From b6c38a4f1bbc1e6aae2f1c13c2c8212c742f8e83 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Sun, 26 Jun 2022 15:13:22 +0800 Subject: [PATCH 01/61] enh: stop query process --- source/client/inc/clientInt.h | 3 +- source/client/src/clientEnv.c | 15 +- source/client/src/clientHb.c | 2 +- source/client/src/clientImpl.c | 28 ++- source/libs/catalog/src/ctgCache.c | 2 - source/libs/catalog/src/ctgDbg.c | 2 +- source/libs/parser/src/parInsert.c | 2 +- source/libs/scheduler/inc/schedulerInt.h | 28 +-- source/libs/scheduler/src/schJob.c | 144 +++++------- source/libs/scheduler/src/scheduler.c | 60 +++-- tests/script/api/batchprepare.c | 22 +- tests/script/api/makefile | 1 + tests/script/api/stopquery.c | 268 +++++++++++++++++++++++ 13 files changed, 422 insertions(+), 155 deletions(-) create mode 100644 tests/script/api/stopquery.c diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index cca79186d0..63f6486197 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -183,7 +183,7 @@ typedef struct SRequestSendRecvBody { void* param; SDataBuf requestMsg; int64_t queryJob; // query job, created according to sql query DAG. - struct SQueryPlan* pDag; // the query dag, generated according to the sql statement. + int32_t subplanNum; SReqResultInfo resInfo; } SRequestSendRecvBody; @@ -300,6 +300,7 @@ void* createRequest(STscObj* pObj, int32_t type); void destroyRequest(SRequestObj* pRequest); SRequestObj* acquireRequest(int64_t rid); int32_t releaseRequest(int64_t rid); +int32_t removeRequest(int64_t rid); char* getDbOfConnection(STscObj* pObj); void setConnectionDB(STscObj* pTscObj, const char* db); diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 9f04e89694..52806343af 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -119,7 +119,7 @@ void closeAllRequests(SHashObj *pRequests) { while (pIter != NULL) { int64_t *rid = pIter; - releaseRequest(*rid); + removeRequest(*rid); pIter = taosHashIterate(pRequests, pIter); } @@ -222,6 +222,12 @@ void doFreeReqResultInfo(SReqResultInfo *pResInfo) { } } +SRequestObj *acquireRequest(int64_t rid) { return (SRequestObj *)taosAcquireRef(clientReqRefPool, rid); } + +int32_t releaseRequest(int64_t rid) { return taosReleaseRef(clientReqRefPool, rid); } + +int32_t removeRequest(int64_t rid) { return taosRemoveRef(clientReqRefPool, rid); } + static void doDestroyRequest(void *p) { assert(p != NULL); SRequestObj *pRequest = (SRequestObj *)p; @@ -239,7 +245,6 @@ static void doDestroyRequest(void *p) { taosMemoryFreeClear(pRequest->pDb); doFreeReqResultInfo(&pRequest->body.resInfo); - qDestroyQueryPlan(pRequest->body.pDag); taosArrayDestroy(pRequest->tableList); taosArrayDestroy(pRequest->dbList); @@ -255,13 +260,9 @@ void destroyRequest(SRequestObj *pRequest) { return; } - taosRemoveRef(clientReqRefPool, pRequest->self); + removeRequest(pRequest->self); } -SRequestObj *acquireRequest(int64_t rid) { return (SRequestObj *)taosAcquireRef(clientReqRefPool, rid); } - -int32_t releaseRequest(int64_t rid) { return taosReleaseRef(clientReqRefPool, rid); } - void taos_init_imp(void) { // In the APIs of other program language, taos_cleanup is not available yet. // So, to make sure taos_cleanup will be invoked to clean up the allocated resource to suppress the valgrind warning. diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 86562fea97..0681b5d0a5 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -320,7 +320,7 @@ int32_t hbBuildQueryDesc(SQueryHbReqBasic *hbBasic, STscObj *pObj) { desc.reqRid = pRequest->self; desc.stableQuery = pRequest->stableQuery; taosGetFqdn(desc.fqdn); - desc.subPlanNum = pRequest->body.pDag ? pRequest->body.pDag->numOfSubplans : 0; + desc.subPlanNum = pRequest->body.subplanNum; if (desc.subPlanNum) { desc.subDesc = taosArrayInit(desc.subPlanNum, sizeof(SQuerySubDesc)); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index b3aaeaea78..a3c818adae 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -876,13 +876,17 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQue break; case QUERY_EXEC_MODE_SCHEDULE: { SArray* pMnodeList = taosArrayInit(4, sizeof(SQueryNodeLoad)); - code = getPlan(pRequest, pQuery, &pRequest->body.pDag, pMnodeList); - if (TSDB_CODE_SUCCESS == code && !pRequest->validateOnly) { - SArray* pNodeList = NULL; - buildSyncExecNodeList(pRequest, &pNodeList, pMnodeList); - - code = scheduleQuery(pRequest, pRequest->body.pDag, pNodeList); - taosArrayDestroy(pNodeList); + SQueryPlan* pDag = NULL; + code = getPlan(pRequest, pQuery, &pDag, pMnodeList); + if (TSDB_CODE_SUCCESS == code) { + pRequest->body.subplanNum = pDag->numOfSubplans; + if (!pRequest->validateOnly) { + SArray* pNodeList = NULL; + buildSyncExecNodeList(pRequest, &pNodeList, pMnodeList); + + code = scheduleQuery(pRequest, pDag, pNodeList); + taosArrayDestroy(pNodeList); + } } taosArrayDestroy(pMnodeList); break; @@ -959,10 +963,13 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData *pResultM .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE}; SAppInstInfo* pAppInfo = getAppInfo(pRequest); - code = qCreateQueryPlan(&cxt, &pRequest->body.pDag, pMnodeList); + SQueryPlan* pDag = NULL; + code = qCreateQueryPlan(&cxt, &pDag, pMnodeList); if (code) { tscError("0x%" PRIx64 " failed to create query plan, code:%s 0x%" PRIx64, pRequest->self, tstrerror(code), pRequest->requestId); + } else { + pRequest->body.subplanNum = pDag->numOfSubplans; } if (TSDB_CODE_SUCCESS == code && !pRequest->validateOnly) { @@ -973,7 +980,7 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData *pResultM .pTrans = pAppInfo->pTransporter, .requestId = pRequest->requestId, .requestObjRefId = pRequest->self}; SSchedulerReq req = {.pConn = &conn, .pNodeList = pNodeList, - .pDag = pRequest->body.pDag, + .pDag = pDag, .sql = pRequest->sqlstr, .startTs = pRequest->metric.start, .fp = schedulerExecCb, @@ -2026,6 +2033,7 @@ void taosAsyncQueryImpl(TAOS *taos, const char *sql, __taos_async_fn_t fp, void if (sqlLen > (size_t)TSDB_MAX_ALLOWED_SQL_LEN) { tscError("sql string exceeds max length:%d", TSDB_MAX_ALLOWED_SQL_LEN); terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT; + releaseTscObj(*(int64_t *)taos); fp(param, NULL, terrno); return; @@ -2035,6 +2043,7 @@ void taosAsyncQueryImpl(TAOS *taos, const char *sql, __taos_async_fn_t fp, void int32_t code = buildRequest(pTscObj, sql, sqlLen, &pRequest); if (code != TSDB_CODE_SUCCESS) { terrno = code; + releaseTscObj(*(int64_t *)taos); fp(param, NULL, terrno); return; } @@ -2043,6 +2052,7 @@ void taosAsyncQueryImpl(TAOS *taos, const char *sql, __taos_async_fn_t fp, void pRequest->body.queryFp = fp; pRequest->body.param = param; doAsyncQuery(pRequest, false); + releaseTscObj(*(int64_t *)taos); } diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 9c9aa4001c..8cd6c7d203 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -1536,8 +1536,6 @@ void ctgClearAllInstance(void) { pIter = taosHashIterate(gCtgMgmt.pCluster, pIter); } - - taosHashClear(gCtgMgmt.pCluster); } void ctgFreeAllInstance(void) { diff --git a/source/libs/catalog/src/ctgDbg.c b/source/libs/catalog/src/ctgDbg.c index 2cb6f0209f..9195747bee 100644 --- a/source/libs/catalog/src/ctgDbg.c +++ b/source/libs/catalog/src/ctgDbg.c @@ -19,7 +19,7 @@ #include "catalogInt.h" extern SCatalogMgmt gCtgMgmt; -SCtgDebug gCTGDebug = {.lockEnable = true, .apiEnable = true}; +SCtgDebug gCTGDebug = {0}; void ctgdUserCallback(SMetaData* pResult, void* param, int32_t code) { ASSERT(*(int32_t*)param == 1); diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index 5d34250444..7cd8398f86 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1757,7 +1757,7 @@ int32_t qBindStmtTagsValue(void* pBlock, void* boundTags, int64_t suid, char* tN } int32_t code = TSDB_CODE_SUCCESS; - SSchema* pSchema = pDataBlock->pTableMeta->schema; + SSchema* pSchema = getTableTagSchema(pDataBlock->pTableMeta); bool isJson = false; STag* pTag = NULL; diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index 6b2570c5b7..843ce7d55a 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -212,7 +212,7 @@ typedef struct SSchJob { SRequestConnInfo conn; SArray *nodeList; // qnode/vnode list, SArray SArray *levels; // starting from 0. SArray - SNodeList *subPlans; // subplan pointer copied from DAG, no need to free it in scheduler + SQueryPlan *pDag; SArray *dataSrcTasks; // SArray int32_t levelIdx; @@ -334,13 +334,13 @@ extern SSchedulerMgmt schMgmt; #define SCH_UNLOCK(type, _lock) (SCH_READ == (type) ? taosRUnLockLatch(_lock) : taosWUnLockLatch(_lock)) -void schDeregisterTaskHb(SSchJob *pJob, SSchTask *pTask); -void schCleanClusterHb(void* pTrans); +void schDeregisterTaskHb(SSchJob *pJob, SSchTask *pTask); +void schCleanClusterHb(void* pTrans); int32_t schLaunchTask(SSchJob *job, SSchTask *task); int32_t schBuildAndSendMsg(SSchJob *job, SSchTask *task, SQueryNodeAddr *addr, int32_t msgType); SSchJob *schAcquireJob(int64_t refId); int32_t schReleaseJob(int64_t refId); -void schFreeFlowCtrl(SSchJob *pJob); +void schFreeFlowCtrl(SSchJob *pJob); int32_t schChkJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel); int32_t schDecTaskFlowQuota(SSchJob *pJob, SSchTask *pTask); int32_t schCheckIncTaskFlowQuota(SSchJob *pJob, SSchTask *pTask, bool *enough); @@ -351,38 +351,40 @@ int32_t schProcessOnTaskFailure(SSchJob *pJob, SSchTask *pTask, int32_t errCode) int32_t schBuildAndSendHbMsg(SQueryNodeEpId *nodeEpId, SArray* taskAction); int32_t schCloneSMsgSendInfo(void *src, void **dst); int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob); -void schFreeJobImpl(void *job); +void schFreeJobImpl(void *job); int32_t schMakeHbRpcCtx(SSchJob *pJob, SSchTask *pTask, SRpcCtx *pCtx); int32_t schEnsureHbConnection(SSchJob *pJob, SSchTask *pTask); int32_t schUpdateHbConnection(SQueryNodeEpId *epId, SSchTrans *trans); int32_t schHandleHbCallback(void *param, const SDataBuf *pMsg, int32_t code); -void schFreeRpcCtx(SRpcCtx *pCtx); +void schFreeRpcCtx(SRpcCtx *pCtx); int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp); -bool schJobNeedToStop(SSchJob *pJob, int8_t *pStatus); +bool schJobNeedToStop(SSchJob *pJob, int8_t *pStatus); int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask); int32_t schSaveJobQueryRes(SSchJob *pJob, SQueryTableRsp *rsp); int32_t schProcessOnExplainDone(SSchJob *pJob, SSchTask *pTask, SRetrieveTableRsp *pRsp); -void schProcessOnDataFetched(SSchJob *job); +void schProcessOnDataFetched(SSchJob *job); int32_t schGetTaskInJob(SSchJob *pJob, uint64_t taskId, SSchTask **pTask); -void schFreeRpcCtxVal(const void *arg); +void schFreeRpcCtxVal(const void *arg); int32_t schMakeBrokenLinkVal(SSchJob *pJob, SSchTask *pTask, SRpcBrokenlinkVal *brokenVal, bool isHb); int32_t schAppendTaskExecNode(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, int32_t execIdx); int32_t schExecStaticExplainJob(SSchedulerReq *pReq, int64_t *job, bool sync); -int32_t schExecJobImpl(SSchedulerReq *pReq, int64_t *job, SQueryResult* pRes, bool sync); +int32_t schExecJobImpl(SSchedulerReq *pReq, SSchJob *pJob, bool sync); int32_t schUpdateJobStatus(SSchJob *pJob, int8_t newStatus); int32_t schCancelJob(SSchJob *pJob); int32_t schProcessOnJobDropped(SSchJob *pJob, int32_t errCode); uint64_t schGenTaskId(void); -void schCloseJobRef(void); +void schCloseJobRef(void); int32_t schExecJob(SSchedulerReq *pReq, int64_t *pJob, SQueryResult *pRes); int32_t schAsyncExecJob(SSchedulerReq *pReq, int64_t *pJob); int32_t schFetchRows(SSchJob *pJob); int32_t schAsyncFetchRows(SSchJob *pJob); int32_t schUpdateTaskHandle(SSchJob *pJob, SSchTask *pTask, bool dropExecNode, void *handle, int32_t execIdx); int32_t schProcessOnTaskStatusRsp(SQueryNodeEpId* pEpId, SArray* pStatusList); -void schFreeSMsgSendInfo(SMsgSendInfo *msgSendInfo); -char* schGetOpStr(SCH_OP_TYPE type); +void schFreeSMsgSendInfo(SMsgSendInfo *msgSendInfo); +char* schGetOpStr(SCH_OP_TYPE type); int32_t schBeginOperation(SSchJob *pJob, SCH_OP_TYPE type, bool sync); +int32_t schInitJob(SSchedulerReq *pReq, SSchJob **pSchJob); +int32_t schSetJobQueryRes(SSchJob* pJob, SQueryResult* pRes); #ifdef __cplusplus diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index 72809e1f93..7843482af3 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -42,7 +42,7 @@ int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel * return TSDB_CODE_SUCCESS; } -int32_t schInitJob(SSchedulerReq *pReq, SSchJob **pSchJob, SQueryResult* pRes, bool syncSchedule) { +int32_t schInitJob(SSchedulerReq *pReq, SSchJob **pSchJob) { int32_t code = 0; int64_t refId = -1; SSchJob *pJob = taosMemoryCalloc(1, sizeof(SSchJob)); @@ -54,12 +54,14 @@ int32_t schInitJob(SSchedulerReq *pReq, SSchJob **pSchJob, SQueryResult* pRes, b pJob->attr.explainMode = pReq->pDag->explainInfo.mode; pJob->conn = *pReq->pConn; pJob->sql = pReq->sql; + pJob->pDag = pReq->pDag; pJob->reqKilled = pReq->reqKilled; - pJob->userRes.queryRes = pRes; pJob->userRes.execFp = pReq->fp; pJob->userRes.userParam = pReq->cbParam; - - if (pReq->pNodeList != NULL) { + + if (pReq->pNodeList == NULL || taosArrayGetSize(pReq->pNodeList) <= 0) { + qDebug("QID:0x%" PRIx64 " input exec nodeList is empty", pReq->pDag->queryId); + } else { pJob->nodeList = taosArrayDup(pReq->pNodeList); } @@ -547,8 +549,6 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { pJob->levelNum = levelNum; pJob->levelIdx = levelNum - 1; - pJob->subPlans = pDag->pSubplans; - SSchLevel level = {0}; SNodeListNode *plans = NULL; int32_t taskNum = 0; @@ -1491,8 +1491,6 @@ void schFreeJobImpl(void *job) { schDropJobAllTasks(pJob); - pJob->subPlans = NULL; // it is a reference to pDag->pSubplans - int32_t numOfLevels = taosArrayGetSize(pJob->levels); for (int32_t i = 0; i < numOfLevels; ++i) { SSchLevel *pLevel = taosArrayGet(pJob->levels, i); @@ -1521,6 +1519,8 @@ void schFreeJobImpl(void *job) { destroyQueryExecRes(&pJob->execRes); + qDestroyQueryPlan(pJob->pDag); + taosMemoryFreeClear(pJob->userRes.queryRes); taosMemoryFreeClear(pJob->resData); taosMemoryFree(pJob); @@ -1533,88 +1533,11 @@ void schFreeJobImpl(void *job) { } } -int32_t schExecJobImpl(SSchedulerReq *pReq, int64_t *job, SQueryResult* pRes, bool sync) { - if (pReq->pNodeList == NULL || taosArrayGetSize(pReq->pNodeList) <= 0) { - qDebug("QID:0x%" PRIx64 " input exec nodeList is empty", pReq->pDag->queryId); - } - - int32_t code = 0; - SSchJob *pJob = NULL; - SCH_ERR_JRET(schInitJob(pReq, &pJob, pRes, sync)); - - qDebug("QID:0x%" PRIx64 " sch job refId 0x%"PRIx64 " started", pReq->pDag->queryId, pJob->refId); - *job = pJob->refId; - - SCH_ERR_JRET(schBeginOperation(pJob, SCH_OP_EXEC, sync)); - - code = schLaunchJob(pJob); - - if (sync) { - SCH_JOB_DLOG("will wait for rsp now, job status:%s", SCH_GET_JOB_STATUS_STR(pJob)); - tsem_wait(&pJob->rspSem); - - schEndOperation(pJob); - } else if (code) { - schPostJobRes(pJob, SCH_OP_EXEC); - } - - SCH_JOB_DLOG("job exec done, job status:%s, jobId:0x%" PRIx64, SCH_GET_JOB_STATUS_STR(pJob), pJob->refId); - - schReleaseJob(pJob->refId); - - SCH_RET(code); - -_return: - - if (!sync) { - pReq->fp(NULL, pReq->cbParam, code); - } - - schReleaseJob(pJob->refId); - - SCH_RET(code); -} - -int32_t schExecJob(SSchedulerReq *pReq, int64_t *pJob, SQueryResult *pRes) { - int32_t code = 0; - - *pJob = 0; - - if (EXPLAIN_MODE_STATIC == pReq->pDag->explainInfo.mode) { - SCH_ERR_JRET(schExecStaticExplainJob(pReq, pJob, true)); - } else { - SCH_ERR_JRET(schExecJobImpl(pReq, pJob, NULL, true)); - } - -_return: - - if (*pJob) { - SSchJob *job = schAcquireJob(*pJob); - schSetJobQueryRes(job, pRes); - schReleaseJob(*pJob); - } - - return code; -} - -int32_t schAsyncExecJob(SSchedulerReq *pReq, int64_t *pJob) { - int32_t code = 0; - - *pJob = 0; - - if (EXPLAIN_MODE_STATIC == pReq->pDag->explainInfo.mode) { - SCH_RET(schExecStaticExplainJob(pReq, pJob, false)); - } - - SCH_ERR_RET(schExecJobImpl(pReq, pJob, NULL, false)); - - return code; -} - -int32_t schExecStaticExplainJob(SSchedulerReq *pReq, int64_t *job, bool sync) { +int32_t schLaunchStaticExplainJob(SSchedulerReq *pReq, SSchJob *pJob, bool sync) { qDebug("QID:0x%" PRIx64 " job started", pReq->pDag->queryId); int32_t code = 0; +/* SSchJob *pJob = taosMemoryCalloc(1, sizeof(SSchJob)); if (NULL == pJob) { qError("QID:0x%" PRIx64 " calloc %d failed", pReq->pDag->queryId, (int32_t)sizeof(SSchJob)); @@ -1625,10 +1548,10 @@ int32_t schExecStaticExplainJob(SSchedulerReq *pReq, int64_t *job, bool sync) { pJob->sql = pReq->sql; pJob->reqKilled = pReq->reqKilled; + pJob->pDag = pReq->pDag; pJob->attr.queryJob = true; pJob->attr.explainMode = pReq->pDag->explainInfo.mode; pJob->queryId = pReq->pDag->queryId; - pJob->subPlans = pReq->pDag->pSubplans; pJob->userRes.execFp = pReq->fp; pJob->userRes.userParam = pReq->cbParam; @@ -1637,11 +1560,14 @@ int32_t schExecStaticExplainJob(SSchedulerReq *pReq, int64_t *job, bool sync) { code = schBeginOperation(pJob, SCH_OP_EXEC, sync); if (code) { pReq->fp(NULL, pReq->cbParam, code); + schFreeJobImpl(pJob); SCH_ERR_RET(code); } - +*/ + SCH_ERR_JRET(qExecStaticExplain(pReq->pDag, (SRetrieveTableRsp **)&pJob->resData)); +/* int64_t refId = taosAddRef(schMgmt.jobRef, pJob); if (refId < 0) { SCH_JOB_ELOG("taosAddRef job failed, error:%s", tstrerror(terrno)); @@ -1656,10 +1582,10 @@ int32_t schExecStaticExplainJob(SSchedulerReq *pReq, int64_t *job, bool sync) { pJob->refId = refId; SCH_JOB_DLOG("job refId:0x%" PRIx64, pJob->refId); +*/ pJob->status = JOB_TASK_STATUS_PARTIAL_SUCCEED; - *job = pJob->refId; SCH_JOB_DLOG("job exec done, job status:%s", SCH_GET_JOB_STATUS_STR(pJob)); if (!sync) { @@ -1668,7 +1594,7 @@ int32_t schExecStaticExplainJob(SSchedulerReq *pReq, int64_t *job, bool sync) { schEndOperation(pJob); } - schReleaseJob(pJob->refId); +// schReleaseJob(pJob->refId); SCH_RET(code); @@ -1715,3 +1641,39 @@ int32_t schAsyncFetchRows(SSchJob *pJob) { } +int32_t schExecJobImpl(SSchedulerReq *pReq, SSchJob *pJob, bool sync) { + int32_t code = 0; + + qDebug("QID:0x%" PRIx64 " sch job refId 0x%"PRIx64 " started", pReq->pDag->queryId, pJob->refId); + + SCH_ERR_JRET(schBeginOperation(pJob, SCH_OP_EXEC, sync)); + + if (EXPLAIN_MODE_STATIC == pReq->pDag->explainInfo.mode) { + code = schLaunchStaticExplainJob(pReq, pJob, true); + } else { + code = schLaunchJob(pJob); + if (sync) { + SCH_JOB_DLOG("will wait for rsp now, job status:%s", SCH_GET_JOB_STATUS_STR(pJob)); + tsem_wait(&pJob->rspSem); + + schEndOperation(pJob); + } else if (code) { + schPostJobRes(pJob, SCH_OP_EXEC); + } + } + + SCH_JOB_DLOG("job exec done, job status:%s, jobId:0x%" PRIx64, SCH_GET_JOB_STATUS_STR(pJob), pJob->refId); + + SCH_RET(code); + +_return: + + if (!sync) { + pReq->fp(NULL, pReq->cbParam, code); + } + + SCH_RET(code); +} + + + diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 57a405ffa3..15a687531c 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -67,33 +67,57 @@ int32_t schedulerInit(SSchedulerCfg *cfg) { return TSDB_CODE_SUCCESS; } -int32_t schedulerExecJob(SSchedulerReq *pReq, int64_t *pJob, SQueryResult *pRes) { +int32_t schedulerExecJob(SSchedulerReq *pReq, int64_t *pJobId, SQueryResult *pRes) { qDebug("scheduler sync exec job start"); + + int32_t code = 0; + SSchJob *pJob = NULL; + SCH_ERR_JRET(schInitJob(pReq, &pJob)); + + *pJobId = pJob->refId; - if (NULL == pReq || NULL == pJob || NULL == pRes) { - SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); - } - - SCH_RET(schExecJob(pReq, pJob, pRes)); -} - -int32_t schedulerAsyncExecJob(SSchedulerReq *pReq, int64_t *pJob) { - qDebug("scheduler async exec job start"); - - int32_t code = 0; - if (NULL == pReq || NULL == pJob) { - SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); - } - - schAsyncExecJob(pReq, pJob); + SCH_ERR_JRET(schExecJobImpl(pReq, pJob, true)); _return: + if (code && NULL == pJob) { + qDestroyQueryPlan(pReq->pDag); + } + + if (pJob) { + schSetJobQueryRes(pJob, pRes); + schReleaseJob(pJob->refId); + } + + return code; +} + +int32_t schedulerAsyncExecJob(SSchedulerReq *pReq, int64_t *pJobId) { + qDebug("scheduler async exec job start"); + + int32_t code = 0; + SSchJob *pJob = NULL; + SCH_ERR_JRET(schInitJob(pReq, &pJob)); + + *pJobId = pJob->refId; + + SCH_ERR_JRET(schExecJobImpl(pReq, pJob, false)); + +_return: + + if (code && NULL == pJob) { + qDestroyQueryPlan(pReq->pDag); + } + + if (pJob) { + schReleaseJob(pJob->refId); + } + if (code != TSDB_CODE_SUCCESS) { pReq->fp(NULL, pReq->cbParam, code); } - SCH_RET(code); + return code; } int32_t schedulerFetchRows(int64_t job, void **pData) { diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index 0e7030b230..b31c39718c 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -915,7 +915,7 @@ int32_t prepareInsertData(BindData *data) { data->colNum = 0; data->colTypes = taosMemoryCalloc(30, sizeof(int32_t)); data->sql = taosMemoryCalloc(1, 1024); - data->pBind = taosMemoryCalloc((allRowNum/gCurCase->bindRowNum)*gCurCase->bindColNum, sizeof(TAOS_MULTI_BIND)); + data->pBind = taosMemoryCalloc((int32_t)(allRowNum/gCurCase->bindRowNum)*gCurCase->bindColNum, sizeof(TAOS_MULTI_BIND)); data->pTags = taosMemoryCalloc(gCurCase->tblNum*gCurCase->bindTagNum, sizeof(TAOS_MULTI_BIND)); data->tsData = taosMemoryMalloc(allRowNum * sizeof(int64_t)); data->boolData = taosMemoryMalloc(allRowNum * sizeof(bool)); @@ -932,7 +932,7 @@ int32_t prepareInsertData(BindData *data) { data->binaryData = taosMemoryMalloc(allRowNum * gVarCharSize); data->binaryLen = taosMemoryMalloc(allRowNum * sizeof(int32_t)); if (gCurCase->bindNullNum) { - data->isNull = taosMemoryCalloc(allRowNum, sizeof(char)); + data->isNull = taosMemoryCalloc((int32_t)allRowNum, sizeof(char)); } for (int32_t i = 0; i < allRowNum; ++i) { @@ -950,7 +950,7 @@ int32_t prepareInsertData(BindData *data) { data->doubleData[i] = (double)(i+1); memset(data->binaryData + gVarCharSize * i, 'a'+i%26, gVarCharLen); if (gCurCase->bindNullNum) { - data->isNull[i] = i % 2; + data->isNull[i] = (char)(i % 2); } data->binaryLen[i] = gVarCharLen; } @@ -979,7 +979,7 @@ int32_t prepareQueryCondData(BindData *data, int32_t tblIdx) { data->colNum = 0; data->colTypes = taosMemoryCalloc(30, sizeof(int32_t)); data->sql = taosMemoryCalloc(1, 1024); - data->pBind = taosMemoryCalloc(bindNum*gCurCase->bindColNum, sizeof(TAOS_MULTI_BIND)); + data->pBind = taosMemoryCalloc((int32_t)bindNum*gCurCase->bindColNum, sizeof(TAOS_MULTI_BIND)); data->tsData = taosMemoryMalloc(bindNum * sizeof(int64_t)); data->boolData = taosMemoryMalloc(bindNum * sizeof(bool)); data->tinyData = taosMemoryMalloc(bindNum * sizeof(int8_t)); @@ -995,7 +995,7 @@ int32_t prepareQueryCondData(BindData *data, int32_t tblIdx) { data->binaryData = taosMemoryMalloc(bindNum * gVarCharSize); data->binaryLen = taosMemoryMalloc(bindNum * sizeof(int32_t)); if (gCurCase->bindNullNum) { - data->isNull = taosMemoryCalloc(bindNum, sizeof(char)); + data->isNull = taosMemoryCalloc((int32_t)bindNum, sizeof(char)); } for (int32_t i = 0; i < bindNum; ++i) { @@ -1013,7 +1013,7 @@ int32_t prepareQueryCondData(BindData *data, int32_t tblIdx) { data->doubleData[i] = (double)(tblIdx*gCurCase->rowNum + rand() % gCurCase->rowNum); memset(data->binaryData + gVarCharSize * i, 'a'+i%26, gVarCharLen); if (gCurCase->bindNullNum) { - data->isNull[i] = i % 2; + data->isNull[i] = (char)(i % 2); } data->binaryLen[i] = gVarCharLen; } @@ -1036,7 +1036,7 @@ int32_t prepareQueryMiscData(BindData *data, int32_t tblIdx) { data->colNum = 0; data->colTypes = taosMemoryCalloc(30, sizeof(int32_t)); data->sql = taosMemoryCalloc(1, 1024); - data->pBind = taosMemoryCalloc(bindNum*gCurCase->bindColNum, sizeof(TAOS_MULTI_BIND)); + data->pBind = taosMemoryCalloc((int32_t)bindNum*gCurCase->bindColNum, sizeof(TAOS_MULTI_BIND)); data->tsData = taosMemoryMalloc(bindNum * sizeof(int64_t)); data->boolData = taosMemoryMalloc(bindNum * sizeof(bool)); data->tinyData = taosMemoryMalloc(bindNum * sizeof(int8_t)); @@ -1052,7 +1052,7 @@ int32_t prepareQueryMiscData(BindData *data, int32_t tblIdx) { data->binaryData = taosMemoryMalloc(bindNum * gVarCharSize); data->binaryLen = taosMemoryMalloc(bindNum * sizeof(int32_t)); if (gCurCase->bindNullNum) { - data->isNull = taosMemoryCalloc(bindNum, sizeof(char)); + data->isNull = taosMemoryCalloc((int32_t)bindNum, sizeof(char)); } for (int32_t i = 0; i < bindNum; ++i) { @@ -1070,7 +1070,7 @@ int32_t prepareQueryMiscData(BindData *data, int32_t tblIdx) { data->doubleData[i] = (double)(tblIdx*gCurCase->rowNum + rand() % gCurCase->rowNum); memset(data->binaryData + gVarCharSize * i, 'a'+i%26, gVarCharLen); if (gCurCase->bindNullNum) { - data->isNull[i] = i % 2; + data->isNull[i] = (char)(i % 2); } data->binaryLen[i] = gVarCharLen; } @@ -1279,7 +1279,7 @@ void bpCheckQueryResult(TAOS_STMT *stmt, TAOS *taos, char *stmtSql, TAOS_MULTI_B } memcpy(&sql[len], p, (int64_t)s - (int64_t)p); - len += (int64_t)s - (int64_t)p; + len += (int32_t)((int64_t)s - (int64_t)p); if (bind[i].is_null && bind[i].is_null[0]) { bpAppendValueString(sql, TSDB_DATA_TYPE_NULL, NULL, 0, &len); @@ -2669,7 +2669,7 @@ int main(int argc, char *argv[]) { TAOS *taos = NULL; - srand(time(NULL)); + srand((unsigned int)time(NULL)); // connect to server if (argc < 2) { diff --git a/tests/script/api/makefile b/tests/script/api/makefile index 46a172cc3a..1f725f17c9 100644 --- a/tests/script/api/makefile +++ b/tests/script/api/makefile @@ -12,6 +12,7 @@ all: $(TARGET) exe: gcc $(CFLAGS) ./batchprepare.c -o $(ROOT)batchprepare $(LFLAGS) + gcc $(CFLAGS) ./stopquery.c -o $(ROOT)stopquery $(LFLAGS) clean: rm $(ROOT)batchprepare diff --git a/tests/script/api/stopquery.c b/tests/script/api/stopquery.c new file mode 100644 index 0000000000..5fa1c3654d --- /dev/null +++ b/tests/script/api/stopquery.c @@ -0,0 +1,268 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// TAOS asynchronous API example +// this example opens multiple tables, insert/retrieve multiple tables +// it is used by TAOS internally for one performance testing +// to compiple: gcc -o asyncdemo asyncdemo.c -ltaos + +#include +#include +#include +#include +#include +#include "taos.h" + + +int points = 5; +int numOfTables = 3; +int tablesInsertProcessed = 0; +int tablesSelectProcessed = 0; +int64_t st, et; + +char hostName[128]; +char dbName[128]; +char tbName[128]; +char runTimes = 1; + +typedef struct { + int id; + TAOS *taos; + char name[16]; + time_t timeStamp; + int value; + int rowsInserted; + int rowsTried; + int rowsRetrieved; +} STable; + +typedef struct SSP_CB_PARAM { + bool fetch; + int32_t *end; +} SSP_CB_PARAM; + +#define CASE_ENTER() do { printf("enter case %s\n", __FUNCTION__); } while (0) +#define CASE_LEAVE() do { printf("leave case %s, runTimes %d\n", __FUNCTION__, runTimes); } while (0) + +static void sqExecSQL(TAOS *taos, char *command) { + int i; + int32_t code = -1; + + TAOS_RES *pSql = taos_query(taos, command); + code = taos_errno(pSql); + if (code != 0) { + fprintf(stderr, "Failed to run %s, reason: %s\n", command, taos_errstr(pSql)); + taos_free_result(pSql); + taos_close(taos); + taos_cleanup(); + exit(EXIT_FAILURE); + } + + taos_free_result(pSql); +} + +void sqExit(char* prefix, const char* errMsg) { + fprintf(stderr, "%s error: %s\n", prefix, errMsg); + exit(1); +} + +void sqStopFetchCb(void *param, TAOS_RES *pRes, int numOfRows) { + SSP_CB_PARAM *qParam = (SSP_CB_PARAM *)param; + taos_stop_query(pRes); + taos_free_result(pRes); + + *qParam->end = 1; +} + +void sqStopQueryCb(void *param, TAOS_RES *pRes, int code) { + SSP_CB_PARAM *qParam = (SSP_CB_PARAM *)param; + if (code == 0 && pRes) { + if (qParam->fetch) { + taos_fetch_rows_a(pRes, sqStopFetchCb, param); + } else { + taos_stop_query(pRes); + taos_free_result(pRes); + *qParam->end = 1; + } + } else { + sqExit("select", taos_errstr(pRes)); + } +} + +void sqFreeFetchCb(void *param, TAOS_RES *pRes, int numOfRows) { + SSP_CB_PARAM *qParam = (SSP_CB_PARAM *)param; + taos_free_result(pRes); + + *qParam->end = 1; +} + +void sqFreeQueryCb(void *param, TAOS_RES *pRes, int code) { + SSP_CB_PARAM *qParam = (SSP_CB_PARAM *)param; + if (code == 0 && pRes) { + if (qParam->fetch) { + taos_fetch_rows_a(pRes, sqFreeFetchCb, param); + } else { + taos_free_result(pRes); + *qParam->end = 1; + } + } else { + sqExit("select", taos_errstr(pRes)); + } +} + + +int sqSyncStopQuery(bool fetch) { + CASE_ENTER(); + for (int32_t i = 0; i < runTimes; ++i) { + char sql[1024] = {0}; + int32_t code = 0; + TAOS *taos = taos_connect(hostName, "root", "taosdata", NULL, 0); + if (taos == NULL) sqExit("taos_connect", taos_errstr(NULL)); + + sprintf(sql, "use %s", dbName); + sqExecSQL(taos, sql); + + sprintf(sql, "select * from %s", tbName); + TAOS_RES* pRes = taos_query(taos, sql); + code = taos_errno(pRes); + if (code) { + sqExit("taos_query", taos_errstr(pRes)); + } + + if (fetch) { + taos_fetch_row(pRes); + } + + taos_stop_query(pRes); + taos_free_result(pRes); + + taos_close(taos); + } + CASE_LEAVE(); +} + +int sqAsyncStopQuery(bool fetch) { + CASE_ENTER(); + for (int32_t i = 0; i < runTimes; ++i) { + char sql[1024] = {0}; + int32_t code = 0; + TAOS *taos = taos_connect(hostName, "root", "taosdata", NULL, 0); + if (taos == NULL) sqExit("taos_connect", taos_errstr(NULL)); + + sprintf(sql, "use %s", dbName); + sqExecSQL(taos, sql); + + sprintf(sql, "select * from %s", tbName); + + int32_t qEnd = 0; + SSP_CB_PARAM param = {0}; + param.fetch = fetch; + param.end = &qEnd; + taos_query_a(taos, sql, sqStopQueryCb, ¶m); + while (0 == qEnd) { + usleep(5000); + } + + taos_close(taos); + } + CASE_LEAVE(); +} + +int sqSyncFreeQuery(bool fetch) { + CASE_ENTER(); + for (int32_t i = 0; i < runTimes; ++i) { + char sql[1024] = {0}; + int32_t code = 0; + TAOS *taos = taos_connect(hostName, "root", "taosdata", NULL, 0); + if (taos == NULL) sqExit("taos_connect", taos_errstr(NULL)); + + sprintf(sql, "use %s", dbName); + sqExecSQL(taos, sql); + + sprintf(sql, "select * from %s", tbName); + TAOS_RES* pRes = taos_query(taos, sql); + code = taos_errno(pRes); + if (code) { + sqExit("taos_query", taos_errstr(pRes)); + } + + if (fetch) { + taos_fetch_row(pRes); + } + + taos_free_result(pRes); + taos_close(taos); + } + CASE_LEAVE(); +} + +int sqAsyncFreeQuery(bool fetch) { + CASE_ENTER(); + for (int32_t i = 0; i < runTimes; ++i) { + char sql[1024] = {0}; + int32_t code = 0; + TAOS *taos = taos_connect(hostName, "root", "taosdata", NULL, 0); + if (taos == NULL) sqExit("taos_connect", taos_errstr(NULL)); + + sprintf(sql, "use %s", dbName); + sqExecSQL(taos, sql); + + sprintf(sql, "select * from %s", tbName); + + int32_t qEnd = 0; + SSP_CB_PARAM param = {0}; + param.fetch = fetch; + param.end = &qEnd; + taos_query_a(taos, sql, sqFreeQueryCb, ¶m); + while (0 == qEnd) { + usleep(5000); + } + + taos_close(taos); + } + CASE_LEAVE(); +} + + +void sqRunAllCase(void) { + sqSyncStopQuery(false); + sqSyncStopQuery(true); + sqAsyncStopQuery(false); + sqAsyncStopQuery(true); + + sqSyncFreeQuery(false); + sqSyncFreeQuery(true); + sqAsyncFreeQuery(false); + sqAsyncFreeQuery(true); + +} + + +int main(int argc, char *argv[]) { + if (argc != 4) { + printf("usage: %s server-ip dbname tablename\n", argv[0]); + exit(0); + } + + strcpy(hostName, argv[1]); + strcpy(dbName, argv[2]); + strcpy(tbName, argv[3]); + + sqRunAllCase(); + + return 0; +} + + From c364f98978b30ba0e42c0ea0c8638be0c61232ef Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Sun, 26 Jun 2022 19:34:50 +0800 Subject: [PATCH 02/61] enh: stop query process --- source/client/inc/clientInt.h | 1 + source/client/src/clientEnv.c | 23 ++-- source/client/src/clientImpl.c | 4 +- source/libs/catalog/src/catalog.c | 19 ++-- tests/script/api/stopquery.c | 168 +++++++++++++++++++++++++++++- 5 files changed, 199 insertions(+), 16 deletions(-) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 63f6486197..cad262a00f 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -301,6 +301,7 @@ void destroyRequest(SRequestObj* pRequest); SRequestObj* acquireRequest(int64_t rid); int32_t releaseRequest(int64_t rid); int32_t removeRequest(int64_t rid); +void doDestroyRequest(void *p); char* getDbOfConnection(STscObj* pObj); void setConnectionDB(STscObj* pTscObj, const char* db); diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 510d231855..500933b68b 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -37,10 +37,12 @@ int32_t clientConnRefPool = -1; static TdThreadOnce tscinit = PTHREAD_ONCE_INIT; volatile int32_t tscInitRes = 0; -static void registerRequest(SRequestObj *pRequest) { +static int32_t registerRequest(SRequestObj *pRequest) { STscObj *pTscObj = acquireTscObj(*(int64_t *)pRequest->pTscObj->id); - - assert(pTscObj != NULL); + if (NULL == pTscObj) { + terrno = TSDB_CODE_TSC_DISCONNECTED; + return terrno; + } // connection has been released already, abort creating request. pRequest->self = taosAddRef(clientReqRefPool, pRequest); @@ -56,6 +58,8 @@ static void registerRequest(SRequestObj *pRequest) { ", current:%d, app current:%d, total:%d, reqId:0x%" PRIx64, pRequest->self, *(int64_t *)pRequest->pTscObj->id, num, currentInst, total, pRequest->requestId); } + + return TSDB_CODE_SUCCESS; } static void deregisterRequest(SRequestObj *pRequest) { @@ -202,7 +206,10 @@ void *createRequest(STscObj *pObj, int32_t type) { pRequest->msgBufLen = ERROR_MSG_BUF_DEFAULT_SIZE; tsem_init(&pRequest->body.rspSem, 0, 0); - registerRequest(pRequest); + if (registerRequest(pRequest)) { + doDestroyRequest(pRequest); + return NULL; + } return pRequest; } @@ -230,12 +237,10 @@ int32_t releaseRequest(int64_t rid) { return taosReleaseRef(clientReqRefPool, ri int32_t removeRequest(int64_t rid) { return taosRemoveRef(clientReqRefPool, rid); } -static void doDestroyRequest(void *p) { +void doDestroyRequest(void *p) { assert(p != NULL); SRequestObj *pRequest = (SRequestObj *)p; - assert(RID_VALID(pRequest->self)); - taosHashRemove(pRequest->pTscObj->pRequests, &pRequest->self, sizeof(pRequest->self)); if (pRequest->body.queryJob != 0) { @@ -253,7 +258,9 @@ static void doDestroyRequest(void *p) { destroyQueryExecRes(&pRequest->body.resInfo.execRes); - deregisterRequest(pRequest); + if (pRequest->self) { + deregisterRequest(pRequest); + } taosMemoryFreeClear(pRequest); } diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index d83ca89c69..00bc7200b1 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -2008,7 +2008,9 @@ void syncCatalogFn(SMetaData* pResult, void* param, int32_t code) { void syncQueryFn(void* param, void* res, int32_t code) { SSyncQueryParam* pParam = param; pParam->pRequest = res; - pParam->pRequest->code = code; + if (pParam->pRequest) { + pParam->pRequest->code = code; + } tsem_post(&pParam->sem); } diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index b459beb658..0c46d6f5fa 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -1127,12 +1127,14 @@ int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion** users, uint32_ } *num = taosHashGetSize(pCtg->userCache); - if (*num > 0) { - *users = taosMemoryCalloc(*num, sizeof(SUserAuthVersion)); - if (NULL == *users) { - ctgError("calloc %d userAuthVersion failed", *num); - CTG_API_LEAVE(TSDB_CODE_OUT_OF_MEMORY); - } + if (*num <= 0) { + CTG_API_LEAVE(TSDB_CODE_SUCCESS); + } + + *users = taosMemoryCalloc(*num, sizeof(SUserAuthVersion)); + if (NULL == *users) { + ctgError("calloc %d userAuthVersion failed", *num); + CTG_API_LEAVE(TSDB_CODE_OUT_OF_MEMORY); } uint32_t i = 0; @@ -1144,6 +1146,11 @@ int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion** users, uint32_ (*users)[i].user[len] = 0; (*users)[i].version = pAuth->version; ++i; + if (i >= *num) { + taosHashCancelIterate(pCtg->userCache, pAuth); + break; + } + pAuth = taosHashIterate(pCtg->userCache, pAuth); } diff --git a/tests/script/api/stopquery.c b/tests/script/api/stopquery.c index 5fa1c3654d..4c7964c983 100644 --- a/tests/script/api/stopquery.c +++ b/tests/script/api/stopquery.c @@ -23,6 +23,7 @@ #include #include #include +#include #include "taos.h" @@ -35,7 +36,7 @@ int64_t st, et; char hostName[128]; char dbName[128]; char tbName[128]; -char runTimes = 1; +int32_t runTimes = 10000; typedef struct { int id; @@ -49,6 +50,7 @@ typedef struct { } STable; typedef struct SSP_CB_PARAM { + TAOS *taos; bool fetch; int32_t *end; } SSP_CB_PARAM; @@ -73,6 +75,16 @@ static void sqExecSQL(TAOS *taos, char *command) { taos_free_result(pSql); } +static void sqExecSQLE(TAOS *taos, char *command) { + int i; + int32_t code = -1; + + TAOS_RES *pSql = taos_query(taos, command); + + taos_free_result(pSql); +} + + void sqExit(char* prefix, const char* errMsg) { fprintf(stderr, "%s error: %s\n", prefix, errMsg); exit(1); @@ -123,6 +135,27 @@ void sqFreeQueryCb(void *param, TAOS_RES *pRes, int code) { } +void sqCloseFetchCb(void *param, TAOS_RES *pRes, int numOfRows) { + SSP_CB_PARAM *qParam = (SSP_CB_PARAM *)param; + taos_close(qParam->taos); + + *qParam->end = 1; +} + +void sqCloseQueryCb(void *param, TAOS_RES *pRes, int code) { + SSP_CB_PARAM *qParam = (SSP_CB_PARAM *)param; + if (code == 0 && pRes) { + if (qParam->fetch) { + taos_fetch_rows_a(pRes, sqFreeFetchCb, param); + } else { + taos_close(qParam->taos); + *qParam->end = 1; + } + } else { + sqExit("select", taos_errstr(pRes)); + } +} + int sqSyncStopQuery(bool fetch) { CASE_ENTER(); for (int32_t i = 0; i < runTimes; ++i) { @@ -131,6 +164,9 @@ int sqSyncStopQuery(bool fetch) { TAOS *taos = taos_connect(hostName, "root", "taosdata", NULL, 0); if (taos == NULL) sqExit("taos_connect", taos_errstr(NULL)); + sprintf(sql, "reset query cache"); + sqExecSQL(taos, sql); + sprintf(sql, "use %s", dbName); sqExecSQL(taos, sql); @@ -161,6 +197,9 @@ int sqAsyncStopQuery(bool fetch) { TAOS *taos = taos_connect(hostName, "root", "taosdata", NULL, 0); if (taos == NULL) sqExit("taos_connect", taos_errstr(NULL)); + sprintf(sql, "reset query cache"); + sqExecSQL(taos, sql); + sprintf(sql, "use %s", dbName); sqExecSQL(taos, sql); @@ -188,6 +227,9 @@ int sqSyncFreeQuery(bool fetch) { TAOS *taos = taos_connect(hostName, "root", "taosdata", NULL, 0); if (taos == NULL) sqExit("taos_connect", taos_errstr(NULL)); + sprintf(sql, "reset query cache"); + sqExecSQL(taos, sql); + sprintf(sql, "use %s", dbName); sqExecSQL(taos, sql); @@ -216,6 +258,9 @@ int sqAsyncFreeQuery(bool fetch) { TAOS *taos = taos_connect(hostName, "root", "taosdata", NULL, 0); if (taos == NULL) sqExit("taos_connect", taos_errstr(NULL)); + sprintf(sql, "reset query cache"); + sqExecSQL(taos, sql); + sprintf(sql, "use %s", dbName); sqExecSQL(taos, sql); @@ -235,8 +280,119 @@ int sqAsyncFreeQuery(bool fetch) { CASE_LEAVE(); } +int sqSyncCloseQuery(bool fetch) { + CASE_ENTER(); + for (int32_t i = 0; i < runTimes; ++i) { + char sql[1024] = {0}; + int32_t code = 0; + TAOS *taos = taos_connect(hostName, "root", "taosdata", NULL, 0); + if (taos == NULL) sqExit("taos_connect", taos_errstr(NULL)); + + sprintf(sql, "reset query cache"); + sqExecSQL(taos, sql); + + sprintf(sql, "use %s", dbName); + sqExecSQL(taos, sql); + + sprintf(sql, "select * from %s", tbName); + TAOS_RES* pRes = taos_query(taos, sql); + code = taos_errno(pRes); + if (code) { + sqExit("taos_query", taos_errstr(pRes)); + } + + if (fetch) { + taos_fetch_row(pRes); + } + + taos_close(taos); + } + CASE_LEAVE(); +} + +int sqAsyncCloseQuery(bool fetch) { + CASE_ENTER(); + for (int32_t i = 0; i < runTimes; ++i) { + char sql[1024] = {0}; + int32_t code = 0; + TAOS *taos = taos_connect(hostName, "root", "taosdata", NULL, 0); + if (taos == NULL) sqExit("taos_connect", taos_errstr(NULL)); + + sprintf(sql, "reset query cache"); + sqExecSQL(taos, sql); + + sprintf(sql, "use %s", dbName); + sqExecSQL(taos, sql); + + sprintf(sql, "select * from %s", tbName); + + int32_t qEnd = 0; + SSP_CB_PARAM param = {0}; + param.fetch = fetch; + param.end = &qEnd; + taos_query_a(taos, sql, sqFreeQueryCb, ¶m); + while (0 == qEnd) { + usleep(5000); + } + } + CASE_LEAVE(); +} + +void *syncQueryThreadFp(void *arg) { + SSP_CB_PARAM* qParam = (SSP_CB_PARAM*)arg; + char sql[1024] = {0}; + int32_t code = 0; + TAOS *taos = taos_connect(hostName, "root", "taosdata", NULL, 0); + if (taos == NULL) sqExit("taos_connect", taos_errstr(NULL)); + + qParam->taos = taos; + + sprintf(sql, "reset query cache"); + sqExecSQLE(taos, sql); + + sprintf(sql, "use %s", dbName); + sqExecSQLE(taos, sql); + + sprintf(sql, "select * from %s", tbName); + TAOS_RES* pRes = taos_query(taos, sql); + + if (qParam->fetch) { + taos_fetch_row(pRes); + } + + taos_free_result(pRes); +} + +void *closeThreadFp(void *arg) { + SSP_CB_PARAM* qParam = (SSP_CB_PARAM*)arg; + while (true) { + if (qParam->taos) { + usleep(rand() % 10000); + taos_close(qParam->taos); + break; + } + usleep(1); + } +} + + +int sqConSyncCloseQuery(bool fetch) { + CASE_ENTER(); + pthread_t qid, cid; + for (int32_t i = 0; i < runTimes; ++i) { + SSP_CB_PARAM param = {0}; + param.fetch = fetch; + pthread_create(&qid, NULL, syncQueryThreadFp, (void*)¶m); + pthread_create(&cid, NULL, closeThreadFp, (void*)¶m); + + pthread_join(qid, NULL); + pthread_join(cid, NULL); + } + CASE_LEAVE(); +} void sqRunAllCase(void) { +/* sqSyncStopQuery(false); sqSyncStopQuery(true); sqAsyncStopQuery(false); @@ -247,6 +403,14 @@ void sqRunAllCase(void) { sqAsyncFreeQuery(false); sqAsyncFreeQuery(true); + sqSyncCloseQuery(false); + sqSyncCloseQuery(true); + sqAsyncCloseQuery(false); + sqAsyncCloseQuery(true); +*/ + sqConSyncCloseQuery(false); + sqConSyncCloseQuery(true); + } @@ -256,6 +420,8 @@ int main(int argc, char *argv[]) { exit(0); } + srand((unsigned int)time(NULL)); + strcpy(hostName, argv[1]); strcpy(dbName, argv[2]); strcpy(tbName, argv[3]); From 73dfd1173a914c2e15cb0f7b277ed3d0cd592b41 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 27 Jun 2022 09:50:42 +0800 Subject: [PATCH 03/61] enh: stop query --- include/client/taos.h | 6 ++--- source/client/inc/clientInt.h | 2 +- source/client/src/clientEnv.c | 22 ++++++++---------- source/client/src/clientImpl.c | 34 ++++++++++++++++++---------- source/client/src/clientMain.c | 18 +++++++++++---- source/client/src/clientMsgHandler.c | 4 ++-- source/client/src/clientSml.c | 22 +++++++++++------- 7 files changed, 65 insertions(+), 43 deletions(-) diff --git a/include/client/taos.h b/include/client/taos.h index 5e7f12de0a..669512e9b1 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -130,10 +130,10 @@ DLL_EXPORT void taos_cleanup(void); DLL_EXPORT int taos_options(TSDB_OPTION option, const void *arg, ...); DLL_EXPORT int taos_init(void); DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port); -DLL_EXPORT TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port); -DLL_EXPORT void taos_close(TAOS *taos); +DLL_EXPORT TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port); +DLL_EXPORT void taos_close(TAOS *taos); -const char *taos_data_type(int type); +const char *taos_data_type(int type); DLL_EXPORT TAOS_STMT *taos_stmt_init(TAOS *taos); DLL_EXPORT int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length); diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index cad262a00f..48efe89361 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -139,7 +139,7 @@ typedef struct STscObj { int8_t connType; int32_t acctId; uint32_t connId; - TAOS* id; // ref ID returned by taosAddRef + int64_t id; // ref ID returned by taosAddRef TdThreadMutex mutex; // used to protect the operation on db int32_t numOfReqs; // number of sqlObj bound to this connection SAppInstInfo* pAppInfo; diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 500933b68b..cad1b4b1cf 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -38,7 +38,7 @@ static TdThreadOnce tscinit = PTHREAD_ONCE_INIT; volatile int32_t tscInitRes = 0; static int32_t registerRequest(SRequestObj *pRequest) { - STscObj *pTscObj = acquireTscObj(*(int64_t *)pRequest->pTscObj->id); + STscObj *pTscObj = acquireTscObj(pRequest->pTscObj->id); if (NULL == pTscObj) { terrno = TSDB_CODE_TSC_DISCONNECTED; return terrno; @@ -56,7 +56,7 @@ static int32_t registerRequest(SRequestObj *pRequest) { int32_t currentInst = atomic_add_fetch_64((int64_t *)&pSummary->currentRequests, 1); tscDebug("0x%" PRIx64 " new Request from connObj:0x%" PRIx64 ", current:%d, app current:%d, total:%d, reqId:0x%" PRIx64, - pRequest->self, *(int64_t *)pRequest->pTscObj->id, num, currentInst, total, pRequest->requestId); + pRequest->self, pRequest->pTscObj->id, num, currentInst, total, pRequest->requestId); } return TSDB_CODE_SUCCESS; @@ -74,8 +74,8 @@ static void deregisterRequest(SRequestObj *pRequest) { int64_t duration = taosGetTimestampUs() - pRequest->metric.start; tscDebug("0x%" PRIx64 " free Request from connObj: 0x%" PRIx64 ", reqId:0x%" PRIx64 " elapsed:%" PRIu64 " ms, current:%d, app current:%d", - pRequest->self, *(int64_t *)pTscObj->id, pRequest->requestId, duration / 1000, num, currentInst); - releaseTscObj(*(int64_t *)pTscObj->id); + pRequest->self, pTscObj->id, pRequest->requestId, duration / 1000, num, currentInst); + releaseTscObj(pTscObj->id); } // todo close the transporter properly @@ -84,7 +84,7 @@ void closeTransporter(STscObj *pTscObj) { return; } - tscDebug("free transporter:%p in connObj: 0x%" PRIx64, pTscObj->pAppInfo->pTransporter, *(int64_t *)pTscObj->id); + tscDebug("free transporter:%p in connObj: 0x%" PRIx64, pTscObj->pAppInfo->pTransporter, pTscObj->id); rpcClose(pTscObj->pAppInfo->pTransporter); } @@ -133,16 +133,15 @@ void closeAllRequests(SHashObj *pRequests) { void destroyTscObj(void *pObj) { STscObj *pTscObj = pObj; - SClientHbKey connKey = {.tscRid = *(int64_t *)pTscObj->id, .connType = pTscObj->connType}; + SClientHbKey connKey = {.tscRid = pTscObj->id, .connType = pTscObj->connType}; hbDeregisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey); int64_t connNum = atomic_sub_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); closeAllRequests(pTscObj->pRequests); schedulerStopQueryHb(pTscObj->pAppInfo->pTransporter); if (0 == connNum) { - // TODO - // closeTransporter(pTscObj); + closeTransporter(pTscObj); } - tscDebug("connObj 0x%" PRIx64 " destroyed, totalConn:%" PRId64, *(int64_t *)pTscObj->id, + tscDebug("connObj 0x%" PRIx64 " p:%p destroyed, totalConn:%" PRId64, pTscObj->id, pTscObj, pTscObj->pAppInfo->numOfConns); taosThreadMutexDestroy(&pTscObj->mutex); taosMemoryFreeClear(pTscObj); @@ -172,11 +171,10 @@ void *createTscObj(const char *user, const char *auth, const char *db, int32_t c } taosThreadMutexInit(&pObj->mutex, NULL); - pObj->id = taosMemoryMalloc(sizeof(int64_t)); - *(int64_t *)pObj->id = taosAddRef(clientConnRefPool, pObj); + pObj->id = taosAddRef(clientConnRefPool, pObj); pObj->schemalessType = 1; - tscDebug("connObj created, 0x%" PRIx64, *(int64_t *)pObj->id); + tscDebug("connObj created, 0x%" PRIx64 ",p:%p", pObj->id, pObj); return pObj; } diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 00bc7200b1..d3510b91ae 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1170,7 +1170,7 @@ STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __t taos_close_internal(pTscObj); pTscObj = NULL; } else { - tscDebug("0x%" PRIx64 " connection is opening, connId:%u, dnodeConn:%p, reqId:0x%" PRIx64, *(int64_t*)pTscObj->id, + tscDebug("0x%" PRIx64 " connection is opening, connId:%u, dnodeConn:%p, reqId:0x%" PRIx64, pTscObj->id, pTscObj->connId, pTscObj->pAppInfo->pTransporter, pRequest->requestId); destroyRequest(pRequest); } @@ -1333,7 +1333,9 @@ TAOS* taos_connect_auth(const char* ip, const char* user, const char* auth, cons STscObj* pObj = taos_connect_internal(ip, user, NULL, auth, db, port, CONN_TYPE__QUERY); if (pObj) { - return pObj->id; + int64_t *rid = taosMemoryCalloc(1, sizeof(int64_t)); + *rid = pObj->id; + return (TAOS*)rid; } return NULL; @@ -2016,11 +2018,18 @@ void syncQueryFn(void* param, void* res, int32_t code) { } void taosAsyncQueryImpl(TAOS* taos, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly) { - STscObj* pTscObj = acquireTscObj(*(int64_t*)taos); + if (NULL == taos) { + terrno = TSDB_CODE_TSC_DISCONNECTED; + fp(param, NULL, terrno); + return; + } + + int64_t rid = *(int64_t*)taos; + STscObj* pTscObj = acquireTscObj(rid); if (pTscObj == NULL || sql == NULL || NULL == fp) { terrno = TSDB_CODE_INVALID_PARA; if (pTscObj) { - releaseTscObj(*(int64_t*)taos); + releaseTscObj(rid); } else { terrno = TSDB_CODE_TSC_DISCONNECTED; } @@ -2032,7 +2041,7 @@ void taosAsyncQueryImpl(TAOS* taos, const char* sql, __taos_async_fn_t fp, void* if (sqlLen > (size_t)TSDB_MAX_ALLOWED_SQL_LEN) { tscError("sql string exceeds max length:%d", TSDB_MAX_ALLOWED_SQL_LEN); terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT; - releaseTscObj(*(int64_t *)taos); + releaseTscObj(rid); fp(param, NULL, terrno); return; @@ -2042,7 +2051,7 @@ void taosAsyncQueryImpl(TAOS* taos, const char* sql, __taos_async_fn_t fp, void* int32_t code = buildRequest(pTscObj, sql, sqlLen, &pRequest); if (code != TSDB_CODE_SUCCESS) { terrno = code; - releaseTscObj(*(int64_t *)taos); + releaseTscObj(rid); fp(param, NULL, terrno); return; } @@ -2051,7 +2060,7 @@ void taosAsyncQueryImpl(TAOS* taos, const char* sql, __taos_async_fn_t fp, void* pRequest->body.queryFp = fp; pRequest->body.param = param; doAsyncQuery(pRequest, false); - releaseTscObj(*(int64_t *)taos); + releaseTscObj(rid); } TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) { @@ -2060,7 +2069,8 @@ TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) { return NULL; } - STscObj* pTscObj = acquireTscObj(*(int64_t*)taos); + int64_t rid = *(int64_t*)taos; + STscObj* pTscObj = acquireTscObj(rid); if (pTscObj == NULL || sql == NULL) { terrno = TSDB_CODE_TSC_DISCONNECTED; return NULL; @@ -2070,16 +2080,16 @@ TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) { SSyncQueryParam* param = taosMemoryCalloc(1, sizeof(SSyncQueryParam)); tsem_init(¶m->sem, 0, 0); - taosAsyncQueryImpl(taos, sql, syncQueryFn, param, validateOnly); + taosAsyncQueryImpl((TAOS*)&rid, sql, syncQueryFn, param, validateOnly); tsem_wait(¶m->sem); - releaseTscObj(*(int64_t*)taos); + releaseTscObj(rid); return param->pRequest; #else size_t sqlLen = strlen(sql); if (sqlLen > (size_t)TSDB_MAX_ALLOWED_SQL_LEN) { - releaseTscObj(*(int64_t*)taos); + releaseTscObj(rid); tscError("sql string exceeds max length:%d", TSDB_MAX_ALLOWED_SQL_LEN); terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT; return NULL; @@ -2087,7 +2097,7 @@ TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) { TAOS_RES* pRes = execQuery(pTscObj, sql, sqlLen, validateOnly); - releaseTscObj(*(int64_t*)taos); + releaseTscObj(rid); return pRes; #endif diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index bbd477fa3b..a24e6faf7f 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -93,7 +93,9 @@ TAOS *taos_connect(const char *ip, const char *user, const char *pass, const cha STscObj *pObj = taos_connect_internal(ip, user, pass, NULL, db, port, CONN_TYPE__QUERY); if (pObj) { - return pObj->id; + int64_t *rid = taosMemoryCalloc(1, sizeof(int64_t)); + *rid = pObj->id; + return (TAOS*)rid; } return NULL; @@ -105,9 +107,9 @@ void taos_close_internal(void *taos) { } STscObj *pTscObj = (STscObj *)taos; - tscDebug("0x%" PRIx64 " try to close connection, numOfReq:%d", *(int64_t *)pTscObj->id, pTscObj->numOfReqs); + tscDebug("0x%" PRIx64 " try to close connection, numOfReq:%d", pTscObj->id, pTscObj->numOfReqs); - taosRemoveRef(clientConnRefPool, *(int64_t *)pTscObj->id); + taosRemoveRef(clientConnRefPool, pTscObj->id); } void taos_close(TAOS *taos) { @@ -880,6 +882,12 @@ void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress) { } int taos_load_table_info(TAOS *taos, const char *tableNameList) { + if (NULL == taos) { + terrno = TSDB_CODE_TSC_DISCONNECTED; + return terrno; + } + + int64_t rid = *(int64_t*)taos; const int32_t MAX_TABLE_NAME_LENGTH = 12 * 1024 * 1024; // 12MB list int32_t code = 0; SRequestObj *pRequest = NULL; @@ -897,7 +905,7 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) { return TSDB_CODE_TSC_INVALID_OPERATION; } - STscObj *pTscObj = acquireTscObj(*(int64_t *)taos); + STscObj *pTscObj = acquireTscObj(rid); if (pTscObj == NULL) { terrno = TSDB_CODE_TSC_DISCONNECTED; return terrno; @@ -942,7 +950,7 @@ _return: taosArrayDestroy(catalogReq.pTableMeta); destroyRequest(pRequest); - releaseTscObj(*(int64_t *)taos); + releaseTscObj(rid); return code; } diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 45a525d124..818762eff6 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -77,7 +77,7 @@ int32_t processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) { for (int32_t i = 0; i < connectRsp.epSet.numOfEps; ++i) { tscDebug("0x%" PRIx64 " epSet.fqdn[%d]:%s port:%d, connObj:0x%" PRIx64, pRequest->requestId, i, - connectRsp.epSet.eps[i].fqdn, connectRsp.epSet.eps[i].port, *(int64_t*)pTscObj->id); + connectRsp.epSet.eps[i].fqdn, connectRsp.epSet.eps[i].port, pTscObj->id); } pTscObj->connId = connectRsp.connId; @@ -91,7 +91,7 @@ int32_t processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) { pTscObj->connType = connectRsp.connType; - hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, *(int64_t*)pTscObj->id, connectRsp.clusterId, connectRsp.connType); + hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, pTscObj->id, connectRsp.clusterId, connectRsp.connType); // pRequest->body.resInfo.pRspMsg = pMsg->pData; tscDebug("0x%" PRIx64 " clusterId:%" PRId64 ", totalConn:%" PRId64, pRequest->requestId, connectRsp.clusterId, diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 7d2bf019d2..1cb0e2d54b 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -309,7 +309,7 @@ static int32_t smlApplySchemaAction(SSmlHandle *info, SSchemaAction *action) { case SCHEMA_ACTION_ADD_COLUMN: { int n = sprintf(result, "alter stable `%s` add column ", action->alterSTable.sTableName); smlBuildColumnDescription(action->alterSTable.field, result + n, capacity - n, &outBytes); - TAOS_RES *res = taos_query(info->taos->id, result); // TODO async doAsyncQuery + TAOS_RES *res = taos_query((TAOS*)&info->taos->id, result); // TODO async doAsyncQuery code = taos_errno(res); const char *errStr = taos_errstr(res); if (code != TSDB_CODE_SUCCESS) { @@ -323,7 +323,7 @@ static int32_t smlApplySchemaAction(SSmlHandle *info, SSchemaAction *action) { case SCHEMA_ACTION_ADD_TAG: { int n = sprintf(result, "alter stable `%s` add tag ", action->alterSTable.sTableName); smlBuildColumnDescription(action->alterSTable.field, result + n, capacity - n, &outBytes); - TAOS_RES *res = taos_query(info->taos->id, result); // TODO async doAsyncQuery + TAOS_RES *res = taos_query((TAOS*)&info->taos->id, result); // TODO async doAsyncQuery code = taos_errno(res); const char *errStr = taos_errstr(res); if (code != TSDB_CODE_SUCCESS) { @@ -337,7 +337,7 @@ static int32_t smlApplySchemaAction(SSmlHandle *info, SSchemaAction *action) { case SCHEMA_ACTION_CHANGE_COLUMN_SIZE: { int n = sprintf(result, "alter stable `%s` modify column ", action->alterSTable.sTableName); smlBuildColumnDescription(action->alterSTable.field, result + n, capacity - n, &outBytes); - TAOS_RES *res = taos_query(info->taos->id, result); // TODO async doAsyncQuery + TAOS_RES *res = taos_query((TAOS*)&info->taos->id, result); // TODO async doAsyncQuery code = taos_errno(res); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " apply schema action. error : %s", info->id, taos_errstr(res)); @@ -350,7 +350,7 @@ static int32_t smlApplySchemaAction(SSmlHandle *info, SSchemaAction *action) { case SCHEMA_ACTION_CHANGE_TAG_SIZE: { int n = sprintf(result, "alter stable `%s` modify tag ", action->alterSTable.sTableName); smlBuildColumnDescription(action->alterSTable.field, result + n, capacity - n, &outBytes); - TAOS_RES *res = taos_query(info->taos->id, result); // TODO async doAsyncQuery + TAOS_RES *res = taos_query((TAOS*)&info->taos->id, result); // TODO async doAsyncQuery code = taos_errno(res); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " apply schema action. error : %s", info->id, taos_errstr(res)); @@ -405,7 +405,7 @@ static int32_t smlApplySchemaAction(SSmlHandle *info, SSchemaAction *action) { pos--; ++freeBytes; outBytes = snprintf(pos, freeBytes, ")"); - TAOS_RES *res = taos_query(info->taos->id, result); + TAOS_RES *res = taos_query((TAOS*)&info->taos->id, result); code = taos_errno(res); if (code != TSDB_CODE_SUCCESS) { uError("SML:0x%" PRIx64 " apply schema action. error : %s", info->id, taos_errstr(res)); @@ -2436,7 +2436,13 @@ static void smlInsertCallback(void *param, void *res, int32_t code) { */ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision) { - STscObj* pTscObj = acquireTscObj(*(int64_t*)taos); + if (NULL == taos) { + terrno = TSDB_CODE_TSC_DISCONNECTED; + return NULL; + } + + int64_t rid = *(int64_t*)taos; + STscObj* pTscObj = acquireTscObj(rid); if (NULL == pTscObj) { terrno = TSDB_CODE_TSC_DISCONNECTED; uError("SML:taos_schemaless_insert invalid taos"); @@ -2445,7 +2451,7 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr SRequestObj* request = (SRequestObj*)createRequest(pTscObj, TSDB_SQL_INSERT); if(!request){ - releaseTscObj(*(int64_t*)taos); + releaseTscObj(rid); uError("SML:taos_schemaless_insert error request is null"); return NULL; } @@ -2533,6 +2539,6 @@ end: // ((STscObj *)taos)->schemalessType = 0; pTscObj->schemalessType = 1; uDebug("resultend:%s", request->msgBuf); - releaseTscObj(*(int64_t*)taos); + releaseTscObj(rid); return (TAOS_RES*)request; } From de06f055c2afb53ba532e56973eea5b1f1beb798 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 27 Jun 2022 13:33:29 +0800 Subject: [PATCH 04/61] enh: stop query --- source/client/inc/clientInt.h | 4 ++- source/client/src/clientEnv.c | 34 ++++++++++++++++------ source/client/src/clientHb.c | 42 ++++++++++++++++++++-------- source/client/src/clientImpl.c | 5 +++- source/client/src/clientMsgHandler.c | 1 - source/libs/catalog/src/ctgAsync.c | 2 ++ tests/script/api/stopquery.c | 1 + 7 files changed, 66 insertions(+), 23 deletions(-) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 48efe89361..9225d67131 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -66,7 +66,7 @@ enum { typedef struct SAppInstInfo SAppInstInfo; typedef struct { - char* key; + char* key; // statistics int32_t reportCnt; int32_t connKeyCnt; @@ -118,6 +118,7 @@ struct SAppInstInfo { uint64_t clusterId; void* pTransporter; SAppHbMgr* pAppHbMgr; + char* instKey; }; typedef struct SAppInfo { @@ -336,6 +337,7 @@ int hbHandleRsp(SClientHbBatchRsp* hbRsp); // cluster level SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo, char* key); void appHbMgrCleanup(void); +void hbRemoveAppHbMrg(SAppHbMgr **pAppHbMgr); // conn level int hbRegisterConn(SAppHbMgr* pAppHbMgr, int64_t tscRefId, int64_t clusterId, int8_t connType); diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index cad1b4b1cf..f5b5c48d47 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -79,13 +79,13 @@ static void deregisterRequest(SRequestObj *pRequest) { } // todo close the transporter properly -void closeTransporter(STscObj *pTscObj) { - if (pTscObj == NULL || pTscObj->pAppInfo->pTransporter == NULL) { +void closeTransporter(SAppInstInfo *pAppInfo) { + if (pAppInfo == NULL || pAppInfo->pTransporter == NULL) { return; } - tscDebug("free transporter:%p in connObj: 0x%" PRIx64, pTscObj->pAppInfo->pTransporter, pTscObj->id); - rpcClose(pTscObj->pAppInfo->pTransporter); + tscDebug("free transporter:%p in app inst %p", pAppInfo->pTransporter, pAppInfo); + rpcClose(pAppInfo->pTransporter); } static bool clientRpcRfp(int32_t code) { @@ -130,6 +130,21 @@ void closeAllRequests(SHashObj *pRequests) { } } +void destroyAppInst(SAppInstInfo* pAppInfo) { + tscDebug("destroy app inst mgr %p", pAppInfo); + + hbRemoveAppHbMrg(&pAppInfo->pAppHbMgr); + taosHashRemove(appInfo.pInstMap, pAppInfo->instKey, strlen(pAppInfo->instKey)); + taosMemoryFreeClear(pAppInfo->instKey); + closeTransporter(pAppInfo); + + taosThreadMutexLock(&pAppInfo->qnodeMutex); + taosArrayDestroy(pAppInfo->pQnodeList); + taosThreadMutexUnlock(&pAppInfo->qnodeMutex); + + taosMemoryFree(pAppInfo); +} + void destroyTscObj(void *pObj) { STscObj *pTscObj = pObj; @@ -138,11 +153,12 @@ void destroyTscObj(void *pObj) { int64_t connNum = atomic_sub_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); closeAllRequests(pTscObj->pRequests); schedulerStopQueryHb(pTscObj->pAppInfo->pTransporter); - if (0 == connNum) { - closeTransporter(pTscObj); - } - tscDebug("connObj 0x%" PRIx64 " p:%p destroyed, totalConn:%" PRId64, pTscObj->id, pTscObj, + tscDebug("connObj 0x%" PRIx64 " p:%p destroyed, remain inst totalConn:%" PRId64, pTscObj->id, pTscObj, pTscObj->pAppInfo->numOfConns); + + if (0 == connNum) { + destroyAppInst(pTscObj->pAppInfo); + } taosThreadMutexDestroy(&pTscObj->mutex); taosMemoryFreeClear(pTscObj); } @@ -174,6 +190,8 @@ void *createTscObj(const char *user, const char *auth, const char *db, int32_t c pObj->id = taosAddRef(clientConnRefPool, pObj); pObj->schemalessType = 1; + atomic_add_fetch_64(&pObj->pAppInfo->numOfConns, 1); + tscDebug("connObj created, 0x%" PRIx64 ",p:%p", pObj->id, pObj); return pObj; } diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 0681b5d0a5..4f4bb83fe5 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -790,22 +790,40 @@ SAppHbMgr *appHbMgrInit(SAppInstInfo *pAppInstInfo, char *key) { return pAppHbMgr; } +void hbFreeAppHbMgr(SAppHbMgr *pTarget) { + void *pIter = taosHashIterate(pTarget->activeInfo, NULL); + while (pIter != NULL) { + SClientHbReq *pOneReq = pIter; + tFreeClientHbReq(pOneReq); + pIter = taosHashIterate(pTarget->activeInfo, pIter); + } + taosHashCleanup(pTarget->activeInfo); + pTarget->activeInfo = NULL; + + taosMemoryFree(pTarget->key); + taosMemoryFree(pTarget); +} + +void hbRemoveAppHbMrg(SAppHbMgr **pAppHbMgr) { + taosThreadMutexLock(&clientHbMgr.lock); + int32_t mgrSize = taosArrayGetSize(clientHbMgr.appHbMgrs); + for (int32_t i = 0; i < mgrSize; ++i) { + SAppHbMgr *pItem = taosArrayGetP(clientHbMgr.appHbMgrs, i); + if (pItem == *pAppHbMgr) { + hbFreeAppHbMgr(*pAppHbMgr); + *pAppHbMgr = NULL; + taosArrayRemove(clientHbMgr.appHbMgrs, i); + break; + } + } + taosThreadMutexUnlock(&clientHbMgr.lock); +} + void appHbMgrCleanup(void) { int sz = taosArrayGetSize(clientHbMgr.appHbMgrs); for (int i = 0; i < sz; i++) { SAppHbMgr *pTarget = taosArrayGetP(clientHbMgr.appHbMgrs, i); - - void *pIter = taosHashIterate(pTarget->activeInfo, NULL); - while (pIter != NULL) { - SClientHbReq *pOneReq = pIter; - tFreeClientHbReq(pOneReq); - pIter = taosHashIterate(pTarget->activeInfo, pIter); - } - taosHashCleanup(pTarget->activeInfo); - pTarget->activeInfo = NULL; - - taosMemoryFree(pTarget->key); - taosMemoryFree(pTarget); + hbFreeAppHbMgr(pTarget); } } diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index d3510b91ae..8c63046323 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -122,7 +122,10 @@ STscObj* taos_connect_internal(const char* ip, const char* user, const char* pas p->pTransporter = openTransporter(user, secretEncrypt, tsNumOfCores); p->pAppHbMgr = appHbMgrInit(p, key); taosHashPut(appInfo.pInstMap, key, strlen(key), &p, POINTER_BYTES); - + p->instKey = key; + key = NULL; + tscDebug("new app inst mgr %p, user:%s, ip:%s, port:%d", p, user, ip, port); + pInst = &p; } diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 818762eff6..5c30df4ae2 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -87,7 +87,6 @@ int32_t processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) { // update the appInstInfo pTscObj->pAppInfo->clusterId = connectRsp.clusterId; - atomic_add_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); pTscObj->connType = connectRsp.connType; diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 6184d13533..139539821c 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -789,6 +789,8 @@ _return: int32_t ctgCallUserCb(void* param) { SCtgJob* pJob = (SCtgJob*)param; + + //taosSsleep(2); (*pJob->userFp)(&pJob->jobRes, pJob->userParam, pJob->jobResCode); diff --git a/tests/script/api/stopquery.c b/tests/script/api/stopquery.c index 4c7964c983..0008aa3303 100644 --- a/tests/script/api/stopquery.c +++ b/tests/script/api/stopquery.c @@ -368,6 +368,7 @@ void *closeThreadFp(void *arg) { while (true) { if (qParam->taos) { usleep(rand() % 10000); + //usleep(1000000); taos_close(qParam->taos); break; } From 49b2c544be13bd6c42c9b201cc9c6b30f36f71e9 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 27 Jun 2022 17:38:02 +0800 Subject: [PATCH 05/61] enh: change refMgt of rpc --- source/libs/transport/inc/transComm.h | 19 ++++++---- source/libs/transport/src/trans.c | 5 ++- source/libs/transport/src/transCli.c | 33 +++++++---------- source/libs/transport/src/transComm.c | 40 +++++++++++++------- source/libs/transport/src/transSvr.c | 53 ++++++++++----------------- 5 files changed, 74 insertions(+), 76 deletions(-) diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 963a85922f..a593dcf817 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -252,7 +252,7 @@ int transSendAsync(SAsyncPool* pool, queue* mq); do { \ if (id > 0) { \ tTrace("handle step1"); \ - SExHandle* exh2 = transAcquireExHandle(refMgt, id); \ + SExHandle* exh2 = transAcquireExHandle(id); \ if (exh2 == NULL || id != exh2->refId) { \ tTrace("handle %p except, may already freed, ignore msg, ref1: %" PRIu64 ", ref2 : %" PRIu64 "", exh1, \ exh2 ? exh2->refId : 0, id); \ @@ -260,7 +260,7 @@ int transSendAsync(SAsyncPool* pool, queue* mq); } \ } else if (id == 0) { \ tTrace("handle step2"); \ - SExHandle* exh2 = transAcquireExHandle(refMgt, id); \ + SExHandle* exh2 = transAcquireExHandle(id); \ if (exh2 == NULL || id == exh2->refId) { \ tTrace("handle %p except, may already freed, ignore msg, ref1: %" PRIu64 ", ref2 : %" PRIu64 "", exh1, id, \ exh2 ? exh2->refId : 0); \ @@ -273,6 +273,7 @@ int transSendAsync(SAsyncPool* pool, queue* mq); goto _return2; \ } \ } while (0) + int transInitBuffer(SConnBuffer* buf); int transClearBuffer(SConnBuffer* buf); int transDestroyBuffer(SConnBuffer* buf); @@ -390,13 +391,15 @@ bool transEpSetIsEqual(SEpSet* a, SEpSet* b); */ void transThreadOnce(); -void transInitEnv(); +void transInit(); +void transCleanup(); + int32_t transOpenExHandleMgt(int size); -void transCloseExHandleMgt(int32_t mgt); -int64_t transAddExHandle(int32_t mgt, void* p); -int32_t transRemoveExHandle(int32_t mgt, int64_t refId); -SExHandle* transAcquireExHandle(int32_t mgt, int64_t refId); -int32_t transReleaseExHandle(int32_t mgt, int64_t refId); +void transCloseExHandleMgt(); +int64_t transAddExHandle(void* p); +int32_t transRemoveExHandle(int64_t refId); +SExHandle* transAcquireExHandle(int64_t refId); +int32_t transReleaseExHandle(int64_t refId); void transDestoryExHandle(void* handle); #ifdef __cplusplus diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 4f7b19b539..d48bd85832 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -36,7 +36,7 @@ static int32_t transValidLocalFqdn(const char* localFqdn, uint32_t* ip) { return 0; } void* rpcOpen(const SRpcInit* pInit) { - transInitEnv(); + rpcInit(); SRpcInfo* pRpc = taosMemoryCalloc(1, sizeof(SRpcInfo)); if (pRpc == NULL) { @@ -82,7 +82,6 @@ void rpcClose(void* arg) { tInfo("start to close rpc"); SRpcInfo* pRpc = (SRpcInfo*)arg; (*taosCloseHandle[pRpc->connType])(pRpc->tcphandle); - transCloseExHandleMgt(pRpc->refMgt); taosMemoryFree(pRpc); return; @@ -170,11 +169,13 @@ void rpcSetDefaultAddr(void* thandle, const char* ip, const char* fqdn) { //} int32_t rpcInit() { + transInit(); // impl later return 0; } void rpcCleanup(void) { // impl later + transCleanup(); return; } diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 7374d1fffc..c960cb4c63 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -15,9 +15,6 @@ #ifdef USE_UV #include "transComm.h" -static int32_t transSCliInst = 0; -static int32_t refMgt = 0; - typedef struct SCliConn { T_REF_DECLARE() uv_connect_t connReq; @@ -503,12 +500,12 @@ static SCliConn* getConnFromPool(void* pool, char* ip, uint32_t port) { } static void allocConnRef(SCliConn* conn, bool update) { if (update) { - transRemoveExHandle(refMgt, conn->refId); + transRemoveExHandle(conn->refId); } SExHandle* exh = taosMemoryCalloc(1, sizeof(SExHandle)); exh->handle = conn; exh->pThrd = conn->hostThrd; - exh->refId = transAddExHandle(refMgt, exh); + exh->refId = transAddExHandle(exh); conn->refId = exh->refId; } static void addConnToPool(void* pool, SCliConn* conn) { @@ -602,9 +599,13 @@ static void cliDestroyConn(SCliConn* conn, bool clear) { tTrace("%s conn %p remove from conn pool", CONN_GET_INST_LABEL(conn), conn); QUEUE_REMOVE(&conn->conn); QUEUE_INIT(&conn->conn); - transRemoveExHandle(refMgt, conn->refId); + transRemoveExHandle(conn->refId); if (clear) { - uv_close((uv_handle_t*)conn->stream, cliDestroy); + if (uv_is_active((uv_handle_t*)conn->stream)) { + uv_close((uv_handle_t*)conn->stream, cliDestroy); + } else { + cliDestroy((uv_handle_t*)conn->stream); + } } } static void cliDestroy(uv_handle_t* handle) { @@ -735,7 +736,7 @@ static void cliHandleQuit(SCliMsg* pMsg, SCliThrd* pThrd) { } static void cliHandleRelease(SCliMsg* pMsg, SCliThrd* pThrd) { int64_t refId = (int64_t)(pMsg->msg.info.handle); - SExHandle* exh = transAcquireExHandle(refMgt, refId); + SExHandle* exh = transAcquireExHandle(refId); if (exh == NULL) { tDebug("%" PRId64 " already release", refId); } @@ -761,7 +762,7 @@ SCliConn* cliGetConn(SCliMsg* pMsg, SCliThrd* pThrd, bool* ignore) { SCliConn* conn = NULL; int64_t refId = (int64_t)(pMsg->msg.info.handle); if (refId != 0) { - SExHandle* exh = transAcquireExHandle(refMgt, refId); + SExHandle* exh = transAcquireExHandle(refId); if (exh == NULL) { *ignore = true; destroyCmsg(pMsg); @@ -769,7 +770,7 @@ SCliConn* cliGetConn(SCliMsg* pMsg, SCliThrd* pThrd, bool* ignore) { // assert(0); } else { conn = exh->handle; - transReleaseExHandle(refMgt, refId); + transReleaseExHandle(refId); } return conn; }; @@ -899,10 +900,6 @@ void* transInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, } cli->pThreadObj[i] = pThrd; } - int ref = atomic_add_fetch_32(&transSCliInst, 1); - if (ref == 1) { - refMgt = transOpenExHandleMgt(50000); - } return cli; } @@ -1086,10 +1083,6 @@ void transCloseClient(void* arg) { } taosMemoryFree(cli->pThreadObj); taosMemoryFree(cli); - int ref = atomic_sub_fetch_32(&transSCliInst, 1); - if (ref == 0) { - transCloseExHandleMgt(refMgt); - } } void transRefCliHandle(void* handle) { if (handle == NULL) { @@ -1111,12 +1104,12 @@ void transUnrefCliHandle(void* handle) { } SCliThrd* transGetWorkThrdFromHandle(int64_t handle) { SCliThrd* pThrd = NULL; - SExHandle* exh = transAcquireExHandle(refMgt, handle); + SExHandle* exh = transAcquireExHandle(handle); if (exh == NULL) { return NULL; } pThrd = exh->pThrd; - transReleaseExHandle(refMgt, handle); + transReleaseExHandle(handle); return pThrd; } SCliThrd* transGetWorkThrd(STrans* trans, int64_t handle) { diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index bff7d79bd3..bc158bb316 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -16,7 +16,9 @@ #include "transComm.h" -// static TdThreadOnce transModuleInit = PTHREAD_ONCE_INIT; +static TdThreadOnce transModuleInit = PTHREAD_ONCE_INIT; + +static int32_t refMgt; int transAuthenticateMsg(void* pMsg, int msgLen, void* pAuth, void* pKey) { T_MD5_CTX context; @@ -478,35 +480,47 @@ bool transEpSetIsEqual(SEpSet* a, SEpSet* b) { return true; } -void transInitEnv() { - // +static void transInitEnv() { + refMgt = transOpenExHandleMgt(50000); uv_os_setenv("UV_TCP_SINGLE_ACCEPT", "1"); } +static void transDestroyEnv() { + // close ref + transCloseExHandleMgt(refMgt); +} +void transInit() { + // init env + taosThreadOnce(&transModuleInit, transInitEnv); +} +void transCleanup() { + // clean env + transDestroyEnv(); +} int32_t transOpenExHandleMgt(int size) { // added into once later return taosOpenRef(size, transDestoryExHandle); } -void transCloseExHandleMgt(int32_t mgt) { +void transCloseExHandleMgt() { // close ref - taosCloseRef(mgt); + taosCloseRef(refMgt); } -int64_t transAddExHandle(int32_t mgt, void* p) { +int64_t transAddExHandle(void* p) { // acquire extern handle - return taosAddRef(mgt, p); + return taosAddRef(refMgt, p); } -int32_t transRemoveExHandle(int32_t mgt, int64_t refId) { +int32_t transRemoveExHandle(int64_t refId) { // acquire extern handle - return taosRemoveRef(mgt, refId); + return taosRemoveRef(refMgt, refId); } -SExHandle* transAcquireExHandle(int32_t mgt, int64_t refId) { +SExHandle* transAcquireExHandle(int64_t refId) { // acquire extern handle - return (SExHandle*)taosAcquireRef(mgt, refId); + return (SExHandle*)taosAcquireRef(refMgt, refId); } -int32_t transReleaseExHandle(int32_t mgt, int64_t refId) { +int32_t transReleaseExHandle(int64_t refId) { // release extern handle - return taosReleaseRef(mgt, refId); + return taosReleaseRef(refMgt, refId); } void transDestoryExHandle(void* handle) { if (handle == NULL) { diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 892d32696e..a78adfb397 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -19,9 +19,7 @@ static TdThreadOnce transModuleInit = PTHREAD_ONCE_INIT; -static char* notify = "a"; -static int32_t tranSSvrInst = 0; -static int32_t refMgt = 0; +static char* notify = "a"; typedef struct { int notifyCount; // @@ -274,7 +272,7 @@ static void uvHandleReq(SSvrConn* pConn) { // 2. once send out data, cli conn released to conn pool immediately // 3. not mixed with persist transMsg.info.ahandle = (void*)pHead->ahandle; - transMsg.info.handle = (void*)transAcquireExHandle(refMgt, pConn->refId); + transMsg.info.handle = (void*)transAcquireExHandle(pConn->refId); transMsg.info.refId = pConn->refId; transMsg.info.traceId = pHead->traceId; @@ -292,7 +290,7 @@ static void uvHandleReq(SSvrConn* pConn) { pConnInfo->clientPort = ntohs(pConn->addr.sin_port); tstrncpy(pConnInfo->user, pConn->user, sizeof(pConnInfo->user)); - transReleaseExHandle(refMgt, pConn->refId); + transReleaseExHandle(pConn->refId); STrans* pTransInst = pConn->pTransInst; (*pTransInst->cfp)(pTransInst->parent, &transMsg, NULL); @@ -523,15 +521,15 @@ void uvWorkerAsyncCb(uv_async_t* handle) { SExHandle* exh1 = transMsg.info.handle; int64_t refId = transMsg.info.refId; - SExHandle* exh2 = transAcquireExHandle(refMgt, refId); + SExHandle* exh2 = transAcquireExHandle(refId); if (exh2 == NULL || exh1 != exh2) { tTrace("handle except msg %p, ignore it", exh1); - transReleaseExHandle(refMgt, refId); + transReleaseExHandle(refId); destroySmsg(msg); continue; } msg->pConn = exh1->handle; - transReleaseExHandle(refMgt, refId); + transReleaseExHandle(refId); (*transAsyncHandle[msg->type])(msg, pThrd); } } @@ -773,8 +771,8 @@ static SSvrConn* createConn(void* hThrd) { SExHandle* exh = taosMemoryMalloc(sizeof(SExHandle)); exh->handle = pConn; exh->pThrd = pThrd; - exh->refId = transAddExHandle(refMgt, exh); - transAcquireExHandle(refMgt, exh->refId); + exh->refId = transAddExHandle(exh); + transAcquireExHandle(exh->refId); pConn->refId = exh->refId; transRefSrvHandle(pConn); @@ -803,14 +801,14 @@ static void destroyConnRegArg(SSvrConn* conn) { } } static int reallocConnRef(SSvrConn* conn) { - transReleaseExHandle(refMgt, conn->refId); - transRemoveExHandle(refMgt, conn->refId); + transReleaseExHandle(conn->refId); + transRemoveExHandle(conn->refId); // avoid app continue to send msg on invalid handle SExHandle* exh = taosMemoryMalloc(sizeof(SExHandle)); exh->handle = conn; exh->pThrd = conn->hostThrd; - exh->refId = transAddExHandle(refMgt, exh); - transAcquireExHandle(refMgt, exh->refId); + exh->refId = transAddExHandle(exh); + transAcquireExHandle(exh->refId); conn->refId = exh->refId; return 0; @@ -822,8 +820,8 @@ static void uvDestroyConn(uv_handle_t* handle) { } SWorkThrd* thrd = conn->hostThrd; - transReleaseExHandle(refMgt, conn->refId); - transRemoveExHandle(refMgt, conn->refId); + transReleaseExHandle(conn->refId); + transRemoveExHandle(conn->refId); tDebug("%s conn %p destroy", transLabel(thrd->pTransInst), conn); // uv_timer_stop(&conn->pTimer); @@ -871,12 +869,6 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, srv->port = port; uv_loop_init(srv->loop); - // taosThreadOnce(&transModuleInit, uvInitEnv); - int ref = atomic_add_fetch_32(&tranSSvrInst, 1); - if (ref == 1) { - refMgt = transOpenExHandleMgt(50000); - } - assert(0 == uv_pipe_init(srv->loop, &srv->pipeListen, 0)); #ifdef WINDOWS char pipeName[64]; @@ -1028,11 +1020,6 @@ void transCloseServer(void* arg) { taosMemoryFree(srv->pipe); taosMemoryFree(srv); - - int ref = atomic_sub_fetch_32(&tranSSvrInst, 1); - if (ref == 0) { - transCloseExHandleMgt(refMgt); - } } void transRefSrvHandle(void* handle) { @@ -1071,11 +1058,11 @@ void transReleaseSrvHandle(void* handle) { tTrace("%s conn %p start to release", transLabel(pThrd->pTransInst), exh->handle); transSendAsync(pThrd->asyncPool, &m->q); - transReleaseExHandle(refMgt, refId); + transReleaseExHandle(refId); return; _return1: tTrace("handle %p failed to send to release handle", exh); - transReleaseExHandle(refMgt, refId); + transReleaseExHandle(refId); return; _return2: tTrace("handle %p failed to send to release handle", exh); @@ -1100,12 +1087,12 @@ void transSendResponse(const STransMsg* msg) { STraceId* trace = (STraceId*)&msg->info.traceId; tGTrace("conn %p start to send resp (1/2)", exh->handle); transSendAsync(pThrd->asyncPool, &m->q); - transReleaseExHandle(refMgt, refId); + transReleaseExHandle(refId); return; _return1: tTrace("handle %p failed to send resp", exh); rpcFreeCont(msg->pCont); - transReleaseExHandle(refMgt, refId); + transReleaseExHandle(refId); return; _return2: tTrace("handle %p failed to send resp", exh); @@ -1129,13 +1116,13 @@ void transRegisterMsg(const STransMsg* msg) { tTrace("%s conn %p start to register brokenlink callback", transLabel(pThrd->pTransInst), exh->handle); transSendAsync(pThrd->asyncPool, &m->q); - transReleaseExHandle(refMgt, refId); + transReleaseExHandle(refId); return; _return1: tTrace("handle %p failed to register brokenlink", exh); rpcFreeCont(msg->pCont); - transReleaseExHandle(refMgt, refId); + transReleaseExHandle(refId); return; _return2: tTrace("handle %p failed to register brokenlink", exh); From 393a5dcf661c4681d4f40ef63b17bf56310a7390 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 27 Jun 2022 18:25:16 +0800 Subject: [PATCH 06/61] enh: change refMgt of rpc --- source/libs/transport/src/trans.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index d48bd85832..3c09877106 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -163,11 +163,6 @@ void rpcSetDefaultAddr(void* thandle, const char* ip, const char* fqdn) { transSetDefaultAddr(thandle, ip, fqdn); } -// void rpcSetMsgTraceId(SRpcMsg* pMsg, STraceId uid) { -// SRpcHandleInfo* pInfo = &pMsg->info; -// pInfo->traceId = uid; -//} - int32_t rpcInit() { transInit(); // impl later From d40271646594a57a69e6a5ce996b18c3adda3dac Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 27 Jun 2022 18:29:28 +0800 Subject: [PATCH 07/61] enh: stop query --- include/client/taos.h | 1 + source/client/inc/clientInt.h | 1 + source/client/src/clientHb.c | 22 +++++++++++++------- source/client/src/clientMain.c | 11 ++++++++++ source/dnode/mnode/impl/src/mndMain.c | 5 +++++ source/libs/catalog/src/ctgAsync.c | 5 +++-- source/libs/catalog/src/ctgCache.c | 14 ++++++------- source/libs/scalar/src/scalar.c | 30 ++++++++++++++++++++++++--- tests/script/api/stopquery.c | 6 +++--- 9 files changed, 72 insertions(+), 23 deletions(-) diff --git a/include/client/taos.h b/include/client/taos.h index 669512e9b1..651faf5dcc 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -163,6 +163,7 @@ DLL_EXPORT TAOS_RES *taos_query(TAOS *taos, const char *sql); DLL_EXPORT TAOS_ROW taos_fetch_row(TAOS_RES *res); DLL_EXPORT int taos_result_precision(TAOS_RES *res); // get the time precision of result DLL_EXPORT void taos_free_result(TAOS_RES *res); +DLL_EXPORT void taos_kill_query(TAOS *taos); DLL_EXPORT int taos_field_count(TAOS_RES *res); DLL_EXPORT int taos_num_fields(TAOS_RES *res); DLL_EXPORT int taos_affected_rows(TAOS_RES *res); diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 9225d67131..3b21258fe1 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -338,6 +338,7 @@ int hbHandleRsp(SClientHbBatchRsp* hbRsp); SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo, char* key); void appHbMgrCleanup(void); void hbRemoveAppHbMrg(SAppHbMgr **pAppHbMgr); +void closeAllRequests(SHashObj *pRequests); // conn level int hbRegisterConn(SAppHbMgr* pAppHbMgr, int64_t tscRefId, int64_t clusterId, int8_t connType); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 4f4bb83fe5..43e83dd501 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -66,25 +66,31 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog if (rsp->vgVersion < 0) { code = catalogRemoveDB(pCatalog, rsp->db, rsp->uid); } else { - SDBVgInfo vgInfo = {0}; - vgInfo.vgVersion = rsp->vgVersion; - vgInfo.hashMethod = rsp->hashMethod; - vgInfo.vgHash = taosHashInit(rsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); - if (NULL == vgInfo.vgHash) { + SDBVgInfo *vgInfo = taosMemoryCalloc(1, sizeof(SDBVgInfo)); + if (NULL == vgInfo) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + vgInfo->vgVersion = rsp->vgVersion; + vgInfo->hashMethod = rsp->hashMethod; + vgInfo->vgHash = taosHashInit(rsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); + if (NULL == vgInfo->vgHash) { + taosMemoryFree(vgInfo); tscError("hash init[%d] failed", rsp->vgNum); return TSDB_CODE_TSC_OUT_OF_MEMORY; } for (int32_t j = 0; j < rsp->vgNum; ++j) { SVgroupInfo *pInfo = taosArrayGet(rsp->pVgroupInfos, j); - if (taosHashPut(vgInfo.vgHash, &pInfo->vgId, sizeof(int32_t), pInfo, sizeof(SVgroupInfo)) != 0) { + if (taosHashPut(vgInfo->vgHash, &pInfo->vgId, sizeof(int32_t), pInfo, sizeof(SVgroupInfo)) != 0) { tscError("hash push failed, errno:%d", errno); - taosHashCleanup(vgInfo.vgHash); + taosHashCleanup(vgInfo->vgHash); + taosMemoryFree(vgInfo); return TSDB_CODE_TSC_OUT_OF_MEMORY; } } - catalogUpdateDBVgInfo(pCatalog, rsp->db, rsp->uid, &vgInfo); + catalogUpdateDBVgInfo(pCatalog, rsp->db, rsp->uid, vgInfo); } if (code) { diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index a24e6faf7f..be3efe0f98 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -181,6 +181,17 @@ void taos_free_result(TAOS_RES *res) { } } +void taos_kill_query(TAOS *taos) { + if (NULL == taos) { + return; + } + int64_t rid = *(int64_t*)taos; + + STscObj* pTscObj = acquireTscObj(rid); + closeAllRequests(pTscObj->pRequests); + releaseTscObj(rid); +} + int taos_field_count(TAOS_RES *res) { if (res == NULL || TD_RES_TMQ_META(res)) { return 0; diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 4c9974ba1a..e03f82ba18 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -527,6 +527,11 @@ int32_t mndProcessSyncMsg(SRpcMsg *pMsg) { static int32_t mndCheckMnodeState(SRpcMsg *pMsg) { if (!IsReq(pMsg)) return 0; + if (pMsg->msgType == TDMT_VND_QUERY || pMsg->msgType == TDMT_VND_QUERY_CONTINUE || + pMsg->msgType == TDMT_VND_QUERY_HEARTBEAT || pMsg->msgType == TDMT_VND_FETCH || + pMsg->msgType == TDMT_VND_DROP_TASK) { + return 0; + } if (mndAcquireRpcRef(pMsg->info.node) == 0) return 0; if (pMsg->msgType == TDMT_MND_MQ_TIMER || pMsg->msgType == TDMT_MND_TELEM_TIMER || pMsg->msgType == TDMT_MND_TRANS_TIMER || pMsg->msgType == TDMT_MND_TTL_TIMER) { diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 139539821c..683cbdb47c 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -789,8 +789,6 @@ _return: int32_t ctgCallUserCb(void* param) { SCtgJob* pJob = (SCtgJob*)param; - - //taosSsleep(2); (*pJob->userFp)(&pJob->jobRes, pJob->userParam, pJob->jobResCode); @@ -827,6 +825,9 @@ _return: qDebug("QID:0x%" PRIx64 " ctg call user callback with rsp %s", pJob->queryId, tstrerror(code)); pJob->jobResCode = code; + + taosSsleep(2); + qDebug("QID:0x%" PRIx64 " ctg after sleep", pJob->queryId); taosAsyncExec(ctgCallUserCb, pJob, NULL); diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 8cd6c7d203..301cdbef20 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -1564,27 +1564,27 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { SCatalog* pCtg = msg->pCtg; if (NULL == dbInfo->vgHash) { - return TSDB_CODE_SUCCESS; + goto _return; } if (dbInfo->vgVersion < 0 || taosHashGetSize(dbInfo->vgHash) <= 0) { ctgError("invalid db vgInfo, dbFName:%s, vgHash:%p, vgVersion:%d, vgHashSize:%d", dbFName, dbInfo->vgHash, dbInfo->vgVersion, taosHashGetSize(dbInfo->vgHash)); - CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + CTG_ERR_JRET(TSDB_CODE_APP_ERROR); } bool newAdded = false; SDbVgVersion vgVersion = {.dbId = msg->dbId, .vgVersion = dbInfo->vgVersion, .numOfTable = dbInfo->numOfTable}; SCtgDBCache *dbCache = NULL; - CTG_ERR_RET(ctgGetAddDBCache(msg->pCtg, dbFName, msg->dbId, &dbCache)); + CTG_ERR_JRET(ctgGetAddDBCache(msg->pCtg, dbFName, msg->dbId, &dbCache)); if (NULL == dbCache) { ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:0x%"PRIx64, dbFName, msg->dbId); - CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); + CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } SCtgVgCache *vgCache = &dbCache->vgCache; - CTG_ERR_RET(ctgWLockVgInfo(msg->pCtg, dbCache)); + CTG_ERR_JRET(ctgWLockVgInfo(msg->pCtg, dbCache)); if (vgCache->vgInfo) { SDBVgInfo *vgInfo = vgCache->vgInfo; @@ -1593,14 +1593,14 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { ctgDebug("db vgVer is old, dbFName:%s, vgVer:%d, curVer:%d", dbFName, dbInfo->vgVersion, vgInfo->vgVersion); ctgWUnlockVgInfo(dbCache); - return TSDB_CODE_SUCCESS; + goto _return; } if (dbInfo->vgVersion == vgInfo->vgVersion && dbInfo->numOfTable == vgInfo->numOfTable) { ctgDebug("no new db vgVer or numOfTable, dbFName:%s, vgVer:%d, numOfTable:%d", dbFName, dbInfo->vgVersion, dbInfo->numOfTable); ctgWUnlockVgInfo(dbCache); - return TSDB_CODE_SUCCESS; + goto _return; } ctgFreeVgInfo(vgInfo); diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 75a9bc3809..cef8ff7075 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -936,6 +936,25 @@ EDealRes sclCalcWalker(SNode* pNode, void* pContext) { return DEAL_RES_ERROR; } +int32_t sclExtendResRows(SScalarParam *pDst, SScalarParam *pSrc, SArray *pBlockList) { + SSDataBlock* pb = taosArrayGetP(pBlockList, 0); + SScalarParam *pLeft = taosMemoryCalloc(1, sizeof(SScalarParam)); + if (NULL == pLeft) { + sclError("calloc %d failed", (int32_t)sizeof(SScalarParam)); + SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + pLeft->numOfRows = pb->info.rows; + colInfoDataEnsureCapacity(pDst->columnData, pb->info.rows); + + _bin_scalar_fn_t OperatorFn = getBinScalarOperatorFn(OP_TYPE_ASSIGN); + OperatorFn(pLeft, pSrc, pDst, TSDB_ORDER_ASC); + + taosMemoryFree(pLeft); + + return TSDB_CODE_SUCCESS; +} + int32_t scalarCalculateConstants(SNode *pNode, SNode **pRes) { if (NULL == pNode) { SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); @@ -983,9 +1002,14 @@ int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst) { SCL_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } - colInfoDataEnsureCapacity(pDst->columnData, res->numOfRows); - colDataAssign(pDst->columnData, res->columnData, res->numOfRows, NULL); - pDst->numOfRows = res->numOfRows; + if (1 == res->numOfRows) { + SCL_ERR_JRET(sclExtendResRows(pDst, res, pBlockList)); + } else { + colInfoDataEnsureCapacity(pDst->columnData, res->numOfRows); + colDataAssign(pDst->columnData, res->columnData, res->numOfRows, NULL); + pDst->numOfRows = res->numOfRows; + } + taosHashRemove(ctx.pRes, (void *)&pNode, POINTER_BYTES); } diff --git a/tests/script/api/stopquery.c b/tests/script/api/stopquery.c index 0008aa3303..addd1272cf 100644 --- a/tests/script/api/stopquery.c +++ b/tests/script/api/stopquery.c @@ -36,7 +36,7 @@ int64_t st, et; char hostName[128]; char dbName[128]; char tbName[128]; -int32_t runTimes = 10000; +int32_t runTimes = 1; typedef struct { int id; @@ -367,8 +367,8 @@ void *closeThreadFp(void *arg) { SSP_CB_PARAM* qParam = (SSP_CB_PARAM*)arg; while (true) { if (qParam->taos) { - usleep(rand() % 10000); - //usleep(1000000); + //usleep(rand() % 10000); + usleep(1000000); taos_close(qParam->taos); break; } From 1d8a0d622678abf331f7553285098a42074f93da Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 27 Jun 2022 21:00:40 +0800 Subject: [PATCH 08/61] feat: refactor rpc code --- source/libs/transport/src/transSvr.c | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index a78adfb397..f1937c93fe 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -140,17 +140,6 @@ static void uvHandleRegister(SSvrMsg* msg, SWorkThrd* thrd); static void (*transAsyncHandle[])(SSvrMsg* msg, SWorkThrd* thrd) = {uvHandleResp, uvHandleQuit, uvHandleRelease, uvHandleRegister, NULL}; -static int32_t exHandlesMgt; - -// void uvInitEnv(); -// void uvOpenExHandleMgt(int size); -// void uvCloseExHandleMgt(); -// int64_t uvAddExHandle(void* p); -// int32_t uvRemoveExHandle(int64_t refId); -// int32_t uvReleaseExHandle(int64_t refId); -// void uvDestoryExHandle(void* handle); -// SExHandle* uvAcquireExHandle(int64_t refId); - static void uvDestroyConn(uv_handle_t* handle); // server and worker thread @@ -787,11 +776,15 @@ static void destroyConn(SSvrConn* conn, bool clear) { transDestroyBuffer(&conn->readBuf); if (clear) { - tTrace("conn %p to be destroyed", conn); - // uv_shutdown_t* req = taosMemoryMalloc(sizeof(uv_shutdown_t)); - uv_close((uv_handle_t*)conn->pTcp, uvDestroyConn); - // uv_close(conn->pTcp) - // uv_shutdown(req, (uv_stream_t*)conn->pTcp, uvShutDownCb); + if (uv_is_active((uv_handle_t*)conn->pTcp)) { + tTrace("conn %p to be destroyed", conn); + // uv_shutdown_t* req = taosMemoryMalloc(sizeof(uv_shutdown_t)); + uv_close((uv_handle_t*)conn->pTcp, uvDestroyConn); + // uv_close(conn->pTcp) + // uv_shutdown(req, (uv_stream_t*)conn->pTcp, uvShutDownCb); + } else { + uvDestroyConn((uv_handle_t*)conn->pTcp); + } } } static void destroyConnRegArg(SSvrConn* conn) { @@ -824,7 +817,6 @@ static void uvDestroyConn(uv_handle_t* handle) { transRemoveExHandle(conn->refId); tDebug("%s conn %p destroy", transLabel(thrd->pTransInst), conn); - // uv_timer_stop(&conn->pTimer); transQueueDestroy(&conn->srvMsgs); QUEUE_REMOVE(&conn->queue); From 1ab7e3a90b70848d1fd8fbbf18d6d402530f9e32 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 27 Jun 2022 21:07:41 +0800 Subject: [PATCH 09/61] feat: refactor rpc code --- source/libs/transport/src/trans.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 3c09877106..a26725c41c 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -165,11 +165,9 @@ void rpcSetDefaultAddr(void* thandle, const char* ip, const char* fqdn) { int32_t rpcInit() { transInit(); - // impl later return 0; } void rpcCleanup(void) { - // impl later transCleanup(); return; } From c9d9c3ffd03a8ae2fc5668780d71ef7cd056d9f1 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 28 Jun 2022 09:28:50 +0800 Subject: [PATCH 10/61] enh: stop query --- source/client/src/clientEnv.c | 5 ++++ source/client/src/clientHb.c | 5 ++++ source/libs/catalog/src/ctgAsync.c | 4 +-- source/libs/catalog/src/ctgCache.c | 2 +- source/libs/scheduler/inc/schedulerInt.h | 4 +-- source/libs/scheduler/src/schJob.c | 32 ++++++------------------ tests/script/api/stopquery.c | 5 ++-- 7 files changed, 23 insertions(+), 34 deletions(-) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index f5b5c48d47..56b675596f 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -132,9 +132,14 @@ void closeAllRequests(SHashObj *pRequests) { void destroyAppInst(SAppInstInfo* pAppInfo) { tscDebug("destroy app inst mgr %p", pAppInfo); + + taosThreadMutexLock(&appInfo.mutex); hbRemoveAppHbMrg(&pAppInfo->pAppHbMgr); taosHashRemove(appInfo.pInstMap, pAppInfo->instKey, strlen(pAppInfo->instKey)); + + taosThreadMutexUnlock(&appInfo.mutex); + taosMemoryFreeClear(pAppInfo->instKey); closeTransporter(pAppInfo); diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 43e83dd501..2de630e181 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -275,8 +275,11 @@ static int32_t hbAsyncCallBack(void *param, const SDataBuf *pMsg, int32_t code) int32_t rspNum = taosArrayGetSize(pRsp.rsps); + taosThreadMutexLock(&appInfo.mutex); + SAppInstInfo **pInst = taosHashGet(appInfo.pInstMap, key, strlen(key)); if (pInst == NULL || NULL == *pInst) { + taosThreadMutexUnlock(&appInfo.mutex); tscError("cluster not exist, key:%s", key); taosMemoryFreeClear(param); tFreeClientHbBatchRsp(&pRsp); @@ -300,6 +303,8 @@ static int32_t hbAsyncCallBack(void *param, const SDataBuf *pMsg, int32_t code) } } + taosThreadMutexUnlock(&appInfo.mutex); + tFreeClientHbBatchRsp(&pRsp); return code; diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 683cbdb47c..455f2bd6a7 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -826,8 +826,8 @@ _return: pJob->jobResCode = code; - taosSsleep(2); - qDebug("QID:0x%" PRIx64 " ctg after sleep", pJob->queryId); + //taosSsleep(2); + //qDebug("QID:0x%" PRIx64 " ctg after sleep", pJob->queryId); taosAsyncExec(ctgCallUserCb, pJob, NULL); diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 301cdbef20..77c1e5b8b1 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -1083,7 +1083,7 @@ int32_t ctgMetaRentUpdate(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t si CTG_LOCK(CTG_WRITE, &slot->lock); if (NULL == slot->meta) { - qError("empty meta slot, id:0x%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); + qDebug("empty meta slot, id:0x%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index 843ce7d55a..a119795787 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -218,9 +218,7 @@ typedef struct SSchJob { int32_t levelIdx; SEpSet dataSrcEps; SHashObj *taskList; - SHashObj *execTasks; // executing tasks, key:taskid, value:SQueryTask* - SHashObj *succTasks; // succeed tasks, key:taskid, value:SQueryTask* - SHashObj *failTasks; // failed tasks, key:taskid, value:SQueryTask* + SHashObj *execTasks; // executing and executed tasks, key:taskid, value:SQueryTask* SHashObj *flowCtrl; // key is ep, element is SSchFlowControl SExplainCtx *explainCtx; diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index 7843482af3..89f355d78c 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -85,20 +85,6 @@ int32_t schInitJob(SSchedulerReq *pReq, SSchJob **pSchJob) { SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - pJob->succTasks = - taosHashInit(pReq->pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); - if (NULL == pJob->succTasks) { - SCH_JOB_ELOG("taosHashInit %d succTasks failed", pReq->pDag->numOfSubplans); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - pJob->failTasks = - taosHashInit(pReq->pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); - if (NULL == pJob->failTasks) { - SCH_JOB_ELOG("taosHashInit %d failTasks failed", pReq->pDag->numOfSubplans); - SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); - } - tsem_init(&pJob->rspSem, 0, 0); refId = taosAddRef(schMgmt.jobRef, pJob); @@ -724,6 +710,7 @@ int32_t schPushTaskToExecList(SSchJob *pJob, SSchTask *pTask) { return TSDB_CODE_SUCCESS; } +/* int32_t schMoveTaskToSuccList(SSchJob *pJob, SSchTask *pTask, bool *moved) { if (0 != taosHashRemove(pJob->execTasks, &pTask->taskId, sizeof(pTask->taskId))) { SCH_TASK_WLOG("remove task from execTask list failed, may not exist, status:%s", SCH_GET_TASK_STATUS_STR(pTask)); @@ -801,6 +788,7 @@ int32_t schMoveTaskToExecList(SSchJob *pJob, SSchTask *pTask, bool *moved) { return TSDB_CODE_SUCCESS; } +*/ int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bool *needRetry) { int8_t status = 0; @@ -1047,9 +1035,7 @@ int32_t schProcessOnTaskFailure(SSchJob *pJob, SSchTask *pTask, int32_t errCode) if (!needRetry) { SCH_TASK_ELOG("task failed and no more retry, code:%s", tstrerror(errCode)); - if (SCH_GET_TASK_STATUS(pTask) == JOB_TASK_STATUS_EXECUTING) { - SCH_ERR_JRET(schMoveTaskToFailList(pJob, pTask, &moved)); - } else { + if (SCH_GET_TASK_STATUS(pTask) != JOB_TASK_STATUS_EXECUTING) { SCH_TASK_ELOG("task not in executing list, status:%s", SCH_GET_TASK_STATUS_STR(pTask)); SCH_ERR_JRET(TSDB_CODE_SCH_STATUS_ERROR); } @@ -1115,8 +1101,6 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { SCH_LOG_TASK_END_TS(pTask); - SCH_ERR_JRET(schMoveTaskToSuccList(pJob, pTask, &moved)); - SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_PARTIAL_SUCCEED); SCH_ERR_JRET(schRecordTaskSucceedNode(pJob, pTask)); @@ -1150,8 +1134,6 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { pJob->fetchTask = pTask; - SCH_ERR_JRET(schMoveTaskToExecList(pJob, pTask, &moved)); - SCH_RET(schProcessOnJobPartialSuccess(pJob)); } @@ -1466,8 +1448,8 @@ void schDropTaskInHashList(SSchJob *pJob, SHashObj *list) { void schDropJobAllTasks(SSchJob *pJob) { schDropTaskInHashList(pJob, pJob->execTasks); - schDropTaskInHashList(pJob, pJob->succTasks); - schDropTaskInHashList(pJob, pJob->failTasks); +// schDropTaskInHashList(pJob, pJob->succTasks); +// schDropTaskInHashList(pJob, pJob->failTasks); } int32_t schCancelJob(SSchJob *pJob) { @@ -1507,8 +1489,8 @@ void schFreeJobImpl(void *job) { schFreeFlowCtrl(pJob); taosHashCleanup(pJob->execTasks); - taosHashCleanup(pJob->failTasks); - taosHashCleanup(pJob->succTasks); +// taosHashCleanup(pJob->failTasks); +// taosHashCleanup(pJob->succTasks); taosHashCleanup(pJob->taskList); taosArrayDestroy(pJob->levels); diff --git a/tests/script/api/stopquery.c b/tests/script/api/stopquery.c index addd1272cf..4c7964c983 100644 --- a/tests/script/api/stopquery.c +++ b/tests/script/api/stopquery.c @@ -36,7 +36,7 @@ int64_t st, et; char hostName[128]; char dbName[128]; char tbName[128]; -int32_t runTimes = 1; +int32_t runTimes = 10000; typedef struct { int id; @@ -367,8 +367,7 @@ void *closeThreadFp(void *arg) { SSP_CB_PARAM* qParam = (SSP_CB_PARAM*)arg; while (true) { if (qParam->taos) { - //usleep(rand() % 10000); - usleep(1000000); + usleep(rand() % 10000); taos_close(qParam->taos); break; } From 6000e2e20b25c4fbfa3d6b1f35449bd3f1f5951b Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 28 Jun 2022 11:45:20 +0800 Subject: [PATCH 11/61] fix: fix rpc quit problem --- source/libs/transport/src/transCli.c | 2 +- source/libs/transport/src/transSvr.c | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index c960cb4c63..4535150240 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -601,7 +601,7 @@ static void cliDestroyConn(SCliConn* conn, bool clear) { QUEUE_INIT(&conn->conn); transRemoveExHandle(conn->refId); if (clear) { - if (uv_is_active((uv_handle_t*)conn->stream)) { + if (!uv_is_closing((uv_handle_t*)conn->stream)) { uv_close((uv_handle_t*)conn->stream, cliDestroy); } else { cliDestroy((uv_handle_t*)conn->stream); diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index f1937c93fe..8bd1c09042 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -431,10 +431,14 @@ static void uvPrepareSendData(SSvrMsg* smsg, uv_buf_t* wb) { } static void uvStartSendRespInternal(SSvrMsg* smsg) { + SSvrConn* pConn = smsg->pConn; + if (pConn->broken) { + return; + } + uv_buf_t wb; uvPrepareSendData(smsg, &wb); - SSvrConn* pConn = smsg->pConn; // uv_timer_stop(&pConn->pTimer); uv_write(&pConn->pWriter, (uv_stream_t*)pConn->pTcp, &wb, 1, uvOnSendCb); } From 383e7a495bd2b533d46fa18b3b7eca1b3c2872b7 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 28 Jun 2022 13:44:50 +0800 Subject: [PATCH 12/61] fix(query): enable output check --- examples/c/stream_demo.c | 7 +- examples/c/tmq.c | 4 +- include/libs/stream/tstream.h | 5 +- source/dnode/vnode/src/tq/tqRead.c | 6 +- source/libs/executor/src/executil.c | 128 +++++++++--------- source/libs/executor/src/timewindowoperator.c | 125 +++++++++-------- source/libs/stream/src/streamExec.c | 15 +- 7 files changed, 152 insertions(+), 138 deletions(-) diff --git a/examples/c/stream_demo.c b/examples/c/stream_demo.c index f5cb7f1120..1e9058d628 100644 --- a/examples/c/stream_demo.c +++ b/examples/c/stream_demo.c @@ -98,9 +98,10 @@ int32_t create_stream() { /*const char* sql = "select min(k), max(k), sum(k) as sum_of_k from st1";*/ /*const char* sql = "select sum(k) from tu1 interval(10m)";*/ /*pRes = tmq_create_stream(pConn, "stream1", "out1", sql);*/ - pRes = taos_query(pConn, - "create stream stream1 trigger at_once into outstb as select _wstartts, sum(k) from st1 partition " - "by tbname interval(10s) "); + pRes = taos_query( + pConn, + "create stream stream1 trigger max_delay 10s into outstb as select _wstartts, sum(k) from st1 partition " + "by tbname session(ts, 10s) "); if (taos_errno(pRes) != 0) { printf("failed to create stream stream1, reason:%s\n", taos_errstr(pRes)); return -1; diff --git a/examples/c/tmq.c b/examples/c/tmq.c index 6fb7e7a1fc..53b259b0af 100644 --- a/examples/c/tmq.c +++ b/examples/c/tmq.c @@ -137,8 +137,8 @@ int32_t create_topic() { } taos_free_result(pRes); - pRes = taos_query(pConn, "create topic topic_ctb_column with meta as database abc1"); - /*pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from st1");*/ + /*pRes = taos_query(pConn, "create topic topic_ctb_column with meta as database abc1");*/ + pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3,t1 from st1 where t1 = 2000"); if (taos_errno(pRes) != 0) { printf("failed to create topic topic_ctb_column, reason:%s\n", taos_errstr(pRes)); return -1; diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index db928f194c..74c885ebcc 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -327,9 +327,8 @@ static FORCE_INLINE int32_t streamTaskInput(SStreamTask* pTask, SStreamQueueItem taosWriteQitem(pTask->inputQueue->queue, pItem); } - if (pItem->type != STREAM_INPUT__TRIGGER && pItem->type != STREAM_INPUT__CHECKPOINT && pTask->triggerParam != 0 && - pTask->triggerStatus == TASK_TRIGGER_STATUS__IN_ACTIVE) { - atomic_store_8(&pTask->triggerStatus, TASK_TRIGGER_STATUS__ACTIVE); + if (pItem->type != STREAM_INPUT__TRIGGER && pItem->type != STREAM_INPUT__CHECKPOINT && pTask->triggerParam != 0) { + atomic_val_compare_exchange_8(&pTask->triggerStatus, TASK_TRIGGER_STATUS__IN_ACTIVE, TASK_TRIGGER_STATUS__ACTIVE); } // TODO: back pressure diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index cbee639911..be2a35c901 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -231,7 +231,7 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle* pHandle) { tInitSubmitBlkIter(&pHandle->msgIter, pHandle->pBlock, &pHandle->blkIter); pBlock->info.groupId = 0; - pBlock->info.uid = pHandle->msgIter.uid; // set the uid of table for submit block + pBlock->info.uid = pHandle->msgIter.uid; pBlock->info.rows = pHandle->msgIter.numOfRows; while ((row = tGetSubmitBlkNext(&pHandle->blkIter)) != NULL) { @@ -251,8 +251,8 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle* pHandle) { } return 0; -FAIL: // todo refactor here - // if (*ppCols) taosArrayDestroy(*ppCols); +FAIL: + tDeleteSSDataBlock(pBlock); return -1; } diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index be6ca5145a..5c2f32126b 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -13,10 +13,10 @@ * along with this program. If not, see . */ -#include "os.h" -#include "index.h" #include "function.h" #include "functionMgt.h" +#include "index.h" +#include "os.h" #include "tdatablock.h" #include "thash.h" #include "tmsg.h" @@ -25,45 +25,41 @@ #include "executorimpl.h" #include "tcompression.h" -void initResultRowInfo(SResultRowInfo *pResultRowInfo) { - pResultRowInfo->size = 0; +void initResultRowInfo(SResultRowInfo* pResultRowInfo) { + pResultRowInfo->size = 0; pResultRowInfo->cur.pageId = -1; } -void cleanupResultRowInfo(SResultRowInfo *pResultRowInfo) { +void cleanupResultRowInfo(SResultRowInfo* pResultRowInfo) { if (pResultRowInfo == NULL) { return; } - for(int32_t i = 0; i < pResultRowInfo->size; ++i) { -// if (pResultRowInfo->pResult[i]) { -// taosMemoryFreeClear(pResultRowInfo->pResult[i]->key); -// } + for (int32_t i = 0; i < pResultRowInfo->size; ++i) { + // if (pResultRowInfo->pResult[i]) { + // taosMemoryFreeClear(pResultRowInfo->pResult[i]->key); + // } } } -void closeAllResultRows(SResultRowInfo *pResultRowInfo) { -// do nothing +void closeAllResultRows(SResultRowInfo* pResultRowInfo) { + // do nothing } -bool isResultRowClosed(SResultRow* pRow) { - return (pRow->closed == true); -} +bool isResultRowClosed(SResultRow* pRow) { return (pRow->closed == true); } -void closeResultRow(SResultRow* pResultRow) { - pResultRow->closed = true; -} +void closeResultRow(SResultRow* pResultRow) { pResultRow->closed = true; } // TODO refactor: use macro SResultRowEntryInfo* getResultEntryInfo(const SResultRow* pRow, int32_t index, const int32_t* offset) { assert(index >= 0 && offset != NULL); - return (SResultRowEntryInfo*)((char*) pRow->pEntryInfo + offset[index]); + return (SResultRowEntryInfo*)((char*)pRow->pEntryInfo + offset[index]); } size_t getResultRowSize(SqlFunctionCtx* pCtx, int32_t numOfOutput) { int32_t rowSize = (numOfOutput * sizeof(SResultRowEntryInfo)) + sizeof(SResultRow); - for(int32_t i = 0; i < numOfOutput; ++i) { + for (int32_t i = 0; i < numOfOutput; ++i) { rowSize += pCtx[i].resDataInfo.interBufSize; } @@ -74,31 +70,29 @@ void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo) { assert(pGroupResInfo != NULL); taosArrayDestroy(pGroupResInfo->pRows); - pGroupResInfo->pRows = NULL; - pGroupResInfo->index = 0; + pGroupResInfo->pRows = NULL; + pGroupResInfo->index = 0; } static int32_t resultrowComparAsc(const void* p1, const void* p2) { - SResKeyPos* pp1 = *(SResKeyPos**) p1; - SResKeyPos* pp2 = *(SResKeyPos**) p2; + SResKeyPos* pp1 = *(SResKeyPos**)p1; + SResKeyPos* pp2 = *(SResKeyPos**)p2; if (pp1->groupId == pp2->groupId) { - int64_t pts1 = *(int64_t*) pp1->key; - int64_t pts2 = *(int64_t*) pp2->key; + int64_t pts1 = *(int64_t*)pp1->key; + int64_t pts2 = *(int64_t*)pp2->key; if (pts1 == pts2) { return 0; } else { - return pts1 < pts2? -1:1; + return pts1 < pts2 ? -1 : 1; } } else { - return pp1->groupId < pp2->groupId? -1:1; + return pp1->groupId < pp2->groupId ? -1 : 1; } } -static int32_t resultrowComparDesc(const void* p1, const void* p2) { - return resultrowComparAsc(p2, p1); -} +static int32_t resultrowComparDesc(const void* p1, const void* p2) { return resultrowComparAsc(p2, p1); } void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, int32_t order) { if (pGroupResInfo->pRows != NULL) { @@ -110,20 +104,20 @@ void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, int pGroupResInfo->pRows = taosArrayInit(10, POINTER_BYTES); size_t keyLen = 0; - while((pData = taosHashIterate(pHashmap, pData)) != NULL) { + while ((pData = taosHashIterate(pHashmap, pData)) != NULL) { void* key = taosHashGetKey(pData, &keyLen); SResKeyPos* p = taosMemoryMalloc(keyLen + sizeof(SResultRowPosition)); - p->groupId = *(uint64_t*) key; - p->pos = *(SResultRowPosition*) pData; + p->groupId = *(uint64_t*)key; + p->pos = *(SResultRowPosition*)pData; memcpy(p->key, (char*)key + sizeof(uint64_t), keyLen - sizeof(uint64_t)); taosArrayPush(pGroupResInfo->pRows, &p); } if (order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC) { - __compar_fn_t fn = (order == TSDB_ORDER_ASC)? resultrowComparAsc:resultrowComparDesc; + __compar_fn_t fn = (order == TSDB_ORDER_ASC) ? resultrowComparAsc : resultrowComparDesc; qsort(pGroupResInfo->pRows->pData, taosArrayGetSize(pGroupResInfo->pRows), POINTER_BYTES, fn); } @@ -155,7 +149,7 @@ int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo) { return 0; } - return (int32_t) taosArrayGetSize(pGroupResInfo->pRows); + return (int32_t)taosArrayGetSize(pGroupResInfo->pRows); } SArray* createSortInfo(SNodeList* pNodeList) { @@ -194,12 +188,13 @@ SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode) { pBlock->info.type = STREAM_INVALID; for (int32_t i = 0; i < numOfCols; ++i) { - SSlotDescNode* pDescNode = (SSlotDescNode*)nodesListGetNode(pNode->pSlots, i); -// if (!pDescNode->output) { // todo disable it temporarily -// continue; -// } + SSlotDescNode* pDescNode = (SSlotDescNode*)nodesListGetNode(pNode->pSlots, i); + if (!pDescNode->output) { // todo disable it temporarily + continue; + } - SColumnInfoData idata = createColumnInfoData(pDescNode->dataType.type, pDescNode->dataType.bytes, pDescNode->slotId); + SColumnInfoData idata = + createColumnInfoData(pDescNode->dataType.type, pDescNode->dataType.bytes, pDescNode->slotId); idata.info.scale = pDescNode->dataType.scale; idata.info.precision = pDescNode->dataType.precision; @@ -211,10 +206,10 @@ SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode) { EDealRes doTranslateTagExpr(SNode** pNode, void* pContext) { SMetaReader* mr = (SMetaReader*)pContext; - if(nodeType(*pNode) == QUERY_NODE_COLUMN){ + if (nodeType(*pNode) == QUERY_NODE_COLUMN) { SColumnNode* pSColumnNode = *(SColumnNode**)pNode; - SValueNode *res = (SValueNode *)nodesMakeNode(QUERY_NODE_VALUE); + SValueNode* res = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); if (NULL == res) { return DEAL_RES_ERROR; } @@ -227,8 +222,8 @@ EDealRes doTranslateTagExpr(SNode** pNode, void* pContext) { const char* p = metaGetTableTagVal(&mr->me, pSColumnNode->node.resType.type, &tagVal); if (p == NULL) { res->node.resType.type = TSDB_DATA_TYPE_NULL; - }else if (pSColumnNode->node.resType.type == TSDB_DATA_TYPE_JSON) { - int32_t len = ((const STag*)p) -> len; + } else if (pSColumnNode->node.resType.type == TSDB_DATA_TYPE_JSON) { + int32_t len = ((const STag*)p)->len; res->datum.p = taosMemoryCalloc(len + 1, 1); memcpy(res->datum.p, p, len); } else if (IS_VAR_DATA_TYPE(pSColumnNode->node.resType.type)) { @@ -240,10 +235,10 @@ EDealRes doTranslateTagExpr(SNode** pNode, void* pContext) { } nodesDestroyNode(*pNode); *pNode = (SNode*)res; - }else if (nodeType(*pNode) == QUERY_NODE_FUNCTION){ - SFunctionNode * pFuncNode = *(SFunctionNode**)pNode; - if(pFuncNode->funcType == FUNCTION_TYPE_TBNAME){ - SValueNode *res = (SValueNode *)nodesMakeNode(QUERY_NODE_VALUE); + } else if (nodeType(*pNode) == QUERY_NODE_FUNCTION) { + SFunctionNode* pFuncNode = *(SFunctionNode**)pNode; + if (pFuncNode->funcType == FUNCTION_TYPE_TBNAME) { + SValueNode* res = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); if (NULL == res) { return DEAL_RES_ERROR; } @@ -263,12 +258,12 @@ EDealRes doTranslateTagExpr(SNode** pNode, void* pContext) { return DEAL_RES_CONTINUE; } -static bool isTableOk(STableKeyInfo* info, SNode *pTagCond, SMeta *metaHandle){ - SMetaReader mr = {0}; +static bool isTableOk(STableKeyInfo* info, SNode* pTagCond, SMeta* metaHandle) { + SMetaReader mr = {0}; metaReaderInit(&mr, metaHandle, 0); metaGetTableEntryByUid(&mr, info->uid); - SNode *pTagCondTmp = nodesCloneNode(pTagCond); + SNode* pTagCondTmp = nodesCloneNode(pTagCond); nodesRewriteExprPostOrder(&pTagCondTmp, doTranslateTagExpr, &mr); metaReaderClear(&mr); @@ -281,7 +276,7 @@ static bool isTableOk(STableKeyInfo* info, SNode *pTagCond, SMeta *metaHandle){ } ASSERT(nodeType(pNew) == QUERY_NODE_VALUE); - SValueNode *pValue = (SValueNode *)pNew; + SValueNode* pValue = (SValueNode*)pNew; ASSERT(pValue->node.resType.type == TSDB_DATA_TYPE_BOOL); bool result = pValue->datum.b; @@ -292,12 +287,12 @@ static bool isTableOk(STableKeyInfo* info, SNode *pTagCond, SMeta *metaHandle){ int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo* pListInfo) { int32_t code = TSDB_CODE_SUCCESS; pListInfo->pTableList = taosArrayInit(8, sizeof(STableKeyInfo)); - if(pListInfo->pTableList == NULL) return TSDB_CODE_OUT_OF_MEMORY; + if (pListInfo->pTableList == NULL) return TSDB_CODE_OUT_OF_MEMORY; uint64_t tableUid = pScanNode->uid; pListInfo->suid = pScanNode->suid; - + SNode* pTagCond = (SNode*)pListInfo->pTagCond; SNode* pTagIndexCond = (SNode*)pListInfo->pTagIndexCond; if (pScanNode->tableType == TSDB_SUPER_TABLE) { @@ -306,7 +301,7 @@ int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo .metaEx = metaHandle, .idx = tsdbGetIdx(metaHandle), .ivtIdx = tsdbGetIvtIdx(metaHandle), .suid = tableUid}; SArray* res = taosArrayInit(8, sizeof(uint64_t)); - //code = doFilterTag(pTagIndexCond, &metaArg, res); + // code = doFilterTag(pTagIndexCond, &metaArg, res); code = TSDB_CODE_INDEX_REBUILDING; if (code == TSDB_CODE_INDEX_REBUILDING) { code = tsdbGetAllTableList(metaHandle, tableUid, pListInfo->pTableList); @@ -328,33 +323,33 @@ int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo code = tsdbGetAllTableList(metaHandle, tableUid, pListInfo->pTableList); } - if(pTagCond){ + if (pTagCond) { int32_t i = 0; - while(i < taosArrayGetSize(pListInfo->pTableList)) { + while (i < taosArrayGetSize(pListInfo->pTableList)) { STableKeyInfo* info = taosArrayGet(pListInfo->pTableList, i); - bool isOk = isTableOk(info, pTagCond, metaHandle); - if(!isOk){ + bool isOk = isTableOk(info, pTagCond, metaHandle); + if (!isOk) { taosArrayRemove(pListInfo->pTableList, i); continue; } i++; } } - }else { // Create one table group. + } else { // Create one table group. STableKeyInfo info = {.lastKey = 0, .uid = tableUid, .groupId = 0}; taosArrayPush(pListInfo->pTableList, &info); } pListInfo->pGroupList = taosArrayInit(4, POINTER_BYTES); - if(pListInfo->pGroupList == NULL) return TSDB_CODE_OUT_OF_MEMORY; + if (pListInfo->pGroupList == NULL) return TSDB_CODE_OUT_OF_MEMORY; - //put into list as default group, remove it if grouping sorting is required later + // put into list as default group, remove it if grouping sorting is required later taosArrayPush(pListInfo->pGroupList, &pListInfo->pTableList); return code; } SArray* extractPartitionColInfo(SNodeList* pNodeList) { - if(!pNodeList) { + if (!pNodeList) { return NULL; } @@ -383,7 +378,6 @@ SArray* extractPartitionColInfo(SNodeList* pNodeList) { return pList; } - SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols, int32_t type) { size_t numOfCols = LIST_LENGTH(pNodeList); @@ -399,7 +393,7 @@ SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNod SColMatchInfo c = {0}; c.output = true; - c.colId = pColNode->colId; + c.colId = pColNode->colId; c.srcSlotId = pColNode->slotId; c.matchType = type; c.targetSlotId = pNode->slotId; @@ -675,8 +669,8 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, } for (int32_t i = 1; i < numOfOutput; ++i) { - (*rowEntryInfoOffset)[i] = - (int32_t)((*rowEntryInfoOffset)[i - 1] + sizeof(SResultRowEntryInfo) + pFuncCtx[i - 1].resDataInfo.interBufSize); + (*rowEntryInfoOffset)[i] = (int32_t)((*rowEntryInfoOffset)[i - 1] + sizeof(SResultRowEntryInfo) + + pFuncCtx[i - 1].resDataInfo.interBufSize); } setSelectValueColumnInfo(pFuncCtx, numOfOutput); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 963e714972..edd1ec0e75 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -34,7 +34,7 @@ typedef struct SWinRes { typedef struct SPullWindowInfo { STimeWindow window; - uint64_t groupId; + uint64_t groupId; } SPullWindowInfo; static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator); @@ -695,11 +695,11 @@ static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t num } void printDataBlock(SSDataBlock* pBlock, const char* flag) { - if (pBlock == NULL){ + if (pBlock == NULL) { qDebug("======printDataBlock Block is Null"); return; } - char *pBuf = NULL; + char* pBuf = NULL; qDebug("%s", dumpBlockData(pBlock, flag, &pBuf)); taosMemoryFree(pBuf); } @@ -1255,10 +1255,10 @@ static void doClearWindows(SAggSupporter* pAggSup, SExprSupp* pSup1, SInterval* TSKEY* tsCols = (TSKEY*)pTsCol->pData; uint64_t* pGpDatas = NULL; if (pBlock->info.type == STREAM_RETRIEVE) { - SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, 2); - pGpDatas = (uint64_t*)pGpCol->pData; + SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, 2); + pGpDatas = (uint64_t*)pGpCol->pData; } - int32_t step = 0; + int32_t step = 0; for (int32_t i = 0; i < pBlock->info.rows; i += step) { SResultRowInfo dumyInfo; dumyInfo.cur.pageId = -1; @@ -1297,8 +1297,8 @@ bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup) { return pSup->maxTs != INT64_MIN && pWin->ekey < pSup->maxTs - pSup->waterMark; } -static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, - SInterval* pInterval, SHashObj* pPullDataMap, SArray* closeWins) { +static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, SInterval* pInterval, + SHashObj* pPullDataMap, SArray* closeWins) { void* pIte = NULL; size_t keyLen = 0; while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) { @@ -1309,15 +1309,18 @@ static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, SResultRowInfo dumyInfo; dumyInfo.cur.pageId = -1; STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, ts, pInterval, pInterval->precision, NULL); - SWinRes winRe = {.ts = win.skey, .groupId = groupId,}; + SWinRes winRe = { + .ts = win.skey, + .groupId = groupId, + }; void* chIds = taosHashGet(pPullDataMap, &winRe, sizeof(SWinRes)); if (isCloseWindow(&win, pSup)) { if (chIds && pPullDataMap) { - SArray* chAy = *(SArray**) chIds; + SArray* chAy = *(SArray**)chIds; int32_t size = taosArrayGetSize(chAy); - qInfo("======window %ld wait child size:%d", win.skey ,size); + qInfo("======window %ld wait child size:%d", win.skey, size); for (int32_t i = 0; i < size; i++) { - qInfo("======window %ld wait chid id:%d", win.skey ,*(int32_t*)taosArrayGet(chAy, i)); + qInfo("======window %ld wait chid id:%d", win.skey, *(int32_t*)taosArrayGet(chAy, i)); } continue; } else if (pPullDataMap) { @@ -1341,7 +1344,7 @@ static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, static void closeChildIntervalWindow(SArray* pChildren, TSKEY maxTs) { int32_t size = taosArrayGetSize(pChildren); for (int32_t i = 0; i < size; i++) { - SOperatorInfo* pChildOp = taosArrayGetP(pChildren, i); + SOperatorInfo* pChildOp = taosArrayGetP(pChildren, i); SStreamFinalIntervalOperatorInfo* pChInfo = pChildOp->info; ASSERT(pChInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE); pChInfo->twAggSup.maxTs = TMAX(pChInfo->twAggSup.maxTs, maxTs); @@ -1431,7 +1434,7 @@ void destroyStreamFinalIntervalOperatorInfo(void* param, int32_t numOfOutput) { SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)param; cleanupBasicInfo(&pInfo->binfo); cleanupAggSup(&pInfo->aggSup); - //it should be empty. + // it should be empty. taosHashCleanup(pInfo->pPullDataMap); taosArrayDestroy(pInfo->pPullWins); blockDataDestroy(pInfo->pPullDataRes); @@ -2236,10 +2239,10 @@ bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup) { return p1 == NULL; } -int32_t getNexWindowPos(SInterval* pInterval, SDataBlockInfo* pBlockInfo, TSKEY* tsCols, - int32_t startPos, TSKEY eKey, STimeWindow* pNextWin) { - int32_t forwardRows = getNumOfRowsInTimeWindow(pBlockInfo, tsCols, startPos, - eKey, binarySearchForKey, NULL, TSDB_ORDER_ASC); +int32_t getNexWindowPos(SInterval* pInterval, SDataBlockInfo* pBlockInfo, TSKEY* tsCols, int32_t startPos, TSKEY eKey, + STimeWindow* pNextWin) { + int32_t forwardRows = + getNumOfRowsInTimeWindow(pBlockInfo, tsCols, startPos, eKey, binarySearchForKey, NULL, TSDB_ORDER_ASC); int32_t prevEndPos = forwardRows - 1 + startPos; return getNextQualifiedWindow(pInterval, pNextWin, pBlockInfo, tsCols, prevEndPos, TSDB_ORDER_ASC); } @@ -2277,8 +2280,11 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc pInfo->interval.precision, NULL); while (1) { if (IS_FINAL_OP(pInfo) && isCloseWindow(&nextWin, &pInfo->twAggSup) && pInfo->pChildren) { - bool ignore = true; - SWinRes winRes = {.ts = nextWin.skey, .groupId = tableGroupId,}; + bool ignore = true; + SWinRes winRes = { + .ts = nextWin.skey, + .groupId = tableGroupId, + }; void* chIds = taosHashGet(pInfo->pPullDataMap, &winRes, sizeof(SWinRes)); if (isDeletedWindow(&nextWin, tableGroupId, &pInfo->aggSup) && !chIds) { SPullWindowInfo pull = {.window = nextWin, .groupId = tableGroupId}; @@ -2289,7 +2295,7 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc int32_t index = -1; SArray* chArray = NULL; if (chIds) { - chArray = *(void**) chIds; + chArray = *(void**)chIds; int32_t chId = getChildIndex(pSDataBlock); index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ); } @@ -2300,7 +2306,7 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc taosHashRemove(pInfo->pPullDataMap, &winRes, sizeof(SWinRes)); } } - if ( index == -1 || pSDataBlock->info.type == STREAM_PUSH_DATA) { + if (index == -1 || pSDataBlock->info.type == STREAM_PUSH_DATA) { ignore = false; } } @@ -2385,17 +2391,17 @@ static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pB if (size - (*pIndex) == 0) { return; } - blockDataEnsureCapacity(pBlock, size - (*pIndex) ); + blockDataEnsureCapacity(pBlock, size - (*pIndex)); ASSERT(3 <= taosArrayGetSize(pBlock->pDataBlock)); for (; (*pIndex) < size; (*pIndex)++) { - SPullWindowInfo* pWin = taosArrayGet(array, (*pIndex) ); - SColumnInfoData* pStartTs = (SColumnInfoData*) taosArrayGet(pBlock->pDataBlock, 0); + SPullWindowInfo* pWin = taosArrayGet(array, (*pIndex)); + SColumnInfoData* pStartTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, 0); colDataAppend(pStartTs, pBlock->info.rows, (const char*)&pWin->window.skey, false); - SColumnInfoData* pEndTs = (SColumnInfoData*) taosArrayGet(pBlock->pDataBlock, 1); + SColumnInfoData* pEndTs = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, 1); colDataAppend(pEndTs, pBlock->info.rows, (const char*)&pWin->window.ekey, false); - SColumnInfoData* pGroupId = (SColumnInfoData*) taosArrayGet(pBlock->pDataBlock, 2); + SColumnInfoData* pGroupId = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, 2); colDataAppend(pGroupId, pBlock->info.rows, (const char*)&pWin->groupId, false); pBlock->info.rows++; } @@ -2408,15 +2414,15 @@ static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pB void processPushEmpty(SSDataBlock* pBlock, SHashObj* pMap) { SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, 0); - TSKEY* tsData = (TSKEY*)pStartCol->pData; + TSKEY* tsData = (TSKEY*)pStartCol->pData; SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, 2); - uint64_t* groupIdData = (uint64_t*)pGroupCol->pData; - int32_t chId = getChildIndex(pBlock); + uint64_t* groupIdData = (uint64_t*)pGroupCol->pData; + int32_t chId = getChildIndex(pBlock); for (int32_t i = 0; i < pBlock->info.rows; i++) { SWinRes winRes = {.ts = tsData[i], .groupId = groupIdData[i]}; - void* chIds = taosHashGet(pMap, &winRes, sizeof(SWinRes)); + void* chIds = taosHashGet(pMap, &winRes, sizeof(SWinRes)); if (chIds) { - SArray* chArray = *(SArray**) chIds; + SArray* chArray = *(SArray**)chIds; int32_t index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ); if (index != -1) { taosArrayRemove(chArray, index); @@ -2484,7 +2490,8 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { printDataBlock(pBlock, IS_FINAL_OP(pInfo) ? "interval Final recv" : "interval Semi recv"); maxTs = TMAX(maxTs, pBlock->info.window.ekey); - if (pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_PUSH_DATA || pBlock->info.type == STREAM_INVALID) { + if (pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_PUSH_DATA || + pBlock->info.type == STREAM_INVALID) { pInfo->binfo.pRes->info.type = pBlock->info.type; } else if (pBlock->info.type == STREAM_CLEAR) { SArray* pUpWins = taosArrayInit(8, sizeof(STimeWindow)); @@ -2513,8 +2520,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { continue; } else if (pBlock->info.type == STREAM_RETRIEVE && !IS_FINAL_OP(pInfo)) { SArray* pUpWins = taosArrayInit(8, sizeof(STimeWindow)); - doClearWindows(&pInfo->aggSup, pSup, &pInfo->interval, 0, pOperator->exprSupp.numOfExprs, - pBlock, pUpWins); + doClearWindows(&pInfo->aggSup, pSup, &pInfo->interval, 0, pOperator->exprSupp.numOfExprs, pBlock, pUpWins); removeResults(pUpWins, pUpdated); taosArrayDestroy(pUpWins); if (taosArrayGetSize(pUpdated) > 0) { @@ -2552,8 +2558,8 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); if (IS_FINAL_OP(pInfo)) { - closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, - &pInfo->interval, pInfo->pPullDataMap, pUpdated); + closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, pInfo->pPullDataMap, + pUpdated); closeChildIntervalWindow(pInfo->pChildren, pInfo->twAggSup.maxTs); } @@ -2874,12 +2880,24 @@ static bool isInWindow(SResultWindowInfo* pWin, TSKEY ts, int64_t gap) { } static SResultWindowInfo* insertNewSessionWindow(SArray* pWinInfos, TSKEY ts, int32_t index) { - SResultWindowInfo win = {.pos.offset = -1, .pos.pageId = -1, .win.skey = ts, .win.ekey = ts, .isOutput = false}; + SResultWindowInfo win = { + .pos.offset = -1, + .pos.pageId = -1, + .win.skey = ts, + .win.ekey = ts, + .isOutput = false, + }; return taosArrayInsert(pWinInfos, index, &win); } static SResultWindowInfo* addNewSessionWindow(SArray* pWinInfos, TSKEY ts) { - SResultWindowInfo win = {.pos.offset = -1, .pos.pageId = -1, .win.skey = ts, .win.ekey = ts, .isOutput = false}; + SResultWindowInfo win = { + .pos.offset = -1, + .pos.pageId = -1, + .win.skey = ts, + .win.ekey = ts, + .isOutput = false, + }; return taosArrayPush(pWinInfos, &win); } @@ -3277,14 +3295,14 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { } else if (pOperator->status == OP_RES_TO_RETURN) { doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator); if (pInfo->pDelRes->info.rows > 0) { - printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo)? "Final Session" : "Single Session"); + printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "Final Session" : "Single Session"); return pInfo->pDelRes; } doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); if (pBInfo->pRes->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); } - printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo)? "Final Session" : "Single Session"); + printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo) ? "Final Session" : "Single Session"); return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; } @@ -3352,11 +3370,11 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator); if (pInfo->pDelRes->info.rows > 0) { - printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo)? "Final Session" : "Single Session"); + printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "Final Session" : "Single Session"); return pInfo->pDelRes; } doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); - printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo)? "Final Session" : "Single Session"); + printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo) ? "Final Session" : "Single Session"); return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; } @@ -4150,16 +4168,16 @@ _error: // merge interval operator typedef struct SMergeIntervalAggOperatorInfo { SIntervalAggOperatorInfo intervalAggOperatorInfo; - SList* groupIntervals; - SListIter groupIntervalsIter; - bool hasGroupId; - uint64_t groupId; - SSDataBlock* prefetchedBlock; - bool inputBlocksFinished; + SList* groupIntervals; + SListIter groupIntervalsIter; + bool hasGroupId; + uint64_t groupId; + SSDataBlock* prefetchedBlock; + bool inputBlocksFinished; } SMergeIntervalAggOperatorInfo; typedef struct SGroupTimeWindow { - uint64_t groupId; + uint64_t groupId; STimeWindow window; } SGroupTimeWindow; @@ -4169,7 +4187,8 @@ void destroyMergeIntervalOperatorInfo(void* param, int32_t numOfOutput) { destroyIntervalOperatorInfo(&miaInfo->intervalAggOperatorInfo, numOfOutput); } -static int32_t finalizeWindowResult(SOperatorInfo* pOperatorInfo, uint64_t tableGroupId, STimeWindow* win, SSDataBlock* pResultBlock) { +static int32_t finalizeWindowResult(SOperatorInfo* pOperatorInfo, uint64_t tableGroupId, STimeWindow* win, + SSDataBlock* pResultBlock) { SMergeIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info; SIntervalAggOperatorInfo* iaInfo = &miaInfo->intervalAggOperatorInfo; SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; @@ -4202,7 +4221,7 @@ static int32_t outputPrevIntervalResult(SOperatorInfo* pOperatorInfo, uint64_t t SListNode* listNode = NULL; while ((listNode = tdListNext(&iter)) != NULL) { SGroupTimeWindow* prevGrpWin = (SGroupTimeWindow*)listNode->data; - if (prevGrpWin->groupId != tableGroupId ) { + if (prevGrpWin->groupId != tableGroupId) { continue; } STimeWindow* prevWin = &prevGrpWin->window; @@ -4445,4 +4464,4 @@ _error: taosMemoryFreeClear(pOperator); pTaskInfo->code = code; return NULL; -} \ No newline at end of file +} diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index c75e6c004a..b3ceb92505 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -46,15 +46,16 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) } if (output == NULL) { if (pItem->type == STREAM_INPUT__DATA_RETRIEVE) { - //SSDataBlock block = {0}; - //block.info.type = STREAM_PUSH_EMPTY; - //block.info.childId = pTask->selfChildId; + SSDataBlock block = {0}; + /*block.info.type = STREAM_PUSH_EMPTY;*/ + // block.info.childId = pTask->selfChildId; SStreamDataBlock* pRetrieveBlock = (SStreamDataBlock*)data; ASSERT(taosArrayGetSize(pRetrieveBlock->blocks) == 1); - SSDataBlock* pBlock = createOneDataBlock(taosArrayGet(pRetrieveBlock->blocks, 0), true); - pBlock->info.type = STREAM_PUSH_EMPTY; - pBlock->info.childId = pTask->selfChildId; - taosArrayPush(pRes, pBlock); + /*SSDataBlock* pBlock = createOneDataBlock(taosArrayGet(pRetrieveBlock->blocks, 0), true);*/ + assignOneDataBlock(&block, taosArrayGet(pRetrieveBlock->blocks, 0)); + block.info.type = STREAM_PUSH_EMPTY; + block.info.childId = pTask->selfChildId; + taosArrayPush(pRes, &block); } break; } From 421efe365b5dfebdebfa0a67c56b05ac3fb29f58 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 28 Jun 2022 14:41:56 +0800 Subject: [PATCH 13/61] fix: fix rpc quit problem --- source/libs/transport/src/transSvr.c | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 2adcd66e7a..c53cab13d5 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -332,19 +332,10 @@ void uvOnTimeoutCb(uv_timer_t* handle) { void uvOnSendCb(uv_write_t* req, int status) { SSvrConn* conn = req->data; - // transClearBuffer(&conn->readBuf); if (status == 0) { tTrace("conn %p data already was written on stream", conn); if (!transQueueEmpty(&conn->srvMsgs)) { SSvrMsg* msg = transQueuePop(&conn->srvMsgs); - // if (msg->type == Release && conn->status != ConnNormal) { - // conn->status = ConnNormal; - // transUnrefSrvHandle(conn); - // reallocConnRef(conn); - // destroySmsg(msg); - // transQueueClear(&conn->srvMsgs); - // return; - //} destroySmsg(msg); // send second data, just use for push if (!transQueueEmpty(&conn->srvMsgs)) { @@ -370,6 +361,7 @@ void uvOnSendCb(uv_write_t* req, int status) { } } } + transUnrefSrvHandle(conn); } else { tError("conn %p failed to write data, %s", conn, uv_err_name(status)); conn->broken = true; @@ -439,7 +431,7 @@ static void uvStartSendRespInternal(SSvrMsg* smsg) { uv_buf_t wb; uvPrepareSendData(smsg, &wb); - // uv_timer_stop(&pConn->pTimer); + transRefSrvHandle(pConn); uv_write(&pConn->pWriter, (uv_stream_t*)pConn->pTcp, &wb, 1, uvOnSendCb); } static void uvStartSendResp(SSvrMsg* smsg) { @@ -780,12 +772,9 @@ static void destroyConn(SSvrConn* conn, bool clear) { transDestroyBuffer(&conn->readBuf); if (clear) { - if (uv_is_active((uv_handle_t*)conn->pTcp)) { + if (!uv_is_closing((uv_handle_t*)conn->pTcp)) { tTrace("conn %p to be destroyed", conn); - // uv_shutdown_t* req = taosMemoryMalloc(sizeof(uv_shutdown_t)); uv_close((uv_handle_t*)conn->pTcp, uvDestroyConn); - // uv_close(conn->pTcp) - // uv_shutdown(req, (uv_stream_t*)conn->pTcp, uvShutDownCb); } else { uvDestroyConn((uv_handle_t*)conn->pTcp); } From 66e8e06861fa0316e46e3b1e4bfeab994c06fd53 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 28 Jun 2022 15:49:33 +0800 Subject: [PATCH 14/61] fix: fix rpc quit problem --- source/dnode/mgmt/node_mgmt/src/dmTransport.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index 7e31cc3144..cf71b7c243 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -264,7 +264,7 @@ int32_t dmInitClient(SDnode *pDnode) { SDnodeTrans *pTrans = &pDnode->trans; SRpcInit rpcInit = {0}; - rpcInit.label = "DND"; + rpcInit.label = "DND-C"; rpcInit.numOfThreads = 1; rpcInit.cfp = (RpcCfp)dmProcessRpcMsg; rpcInit.sessions = 1024; @@ -298,7 +298,7 @@ int32_t dmInitServer(SDnode *pDnode) { SRpcInit rpcInit = {0}; strncpy(rpcInit.localFqdn, tsLocalFqdn, strlen(tsLocalFqdn)); rpcInit.localPort = tsServerPort; - rpcInit.label = "DND"; + rpcInit.label = "DND-S"; rpcInit.numOfThreads = tsNumOfRpcThreads; rpcInit.cfp = (RpcCfp)dmProcessRpcMsg; rpcInit.sessions = tsMaxShellConns; From c0183e1fa8962b8630b2f45c5ae232837ae3e05f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 28 Jun 2022 15:50:47 +0800 Subject: [PATCH 15/61] fix: duplicate response while set standby in vnode --- source/dnode/mgmt/mgmt_vnode/src/vmWorker.c | 8 +------- source/dnode/vnode/src/vnd/vnodeSync.c | 2 +- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 5e1ef23f1c..91ef292360 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -107,13 +107,7 @@ static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOf const STraceId *trace = &pMsg->info.traceId; dGTrace("vgId:%d, msg:%p get from vnode-sync queue", pVnode->vgId, pMsg); - int32_t code = vnodeProcessSyncReq(pVnode->pImpl, pMsg, NULL); - if (code != 0) { - if (terrno != 0) code = terrno; - dGError("vgId:%d, msg:%p failed to sync since %s", pVnode->vgId, pMsg, terrstr()); - vmSendRsp(pMsg, code); - } - + int32_t code = vnodeProcessSyncReq(pVnode->pImpl, pMsg, NULL); // no response here dGTrace("vgId:%d, msg:%p is freed, code:0x%x", pVnode->vgId, pMsg, code); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index b998f40195..7cf6ccd13f 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -73,7 +73,7 @@ static int32_t vnodeSetStandBy(SVnode *pVnode) { vInfo("vgId:%d, set standby success", TD_VID(pVnode)); return 0; } else { - vError("vgId:%d, failed to set standby since %s", TD_VID(pVnode), terrstr()); + vError("vgId:%d, failed to set standby after leader transfer since %s", TD_VID(pVnode), terrstr()); return -1; } } From 58803fafcfba4bc1c48d9648540d087ea2471e0d Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Tue, 28 Jun 2022 13:59:49 +0800 Subject: [PATCH 16/61] fix(query): check datablock should check output flag --- examples/c/tmq.c | 2 +- include/common/tdatablock.h | 7 +- include/common/tmsg.h | 1 + source/client/src/clientMsgHandler.c | 28 +++-- source/common/src/tdatablock.c | 9 +- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 26 ++--- source/dnode/mnode/impl/src/mndShow.c | 4 +- source/dnode/vnode/src/tq/tqExec.c | 2 +- source/libs/command/src/command.c | 121 +++++++++----------- source/libs/command/src/explain.c | 70 +++++------ source/libs/executor/inc/executil.h | 82 ++++++------- source/libs/executor/src/dataDispatcher.c | 21 ++-- source/libs/executor/src/executil.c | 6 +- source/libs/executor/src/executorimpl.c | 82 ++++++------- source/libs/stream/src/streamData.c | 4 +- source/libs/stream/src/streamDispatch.c | 4 +- 16 files changed, 230 insertions(+), 239 deletions(-) diff --git a/examples/c/tmq.c b/examples/c/tmq.c index 53b259b0af..68b0492ace 100644 --- a/examples/c/tmq.c +++ b/examples/c/tmq.c @@ -138,7 +138,7 @@ int32_t create_topic() { taos_free_result(pRes); /*pRes = taos_query(pConn, "create topic topic_ctb_column with meta as database abc1");*/ - pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3,t1 from st1 where t1 = 2000"); + pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from st1 where t1 = 2000"); if (taos_errno(pRes) != 0) { printf("failed to create topic topic_ctb_column, reason:%s\n", taos_errstr(pRes)); return -1; diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 6653fdd9cd..72e0af657c 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -224,7 +224,7 @@ size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize); int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n); int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src); -int32_t copyDataBlock(SSDataBlock* dst, const SSDataBlock* src); +int32_t copyDataBlock(SSDataBlock* dst, const SSDataBlock* src); SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData); SSDataBlock* createDataBlock(); int32_t blockDataAppendColInfo(SSDataBlock* pBlock, SColumnInfoData* pColInfoData); @@ -232,9 +232,8 @@ int32_t blockDataAppendColInfo(SSDataBlock* pBlock, SColumnInfoData* pColIn SColumnInfoData createColumnInfoData(int16_t type, int32_t bytes, int16_t colId); SColumnInfoData* bdGetColumnInfoData(SSDataBlock* pBlock, int32_t index); -void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, - int8_t needCompress); -const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData); +void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, int8_t needCompress); +const char* blockDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData); void blockDebugShowDataBlocks(const SArray* dataBlocks, const char* flag); // for debug diff --git a/include/common/tmsg.h b/include/common/tmsg.h index c5b0b89311..0afcaaa50d 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -623,6 +623,7 @@ typedef struct { col_id_t colId; int16_t slotId; }; + bool output; // TODO remove it later int16_t type; int32_t bytes; diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 45a525d124..94bd07a18a 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -67,11 +67,11 @@ int32_t processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) { dstEpSet.eps[dstEpSet.inUse].fqdn); } else if (connectRsp.dnodeNum > 1 && !isEpsetEqual(&pTscObj->pAppInfo->mgmtEp.epSet, &connectRsp.epSet)) { SEpSet* pOrig = &pTscObj->pAppInfo->mgmtEp.epSet; - SEp* pOrigEp = &pOrig->eps[pOrig->inUse]; - SEp* pNewEp = &connectRsp.epSet.eps[connectRsp.epSet.inUse]; - tscDebug("mnode epset updated from %d/%d=>%s:%d to %d/%d=>%s:%d in connRsp", - pOrig->inUse, pOrig->numOfEps, pOrigEp->fqdn, pOrigEp->port, - connectRsp.epSet.inUse, connectRsp.epSet.numOfEps, pNewEp->fqdn, pNewEp->port); + SEp* pOrigEp = &pOrig->eps[pOrig->inUse]; + SEp* pNewEp = &connectRsp.epSet.eps[connectRsp.epSet.inUse]; + tscDebug("mnode epset updated from %d/%d=>%s:%d to %d/%d=>%s:%d in connRsp", pOrig->inUse, pOrig->numOfEps, + pOrigEp->fqdn, pOrigEp->port, connectRsp.epSet.inUse, connectRsp.epSet.numOfEps, pNewEp->fqdn, + pNewEp->port); updateEpSet_s(&pTscObj->pAppInfo->mgmtEp, &connectRsp.epSet); } @@ -308,13 +308,13 @@ static int32_t buildShowVariablesBlock(SArray* pVars, SSDataBlock** block) { blockDataEnsureCapacity(pBlock, numOfCfg); for (int32_t i = 0, c = 0; i < numOfCfg; ++i, c = 0) { - SVariablesInfo *pInfo = taosArrayGet(pVars, i); + SVariablesInfo* pInfo = taosArrayGet(pVars, i); char name[TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(name, pInfo->name, TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE); - SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, c++); + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, c++); colDataAppend(pColInfo, i, name, false); - + char value[TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(value, pInfo->value, TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE); pColInfo = taosArrayGet(pBlock->pDataBlock, c++); @@ -324,14 +324,13 @@ static int32_t buildShowVariablesBlock(SArray* pVars, SSDataBlock** block) { pBlock->info.rows = numOfCfg; *block = pBlock; - + return TSDB_CODE_SUCCESS; } - static int32_t buildShowVariablesRsp(SArray* pVars, SRetrieveTableRsp** pRsp) { SSDataBlock* pBlock = NULL; - int32_t code = buildShowVariablesBlock(pVars, &pBlock); + int32_t code = buildShowVariablesBlock(pVars, &pBlock); if (code) { return code; } @@ -351,7 +350,7 @@ static int32_t buildShowVariablesRsp(SArray* pVars, SRetrieveTableRsp** pRsp) { (*pRsp)->numOfCols = htonl(SHOW_VARIABLES_RESULT_COLS); int32_t len = 0; - blockCompressEncode(pBlock, (*pRsp)->data, &len, SHOW_VARIABLES_RESULT_COLS, false); + blockEncode(pBlock, (*pRsp)->data, &len, SHOW_VARIABLES_RESULT_COLS, false); ASSERT(len == rspSize - sizeof(SRetrieveTableRsp)); blockDataDestroy(pBlock); @@ -363,7 +362,7 @@ int32_t processShowVariablesRsp(void* param, const SDataBuf* pMsg, int32_t code) if (code != TSDB_CODE_SUCCESS) { setErrno(pRequest, code); } else { - SShowVariablesRsp rsp = {0}; + SShowVariablesRsp rsp = {0}; SRetrieveTableRsp* pRes = NULL; code = tDeserializeSShowVariablesRsp(pMsg->pData, pMsg->len, &rsp); if (TSDB_CODE_SUCCESS == code) { @@ -372,7 +371,7 @@ int32_t processShowVariablesRsp(void* param, const SDataBuf* pMsg, int32_t code) if (TSDB_CODE_SUCCESS == code) { code = setQueryResultFromRsp(&pRequest->body.resInfo, pRes, false, false); } - + tFreeSShowVariablesRsp(&rsp); } @@ -384,7 +383,6 @@ int32_t processShowVariablesRsp(void* param, const SDataBuf* pMsg, int32_t code) return code; } - __async_send_cb_fn_t getMsgRspHandle(int32_t msgType) { switch (msgType) { case TDMT_MND_CONNECT: diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 1ec298ee15..dd9500e8d8 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1613,7 +1613,8 @@ void blockDebugShowDataBlocks(const SArray* dataBlocks, const char* flag) { size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); int32_t rows = pDataBlock->info.rows; - printf("%s |block type %d |child id %d|group id %zX\n", flag, (int32_t)pDataBlock->info.type, pDataBlock->info.childId, pDataBlock->info.groupId); + printf("%s |block type %d |child id %d|group id %zX\n", flag, (int32_t)pDataBlock->info.type, + pDataBlock->info.childId, pDataBlock->info.groupId); for (int32_t j = 0; j < rows; j++) { printf("%s |", flag); for (int32_t k = 0; k < numOfCols; k++) { @@ -1902,8 +1903,7 @@ char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId) { return rname.childTableName; } -void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, - int8_t needCompress) { +void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, int8_t needCompress) { // todo extract method int32_t* actualLen = (int32_t*)data; data += sizeof(int32_t); @@ -1913,6 +1913,7 @@ void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); + *((int16_t*)data) = pColInfoData->info.type; data += sizeof(int16_t); @@ -1960,7 +1961,7 @@ void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen *groupId = pBlock->info.groupId; } -const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData) { +const char* blockDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData) { const char* pStart = pData; int32_t dataLen = *(int32_t*)pStart; diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 980749ca70..234a133243 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -17,7 +17,6 @@ #include "dmInt.h" #include "systable.h" - extern SConfig *tsCfg; static void dmUpdateDnodeCfg(SDnodeMgmt *pMgmt, SDnodeCfg *pCfg) { @@ -83,7 +82,7 @@ void dmSendStatusReq(SDnodeMgmt *pMgmt) { (*pMgmt->getVnodeLoadsFp)(&vinfo); req.pVloads = vinfo.pVloads; - SMonMloadInfo minfo = {0}; + SMonMloadInfo minfo = {0}; (*pMgmt->getMnodeLoadsFp)(&minfo); req.mload = minfo.load; @@ -186,10 +185,10 @@ int32_t dmProcessServerRunStatus(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { return 0; } -SSDataBlock* dmBuildVariablesBlock(void) { - SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); +SSDataBlock *dmBuildVariablesBlock(void) { + SSDataBlock *pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); size_t size = 0; - const SSysTableMeta* pMeta = NULL; + const SSysTableMeta *pMeta = NULL; getInfosDbMeta(&pMeta, &size); int32_t index = 0; @@ -215,7 +214,7 @@ SSDataBlock* dmBuildVariablesBlock(void) { return pBlock; } -int32_t dmAppendVariablesToBlock(SSDataBlock* pBlock, int32_t dnodeId) { +int32_t dmAppendVariablesToBlock(SSDataBlock *pBlock, int32_t dnodeId) { int32_t numOfCfg = taosArrayGetSize(tsCfg->array); int32_t numOfRows = 0; blockDataEnsureCapacity(pBlock, numOfCfg); @@ -230,8 +229,8 @@ int32_t dmAppendVariablesToBlock(SSDataBlock* pBlock, int32_t dnodeId) { STR_WITH_MAXSIZE_TO_VARSTR(name, pItem->name, TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE); pColInfo = taosArrayGet(pBlock->pDataBlock, c++); colDataAppend(pColInfo, i, name, false); - - char value[TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE] = {0}; + + char value[TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE] = {0}; int32_t valueLen = 0; cfgDumpItemValue(pItem, &value[VARSTR_HEADER_SIZE], TSDB_CONFIG_VALUE_LEN, &valueLen); varDataSetLen(value, valueLen); @@ -241,9 +240,8 @@ int32_t dmAppendVariablesToBlock(SSDataBlock* pBlock, int32_t dnodeId) { numOfRows++; } - pBlock->info.rows = numOfRows; - + return TSDB_CODE_SUCCESS; } @@ -267,7 +265,7 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { return -1; } - SSDataBlock* pBlock = dmBuildVariablesBlock(); + SSDataBlock *pBlock = dmBuildVariablesBlock(); dmAppendVariablesToBlock(pBlock, pMgmt->pData->dnodeId); @@ -283,14 +281,14 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { return -1; } - char *pStart = pRsp->data; + char *pStart = pRsp->data; *(int32_t *)pStart = htonl(numOfCols); pStart += sizeof(int32_t); // number of columns for (int32_t i = 0; i < numOfCols; ++i) { SSysTableSchema *pSchema = (SSysTableSchema *)pStart; SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, i); - + pSchema->bytes = htonl(pColInfo->info.bytes); pSchema->colId = htons(pColInfo->info.colId); pSchema->type = pColInfo->info.type; @@ -299,7 +297,7 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { } int32_t len = 0; - blockCompressEncode(pBlock, pStart, &len, numOfCols, false); + blockEncode(pBlock, pStart, &len, numOfCols, false); pRsp->numOfRows = htonl(pBlock->info.rows); pRsp->precision = TSDB_TIME_PRECISION_MILLI; // millisecond time precision diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 164bcc7d60..6014adbe95 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -15,8 +15,8 @@ #define _DEFAULT_SOURCE #include "mndShow.h" -#include "systable.h" #include "mndPrivilege.h" +#include "systable.h" #define SHOW_STEP_SIZE 100 @@ -307,7 +307,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { } int32_t len = 0; - blockCompressEncode(pBlock, pStart, &len, pShow->pMeta->numOfColumns, false); + blockEncode(pBlock, pStart, &len, pShow->pMeta->numOfColumns, false); } pRsp->numOfRows = htonl(rowsRead); diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index afeeeab500..c7d7c787a2 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -29,7 +29,7 @@ static int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataBlkRsp* pRs // TODO enable compress int32_t actualLen = 0; - blockCompressEncode(pBlock, pRetrieve->data, &actualLen, taosArrayGetSize(pBlock->pDataBlock), false); + blockEncode(pBlock, pRetrieve->data, &actualLen, taosArrayGetSize(pBlock->pDataBlock), false); actualLen += sizeof(SRetrieveTableRsp); ASSERT(actualLen <= dataStrLen); taosArrayPush(pRsp->blockDataLen, &actualLen); diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index cb14b916af..d6b6456515 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -18,8 +18,8 @@ #include "tdatablock.h" #include "tglobal.h" -extern SConfig *tsCfg; -static int32_t getSchemaBytes(const SSchema* pSchema) { +extern SConfig* tsCfg; +static int32_t getSchemaBytes(const SSchema* pSchema) { switch (pSchema->type) { case TSDB_DATA_TYPE_BINARY: return (pSchema->bytes - VARSTR_HEADER_SIZE); @@ -104,7 +104,7 @@ static int32_t execDescribe(SNode* pStmt, SRetrieveTableRsp** pRsp) { (*pRsp)->numOfCols = htonl(DESCRIBE_RESULT_COLS); int32_t len = 0; - blockCompressEncode(pBlock, (*pRsp)->data, &len, DESCRIBE_RESULT_COLS, false); + blockEncode(pBlock, (*pRsp)->data, &len, DESCRIBE_RESULT_COLS, false); ASSERT(len == rspSize - sizeof(SRetrieveTableRsp)); blockDataDestroy(pBlock); @@ -113,9 +113,8 @@ static int32_t execDescribe(SNode* pStmt, SRetrieveTableRsp** pRsp) { static int32_t execResetQueryCache() { return catalogClearCache(); } - static SSDataBlock* buildCreateDBResultDataBlock() { - SSDataBlock* pBlock = createDataBlock(); + SSDataBlock* pBlock = createDataBlock(); SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_DB_RESULT_COLS, 1); blockDataAppendColInfo(pBlock, &infoData); @@ -149,14 +148,14 @@ int64_t getValOfDiffPrecision(int8_t unit, int64_t val) { return v; } -char *buildRetension(SArray *pRetension) { +char* buildRetension(SArray* pRetension) { size_t size = taosArrayGetSize(pRetension); if (size == 0) { return NULL; } - char *p1 = taosMemoryCalloc(1, 100); - SRetention *p = taosArrayGet(pRetension, 0); + char* p1 = taosMemoryCalloc(1, 100); + SRetention* p = taosArrayGet(pRetension, 0); int32_t len = 0; @@ -185,8 +184,7 @@ char *buildRetension(SArray *pRetension) { return p1; } - -static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char *dbFName, SDbCfgInfo* pCfg) { +static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbFName, SDbCfgInfo* pCfg) { blockDataEnsureCapacity(pBlock, 1); pBlock->info.rows = 1; @@ -198,7 +196,7 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char *dbFName, S SColumnInfoData* pCol2 = taosArrayGet(pBlock->pDataBlock, 1); char buf2[SHOW_CREATE_DB_RESULT_FIELD2_LEN] = {0}; int32_t len = 0; - char *prec = NULL; + char* prec = NULL; switch (pCfg->precision) { case TSDB_TIME_PRECISION_MILLI: prec = TSDB_TIME_PRECISION_MILLI_STR; @@ -214,15 +212,16 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char *dbFName, S break; } - char *retentions = buildRetension(pCfg->pRetensions); - - len += sprintf(buf2 + VARSTR_HEADER_SIZE, "CREATE DATABASE `%s` BUFFER %d CACHELAST %d COMP %d DURATION %dm " - "FSYNC %d MAXROWS %d MINROWS %d KEEP %dm,%dm,%dm PAGES %d PAGESIZE %d PRECISION '%s' REPLICA %d " - "STRICT %d WAL %d VGROUPS %d SINGLE_STABLE %d", - dbFName, pCfg->buffer, pCfg->cacheLastRow, pCfg->compression, pCfg->daysPerFile, - pCfg->fsyncPeriod, pCfg->maxRows, pCfg->minRows, pCfg->daysToKeep0, pCfg->daysToKeep1, pCfg->daysToKeep2, - pCfg->pages, pCfg->pageSize, prec, pCfg->replications, pCfg->strict, pCfg->walLevel, pCfg->numOfVgroups, - 1 == pCfg->numOfStables); + char* retentions = buildRetension(pCfg->pRetensions); + + len += sprintf(buf2 + VARSTR_HEADER_SIZE, + "CREATE DATABASE `%s` BUFFER %d CACHELAST %d COMP %d DURATION %dm " + "FSYNC %d MAXROWS %d MINROWS %d KEEP %dm,%dm,%dm PAGES %d PAGESIZE %d PRECISION '%s' REPLICA %d " + "STRICT %d WAL %d VGROUPS %d SINGLE_STABLE %d", + dbFName, pCfg->buffer, pCfg->cacheLastRow, pCfg->compression, pCfg->daysPerFile, pCfg->fsyncPeriod, + pCfg->maxRows, pCfg->minRows, pCfg->daysToKeep0, pCfg->daysToKeep1, pCfg->daysToKeep2, pCfg->pages, + pCfg->pageSize, prec, pCfg->replications, pCfg->strict, pCfg->walLevel, pCfg->numOfVgroups, + 1 == pCfg->numOfStables); if (retentions) { len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, " RETENTIONS %s", retentions); @@ -230,11 +229,10 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char *dbFName, S } (varDataLen(buf2)) = len; - + colDataAppend(pCol2, 0, buf2, false); } - static int32_t execShowCreateDatabase(SShowCreateDatabaseStmt* pStmt, SRetrieveTableRsp** pRsp) { SSDataBlock* pBlock = buildCreateDBResultDataBlock(); setCreateDBResultIntoDataBlock(pBlock, pStmt->dbName, pStmt->pCfg); @@ -254,7 +252,7 @@ static int32_t execShowCreateDatabase(SShowCreateDatabaseStmt* pStmt, SRetrieveT (*pRsp)->numOfCols = htonl(SHOW_CREATE_DB_RESULT_COLS); int32_t len = 0; - blockCompressEncode(pBlock, (*pRsp)->data, &len, SHOW_CREATE_DB_RESULT_COLS, false); + blockEncode(pBlock, (*pRsp)->data, &len, SHOW_CREATE_DB_RESULT_COLS, false); ASSERT(len == rspSize - sizeof(SRetrieveTableRsp)); blockDataDestroy(pBlock); @@ -276,14 +274,14 @@ static SSDataBlock* buildCreateTbResultDataBlock() { void appendColumnFields(char* buf, int32_t* len, STableCfg* pCfg) { for (int32_t i = 0; i < pCfg->numOfColumns; ++i) { SSchema* pSchema = pCfg->pSchemas + i; - char type[32]; + char type[32]; sprintf(type, "%s", tDataTypes[pSchema->type].name); if (TSDB_DATA_TYPE_VARCHAR == pSchema->type) { sprintf(type + strlen(type), "(%d)", (int32_t)(pSchema->bytes - VARSTR_HEADER_SIZE)); } else if (TSDB_DATA_TYPE_NCHAR == pSchema->type) { - sprintf(type + strlen(type), "(%d)", (int32_t)((pSchema->bytes - VARSTR_HEADER_SIZE)/TSDB_NCHAR_SIZE)); + sprintf(type + strlen(type), "(%d)", (int32_t)((pSchema->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); } - + *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s`%s` %s", ((i > 0) ? ", " : ""), pSchema->name, type); } } @@ -291,19 +289,18 @@ void appendColumnFields(char* buf, int32_t* len, STableCfg* pCfg) { void appendTagFields(char* buf, int32_t* len, STableCfg* pCfg) { for (int32_t i = 0; i < pCfg->numOfTags; ++i) { SSchema* pSchema = pCfg->pSchemas + pCfg->numOfColumns + i; - char type[32]; + char type[32]; sprintf(type, "%s", tDataTypes[pSchema->type].name); if (TSDB_DATA_TYPE_VARCHAR == pSchema->type) { sprintf(type + strlen(type), "(%d)", (int32_t)(pSchema->bytes - VARSTR_HEADER_SIZE)); } else if (TSDB_DATA_TYPE_NCHAR == pSchema->type) { - sprintf(type + strlen(type), "(%d)", (int32_t)((pSchema->bytes - VARSTR_HEADER_SIZE)/TSDB_NCHAR_SIZE)); + sprintf(type + strlen(type), "(%d)", (int32_t)((pSchema->bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); } *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s`%s` %s", ((i > 0) ? ", " : ""), pSchema->name, type); } } - void appendTagNameFields(char* buf, int32_t* len, STableCfg* pCfg) { for (int32_t i = 0; i < pCfg->numOfTags; ++i) { SSchema* pSchema = pCfg->pSchemas + pCfg->numOfColumns + i; @@ -311,13 +308,12 @@ void appendTagNameFields(char* buf, int32_t* len, STableCfg* pCfg) { } } - int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) { - SArray *pTagVals = NULL; - STag *pTag = (STag*)pCfg->pTags; - + SArray* pTagVals = NULL; + STag* pTag = (STag*)pCfg->pTags; + if (pCfg->pTags && pTag->flags & TD_TAG_JSON) { - char *pJson = parseTagDatatoJson(pTag); + char* pJson = parseTagDatatoJson(pTag); if (pJson) { *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s", pJson); taosMemoryFree(pJson); @@ -325,8 +321,8 @@ int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) { return TSDB_CODE_SUCCESS; } - - int32_t code = tTagToValArray((const STag *)pCfg->pTags, &pTagVals); + + int32_t code = tTagToValArray((const STag*)pCfg->pTags, &pTagVals); if (code) { return code; } @@ -339,20 +335,20 @@ int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) { if (i > 0) { *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, ", "); } - + if (j >= valueNum) { *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "NULL"); continue; } - - STagVal *pTagVal = (STagVal *)taosArrayGet(pTagVals, j); + + STagVal* pTagVal = (STagVal*)taosArrayGet(pTagVals, j); if (pSchema->colId > pTagVal->cid) { qError("tag value and column mismatch, schemaId:%d, valId:%d", pSchema->colId, pTagVal->cid); taosArrayDestroy(pTagVals); return TSDB_CODE_APP_ERROR; } else if (pSchema->colId == pTagVal->cid) { - char type = pTagVal->type; - int32_t tlen = 0; + char type = pTagVal->type; + int32_t tlen = 0; if (IS_VAR_DATA_TYPE(type)) { dataConverToStr(buf + VARSTR_HEADER_SIZE + *len, type, pTagVal->pData, pTagVal->nData, &tlen); @@ -364,7 +360,6 @@ int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) { } else { *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "NULL"); } - /* if (type == TSDB_DATA_TYPE_BINARY) { @@ -372,7 +367,7 @@ int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) { if (num) { *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, ", "); } - + memcpy(buf + VARSTR_HEADER_SIZE + *len, pTagVal->pData, pTagVal->nData); *len += pTagVal->nData; } @@ -397,7 +392,7 @@ int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) { taosArrayDestroy(pTagVals); - return TSDB_CODE_SUCCESS; + return TSDB_CODE_SUCCESS; } void appendTableOptions(char* buf, int32_t* len, STableCfg* pCfg) { @@ -426,7 +421,7 @@ void appendTableOptions(char* buf, int32_t* len, STableCfg* pCfg) { *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, " ROLLUP("); for (int32_t i = 0; i < funcNum; ++i) { char* pFunc = taosArrayGet(pCfg->pFuncs, i); - *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s%s", ((i > 0) ? ", " : ""), pFunc); + *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s%s", ((i > 0) ? ", " : ""), pFunc); } *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, ")"); } @@ -436,7 +431,7 @@ void appendTableOptions(char* buf, int32_t* len, STableCfg* pCfg) { } } -static int32_t setCreateTBResultIntoDataBlock(SSDataBlock* pBlock, char *tbName, STableCfg* pCfg) { +static int32_t setCreateTBResultIntoDataBlock(SSDataBlock* pBlock, char* tbName, STableCfg* pCfg) { int32_t code = 0; blockDataEnsureCapacity(pBlock, 1); pBlock->info.rows = 1; @@ -454,7 +449,7 @@ static int32_t setCreateTBResultIntoDataBlock(SSDataBlock* pBlock, char *tbName, len += sprintf(buf2 + VARSTR_HEADER_SIZE, "CREATE STABLE `%s` (", tbName); appendColumnFields(buf2, &len, pCfg); len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, ") TAGS ("); - appendTagFields(buf2, &len, pCfg); + appendTagFields(buf2, &len, pCfg); len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, ")"); appendTableOptions(buf2, &len, pCfg); } else if (TSDB_CHILD_TABLE == pCfg->tableType) { @@ -474,16 +469,15 @@ static int32_t setCreateTBResultIntoDataBlock(SSDataBlock* pBlock, char *tbName, } varDataLen(buf2) = len; - + colDataAppend(pCol2, 0, buf2, false); return TSDB_CODE_SUCCESS; } - static int32_t execShowCreateTable(SShowCreateTableStmt* pStmt, SRetrieveTableRsp** pRsp) { SSDataBlock* pBlock = buildCreateTbResultDataBlock(); - int32_t code = setCreateTBResultIntoDataBlock(pBlock, pStmt->tableName, pStmt->pCfg); + int32_t code = setCreateTBResultIntoDataBlock(pBlock, pStmt->tableName, pStmt->pCfg); if (code) { return code; } @@ -503,7 +497,7 @@ static int32_t execShowCreateTable(SShowCreateTableStmt* pStmt, SRetrieveTableRs (*pRsp)->numOfCols = htonl(SHOW_CREATE_TB_RESULT_COLS); int32_t len = 0; - blockCompressEncode(pBlock, (*pRsp)->data, &len, SHOW_CREATE_TB_RESULT_COLS, false); + blockEncode(pBlock, (*pRsp)->data, &len, SHOW_CREATE_TB_RESULT_COLS, false); ASSERT(len == rspSize - sizeof(SRetrieveTableRsp)); blockDataDestroy(pBlock); @@ -516,17 +510,17 @@ static int32_t execShowCreateSTable(SShowCreateTableStmt* pStmt, SRetrieveTableR terrno = TSDB_CODE_TSC_NOT_STABLE_ERROR; return terrno; } - + return execShowCreateTable(pStmt, pRsp); } -static int32_t execAlterLocal(SAlterLocalStmt* pStmt) { +static int32_t execAlterLocal(SAlterLocalStmt* pStmt) { if (cfgSetItem(tsCfg, pStmt->config, pStmt->value, CFG_STYPE_ALTER_CMD)) { - return terrno; + return terrno; } if (taosSetCfg(tsCfg, pStmt->config)) { - return terrno; + return terrno; } return TSDB_CODE_SUCCESS; @@ -551,21 +545,20 @@ static SSDataBlock* buildLocalVariablesResultDataBlock() { return pBlock; } - int32_t setLocalVariablesResultIntoDataBlock(SSDataBlock* pBlock) { int32_t numOfCfg = taosArrayGetSize(tsCfg->array); int32_t numOfRows = 0; blockDataEnsureCapacity(pBlock, numOfCfg); for (int32_t i = 0, c = 0; i < numOfCfg; ++i, c = 0) { - SConfigItem *pItem = taosArrayGet(tsCfg->array, i); + SConfigItem* pItem = taosArrayGet(tsCfg->array, i); char name[TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE] = {0}; STR_WITH_MAXSIZE_TO_VARSTR(name, pItem->name, TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE); - SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, c++); + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, c++); colDataAppend(pColInfo, i, name, false); - - char value[TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE] = {0}; + + char value[TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE] = {0}; int32_t valueLen = 0; cfgDumpItemValue(pItem, &value[VARSTR_HEADER_SIZE], TSDB_CONFIG_VALUE_LEN, &valueLen); varDataSetLen(value, valueLen); @@ -575,16 +568,14 @@ int32_t setLocalVariablesResultIntoDataBlock(SSDataBlock* pBlock) { numOfRows++; } - pBlock->info.rows = numOfRows; - + return TSDB_CODE_SUCCESS; } - static int32_t execShowLocalVariables(SRetrieveTableRsp** pRsp) { SSDataBlock* pBlock = buildLocalVariablesResultDataBlock(); - int32_t code = setLocalVariablesResultIntoDataBlock(pBlock); + int32_t code = setLocalVariablesResultIntoDataBlock(pBlock); if (code) { return code; } @@ -604,7 +595,7 @@ static int32_t execShowLocalVariables(SRetrieveTableRsp** pRsp) { (*pRsp)->numOfCols = htonl(SHOW_LOCAL_VARIABLES_RESULT_COLS); int32_t len = 0; - blockCompressEncode(pBlock, (*pRsp)->data, &len, SHOW_LOCAL_VARIABLES_RESULT_COLS, false); + blockEncode(pBlock, (*pRsp)->data, &len, SHOW_LOCAL_VARIABLES_RESULT_COLS, false); ASSERT(len == rspSize - sizeof(SRetrieveTableRsp)); blockDataDestroy(pBlock); diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index ef4ac0b639..7af36a0842 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -13,11 +13,11 @@ * along with this program. If not, see . */ -#include "tdatablock.h" #include "commandInt.h" #include "plannodes.h" #include "query.h" #include "tcommon.h" +#include "tdatablock.h" int32_t qExplainGenerateResNode(SPhysiNode *pNode, SExplainGroup *group, SExplainResNode **pRes); int32_t qExplainAppendGroupResRows(void *pCtx, int32_t groupId, int32_t level); @@ -216,7 +216,7 @@ int32_t qExplainGenerateResChildren(SPhysiNode *pNode, SExplainGroup *group, SNo SExplainResNode *pResNode = NULL; FOREACH(node, pPhysiChildren) { QRY_ERR_RET(qExplainGenerateResNode((SPhysiNode *)node, group, &pResNode)); - QRY_ERR_RET(nodesListAppend(*pChildren, (SNode*)pResNode)); + QRY_ERR_RET(nodesListAppend(*pChildren, (SNode *)pResNode)); } return TSDB_CODE_SUCCESS; @@ -232,14 +232,14 @@ int32_t qExplainGenerateResNodeExecInfo(SArray **pExecInfo, SExplainGroup *group SExplainRsp *rsp = NULL; for (int32_t i = 0; i < group->nodeNum; ++i) { rsp = taosArrayGet(group->nodeExecInfo, i); -/* - if (group->physiPlanExecIdx >= rsp->numOfPlans) { - qError("physiPlanIdx %d exceed plan num %d", group->physiPlanExecIdx, rsp->numOfPlans); - return TSDB_CODE_QRY_APP_ERROR; - } + /* + if (group->physiPlanExecIdx >= rsp->numOfPlans) { + qError("physiPlanIdx %d exceed plan num %d", group->physiPlanExecIdx, rsp->numOfPlans); + return TSDB_CODE_QRY_APP_ERROR; + } - taosArrayPush(*pExecInfo, rsp->subplanInfo + group->physiPlanExecIdx); -*/ + taosArrayPush(*pExecInfo, rsp->subplanInfo + group->physiPlanExecIdx); + */ taosArrayPush(*pExecInfo, rsp->subplanInfo); } @@ -426,23 +426,23 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i if (EXPLAIN_MODE_ANALYZE == ctx->mode) { EXPLAIN_ROW_NEW(level + 1, "I/O: "); - int32_t nodeNum = taosArrayGetSize(pResNode->pExecInfo); + int32_t nodeNum = taosArrayGetSize(pResNode->pExecInfo); struct STableScanAnalyzeInfo info = {0}; int32_t maxIndex = 0; int32_t totalRows = 0; - for(int32_t i = 0; i < nodeNum; ++i) { + for (int32_t i = 0; i < nodeNum; ++i) { SExplainExecInfo *execInfo = taosArrayGet(pResNode->pExecInfo, i); STableScanAnalyzeInfo *pScanInfo = (STableScanAnalyzeInfo *)execInfo->verboseInfo; - info.totalBlocks += pScanInfo->totalBlocks; - info.loadBlocks += pScanInfo->loadBlocks; - info.totalRows += pScanInfo->totalRows; - info.skipBlocks += pScanInfo->skipBlocks; - info.filterTime += pScanInfo->filterTime; - info.loadBlockStatis += pScanInfo->loadBlockStatis; + info.totalBlocks += pScanInfo->totalBlocks; + info.loadBlocks += pScanInfo->loadBlocks; + info.totalRows += pScanInfo->totalRows; + info.skipBlocks += pScanInfo->skipBlocks; + info.filterTime += pScanInfo->filterTime; + info.loadBlockStatis += pScanInfo->loadBlockStatis; info.totalCheckedRows += pScanInfo->totalCheckedRows; - info.filterOutBlocks += pScanInfo->filterOutBlocks; + info.filterOutBlocks += pScanInfo->filterOutBlocks; if (pScanInfo->totalRows > totalRows) { totalRows = pScanInfo->totalRows; @@ -468,13 +468,14 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - //Rows out: Avg 4166.7 rows x 24 workers. Max 4187 rows (seg7) with 0.220 ms to first row, 1.738 ms to end, start offset by 1.470 ms. + // Rows out: Avg 4166.7 rows x 24 workers. Max 4187 rows (seg7) with 0.220 ms to first row, 1.738 ms to end, + // start offset by 1.470 ms. SExplainExecInfo *execInfo = taosArrayGet(pResNode->pExecInfo, maxIndex); STableScanAnalyzeInfo *p1 = (STableScanAnalyzeInfo *)execInfo->verboseInfo; EXPLAIN_ROW_NEW(level + 1, " "); - EXPLAIN_ROW_APPEND("max_row_task=%d, total_rows:%" PRId64 ", ep:%s (cost=%.3f..%.3f)", maxIndex, p1->totalRows, "tbd", - execInfo->startupCost, execInfo->totalCost); + EXPLAIN_ROW_APPEND("max_row_task=%d, total_rows:%" PRId64 ", ep:%s (cost=%.3f..%.3f)", maxIndex, p1->totalRows, + "tbd", execInfo->startupCost, execInfo->totalCost); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_END(); @@ -752,7 +753,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_NEW(level + 1, "Sort Key: "); if (pResNode->pExecInfo) { for (int32_t i = 0; i < LIST_LENGTH(pSortNode->pSortKeys); ++i) { - SOrderByExprNode *ptn = (SOrderByExprNode*)nodesListGetNode(pSortNode->pSortKeys, i); + SOrderByExprNode *ptn = (SOrderByExprNode *)nodesListGetNode(pSortNode->pSortKeys, i); EXPLAIN_ROW_APPEND("%s ", nodesGetNameFromColumnNode(ptn->pExpr)); } } @@ -907,16 +908,16 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); if (pFillNode->pValues) { - SNodeListNode *pValues = (SNodeListNode*)pFillNode->pValues; + SNodeListNode *pValues = (SNodeListNode *)pFillNode->pValues; EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILL_VALUE_FORMAT); - SNode* tNode = NULL; + SNode *tNode = NULL; int32_t i = 0; FOREACH(tNode, pValues->pNodeList) { if (i) { EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); } - SValueNode* tValue = (SValueNode*)tNode; - char *value = nodesGetStrValueFromNode(tValue); + SValueNode *tValue = (SValueNode *)tNode; + char *value = nodesGetStrValueFromNode(tValue); EXPLAIN_ROW_APPEND(EXPLAIN_STRING_TYPE_FORMAT, value); taosMemoryFree(value); ++i; @@ -926,8 +927,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); } - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIMERANGE_FORMAT, pFillNode->timeRange.skey, - pFillNode->timeRange.ekey); + EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIMERANGE_FORMAT, pFillNode->timeRange.skey, pFillNode->timeRange.ekey); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); @@ -1070,13 +1070,13 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); - + if (EXPLAIN_MODE_ANALYZE == ctx->mode) { // sort key EXPLAIN_ROW_NEW(level + 1, "Merge Key: "); if (pResNode->pExecInfo) { for (int32_t i = 0; i < LIST_LENGTH(pMergeNode->pMergeKeys); ++i) { - SOrderByExprNode *ptn = (SOrderByExprNode*)nodesListGetNode(pMergeNode->pMergeKeys, i); + SOrderByExprNode *ptn = (SOrderByExprNode *)nodesListGetNode(pMergeNode->pMergeKeys, i); EXPLAIN_ROW_APPEND("%s ", nodesGetNameFromColumnNode(ptn->pExpr)); } } @@ -1115,7 +1115,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_NEW(level + 1, EXPLAIN_MERGE_KEYS_FORMAT); for (int32_t i = 0; i < LIST_LENGTH(pMergeNode->pMergeKeys); ++i) { - SOrderByExprNode *ptn = (SOrderByExprNode*)nodesListGetNode(pMergeNode->pMergeKeys, i); + SOrderByExprNode *ptn = (SOrderByExprNode *)nodesListGetNode(pMergeNode->pMergeKeys, i); EXPLAIN_ROW_APPEND("%s ", nodesGetNameFromColumnNode(ptn->pExpr)); } EXPLAIN_ROW_END(); @@ -1130,7 +1130,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i } } break; - } + } default: qError("not supported physical node type %d", pNode->type); return TSDB_CODE_QRY_APP_ERROR; @@ -1190,12 +1190,12 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - SSDataBlock *pBlock = createDataBlock(); + SSDataBlock *pBlock = createDataBlock(); SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, TSDB_EXPLAIN_RESULT_ROW_SIZE, 1); blockDataAppendColInfo(pBlock, &infoData); blockDataEnsureCapacity(pBlock, rowNum); - SColumnInfoData* pInfoData = taosArrayGet(pBlock->pDataBlock, 0); + SColumnInfoData *pInfoData = taosArrayGet(pBlock->pDataBlock, 0); char buf[1024] = {0}; for (int32_t i = 0; i < rowNum; ++i) { @@ -1219,7 +1219,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { rsp->numOfRows = htonl(rowNum); int32_t len = 0; - blockCompressEncode(pBlock, rsp->data, &len, taosArrayGetSize(pBlock->pDataBlock), 0); + blockEncode(pBlock, rsp->data, &len, taosArrayGetSize(pBlock->pDataBlock), 0); ASSERT(len == rspSize - sizeof(SRetrieveTableRsp)); rsp->compLen = htonl(len); diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index f3e1eb47e8..df961c00fa 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -25,37 +25,36 @@ #define SET_RES_WINDOW_KEY(_k, _ori, _len, _uid) \ do { \ assert(sizeof(_uid) == sizeof(uint64_t)); \ - *(uint64_t *)(_k) = (_uid); \ + *(uint64_t*)(_k) = (_uid); \ memcpy((_k) + sizeof(uint64_t), (_ori), (_len)); \ } while (0) -#define SET_RES_EXT_WINDOW_KEY(_k, _ori, _len, _uid, _buf) \ - do { \ - assert(sizeof(_uid) == sizeof(uint64_t)); \ - *(void **)(_k) = (_buf); \ - *(uint64_t *)((_k) + POINTER_BYTES) = (_uid); \ - memcpy((_k) + POINTER_BYTES + sizeof(uint64_t), (_ori), (_len)); \ +#define SET_RES_EXT_WINDOW_KEY(_k, _ori, _len, _uid, _buf) \ + do { \ + assert(sizeof(_uid) == sizeof(uint64_t)); \ + *(void**)(_k) = (_buf); \ + *(uint64_t*)((_k) + POINTER_BYTES) = (_uid); \ + memcpy((_k) + POINTER_BYTES + sizeof(uint64_t), (_ori), (_len)); \ } while (0) - -#define GET_RES_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t)) +#define GET_RES_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t)) #define GET_RES_EXT_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t) + POINTER_BYTES) -#define GET_TASKID(_t) (((SExecTaskInfo*)(_t))->id.str) +#define GET_TASKID(_t) (((SExecTaskInfo*)(_t))->id.str) typedef struct SGroupResInfo { int32_t index; - SArray* pRows; // SArray + SArray* pRows; // SArray } SGroupResInfo; typedef struct SResultRow { - int32_t pageId; // pageId & rowId is the position of current result in disk-based output buffer - int32_t offset:29; // row index in buffer page - bool startInterp; // the time window start timestamp has done the interpolation already. - bool endInterp; // the time window end timestamp has done the interpolation already. - bool closed; // this result status: closed or opened - uint32_t numOfRows; // number of rows of current time window - STimeWindow win; + int32_t pageId; // pageId & rowId is the position of current result in disk-based output buffer + int32_t offset : 29; // row index in buffer page + bool startInterp; // the time window start timestamp has done the interpolation already. + bool endInterp; // the time window end timestamp has done the interpolation already. + bool closed; // this result status: closed or opened + uint32_t numOfRows; // number of rows of current time window + STimeWindow win; struct SResultRowEntryInfo pEntryInfo[]; // For each result column, there is a resultInfo } SResultRow; @@ -66,57 +65,58 @@ typedef struct SResultRowPosition { typedef struct SResKeyPos { SResultRowPosition pos; - uint64_t groupId; - char key[]; + uint64_t groupId; + char key[]; } SResKeyPos; typedef struct SResultRowInfo { - int32_t size; // number of result set + int32_t size; // number of result set SResultRowPosition cur; - SList* openWindow; + SList* openWindow; } SResultRowInfo; struct SqlFunctionCtx; -size_t getResultRowSize(struct SqlFunctionCtx* pCtx, int32_t numOfOutput); -void initResultRowInfo(SResultRowInfo* pResultRowInfo); -void cleanupResultRowInfo(SResultRowInfo* pResultRowInfo); +size_t getResultRowSize(struct SqlFunctionCtx* pCtx, int32_t numOfOutput); +void initResultRowInfo(SResultRowInfo* pResultRowInfo); +void cleanupResultRowInfo(SResultRowInfo* pResultRowInfo); -void closeAllResultRows(SResultRowInfo* pResultRowInfo); +void closeAllResultRows(SResultRowInfo* pResultRowInfo); -void initResultRow(SResultRow *pResultRow); -void closeResultRow(SResultRow* pResultRow); -bool isResultRowClosed(SResultRow* pResultRow); +void initResultRow(SResultRow* pResultRow); +void closeResultRow(SResultRow* pResultRow); +bool isResultRowClosed(SResultRow* pResultRow); struct SResultRowEntryInfo* getResultEntryInfo(const SResultRow* pRow, int32_t index, const int32_t* offset); -static FORCE_INLINE SResultRow *getResultRowByPos(SDiskbasedBuf* pBuf, SResultRowPosition* pos) { - SFilePage* bufPage = (SFilePage*) getBufPage(pBuf, pos->pageId); +static FORCE_INLINE SResultRow* getResultRowByPos(SDiskbasedBuf* pBuf, SResultRowPosition* pos) { + SFilePage* bufPage = (SFilePage*)getBufPage(pBuf, pos->pageId); SResultRow* pRow = (SResultRow*)((char*)bufPage + pos->offset); return pRow; } -void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, int32_t order); -void initMultiResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList); +void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, int32_t order); +void initMultiResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList); -void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo); -bool hasDataInGroupInfo(SGroupResInfo* pGroupResInfo); +void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo); +bool hasDataInGroupInfo(SGroupResInfo* pGroupResInfo); int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo); SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode); EDealRes doTranslateTagExpr(SNode** pNode, void* pContext); -int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo* pListInfo); -SArray* createSortInfo(SNodeList* pNodeList); -SArray* extractPartitionColInfo(SNodeList* pNodeList); -SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols, int32_t type); +int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo* pListInfo); +SArray* createSortInfo(SNodeList* pNodeList); +SArray* extractPartitionColInfo(SNodeList* pNodeList); +SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols, + int32_t type); SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* numOfExprs); SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowEntryInfoOffset); -void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray* pCols, bool outputEveryColumn); -void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow); +void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray* pCols, bool outputEveryColumn); +void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow); SInterval extractIntervalInfo(const STableScanPhysiNode* pTableScanNode); SColumn extractColumnFromColumnNode(SColumnNode* pColNode); diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index 802f9ea5b5..5ee222efb7 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -69,14 +69,16 @@ static bool needCompress(const SSDataBlock* pData, int32_t numOfCols) { // data format: // +----------------+--------------+----------+--------------------------------------------+--------------------------------------+-------------+-----------+-------------+-----------+ -// |SDataCacheEntry | total length | group id | col1_schema | col2_schema | col3_schema ...| column#1 length, column#2 length ... | col1 bitmap | col1 data | col2 bitmap | col2 data | .... -// | | (4 bytes) |(8 bytes) |(sizeof(int16_t)+sizeof(int32_t))*numOfCols | sizeof(int32_t) * numOfCols | actual size | | actual size | | +// |SDataCacheEntry | total length | group id | col1_schema | col2_schema | col3_schema ...| column#1 length, column#2 +// length ... | col1 bitmap | col1 data | col2 bitmap | col2 data | .... | | (4 bytes) |(8 bytes) +// |(sizeof(int16_t)+sizeof(int32_t))*numOfCols | sizeof(int32_t) * numOfCols | actual size | | +// actual size | | // +----------------+--------------+----------+--------------------------------------------+--------------------------------------+-------------+-----------+-------------+-----------+ // The length of bitmap is decided by number of rows of this data block, and the length of each column data is // recorded in the first segment, next to the struct header static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pInput, SDataDispatchBuf* pBuf) { int32_t numOfCols = 0; - SNode* pNode; + SNode* pNode; FOREACH(pNode, pHandle->pSchema->pSlots) { SSlotDescNode* pSlotDesc = (SSlotDescNode*)pNode; if (pSlotDesc->output) { @@ -90,12 +92,12 @@ static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pIn pEntry->dataLen = 0; pBuf->useSize = sizeof(SDataCacheEntry); - blockCompressEncode(pInput->pData, pEntry->data, &pEntry->dataLen, numOfCols, pEntry->compressed); + blockEncode(pInput->pData, pEntry->data, &pEntry->dataLen, numOfCols, pEntry->compressed); pBuf->useSize += pEntry->dataLen; - - atomic_add_fetch_64(&pHandle->cachedSize, pEntry->dataLen); - atomic_add_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen); + + atomic_add_fetch_64(&pHandle->cachedSize, pEntry->dataLen); + atomic_add_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen); } static bool allocBuf(SDataDispatchHandle* pDispatcher, const SInputData* pInput, SDataDispatchBuf* pBuf) { @@ -187,8 +189,8 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { pOutput->numOfCols = pEntry->numOfCols; pOutput->compressed = pEntry->compressed; - atomic_sub_fetch_64(&pDispatcher->cachedSize, pEntry->dataLen); - atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen); + atomic_sub_fetch_64(&pDispatcher->cachedSize, pEntry->dataLen); + atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen); taosMemoryFreeClear(pDispatcher->nextOutput.pData); // todo persistent pOutput->bufStatus = updateStatus(pDispatcher); @@ -198,7 +200,6 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { pOutput->precision = pDispatcher->pSchema->precision; taosThreadMutexUnlock(&pDispatcher->mutex); - return TSDB_CODE_SUCCESS; } diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 5c2f32126b..2561516f6a 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -189,9 +189,9 @@ SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode) { for (int32_t i = 0; i < numOfCols; ++i) { SSlotDescNode* pDescNode = (SSlotDescNode*)nodesListGetNode(pNode->pSlots, i); - if (!pDescNode->output) { // todo disable it temporarily - continue; - } + /*if (!pDescNode->output) { // todo disable it temporarily*/ + /*continue;*/ + /*}*/ SColumnInfoData idata = createColumnInfoData(pDescNode->dataType.type, pDescNode->dataType.bytes, pDescNode->slotId); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index ad0d5c1447..706fc87520 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2048,7 +2048,7 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLo if (pColList == NULL) { // data from other sources blockDataCleanup(pRes); // blockDataEnsureCapacity(pRes, numOfRows); - blockCompressDecode(pRes, numOfOutput, numOfRows, pData); + blockDecode(pRes, numOfOutput, numOfRows, pData); } else { // extract data according to pColList ASSERT(numOfOutput == taosArrayGetSize(pColList)); char* pStart = pData; @@ -2072,7 +2072,7 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLo blockDataAppendColInfo(pBlock, &idata); } - blockCompressDecode(pBlock, numOfCols, numOfRows, pStart); + blockDecode(pBlock, numOfCols, numOfRows, pStart); blockDataEnsureCapacity(pRes, numOfRows); // data from mnode @@ -2934,7 +2934,7 @@ int32_t aggEncodeResultRow(SOperatorInfo* pOperator, char** result, int32_t* len *length = 0; return TSDB_CODE_SUCCESS; } - + *result = (char*)taosMemoryCalloc(1, totalSize); if (*result == NULL) { return TSDB_CODE_OUT_OF_MEMORY; @@ -3898,60 +3898,59 @@ int32_t extractTableSchemaVersion(SReadHandle* pHandle, uint64_t uid, SExecTaskI return TSDB_CODE_SUCCESS; } -static int32_t sortTableGroup(STableListInfo* pTableListInfo, int32_t groupNum){ +static int32_t sortTableGroup(STableListInfo* pTableListInfo, int32_t groupNum) { taosArrayClear(pTableListInfo->pGroupList); - SArray *sortSupport = taosArrayInit(groupNum, sizeof(uint64_t)); - if(sortSupport == NULL) return TSDB_CODE_OUT_OF_MEMORY; + SArray* sortSupport = taosArrayInit(groupNum, sizeof(uint64_t)); + if (sortSupport == NULL) return TSDB_CODE_OUT_OF_MEMORY; for (int32_t i = 0; i < taosArrayGetSize(pTableListInfo->pTableList); i++) { STableKeyInfo* info = taosArrayGet(pTableListInfo->pTableList, i); - uint64_t* groupId = taosHashGet(pTableListInfo->map, &info->uid, sizeof(uint64_t)); + uint64_t* groupId = taosHashGet(pTableListInfo->map, &info->uid, sizeof(uint64_t)); int32_t index = taosArraySearchIdx(sortSupport, groupId, compareUint64Val, TD_EQ); - if (index == -1){ - void *p = taosArraySearch(sortSupport, groupId, compareUint64Val, TD_GT); - SArray *tGroup = taosArrayInit(8, sizeof(STableKeyInfo)); - if(tGroup == NULL) { + if (index == -1) { + void* p = taosArraySearch(sortSupport, groupId, compareUint64Val, TD_GT); + SArray* tGroup = taosArrayInit(8, sizeof(STableKeyInfo)); + if (tGroup == NULL) { taosArrayDestroy(sortSupport); return TSDB_CODE_OUT_OF_MEMORY; } - if(taosArrayPush(tGroup, info) == NULL){ + if (taosArrayPush(tGroup, info) == NULL) { qError("taos push info array error"); taosArrayDestroy(sortSupport); return TSDB_CODE_QRY_APP_ERROR; } - if(p == NULL){ - if(taosArrayPush(sortSupport, groupId) != NULL){ + if (p == NULL) { + if (taosArrayPush(sortSupport, groupId) != NULL) { qError("taos push support array error"); taosArrayDestroy(sortSupport); return TSDB_CODE_QRY_APP_ERROR; } - if(taosArrayPush(pTableListInfo->pGroupList, &tGroup) != NULL){ + if (taosArrayPush(pTableListInfo->pGroupList, &tGroup) != NULL) { qError("taos push group array error"); taosArrayDestroy(sortSupport); return TSDB_CODE_QRY_APP_ERROR; } - }else{ + } else { int32_t pos = TARRAY_ELEM_IDX(sortSupport, p); - if(taosArrayInsert(sortSupport, pos, groupId) == NULL){ + if (taosArrayInsert(sortSupport, pos, groupId) == NULL) { qError("taos insert support array error"); taosArrayDestroy(sortSupport); return TSDB_CODE_QRY_APP_ERROR; } - if(taosArrayInsert(pTableListInfo->pGroupList, pos, &tGroup) == NULL){ + if (taosArrayInsert(pTableListInfo->pGroupList, pos, &tGroup) == NULL) { qError("taos insert group array error"); taosArrayDestroy(sortSupport); return TSDB_CODE_QRY_APP_ERROR; } } - }else{ + } else { SArray* tGroup = (SArray*)taosArrayGetP(pTableListInfo->pGroupList, index); - if(taosArrayPush(tGroup, info) == NULL){ + if (taosArrayPush(tGroup, info) == NULL) { qError("taos push uid array error"); taosArrayDestroy(sortSupport); return TSDB_CODE_QRY_APP_ERROR; } } - } taosArrayDestroy(sortSupport); return TDB_CODE_SUCCESS; @@ -3969,9 +3968,9 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, int32_t keyLen = 0; void* keyBuf = NULL; - SNode* node; + SNode* node; FOREACH(node, group) { - SExprNode *pExpr = (SExprNode *)node; + SExprNode* pExpr = (SExprNode*)node; keyLen += pExpr->resType.bytes; } @@ -3990,15 +3989,15 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, metaReaderInit(&mr, pHandle->meta, 0); metaGetTableEntryByUid(&mr, info->uid); - SNodeList *groupNew = nodesCloneList(group); + SNodeList* groupNew = nodesCloneList(group); nodesRewriteExprsPostOrder(groupNew, doTranslateTagExpr, &mr); char* isNull = (char*)keyBuf; char* pStart = (char*)keyBuf + nullFlagSize; - SNode* pNode; + SNode* pNode; int32_t index = 0; - FOREACH(pNode, groupNew){ + FOREACH(pNode, groupNew) { SNode* pNew = NULL; int32_t code = scalarCalculateConstants(pNode, &pNew); if (TSDB_CODE_SUCCESS == code) { @@ -4010,15 +4009,15 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, } ASSERT(nodeType(pNew) == QUERY_NODE_VALUE); - SValueNode *pValue = (SValueNode *)pNew; + SValueNode* pValue = (SValueNode*)pNew; if (pValue->node.resType.type == TSDB_DATA_TYPE_NULL) { isNull[index++] = 1; continue; } else { isNull[index++] = 0; - char* data = nodesGetValueFromNode(pValue); - if (pValue->node.resType.type == TSDB_DATA_TYPE_JSON){ + char* data = nodesGetValueFromNode(pValue); + if (pValue->node.resType.type == TSDB_DATA_TYPE_JSON) { int32_t len = getJsonValueLen(data); memcpy(pStart, data, len); pStart += len; @@ -4031,7 +4030,7 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, } } } - int32_t len = (int32_t)(pStart - (char*)keyBuf); + int32_t len = (int32_t)(pStart - (char*)keyBuf); uint64_t groupId = calcGroupId(keyBuf, len); taosHashPut(pTableListInfo->map, &(info->uid), sizeof(uint64_t), &groupId, sizeof(uint64_t)); info->groupId = groupId; @@ -4042,7 +4041,7 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, } taosMemoryFree(keyBuf); - if(pTableListInfo->needSortTableByGroupId){ + if (pTableListInfo->needSortTableByGroupId) { return sortTableGroup(pTableListInfo, groupNum); } @@ -4050,7 +4049,8 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, } SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, - uint64_t queryId, uint64_t taskId, STableListInfo* pTableListInfo, const char* pUser) { + uint64_t queryId, uint64_t taskId, STableListInfo* pTableListInfo, + const char* pUser) { int32_t type = nodeType(pPhyNode); if (pPhyNode->pChildren == NULL || LIST_LENGTH(pPhyNode->pChildren) == 0) { @@ -4058,7 +4058,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode; int32_t code = createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId); - if(code){ + if (code) { pTaskInfo->code = code; return NULL; } @@ -4076,7 +4076,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN == type) { STableMergeScanPhysiNode* pTableScanNode = (STableMergeScanPhysiNode*)pPhyNode; int32_t code = createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId); - if(code){ + if (code) { return NULL; } code = extractTableSchemaVersion(pHandle, pTableScanNode->scan.uid, pTaskInfo); @@ -4085,7 +4085,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo return NULL; } - SOperatorInfo* pOperator = createTableMergeScanOperatorInfo(pTableScanNode, pTableListInfo, pHandle, pTaskInfo, queryId, taskId); + SOperatorInfo* pOperator = + createTableMergeScanOperatorInfo(pTableScanNode, pTableListInfo, pHandle, pTaskInfo, queryId, taskId); STableScanInfo* pScanInfo = pOperator->info; pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder; @@ -4105,7 +4106,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId); } - SOperatorInfo* pOperator = createStreamScanOperatorInfo(pHandle, pTableScanNode, pTaskInfo, &twSup, queryId, taskId); + SOperatorInfo* pOperator = + createStreamScanOperatorInfo(pHandle, pTableScanNode, pTaskInfo, &twSup, queryId, taskId); return pOperator; } else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) { @@ -4603,8 +4605,8 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead (*pTaskInfo)->sql = sql; (*pTaskInfo)->tableqinfoList.pTagCond = pPlan->pTagCond; (*pTaskInfo)->tableqinfoList.pTagIndexCond = pPlan->pTagIndexCond; - (*pTaskInfo)->pRoot = - createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId, &(*pTaskInfo)->tableqinfoList, pPlan->user); + (*pTaskInfo)->pRoot = createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId, + &(*pTaskInfo)->tableqinfoList, pPlan->user); if (NULL == (*pTaskInfo)->pRoot) { code = (*pTaskInfo)->code; @@ -4622,8 +4624,8 @@ _complete: static void doDestroyTableList(STableListInfo* pTableqinfoList) { taosArrayDestroy(pTableqinfoList->pTableList); taosHashCleanup(pTableqinfoList->map); - if(pTableqinfoList->needSortTableByGroupId){ - for(int32_t i = 0; i < taosArrayGetSize(pTableqinfoList->pGroupList); i++){ + if (pTableqinfoList->needSortTableByGroupId) { + for (int32_t i = 0; i < taosArrayGetSize(pTableqinfoList->pGroupList); i++) { SArray* tmp = taosArrayGetP(pTableqinfoList->pGroupList, i); taosArrayDestroy(tmp); } diff --git a/source/libs/stream/src/streamData.c b/source/libs/stream/src/streamData.c index 3efe3e6c9c..e9b311db9a 100644 --- a/source/libs/stream/src/streamData.c +++ b/source/libs/stream/src/streamData.c @@ -30,7 +30,7 @@ int32_t streamDispatchReqToData(const SStreamDispatchReq* pReq, SStreamDataBlock /*int32_t len = *(int32_t*)taosArrayGet(pReq->dataLen, i);*/ SRetrieveTableRsp* pRetrieve = taosArrayGetP(pReq->data, i); SSDataBlock* pDataBlock = taosArrayGet(pArray, i); - blockCompressDecode(pDataBlock, htonl(pRetrieve->numOfCols), htonl(pRetrieve->numOfRows), pRetrieve->data); + blockDecode(pDataBlock, htonl(pRetrieve->numOfCols), htonl(pRetrieve->numOfRows), pRetrieve->data); // TODO: refactor pDataBlock->info.window.skey = be64toh(pRetrieve->skey); pDataBlock->info.window.ekey = be64toh(pRetrieve->ekey); @@ -50,7 +50,7 @@ int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock taosArraySetSize(pArray, 1); SRetrieveTableRsp* pRetrieve = pReq->pRetrieve; SSDataBlock* pDataBlock = taosArrayGet(pArray, 0); - blockCompressDecode(pDataBlock, htonl(pRetrieve->numOfCols), htonl(pRetrieve->numOfRows), pRetrieve->data); + blockDecode(pDataBlock, htonl(pRetrieve->numOfCols), htonl(pRetrieve->numOfRows), pRetrieve->data); // TODO: refactor pDataBlock->info.window.skey = be64toh(pRetrieve->skey); pDataBlock->info.window.ekey = be64toh(pRetrieve->ekey); diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index a5e9b8edd9..8034840fce 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -108,7 +108,7 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) pRetrieve->ekey = htobe64(pBlock->info.window.ekey); int32_t actualLen = 0; - blockCompressEncode(pBlock, pRetrieve->data, &actualLen, numOfCols, false); + blockEncode(pBlock, pRetrieve->data, &actualLen, numOfCols, false); SStreamRetrieveReq req = { .streamId = pTask->streamId, @@ -181,7 +181,7 @@ static int32_t streamAddBlockToDispatchMsg(const SSDataBlock* pBlock, SStreamDis pRetrieve->numOfCols = htonl(numOfCols); int32_t actualLen = 0; - blockCompressEncode(pBlock, pRetrieve->data, &actualLen, numOfCols, false); + blockEncode(pBlock, pRetrieve->data, &actualLen, numOfCols, false); actualLen += sizeof(SRetrieveTableRsp); ASSERT(actualLen <= dataStrLen); taosArrayPush(pReq->dataLen, &actualLen); From bb95a3427eb25fc35503816edfc2d93738c87261 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 28 Jun 2022 19:19:40 +0800 Subject: [PATCH 17/61] fix: fix rpc quit problem --- source/libs/transport/src/trans.c | 1 + source/libs/transport/src/transCli.c | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index a26725c41c..acf9b6b4a7 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -83,6 +83,7 @@ void rpcClose(void* arg) { SRpcInfo* pRpc = (SRpcInfo*)arg; (*taosCloseHandle[pRpc->connType])(pRpc->tcphandle); taosMemoryFree(pRpc); + tInfo("finish to close rpc"); return; } diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index b0d9b85625..5274c5c593 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -316,8 +316,9 @@ void cliHandleResp(SCliConn* conn) { if (CONN_NO_PERSIST_BY_APP(conn)) { pMsg = transQueuePop(&conn->cliMsgs); - pCtx = pMsg->ctx; - transMsg.info.ahandle = pCtx->ahandle; + + pCtx = pMsg ? pMsg->ctx : NULL; + transMsg.info.ahandle = pCtx ? pCtx->ahandle : NULL; tDebug("%s conn %p get ahandle %p, persist: 0", CONN_GET_INST_LABEL(conn), conn, transMsg.info.ahandle); } else { uint64_t ahandle = (uint64_t)pHead->ahandle; @@ -501,6 +502,7 @@ static SCliConn* getConnFromPool(void* pool, char* ip, uint32_t port) { static void allocConnRef(SCliConn* conn, bool update) { if (update) { transRemoveExHandle(conn->refId); + conn->refId = -1; } SExHandle* exh = taosMemoryCalloc(1, sizeof(SExHandle)); exh->handle = conn; @@ -600,6 +602,8 @@ static void cliDestroyConn(SCliConn* conn, bool clear) { QUEUE_REMOVE(&conn->conn); QUEUE_INIT(&conn->conn); transRemoveExHandle(conn->refId); + conn->refId = -1; + if (clear) { if (!uv_is_closing((uv_handle_t*)conn->stream)) { uv_close((uv_handle_t*)conn->stream, cliDestroy); @@ -609,8 +613,11 @@ static void cliDestroyConn(SCliConn* conn, bool clear) { } } static void cliDestroy(uv_handle_t* handle) { + if (uv_handle_get_type(handle) != UV_TCP || handle->data == NULL) return; SCliConn* conn = handle->data; + transRemoveExHandle(conn->refId); taosMemoryFree(conn->ip); + conn->stream->data = NULL; taosMemoryFree(conn->stream); transCtxCleanup(&conn->ctx); transQueueDestroy(&conn->cliMsgs); @@ -968,7 +975,7 @@ void cliSendQuit(SCliThrd* thrd) { } void cliWalkCb(uv_handle_t* handle, void* arg) { if (!uv_is_closing(handle)) { - uv_close(handle, NULL); + uv_close(handle, cliDestroy); } } @@ -1106,8 +1113,11 @@ SCliThrd* transGetWorkThrdFromHandle(int64_t handle) { SCliThrd* pThrd = NULL; SExHandle* exh = transAcquireExHandle(handle); if (exh == NULL) { + tTrace("no, no %" PRId64 "", handle); return NULL; } + tTrace("YY %" PRId64 "", handle); + pThrd = exh->pThrd; transReleaseExHandle(handle); return pThrd; From c1690655fc20fea391d58aeb966827710dac169e Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 28 Jun 2022 19:21:04 +0800 Subject: [PATCH 18/61] fix: fix rpc quit problem --- source/libs/transport/src/transCli.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 5274c5c593..7d7e459aa8 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1113,11 +1113,8 @@ SCliThrd* transGetWorkThrdFromHandle(int64_t handle) { SCliThrd* pThrd = NULL; SExHandle* exh = transAcquireExHandle(handle); if (exh == NULL) { - tTrace("no, no %" PRId64 "", handle); return NULL; } - tTrace("YY %" PRId64 "", handle); - pThrd = exh->pThrd; transReleaseExHandle(handle); return pThrd; From 1fc9eaadd08c7ff8ee62962d3daa2a8422b73be3 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 28 Jun 2022 20:30:17 +0800 Subject: [PATCH 19/61] fix: fix rpc quit problem --- source/client/src/clientEnv.c | 14 +++++++------- source/libs/transport/src/transCli.c | 8 +++++--- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 22b491994e..9bb9c6da05 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -40,7 +40,7 @@ volatile int32_t tscInitRes = 0; static int32_t registerRequest(SRequestObj *pRequest) { STscObj *pTscObj = acquireTscObj(pRequest->pTscObj->id); if (NULL == pTscObj) { - terrno = TSDB_CODE_TSC_DISCONNECTED; + terrno = TSDB_CODE_TSC_DISCONNECTED; return terrno; } @@ -91,7 +91,7 @@ void closeTransporter(SAppInstInfo *pAppInfo) { static bool clientRpcRfp(int32_t code, tmsg_t msgType) { if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED || code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY) { - if (msgType == TDMT_VND_QUERY || msgType == TDMT_VND_FETCH) { + if (/*msgType == TDMT_VND_QUERY ||*/ msgType == TDMT_VND_FETCH) { return false; } return true; @@ -133,11 +133,11 @@ void closeAllRequests(SHashObj *pRequests) { } } -void destroyAppInst(SAppInstInfo* pAppInfo) { +void destroyAppInst(SAppInstInfo *pAppInfo) { tscDebug("destroy app inst mgr %p", pAppInfo); taosThreadMutexLock(&appInfo.mutex); - + hbRemoveAppHbMrg(&pAppInfo->pAppHbMgr); taosHashRemove(appInfo.pInstMap, pAppInfo->instKey, strlen(pAppInfo->instKey)); @@ -145,10 +145,10 @@ void destroyAppInst(SAppInstInfo* pAppInfo) { taosMemoryFreeClear(pAppInfo->instKey); closeTransporter(pAppInfo); - + taosThreadMutexLock(&pAppInfo->qnodeMutex); taosArrayDestroy(pAppInfo->pQnodeList); - taosThreadMutexUnlock(&pAppInfo->qnodeMutex); + taosThreadMutexUnlock(&pAppInfo->qnodeMutex); taosMemoryFree(pAppInfo); } @@ -161,7 +161,7 @@ void destroyTscObj(void *pObj) { int64_t connNum = atomic_sub_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); closeAllRequests(pTscObj->pRequests); schedulerStopQueryHb(pTscObj->pAppInfo->pTransporter); - tscDebug("connObj 0x%" PRIx64 " p:%p destroyed, remain inst totalConn:%" PRId64, pTscObj->id, pTscObj, + tscDebug("connObj 0x%" PRIx64 " p:%p destroyed, remain inst totalConn:%" PRId64, pTscObj->id, pTscObj, pTscObj->pAppInfo->numOfConns); if (0 == connNum) { diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 7d7e459aa8..612f0cc3fd 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -464,8 +464,7 @@ void* destroyConnPool(void* pool) { SConnList* connList = taosHashIterate((SHashObj*)pool, NULL); while (connList != NULL) { while (!QUEUE_IS_EMPTY(&connList->conn)) { - queue* h = QUEUE_HEAD(&connList->conn); - // QUEUE_REMOVE(h); + queue* h = QUEUE_HEAD(&connList->conn); SCliConn* c = QUEUE_DATA(h, SCliConn, conn); cliDestroyConn(c, true); } @@ -613,7 +612,10 @@ static void cliDestroyConn(SCliConn* conn, bool clear) { } } static void cliDestroy(uv_handle_t* handle) { - if (uv_handle_get_type(handle) != UV_TCP || handle->data == NULL) return; + if (uv_handle_get_type(handle) != UV_TCP || handle->data == NULL) { + return; + } + SCliConn* conn = handle->data; transRemoveExHandle(conn->refId); taosMemoryFree(conn->ip); From 3625a93df1b216dc39f4219c29500405dd104b71 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 28 Jun 2022 20:32:06 +0800 Subject: [PATCH 20/61] fix: fix rpc quit problem --- source/client/src/clientEnv.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 9bb9c6da05..014f8f55ac 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -91,7 +91,7 @@ void closeTransporter(SAppInstInfo *pAppInfo) { static bool clientRpcRfp(int32_t code, tmsg_t msgType) { if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED || code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY) { - if (/*msgType == TDMT_VND_QUERY ||*/ msgType == TDMT_VND_FETCH) { + if (msgType == TDMT_VND_QUERY || msgType == TDMT_VND_FETCH) { return false; } return true; From 0c85bd1e989a8ef6e8f5a24c96552fc5cc4957e4 Mon Sep 17 00:00:00 2001 From: tomchon Date: Tue, 28 Jun 2022 21:41:32 +0800 Subject: [PATCH 21/61] test: modify testcase of 2 mnodes --- tests/system-test/6-cluster/5dnode2mnode.py | 134 ++++++-------------- 1 file changed, 38 insertions(+), 96 deletions(-) diff --git a/tests/system-test/6-cluster/5dnode2mnode.py b/tests/system-test/6-cluster/5dnode2mnode.py index e08e738be6..509e535f01 100644 --- a/tests/system-test/6-cluster/5dnode2mnode.py +++ b/tests/system-test/6-cluster/5dnode2mnode.py @@ -9,28 +9,22 @@ from util.sql import * from util.cases import * from util.dnodes import TDDnodes from util.dnodes import TDDnode +from util.cluster import * + import time import socket import subprocess +sys.path.append("./6-cluster") -class MyDnodes(TDDnodes): - def __init__(self ,dnodes_lists): - super(MyDnodes,self).__init__() - self.dnodes = dnodes_lists # dnode must be TDDnode instance - self.simDeployed = False - +from clusterCommonCreate import * +from clusterCommonCheck import * + class TDTestCase: - noConn = True - def init(self,conn ,logSql): tdLog.debug(f"start to excute {__file__}") - self.TDDnodes = None - self.depoly_cluster(5) - self.master_dnode = self.TDDnodes.dnodes[0] - self.host=self.master_dnode.cfgDict["fqdn"] - conn1 = taos.connect(self.master_dnode.cfgDict["fqdn"] , config=self.master_dnode.cfgDir) - tdSql.init(conn1.cursor()) - + tdSql.init(conn.cursor()) + self.host = socket.gethostname() + def getBuildPath(self): selfPath = os.path.dirname(os.path.realpath(__file__)) @@ -41,90 +35,12 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files or "taosd.exe" in files): + if ("taosd" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] break return buildPath - - - def depoly_cluster(self ,dnodes_nums): - - testCluster = False - valgrind = 0 - hostname = socket.gethostname() - dnodes = [] - start_port = 6030 - start_port_sec = 6130 - - for num in range(1, dnodes_nums+1): - dnode = TDDnode(num) - dnode.addExtraCfg("firstEp", f"{hostname}:{start_port}") - dnode.addExtraCfg("fqdn", f"{hostname}") - dnode.addExtraCfg("serverPort", f"{start_port + (num-1)*100}") - dnode.addExtraCfg("monitorFqdn", hostname) - dnode.addExtraCfg("monitorPort", 7043) - dnode.addExtraCfg("secondEp", f"{hostname}:{start_port_sec}") - - dnodes.append(dnode) - - self.TDDnodes = MyDnodes(dnodes) - self.TDDnodes.init("") - self.TDDnodes.setTestCluster(testCluster) - self.TDDnodes.setValgrind(valgrind) - self.TDDnodes.stopAll() - for dnode in self.TDDnodes.dnodes: - self.TDDnodes.deploy(dnode.index,{}) - - for dnode in self.TDDnodes.dnodes: - self.TDDnodes.starttaosd(dnode.index) - - # create cluster - for dnode in self.TDDnodes.dnodes[1:]: - # print(dnode.cfgDict) - dnode_id = dnode.cfgDict["fqdn"] + ":" +dnode.cfgDict["serverPort"] - dnode_first_host = dnode.cfgDict["firstEp"].split(":")[0] - dnode_first_port = dnode.cfgDict["firstEp"].split(":")[-1] - cmd = f"{self.getBuildPath()}/build/bin/taos -h {dnode_first_host} -P {dnode_first_port} -s \"create dnode \\\"{dnode_id}\\\"\"" - print(cmd) - os.system(cmd) - - time.sleep(2) - tdLog.info(" create cluster done! ") - - def five_dnode_one_mnode(self): - tdSql.query("show dnodes;") - tdSql.checkData(0,1,'%s:6030'%self.host) - tdSql.checkData(4,1,'%s:6430'%self.host) - tdSql.checkData(0,4,'ready') - tdSql.checkData(4,4,'ready') - tdSql.query("show mnodes;") - tdSql.checkData(0,1,'%s:6030'%self.host) - tdSql.checkData(0,2,'leader') - tdSql.checkData(0,3,'ready') - - - tdSql.error("create mnode on dnode 1;") - tdSql.error("drop mnode on dnode 1;") - - tdSql.execute("drop database if exists db") - tdSql.execute("create database if not exists db replica 1 duration 300") - tdSql.execute("use db") - tdSql.execute( - '''create table stb1 - (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) - tags (t1 int) - ''' - ) - tdSql.execute( - ''' - create table t1 - (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) - ''' - ) - for i in range(4): - tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') def five_dnode_two_mnode(self): tdSql.query("show dnodes;") @@ -187,6 +103,34 @@ class TDTestCase: tdSql.error("create mnode on dnode 2") tdSql.query("show dnodes;") print(tdSql.queryResult) + clusterComCheck.checkDnodes(5) + # restart all taosd + tdDnodes=cluster.dnodes + + # stop follower + tdLog.info("stop follower") + tdDnodes[1].stoptaosd() + if cluster.checkConnectStatus(0) : + print("cluster also work") + + # start follower + tdLog.info("start follower") + tdDnodes[1].starttaosd() + if clusterComCheck.checkMnodeStatus(2) : + print("both mnodes are ready") + + # stop leader + tdLog.info("stop leader") + tdDnodes[0].stoptaosd() + try: + cluster.checkConnectStatus(2) + tdLog.exit(" The election still succeeds when leader of both mnodes are killed ") + except Exception: + pass + tdLog.info("start leader") + tdDnodes[0].starttaosd() + if clusterComCheck.checkMnodeStatus(2) : + print("both mnodes are ready") # # fisrt drop follower of mnode # BUG @@ -229,8 +173,6 @@ class TDTestCase: def run(self): - print(self.master_dnode.cfgDict) - self.five_dnode_one_mnode() self.five_dnode_two_mnode() From 5ddcb862758d86a9b44637145e3d5b7d3754365a Mon Sep 17 00:00:00 2001 From: tomchon Date: Tue, 28 Jun 2022 21:43:24 +0800 Subject: [PATCH 22/61] test:modify testcase of insertWithMoreVgroups.py --- tests/system-test/1-insert/insertWithMoreVgroup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/1-insert/insertWithMoreVgroup.py b/tests/system-test/1-insert/insertWithMoreVgroup.py index 1ebbc1fc57..aad334f72a 100644 --- a/tests/system-test/1-insert/insertWithMoreVgroup.py +++ b/tests/system-test/1-insert/insertWithMoreVgroup.py @@ -81,7 +81,7 @@ class TDTestCase: user = "root" password = "taosdata" port =6030 - con=taos.connect(host=host, user=user, password=password, config=cfg ,port=port,config="/home/chr/TDengine/sim/psim/cfg/") + con=taos.connect(host=host, user=user, password=password, config=cfg ,port=port) cur=con.cursor() print(cur) return cur From 51fc9a7b24d8f0d716c1659c73c4ca25383de254 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 29 Jun 2022 09:52:23 +0800 Subject: [PATCH 23/61] fix: fix explain issue --- source/libs/scheduler/src/schJob.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index 89f355d78c..53f9a934f8 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -1631,7 +1631,7 @@ int32_t schExecJobImpl(SSchedulerReq *pReq, SSchJob *pJob, bool sync) { SCH_ERR_JRET(schBeginOperation(pJob, SCH_OP_EXEC, sync)); if (EXPLAIN_MODE_STATIC == pReq->pDag->explainInfo.mode) { - code = schLaunchStaticExplainJob(pReq, pJob, true); + code = schLaunchStaticExplainJob(pReq, pJob, sync); } else { code = schLaunchJob(pJob); if (sync) { From 32c00b380170ce51e5fe793642ee43579d3105e2 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 29 Jun 2022 10:04:02 +0800 Subject: [PATCH 24/61] modify case --- source/client/src/clientEnv.c | 3 -- source/client/src/clientImpl.c | 44 +++++++++---------- source/dnode/mgmt/node_mgmt/src/dmTransport.c | 3 -- .../drop_dnode_has_multi_vnode_replica1.sim | 2 +- 4 files changed, 23 insertions(+), 29 deletions(-) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 014f8f55ac..d150d68c8d 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -91,9 +91,6 @@ void closeTransporter(SAppInstInfo *pAppInfo) { static bool clientRpcRfp(int32_t code, tmsg_t msgType) { if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED || code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY) { - if (msgType == TDMT_VND_QUERY || msgType == TDMT_VND_FETCH) { - return false; - } return true; } else { return false; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 8c63046323..ce993d7d6d 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -125,7 +125,7 @@ STscObj* taos_connect_internal(const char* ip, const char* user, const char* pas p->instKey = key; key = NULL; tscDebug("new app inst mgr %p, user:%s, ip:%s, port:%d", p, user, ip, port); - + pInst = &p; } @@ -620,12 +620,12 @@ int32_t scheduleAsyncQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNod .requestId = pRequest->requestId, .requestObjRefId = pRequest->self}; SSchedulerReq req = {.pConn = &conn, - .pNodeList = pNodeList, - .pDag = pDag, - .sql = pRequest->sqlstr, - .startTs = pRequest->metric.start, - .fp = schdExecCallback, - .cbParam = &res}; + .pNodeList = pNodeList, + .pDag = pDag, + .sql = pRequest->sqlstr, + .startTs = pRequest->metric.start, + .fp = schdExecCallback, + .cbParam = &res}; int32_t code = schedulerAsyncExecJob(&req, &pRequest->body.queryJob); @@ -672,13 +672,13 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList .requestId = pRequest->requestId, .requestObjRefId = pRequest->self}; SSchedulerReq req = {.pConn = &conn, - .pNodeList = pNodeList, - .pDag = pDag, - .sql = pRequest->sqlstr, - .startTs = pRequest->metric.start, - .fp = NULL, - .cbParam = NULL, - .reqKilled = &pRequest->killed}; + .pNodeList = pNodeList, + .pDag = pDag, + .sql = pRequest->sqlstr, + .startTs = pRequest->metric.start, + .fp = NULL, + .cbParam = NULL, + .reqKilled = &pRequest->killed}; int32_t code = schedulerExecJob(&req, &pRequest->body.queryJob, &res); pRequest->body.resInfo.execRes = res.res; @@ -877,15 +877,15 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQue } break; case QUERY_EXEC_MODE_SCHEDULE: { - SArray* pMnodeList = taosArrayInit(4, sizeof(SQueryNodeLoad)); + SArray* pMnodeList = taosArrayInit(4, sizeof(SQueryNodeLoad)); SQueryPlan* pDag = NULL; code = getPlan(pRequest, pQuery, &pDag, pMnodeList); - if (TSDB_CODE_SUCCESS == code) { + if (TSDB_CODE_SUCCESS == code) { pRequest->body.subplanNum = pDag->numOfSubplans; if (!pRequest->validateOnly) { SArray* pNodeList = NULL; buildSyncExecNodeList(pRequest, &pNodeList, pMnodeList); - + code = scheduleQuery(pRequest, pDag, pNodeList); taosArrayDestroy(pNodeList); } @@ -966,7 +966,7 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultM .pUser = pRequest->pTscObj->user}; SAppInstInfo* pAppInfo = getAppInfo(pRequest); - SQueryPlan* pDag = NULL; + SQueryPlan* pDag = NULL; code = qCreateQueryPlan(&cxt, &pDag, pMnodeList); if (code) { tscError("0x%" PRIx64 " failed to create query plan, code:%s 0x%" PRIx64, pRequest->self, tstrerror(code), @@ -1336,7 +1336,7 @@ TAOS* taos_connect_auth(const char* ip, const char* user, const char* auth, cons STscObj* pObj = taos_connect_internal(ip, user, NULL, auth, db, port, CONN_TYPE__QUERY); if (pObj) { - int64_t *rid = taosMemoryCalloc(1, sizeof(int64_t)); + int64_t* rid = taosMemoryCalloc(1, sizeof(int64_t)); *rid = pObj->id; return (TAOS*)rid; } @@ -2027,7 +2027,7 @@ void taosAsyncQueryImpl(TAOS* taos, const char* sql, __taos_async_fn_t fp, void* return; } - int64_t rid = *(int64_t*)taos; + int64_t rid = *(int64_t*)taos; STscObj* pTscObj = acquireTscObj(rid); if (pTscObj == NULL || sql == NULL || NULL == fp) { terrno = TSDB_CODE_INVALID_PARA; @@ -2063,7 +2063,7 @@ void taosAsyncQueryImpl(TAOS* taos, const char* sql, __taos_async_fn_t fp, void* pRequest->body.queryFp = fp; pRequest->body.param = param; doAsyncQuery(pRequest, false); - releaseTscObj(rid); + releaseTscObj(rid); } TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) { @@ -2072,7 +2072,7 @@ TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly) { return NULL; } - int64_t rid = *(int64_t*)taos; + int64_t rid = *(int64_t*)taos; STscObj* pTscObj = acquireTscObj(rid); if (pTscObj == NULL || sql == NULL) { terrno = TSDB_CODE_TSC_DISCONNECTED; diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index cf71b7c243..1c5d74bb94 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -251,9 +251,6 @@ static inline void dmReleaseHandle(SRpcHandleInfo *pHandle, int8_t type) { static bool rpcRfp(int32_t code, tmsg_t msgType) { if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED || code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY) { - if (msgType == TDMT_VND_QUERY || msgType == TDMT_VND_FETCH) { - return false; - } return true; } else { return false; diff --git a/tests/script/tsim/dnode/drop_dnode_has_multi_vnode_replica1.sim b/tests/script/tsim/dnode/drop_dnode_has_multi_vnode_replica1.sim index 78bca45f31..20eac3836c 100644 --- a/tests/script/tsim/dnode/drop_dnode_has_multi_vnode_replica1.sim +++ b/tests/script/tsim/dnode/drop_dnode_has_multi_vnode_replica1.sim @@ -124,7 +124,7 @@ if $data(5)[3] != 3 then return -1 endi -#sql reset query cache +sql reset query cache print =============== step4: select data sql show d1.tables From c5b3183261c5c2635e8d8ae20b4fb4e5f9347403 Mon Sep 17 00:00:00 2001 From: tomchon Date: Wed, 29 Jun 2022 10:05:05 +0800 Subject: [PATCH 25/61] test:modify ways to start 5dnode2mnode.py --- tests/system-test/fulltest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index b0b8aa7108..b76d73b54c 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -116,7 +116,7 @@ python3 ./test.py -f 2-query/function_null.py python3 ./test.py -f 2-query/queryQnode.py python3 ./test.py -f 6-cluster/5dnode1mnode.py -python3 ./test.py -f 6-cluster/5dnode2mnode.py +python3 ./test.py -f 6-cluster/5dnode2mnode.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3 # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 5 -M 3 # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 5 -M 3 From 286a7c6d95bf9bbd75f7c23202291891a99057d1 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 29 Jun 2022 10:22:28 +0800 Subject: [PATCH 26/61] fix: fix rpc quit problem --- tests/system-test/2-query/json_tag.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system-test/2-query/json_tag.py b/tests/system-test/2-query/json_tag.py index 9b96b6ebd0..ff6a249d3e 100644 --- a/tests/system-test/2-query/json_tag.py +++ b/tests/system-test/2-query/json_tag.py @@ -199,7 +199,7 @@ class TDTestCase: tdSql.checkData(0, 0, "true") # test select json tag->'key', value is null tdSql.query("select jtag->'tag1' from jsons1_4") - tdSql.checkData(0, 0, "null") + tdSql.checkData(0, 0, None) # test select json tag->'key', value is double tdSql.query("select jtag->'tag1' from jsons1_5") tdSql.checkData(0, 0, "1.232000000") From 6dc7eaa107a5de21aea2531c58ffa4709cf3d2d0 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 29 Jun 2022 11:45:18 +0800 Subject: [PATCH 27/61] modify case --- source/libs/transport/inc/transComm.h | 1 - source/libs/transport/src/trans.c | 2 +- source/libs/transport/src/transSvr.c | 2 -- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 4b5f11a497..d656d87b12 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -295,7 +295,6 @@ void transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pMsg, STra void transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pMsg, STransMsg* pRsp); void transSendResponse(const STransMsg* msg); void transRegisterMsg(const STransMsg* msg); -int transGetConnInfo(void* thandle, STransHandleInfo* pInfo); void transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn); void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle); diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index acf9b6b4a7..6082b7f608 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -141,7 +141,7 @@ void rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pMsg, SRpcMsg* pRsp) { } void rpcSendResponse(const SRpcMsg* pMsg) { transSendResponse(pMsg); } -int32_t rpcGetConnInfo(void* thandle, SRpcConnInfo* pInfo) { return transGetConnInfo((void*)thandle, pInfo); } +int32_t rpcGetConnInfo(void* thandle, SRpcConnInfo* pInfo) { return 0; } void rpcRefHandle(void* handle, int8_t type) { assert(type == TAOS_CONN_SERVER || type == TAOS_CONN_CLIENT); diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index c53cab13d5..df0c3bed45 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -1114,6 +1114,4 @@ _return2: rpcFreeCont(msg->pCont); } -int transGetConnInfo(void* thandle, STransHandleInfo* pConnInfo) { return -1; } - #endif From 41f24314d0b789ef28ff30208e2d582ffeac5881 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 29 Jun 2022 15:44:30 +0800 Subject: [PATCH 28/61] refactor(sync): add SYNC_TERM_INVALID --- source/libs/sync/inc/syncInt.h | 1 - source/libs/sync/src/syncAppendEntries.c | 8 +- source/libs/sync/src/syncMain.c | 162 +++-------------------- source/libs/sync/src/syncRaftLog.c | 43 ++++-- source/libs/sync/src/syncRequestVote.c | 4 + 5 files changed, 57 insertions(+), 161 deletions(-) diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index d351dc50f4..d58ae83dd3 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -221,7 +221,6 @@ void syncNodeVoteForSelf(SSyncNode* pSyncNode); // snapshot -------------- bool syncNodeHasSnapshot(SSyncNode* pSyncNode); -bool syncNodeIsIndexInSnapshot(SSyncNode* pSyncNode, SyncIndex index); SyncIndex syncNodeGetLastIndex(SSyncNode* pSyncNode); SyncTerm syncNodeGetLastTerm(SSyncNode* pSyncNode); diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index ba3dd66550..a10b81d165 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -860,7 +860,9 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs } code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry); - ASSERT(code == 0); + if (code != 0) { + return -1; + } // pre commit code = syncNodePreCommit(ths, pAppendEntry); @@ -971,7 +973,9 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs ASSERT(pAppendEntry != NULL); code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pAppendEntry); - ASSERT(code == 0); + if (code != 0) { + return -1; + } // pre commit code = syncNodePreCommit(ths, pAppendEntry); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index f6768bf494..cc2b5a7706 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1935,19 +1935,6 @@ bool syncNodeHasSnapshot(SSyncNode* pSyncNode) { return ret; } -#if 0 -bool syncNodeIsIndexInSnapshot(SSyncNode* pSyncNode, SyncIndex index) { - ASSERT(syncNodeHasSnapshot(pSyncNode)); - ASSERT(pSyncNode->pFsm->FpGetSnapshotInfo != NULL); - ASSERT(index >= SYNC_INDEX_BEGIN); - - SSnapshot snapshot; - pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot); - bool b = (index <= snapshot.lastApplyIndex); - return b; -} -#endif - SyncIndex syncNodeGetLastIndex(SSyncNode* pSyncNode) { SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0}; if (pSyncNode->pFsm->FpGetSnapshotInfo != NULL) { @@ -2004,21 +1991,6 @@ SyncIndex syncNodeGetPreIndex(SSyncNode* pSyncNode, SyncIndex index) { return preIndex; } -/* -SyncIndex syncNodeGetPreIndex(SSyncNode* pSyncNode, SyncIndex index) { - ASSERT(index >= SYNC_INDEX_BEGIN); - - SyncIndex syncStartIndex = syncNodeSyncStartIndex(pSyncNode); - if (index > syncStartIndex) { - syncNodeLog3("syncNodeGetPreIndex", pSyncNode); - ASSERT(0); - } - - SyncIndex preIndex = index - 1; - return preIndex; -} -*/ - SyncTerm syncNodeGetPreTerm(SSyncNode* pSyncNode, SyncIndex index) { if (index < SYNC_INDEX_BEGIN) { return SYNC_TERM_INVALID; @@ -2056,112 +2028,6 @@ SyncTerm syncNodeGetPreTerm(SSyncNode* pSyncNode, SyncIndex index) { return SYNC_TERM_INVALID; } -#if 0 -SyncTerm syncNodeGetPreTerm(SSyncNode* pSyncNode, SyncIndex index) { - ASSERT(index >= SYNC_INDEX_BEGIN); - - SyncIndex syncStartIndex = syncNodeSyncStartIndex(pSyncNode); - if (index > syncStartIndex) { - syncNodeLog3("syncNodeGetPreTerm", pSyncNode); - ASSERT(0); - } - - if (index == SYNC_INDEX_BEGIN) { - return 0; - } - - SyncTerm preTerm = 0; - SyncIndex preIndex = index - 1; - SSyncRaftEntry* pPreEntry = NULL; - int32_t code = pSyncNode->pLogStore->syncLogGetEntry(pSyncNode->pLogStore, preIndex, &pPreEntry); - if (code == 0) { - ASSERT(pPreEntry != NULL); - preTerm = pPreEntry->term; - taosMemoryFree(pPreEntry); - return preTerm; - } else { - if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST) { - SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0, .lastConfigIndex = -1}; - if (pSyncNode->pFsm->FpGetSnapshotInfo != NULL) { - pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot); - if (snapshot.lastApplyIndex == preIndex) { - return snapshot.lastApplyTerm; - } - } - } - } - - ASSERT(0); - return -1; -} -#endif - -#if 0 -SyncTerm syncNodeGetPreTerm(SSyncNode* pSyncNode, SyncIndex index) { - ASSERT(index >= SYNC_INDEX_BEGIN); - - SyncIndex syncStartIndex = syncNodeSyncStartIndex(pSyncNode); - if (index > syncStartIndex) { - syncNodeLog3("syncNodeGetPreTerm", pSyncNode); - ASSERT(0); - } - - if (index == SYNC_INDEX_BEGIN) { - return 0; - } - - SyncTerm preTerm = 0; - if (syncNodeHasSnapshot(pSyncNode)) { - // has snapshot - SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0, .lastConfigIndex = -1}; - if (pSyncNode->pFsm->FpGetSnapshotInfo != NULL) { - pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot); - } - - if (index > snapshot.lastApplyIndex + 1) { - // should be log preTerm - SSyncRaftEntry* pPreEntry = NULL; - int32_t code = pSyncNode->pLogStore->syncLogGetEntry(pSyncNode->pLogStore, index - 1, &pPreEntry); - ASSERT(code == 0); - ASSERT(pPreEntry != NULL); - - preTerm = pPreEntry->term; - taosMemoryFree(pPreEntry); - - } else if (index == snapshot.lastApplyIndex + 1) { - preTerm = snapshot.lastApplyTerm; - - } else { - // maybe snapshot change - sError("sync get pre term, bad scene. index:%ld", index); - logStoreLog2("sync get pre term, bad scene", pSyncNode->pLogStore); - - SSyncRaftEntry* pPreEntry = NULL; - int32_t code = pSyncNode->pLogStore->syncLogGetEntry(pSyncNode->pLogStore, index - 1, &pPreEntry); - ASSERT(code == 0); - ASSERT(pPreEntry != NULL); - - preTerm = pPreEntry->term; - taosMemoryFree(pPreEntry); - } - - } else { - // no snapshot - ASSERT(index > SYNC_INDEX_BEGIN); - - SSyncRaftEntry* pPreEntry = NULL; - int32_t code = pSyncNode->pLogStore->syncLogGetEntry(pSyncNode->pLogStore, index - 1, &pPreEntry); - ASSERT(code == 0); - ASSERT(pPreEntry != NULL); - - preTerm = pPreEntry->term; - taosMemoryFree(pPreEntry); - } - - return preTerm; -} -#endif - // get pre index and term of "index" int32_t syncNodeGetPreIndexTerm(SSyncNode* pSyncNode, SyncIndex index, SyncIndex* pPreIndex, SyncTerm* pPreTerm) { *pPreIndex = syncNodeGetPreIndex(pSyncNode, index); @@ -2351,8 +2217,8 @@ static int32_t syncNodeAppendNoop(SSyncNode* ths) { ASSERT(pEntry != NULL); if (ths->state == TAOS_SYNC_STATE_LEADER) { - // ths->pLogStore->appendEntry(ths->pLogStore, pEntry); - ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry); + int32_t code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry); + ASSERT(code == 0); syncNodeReplicate(ths); } @@ -2406,6 +2272,7 @@ int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg) { // int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg, SyncIndex* pRetIndex) { int32_t ret = 0; + int32_t code = 0; syncClientRequestLog2("==syncNodeOnClientRequestCb==", pMsg); SyncIndex index = ths->pLogStore->syncLogWriteIndex(ths->pLogStore); @@ -2414,18 +2281,24 @@ int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg, SyncI ASSERT(pEntry != NULL); if (ths->state == TAOS_SYNC_STATE_LEADER) { - // ths->pLogStore->appendEntry(ths->pLogStore, pEntry); - ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry); + // append entry + code = ths->pLogStore->syncLogAppendEntry(ths->pLogStore, pEntry); + if (code != 0) { + // del resp mgr, call FpCommitCb + ASSERT(0); + return -1; + } - // start replicate right now! - syncNodeReplicate(ths); + // if mulit replica, start replicate right now + if (ths->replicaNum > 1) { + syncNodeReplicate(ths); + } // pre commit SRpcMsg rpcMsg; syncEntry2OriginalRpc(pEntry, &rpcMsg); if (ths->pFsm != NULL) { - // if (ths->pFsm->FpPreCommitCb != NULL && pEntry->originalRpcType != TDMT_SYNC_NOOP) { if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pEntry->originalRpcType)) { SFsmCbMeta cbMeta = {0}; cbMeta.index = pEntry->index; @@ -2439,8 +2312,10 @@ int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg, SyncI } rpcFreeCont(rpcMsg.pCont); - // only myself, maybe commit - syncMaybeAdvanceCommitIndex(ths); + // if only myself, maybe commit right now + if (ths->replicaNum == 1) { + syncMaybeAdvanceCommitIndex(ths); + } } else { // pre commit @@ -2448,7 +2323,6 @@ int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg, SyncI syncEntry2OriginalRpc(pEntry, &rpcMsg); if (ths->pFsm != NULL) { - // if (ths->pFsm->FpPreCommitCb != NULL && pEntry->originalRpcType != TDMT_SYNC_NOOP) { if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pEntry->originalRpcType)) { SFsmCbMeta cbMeta = {0}; cbMeta.index = pEntry->index; diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index a574c8cc1e..a026892629 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -168,6 +168,9 @@ static SyncIndex raftLogWriteIndex(struct SSyncLogStore* pLogStore) { return lastVer + 1; } +// if success, return last term +// if not log, return 0 +// if error, return SYNC_TERM_INVALID static SyncTerm raftLogLastTerm(struct SSyncLogStore* pLogStore) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; @@ -176,15 +179,17 @@ static SyncTerm raftLogLastTerm(struct SSyncLogStore* pLogStore) { } else { SSyncRaftEntry* pLastEntry; int32_t code = raftLogGetLastEntry(pLogStore, &pLastEntry); - ASSERT(code == 0); - ASSERT(pLastEntry != NULL); - - SyncTerm lastTerm = pLastEntry->term; - taosMemoryFree(pLastEntry); - return lastTerm; + if (code == 0 && pLastEntry != NULL) { + SyncTerm lastTerm = pLastEntry->term; + taosMemoryFree(pLastEntry); + return lastTerm; + } else { + return SYNC_TERM_INVALID; + } } - return 0; + // can not be here! + return SYNC_TERM_INVALID; } static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { @@ -218,16 +223,21 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr ASSERT(0); } - walFsync(pWal, true); + // walFsync(pWal, true); - char eventLog[128]; - snprintf(eventLog, sizeof(eventLog), "write index:%ld, type:%s,%d, type2:%s,%d", pEntry->index, - TMSG_INFO(pEntry->msgType), pEntry->msgType, TMSG_INFO(pEntry->originalRpcType), pEntry->originalRpcType); - syncNodeEventLog(pData->pSyncNode, eventLog); + do { + char eventLog[128]; + snprintf(eventLog, sizeof(eventLog), "write index:%ld, type:%s,%d, type2:%s,%d", pEntry->index, + TMSG_INFO(pEntry->msgType), pEntry->msgType, TMSG_INFO(pEntry->originalRpcType), pEntry->originalRpcType); + syncNodeEventLog(pData->pSyncNode, eventLog); + } while (0); return code; } +// entry found, return 0 +// entry not found, return -1, terrno = TSDB_CODE_WAL_LOG_NOT_EXIST +// other error, return -1 static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncRaftEntry** ppEntry) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; @@ -238,6 +248,7 @@ static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, // SWalReadHandle* pWalHandle = walOpenReadHandle(pWal); SWalReadHandle* pWalHandle = pData->pWalHandle; if (pWalHandle == NULL) { + terrno = TSDB_CODE_SYN_INTERNAL_ERROR; return -1; } @@ -309,6 +320,9 @@ static int32_t raftLogTruncate(struct SSyncLogStore* pLogStore, SyncIndex fromIn return code; } +// entry found, return 0 +// entry not found, return -1, terrno = TSDB_CODE_WAL_LOG_NOT_EXIST +// other error, return -1 static int32_t raftLogGetLastEntry(SSyncLogStore* pLogStore, SSyncRaftEntry** ppLastEntry) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; @@ -320,7 +334,8 @@ static int32_t raftLogGetLastEntry(SSyncLogStore* pLogStore, SSyncRaftEntry** pp return -1; } else { SyncIndex lastIndex = raftLogLastIndex(pLogStore); - int32_t code = raftLogGetEntry(pLogStore, lastIndex, ppLastEntry); + ASSERT(lastIndex >= SYNC_INDEX_BEGIN); + int32_t code = raftLogGetEntry(pLogStore, lastIndex, ppLastEntry); return code; } @@ -356,7 +371,7 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { ASSERT(0); } - walFsync(pWal, true); + // walFsync(pWal, true); char eventLog[128]; snprintf(eventLog, sizeof(eventLog), "old write index:%ld, type:%s,%d, type2:%s,%d", pEntry->index, diff --git a/source/libs/sync/src/syncRequestVote.c b/source/libs/sync/src/syncRequestVote.c index 95dec6cb83..d272e0175f 100644 --- a/source/libs/sync/src/syncRequestVote.c +++ b/source/libs/sync/src/syncRequestVote.c @@ -156,6 +156,10 @@ static bool syncNodeOnRequestVoteLogOK(SSyncNode* pSyncNode, SyncRequestVote* pM SyncTerm myLastTerm = syncNodeGetLastTerm(pSyncNode); SyncIndex myLastIndex = syncNodeGetLastIndex(pSyncNode); + if (myLastTerm == SYNC_TERM_INVALID) { + return false; + } + if (pMsg->lastLogTerm > myLastTerm) { return true; } From 16c6427e7b3b9c3fb94273bfe9331b93cb914b3e Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 29 Jun 2022 16:29:12 +0800 Subject: [PATCH 29/61] fix: fix scheduler callback issue --- source/libs/scheduler/src/scheduler.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 15a687531c..8b125811cf 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -113,10 +113,6 @@ _return: schReleaseJob(pJob->refId); } - if (code != TSDB_CODE_SUCCESS) { - pReq->fp(NULL, pReq->cbParam, code); - } - return code; } From 3440822ebf6b6196a060949f3f4cf9ea53e238ce Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 29 Jun 2022 16:40:36 +0800 Subject: [PATCH 30/61] refactor(sync): add some comments --- source/libs/sync/src/syncMain.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index cc2b5a7706..d32153e5ed 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1923,6 +1923,8 @@ void syncNodeVoteForSelf(SSyncNode* pSyncNode) { } // snapshot -------------- + +// return if has a snapshot bool syncNodeHasSnapshot(SSyncNode* pSyncNode) { bool ret = false; SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0, .lastConfigIndex = -1}; @@ -1935,8 +1937,10 @@ bool syncNodeHasSnapshot(SSyncNode* pSyncNode) { return ret; } +// return max(logLastIndex, snapshotLastIndex) +// if no snapshot and log, return -1 SyncIndex syncNodeGetLastIndex(SSyncNode* pSyncNode) { - SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0}; + SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0, .lastConfigIndex = -1}; if (pSyncNode->pFsm->FpGetSnapshotInfo != NULL) { pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot); } @@ -1946,6 +1950,8 @@ SyncIndex syncNodeGetLastIndex(SSyncNode* pSyncNode) { return lastIndex; } +// return the last term of snapshot and log +// if error, return SYNC_TERM_INVALID (by syncLogLastTerm) SyncTerm syncNodeGetLastTerm(SSyncNode* pSyncNode) { SyncTerm lastTerm = 0; if (syncNodeHasSnapshot(pSyncNode)) { @@ -1977,11 +1983,14 @@ int32_t syncNodeGetLastIndexTerm(SSyncNode* pSyncNode, SyncIndex* pLastIndex, Sy return 0; } +// return append-entries first try index SyncIndex syncNodeSyncStartIndex(SSyncNode* pSyncNode) { SyncIndex syncStartIndex = syncNodeGetLastIndex(pSyncNode) + 1; return syncStartIndex; } +// if index > 0, return index - 1 +// else, return -1 SyncIndex syncNodeGetPreIndex(SSyncNode* pSyncNode, SyncIndex index) { SyncIndex preIndex = index - 1; if (preIndex < SYNC_INDEX_INVALID) { @@ -1991,6 +2000,10 @@ SyncIndex syncNodeGetPreIndex(SSyncNode* pSyncNode, SyncIndex index) { return preIndex; } +// if index < 0, return SYNC_TERM_INVALID +// if index == 0, return 0 +// if index > 0, return preTerm +// if error, return SYNC_TERM_INVALID SyncTerm syncNodeGetPreTerm(SSyncNode* pSyncNode, SyncIndex index) { if (index < SYNC_INDEX_BEGIN) { return SYNC_TERM_INVALID; From 4c82ce1810ef2424d04c468d60d59af93d9e0193 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 29 Jun 2022 17:00:48 +0800 Subject: [PATCH 31/61] fix: fix request freed issue --- include/libs/scheduler/scheduler.h | 14 ++-- source/client/src/clientImpl.c | 78 +++++-------------- source/libs/scheduler/inc/schedulerInt.h | 63 +++++++-------- source/libs/scheduler/src/schJob.c | 13 ++-- source/libs/scheduler/src/scheduler.c | 2 +- source/libs/scheduler/test/schedulerTests.cpp | 20 ++--- 6 files changed, 78 insertions(+), 112 deletions(-) diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index ecb21335b9..be3d16ab0d 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -69,18 +69,20 @@ typedef struct SSchdFetchParam { int32_t* code; } SSchdFetchParam; -typedef void (*schedulerExecCallback)(SQueryResult* pResult, void* param, int32_t code); -typedef void (*schedulerFetchCallback)(void* pResult, void* param, int32_t code); +typedef void (*schedulerExecFp)(SQueryResult* pResult, void* param, int32_t code); +typedef void (*schedulerFetchFp)(void* pResult, void* param, int32_t code); +typedef bool (*schedulerChkKillFp)(void* param); typedef struct SSchedulerReq { - bool *reqKilled; SRequestConnInfo *pConn; SArray *pNodeList; SQueryPlan *pDag; const char *sql; int64_t startTs; - schedulerExecCallback fp; - void* cbParam; + schedulerExecFp execFp; + void* execParam; + schedulerChkKillFp chkKillFp; + void* chkKillParam; } SSchedulerReq; @@ -110,7 +112,7 @@ int32_t schedulerExecJob(SSchedulerReq *pReq, int64_t *pJob, SQueryResult *pRes) */ int32_t schedulerFetchRows(int64_t job, void **data); -void schedulerAsyncFetchRows(int64_t job, schedulerFetchCallback fp, void* param); +void schedulerAsyncFetchRows(int64_t job, schedulerFetchFp fp, void* param); int32_t schedulerGetTasksStatus(int64_t job, SArray *pSub); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index ce993d7d6d..09fb73cfba 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -55,6 +55,18 @@ static char* getClusterKey(const char* user, const char* auth, const char* ip, i return strdup(key); } +bool chkRequestKilled(void* param) { + bool killed = false; + SRequestObj* pRequest = acquireRequest((int64_t)param); + if (NULL == pRequest || pRequest->killed) { + killed = true; + } + + releaseRequest((int64_t)param); + + return killed; +} + static STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __taos_async_fn_t fp, void* param, SAppInstInfo* pAppInfo, int connType); @@ -612,58 +624,6 @@ _return: return code; } -int32_t scheduleAsyncQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList) { - tsem_init(&schdRspSem, 0, 0); - - SQueryResult res = {.code = 0, .numOfRows = 0}; - SRequestConnInfo conn = {.pTrans = pRequest->pTscObj->pAppInfo->pTransporter, - .requestId = pRequest->requestId, - .requestObjRefId = pRequest->self}; - SSchedulerReq req = {.pConn = &conn, - .pNodeList = pNodeList, - .pDag = pDag, - .sql = pRequest->sqlstr, - .startTs = pRequest->metric.start, - .fp = schdExecCallback, - .cbParam = &res}; - - int32_t code = schedulerAsyncExecJob(&req, &pRequest->body.queryJob); - - pRequest->body.resInfo.execRes = res.res; - - while (true) { - if (code != TSDB_CODE_SUCCESS) { - if (pRequest->body.queryJob != 0) { - schedulerFreeJob(pRequest->body.queryJob, 0); - } - - pRequest->code = code; - terrno = code; - return pRequest->code; - } else { - tsem_wait(&schdRspSem); - - if (res.code) { - code = res.code; - } else { - break; - } - } - } - - if (TDMT_VND_SUBMIT == pRequest->type || TDMT_VND_CREATE_TABLE == pRequest->type) { - pRequest->body.resInfo.numOfRows = res.numOfRows; - - if (pRequest->body.queryJob != 0) { - schedulerFreeJob(pRequest->body.queryJob, 0); - } - } - - pRequest->code = res.code; - terrno = res.code; - return pRequest->code; -} - int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList) { void* pTransporter = pRequest->pTscObj->pAppInfo->pTransporter; @@ -676,9 +636,10 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList .pDag = pDag, .sql = pRequest->sqlstr, .startTs = pRequest->metric.start, - .fp = NULL, - .cbParam = NULL, - .reqKilled = &pRequest->killed}; + .execFp = NULL, + .execParam = NULL, + .chkKillFp = chkRequestKilled, + .chkKillParam = (void*)pRequest->self}; int32_t code = schedulerExecJob(&req, &pRequest->body.queryJob, &res); pRequest->body.resInfo.execRes = res.res; @@ -986,9 +947,10 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultM .pDag = pDag, .sql = pRequest->sqlstr, .startTs = pRequest->metric.start, - .fp = schedulerExecCb, - .cbParam = pRequest, - .reqKilled = &pRequest->killed}; + .execFp = schedulerExecCb, + .execParam = pRequest, + .chkKillFp = chkRequestKilled, + .chkKillParam = (void*)pRequest->self}; code = schedulerAsyncExecJob(&req, &pRequest->body.queryJob); taosArrayDestroy(pNodeList); } else { diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index a119795787..5998ab6965 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -99,8 +99,8 @@ typedef struct SSchStat { typedef struct SSchResInfo { SQueryResult* queryRes; void** fetchRes; - schedulerExecCallback execFp; - schedulerFetchCallback fetchFp; + schedulerExecFp execFp; + schedulerFetchFp fetchFp; void* userParam; } SSchResInfo; @@ -204,37 +204,38 @@ typedef struct { } SSchOpStatus; typedef struct SSchJob { - int64_t refId; - uint64_t queryId; - SSchJobAttr attr; - int32_t levelNum; - int32_t taskNum; - SRequestConnInfo conn; - SArray *nodeList; // qnode/vnode list, SArray - SArray *levels; // starting from 0. SArray - SQueryPlan *pDag; + int64_t refId; + uint64_t queryId; + SSchJobAttr attr; + int32_t levelNum; + int32_t taskNum; + SRequestConnInfo conn; + SArray *nodeList; // qnode/vnode list, SArray + SArray *levels; // starting from 0. SArray + SQueryPlan *pDag; - SArray *dataSrcTasks; // SArray - int32_t levelIdx; - SEpSet dataSrcEps; - SHashObj *taskList; - SHashObj *execTasks; // executing and executed tasks, key:taskid, value:SQueryTask* - SHashObj *flowCtrl; // key is ep, element is SSchFlowControl + SArray *dataSrcTasks; // SArray + int32_t levelIdx; + SEpSet dataSrcEps; + SHashObj *taskList; + SHashObj *execTasks; // executing and executed tasks, key:taskid, value:SQueryTask* + SHashObj *flowCtrl; // key is ep, element is SSchFlowControl - SExplainCtx *explainCtx; - int8_t status; - SQueryNodeAddr resNode; - tsem_t rspSem; - SSchOpStatus opStatus; - bool *reqKilled; - SSchTask *fetchTask; - int32_t errCode; - SRWLatch resLock; - SQueryExecRes execRes; - void *resData; //TODO free it or not - int32_t resNumOfRows; - SSchResInfo userRes; - const char *sql; + SExplainCtx *explainCtx; + int8_t status; + SQueryNodeAddr resNode; + tsem_t rspSem; + SSchOpStatus opStatus; + schedulerChkKillFp chkKillFp; + void* chkKillParam; + SSchTask *fetchTask; + int32_t errCode; + SRWLatch resLock; + SQueryExecRes execRes; + void *resData; //TODO free it or not + int32_t resNumOfRows; + SSchResInfo userRes; + const char *sql; SQueryProfileSummary summary; } SSchJob; diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index 53f9a934f8..bd3a944c3f 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -55,9 +55,10 @@ int32_t schInitJob(SSchedulerReq *pReq, SSchJob **pSchJob) { pJob->conn = *pReq->pConn; pJob->sql = pReq->sql; pJob->pDag = pReq->pDag; - pJob->reqKilled = pReq->reqKilled; - pJob->userRes.execFp = pReq->fp; - pJob->userRes.userParam = pReq->cbParam; + pJob->chkKillFp = pReq->chkKillFp; + pJob->chkKillParam = pReq->chkKillParam; + pJob->userRes.execFp = pReq->execFp; + pJob->userRes.userParam = pReq->execParam; if (pReq->pNodeList == NULL || taosArrayGetSize(pReq->pNodeList) <= 0) { qDebug("QID:0x%" PRIx64 " input exec nodeList is empty", pReq->pDag->queryId); @@ -182,7 +183,7 @@ FORCE_INLINE bool schJobNeedToStop(SSchJob *pJob, int8_t *pStatus) { *pStatus = status; } - if (*pJob->reqKilled) { + if ((*pJob->chkKillFp)(pJob->chkKillParam)) { schUpdateJobStatus(pJob, JOB_TASK_STATUS_DROPPING); schUpdateJobErrCode(pJob, TSDB_CODE_TSC_QUERY_KILLED); @@ -1584,7 +1585,7 @@ _return: schEndOperation(pJob); if (!sync) { - pReq->fp(NULL, pReq->cbParam, code); + pReq->execFp(NULL, pReq->execParam, code); } schFreeJobImpl(pJob); @@ -1651,7 +1652,7 @@ int32_t schExecJobImpl(SSchedulerReq *pReq, SSchJob *pJob, bool sync) { _return: if (!sync) { - pReq->fp(NULL, pReq->cbParam, code); + pReq->execFp(NULL, pReq->execParam, code); } SCH_RET(code); diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 8b125811cf..74ddc89b40 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -140,7 +140,7 @@ int32_t schedulerFetchRows(int64_t job, void **pData) { SCH_RET(code); } -void schedulerAsyncFetchRows(int64_t job, schedulerFetchCallback fp, void* param) { +void schedulerAsyncFetchRows(int64_t job, schedulerFetchFp fp, void* param) { qDebug("scheduler async fetch rows start"); int32_t code = 0; diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index e5cc3cd481..b372ee3ead 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -511,8 +511,8 @@ void* schtRunJobThread(void *aa) { req.pNodeList = qnodeList; req.pDag = &dag; req.sql = "select * from tb"; - req.fp = schtQueryCb; - req.cbParam = &queryDone; + req.execFp = schtQueryCb; + req.execParam = &queryDone; code = schedulerAsyncExecJob(&req, &queryJobRefId); assert(code == 0); @@ -663,8 +663,8 @@ TEST(queryTest, normalCase) { req.pNodeList = qnodeList; req.pDag = &dag; req.sql = "select * from tb"; - req.fp = schtQueryCb; - req.cbParam = &queryDone; + req.execFp = schtQueryCb; + req.execParam = &queryDone; code = schedulerAsyncExecJob(&req, &job); ASSERT_EQ(code, 0); @@ -767,8 +767,8 @@ TEST(queryTest, readyFirstCase) { req.pNodeList = qnodeList; req.pDag = &dag; req.sql = "select * from tb"; - req.fp = schtQueryCb; - req.cbParam = &queryDone; + req.execFp = schtQueryCb; + req.execParam = &queryDone; code = schedulerAsyncExecJob(&req, &job); ASSERT_EQ(code, 0); @@ -874,8 +874,8 @@ TEST(queryTest, flowCtrlCase) { req.pNodeList = qnodeList; req.pDag = &dag; req.sql = "select * from tb"; - req.fp = schtQueryCb; - req.cbParam = &queryDone; + req.execFp = schtQueryCb; + req.execParam = &queryDone; code = schedulerAsyncExecJob(&req, &job); ASSERT_EQ(code, 0); @@ -987,8 +987,8 @@ TEST(insertTest, normalCase) { req.pNodeList = qnodeList; req.pDag = &dag; req.sql = "insert into tb values(now,1)"; - req.fp = schtQueryCb; - req.cbParam = NULL; + req.execFp = schtQueryCb; + req.execParam = NULL; code = schedulerExecJob(&req, &insertJobRefId, &res); ASSERT_EQ(code, 0); From 290d5b3f1445c6315eda83811e5aac7ab787742c Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 29 Jun 2022 17:50:33 +0800 Subject: [PATCH 32/61] refactor(query): function adoption to us/ns db precision TD-16436 --- source/libs/function/src/builtins.c | 17 +++++++++++++++++ source/libs/scalar/src/sclfunc.c | 4 ++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index e53776bc7c..640c160203 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1379,6 +1379,20 @@ static bool validateTimezoneFormat(const SValueNode* pVal) { return true; } +void static addDbPrecisonParam(SNodeList* pList, uint8_t precision) { + SValueNode* pVal = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); + pVal->literal = NULL; + pVal->isDuration = false; + pVal->translate = true; + pVal->node.resType.type = TSDB_DATA_TYPE_TINYINT; + pVal->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_TINYINT].bytes; + pVal->node.resType.precision = precision; + pVal->datum.i = (int64_t)precision; + pVal->typeData = (int64_t)precision; + + nodesListAppend(pList, (SNode*)pVal); +} + void static addTimezoneParam(SNodeList* pList) { char buf[6] = {0}; time_t t = taosTime(NULL); @@ -1462,6 +1476,9 @@ static int32_t translateTimeTruncate(SFunctionNode* pFunc, char* pErrBuf, int32_ return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } + uint8_t dbPrec = pFunc->node.resType.precision; + addDbPrecisonParam(pFunc->pParameterList, dbPrec); + pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes, .type = TSDB_DATA_TYPE_TIMESTAMP}; return TSDB_CODE_SUCCESS; diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 80bebafef2..57962abc01 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -1192,10 +1192,10 @@ int32_t toJsonFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOu int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { int32_t type = GET_PARAM_TYPE(&pInput[0]); - int32_t timePrec = GET_PARAM_PRECISON(&pInput[0]); - int64_t timeUnit, timeVal = 0; + int64_t timeUnit, timePrec, timeVal = 0; GET_TYPED_DATA(timeUnit, int64_t, GET_PARAM_TYPE(&pInput[1]), pInput[1].columnData->pData); + GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[2]), pInput[2].columnData->pData); int64_t factor = (timePrec == TSDB_TIME_PRECISION_MILLI) ? 1000 : (timePrec == TSDB_TIME_PRECISION_MICRO ? 1000000 : 1000000000); From c3b2b98454a52806f2fabaa0e48921326f3cedc0 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 29 Jun 2022 19:18:15 +0800 Subject: [PATCH 33/61] refactor(sync): delete some asserts --- include/libs/sync/syncTools.h | 17 +++++++++++++++++ source/libs/sync/src/syncElection.c | 6 +++++- source/libs/sync/src/syncRaftCfg.c | 19 ++++++++++++++----- 3 files changed, 36 insertions(+), 6 deletions(-) diff --git a/include/libs/sync/syncTools.h b/include/libs/sync/syncTools.h index 5d892352d6..46f279ed85 100644 --- a/include/libs/sync/syncTools.h +++ b/include/libs/sync/syncTools.h @@ -324,6 +324,23 @@ void syncAppendEntriesPrint2(char* s, const SyncAppendEntries* pMsg); void syncAppendEntriesLog(const SyncAppendEntries* pMsg); void syncAppendEntriesLog2(char* s, const SyncAppendEntries* pMsg); +// --------------------------------------------- +typedef struct SyncAppendEntriesBatch { + uint32_t bytes; + int32_t vgId; + uint32_t msgType; + SRaftId srcId; + SRaftId destId; + // private data + SyncTerm term; + SyncIndex prevLogIndex; + SyncTerm prevLogTerm; + SyncIndex commitIndex; + SyncTerm privateTerm; + uint32_t dataLen; + char data[]; +} SyncAppendEntriesBatch; + // --------------------------------------------- typedef struct SyncAppendEntriesReply { uint32_t bytes; diff --git a/source/libs/sync/src/syncElection.c b/source/libs/sync/src/syncElection.c index 738b17b9cb..816430b5b5 100644 --- a/source/libs/sync/src/syncElection.c +++ b/source/libs/sync/src/syncElection.c @@ -75,7 +75,11 @@ int32_t syncNodeElect(SSyncNode* pSyncNode) { if (pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER) { syncNodeFollower2Candidate(pSyncNode); } - ASSERT(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE); + + if (pSyncNode->state != TAOS_SYNC_STATE_CANDIDATE) { + syncNodeErrorLog(pSyncNode, "not candidate, can not elect"); + return -1; + } // start election raftStoreNextTerm(pSyncNode->pRaftStore); diff --git a/source/libs/sync/src/syncRaftCfg.c b/source/libs/sync/src/syncRaftCfg.c index 9d16bed6c1..43890c196c 100644 --- a/source/libs/sync/src/syncRaftCfg.c +++ b/source/libs/sync/src/syncRaftCfg.c @@ -101,7 +101,7 @@ cJSON *syncCfg2Json(SSyncCfg *pSyncCfg) { char *syncCfg2Str(SSyncCfg *pSyncCfg) { cJSON *pJson = syncCfg2Json(pSyncCfg); - char * serialized = cJSON_Print(pJson); + char *serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } @@ -109,7 +109,7 @@ char *syncCfg2Str(SSyncCfg *pSyncCfg) { char *syncCfg2SimpleStr(SSyncCfg *pSyncCfg) { if (pSyncCfg != NULL) { int32_t len = 512; - char * s = taosMemoryMalloc(len); + char *s = taosMemoryMalloc(len); memset(s, 0, len); snprintf(s, len, "{replica-num:%d, my-index:%d, ", pSyncCfg->replicaNum, pSyncCfg->myIndex); @@ -205,7 +205,7 @@ cJSON *raftCfg2Json(SRaftCfg *pRaftCfg) { char *raftCfg2Str(SRaftCfg *pRaftCfg) { cJSON *pJson = raftCfg2Json(pRaftCfg); - char * serialized = cJSON_Print(pJson); + char *serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } @@ -214,7 +214,16 @@ int32_t raftCfgCreateFile(SSyncCfg *pCfg, SRaftCfgMeta meta, const char *path) { ASSERT(pCfg != NULL); TdFilePtr pFile = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE); - ASSERT(pFile != NULL); + if (pFile == NULL) { + int32_t err = terrno; + const char *errStr = tstrerror(err); + int32_t sysErr = errno; + const char *sysErrStr = strerror(errno); + sError("create raft cfg file error, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", err, err, errStr, sysErr, sysErrStr); + ASSERT(0); + + return -1; + } SRaftCfg raftCfg; raftCfg.cfg = *pCfg; @@ -271,7 +280,7 @@ int32_t raftCfgFromJson(const cJSON *pRoot, SRaftCfg *pRaftCfg) { (pRaftCfg->configIndexArr)[i] = atoll(pIndex->valuestring); } - cJSON * pJsonSyncCfg = cJSON_GetObjectItem(pJson, "SSyncCfg"); + cJSON *pJsonSyncCfg = cJSON_GetObjectItem(pJson, "SSyncCfg"); int32_t code = syncCfgFromJson(pJsonSyncCfg, &(pRaftCfg->cfg)); ASSERT(code == 0); From 4e8a925da42a815c7ca71ce913bac3fdcf65ccd8 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Wed, 29 Jun 2022 19:21:39 +0800 Subject: [PATCH 34/61] fix: install script for3.0 (#14363) * fix: remove sudo from directory maniplation for 3.0 * fix: remove more sudo from directory maniplation --- packaging/release.sh | 18 ++++++------------ packaging/rpm/makerpm.sh | 0 2 files changed, 6 insertions(+), 12 deletions(-) mode change 100644 => 100755 packaging/rpm/makerpm.sh diff --git a/packaging/release.sh b/packaging/release.sh index 00a4ad7009..1d9f6dfc68 100755 --- a/packaging/release.sh +++ b/packaging/release.sh @@ -181,15 +181,9 @@ cd "${curr_dir}" # 2. cmake executable file compile_dir="${top_dir}/debug" -if [ -d ${compile_dir} ]; then - ${csudo}rm -rf ${compile_dir} -fi +${csudo}rm -rf ${compile_dir} -if [ "$osType" != "Darwin" ]; then - ${csudo}mkdir -p ${compile_dir} -else - mkdir -p ${compile_dir} -fi +mkdir -p ${compile_dir} cd ${compile_dir} if [[ "$allocator" == "jemalloc" ]]; then @@ -255,9 +249,9 @@ if [ "$osType" != "Darwin" ]; then echo "====do deb package for the ubuntu system====" output_dir="${top_dir}/debs" if [ -d ${output_dir} ]; then - ${csudo}rm -rf ${output_dir} + rm -rf ${output_dir} fi - ${csudo}mkdir -p ${output_dir} + mkdir -p ${output_dir} cd ${script_dir}/deb ${csudo}./makedeb.sh ${compile_dir} ${output_dir} ${verNumber} ${cpuType} ${osType} ${verMode} ${verType} @@ -280,9 +274,9 @@ if [ "$osType" != "Darwin" ]; then echo "====do rpm package for the centos system====" output_dir="${top_dir}/rpms" if [ -d ${output_dir} ]; then - ${csudo}rm -rf ${output_dir} + rm -rf ${output_dir} fi - ${csudo}mkdir -p ${output_dir} + mkdir -p ${output_dir} cd ${script_dir}/rpm ${csudo}./makerpm.sh ${compile_dir} ${output_dir} ${verNumber} ${cpuType} ${osType} ${verMode} ${verType} diff --git a/packaging/rpm/makerpm.sh b/packaging/rpm/makerpm.sh old mode 100644 new mode 100755 From 9e755698707a76636526b67251b82b804e0c8c2d Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 29 Jun 2022 17:50:33 +0800 Subject: [PATCH 35/61] refactor(query): function adoption to us/ns db precision TD-16436 --- source/libs/function/src/builtins.c | 5 +++++ source/libs/function/src/builtinsimpl.c | 2 +- source/libs/scalar/src/sclfunc.c | 8 +++++--- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 640c160203..6fa7d10ca3 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1476,6 +1476,7 @@ static int32_t translateTimeTruncate(SFunctionNode* pFunc, char* pErrBuf, int32_ return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } + //add database precision as param uint8_t dbPrec = pFunc->node.resType.precision; addDbPrecisonParam(pFunc->pParameterList, dbPrec); @@ -1503,6 +1504,10 @@ static int32_t translateTimeDiff(SFunctionNode* pFunc, char* pErrBuf, int32_t le } } + //add database precision as param + uint8_t dbPrec = pFunc->node.resType.precision; + addDbPrecisonParam(pFunc->pParameterList, dbPrec); + pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT}; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 7376715968..e40ab3784d 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -5525,7 +5525,7 @@ int32_t groupKeyFunction(SqlFunctionCtx* pCtx) { int32_t startIndex = pInput->startRowIndex; - //escape rest of data blocks to avoid first entry be overwritten. + //escape rest of data blocks to avoid first entry to be overwritten. if (pInfo->hasResult) { goto _group_key_over; } diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 57962abc01..b7c171db2e 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -1380,10 +1380,12 @@ int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarPara } int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { - int32_t timePrec = GET_PARAM_PRECISON(&pInput[0]); - int64_t timeUnit = -1, timeVal[2] = {0}; - if (inputNum == 3) { + int64_t timeUnit = -1, timePrec, timeVal[2] = {0}; + if (inputNum == 4) { GET_TYPED_DATA(timeUnit, int64_t, GET_PARAM_TYPE(&pInput[2]), pInput[2].columnData->pData); + GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[3]), pInput[3].columnData->pData); + } else { + GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[2]), pInput[2].columnData->pData); } int32_t numOfRows = 0; From 5e31c9e8fe664af8be2c4292a65dad9380802e8a Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 29 Jun 2022 17:50:33 +0800 Subject: [PATCH 36/61] refactor(query): function adoption to us/ns db precision TD-16436 --- source/libs/function/src/builtins.c | 4 ++++ source/libs/scalar/src/sclfunc.c | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 6fa7d10ca3..440e12c3d7 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1460,6 +1460,10 @@ static int32_t translateToUnixtimestamp(SFunctionNode* pFunc, char* pErrBuf, int return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } + //add database precision as param + uint8_t dbPrec = pFunc->node.resType.precision; + addDbPrecisonParam(pFunc->pParameterList, dbPrec); + pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT}; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index b7c171db2e..dc4bf33a2b 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -1116,7 +1116,8 @@ int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam * int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { int32_t type = GET_PARAM_TYPE(pInput); - int32_t timePrec = GET_PARAM_PRECISON(pInput); + int64_t timePrec; + GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[1]), pInput[1].columnData->pData); for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { if (colDataIsNull_s(pInput[0].columnData, i)) { From 8faf37f1d00a6a3a7f0d864c09cb370047a47985 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 29 Jun 2022 20:30:16 +0800 Subject: [PATCH 37/61] fix:error if num of line is invalid --- source/client/src/clientSml.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 7d2bf019d2..8a9905935a 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -2453,7 +2453,6 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr pTscObj->schemalessType = 1; SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; - int cnt = ceil(((double)numLines) / LINE_BATCH); Params params; params.request = request; tsem_init(¶ms.sem, 0, 0); @@ -2490,6 +2489,15 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr goto end; } + if(protocol == TSDB_SML_JSON_PROTOCOL){ + numLines = 1; + }else if(numLines <= 0){ + request->code = TSDB_CODE_SML_INVALID_DATA; + smlBuildInvalidDataMsg(&msg, "line num is invalid", NULL); + goto end; + } + + int cnt = ceil(((double)numLines) / LINE_BATCH); for (int i = 0; i < cnt; ++i) { SRequestObj* req = (SRequestObj*)createRequest(pTscObj, TSDB_SQL_INSERT); if(!req){ From 0eed30b0f028a68bd08ad23d26e21d5a81509137 Mon Sep 17 00:00:00 2001 From: tomchon Date: Wed, 29 Jun 2022 20:46:42 +0800 Subject: [PATCH 38/61] test:modify testcase of muti-mondes --- .../system-test/6-cluster/5dnode3mnodeStop.py | 14 +-- .../6-cluster/5dnode3mnodeStopLoop.py | 118 ++++++++++++++++++ tests/system-test/fulltest.sh | 5 +- 3 files changed, 122 insertions(+), 15 deletions(-) create mode 100644 tests/system-test/6-cluster/5dnode3mnodeStopLoop.py diff --git a/tests/system-test/6-cluster/5dnode3mnodeStop.py b/tests/system-test/6-cluster/5dnode3mnodeStop.py index 1b55176e1d..f932e5537e 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeStop.py +++ b/tests/system-test/6-cluster/5dnode3mnodeStop.py @@ -88,7 +88,7 @@ class TDTestCase: tdLog.info("Confirm the status of the dnode again") tdSql.error("create mnode on dnode 2") tdSql.query("show dnodes;") - print(tdSql.queryResult) + # print(tdSql.queryResult) clusterComCheck.checkDnodes(dnodenumbers) # restart all taosd tdDnodes=cluster.dnodes @@ -108,18 +108,6 @@ class TDTestCase: tdDnodes[0].starttaosd() clusterComCheck.checkMnodeStatus(3) - tdLog.info("Take turns stopping all dnodes ") - # seperate vnode and mnode in different dnodes. - # create database and stable - stopcount =0 - while stopcount <= 2: - tdLog.info("first restart loop") - for i in range(dnodenumbers): - tdDnodes[i].stoptaosd() - tdDnodes[i].starttaosd() - stopcount+=1 - clusterComCheck.checkDnodes(dnodenumbers) - clusterComCheck.checkMnodeStatus(3) def run(self): # print(self.master_dnode.cfgDict) diff --git a/tests/system-test/6-cluster/5dnode3mnodeStopLoop.py b/tests/system-test/6-cluster/5dnode3mnodeStopLoop.py new file mode 100644 index 0000000000..e0c91e5ac4 --- /dev/null +++ b/tests/system-test/6-cluster/5dnode3mnodeStopLoop.py @@ -0,0 +1,118 @@ +from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE +import taos +import sys +import time +import os + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.dnodes import TDDnodes +from util.dnodes import TDDnode +from util.cluster import * +from test import tdDnodes +sys.path.append("./6-cluster") + +from clusterCommonCreate import * +from clusterCommonCheck import * +import time +import socket +import subprocess +from multiprocessing import Process + + +class TDTestCase: + + def init(self,conn ,logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + self.host = socket.gethostname() + + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def fiveDnodeThreeMnode(self,dnodenumbers,mnodeNums,restartNumber): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'db', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'replica': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbNum': 1, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 10, + 'showMsg': 1, + 'showRow': 1} + dnodenumbers=int(dnodenumbers) + mnodeNums=int(mnodeNums) + dbNumbers = int(dnodenumbers * restartNumber) + + tdLog.info("first check dnode and mnode") + tdSql.query("show dnodes;") + tdSql.checkData(0,1,'%s:6030'%self.host) + tdSql.checkData(4,1,'%s:6430'%self.host) + clusterComCheck.checkDnodes(dnodenumbers) + clusterComCheck.checkMnodeStatus(1) + + # fisr add three mnodes; + tdLog.info("fisr add three mnodes and check mnode status") + tdSql.execute("create mnode on dnode 2") + clusterComCheck.checkMnodeStatus(2) + tdSql.execute("create mnode on dnode 3") + clusterComCheck.checkMnodeStatus(3) + + # add some error operations and + tdLog.info("Confirm the status of the dnode again") + tdSql.error("create mnode on dnode 2") + tdSql.query("show dnodes;") + # print(tdSql.queryResult) + clusterComCheck.checkDnodes(dnodenumbers) + # restart all taosd + tdDnodes=cluster.dnodes + + tdLog.info("Take turns stopping all dnodes ") + # seperate vnode and mnode in different dnodes. + # create database and stable + stopcount =0 + while stopcount <= 2: + tdLog.info(" restart loop: %d"%stopcount ) + for i in range(dnodenumbers): + tdDnodes[i].stoptaosd() + tdDnodes[i].starttaosd() + stopcount+=1 + clusterComCheck.checkDnodes(dnodenumbers) + clusterComCheck.checkMnodeStatus(3) + + def run(self): + # print(self.master_dnode.cfgDict) + self.fiveDnodeThreeMnode(5,3,1) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index b76d73b54c..436c2ed349 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -118,15 +118,16 @@ python3 ./test.py -f 2-query/queryQnode.py python3 ./test.py -f 6-cluster/5dnode1mnode.py python3 ./test.py -f 6-cluster/5dnode2mnode.py -N 5 -M 3 python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeStopLoop.py -N 5 -M 3 # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 5 -M 3 # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 5 -M 3 -python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 5 -M 3 +# python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 5 -M 3 # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py -N 5 -M 3 # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py -N 5 -M 3 # python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py -N 5 -M 3 # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeStopInsert.py # python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5 -python3 test.py -f 6-cluster/5dnode3mnodeStopConnect.py -N 5 -M 3 +# python3 test.py -f 6-cluster/5dnode3mnodeStopConnect.py -N 5 -M 3 python3 ./test.py -f 7-tmq/basic5.py From 4c23223447f7c8d4771939e1b0a740e3f5e3be26 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 29 Jun 2022 17:50:33 +0800 Subject: [PATCH 39/61] refactor(query): function adoption to us/ns db precision TD-16436 --- source/libs/function/src/builtins.c | 39 ++++++++++++++++------------- source/libs/scalar/src/sclfunc.c | 10 ++++++-- 2 files changed, 30 insertions(+), 19 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 440e12c3d7..bfb638360d 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -39,6 +39,20 @@ static int32_t invaildFuncParaValueErrMsg(char* pErrBuf, int32_t len, const char return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_PARA_VALUE, "Invalid parameter value : %s", pFuncName); } +void static addDbPrecisonParam(SNodeList** pList, uint8_t precision) { + SValueNode* pVal = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); + pVal->literal = NULL; + pVal->isDuration = false; + pVal->translate = true; + pVal->node.resType.type = TSDB_DATA_TYPE_TINYINT; + pVal->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_TINYINT].bytes; + pVal->node.resType.precision = precision; + pVal->datum.i = (int64_t)precision; + pVal->typeData = (int64_t)precision; + + nodesListMakeAppend(pList, (SNode*)pVal); +} + // There is only one parameter of numeric type, and the return type is parameter type static int32_t translateInOutNum(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { if (1 != LIST_LENGTH(pFunc->pParameterList)) { @@ -222,6 +236,11 @@ static int32_t translateWduration(SFunctionNode* pFunc, char* pErrBuf, int32_t l static int32_t translateTimePseudoColumn(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { // pseudo column do not need to check parameters + + //add database precision as param + uint8_t dbPrec = pFunc->node.resType.precision; + addDbPrecisonParam(&pFunc->pParameterList, dbPrec); + pFunc->node.resType = (SDataType){.bytes = sizeof(int64_t), .type = TSDB_DATA_TYPE_TIMESTAMP}; return TSDB_CODE_SUCCESS; } @@ -1379,20 +1398,6 @@ static bool validateTimezoneFormat(const SValueNode* pVal) { return true; } -void static addDbPrecisonParam(SNodeList* pList, uint8_t precision) { - SValueNode* pVal = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); - pVal->literal = NULL; - pVal->isDuration = false; - pVal->translate = true; - pVal->node.resType.type = TSDB_DATA_TYPE_TINYINT; - pVal->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_TINYINT].bytes; - pVal->node.resType.precision = precision; - pVal->datum.i = (int64_t)precision; - pVal->typeData = (int64_t)precision; - - nodesListAppend(pList, (SNode*)pVal); -} - void static addTimezoneParam(SNodeList* pList) { char buf[6] = {0}; time_t t = taosTime(NULL); @@ -1462,7 +1467,7 @@ static int32_t translateToUnixtimestamp(SFunctionNode* pFunc, char* pErrBuf, int //add database precision as param uint8_t dbPrec = pFunc->node.resType.precision; - addDbPrecisonParam(pFunc->pParameterList, dbPrec); + addDbPrecisonParam(&pFunc->pParameterList, dbPrec); pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT}; return TSDB_CODE_SUCCESS; @@ -1482,7 +1487,7 @@ static int32_t translateTimeTruncate(SFunctionNode* pFunc, char* pErrBuf, int32_ //add database precision as param uint8_t dbPrec = pFunc->node.resType.precision; - addDbPrecisonParam(pFunc->pParameterList, dbPrec); + addDbPrecisonParam(&pFunc->pParameterList, dbPrec); pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_TIMESTAMP].bytes, .type = TSDB_DATA_TYPE_TIMESTAMP}; @@ -1510,7 +1515,7 @@ static int32_t translateTimeDiff(SFunctionNode* pFunc, char* pErrBuf, int32_t le //add database precision as param uint8_t dbPrec = pFunc->node.resType.precision; - addDbPrecisonParam(pFunc->pParameterList, dbPrec); + addDbPrecisonParam(&pFunc->pParameterList, dbPrec); pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT}; return TSDB_CODE_SUCCESS; diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index dc4bf33a2b..d350a380e0 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -1515,7 +1515,10 @@ int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *p } int32_t nowFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { - int64_t ts = taosGetTimestamp(TSDB_TIME_PRECISION_MILLI); + int64_t timePrec; + GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[0]), pInput[0].columnData->pData); + + int64_t ts = taosGetTimestamp(timePrec); for (int32_t i = 0; i < pInput->numOfRows; ++i) { colDataAppendInt64(pOutput->columnData, i, &ts); } @@ -1524,7 +1527,10 @@ int32_t nowFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu } int32_t todayFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { - int64_t ts = taosGetTimestampToday(TSDB_TIME_PRECISION_MILLI); + int64_t timePrec; + GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[1]), pInput[1].columnData->pData); + + int64_t ts = taosGetTimestampToday(timePrec); for (int32_t i = 0; i < pInput->numOfRows; ++i) { colDataAppendInt64(pOutput->columnData, i, &ts); } From b1670f8b82efad7c24e964e5c8ae35e83c484f0e Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Wed, 29 Jun 2022 17:50:33 +0800 Subject: [PATCH 40/61] refactor(query): function adoption to us/ns db precision TD-16436 --- source/libs/scalar/src/sclfunc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index d350a380e0..76c0e48740 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -1528,7 +1528,7 @@ int32_t nowFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu int32_t todayFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { int64_t timePrec; - GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[1]), pInput[1].columnData->pData); + GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[0]), pInput[0].columnData->pData); int64_t ts = taosGetTimestampToday(timePrec); for (int32_t i = 0; i < pInput->numOfRows; ++i) { From c2955a807b6c60bd6a6c105ca0477dcddd952971 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 29 Jun 2022 20:57:15 +0800 Subject: [PATCH 41/61] feat(tmq): offset support snapshot --- examples/c/tmq.c | 4 +- include/common/tcommon.h | 17 +- include/common/tmsg.h | 70 ++- include/libs/executor/executor.h | 7 - include/libs/stream/tstream.h | 19 +- source/client/inc/clientInt.h | 25 +- source/client/src/clientMain.c | 10 +- source/client/src/tmq.c | 414 ++++++++---------- source/common/src/tmsg.c | 168 +++++-- source/dnode/vnode/inc/vnode.h | 20 +- source/dnode/vnode/src/inc/tq.h | 19 +- source/dnode/vnode/src/sma/smaRollup.c | 14 +- source/dnode/vnode/src/tq/tq.c | 299 +++++++++++-- source/dnode/vnode/src/tq/tqExec.c | 35 +- source/dnode/vnode/src/tq/tqOffset.c | 4 +- source/dnode/vnode/src/tq/tqPush.c | 12 +- source/dnode/vnode/src/tq/tqRead.c | 29 +- source/dnode/vnode/src/vnd/vnodeSvr.c | 2 +- source/libs/executor/src/executor.c | 10 +- source/libs/executor/src/scanoperator.c | 12 +- source/libs/executor/src/timewindowoperator.c | 26 +- source/libs/function/src/builtins.c | 25 +- source/libs/stream/src/streamExec.c | 19 +- source/libs/stream/src/streamTask.c | 8 +- 24 files changed, 810 insertions(+), 458 deletions(-) diff --git a/examples/c/tmq.c b/examples/c/tmq.c index 68b0492ace..4226587d56 100644 --- a/examples/c/tmq.c +++ b/examples/c/tmq.c @@ -138,7 +138,7 @@ int32_t create_topic() { taos_free_result(pRes); /*pRes = taos_query(pConn, "create topic topic_ctb_column with meta as database abc1");*/ - pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from st1 where t1 = 2000"); + pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from st1"); if (taos_errno(pRes) != 0) { printf("failed to create topic topic_ctb_column, reason:%s\n", taos_errstr(pRes)); return -1; @@ -225,7 +225,7 @@ void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) { } int32_t cnt = 0; while (running) { - TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, 0); + TAOS_RES* tmqmessage = tmq_consumer_poll(tmq, -1); if (tmqmessage) { cnt++; msg_process(tmqmessage); diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 928fe0aa0e..54044f7d89 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -25,10 +25,11 @@ extern "C" { #endif +// TODO remove it enum { - TMQ_CONF__RESET_OFFSET__LATEST = -1, - TMQ_CONF__RESET_OFFSET__EARLIEAST = -2, TMQ_CONF__RESET_OFFSET__NONE = -3, + TMQ_CONF__RESET_OFFSET__EARLIEAST = -2, + TMQ_CONF__RESET_OFFSET__LATEST = -1, }; enum { @@ -39,6 +40,16 @@ enum { TMQ_MSG_TYPE__END_RSP, }; +enum { + STREAM_INPUT__DATA_SUBMIT = 1, + STREAM_INPUT__DATA_BLOCK, + STREAM_INPUT__DATA_SCAN, + STREAM_INPUT__DATA_RETRIEVE, + STREAM_INPUT__TRIGGER, + STREAM_INPUT__CHECKPOINT, + STREAM_INPUT__DROP, +}; + typedef enum EStreamType { STREAM_NORMAL = 1, STREAM_INVERT, @@ -48,7 +59,7 @@ typedef enum EStreamType { STREAM_DELETE, STREAM_RETRIEVE, STREAM_PUSH_DATA, - STREAM_PUSH_EMPTY, + STREAM_PUSH_OVER, } EStreamType; typedef struct { diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 0afcaaa50d..71867d9741 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2462,22 +2462,37 @@ int32_t tDecodeSMqCMCommitOffsetReq(SDecoder* decoder, SMqCMCommitOffsetReq* pRe // tqOffset enum { - TMQ_OFFSET__SNAPSHOT = 1, - TMQ_OFFSET__LOG, + TMQ_OFFSET__RESET_NONE = -3, + TMQ_OFFSET__RESET_EARLIEAST = -2, + TMQ_OFFSET__RESET_LATEST = -1, + TMQ_OFFSET__LOG = 1, + TMQ_OFFSET__SNAPSHOT_DATA = 2, + TMQ_OFFSET__SNAPSHOT_META = 3, }; typedef struct { int8_t type; union { + // snapshot data struct { int64_t uid; int64_t ts; }; + // log struct { int64_t version; }; }; - char subKey[TSDB_SUBSCRIBE_KEY_LEN]; +} STqOffsetVal; + +int32_t tEncodeSTqOffsetVal(SEncoder* pEncoder, const STqOffsetVal* pOffsetVal); +int32_t tDecodeSTqOffsetVal(SDecoder* pDecoder, STqOffsetVal* pOffsetVal); +int32_t tFormatOffset(char* buf, int32_t maxLen, const STqOffsetVal* pVal); +bool tOffsetEqual(const STqOffsetVal* pLeft, const STqOffsetVal* pRight); + +typedef struct { + STqOffsetVal val; + char subKey[TSDB_SUBSCRIBE_KEY_LEN]; } STqOffset; int32_t tEncodeSTqOffset(SEncoder* pEncoder, const STqOffset* pOffset); @@ -2710,7 +2725,8 @@ typedef struct { uint64_t reqId; int64_t consumerId; int64_t timeout; - int64_t currentOffset; + // int64_t currentOffset; + STqOffsetVal reqOffset; } SMqPollReq; typedef struct { @@ -2779,12 +2795,14 @@ static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) { } typedef struct { - SMqRspHead head; - int64_t reqOffset; - int64_t rspOffset; - int16_t resMsgType; - int32_t metaRspLen; - void* metaRsp; + SMqRspHead head; + int64_t reqOffset; + int64_t rspOffset; + STqOffsetVal reqOffsetNew; + STqOffsetVal rspOffsetNew; + int16_t resMsgType; + int32_t metaRspLen; + void* metaRsp; } SMqMetaRsp; static FORCE_INLINE int32_t tEncodeSMqMetaRsp(void** buf, const SMqMetaRsp* pRsp) { @@ -2806,6 +2824,24 @@ static FORCE_INLINE void* tDecodeSMqMetaRsp(const void* buf, SMqMetaRsp* pRsp) { return (void*)buf; } +typedef struct { + SMqRspHead head; + STqOffsetVal reqOffset; + STqOffsetVal rspOffset; + int32_t skipLogNum; + int32_t blockNum; + int8_t withTbName; + int8_t withSchema; + SArray* blockDataLen; + SArray* blockData; + SArray* blockTbName; + SArray* blockSchema; +} SMqDataRsp; + +int32_t tEncodeSMqDataRsp(SEncoder* pEncoder, const SMqDataRsp* pRsp); +int32_t tDecodeSMqDataRsp(SDecoder* pDecoder, SMqDataRsp* pRsp); + +#if 0 typedef struct { SMqRspHead head; int64_t reqOffset; @@ -2814,13 +2850,10 @@ typedef struct { int32_t blockNum; int8_t withTbName; int8_t withSchema; - int8_t withTag; - SArray* blockDataLen; // SArray - SArray* blockData; // SArray - SArray* blockTbName; // SArray - SArray* blockSchema; // SArray - SArray* blockTags; // SArray - SArray* blockTagSchema; // SArray + SArray* blockDataLen; // SArray + SArray* blockData; // SArray + SArray* blockTbName; // SArray + SArray* blockSchema; // SArray } SMqDataBlkRsp; static FORCE_INLINE int32_t tEncodeSMqDataBlkRsp(void** buf, const SMqDataBlkRsp* pRsp) { @@ -2832,7 +2865,6 @@ static FORCE_INLINE int32_t tEncodeSMqDataBlkRsp(void** buf, const SMqDataBlkRsp if (pRsp->blockNum != 0) { tlen += taosEncodeFixedI8(buf, pRsp->withTbName); tlen += taosEncodeFixedI8(buf, pRsp->withSchema); - tlen += taosEncodeFixedI8(buf, pRsp->withTag); for (int32_t i = 0; i < pRsp->blockNum; i++) { int32_t bLen = *(int32_t*)taosArrayGet(pRsp->blockDataLen, i); @@ -2862,7 +2894,6 @@ static FORCE_INLINE void* tDecodeSMqDataBlkRsp(const void* buf, SMqDataBlkRsp* p pRsp->blockDataLen = taosArrayInit(pRsp->blockNum, sizeof(int32_t)); buf = taosDecodeFixedI8(buf, &pRsp->withTbName); buf = taosDecodeFixedI8(buf, &pRsp->withSchema); - buf = taosDecodeFixedI8(buf, &pRsp->withTag); if (pRsp->withTbName) { pRsp->blockTbName = taosArrayInit(pRsp->blockNum, sizeof(void*)); } @@ -2891,6 +2922,7 @@ static FORCE_INLINE void* tDecodeSMqDataBlkRsp(const void* buf, SMqDataBlkRsp* p } return (void*)buf; } +#endif typedef struct { SMqRspHead head; diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 00acc4741d..564c208c69 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -36,15 +36,8 @@ typedef struct SReadHandle { void* vnode; void* mnd; SMsgCb* pMsgCb; -// int8_t initTsdbReader; } SReadHandle; -enum { - STREAM_DATA_TYPE_SUBMIT_BLOCK = 1, - STREAM_DATA_TYPE_SSDATA_BLOCK = 2, - STREAM_DATA_TYPE_FROM_SNAPSHOT = 3, -}; - typedef enum { OPTR_EXEC_MODEL_BATCH = 0x1, OPTR_EXEC_MODEL_STREAM = 0x2, diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 74c885ebcc..67074c789e 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -55,15 +55,6 @@ enum { TASK_OUTPUT_STATUS__BLOCKED, }; -enum { - STREAM_INPUT__DATA_SUBMIT = 1, - STREAM_INPUT__DATA_BLOCK, - STREAM_INPUT__DATA_RETRIEVE, - STREAM_INPUT__TRIGGER, - STREAM_INPUT__CHECKPOINT, - STREAM_INPUT__DROP, -}; - typedef struct { int8_t type; } SStreamQueueItem; @@ -152,10 +143,6 @@ typedef struct { void* executor; } STaskExec; -typedef struct { - int32_t taskId; -} STaskDispatcherInplace; - typedef struct { int32_t taskId; int32_t nodeId; @@ -208,7 +195,6 @@ enum { enum { TASK_DISPATCH__NONE = 1, - TASK_DISPATCH__INPLACE, TASK_DISPATCH__FIXED, TASK_DISPATCH__SHUFFLE, }; @@ -260,7 +246,7 @@ struct SStreamTask { // exec STaskExec exec; - // TODO: merge sink and dispatch + // TODO: unify sink and dispatch // local sink union { @@ -269,9 +255,8 @@ struct SStreamTask { STaskSinkFetch fetchSink; }; - // dispatch + // remote dispatcher union { - STaskDispatcherInplace inplaceDispatcher; STaskDispatcherFixedEp fixedEpDispatcher; STaskDispatcherShuffle shuffleDispatcher; }; diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index cca79186d0..5ffc9ecae8 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -54,11 +54,10 @@ enum { RES_TYPE__TMQ_META, }; -#define SHOW_VARIABLES_RESULT_COLS 2 +#define SHOW_VARIABLES_RESULT_COLS 2 #define SHOW_VARIABLES_RESULT_FIELD1_LEN (TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE) #define SHOW_VARIABLES_RESULT_FIELD2_LEN (TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE) - #define TD_RES_QUERY(res) (*(int8_t*)res == RES_TYPE__QUERY) #define TD_RES_TMQ(res) (*(int8_t*)res == RES_TYPE__TMQ) #define TD_RES_TMQ_META(res) (*(int8_t*)res == RES_TYPE__TMQ_META) @@ -194,7 +193,7 @@ typedef struct { int32_t vgId; SSchemaWrapper schema; int32_t resIter; - SMqDataBlkRsp rsp; + SMqDataRsp rsp; SReqResultInfo resInfo; } SMqRspObj; @@ -238,18 +237,18 @@ typedef struct SSyncQueryParam { void* doAsyncFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4); void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4); -void doSetOneRowPtr(SReqResultInfo* pResultInfo); -void setResPrecision(SReqResultInfo* pResInfo, int32_t precision); -int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4, - bool freeAfterUse); -void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols); -void doFreeReqResultInfo(SReqResultInfo* pResInfo); -int32_t transferTableNameList(const char* tbList, int32_t acctId, char* dbName, SArray** pReq); -void syncCatalogFn(SMetaData* pResult, void* param, int32_t code); +void doSetOneRowPtr(SReqResultInfo* pResultInfo); +void setResPrecision(SReqResultInfo* pResInfo, int32_t precision); +int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4, + bool freeAfterUse); +void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols); +void doFreeReqResultInfo(SReqResultInfo* pResInfo); +int32_t transferTableNameList(const char* tbList, int32_t acctId, char* dbName, SArray** pReq); +void syncCatalogFn(SMetaData* pResult, void* param, int32_t code); SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen, bool validateOnly); -TAOS_RES *taosQueryImpl(TAOS *taos, const char *sql, bool validateOnly); -void taosAsyncQueryImpl(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param, bool validateOnly); +TAOS_RES* taosQueryImpl(TAOS* taos, const char* sql, bool validateOnly); +void taosAsyncQueryImpl(TAOS* taos, const char* sql, __taos_async_fn_t fp, void* param, bool validateOnly); static FORCE_INLINE SReqResultInfo* tmqGetCurResInfo(TAOS_RES* res) { SMqRspObj* msg = (SMqRspObj*)res; diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 52574dcc9f..90df3eb31a 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -81,16 +81,16 @@ void taos_cleanup(void) { taosCloseLog(); } -static setConfRet taos_set_config_imp(const char *config){ +static setConfRet taos_set_config_imp(const char *config) { setConfRet ret = {SET_CONF_RET_SUCC, {0}}; // TODO: need re-implementation return ret; } -setConfRet taos_set_config(const char *config){ -// TODO pthread_mutex_lock(&setConfMutex); +setConfRet taos_set_config(const char *config) { + // TODO pthread_mutex_lock(&setConfMutex); setConfRet ret = taos_set_config_imp(config); -// pthread_mutex_unlock(&setConfMutex); + // pthread_mutex_unlock(&setConfMutex); return ret; } @@ -179,8 +179,6 @@ void taos_free_result(TAOS_RES *res) { SMqRspObj *pRsp = (SMqRspObj *)res; if (pRsp->rsp.blockData) taosArrayDestroyP(pRsp->rsp.blockData, taosMemoryFree); if (pRsp->rsp.blockDataLen) taosArrayDestroy(pRsp->rsp.blockDataLen); - if (pRsp->rsp.blockTags) taosArrayDestroy(pRsp->rsp.blockTags); - if (pRsp->rsp.blockTagSchema) taosArrayDestroy(pRsp->rsp.blockTagSchema); if (pRsp->rsp.withTbName) taosArrayDestroyP(pRsp->rsp.blockTbName, taosMemoryFree); if (pRsp->rsp.withSchema) taosArrayDestroyP(pRsp->rsp.blockSchema, (FDelete)tDeleteSSchemaWrapper); pRsp->resInfo.pRspMsg = NULL; diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 637a7ee5dd..4cef25b9ce 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -126,8 +126,10 @@ typedef struct { // statistics int64_t pollCnt; // offset - int64_t committedOffset; - int64_t currentOffset; + /*int64_t committedOffset;*/ + /*int64_t currentOffset;*/ + STqOffsetVal committedOffsetNew; + STqOffsetVal currentOffsetNew; // connection info int32_t vgId; int32_t vgStatus; @@ -152,8 +154,8 @@ typedef struct { SMqClientVg* vgHandle; SMqClientTopic* topicHandle; union { - SMqDataBlkRsp dataRsp; - SMqMetaRsp metaRsp; + SMqDataRsp dataRsp; + SMqMetaRsp metaRsp; }; } SMqPollRspWrapper; @@ -179,6 +181,7 @@ typedef struct { tsem_t rspSem; } SMqPollCbParam; +#if 0 typedef struct { tmq_t* tmq; int8_t async; @@ -190,12 +193,13 @@ typedef struct { SArray* offsets; void* userParam; } SMqCommitCbParam; +#endif typedef struct { - tmq_t* tmq; - int8_t automatic; - int8_t async; - int8_t freeOffsets; + tmq_t* tmq; + int8_t automatic; + int8_t async; + /*int8_t freeOffsets;*/ int32_t waitingRspNum; int32_t totalRspNum; int32_t rspErr; @@ -351,6 +355,7 @@ static int32_t tmqMakeTopicVgKey(char* dst, const char* topicName, int32_t vg) { return sprintf(dst, "%s:%d", topicName, vg); } +#if 0 int32_t tmqCommitCb(void* param, const SDataBuf* pMsg, int32_t code) { SMqCommitCbParam* pParam = (SMqCommitCbParam*)param; pParam->rspErr = code; @@ -371,6 +376,7 @@ int32_t tmqCommitCb(void* param, const SDataBuf* pMsg, int32_t code) { } return 0; } +#endif int32_t tmqCommitCb2(void* param, const SDataBuf* pBuf, int32_t code) { SMqCommitCbParam2* pParam = (SMqCommitCbParam2*)param; @@ -413,139 +419,148 @@ int32_t tmqCommitCb2(void* param, const SDataBuf* pBuf, int32_t code) { return 0; } +static int32_t tmqSendCommitReq(tmq_t* tmq, SMqClientVg* pVg, SMqClientTopic* pTopic, SMqCommitCbParamSet* pParamSet) { + STqOffset* pOffset = taosMemoryCalloc(1, sizeof(STqOffset)); + if (pOffset == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pOffset->val = pVg->currentOffsetNew; + + int32_t groupLen = strlen(tmq->groupId); + memcpy(pOffset->subKey, tmq->groupId, groupLen); + pOffset->subKey[groupLen] = TMQ_SEPARATOR; + strcpy(pOffset->subKey + groupLen + 1, pTopic->topicName); + + int32_t len; + int32_t code; + tEncodeSize(tEncodeSTqOffset, pOffset, len, code); + if (code < 0) { + ASSERT(0); + return -1; + } + void* buf = taosMemoryCalloc(1, sizeof(SMsgHead) + len); + if (buf == NULL) return -1; + ((SMsgHead*)buf)->vgId = htonl(pVg->vgId); + + void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + + SEncoder encoder; + tEncoderInit(&encoder, abuf, len); + tEncodeSTqOffset(&encoder, pOffset); + + // build param + SMqCommitCbParam2* pParam = taosMemoryCalloc(1, sizeof(SMqCommitCbParam2)); + pParam->params = pParamSet; + pParam->pOffset = pOffset; + + // build send info + SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); + if (pMsgSendInfo == NULL) { + return -1; + } + pMsgSendInfo->msgInfo = (SDataBuf){ + .pData = buf, + .len = sizeof(SMsgHead) + len, + .handle = NULL, + }; + + tscDebug("consumer %ld commit offset of %s on vg %d, offset is %ld", tmq->consumerId, pOffset->subKey, pVg->vgId, + pOffset->val.version); + + // TODO: put into cb + pVg->committedOffsetNew = pVg->currentOffsetNew; + + pMsgSendInfo->requestId = generateRequestId(); + pMsgSendInfo->requestObjRefId = 0; + pMsgSendInfo->param = pParam; + pMsgSendInfo->fp = tmqCommitCb2; + pMsgSendInfo->msgType = TDMT_VND_MQ_COMMIT_OFFSET; + // send msg + + int64_t transporterId = 0; + asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, pMsgSendInfo); + pParamSet->waitingRspNum++; + pParamSet->totalRspNum++; + return 0; +} + +int32_t tmqCommitMsgImpl(tmq_t* tmq, const TAOS_RES* msg, int8_t async, tmq_commit_cb* userCb, void* userParam) { + char* topic; + int32_t vgId; + ASSERT(msg != NULL); + if (TD_RES_TMQ(msg)) { + SMqRspObj* pRspObj = (SMqRspObj*)msg; + topic = pRspObj->topic; + vgId = pRspObj->vgId; + } else if (TD_RES_TMQ_META(msg)) { + SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)msg; + topic = pMetaRspObj->topic; + vgId = pMetaRspObj->vgId; + } else { + return TSDB_CODE_TMQ_INVALID_MSG; + } + + SMqCommitCbParamSet* pParamSet = taosMemoryCalloc(1, sizeof(SMqCommitCbParamSet)); + if (pParamSet == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + pParamSet->tmq = tmq; + pParamSet->automatic = 0; + pParamSet->async = async; + /*pParamSet->freeOffsets = 1;*/ + pParamSet->userCb = userCb; + pParamSet->userParam = userParam; + tsem_init(&pParamSet->rspSem, 0, 0); + + for (int32_t i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { + SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); + if (strcmp(pTopic->topicName, topic) != 0) continue; + for (int32_t j = 0; j < taosArrayGetSize(pTopic->vgs); j++) { + SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); + if (pVg->vgId != vgId) continue; + + if (pVg->currentOffsetNew.type > 0 && !tOffsetEqual(&pVg->currentOffsetNew, &pVg->committedOffsetNew)) { + if (tmqSendCommitReq(tmq, pVg, pTopic, pParamSet) < 0) { + goto FAIL; + } + goto HANDLE_RSP; + } + } + } + + int32_t code = -1; + +HANDLE_RSP: + if (pParamSet->totalRspNum == 0) { + tsem_destroy(&pParamSet->rspSem); + taosMemoryFree(pParamSet); + return 0; + } + + if (!async) { + tsem_wait(&pParamSet->rspSem); + code = pParamSet->rspErr; + tsem_destroy(&pParamSet->rspSem); + return code; + } else { + code = 0; + } + +FAIL: + if (code != 0 && async) { + userCb(tmq, code, userParam); + } + return 0; +} + int32_t tmqCommitInner2(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_t async, tmq_commit_cb* userCb, void* userParam) { int32_t code = -1; if (msg != NULL) { - char* topic; - int32_t vgId; - if (TD_RES_TMQ(msg)) { - SMqRspObj* pRspObj = (SMqRspObj*)msg; - topic = pRspObj->topic; - vgId = pRspObj->vgId; - } else if (TD_RES_TMQ_META(msg)) { - SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)msg; - topic = pMetaRspObj->topic; - vgId = pMetaRspObj->vgId; - } else { - return TSDB_CODE_TMQ_INVALID_MSG; - } - - SMqCommitCbParamSet* pParamSet = taosMemoryCalloc(1, sizeof(SMqCommitCbParamSet)); - if (pParamSet == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - pParamSet->tmq = tmq; - pParamSet->automatic = automatic; - pParamSet->async = async; - pParamSet->freeOffsets = 1; - pParamSet->userCb = userCb; - pParamSet->userParam = userParam; - tsem_init(&pParamSet->rspSem, 0, 0); - - for (int32_t i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { - SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); - if (strcmp(pTopic->topicName, topic) == 0) { - for (int32_t j = 0; j < taosArrayGetSize(pTopic->vgs); j++) { - SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); - if (pVg->vgId == vgId) { - if (pVg->currentOffset < 0 || pVg->committedOffset == pVg->currentOffset) { - tscDebug("consumer %ld skip commit for topic %s vg %d, current offset is %ld, committed offset is %ld", - tmq->consumerId, pTopic->topicName, pVg->vgId, pVg->currentOffset, pVg->committedOffset); - - return 0; - } - - STqOffset* pOffset = taosMemoryCalloc(1, sizeof(STqOffset)); - if (pOffset == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - pOffset->type = TMQ_OFFSET__LOG; - pOffset->version = pVg->currentOffset; - - int32_t groupLen = strlen(tmq->groupId); - memcpy(pOffset->subKey, tmq->groupId, groupLen); - pOffset->subKey[groupLen] = TMQ_SEPARATOR; - strcpy(pOffset->subKey + groupLen + 1, pTopic->topicName); - - int32_t len; - int32_t code; - tEncodeSize(tEncodeSTqOffset, pOffset, len, code); - if (code < 0) { - ASSERT(0); - } - void* buf = taosMemoryCalloc(1, sizeof(SMsgHead) + len); - ((SMsgHead*)buf)->vgId = htonl(pVg->vgId); - - void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); - - SEncoder encoder; - tEncoderInit(&encoder, abuf, len); - tEncodeSTqOffset(&encoder, pOffset); - - // build param - SMqCommitCbParam2* pParam = taosMemoryCalloc(1, sizeof(SMqCommitCbParam2)); - pParam->params = pParamSet; - pParam->pOffset = pOffset; - - // build send info - SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); - if (pMsgSendInfo == NULL) { - // TODO - continue; - } - pMsgSendInfo->msgInfo = (SDataBuf){ - .pData = buf, - .len = sizeof(SMsgHead) + len, - .handle = NULL, - }; - - tscDebug("consumer %ld commit offset of %s on vg %d, offset is %ld", tmq->consumerId, pOffset->subKey, - pVg->vgId, pOffset->version); - - // TODO: put into cb - pVg->committedOffset = pVg->currentOffset; - - pMsgSendInfo->requestId = generateRequestId(); - pMsgSendInfo->requestObjRefId = 0; - pMsgSendInfo->param = pParam; - pMsgSendInfo->fp = tmqCommitCb2; - pMsgSendInfo->msgType = TDMT_VND_MQ_COMMIT_OFFSET; - // send msg - - int64_t transporterId = 0; - asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, pMsgSendInfo); - pParamSet->waitingRspNum++; - pParamSet->totalRspNum++; - } - } - } - } - if (pParamSet->totalRspNum == 0) { - tsem_destroy(&pParamSet->rspSem); - taosMemoryFree(pParamSet); - return 0; - } - - if (!async) { - tsem_wait(&pParamSet->rspSem); - code = pParamSet->rspErr; - tsem_destroy(&pParamSet->rspSem); - } else { - code = 0; - } - - if (code != 0 && async) { - if (automatic) { - tmq->commitCb(tmq, code, tmq->commitCbUserParam); - } else { - userCb(tmq, code, userParam); - } - } - return 0; + return tmqCommitMsgImpl(tmq, msg, async, userCb, userParam); } SMqCommitCbParamSet* pParamSet = taosMemoryCalloc(1, sizeof(SMqCommitCbParamSet)); @@ -556,7 +571,7 @@ int32_t tmqCommitInner2(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_ pParamSet->tmq = tmq; pParamSet->automatic = automatic; pParamSet->async = async; - pParamSet->freeOffsets = 1; + /*pParamSet->freeOffsets = 1;*/ pParamSet->userCb = userCb; pParamSet->userParam = userParam; tsem_init(&pParamSet->rspSem, 0, 0); @@ -572,75 +587,11 @@ int32_t tmqCommitInner2(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_ tscDebug("consumer %ld begin commit for topic %s, vgId %d", tmq->consumerId, pTopic->topicName, pVg->vgId); - /*if (pVg->currentOffset < 0) {*/ - if (pVg->currentOffset < 0 || pVg->committedOffset == pVg->currentOffset) { - tscDebug("consumer %ld skip commit for topic %s vg %d, current offset is %ld, committed offset is %ld", - tmq->consumerId, pTopic->topicName, pVg->vgId, pVg->currentOffset, pVg->committedOffset); - - continue; + if (pVg->currentOffsetNew.type > 0 && !tOffsetEqual(&pVg->currentOffsetNew, &pVg->committedOffsetNew)) { + if (tmqSendCommitReq(tmq, pVg, pTopic, pParamSet) < 0) { + continue; + } } - STqOffset* pOffset = taosMemoryCalloc(1, sizeof(STqOffset)); - if (pOffset == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - pOffset->type = TMQ_OFFSET__LOG; - pOffset->version = pVg->currentOffset; - - int32_t groupLen = strlen(tmq->groupId); - memcpy(pOffset->subKey, tmq->groupId, groupLen); - pOffset->subKey[groupLen] = TMQ_SEPARATOR; - strcpy(pOffset->subKey + groupLen + 1, pTopic->topicName); - - int32_t len; - int32_t code; - tEncodeSize(tEncodeSTqOffset, pOffset, len, code); - if (code < 0) { - ASSERT(0); - } - void* buf = taosMemoryCalloc(1, sizeof(SMsgHead) + len); - ((SMsgHead*)buf)->vgId = htonl(pVg->vgId); - - void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); - - SEncoder encoder; - tEncoderInit(&encoder, abuf, len); - tEncodeSTqOffset(&encoder, pOffset); - - // build param - SMqCommitCbParam2* pParam = taosMemoryCalloc(1, sizeof(SMqCommitCbParam2)); - pParam->params = pParamSet; - pParam->pOffset = pOffset; - - // build send info - SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); - if (pMsgSendInfo == NULL) { - // TODO - continue; - } - pMsgSendInfo->msgInfo = (SDataBuf){ - .pData = buf, - .len = sizeof(SMsgHead) + len, - .handle = NULL, - }; - - tscDebug("consumer %ld commit offset of %s on vg %d, offset is %ld", tmq->consumerId, pOffset->subKey, pVg->vgId, - pOffset->version); - - // TODO: put into cb - pVg->committedOffset = pVg->currentOffset; - - pMsgSendInfo->requestId = generateRequestId(); - pMsgSendInfo->requestObjRefId = 0; - pMsgSendInfo->param = pParam; - pMsgSendInfo->fp = tmqCommitCb2; - pMsgSendInfo->msgType = TDMT_VND_MQ_COMMIT_OFFSET; - // send msg - - int64_t transporterId = 0; - asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, pMsgSendInfo); - pParamSet->waitingRspNum++; - pParamSet->totalRspNum++; } } @@ -1174,7 +1125,10 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { if (rspType == TMQ_MSG_TYPE__POLL_RSP) { memcpy(&pRspWrapper->dataRsp, pMsg->pData, sizeof(SMqRspHead)); - tDecodeSMqDataBlkRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRspWrapper->dataRsp); + SDecoder decoder; + tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); + tDecodeSMqDataRsp(&decoder, &pRspWrapper->dataRsp); + /*tDecodeSMqDataBlkRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRspWrapper->dataRsp);*/ } else { ASSERT(rspType == TMQ_MSG_TYPE__POLL_META_RSP); memcpy(&pRspWrapper->metaRsp, pMsg->pData, sizeof(SMqRspHead)); @@ -1226,9 +1180,11 @@ bool tmqUpdateEp2(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { for (int32_t j = 0; j < vgNumCur; j++) { SMqClientVg* pVgCur = taosArrayGet(pTopicCur->vgs, j); sprintf(vgKey, "%s:%d", pTopicCur->topicName, pVgCur->vgId); - tscDebug("consumer %ld epoch %d vg %d vgKey is %s, offset is %ld", tmq->consumerId, epoch, pVgCur->vgId, vgKey, - pVgCur->currentOffset); - taosHashPut(pHash, vgKey, strlen(vgKey), &pVgCur->currentOffset, sizeof(int64_t)); + char buf[50]; + tFormatOffset(buf, 50, &pVgCur->currentOffsetNew); + tscDebug("consumer %ld epoch %d vg %d vgKey is %s, offset is %s", tmq->consumerId, epoch, pVgCur->vgId, vgKey, + buf); + taosHashPut(pHash, vgKey, strlen(vgKey), &pVgCur->currentOffsetNew, sizeof(STqOffsetVal)); } } } @@ -1257,7 +1213,7 @@ bool tmqUpdateEp2(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { pVgEp->vgId, offset, vgKey); SMqClientVg clientVg = { .pollCnt = 0, - .currentOffset = offset, + .currentOffsetNew.type = tmq->resetOffsetCfg, .vgId = pVgEp->vgId, .epSet = pVgEp->epSet, .vgStatus = TMQ_VG_STATUS__IDLE, @@ -1281,7 +1237,7 @@ bool tmqUpdateEp2(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { return set; } -#if 1 +#if 0 bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { /*printf("call update ep %d\n", epoch);*/ bool set = false; @@ -1516,16 +1472,16 @@ int32_t tmq_seek(tmq_t* tmq, const tmq_topic_vgroup_t* offset) { #endif SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t timeout, SMqClientTopic* pTopic, SMqClientVg* pVg) { - int64_t reqOffset; - if (pVg->currentOffset >= 0) { - reqOffset = pVg->currentOffset; - } else { - /*if (tmq->resetOffsetCfg == TMQ_CONF__RESET_OFFSET__NONE) {*/ - /*tscError("unable to poll since no committed offset but reset offset is set to none");*/ - /*return NULL;*/ - /*}*/ - reqOffset = tmq->resetOffsetCfg; - } + /*int64_t reqOffset;*/ + /*if (pVg->currentOffset >= 0) {*/ + /*reqOffset = pVg->currentOffset;*/ + /*} else {*/ + /*if (tmq->resetOffsetCfg == TMQ_CONF__RESET_OFFSET__NONE) {*/ + /*tscError("unable to poll since no committed offset but reset offset is set to none");*/ + /*return NULL;*/ + /*}*/ + /*reqOffset = tmq->resetOffsetCfg;*/ + /*}*/ SMqPollReq* pReq = taosMemoryCalloc(1, sizeof(SMqPollReq)); if (pReq == NULL) { @@ -1544,7 +1500,8 @@ SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t timeout, SMqClientTopic* pReq->timeout = timeout; pReq->consumerId = tmq->consumerId; pReq->epoch = tmq->epoch; - pReq->currentOffset = reqOffset; + /*pReq->currentOffset = reqOffset;*/ + pReq->reqOffset = pVg->currentOffsetNew; pReq->reqId = generateRequestId(); pReq->useSnapshot = tmq->useSnapshot; @@ -1572,7 +1529,7 @@ SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper) { tstrncpy(pRspObj->db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN); pRspObj->vgId = pWrapper->vgHandle->vgId; pRspObj->resIter = -1; - memcpy(&pRspObj->rsp, &pWrapper->dataRsp, sizeof(SMqDataBlkRsp)); + memcpy(&pRspObj->rsp, &pWrapper->dataRsp, sizeof(SMqDataRsp)); pRspObj->resInfo.totalRows = 0; pRspObj->resInfo.precision = TSDB_TIME_PRECISION_MILLI; @@ -1645,8 +1602,11 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { int64_t transporterId = 0; /*printf("send poll\n");*/ - tscDebug("consumer %ld send poll to %s : vg %d, epoch %d, req offset %ld, reqId %lu", tmq->consumerId, - pTopic->topicName, pVg->vgId, tmq->epoch, pVg->currentOffset, pReq->reqId); + + char offsetFormatBuf[50]; + tFormatOffset(offsetFormatBuf, 50, &pVg->currentOffsetNew); + tscDebug("consumer %ld send poll to %s : vg %d, epoch %d, req offset %s, reqId %lu", tmq->consumerId, + pTopic->topicName, pVg->vgId, tmq->epoch, offsetFormatBuf, pReq->reqId); /*printf("send vg %d %ld\n", pVg->vgId, pVg->currentOffset);*/ asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo); pVg->pollCnt++; @@ -1695,7 +1655,7 @@ void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { if (pollRspWrapper->dataRsp.head.epoch == consumerEpoch) { SMqClientVg* pVg = pollRspWrapper->vgHandle; /*printf("vg %d offset %ld up to %ld\n", pVg->vgId, pVg->currentOffset, rspMsg->msg.rspOffset);*/ - pVg->currentOffset = pollRspWrapper->dataRsp.rspOffset; + pVg->currentOffsetNew = pollRspWrapper->dataRsp.rspOffset; atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); if (pollRspWrapper->dataRsp.blockNum == 0) { taosFreeQitem(pollRspWrapper); @@ -1717,7 +1677,7 @@ void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { if (pollRspWrapper->metaRsp.head.epoch == consumerEpoch) { SMqClientVg* pVg = pollRspWrapper->vgHandle; /*printf("vg %d offset %ld up to %ld\n", pVg->vgId, pVg->currentOffset, rspMsg->msg.rspOffset);*/ - pVg->currentOffset = pollRspWrapper->metaRsp.rspOffset; + pVg->currentOffsetNew = pollRspWrapper->metaRsp.rspOffsetNew; atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); // build rsp SMqMetaRspObj* pRsp = tmqBuildMetaRspFromWrapper(pollRspWrapper); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index e9b5c67d76..8c1daae3fa 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -2300,7 +2300,6 @@ int32_t tDeserializeSServerVerRsp(void *buf, int32_t bufLen, SServerVerRsp *pRsp return 0; } - int32_t tSerializeSQnodeListRsp(void *buf, int32_t bufLen, SQnodeListRsp *pRsp) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -2387,7 +2386,6 @@ int32_t tDeserializeSDnodeListRsp(void *buf, int32_t bufLen, SDnodeListRsp *pRsp void tFreeSDnodeListRsp(SDnodeListRsp *pRsp) { taosArrayDestroy(pRsp->dnodeList); } - int32_t tSerializeSCompactDbReq(void *buf, int32_t bufLen, SCompactDbReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -2909,20 +2907,19 @@ int32_t tDeserializeSShowVariablesReq(void *buf, int32_t bufLen, SShowVariablesR return 0; } -int32_t tEncodeSVariablesInfo(SEncoder* pEncoder, SVariablesInfo* pInfo) { +int32_t tEncodeSVariablesInfo(SEncoder *pEncoder, SVariablesInfo *pInfo) { if (tEncodeCStr(pEncoder, pInfo->name) < 0) return -1; if (tEncodeCStr(pEncoder, pInfo->value) < 0) return -1; return 0; } -int32_t tDecodeSVariablesInfo(SDecoder* pDecoder, SVariablesInfo* pInfo) { +int32_t tDecodeSVariablesInfo(SDecoder *pDecoder, SVariablesInfo *pInfo) { if (tDecodeCStrTo(pDecoder, pInfo->name) < 0) return -1; if (tDecodeCStrTo(pDecoder, pInfo->value) < 0) return -1; return 0; } - -int32_t tSerializeSShowVariablesRsp(void* buf, int32_t bufLen, SShowVariablesRsp* pRsp) { +int32_t tSerializeSShowVariablesRsp(void *buf, int32_t bufLen, SShowVariablesRsp *pRsp) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -2930,7 +2927,7 @@ int32_t tSerializeSShowVariablesRsp(void* buf, int32_t bufLen, SShowVariablesRsp int32_t varNum = taosArrayGetSize(pRsp->variables); if (tEncodeI32(&encoder, varNum) < 0) return -1; for (int32_t i = 0; i < varNum; ++i) { - SVariablesInfo* pInfo = taosArrayGet(pRsp->variables, i); + SVariablesInfo *pInfo = taosArrayGet(pRsp->variables, i); if (tEncodeSVariablesInfo(&encoder, pInfo) < 0) return -1; } tEndEncode(&encoder); @@ -2940,7 +2937,7 @@ int32_t tSerializeSShowVariablesRsp(void* buf, int32_t bufLen, SShowVariablesRsp return tlen; } -int32_t tDeserializeSShowVariablesRsp(void* buf, int32_t bufLen, SShowVariablesRsp* pRsp) { +int32_t tDeserializeSShowVariablesRsp(void *buf, int32_t bufLen, SShowVariablesRsp *pRsp) { SDecoder decoder = {0}; tDecoderInit(&decoder, buf, bufLen); @@ -2962,11 +2959,11 @@ int32_t tDeserializeSShowVariablesRsp(void* buf, int32_t bufLen, SShowVariablesR return 0; } -void tFreeSShowVariablesRsp(SShowVariablesRsp* pRsp) { +void tFreeSShowVariablesRsp(SShowVariablesRsp *pRsp) { if (NULL == pRsp) { return; } - + taosArrayDestroy(pRsp->variables); } @@ -5360,30 +5357,149 @@ void tFreeSMAlterStbRsp(SMAlterStbRsp *pRsp) { } } -int32_t tEncodeSTqOffset(SEncoder *pEncoder, const STqOffset *pOffset) { - if (tEncodeI8(pEncoder, pOffset->type) < 0) return -1; - if (pOffset->type == TMQ_OFFSET__SNAPSHOT) { - if (tEncodeI64(pEncoder, pOffset->uid) < 0) return -1; - if (tEncodeI64(pEncoder, pOffset->ts) < 0) return -1; - } else if (pOffset->type == TMQ_OFFSET__LOG) { - if (tEncodeI64(pEncoder, pOffset->version) < 0) return -1; +int32_t tEncodeSTqOffsetVal(SEncoder *pEncoder, const STqOffsetVal *pOffsetVal) { + if (tEncodeI8(pEncoder, pOffsetVal->type) < 0) return -1; + if (pOffsetVal->type == TMQ_OFFSET__SNAPSHOT_DATA) { + if (tEncodeI64(pEncoder, pOffsetVal->uid) < 0) return -1; + if (tEncodeI64(pEncoder, pOffsetVal->ts) < 0) return -1; + } else if (pOffsetVal->type == TMQ_OFFSET__LOG) { + if (tEncodeI64(pEncoder, pOffsetVal->version) < 0) return -1; + } else if (pOffsetVal->type < 0) { + // do nothing } else { ASSERT(0); } + return 0; +} + +int32_t tDecodeSTqOffsetVal(SDecoder *pDecoder, STqOffsetVal *pOffsetVal) { + if (tDecodeI8(pDecoder, &pOffsetVal->type) < 0) return -1; + if (pOffsetVal->type == TMQ_OFFSET__SNAPSHOT_DATA) { + if (tDecodeI64(pDecoder, &pOffsetVal->uid) < 0) return -1; + if (tDecodeI64(pDecoder, &pOffsetVal->ts) < 0) return -1; + } else if (pOffsetVal->type == TMQ_OFFSET__LOG) { + if (tDecodeI64(pDecoder, &pOffsetVal->version) < 0) return -1; + } else if (pOffsetVal->type < 0) { + // do nothing + } else { + ASSERT(0); + } + return 0; +} + +#if 1 +int32_t tFormatOffset(char *buf, int32_t maxLen, const STqOffsetVal *pVal) { + if (pVal->type == TMQ_OFFSET__RESET_NONE) { + snprintf(buf, maxLen, "offset(reset to none)"); + } else if (pVal->type == TMQ_OFFSET__RESET_EARLIEAST) { + snprintf(buf, maxLen, "offset(reset to earlieast)"); + } else if (pVal->type == TMQ_OFFSET__RESET_LATEST) { + snprintf(buf, maxLen, "offset(reset to latest)"); + } else if (pVal->type == TMQ_OFFSET__LOG) { + snprintf(buf, maxLen, "offset(log) ver:%ld", pVal->version); + } else if (pVal->type == TMQ_OFFSET__SNAPSHOT_DATA) { + snprintf(buf, maxLen, "offset(snapshot data) uid:%ld, ts:%ld", pVal->uid, pVal->ts); + } else if (pVal->type == TMQ_OFFSET__SNAPSHOT_META) { + snprintf(buf, maxLen, "offset(snapshot meta) uid:%ld, ts:%ld", pVal->uid, pVal->ts); + } else { + ASSERT(0); + } + return 0; +} +#endif + +bool tOffsetEqual(const STqOffsetVal *pLeft, const STqOffsetVal *pRight) { + if (pLeft->type == pRight->type) { + if (pLeft->type == TMQ_OFFSET__LOG) { + return pLeft->version == pRight->version; + } else if (pLeft->type == TMQ_OFFSET__SNAPSHOT_DATA) { + return pLeft->uid == pRight->uid && pLeft->ts == pRight->ts; + } else if (pLeft->type == TMQ_OFFSET__SNAPSHOT_META) { + ASSERT(0); + // TODO + return pLeft->uid == pRight->uid && pLeft->ts == pRight->ts; + } + } + return false; +} + +int32_t tEncodeSTqOffset(SEncoder *pEncoder, const STqOffset *pOffset) { + if (tEncodeSTqOffsetVal(pEncoder, &pOffset->val) < 0) return -1; if (tEncodeCStr(pEncoder, pOffset->subKey) < 0) return -1; return 0; } int32_t tDecodeSTqOffset(SDecoder *pDecoder, STqOffset *pOffset) { - if (tDecodeI8(pDecoder, &pOffset->type) < 0) return -1; - if (pOffset->type == TMQ_OFFSET__SNAPSHOT) { - if (tDecodeI64(pDecoder, &pOffset->uid) < 0) return -1; - if (tDecodeI64(pDecoder, &pOffset->ts) < 0) return -1; - } else if (pOffset->type == TMQ_OFFSET__LOG) { - if (tDecodeI64(pDecoder, &pOffset->version) < 0) return -1; - } else { - ASSERT(0); - } + if (tDecodeSTqOffsetVal(pDecoder, &pOffset->val) < 0) return -1; if (tDecodeCStrTo(pDecoder, pOffset->subKey) < 0) return -1; return 0; } + +int32_t tEncodeSMqDataRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) { + if (tEncodeSTqOffsetVal(pEncoder, &pRsp->reqOffset) < 0) return -1; + if (tEncodeSTqOffsetVal(pEncoder, &pRsp->rspOffset) < 0) return -1; + if (tEncodeI32(pEncoder, pRsp->skipLogNum) < 0) return -1; + if (tEncodeI32(pEncoder, pRsp->blockNum) < 0) return -1; + if (pRsp->blockNum != 0) { + if (tEncodeI8(pEncoder, pRsp->withTbName) < 0) return -1; + if (tEncodeI8(pEncoder, pRsp->withSchema) < 0) return -1; + + for (int32_t i = 0; i < pRsp->blockNum; i++) { + int32_t bLen = *(int32_t *)taosArrayGet(pRsp->blockDataLen, i); + void *data = taosArrayGetP(pRsp->blockData, i); + if (tEncodeBinary(pEncoder, (const uint8_t *)data, bLen) < 0) return -1; + if (pRsp->withSchema) { + SSchemaWrapper *pSW = (SSchemaWrapper *)taosArrayGetP(pRsp->blockSchema, i); + if (tEncodeSSchemaWrapper(pEncoder, pSW) < 0) return -1; + } + if (pRsp->withTbName) { + char *tbName = (char *)taosArrayGetP(pRsp->blockTbName, i); + if (tEncodeCStr(pEncoder, tbName) < 0) return -1; + } + } + } + return 0; +} + +int32_t tDecodeSMqDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) { + if (tDecodeSTqOffsetVal(pDecoder, &pRsp->reqOffset) < 0) return -1; + if (tDecodeSTqOffsetVal(pDecoder, &pRsp->rspOffset) < 0) return -1; + if (tDecodeI32(pDecoder, &pRsp->skipLogNum) < 0) return -1; + if (tDecodeI32(pDecoder, &pRsp->blockNum) < 0) return -1; + if (pRsp->blockNum != 0) { + pRsp->blockData = taosArrayInit(pRsp->blockNum, sizeof(void *)); + pRsp->blockDataLen = taosArrayInit(pRsp->blockNum, sizeof(int32_t)); + if (tDecodeI8(pDecoder, &pRsp->withTbName) < 0) return -1; + if (tDecodeI8(pDecoder, &pRsp->withSchema) < 0) return -1; + if (pRsp->withTbName) { + pRsp->blockTbName = taosArrayInit(pRsp->blockNum, sizeof(void *)); + } + if (pRsp->withSchema) { + pRsp->blockSchema = taosArrayInit(pRsp->blockNum, sizeof(void *)); + } + + for (int32_t i = 0; i < pRsp->blockNum; i++) { + void *data; + uint64_t bLen; + if (tDecodeBinaryAlloc(pDecoder, &data, &bLen) < 0) return -1; + taosArrayPush(pRsp->blockData, &data); + int32_t len = bLen; + taosArrayPush(pRsp->blockDataLen, &len); + + if (pRsp->withSchema) { + SSchemaWrapper *pSW = (SSchemaWrapper *)taosMemoryCalloc(1, sizeof(SSchemaWrapper)); + if (pSW == NULL) return -1; + if (tDecodeSSchemaWrapper(pDecoder, pSW) < 0) return -1; + taosArrayPush(pRsp->blockSchema, &pSW); + } + + if (pRsp->withTbName) { + char *tbName; + if (tDecodeCStrAlloc(pDecoder, &tbName) < 0) return -1; + taosArrayPush(pRsp->blockTbName, &tbName); + } + } + } + return 0; +} + diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 5c2d2cd712..cef0bbbc01 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -139,19 +139,19 @@ void tsdbCleanupReadHandle(tsdbReaderT queryHandle); // tq -typedef struct STqReadHandle STqReadHandle; +typedef struct STqReadHandle SStreamReader; -STqReadHandle *tqInitSubmitMsgScanner(SMeta *pMeta); +SStreamReader *tqInitSubmitMsgScanner(SMeta *pMeta); -void tqReadHandleSetColIdList(STqReadHandle *pReadHandle, SArray *pColIdList); -int32_t tqReadHandleSetTbUidList(STqReadHandle *pHandle, const SArray *tbUidList); -int32_t tqReadHandleAddTbUidList(STqReadHandle *pHandle, const SArray *tbUidList); -int32_t tqReadHandleRemoveTbUidList(STqReadHandle *pHandle, const SArray *tbUidList); +void tqReadHandleSetColIdList(SStreamReader *pReadHandle, SArray *pColIdList); +int32_t tqReadHandleSetTbUidList(SStreamReader *pHandle, const SArray *tbUidList); +int32_t tqReadHandleAddTbUidList(SStreamReader *pHandle, const SArray *tbUidList); +int32_t tqReadHandleRemoveTbUidList(SStreamReader *pHandle, const SArray *tbUidList); -int32_t tqReadHandleSetMsg(STqReadHandle *pHandle, SSubmitReq *pMsg, int64_t ver); -bool tqNextDataBlock(STqReadHandle *pHandle); -bool tqNextDataBlockFilterOut(STqReadHandle *pHandle, SHashObj *filterOutUids); -int32_t tqRetrieveDataBlock(SSDataBlock *pBlock, STqReadHandle *pHandle); +int32_t tqReadHandleSetMsg(SStreamReader *pHandle, SSubmitReq *pMsg, int64_t ver); +bool tqNextDataBlock(SStreamReader *pHandle); +bool tqNextDataBlockFilterOut(SStreamReader *pHandle, SHashObj *filterOutUids); +int32_t tqRetrieveDataBlock(SSDataBlock *pBlock, SStreamReader *pHandle); // sma int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days); diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 59c3e95b9c..38a4f28041 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -39,6 +39,16 @@ extern "C" { #define tqInfo(...) do { if (tqDebugFlag & DEBUG_INFO) { taosPrintLog("TQ ", DEBUG_INFO, 255, __VA_ARGS__); }} while(0) #define tqDebug(...) do { if (tqDebugFlag & DEBUG_DEBUG) { taosPrintLog("TQ ", DEBUG_DEBUG, tqDebugFlag, __VA_ARGS__); }} while(0) #define tqTrace(...) do { if (tqDebugFlag & DEBUG_TRACE) { taosPrintLog("TQ ", DEBUG_TRACE, tqDebugFlag, __VA_ARGS__); }} while(0) + +#define IS_META_MSG(x) ( \ + x == TDMT_VND_CREATE_STB \ + || x == TDMT_VND_ALTER_STB \ + || x == TDMT_VND_DROP_STB \ + || x == TDMT_VND_CREATE_TABLE \ + || x == TDMT_VND_ALTER_TABLE \ + || x == TDMT_VND_DROP_TABLE \ + || x == TDMT_VND_DROP_TTL_TABLE \ +) // clang-format on typedef struct STqOffsetStore STqOffsetStore; @@ -101,12 +111,13 @@ typedef struct { typedef struct { int8_t subType; - STqReadHandle* pExecReader[5]; + SStreamReader* pExecReader[5]; union { STqExecCol execCol; STqExecTb execTb; STqExecDb execDb; }; + } STqExecHandle; typedef struct { @@ -149,9 +160,9 @@ static STqMgmt tqMgmt = {0}; int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalHead** pHeadWithCkSum); // tqExec -int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkRsp* pRsp, int32_t workerId); -int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataBlkRsp* pRsp, int32_t workerId); -int32_t tqSendPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataBlkRsp* pRsp); +int32_t tqLogScanExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataRsp* pRsp, int32_t workerId); +int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, int32_t workerId); +int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp); // tqMeta int32_t tqMetaOpen(STQ* pTq); diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index ed5b6f4055..e1e195f14c 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -306,7 +306,7 @@ int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con return TSDB_CODE_FAILED; } - STqReadHandle *pReadHandle = tqInitSubmitMsgScanner(pMeta); + SStreamReader *pReadHandle = tqInitSubmitMsgScanner(pMeta); if (!pReadHandle) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _err; @@ -590,8 +590,8 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { tdRefSmaStat(pSma, (SSmaStat *)pStat); SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL}; - qSetStreamInput(pItem->taskInfo, &dataBlock, STREAM_DATA_TYPE_SSDATA_BLOCK, false); - tdFetchAndSubmitRSmaResult(pItem, STREAM_DATA_TYPE_SSDATA_BLOCK); + qSetStreamInput(pItem->taskInfo, &dataBlock, STREAM_INPUT__DATA_BLOCK, false); + tdFetchAndSubmitRSmaResult(pItem, STREAM_INPUT__DATA_BLOCK); tdUnRefSmaStat(pSma, (SSmaStat *)pStat); @@ -611,12 +611,12 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType smaDebug("vgId:%d, execute rsma %" PRIi8 " task for qTaskInfo:%p suid:%" PRIu64, SMA_VID(pSma), level, pItem->taskInfo, suid); - if (qSetStreamInput(pItem->taskInfo, pMsg, inputType, true) < 0) { // STREAM_DATA_TYPE_SUBMIT_BLOCK + if (qSetStreamInput(pItem->taskInfo, pMsg, inputType, true) < 0) { // INPUT__DATA_SUBMIT smaError("vgId:%d, rsma % " PRIi8 " qSetStreamInput failed since %s", SMA_VID(pSma), level, tstrerror(terrno)); return TSDB_CODE_FAILED; } - tdFetchAndSubmitRSmaResult(pItem, STREAM_DATA_TYPE_SUBMIT_BLOCK); + tdFetchAndSubmitRSmaResult(pItem, STREAM_INPUT__DATA_SUBMIT); atomic_store_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE); smaDebug("vgId:%d, process rsma insert", SMA_VID(pSma)); @@ -654,7 +654,7 @@ static int32_t tdExecuteRSma(SSma *pSma, const void *pMsg, int32_t inputType, tb return TSDB_CODE_SUCCESS; } - if (inputType == STREAM_DATA_TYPE_SUBMIT_BLOCK) { + if (inputType == STREAM_INPUT__DATA_SUBMIT) { tdExecuteRSmaImpl(pSma, pMsg, inputType, &pRSmaInfo->items[0], suid, TSDB_RETENTION_L1); tdExecuteRSmaImpl(pSma, pMsg, inputType, &pRSmaInfo->items[1], suid, TSDB_RETENTION_L2); } @@ -675,7 +675,7 @@ int32_t tdProcessRSmaSubmit(SSma *pSma, void *pMsg, int32_t inputType) { return TSDB_CODE_SUCCESS; } - if (inputType == STREAM_DATA_TYPE_SUBMIT_BLOCK) { + if (inputType == STREAM_INPUT__DATA_SUBMIT) { STbUidStore uidStore = {0}; tdFetchSubmitReqSuids(pMsg, &uidStore); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 766cf7af35..f36998d606 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -113,8 +113,23 @@ int32_t tqSendMetaPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, return 0; } -int32_t tqSendPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataBlkRsp* pRsp) { - int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqDataBlkRsp(NULL, pRsp); +int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp) { + ASSERT(taosArrayGetSize(pRsp->blockData) == pRsp->blockNum); + ASSERT(taosArrayGetSize(pRsp->blockDataLen) == pRsp->blockNum); + + if (pRsp->withSchema) { + ASSERT(taosArrayGetSize(pRsp->blockSchema) == pRsp->blockNum); + } else { + ASSERT(taosArrayGetSize(pRsp->blockSchema) == 0); + } + + int32_t len; + int32_t code; + tEncodeSize(tEncodeSMqDataRsp, pRsp, len, code); + if (code < 0) { + return -1; + } + int32_t tlen = sizeof(SMqRspHead) + len; void* buf = rpcMallocCont(tlen); if (buf == NULL) { return -1; @@ -125,18 +140,26 @@ int32_t tqSendPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, con ((SMqRspHead*)buf)->consumerId = pReq->consumerId; void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); - tEncodeSMqDataBlkRsp(&abuf, pRsp); - SRpcMsg resp = { + SEncoder encoder; + tEncoderInit(&encoder, abuf, tlen); + tEncodeSMqDataRsp(&encoder, pRsp); + /*tEncodeSMqDataBlkRsp(&abuf, pRsp);*/ + + SRpcMsg rsp = { .info = pMsg->info, .pCont = buf, .contLen = tlen, .code = 0, }; - tmsgSendRsp(&resp); + tmsgSendRsp(&rsp); - tqDebug("vg %d from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %ld, rspOffset: %ld", - TD_VID(pTq->pVnode), pReq->consumerId, pReq->epoch, pRsp->blockNum, pRsp->reqOffset, pRsp->rspOffset); + char buf1[50]; + char buf2[50]; + tFormatOffset(buf1, 50, &pRsp->reqOffset); + tFormatOffset(buf2, 50, &pRsp->rspOffset); + tqDebug("vg %d from consumer %ld (epoch %d) send rsp, block num: %d, reqOffset: %s, rspOffset: %s", + TD_VID(pTq->pVnode), pReq->consumerId, pReq->epoch, pRsp->blockNum, buf1, buf2); return 0; } @@ -151,17 +174,17 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, char* msg, int32_t msgLen) { } tDecoderClear(&decoder); - if (offset.type == TMQ_OFFSET__SNAPSHOT) { + if (offset.val.type == TMQ_OFFSET__SNAPSHOT_DATA) { tqDebug("receive offset commit msg to %s on vg %d, offset(type:snapshot) uid: %ld, ts: %ld", offset.subKey, - TD_VID(pTq->pVnode), offset.uid, offset.ts); - } else if (offset.type == TMQ_OFFSET__LOG) { + TD_VID(pTq->pVnode), offset.val.uid, offset.val.ts); + } else if (offset.val.type == TMQ_OFFSET__LOG) { tqDebug("receive offset commit msg to %s on vg %d, offset(type:log) version: %ld", offset.subKey, - TD_VID(pTq->pVnode), offset.version); + TD_VID(pTq->pVnode), offset.val.version); } else { ASSERT(0); } STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, offset.subKey); - if (pOffset == NULL || pOffset->version < offset.version) { + if (pOffset == NULL || pOffset->val.version < offset.val.version) { if (tqOffsetWrite(pTq->pOffsetStore, &offset) < 0) { ASSERT(0); return -1; @@ -171,6 +194,237 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, char* msg, int32_t msgLen) { return 0; } +static int32_t tqInitDataRsp(SMqDataRsp* pRsp, const SMqPollReq* pReq, int8_t subType) { + pRsp->reqOffset = pReq->reqOffset; + + pRsp->blockData = taosArrayInit(0, sizeof(void*)); + pRsp->blockDataLen = taosArrayInit(0, sizeof(int32_t)); + + if (pRsp->blockData == NULL || pRsp->blockDataLen == NULL) { + return -1; + } + + pRsp->withTbName = pReq->withTbName; + if (pRsp->withTbName) { + pRsp->blockTbName = taosArrayInit(0, sizeof(void*)); + if (pRsp->blockTbName == NULL) { + // TODO free + return -1; + } + } + + if (subType == TOPIC_SUB_TYPE__COLUMN) { + pRsp->withSchema = false; + } else { + pRsp->withSchema = true; + pRsp->blockSchema = taosArrayInit(0, sizeof(void*)); + if (pRsp->blockSchema == NULL) { + // TODO free + return -1; + } + } + return 0; +} + +static int32_t tqInitMetaRsp(SMqMetaRsp* pRsp, const SMqPollReq* pReq) { return 0; } + +static FORCE_INLINE void tqOffsetResetToData(STqOffsetVal* pOffsetVal, int64_t uid, int64_t ts) { + pOffsetVal->type = TMQ_OFFSET__SNAPSHOT_DATA; + pOffsetVal->uid = uid; + pOffsetVal->ts = ts; +} + +static FORCE_INLINE void tqOffsetResetToLog(STqOffsetVal* pOffsetVal, int64_t ver) { + pOffsetVal->type = TMQ_OFFSET__LOG; + pOffsetVal->version = ver; +} + +int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { + SMqPollReq* pReq = pMsg->pCont; + int64_t consumerId = pReq->consumerId; + int64_t timeout = pReq->timeout; + int32_t reqEpoch = pReq->epoch; + int32_t code = 0; + STqOffsetVal reqOffset = pReq->reqOffset; + STqOffsetVal fetchOffsetNew; + + // 1.find handle + char buf[50]; + tFormatOffset(buf, 50, &reqOffset); + tqDebug("tmq poll: consumer %ld (epoch %d) recv poll req in vg %d, req offset %s", consumerId, pReq->epoch, + TD_VID(pTq->pVnode), buf); + + STqHandle* pHandle = taosHashGet(pTq->handles, pReq->subKey, strlen(pReq->subKey)); + /*ASSERT(pHandle);*/ + if (pHandle == NULL) { + tqError("tmq poll: no consumer handle for consumer %ld in vg %d, subkey %s", consumerId, TD_VID(pTq->pVnode), + pReq->subKey); + return -1; + } + + // check rebalance + if (pHandle->consumerId != consumerId) { + tqError("tmq poll: consumer handle mismatch for consumer %ld in vg %d, subkey %s, handle consumer id %ld", + consumerId, TD_VID(pTq->pVnode), pReq->subKey, pHandle->consumerId); + return -1; + } + + // update epoch if need + int32_t consumerEpoch = atomic_load_32(&pHandle->epoch); + while (consumerEpoch < reqEpoch) { + consumerEpoch = atomic_val_compare_exchange_32(&pHandle->epoch, consumerEpoch, reqEpoch); + } + + // 2.reset offset if needed + if (reqOffset.type > 0) { + fetchOffsetNew = reqOffset; + } else { + STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, pReq->subKey); + if (pOffset != NULL) { + fetchOffsetNew = pOffset->val; + char formatBuf[50]; + tFormatOffset(formatBuf, 50, &fetchOffsetNew); + tqDebug("tmq poll: consumer %ld, offset reset to %s", consumerId, formatBuf); + } else { + if (reqOffset.type == TMQ_OFFSET__RESET_EARLIEAST) { + if (pReq->useSnapshot) { + if (!pHandle->fetchMeta) { + tqOffsetResetToData(&fetchOffsetNew, 0, 0); + } else { + // reset to meta + ASSERT(0); + } + } else { + tqOffsetResetToLog(&fetchOffsetNew, walGetFirstVer(pTq->pVnode->pWal)); + } + } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) { + tqOffsetResetToLog(&fetchOffsetNew, walGetLastVer(pTq->pVnode->pWal)); + } else if (reqOffset.type == TMQ_OFFSET__RESET_NONE) { + tqError("tmq poll: no offset committed for consumer %ld in vg %d, subkey %s, reset none failed", consumerId, + TD_VID(pTq->pVnode), pReq->subKey); + terrno = TSDB_CODE_TQ_NO_COMMITTED_OFFSET; + return -1; + } + } + } + + // 3.query + SMqDataRsp dataRsp = {0}; + tqInitDataRsp(&dataRsp, pReq, pHandle->execHandle.subType); + + if (fetchOffsetNew.type == TMQ_OFFSET__LOG) { + int64_t fetchVer = fetchOffsetNew.version + 1; + SWalHead* pHeadWithCkSum = taosMemoryMalloc(sizeof(SWalHead) + 2048); + if (pHeadWithCkSum == NULL) { + return -1; + } + + walSetReaderCapacity(pHandle->pWalReader, 2048); + + while (1) { + consumerEpoch = atomic_load_32(&pHandle->epoch); + if (consumerEpoch > reqEpoch) { + tqWarn("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, found new consumer epoch %d, discard req epoch %d", + consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchVer, consumerEpoch, reqEpoch); + break; + } + + if (tqFetchLog(pTq, pHandle, &fetchVer, &pHeadWithCkSum) < 0) { + // TODO add push mgr + + tqOffsetResetToLog(&dataRsp.rspOffset, fetchVer); + if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) { + code = -1; + } + goto OVER; + } + + SWalReadHead* pHead = &pHeadWithCkSum->head; + + tqDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d offset %ld msgType %d", consumerId, pReq->epoch, + TD_VID(pTq->pVnode), fetchVer, pHead->msgType); + + if (pHead->msgType == TDMT_VND_SUBMIT) { + SSubmitReq* pCont = (SSubmitReq*)&pHead->body; + + if (tqLogScanExec(pTq, &pHandle->execHandle, pCont, &dataRsp, workerId) < 0) { + ASSERT(0); + } + // TODO batch optimization: + // TODO continue scan until meeting batch requirement + if (dataRsp.blockNum > 0 /* threshold */) { + tqOffsetResetToLog(&dataRsp.rspOffset, fetchVer); + + if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) { + code = -1; + } + + } else { + fetchVer++; + } + goto OVER; + + } else { + ASSERT(pHandle->fetchMeta); + ASSERT(IS_META_MSG(pHead->msgType)); + tqInfo("fetch meta msg, ver: %ld, type: %d", pHead->version, pHead->msgType); + SMqMetaRsp metaRsp = {0}; + metaRsp.reqOffset = pReq->reqOffset.version; + /*tqOffsetResetToLog(&metaR)*/ + metaRsp.rspOffset = fetchVer; + metaRsp.resMsgType = pHead->msgType; + metaRsp.metaRspLen = pHead->bodyLen; + metaRsp.metaRsp = pHead->body; + if (tqSendMetaPollRsp(pTq, pMsg, pReq, &metaRsp) < 0) { + code = -1; + goto OVER; + } + code = 0; + goto OVER; + } + } + + taosMemoryFree(pHeadWithCkSum); + } else if (fetchOffsetNew.type == TMQ_OFFSET__SNAPSHOT_DATA) { + // 1. set uid and ts + // 2. get data (rebuild reader if needed) + // 3. get new uid and ts + + char formatBuf[50]; + tFormatOffset(formatBuf, 50, &dataRsp.reqOffset); + tqInfo("retrieve using snapshot req offset %s", formatBuf); + if (tqScanSnapshot(pTq, &pHandle->execHandle, &dataRsp, workerId) < 0) { + ASSERT(0); + } + + // 4. send rsp + if (dataRsp.blockNum != 0) { + tqOffsetResetToData(&dataRsp.rspOffset, 0, 0); + if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) { + code = -1; + } + } + } else if (fetchOffsetNew.type == TMQ_OFFSET__SNAPSHOT_META) { + ASSERT(0); + } + +OVER: + // TODO wrap in destroy func + taosArrayDestroy(dataRsp.blockDataLen); + taosArrayDestroyP(dataRsp.blockData, (FDelete)taosMemoryFree); + + if (dataRsp.withSchema) { + taosArrayDestroyP(dataRsp.blockSchema, (FDelete)tDeleteSSchemaWrapper); + } + + if (dataRsp.withTbName) { + taosArrayDestroyP(dataRsp.blockTbName, (FDelete)taosMemoryFree); + } + + return code; +} + +#if 0 int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { SMqPollReq* pReq = pMsg->pCont; int64_t consumerId = pReq->consumerId; @@ -185,10 +439,10 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { } else { STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, pReq->subKey); if (pOffset != NULL) { - ASSERT(pOffset->type == TMQ_OFFSET__LOG); + ASSERT(pOffset->val.type == TMQ_OFFSET__LOG); tqDebug("consumer %ld, restore offset of %s on vg %d, offset(type:log) version: %ld", consumerId, pReq->subKey, - TD_VID(pTq->pVnode), pOffset->version); - fetchOffset = pOffset->version + 1; + TD_VID(pTq->pVnode), pOffset->val.version); + fetchOffset = pOffset->val.version + 1; } else { if (pReq->currentOffset == TMQ_CONF__RESET_OFFSET__EARLIEAST) { fetchOffset = walGetFirstVer(pTq->pWal); @@ -241,12 +495,11 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { if (rsp.withTbName) { rsp.blockTbName = taosArrayInit(0, sizeof(void*)); } + if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { rsp.withSchema = false; - rsp.withTag = false; } else { rsp.withSchema = true; - rsp.withTag = false; rsp.blockSchema = taosArrayInit(0, sizeof(void*)); } @@ -302,10 +555,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { } } else { ASSERT(pHandle->fetchMeta); - ASSERT(pHead->msgType == TDMT_VND_CREATE_STB || pHead->msgType == TDMT_VND_ALTER_STB || - pHead->msgType == TDMT_VND_DROP_STB || pHead->msgType == TDMT_VND_CREATE_TABLE || - pHead->msgType == TDMT_VND_ALTER_TABLE || pHead->msgType == TDMT_VND_DROP_TABLE || - pHead->msgType == TDMT_VND_DROP_TTL_TABLE); + ASSERT(IS_META_MSG(pHead->msgType)); tqInfo("fetch meta msg, ver: %ld, type: %d", pHead->version, pHead->msgType); SMqMetaRsp metaRsp = {0}; metaRsp.reqOffset = pReq->currentOffset; @@ -341,7 +591,7 @@ SEND_RSP: rsp.rspOffset = fetchOffset; - if (tqSendPollRsp(pTq, pMsg, pReq, &rsp) < 0) { + if (tqSendDataRsp(pTq, pMsg, pReq, &rsp) < 0) { code = -1; } OVER: @@ -359,6 +609,7 @@ OVER: return code; } +#endif int32_t tqProcessVgDeleteReq(STQ* pTq, char* msg, int32_t msgLen) { SMqVDeleteReq* pReq = (SMqVDeleteReq*)msg; @@ -403,7 +654,6 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) { .reader = pHandle->execHandle.pExecReader[i], .meta = pTq->pVnode->pMeta, .vnode = pTq->pVnode, -// .initTsdbReader = 1, }; pHandle->execHandle.execCol.task[i] = qCreateStreamExecTaskInfo(pHandle->execHandle.execCol.qmsg, &handle); ASSERT(pHandle->execHandle.execCol.task[i]); @@ -474,12 +724,11 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, char* msg, int32_t msgLen) { if (pTask->execType != TASK_EXEC__NONE) { // expand runners if (pTask->isDataScan) { - STqReadHandle* pStreamReader = tqInitSubmitMsgScanner(pTq->pVnode->pMeta); + SStreamReader* pStreamReader = tqInitSubmitMsgScanner(pTq->pVnode->pMeta); SReadHandle handle = { .reader = pStreamReader, .meta = pTq->pVnode->pMeta, .vnode = pTq->pVnode, -// .initTsdbReader = 1, }; /*pTask->exec.inputHandle = pStreamReader;*/ pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle); diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index c7d7c787a2..eb09199c70 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -15,7 +15,7 @@ #include "tq.h" -static int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataBlkRsp* pRsp) { +static int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp) { int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock); void* buf = taosMemoryCalloc(1, dataStrLen); if (buf == NULL) return -1; @@ -37,7 +37,7 @@ static int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataBlkRsp* pRs return 0; } -static int32_t tqAddBlockSchemaToRsp(const STqExecHandle* pExec, int32_t workerId, SMqDataBlkRsp* pRsp) { +static int32_t tqAddBlockSchemaToRsp(const STqExecHandle* pExec, int32_t workerId, SMqDataRsp* pRsp) { SSchemaWrapper* pSW = tCloneSSchemaWrapper(pExec->pExecReader[workerId]->pSchemaWrapper); if (pSW == NULL) { return -1; @@ -46,10 +46,9 @@ static int32_t tqAddBlockSchemaToRsp(const STqExecHandle* pExec, int32_t workerI return 0; } -static int32_t tqAddTbNameToRsp(const STQ* pTq, const STqExecHandle* pExec, SMqDataBlkRsp* pRsp, int32_t workerId) { +static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, SMqDataRsp* pRsp, int32_t workerId) { SMetaReader mr = {0}; metaReaderInit(&mr, pTq->pVnode->pMeta, 0); - int64_t uid = pExec->pExecReader[workerId]->msgIter.uid; if (metaGetTableEntryByUid(&mr, uid) < 0) { ASSERT(0); return -1; @@ -60,13 +59,13 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, const STqExecHandle* pExec, SMqD return 0; } -int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataBlkRsp* pRsp, int32_t workerId) { +int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataRsp* pRsp, int32_t workerId) { ASSERT(pExec->subType == TOPIC_SUB_TYPE__COLUMN); qTaskInfo_t task = pExec->execCol.task[workerId]; + // TODO set uid and ts if (qStreamScanSnapshot(task) < 0) { ASSERT(0); } - // set version while (1) { SSDataBlock* pDataBlock = NULL; uint64_t ts = 0; @@ -79,17 +78,24 @@ int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataBlkRsp* pRsp ASSERT(taosArrayGetSize(pDataBlock->pDataBlock) != 0); tqAddBlockDataToRsp(pDataBlock, pRsp); + + if (pRsp->withTbName) { + // TODO + pRsp->withTbName = 0; + /*int64_t uid = 0;*/ + /*tqAddTbNameToRsp(pTq, uid, pRsp, workerId);*/ + } pRsp->blockNum++; } return 0; } -int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkRsp* pRsp, int32_t workerId) { +int32_t tqLogScanExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataRsp* pRsp, int32_t workerId) { if (pExec->subType == TOPIC_SUB_TYPE__COLUMN) { qTaskInfo_t task = pExec->execCol.task[workerId]; ASSERT(task); - qSetStreamInput(task, pReq, STREAM_DATA_TYPE_SUBMIT_BLOCK, false); + qSetStreamInput(task, pReq, STREAM_INPUT__DATA_SUBMIT, false); while (1) { SSDataBlock* pDataBlock = NULL; uint64_t ts = 0; @@ -102,13 +108,14 @@ int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkR tqAddBlockDataToRsp(pDataBlock, pRsp); if (pRsp->withTbName) { - tqAddTbNameToRsp(pTq, pExec, pRsp, workerId); + int64_t uid = pExec->pExecReader[workerId]->msgIter.uid; + tqAddTbNameToRsp(pTq, uid, pRsp, workerId); } pRsp->blockNum++; } } else if (pExec->subType == TOPIC_SUB_TYPE__TABLE) { pRsp->withSchema = 1; - STqReadHandle* pReader = pExec->pExecReader[workerId]; + SStreamReader* pReader = pExec->pExecReader[workerId]; tqReadHandleSetMsg(pReader, pReq, 0); while (tqNextDataBlock(pReader)) { SSDataBlock block = {0}; @@ -118,14 +125,15 @@ int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkR } tqAddBlockDataToRsp(&block, pRsp); if (pRsp->withTbName) { - tqAddTbNameToRsp(pTq, pExec, pRsp, workerId); + int64_t uid = pExec->pExecReader[workerId]->msgIter.uid; + tqAddTbNameToRsp(pTq, uid, pRsp, workerId); } tqAddBlockSchemaToRsp(pExec, workerId, pRsp); pRsp->blockNum++; } } else if (pExec->subType == TOPIC_SUB_TYPE__DB) { pRsp->withSchema = 1; - STqReadHandle* pReader = pExec->pExecReader[workerId]; + SStreamReader* pReader = pExec->pExecReader[workerId]; tqReadHandleSetMsg(pReader, pReq, 0); while (tqNextDataBlockFilterOut(pReader, pExec->execDb.pFilterOutTbUid)) { SSDataBlock block = {0}; @@ -135,7 +143,8 @@ int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkR } tqAddBlockDataToRsp(&block, pRsp); if (pRsp->withTbName) { - tqAddTbNameToRsp(pTq, pExec, pRsp, workerId); + int64_t uid = pExec->pExecReader[workerId]->msgIter.uid; + tqAddTbNameToRsp(pTq, uid, pRsp, workerId); } tqAddBlockSchemaToRsp(pExec, workerId, pRsp); pRsp->blockNum++; diff --git a/source/dnode/vnode/src/tq/tqOffset.c b/source/dnode/vnode/src/tq/tqOffset.c index e536e87032..16b8872098 100644 --- a/source/dnode/vnode/src/tq/tqOffset.c +++ b/source/dnode/vnode/src/tq/tqOffset.c @@ -92,8 +92,8 @@ STqOffset* tqOffsetRead(STqOffsetStore* pStore, const char* subscribeKey) { } int32_t tqOffsetWrite(STqOffsetStore* pStore, const STqOffset* pOffset) { - ASSERT(pOffset->type == TMQ_OFFSET__LOG); - ASSERT(pOffset->version >= 0); + /*ASSERT(pOffset->val.type == TMQ_OFFSET__LOG);*/ + /*ASSERT(pOffset->val.version >= 0);*/ return taosHashPut(pStore->pHash, pOffset->subKey, strlen(pOffset->subKey), pOffset, sizeof(STqOffset)); } diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index 2ec627bd5c..bd7cda4de3 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -15,16 +15,17 @@ #include "tq.h" +#if 0 void tqTmrRspFunc(void* param, void* tmrId) { STqHandle* pHandle = (STqHandle*)param; atomic_store_8(&pHandle->pushHandle.tmrStopped, 1); } -static int32_t tqLoopExecFromQueue(STQ* pTq, STqHandle* pHandle, SStreamDataSubmit** ppSubmit, SMqDataBlkRsp* pRsp) { +static int32_t tqLoopExecFromQueue(STQ* pTq, STqHandle* pHandle, SStreamDataSubmit** ppSubmit, SMqDataRsp* pRsp) { SStreamDataSubmit* pSubmit = *ppSubmit; while (pSubmit != NULL) { ASSERT(pSubmit->ver == pHandle->pushHandle.processedVer + 1); - if (tqDataExec(pTq, &pHandle->execHandle, pSubmit->data, pRsp, 0) < 0) { + if (tqLogScanExec(pTq, &pHandle->execHandle, pSubmit->data, pRsp, 0) < 0) { /*ASSERT(0);*/ } // update processed @@ -43,7 +44,7 @@ static int32_t tqLoopExecFromQueue(STQ* pTq, STqHandle* pHandle, SStreamDataSubm } int32_t tqExecFromInputQ(STQ* pTq, STqHandle* pHandle) { - SMqDataBlkRsp rsp = {0}; + SMqDataRsp rsp = {0}; // 1. guard and set status executing int8_t execStatus = atomic_val_compare_exchange_8(&pHandle->pushHandle.execStatus, TASK_EXEC_STATUS__IDLE, TASK_EXEC_STATUS__EXECUTING); @@ -175,13 +176,13 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_ taosWLockLatch(&pHandle->pushHandle.lock); - SMqDataBlkRsp rsp = {0}; + SMqDataRsp rsp = {0}; rsp.reqOffset = pHandle->pushHandle.reqOffset; rsp.blockData = taosArrayInit(0, sizeof(void*)); rsp.blockDataLen = taosArrayInit(0, sizeof(int32_t)); if (msgType == TDMT_VND_SUBMIT) { - tqDataExec(pTq, &pHandle->execHandle, pReq, &rsp, workerId); + tqLogScanExec(pTq, &pHandle->execHandle, pReq, &rsp, workerId); } else { // TODO ASSERT(0); @@ -233,6 +234,7 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_ return 0; } +#endif int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) { if (msgType == TDMT_VND_SUBMIT) { diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index be2a35c901..7d7b9636df 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -44,10 +44,7 @@ int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalHead* } else { if (pHandle->fetchMeta) { SWalReadHead* pHead = &((*ppHeadWithCkSum)->head); - if (pHead->msgType == TDMT_VND_CREATE_STB || pHead->msgType == TDMT_VND_ALTER_STB || - pHead->msgType == TDMT_VND_DROP_STB || pHead->msgType == TDMT_VND_CREATE_TABLE || - pHead->msgType == TDMT_VND_ALTER_TABLE || pHead->msgType == TDMT_VND_DROP_TABLE || - pHead->msgType == TDMT_VND_DROP_TTL_TABLE) { + if (IS_META_MSG(pHead->msgType)) { code = walFetchBody(pHandle->pWalReader, ppHeadWithCkSum); if (code < 0) { @@ -76,8 +73,8 @@ END: return code; } -STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta) { - STqReadHandle* pReadHandle = taosMemoryMalloc(sizeof(STqReadHandle)); +SStreamReader* tqInitSubmitMsgScanner(SMeta* pMeta) { + SStreamReader* pReadHandle = taosMemoryMalloc(sizeof(SStreamReader)); if (pReadHandle == NULL) { return NULL; } @@ -85,15 +82,15 @@ STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta) { pReadHandle->pMsg = NULL; pReadHandle->ver = -1; pReadHandle->pColIdList = NULL; - pReadHandle->cachedSchemaVer = -1; - pReadHandle->cachedSchemaSuid = -1; + pReadHandle->cachedSchemaVer = 0; + pReadHandle->cachedSchemaSuid = 0; pReadHandle->pSchema = NULL; pReadHandle->pSchemaWrapper = NULL; pReadHandle->tbIdHash = NULL; return pReadHandle; } -int32_t tqReadHandleSetMsg(STqReadHandle* pReadHandle, SSubmitReq* pMsg, int64_t ver) { +int32_t tqReadHandleSetMsg(SStreamReader* pReadHandle, SSubmitReq* pMsg, int64_t ver) { pReadHandle->pMsg = pMsg; if (tInitSubmitMsgIter(pMsg, &pReadHandle->msgIter) < 0) return -1; @@ -108,7 +105,7 @@ int32_t tqReadHandleSetMsg(STqReadHandle* pReadHandle, SSubmitReq* pMsg, int64_t return 0; } -bool tqNextDataBlock(STqReadHandle* pHandle) { +bool tqNextDataBlock(SStreamReader* pHandle) { if (pHandle->pMsg == NULL) return false; while (1) { if (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) < 0) { @@ -130,7 +127,7 @@ bool tqNextDataBlock(STqReadHandle* pHandle) { return false; } -bool tqNextDataBlockFilterOut(STqReadHandle* pHandle, SHashObj* filterOutUids) { +bool tqNextDataBlockFilterOut(SStreamReader* pHandle, SHashObj* filterOutUids) { while (1) { if (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) < 0) { return false; @@ -146,7 +143,7 @@ bool tqNextDataBlockFilterOut(STqReadHandle* pHandle, SHashObj* filterOutUids) { return false; } -int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle* pHandle) { +int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, SStreamReader* pHandle) { // TODO: cache multiple schema int32_t sversion = htonl(pHandle->pBlock->sversion); if (pHandle->cachedSchemaSuid == 0 || pHandle->cachedSchemaVer != sversion || @@ -256,9 +253,9 @@ FAIL: return -1; } -void tqReadHandleSetColIdList(STqReadHandle* pReadHandle, SArray* pColIdList) { pReadHandle->pColIdList = pColIdList; } +void tqReadHandleSetColIdList(SStreamReader* pReadHandle, SArray* pColIdList) { pReadHandle->pColIdList = pColIdList; } -int tqReadHandleSetTbUidList(STqReadHandle* pHandle, const SArray* tbUidList) { +int tqReadHandleSetTbUidList(SStreamReader* pHandle, const SArray* tbUidList) { if (pHandle->tbIdHash) { taosHashClear(pHandle->tbIdHash); } @@ -277,7 +274,7 @@ int tqReadHandleSetTbUidList(STqReadHandle* pHandle, const SArray* tbUidList) { return 0; } -int tqReadHandleAddTbUidList(STqReadHandle* pHandle, const SArray* tbUidList) { +int tqReadHandleAddTbUidList(SStreamReader* pHandle, const SArray* tbUidList) { if (pHandle->tbIdHash == NULL) { pHandle->tbIdHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); if (pHandle->tbIdHash == NULL) { @@ -294,7 +291,7 @@ int tqReadHandleAddTbUidList(STqReadHandle* pHandle, const SArray* tbUidList) { return 0; } -int tqReadHandleRemoveTbUidList(STqReadHandle* pHandle, const SArray* tbUidList) { +int tqReadHandleRemoveTbUidList(SStreamReader* pHandle, const SArray* tbUidList) { ASSERT(pHandle->tbIdHash != NULL); for (int32_t i = 0; i < taosArrayGetSize(tbUidList); i++) { diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 72c766e0ae..d27fcb4f03 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -780,7 +780,7 @@ _exit: // TODO: the partial success scenario and the error case // TODO: refactor if ((terrno == TSDB_CODE_SUCCESS) && (pRsp->code == TSDB_CODE_SUCCESS)) { - tdProcessRSmaSubmit(pVnode->pSma, pReq, STREAM_DATA_TYPE_SUBMIT_BLOCK); + tdProcessRSmaSubmit(pVnode->pSma, pReq, STREAM_INPUT__DATA_SUBMIT); } return 0; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 6de364e63a..31edc46b4d 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -44,12 +44,12 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu // prevent setting a different type of block pInfo->blockType = type; - if (type == STREAM_DATA_TYPE_SUBMIT_BLOCK) { + if (type == STREAM_INPUT__DATA_SUBMIT) { if (tqReadHandleSetMsg(pInfo->streamBlockReader, input, 0) < 0) { qError("submit msg messed up when initing stream block, %s" PRIx64, id); return TSDB_CODE_QRY_APP_ERROR; } - } else if (type == STREAM_DATA_TYPE_SSDATA_BLOCK) { + } else if (type == STREAM_INPUT__DATA_BLOCK) { for (int32_t i = 0; i < numOfBlocks; ++i) { SSDataBlock* pDataBlock = &((SSDataBlock*)input)[i]; @@ -60,9 +60,9 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu taosArrayAddAll(p->pDataBlock, pDataBlock->pDataBlock); taosArrayPush(pInfo->pBlockLists, &p); } - } else if (type == STREAM_DATA_TYPE_FROM_SNAPSHOT) { + } else if (type == STREAM_INPUT__DATA_SCAN) { // do nothing - ASSERT(pInfo->blockType == STREAM_DATA_TYPE_FROM_SNAPSHOT); + ASSERT(pInfo->blockType == STREAM_INPUT__DATA_SCAN); } else { ASSERT(0); } @@ -76,7 +76,7 @@ int32_t qStreamScanSnapshot(qTaskInfo_t tinfo) { return TSDB_CODE_QRY_APP_ERROR; } SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - return doSetStreamBlock(pTaskInfo->pRoot, NULL, 0, STREAM_DATA_TYPE_FROM_SNAPSHOT, 0, NULL); + return doSetStreamBlock(pTaskInfo->pRoot, NULL, 0, STREAM_INPUT__DATA_SCAN, 0, NULL); } int32_t qSetStreamInput(qTaskInfo_t tinfo, const void* input, int32_t type, bool assignUid) { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index c51ef44154..8cfdb22313 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -962,7 +962,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { size_t total = taosArrayGetSize(pInfo->pBlockLists); // TODO: refactor - if (pInfo->blockType == STREAM_DATA_TYPE_SSDATA_BLOCK) { + if (pInfo->blockType == STREAM_INPUT__DATA_BLOCK) { if (pInfo->validBlockIndex >= total) { /*doClearBufferedBlocks(pInfo);*/ pOperator->status = OP_EXEC_DONE; @@ -973,7 +973,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { SSDataBlock* pBlock = taosArrayGetP(pInfo->pBlockLists, current); blockDataUpdateTsWindow(pBlock, 0); if (pBlock->info.type == STREAM_RETRIEVE) { - pInfo->blockType = STREAM_DATA_TYPE_SUBMIT_BLOCK; + pInfo->blockType = STREAM_INPUT__DATA_SUBMIT; pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RETRIEVE; copyDataBlock(pInfo->pPullDataRes, pBlock); pInfo->pullDataResIndex = 0; @@ -981,7 +981,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { updateInfoAddCloseWindowSBF(pInfo->pUpdateInfo); } return pBlock; - } else if (pInfo->blockType == STREAM_DATA_TYPE_SUBMIT_BLOCK) { + } else if (pInfo->blockType == STREAM_INPUT__DATA_SUBMIT) { if (pInfo->scanMode == STREAM_SCAN_FROM_RES) { blockDataDestroy(pInfo->pUpdateRes); pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; @@ -1133,7 +1133,9 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { return (pBlockInfo->rows == 0) ? NULL : pInfo->pRes; - } else if (pInfo->blockType == STREAM_DATA_TYPE_FROM_SNAPSHOT) { + } else if (pInfo->blockType == STREAM_INPUT__DATA_SCAN) { + // check reader last status + // if not match, reset status SSDataBlock* pResult = doTableScan(pInfo->pSnapshotReadOp); return pResult && pResult->info.rows > 0 ? pResult : NULL; @@ -1213,7 +1215,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pInfo->tableUid = pScanPhyNode->uid; // set the extract column id to streamHandle - tqReadHandleSetColIdList((STqReadHandle*)pHandle->reader, pColIds); + tqReadHandleSetColIdList((SStreamReader*)pHandle->reader, pColIds); SArray* tableIdList = extractTableIdList(&pTaskInfo->tableqinfoList); int32_t code = tqReadHandleSetTbUidList(pHandle->reader, tableIdList); if (code != 0) { diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index edd1ec0e75..6d287ef887 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -970,9 +970,8 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) { getTableScanInfo(pOperator, &pInfo->order, &scanFlag); if (pInfo->scalarSupp.pExprInfo != NULL) { - SExprSupp* pExprSup =& pInfo->scalarSupp; - projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, - pExprSup->numOfExprs, NULL); + SExprSupp* pExprSup = &pInfo->scalarSupp; + projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); } // the pDataBlock are always the same one, no need to call this again @@ -1512,23 +1511,24 @@ void increaseTs(SqlFunctionCtx* pCtx) { SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, - STimeWindowAggSupp* pTwAggSupp, SIntervalPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, bool isStream) { + STimeWindowAggSupp* pTwAggSupp, SIntervalPhysiNode* pPhyNode, + SExecTaskInfo* pTaskInfo, bool isStream) { SIntervalAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SIntervalAggOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } - pInfo->win = pTaskInfo->window; - pInfo->order = TSDB_ORDER_ASC; - pInfo->interval = *pInterval; + pInfo->win = pTaskInfo->window; + pInfo->order = TSDB_ORDER_ASC; + pInfo->interval = *pInterval; pInfo->execModel = pTaskInfo->execModel; - pInfo->twAggSup = *pTwAggSupp; + pInfo->twAggSup = *pTwAggSupp; if (pPhyNode->window.pExprs != NULL) { int32_t numOfScalar = 0; SExprInfo* pScalarExprInfo = createExprInfo(pPhyNode->window.pExprs, NULL, &numOfScalar); - int32_t code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar); + int32_t code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -2498,10 +2498,10 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { doClearWindows(&pInfo->aggSup, pSup, &pInfo->interval, pInfo->primaryTsIndex, pOperator->exprSupp.numOfExprs, pBlock, pUpWins); if (IS_FINAL_OP(pInfo)) { - int32_t childIndex = getChildIndex(pBlock); - SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex); + int32_t childIndex = getChildIndex(pBlock); + SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex); SStreamFinalIntervalOperatorInfo* pChildInfo = pChildOp->info; - SExprSupp* pChildSup = &pChildOp->exprSupp; + SExprSupp* pChildSup = &pChildOp->exprSupp; doClearWindows(&pChildInfo->aggSup, pChildSup, &pChildInfo->interval, pChildInfo->primaryTsIndex, pChildSup->numOfExprs, pBlock, NULL); @@ -2527,7 +2527,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { break; } continue; - } else if (pBlock->info.type == STREAM_PUSH_EMPTY && IS_FINAL_OP(pInfo)) { + } else if (pBlock->info.type == STREAM_PUSH_OVER && IS_FINAL_OP(pInfo)) { processPushEmpty(pBlock, pInfo->pPullDataMap); continue; } diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 699a050aea..1ff1cbe833 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1052,7 +1052,8 @@ static int32_t translateUniqueMode(SFunctionNode* pFunc, char* pErrBuf, int32_t SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0); if (!nodesExprHasColumn(pPara)) { - return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, "The parameters of %s must contain columns", isUnique ? "UNIQUE" : "MODE"); + return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, "The parameters of %s must contain columns", + isUnique ? "UNIQUE" : "MODE"); } pFunc->node.resType = ((SExprNode*)pPara)->resType; @@ -1228,19 +1229,19 @@ static int32_t translateSubstr(SFunctionNode* pFunc, char* pErrBuf, int32_t len) static int32_t translateCast(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { // The number of parameters has been limited by the syntax definition - //uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; + // uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; // The function return type has been set during syntax parsing uint8_t para2Type = pFunc->node.resType.type; - //if (para2Type != TSDB_DATA_TYPE_BIGINT && para2Type != TSDB_DATA_TYPE_UBIGINT && - // para2Type != TSDB_DATA_TYPE_VARCHAR && para2Type != TSDB_DATA_TYPE_NCHAR && - // para2Type != TSDB_DATA_TYPE_TIMESTAMP) { - // return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); - //} - //if ((para2Type == TSDB_DATA_TYPE_TIMESTAMP && IS_VAR_DATA_TYPE(para1Type)) || - // (para2Type == TSDB_DATA_TYPE_BINARY && para1Type == TSDB_DATA_TYPE_NCHAR)) { - // return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); - //} + // if (para2Type != TSDB_DATA_TYPE_BIGINT && para2Type != TSDB_DATA_TYPE_UBIGINT && + // para2Type != TSDB_DATA_TYPE_VARCHAR && para2Type != TSDB_DATA_TYPE_NCHAR && + // para2Type != TSDB_DATA_TYPE_TIMESTAMP) { + // return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + // } + // if ((para2Type == TSDB_DATA_TYPE_TIMESTAMP && IS_VAR_DATA_TYPE(para1Type)) || + // (para2Type == TSDB_DATA_TYPE_BINARY && para1Type == TSDB_DATA_TYPE_NCHAR)) { + // return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + // } int32_t para2Bytes = pFunc->node.resType.bytes; if (IS_VAR_DATA_TYPE(para2Type)) { @@ -1639,7 +1640,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "leastsquares", .type = FUNCTION_TYPE_LEASTSQUARES, - .classification = FUNC_MGT_AGG_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_FORBID_STREAM_FUNC, .translateFunc = translateLeastSQR, .getEnvFunc = getLeastSQRFuncEnv, .initFunc = leastSQRFunctionSetup, diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index b3ceb92505..3aee636ac9 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -17,21 +17,20 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) { void* exec = pTask->exec.executor; - bool hasData = false; // set input SStreamQueueItem* pItem = (SStreamQueueItem*)data; if (pItem->type == STREAM_INPUT__TRIGGER) { SStreamTrigger* pTrigger = (SStreamTrigger*)data; - qSetMultiStreamInput(exec, pTrigger->pBlock, 1, STREAM_DATA_TYPE_SSDATA_BLOCK, false); + qSetMultiStreamInput(exec, pTrigger->pBlock, 1, STREAM_INPUT__DATA_BLOCK, false); } else if (pItem->type == STREAM_INPUT__DATA_SUBMIT) { ASSERT(pTask->isDataScan); SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)data; - qSetStreamInput(exec, pSubmit->data, STREAM_DATA_TYPE_SUBMIT_BLOCK, false); + qSetStreamInput(exec, pSubmit->data, STREAM_INPUT__DATA_SUBMIT, false); } else if (pItem->type == STREAM_INPUT__DATA_BLOCK || pItem->type == STREAM_INPUT__DATA_RETRIEVE) { SStreamDataBlock* pBlock = (SStreamDataBlock*)data; SArray* blocks = pBlock->blocks; - qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_DATA_TYPE_SSDATA_BLOCK, false); + qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_INPUT__DATA_BLOCK, false); } else if (pItem->type == STREAM_INPUT__DROP) { // TODO exec drop return 0; @@ -46,20 +45,16 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) } if (output == NULL) { if (pItem->type == STREAM_INPUT__DATA_RETRIEVE) { - SSDataBlock block = {0}; - /*block.info.type = STREAM_PUSH_EMPTY;*/ - // block.info.childId = pTask->selfChildId; + SSDataBlock block = {0}; SStreamDataBlock* pRetrieveBlock = (SStreamDataBlock*)data; ASSERT(taosArrayGetSize(pRetrieveBlock->blocks) == 1); - /*SSDataBlock* pBlock = createOneDataBlock(taosArrayGet(pRetrieveBlock->blocks, 0), true);*/ assignOneDataBlock(&block, taosArrayGet(pRetrieveBlock->blocks, 0)); - block.info.type = STREAM_PUSH_EMPTY; + block.info.type = STREAM_PUSH_OVER; block.info.childId = pTask->selfChildId; taosArrayPush(pRes, &block); } break; } - hasData = true; if (output->info.type == STREAM_RETRIEVE) { if (streamBroadcastToChildren(pTask, output) < 0) { @@ -73,9 +68,6 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) assignOneDataBlock(&block, output); block.info.childId = pTask->selfChildId; taosArrayPush(pRes, &block); - /*SSDataBlock* outputCopy = createOneDataBlock(output, true);*/ - /*outputCopy->info.childId = pTask->selfChildId;*/ - /*taosArrayPush(pRes, outputCopy);*/ } return 0; } @@ -164,4 +156,3 @@ FAIL: atomic_store_8(&pTask->execStatus, TASK_EXEC_STATUS__IDLE); return -1; } - diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 1daa9bd50e..97aa182dc9 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -86,9 +86,7 @@ int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) { ASSERT(pTask->sinkType == TASK_SINK__NONE); } - if (pTask->dispatchType == TASK_DISPATCH__INPLACE) { - if (tEncodeI32(pEncoder, pTask->inplaceDispatcher.taskId) < 0) return -1; - } else if (pTask->dispatchType == TASK_DISPATCH__FIXED) { + if (pTask->dispatchType == TASK_DISPATCH__FIXED) { if (tEncodeI32(pEncoder, pTask->fixedEpDispatcher.taskId) < 0) return -1; if (tEncodeI32(pEncoder, pTask->fixedEpDispatcher.nodeId) < 0) return -1; if (tEncodeSEpSet(pEncoder, &pTask->fixedEpDispatcher.epSet) < 0) return -1; @@ -147,9 +145,7 @@ int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask) { ASSERT(pTask->sinkType == TASK_SINK__NONE); } - if (pTask->dispatchType == TASK_DISPATCH__INPLACE) { - if (tDecodeI32(pDecoder, &pTask->inplaceDispatcher.taskId) < 0) return -1; - } else if (pTask->dispatchType == TASK_DISPATCH__FIXED) { + if (pTask->dispatchType == TASK_DISPATCH__FIXED) { if (tDecodeI32(pDecoder, &pTask->fixedEpDispatcher.taskId) < 0) return -1; if (tDecodeI32(pDecoder, &pTask->fixedEpDispatcher.nodeId) < 0) return -1; if (tDecodeSEpSet(pDecoder, &pTask->fixedEpDispatcher.epSet) < 0) return -1; From efb2cd27c75fc84d45b393868605889957adb183 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 29 Jun 2022 21:32:06 +0800 Subject: [PATCH 42/61] refactor: add tref to solve the timer not stopped problem --- source/dnode/vnode/src/inc/sma.h | 21 +++++++++- source/dnode/vnode/src/inc/vnodeInt.h | 1 + source/dnode/vnode/src/sma/smaEnv.c | 55 ++++++++++++++++++++------ source/dnode/vnode/src/sma/smaRollup.c | 43 +++++++++++++------- source/dnode/vnode/src/sma/smaUtil.c | 8 ++-- 5 files changed, 95 insertions(+), 33 deletions(-) diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index 23ad70bad3..5e5fbe861e 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -28,7 +28,8 @@ extern "C" { #define smaError(...) do { if (smaDebugFlag & DEBUG_ERROR) { taosPrintLog("SMA ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} while(0) #define smaWarn(...) do { if (smaDebugFlag & DEBUG_WARN) { taosPrintLog("SMA WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} while(0) #define smaInfo(...) do { if (smaDebugFlag & DEBUG_INFO) { taosPrintLog("SMA ", DEBUG_INFO, 255, __VA_ARGS__); }} while(0) -#define smaDebug(...) do { if (smaDebugFlag & DEBUG_DEBUG) { taosPrintLog("SMA ", DEBUG_DEBUG, tsdbDebugFlag, __VA_ARGS__); }} while(0) +// #define smaDebug(...) do { if (smaDebugFlag & DEBUG_DEBUG) { taosPrintLog("SMA ", DEBUG_DEBUG, tsdbDebugFlag, __VA_ARGS__); }} while(0) +#define smaDebug(...) do { if (smaDebugFlag & DEBUG_WARN) { taosPrintLog("SMA WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} while(0) #define smaTrace(...) do { if (smaDebugFlag & DEBUG_TRACE) { taosPrintLog("SMA ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0) // clang-format on @@ -46,6 +47,11 @@ struct SSmaEnv { SSmaStat *pStat; }; +typedef struct { + int32_t smaRef; + int32_t refId; +} SSmaMgmt; + #define SMA_ENV_LOCK(env) ((env)->lock) #define SMA_ENV_TYPE(env) ((env)->type) #define SMA_ENV_STAT(env) ((env)->pStat) @@ -58,6 +64,7 @@ struct STSmaStat { struct SRSmaStat { SSma *pSma; + int64_t refId; void *tmrHandle; tmr_h tmrId; int32_t tmrSeconds; @@ -73,6 +80,7 @@ struct SSmaStat { }; T_REF_DECLARE() }; + #define SMA_TSMA_STAT(s) (&(s)->tsmaStat) #define SMA_RSMA_STAT(s) (&(s)->rsmaStat) #define RSMA_INFO_HASH(r) ((r)->rsmaInfoHash) @@ -80,6 +88,7 @@ struct SSmaStat { #define RSMA_TMR_HANDLE(r) ((r)->tmrHandle) #define RSMA_TRIGGER_STAT(r) (&(r)->triggerStat) #define RSMA_RUNNING_STAT(r) (&(r)->runningStat) +#define RSMA_REF_ID(r) ((r)->refId) enum { TASK_TRIGGER_STAT_INIT = 0, @@ -192,10 +201,18 @@ typedef struct STFInfo STFInfo; typedef struct STFile STFile; struct STFInfo { + // common fields uint32_t magic; uint32_t ftype; uint32_t fver; int64_t fsize; + + // specific fields + union { + struct { + int64_t applyVer[2]; + } qTaskInfo; + }; }; struct STFile { @@ -230,7 +247,7 @@ int32_t tdUpdateTFileHeader(STFile *pTFile); void tdUpdateTFileMagic(STFile *pTFile, void *pCksm); void tdCloseTFile(STFile *pTFile); -void tdGetVndFileName(int32_t vid, const char *dname, const char *fname, char *outputName); +void tdGetVndFileName(int32_t vgId, const char *dname, const char *fname, char *outputName); #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index baead763ad..6bd7d4edd1 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -27,6 +27,7 @@ #include "tdatablock.h" #include "tdb.h" #include "tencode.h" +#include "tref.h" #include "tfs.h" #include "tglobal.h" #include "tjson.h" diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c index 7f115633b9..36ee20fe25 100644 --- a/source/dnode/vnode/src/sma/smaEnv.c +++ b/source/dnode/vnode/src/sma/smaEnv.c @@ -18,6 +18,9 @@ typedef struct SSmaStat SSmaStat; #define RSMA_TASK_INFO_HASH_SLOT 8 +#define SMA_MGMT_REF_NUM 1024 + +extern SSmaMgmt smaMgmt; // declaration of static functions @@ -25,6 +28,7 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *p static SSmaEnv *tdNewSmaEnv(const SSma *pSma, int8_t smaType, const char *path); static int32_t tdInitSmaEnv(SSma *pSma, int8_t smaType, const char *path, SSmaEnv **pEnv); static void *tdFreeTSmaStat(STSmaStat *pStat); +static void tdDestroyRSmaStat(void *pRSmaStat); // implementation @@ -128,6 +132,22 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS if (smaType == TSDB_SMA_TYPE_ROLLUP) { SRSmaStat *pRSmaStat = (SRSmaStat *)(*pSmaStat); pRSmaStat->pSma = (SSma *)pSma; + + // init smaMgmt + smaMgmt.smaRef = taosOpenRef(SMA_MGMT_REF_NUM, tdDestroyRSmaStat); + if (smaMgmt.refId < 0) { + smaError("init smaRef failed, num:%d", SMA_MGMT_REF_NUM); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + + int64_t refId = taosAddRef(smaMgmt.smaRef, pRSmaStat); + if (refId < 0) { + smaError("taosAddRef smaRef failed, since:%s", tstrerror(terrno)); + return TSDB_CODE_FAILED; + } + pRSmaStat->refId = refId; + // init timer RSMA_TMR_HANDLE(pRSmaStat) = taosTmrInit(10000, 100, 10000, "RSMA"); if (!RSMA_TMR_HANDLE(pRSmaStat)) { @@ -150,6 +170,7 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS } else { ASSERT(0); } + } return TSDB_CODE_SUCCESS; } @@ -169,9 +190,10 @@ static void *tdFreeTSmaStat(STSmaStat *pStat) { return NULL; } -static void tdDestroyRSmaStat(SRSmaStat *pStat) { - if (pStat) { - smaDebug("vgId:%d destroy rsma stat", SMA_VID(pStat->pSma)); +static void tdDestroyRSmaStat(void *pRSmaStat) { + if (pRSmaStat) { + SRSmaStat *pStat = (SRSmaStat *)pRSmaStat; + smaDebug("vgId:%d %s:%d destroy rsma stat %p", SMA_VID(pStat->pSma), __func__, __LINE__, pRSmaStat); // step 1: set persistence task cancelled atomic_store_8(RSMA_TRIGGER_STAT(pStat), TASK_TRIGGER_STAT_CANCELLED); @@ -183,9 +205,10 @@ static void tdDestroyRSmaStat(SRSmaStat *pStat) { if (atomic_load_8(RSMA_RUNNING_STAT(pStat)) == 1) { while (1) { if (atomic_load_8(RSMA_TRIGGER_STAT(pStat)) == TASK_TRIGGER_STAT_FINISHED) { + smaDebug("rsma, persist task finished already"); break; } else { - smaDebug("not destroyed since rsma stat in %" PRIi8, atomic_load_8(RSMA_TRIGGER_STAT(pStat))); + smaDebug("rsma, persist task not finished yet since rsma stat in %" PRIi8, atomic_load_8(RSMA_TRIGGER_STAT(pStat))); } ++nLoops; if (nLoops > 1000) { @@ -209,7 +232,10 @@ static void tdDestroyRSmaStat(SRSmaStat *pStat) { nLoops = 0; while (1) { if (T_REF_VAL_GET((SSmaStat *)pStat) == 0) { + smaDebug("rsma, all fetch task finished already"); break; + } else { + smaDebug("rsma, fetch tasks not all finished yet"); } ++nLoops; if (nLoops > 1000) { @@ -222,18 +248,17 @@ static void tdDestroyRSmaStat(SRSmaStat *pStat) { if (RSMA_TMR_HANDLE(pStat)) { taosTmrCleanUp(RSMA_TMR_HANDLE(pStat)); } - } -} -static void *tdFreeRSmaStat(SRSmaStat *pStat) { - tdDestroyRSmaStat(pStat); - taosMemoryFreeClear(pStat); - return NULL; + } } void *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType) { tdDestroySmaState(pSmaStat, smaType); - taosMemoryFreeClear(pSmaStat); + if (smaType == TSDB_SMA_TYPE_TIME_RANGE) { + taosMemoryFreeClear(pSmaStat); + } + // tref used to free rsma stat + return NULL; } @@ -243,17 +268,21 @@ void *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType) { * @param pSmaStat * @return int32_t */ + int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType) { if (pSmaStat) { if (smaType == TSDB_SMA_TYPE_TIME_RANGE) { tdDestroyTSmaStat(SMA_TSMA_STAT(pSmaStat)); } else if (smaType == TSDB_SMA_TYPE_ROLLUP) { - tdDestroyRSmaStat(SMA_RSMA_STAT(pSmaStat)); + SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pSmaStat); + if (taosRemoveRef(smaMgmt.smaRef, RSMA_REF_ID(pRSmaStat)) < 0) { + smaError("remove refId from smaRef failed, refId:0x%" PRIx64, RSMA_REF_ID(pRSmaStat)); + } } else { ASSERT(0); } } - return TSDB_CODE_SUCCESS; + return 0; } int32_t tdLockSma(SSma *pSma) { diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 4402f37c34..62dddb2da7 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -18,9 +18,13 @@ #define RSMA_QTASKINFO_PERSIST_MS 7200000 #define RSMA_QTASKINFO_BUFSIZE 32768 #define RSMA_QTASKINFO_HEAD_LEN (sizeof(int32_t) + sizeof(int8_t) + sizeof(int64_t)) // len + type + suid -typedef enum { TD_QTASK_TMP_FILE = 0, TD_QTASK_CUR_FILE } TD_QTASK_FILE_T; -static const char *tdQTaskInfoFname[] = {"qtaskinfo.t", "qtaskinfo"}; +SSmaMgmt smaMgmt = { + .smaRef = -1, +}; + +typedef enum { TD_QTASK_TMP_F = 0, TD_QTASK_CUR_F} TD_QTASK_FILE_T; +static const char *tdQTaskInfoFname[] = {"qtaskinfo.t", "qtaskinfo"}; typedef struct SRSmaQTaskInfoItem SRSmaQTaskInfoItem; typedef struct SRSmaQTaskInfoIter SRSmaQTaskInfoIter; @@ -80,6 +84,10 @@ struct SRSmaQTaskInfoIter { int32_t nBufPos; }; +static void tdRSmaQTaskInfoGetFName(int32_t vgId, int8_t ftype, char *outputName) { + tdGetVndFileName(vgId, VNODE_RSMA_DIR, tdQTaskInfoFname[ftype], outputName); +} + static FORCE_INLINE int32_t tdRSmaQTaskInfoContLen(int32_t lenWithHead) { return lenWithHead - RSMA_QTASKINFO_HEAD_LEN; } @@ -761,7 +769,7 @@ static int32_t tdRSmaRestoreQTaskInfoReload(SSma *pSma) { STFile tFile = {0}; char qTaskInfoFName[TSDB_FILENAME_LEN]; - tdRSmaQTaskInfoGetFName(TD_VID(pVnode), TD_QTASK_CUR_FILE, qTaskInfoFName); + tdRSmaQTaskInfoGetFName(TD_VID(pVnode), TD_QTASK_TMP_F, qTaskInfoFName); if (tdInitTFile(&tFile, pVnode->pTfs, qTaskInfoFName) < 0) { goto _err; } @@ -1003,10 +1011,6 @@ static int32_t tdRSmaQTaskInfoRestore(SSma *pSma, SRSmaQTaskInfoIter *pIter) { return TSDB_CODE_SUCCESS; } -static void tdRSmaQTaskInfoGetFName(int32_t vid, int8_t ftype, char *outputName) { - tdGetVndFileName(vid, VNODE_RSMA_DIR, tdQTaskInfoFname[ftype], outputName); -} - static void *tdRSmaPersistExec(void *param) { setThreadName("rsma-task-persist"); SRSmaStat *pRSmaStat = param; @@ -1063,7 +1067,7 @@ static void *tdRSmaPersistExec(void *param) { if (!isFileCreated) { char qTaskInfoFName[TSDB_FILENAME_LEN]; - tdRSmaQTaskInfoGetFName(vid, TD_QTASK_TMP_FILE, qTaskInfoFName); + tdRSmaQTaskInfoGetFName(vid, TD_QTASK_TMP_F, qTaskInfoFName); tdInitTFile(&tFile, pTfs, qTaskInfoFName); tdCreateTFile(&tFile, pTfs, true, -1); @@ -1079,8 +1083,8 @@ static void *tdRSmaPersistExec(void *param) { ASSERT(headLen <= RSMA_QTASKINFO_HEAD_LEN); tdAppendTFile(&tFile, (void *)&tmpBuf, headLen, &toffset); - smaDebug("vgId:%d, table %" PRIi64 " level %d head part len:%d appended to offset:%" PRIi64, vid, pRSmaInfo->suid, - i + 1, headLen, toffset); + smaDebug("vgId:%d, table %" PRIi64 " level %d head part(len:%d) appended to offset:%" PRIi64, vid, + pRSmaInfo->suid, i + 1, headLen, toffset); tdAppendTFile(&tFile, pOutput, len, &toffset); smaDebug("vgId:%d, table %" PRIi64 " level %d body part len:%d appended to offset:%" PRIi64, vid, pRSmaInfo->suid, i + 1, len, toffset); @@ -1106,8 +1110,8 @@ _normal: char newFName[TSDB_FILENAME_LEN]; strncpy(newFName, TD_TFILE_FULL_NAME(&tFile), TSDB_FILENAME_LEN); - char *pos = strstr(newFName, tdQTaskInfoFname[TD_QTASK_TMP_FILE]); - strncpy(pos, tdQTaskInfoFname[TD_QTASK_CUR_FILE], TSDB_FILENAME_LEN - POINTER_DISTANCE(pos, newFName)); + char *pos = strstr(newFName, tdQTaskInfoFname[TD_QTASK_TMP_F]); + strncpy(pos, tdQTaskInfoFname[TD_QTASK_TMP_F], TSDB_FILENAME_LEN - POINTER_DISTANCE(pos, newFName)); if (taosRenameFile(TD_TFILE_FULL_NAME(&tFile), newFName) != 0) { smaError("vgId:%d, failed to rename %s to %s", vid, TD_TFILE_FULL_NAME(&tFile), newFName); goto _err; @@ -1134,6 +1138,7 @@ _end: ASSERT(0); } atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 0); + taosReleaseRef(smaMgmt.smaRef, pRSmaStat->refId); taosThreadExit(NULL); return NULL; } @@ -1159,6 +1164,7 @@ static void tdRSmaPersistTask(SRSmaStat *pRSmaStat) { ASSERT(0); } atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 0); + taosReleaseRef(smaMgmt.smaRef, pRSmaStat->refId); } taosThreadAttrDestroy(&thAttr); @@ -1171,9 +1177,16 @@ static void tdRSmaPersistTask(SRSmaStat *pRSmaStat) { * @param tmrId */ static void tdRSmaPersistTrigger(void *param, void *tmrId) { - SRSmaStat *pRSmaStat = param; + SRSmaStat *rsmaStat = param; + int64_t refId = rsmaStat->refId; - int8_t tmrStat = + SRSmaStat *pRSmaStat = (SRSmaStat *)taosAcquireRef(smaMgmt.smaRef, refId); + if(!pRSmaStat) { + smaDebug("rsma persistence task not start since already destroyed"); + return; + } + + int8_t tmrStat = atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_ACTIVE, TASK_TRIGGER_STAT_INACTIVE); switch (tmrStat) { case TASK_TRIGGER_STAT_ACTIVE: { @@ -1191,6 +1204,7 @@ static void tdRSmaPersistTrigger(void *param, void *tmrId) { } else { atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 0); } + return; } break; case TASK_TRIGGER_STAT_CANCELLED: { atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_FINISHED); @@ -1206,4 +1220,5 @@ static void tdRSmaPersistTrigger(void *param, void *tmrId) { smaWarn("rsma persistence not start since unknown stat %" PRIi8, tmrStat); } break; } + taosReleaseRef(smaMgmt.smaRef, refId); } diff --git a/source/dnode/vnode/src/sma/smaUtil.c b/source/dnode/vnode/src/sma/smaUtil.c index 5f78aadddf..17bc2cdaca 100644 --- a/source/dnode/vnode/src/sma/smaUtil.c +++ b/source/dnode/vnode/src/sma/smaUtil.c @@ -141,8 +141,8 @@ int64_t tdAppendTFile(STFile *pTFile, void *buf, int64_t nbyte, int64_t *offset) } #if 1 - smaDebug("append to file %s, offset:%" PRIi64 " + nbyte:%" PRIi64 " =%" PRIi64, TD_TFILE_FULL_NAME(pTFile), toffset, - nbyte, toffset + nbyte); + smaDebug("append to file %s, offset:%" PRIi64 " nbyte:%" PRIi64 " fsize:%" PRIi64, TD_TFILE_FULL_NAME(pTFile), + toffset, nbyte, toffset + nbyte); #endif ASSERT(pTFile->info.fsize == toffset); @@ -179,8 +179,8 @@ void tdCloseTFile(STFile *pTFile) { } } -void tdGetVndFileName(int32_t vid, const char *dname, const char *fname, char *outputName) { - snprintf(outputName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s/%s", vid, dname, fname); +void tdGetVndFileName(int32_t vgId, const char *dname, const char *fname, char *outputName) { + snprintf(outputName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s/%s", vgId, dname, fname); } int32_t tdInitTFile(STFile *pTFile, STfs *pTfs, const char *fname) { From 6600c4725e4e2ed38b156c2f0d2e5caa01cf28dc Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Wed, 29 Jun 2022 21:35:04 +0800 Subject: [PATCH 43/61] refactor: code optimization --- source/dnode/vnode/src/inc/sma.h | 3 +-- source/dnode/vnode/src/sma/smaEnv.c | 5 ++--- source/dnode/vnode/src/sma/smaRollup.c | 17 +++++++++-------- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index 5e5fbe861e..3e15fd9c57 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -28,8 +28,7 @@ extern "C" { #define smaError(...) do { if (smaDebugFlag & DEBUG_ERROR) { taosPrintLog("SMA ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} while(0) #define smaWarn(...) do { if (smaDebugFlag & DEBUG_WARN) { taosPrintLog("SMA WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} while(0) #define smaInfo(...) do { if (smaDebugFlag & DEBUG_INFO) { taosPrintLog("SMA ", DEBUG_INFO, 255, __VA_ARGS__); }} while(0) -// #define smaDebug(...) do { if (smaDebugFlag & DEBUG_DEBUG) { taosPrintLog("SMA ", DEBUG_DEBUG, tsdbDebugFlag, __VA_ARGS__); }} while(0) -#define smaDebug(...) do { if (smaDebugFlag & DEBUG_WARN) { taosPrintLog("SMA WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} while(0) +#define smaDebug(...) do { if (smaDebugFlag & DEBUG_DEBUG) { taosPrintLog("SMA ", DEBUG_DEBUG, tsdbDebugFlag, __VA_ARGS__); }} while(0) #define smaTrace(...) do { if (smaDebugFlag & DEBUG_TRACE) { taosPrintLog("SMA ", DEBUG_TRACE, tsdbDebugFlag, __VA_ARGS__); }} while(0) // clang-format on diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c index 36ee20fe25..e70e79fb6f 100644 --- a/source/dnode/vnode/src/sma/smaEnv.c +++ b/source/dnode/vnode/src/sma/smaEnv.c @@ -170,7 +170,6 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS } else { ASSERT(0); } - } return TSDB_CODE_SUCCESS; } @@ -208,7 +207,8 @@ static void tdDestroyRSmaStat(void *pRSmaStat) { smaDebug("rsma, persist task finished already"); break; } else { - smaDebug("rsma, persist task not finished yet since rsma stat in %" PRIi8, atomic_load_8(RSMA_TRIGGER_STAT(pStat))); + smaDebug("rsma, persist task not finished yet since rsma stat in %" PRIi8, + atomic_load_8(RSMA_TRIGGER_STAT(pStat))); } ++nLoops; if (nLoops > 1000) { @@ -248,7 +248,6 @@ static void tdDestroyRSmaStat(void *pRSmaStat) { if (RSMA_TMR_HANDLE(pStat)) { taosTmrCleanUp(RSMA_TMR_HANDLE(pStat)); } - } } diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 62dddb2da7..bd74bbd639 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -23,8 +23,8 @@ SSmaMgmt smaMgmt = { .smaRef = -1, }; -typedef enum { TD_QTASK_TMP_F = 0, TD_QTASK_CUR_F} TD_QTASK_FILE_T; -static const char *tdQTaskInfoFname[] = {"qtaskinfo.t", "qtaskinfo"}; +typedef enum { TD_QTASK_TMP_F = 0, TD_QTASK_CUR_F } TD_QTASK_FILE_T; +static const char *tdQTaskInfoFname[] = {"qtaskinfo.t", "qtaskinfo"}; typedef struct SRSmaQTaskInfoItem SRSmaQTaskInfoItem; typedef struct SRSmaQTaskInfoIter SRSmaQTaskInfoIter; @@ -805,9 +805,9 @@ _err: /** * @brief reload ts data from checkpoint - * - * @param pSma - * @return int32_t + * + * @param pSma + * @return int32_t */ static int32_t tdRSmaRestoreTSDataReload(SSma *pSma) { // TODO @@ -869,7 +869,8 @@ static int32_t tdRSmaQTaskInfoItemRestore(SSma *pSma, const SRSmaQTaskInfoItem * pItem->type, terrstr(terrno)); return TSDB_CODE_FAILED; } - smaDebug("vgId:%d, restore rsma task success for table:%" PRIi64 " level %d", SMA_VID(pSma), pItem->suid, pItem->type); + smaDebug("vgId:%d, restore rsma task success for table:%" PRIi64 " level %d", SMA_VID(pSma), pItem->suid, + pItem->type); return TSDB_CODE_SUCCESS; } @@ -1181,12 +1182,12 @@ static void tdRSmaPersistTrigger(void *param, void *tmrId) { int64_t refId = rsmaStat->refId; SRSmaStat *pRSmaStat = (SRSmaStat *)taosAcquireRef(smaMgmt.smaRef, refId); - if(!pRSmaStat) { + if (!pRSmaStat) { smaDebug("rsma persistence task not start since already destroyed"); return; } - int8_t tmrStat = + int8_t tmrStat = atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_ACTIVE, TASK_TRIGGER_STAT_INACTIVE); switch (tmrStat) { case TASK_TRIGGER_STAT_ACTIVE: { From e74187248bfae7d56924b4c6c98524a79b1d68b4 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 29 Jun 2022 22:29:02 +0800 Subject: [PATCH 44/61] fix(tmq): offset --- source/client/src/tmq.c | 16 ++++++++-------- source/dnode/vnode/src/tq/tq.c | 4 ++-- tests/script/tsim/tmq/basic2Of2ConsOverlap.sim | 2 ++ 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 4cef25b9ce..ec2ebdfae9 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -1124,10 +1124,10 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { pRspWrapper->topicHandle = pTopic; if (rspType == TMQ_MSG_TYPE__POLL_RSP) { - memcpy(&pRspWrapper->dataRsp, pMsg->pData, sizeof(SMqRspHead)); SDecoder decoder; tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); tDecodeSMqDataRsp(&decoder, &pRspWrapper->dataRsp); + memcpy(&pRspWrapper->dataRsp, pMsg->pData, sizeof(SMqRspHead)); /*tDecodeSMqDataBlkRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRspWrapper->dataRsp);*/ } else { ASSERT(rspType == TMQ_MSG_TYPE__POLL_META_RSP); @@ -1138,7 +1138,7 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { taosMemoryFree(pMsg->pData); tscDebug("consumer %ld recv poll: vg %d, req offset %ld, rsp offset %ld, type %d", tmq->consumerId, pVg->vgId, - pRspWrapper->dataRsp.reqOffset, pRspWrapper->dataRsp.rspOffset, rspType); + pRspWrapper->dataRsp.reqOffset.version, pRspWrapper->dataRsp.rspOffset.version, rspType); taosWriteQitem(tmq->mqueue, pRspWrapper); tsem_post(&tmq->rspSem); @@ -1203,17 +1203,17 @@ bool tmqUpdateEp2(tmq_t* tmq, int32_t epoch, SMqAskEpRsp* pRsp) { for (int32_t j = 0; j < vgNumGet; j++) { SMqSubVgEp* pVgEp = taosArrayGet(pTopicEp->vgs, j); sprintf(vgKey, "%s:%d", topic.topicName, pVgEp->vgId); - int64_t* pOffset = taosHashGet(pHash, vgKey, strlen(vgKey)); - int64_t offset = tmq->resetOffsetCfg; + STqOffsetVal* pOffset = taosHashGet(pHash, vgKey, strlen(vgKey)); + STqOffsetVal offsetNew = {.type = tmq->resetOffsetCfg}; if (pOffset != NULL) { - offset = *pOffset; + offsetNew = *pOffset; } - tscDebug("consumer %ld(epoch %d) offset of vg %d updated to %ld, vgKey is %s", tmq->consumerId, epoch, - pVgEp->vgId, offset, vgKey); + /*tscDebug("consumer %ld(epoch %d) offset of vg %d updated to %ld, vgKey is %s", tmq->consumerId, epoch,*/ + /*pVgEp->vgId, offset, vgKey);*/ SMqClientVg clientVg = { .pollCnt = 0, - .currentOffsetNew.type = tmq->resetOffsetCfg, + .currentOffsetNew = offsetNew, .vgId = pVgEp->vgId, .epSet = pVgEp->epSet, .vgStatus = TMQ_VG_STATUS__IDLE, diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index ac36dbf774..aadacf1e72 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -142,7 +142,7 @@ int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, con void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); SEncoder encoder; - tEncoderInit(&encoder, abuf, tlen); + tEncoderInit(&encoder, abuf, len); tEncodeSMqDataRsp(&encoder, pRsp); /*tEncodeSMqDataBlkRsp(&abuf, pRsp);*/ @@ -332,7 +332,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { if (tqFetchLog(pTq, pHandle, &fetchVer, &pHeadWithCkSum) < 0) { // TODO add push mgr - tqOffsetResetToLog(&dataRsp.rspOffset, fetchVer); + tqOffsetResetToLog(&dataRsp.rspOffset, fetchVer - 1); if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) { code = -1; } diff --git a/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim b/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim index 480cf520d9..d796d00f03 100644 --- a/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim +++ b/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim @@ -134,6 +134,8 @@ endi $totalMsgCons = $totalMsgOfOneTopic + $totalMsgOfStb $sumOfMsgCnt = $data[0][2] + $data[1][2] if $sumOfMsgCnt != $totalMsgCons then + print total: $totalMsgCons + print sum: $sumOfMsgCnt return -1 endi From 94c3eb29d26d7296b8702e60cc16187c5f2ed574 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 29 Jun 2022 23:25:04 +0800 Subject: [PATCH 45/61] fix(tmq): rsp error offset --- source/client/src/tmq.c | 4 ++-- source/dnode/vnode/src/tq/tq.c | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index ec2ebdfae9..5493628e74 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -514,6 +514,8 @@ int32_t tmqCommitMsgImpl(tmq_t* tmq, const TAOS_RES* msg, int8_t async, tmq_comm pParamSet->userParam = userParam; tsem_init(&pParamSet->rspSem, 0, 0); + int32_t code = -1; + for (int32_t i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); if (strcmp(pTopic->topicName, topic) != 0) continue; @@ -530,8 +532,6 @@ int32_t tmqCommitMsgImpl(tmq_t* tmq, const TAOS_RES* msg, int8_t async, tmq_comm } } - int32_t code = -1; - HANDLE_RSP: if (pParamSet->totalRspNum == 0) { tsem_destroy(&pParamSet->rspSem); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index aadacf1e72..a08515ed5c 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -332,7 +332,8 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { if (tqFetchLog(pTq, pHandle, &fetchVer, &pHeadWithCkSum) < 0) { // TODO add push mgr - tqOffsetResetToLog(&dataRsp.rspOffset, fetchVer - 1); + tqOffsetResetToLog(&dataRsp.rspOffset, fetchVer); + ASSERT(dataRsp.rspOffset.version >= dataRsp.reqOffset.version); if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) { code = -1; } @@ -354,6 +355,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { // TODO continue scan until meeting batch requirement if (dataRsp.blockNum > 0 /* threshold */) { tqOffsetResetToLog(&dataRsp.rspOffset, fetchVer); + ASSERT(dataRsp.rspOffset.version >= dataRsp.reqOffset.version); if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) { code = -1; From 3f00a6db8ec95731d18600f7713faba01c7ab256 Mon Sep 17 00:00:00 2001 From: Xuefeng Tan <1172915550@qq.com> Date: Wed, 29 Jun 2022 23:26:10 +0800 Subject: [PATCH 46/61] feat(taosAdapter): remake REST api (#14351) --- tools/taosadapter | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/taosadapter b/tools/taosadapter index c1dc11f3ca..c885e967e4 160000 --- a/tools/taosadapter +++ b/tools/taosadapter @@ -1 +1 @@ -Subproject commit c1dc11f3cae64adf31dbd9a954ef8372d1e8f671 +Subproject commit c885e967e490105999b84d009a15168728dfafaf From cf35174d539d5b0b59ff6776af9121ce5c5d95c7 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 30 Jun 2022 10:23:35 +0800 Subject: [PATCH 47/61] feat: the last_row function supports all scenes --- include/libs/function/functionMgt.h | 1 + include/libs/nodes/plannodes.h | 1 + source/libs/function/src/builtins.c | 13 +++++++ source/libs/planner/src/planLogicCreater.c | 8 ++-- source/libs/planner/src/planOptimizer.c | 43 +++++++++++++++++++++- source/libs/planner/src/planSpliter.c | 2 + source/libs/planner/test/planBasicTest.cpp | 2 + 7 files changed, 66 insertions(+), 4 deletions(-) diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 1ed78750d1..7697dd5047 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -125,6 +125,7 @@ typedef enum EFunctionType { FUNCTION_TYPE_BLOCK_DIST_INFO, // block distribution pseudo column function FUNCTION_TYPE_TO_COLUMN, FUNCTION_TYPE_GROUP_KEY, + FUNCTION_TYPE_CACHE_LAST_ROW, // distributed splitting functions FUNCTION_TYPE_APERCENTILE_PARTIAL = 4000, diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 4671c8b81e..f8d79a863f 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -91,6 +91,7 @@ typedef struct SAggLogicNode { SLogicNode node; SNodeList* pGroupKeys; SNodeList* pAggFuncs; + bool hasLastRow; } SAggLogicNode; typedef struct SProjectLogicNode { diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index fbebb12cc3..85a2ab22b6 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1882,6 +1882,19 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .name = "last_row", .type = FUNCTION_TYPE_LAST_ROW, .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC, + .translateFunc = translateFirstLast, + .getEnvFunc = getFirstLastFuncEnv, + .initFunc = functionSetup, + .processFunc = lastFunction, + .finalizeFunc = firstLastFinalize, + .pPartialFunc = "_last_partial", + .pMergeFunc = "_last_merge", + .combineFunc = lastCombine, + }, + { + .name = "_cache_last_row", + .type = FUNCTION_TYPE_CACHE_LAST_ROW, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC, .translateFunc = translateLastRow, .getEnvFunc = getMinmaxFuncEnv, .initFunc = minmaxFunctionSetup, diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index ef8b109b62..05256e2696 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -160,9 +160,9 @@ static EScanType getScanType(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SNod return SCAN_TYPE_STREAM; } - if (pSelect->hasLastRowFunc) { - return SCAN_TYPE_LAST_ROW; - } + // if (pSelect->hasLastRowFunc) { + // return SCAN_TYPE_LAST_ROW; + // } if (NULL == pScanCols) { // select count(*) from t @@ -474,6 +474,8 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, return TSDB_CODE_OUT_OF_MEMORY; } + pAgg->hasLastRow = pSelect->hasLastRowFunc; + int32_t code = TSDB_CODE_SUCCESS; // set grouyp keys, agg funcs and having conditions diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 42f7f744c5..b733d75fb2 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1616,6 +1616,46 @@ static int32_t rewriteUniqueOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLog return rewriteUniqueOptimizeImpl(pCxt, pLogicSubplan, pIndef); } +static bool lastRowScanOptMayBeOptimized(SLogicNode* pNode) { + if (QUERY_NODE_LOGIC_PLAN_AGG != nodeType(pNode) || !(((SAggLogicNode*)pNode)->hasLastRow) || + NULL != ((SAggLogicNode*)pNode)->pGroupKeys || 1 != LIST_LENGTH(pNode->pChildren) || + QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(nodesListGetNode(pNode->pChildren, 0)) || + NULL != ((SScanLogicNode*)nodesListGetNode(pNode->pChildren, 0))->node.pConditions) { + return false; + } + + SNode* pFunc = NULL; + FOREACH(pFunc, ((SAggLogicNode*)pNode)->pAggFuncs) { + if (FUNCTION_TYPE_LAST_ROW != ((SFunctionNode*)pFunc)->funcType) { + return false; + } + } + + return true; +} + +static int32_t lastRowScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { + SAggLogicNode* pAgg = (SAggLogicNode*)optFindPossibleNode(pLogicSubplan->pNode, lastRowScanOptMayBeOptimized); + + if (NULL == pAgg) { + return TSDB_CODE_SUCCESS; + } + + SNode* pNode = NULL; + FOREACH(pNode, pAgg->pAggFuncs) { + SFunctionNode* pFunc = (SFunctionNode*)pNode; + int32_t len = snprintf(pFunc->functionName, sizeof(pFunc->functionName), "_cache_last_row"); + pFunc->functionName[len] = '\0'; + fmGetFuncInfo(pFunc, NULL, 0); + } + pAgg->hasLastRow = false; + + ((SScanLogicNode*)nodesListGetNode(pAgg->node.pChildren, 0))->scanType = SCAN_TYPE_LAST_ROW; + + pCxt->optimized = true; + return TSDB_CODE_SUCCESS; +} + // merge projects static bool mergeProjectsMayBeOptimized(SLogicNode* pNode) { if (QUERY_NODE_LOGIC_PLAN_PROJECT != nodeType(pNode) || 1 != LIST_LENGTH(pNode->pChildren)) { @@ -1704,7 +1744,8 @@ static const SOptimizeRule optimizeRuleSet[] = { {.pName = "EliminateProject", .optimizeFunc = eliminateProjOptimize}, {.pName = "EliminateSetOperator", .optimizeFunc = eliminateSetOpOptimize}, {.pName = "RewriteTail", .optimizeFunc = rewriteTailOptimize}, - {.pName = "RewriteUnique", .optimizeFunc = rewriteUniqueOptimize} + {.pName = "RewriteUnique", .optimizeFunc = rewriteUniqueOptimize}, + {.pName = "LastRowScan", .optimizeFunc = lastRowScanOptimize} }; // clang-format on diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 9d23df5bda..2bc226804f 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -197,6 +197,8 @@ static bool stbSplNeedSplit(bool streamQuery, SLogicNode* pNode) { return stbSplIsMultiTbScan(streamQuery, (SScanLogicNode*)pNode); case QUERY_NODE_LOGIC_PLAN_JOIN: return !(((SJoinLogicNode*)pNode)->isSingleTableJoin); + // case QUERY_NODE_LOGIC_PLAN_PARTITION: + // return stbSplHasMultiTbScan(streamQuery, pNode); case QUERY_NODE_LOGIC_PLAN_AGG: return !stbSplHasGatherExecFunc(((SAggLogicNode*)pNode)->pAggFuncs) && stbSplHasMultiTbScan(streamQuery, pNode); case QUERY_NODE_LOGIC_PLAN_WINDOW: diff --git a/source/libs/planner/test/planBasicTest.cpp b/source/libs/planner/test/planBasicTest.cpp index ff725c444e..b886fca2af 100644 --- a/source/libs/planner/test/planBasicTest.cpp +++ b/source/libs/planner/test/planBasicTest.cpp @@ -99,6 +99,8 @@ TEST_F(PlanBasicTest, lastRowFunc) { run("SELECT LAST_ROW(c1, c2) FROM t1"); run("SELECT LAST_ROW(c1) FROM st1"); + + run("SELECT LAST_ROW(c1), SUM(c3) FROM t1"); } TEST_F(PlanBasicTest, sampleFunc) { From 2caa9fdf7dc5ed1b5b364a0303c5d8342fa0844a Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Thu, 30 Jun 2022 10:53:15 +0800 Subject: [PATCH 48/61] test: add test case --- tests/system-test/7-tmq/tmqCommon.py | 16 +++++++++++++ tests/test/c/tmqSim.c | 34 ++++++++++++++++++++++++---- 2 files changed, 45 insertions(+), 5 deletions(-) diff --git a/tests/system-test/7-tmq/tmqCommon.py b/tests/system-test/7-tmq/tmqCommon.py index 9254f57c40..b545340153 100644 --- a/tests/system-test/7-tmq/tmqCommon.py +++ b/tests/system-test/7-tmq/tmqCommon.py @@ -76,6 +76,22 @@ class TMQCom: resultList.append(tdSql.getData(i , 3)) return resultList + + def selectConsumeMsgResult(self,expectRows,cdbName='cdb'): + resultList=[] + while 1: + tdSql.query("select * from %s.consumeresult"%cdbName) + #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) + if tdSql.getRows() == expectRows: + break + else: + time.sleep(5) + + for i in range(expectRows): + tdLog.info ("consume id: %d, consume msgs: %d, consume rows: %d"%(tdSql.getData(i , 1), tdSql.getData(i , 2), tdSql.getData(i , 3))) + resultList.append(tdSql.getData(i , 2)) + + return resultList def startTmqSimProcess(self,pollDelay,dbName,showMsg=1,showRow=1,cdbName='cdb',valgrind=0,alias=0): buildPath = tdCom.getBuildPath() diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index 81fa72d15a..d21b1faba4 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -635,6 +635,9 @@ void loop_consume(SThreadInfo* pInfo) { } } + uint64_t lastPrintTime = taosGetTimestampMs(); + uint64_t startTs = taosGetTimestampMs(); + int32_t consumeDelay = g_stConfInfo.consumeDelay == -1 ? -1 : (g_stConfInfo.consumeDelay * 1000); while (running) { TAOS_RES* tmqMsg = tmq_consumer_poll(pInfo->tmq, consumeDelay); @@ -646,7 +649,15 @@ void loop_consume(SThreadInfo* pInfo) { taos_free_result(tmqMsg); totalMsgs++; - + + int64_t currentPrintTime = taosGetTimestampMs(); + if (currentPrintTime - lastPrintTime > 10 * 1000) { + taosFprintfFile(g_fp, + "consumer id %d has currently poll total msgs: %" PRId64 "\n", + pInfo->consumerId, totalMsgs); + lastPrintTime = currentPrintTime; + } + if (0 == once_flag) { once_flag = 1; notifyMainScript(pInfo, NOTIFY_CMD_START_CONSUM); @@ -663,7 +674,7 @@ void loop_consume(SThreadInfo* pInfo) { break; } } - + if (0 == running) { taosFprintfFile(g_fp, "receive stop signal and not continue consume\n"); } @@ -676,8 +687,6 @@ void loop_consume(SThreadInfo* pInfo) { } void* consumeThreadFunc(void* param) { - int32_t totalMsgs = 0; - SThreadInfo* pInfo = (SThreadInfo*)param; pInfo->taos = taos_connect(NULL, "root", "taosdata", NULL, 0); @@ -859,12 +868,27 @@ int main(int32_t argc, char* argv[]) { (void*)(&(g_stConfInfo.stThreads[i]))); } + int64_t start = taosGetTimestampUs(); + for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { taosThreadJoin(g_stConfInfo.stThreads[i].thread, NULL); taosThreadClear(&g_stConfInfo.stThreads[i].thread); } - // printf("consumer: %d, cosumer1: %d\n", totalMsgs, pInfo->consumeMsgCnt); + int64_t end = taosGetTimestampUs(); + + int64_t totalMsgs = 0; + for (int32_t i = 0; i < g_stConfInfo.numOfThread; i++) { + totalMsgs += g_stConfInfo.stThreads[i].consumeMsgCnt; + } + + int64_t t = end - start; + if (0 == t) t = 1; + + double tInMs = (double)t / 1000000.0; + taosFprintfFile(g_fp, + "Spent %.4f seconds to poll msgs: %" PRIu64 " with %d thread(s), throughput: %.2f msgs/second\n\n", + tInMs, totalMsgs, g_stConfInfo.numOfThread, (double)(totalMsgs / tInMs)); taosFprintfFile(g_fp, "==== close tmqlog ====\n"); taosCloseFile(&g_fp); From 2ad65998107b88498917bcb902500e9912c4efef Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 30 Jun 2022 11:04:24 +0800 Subject: [PATCH 49/61] feat: partition by distributed split --- source/libs/planner/src/planLogicCreater.c | 10 +++------- source/libs/planner/src/planPhysiCreater.c | 2 +- source/libs/planner/src/planSpliter.c | 19 ++++++++++++++++--- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 05256e2696..d170482c48 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -154,16 +154,12 @@ static int32_t createSelectRootLogicNode(SLogicPlanContext* pCxt, SSelectStmt* p return createRootLogicNode(pCxt, pSelect, pSelect->precision, (FCreateLogicNode)func, pRoot); } -static EScanType getScanType(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SNodeList* pScanPseudoCols, - SNodeList* pScanCols, int8_t tableType) { +static EScanType getScanType(SLogicPlanContext* pCxt, SNodeList* pScanPseudoCols, SNodeList* pScanCols, + int8_t tableType) { if (pCxt->pPlanCxt->topicQuery || pCxt->pPlanCxt->streamQuery) { return SCAN_TYPE_STREAM; } - // if (pSelect->hasLastRowFunc) { - // return SCAN_TYPE_LAST_ROW; - // } - if (NULL == pScanCols) { // select count(*) from t return NULL == pScanPseudoCols @@ -279,7 +275,7 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect code = rewriteExprsForSelect(pScan->pScanPseudoCols, pSelect, SQL_CLAUSE_FROM); } - pScan->scanType = getScanType(pCxt, pSelect, pScan->pScanPseudoCols, pScan->pScanCols, pScan->tableType); + pScan->scanType = getScanType(pCxt, pScan->pScanPseudoCols, pScan->pScanCols, pScan->tableType); if (TSDB_CODE_SUCCESS == code) { code = addPrimaryKeyCol(pScan->tableId, &pScan->pScanCols); diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 46747af3a9..0f19db26a5 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -1344,7 +1344,7 @@ static int32_t createMergePhysiNode(SPhysiPlanContext* pCxt, SMergeLogicNode* pM } } - if (TSDB_CODE_SUCCESS == code) { + if (TSDB_CODE_SUCCESS == code && NULL != pMergeLogicNode->pMergeKeys) { code = setListSlotId(pCxt, pMerge->node.pOutputDataBlockDesc->dataBlockId, -1, pMergeLogicNode->pMergeKeys, &pMerge->pMergeKeys); } diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 2bc226804f..60c04c2c30 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -197,8 +197,8 @@ static bool stbSplNeedSplit(bool streamQuery, SLogicNode* pNode) { return stbSplIsMultiTbScan(streamQuery, (SScanLogicNode*)pNode); case QUERY_NODE_LOGIC_PLAN_JOIN: return !(((SJoinLogicNode*)pNode)->isSingleTableJoin); - // case QUERY_NODE_LOGIC_PLAN_PARTITION: - // return stbSplHasMultiTbScan(streamQuery, pNode); + case QUERY_NODE_LOGIC_PLAN_PARTITION: + return stbSplHasMultiTbScan(streamQuery, pNode); case QUERY_NODE_LOGIC_PLAN_AGG: return !stbSplHasGatherExecFunc(((SAggLogicNode*)pNode)->pAggFuncs) && stbSplHasMultiTbScan(streamQuery, pNode); case QUERY_NODE_LOGIC_PLAN_WINDOW: @@ -433,7 +433,7 @@ static int32_t stbSplSplitIntervalForBatch(SSplitContext* pCxt, SStableSplitInfo SNodeList* pMergeKeys = NULL; code = stbSplCreateMergeKeysByPrimaryKey(((SWindowLogicNode*)pInfo->pSplitNode)->pTspk, &pMergeKeys); if (TSDB_CODE_SUCCESS == code) { - code = stbSplCreateMergeNode(pCxt, NULL, pInfo->pSplitNode, pMergeKeys, pPartWindow, false); + code = stbSplCreateMergeNode(pCxt, NULL, pInfo->pSplitNode, pMergeKeys, pPartWindow, true); } if (TSDB_CODE_SUCCESS != code) { nodesDestroyList(pMergeKeys); @@ -889,6 +889,16 @@ static int32_t stbSplSplitJoinNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) return code; } +static int32_t stbSplSplitPartitionNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) { + int32_t code = stbSplCreateMergeNode(pCxt, pInfo->pSubplan, pInfo->pSplitNode, NULL, pInfo->pSplitNode, true); + if (TSDB_CODE_SUCCESS == code) { + code = nodesListMakeStrictAppend(&pInfo->pSubplan->pChildren, + (SNode*)splCreateScanSubplan(pCxt, pInfo->pSplitNode, SPLIT_FLAG_STABLE_SPLIT)); + } + ++(pCxt->groupId); + return code; +} + static int32_t stableSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) { if (pCxt->pPlanCxt->rSmaQuery) { return TSDB_CODE_SUCCESS; @@ -907,6 +917,9 @@ static int32_t stableSplit(SSplitContext* pCxt, SLogicSubplan* pSubplan) { case QUERY_NODE_LOGIC_PLAN_JOIN: code = stbSplSplitJoinNode(pCxt, &info); break; + case QUERY_NODE_LOGIC_PLAN_PARTITION: + code = stbSplSplitPartitionNode(pCxt, &info); + break; case QUERY_NODE_LOGIC_PLAN_AGG: code = stbSplSplitAggNode(pCxt, &info); break; From d53e1982add2e668b818eab16529003a6324d702 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Thu, 30 Jun 2022 11:18:21 +0800 Subject: [PATCH 50/61] refactor: tref used for rsma fetch task --- source/dnode/vnode/src/inc/sma.h | 13 ++-- source/dnode/vnode/src/sma/smaEnv.c | 2 +- source/dnode/vnode/src/sma/smaRollup.c | 95 ++++++++++++++------------ 3 files changed, 57 insertions(+), 53 deletions(-) diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index 3e15fd9c57..b455d779c1 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -48,7 +48,6 @@ struct SSmaEnv { typedef struct { int32_t smaRef; - int32_t refId; } SSmaMgmt; #define SMA_ENV_LOCK(env) ((env)->lock) @@ -63,12 +62,12 @@ struct STSmaStat { struct SRSmaStat { SSma *pSma; - int64_t refId; - void *tmrHandle; - tmr_h tmrId; - int32_t tmrSeconds; - int8_t triggerStat; - int8_t runningStat; + int64_t refId; // shared by persistence/fetch tasks + void *tmrHandle; // for persistence task + tmr_h tmrId; // for persistence task + int32_t tmrSeconds; // for persistence task + int8_t triggerStat; // for persistence task + int8_t runningStat; // for persistence task SHashObj *rsmaInfoHash; // key: stbUid, value: SRSmaInfo; }; diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c index e70e79fb6f..120d6612a2 100644 --- a/source/dnode/vnode/src/sma/smaEnv.c +++ b/source/dnode/vnode/src/sma/smaEnv.c @@ -135,7 +135,7 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS // init smaMgmt smaMgmt.smaRef = taosOpenRef(SMA_MGMT_REF_NUM, tdDestroyRSmaStat); - if (smaMgmt.refId < 0) { + if (smaMgmt.smaRef < 0) { smaError("init smaRef failed, num:%d", SMA_MGMT_REF_NUM); terrno = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_FAILED; diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index bd74bbd639..bf662ddc05 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -50,6 +50,7 @@ static int32_t tdRSmaRestoreTSDataReload(SSma *pSma); struct SRSmaInfoItem { SRSmaInfo *pRsmaInfo; + int64_t refId; void *taskInfo; // qTaskInfo_t tmr_h tmrId; int8_t level; @@ -60,11 +61,14 @@ struct SRSmaInfoItem { struct SRSmaInfo { STSchema *pTSchema; - SSma *pSma; + SRSmaStat *pStat; int64_t suid; SRSmaInfoItem items[TSDB_RETENTION_L2]; }; +#define RSMA_INFO_SMA(r) ((r)->pStat->pSma) +#define RSMA_INFO_STAT(r) ((r)->pStat) + struct SRSmaQTaskInfoItem { int32_t len; int8_t type; @@ -107,22 +111,21 @@ static FORCE_INLINE void tdFreeTaskHandle(qTaskInfo_t *taskHandle, int32_t vgId, void *tdFreeRSmaInfo(SRSmaInfo *pInfo) { if (pInfo) { + SSma *pSma = RSMA_INFO_SMA(pInfo); for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { SRSmaInfoItem *pItem = &pInfo->items[i]; if (pItem->taskInfo) { - smaDebug("vgId:%d, stb %" PRIi64 " stop fetch-timer %p level %d", SMA_VID(pInfo->pSma), pInfo->suid, - pItem->tmrId, i + 1); + smaDebug("vgId:%d, stb %" PRIi64 " stop fetch-timer %p level %d", SMA_VID(pSma), pInfo->suid, pItem->tmrId, + i + 1); taosTmrStopA(&pItem->tmrId); - tdFreeTaskHandle(&pItem->taskInfo, SMA_VID(pInfo->pSma), i + 1); + tdFreeTaskHandle(&pItem->taskInfo, SMA_VID(pSma), i + 1); } else { - smaDebug("vgId:%d, stb %" PRIi64 " no need to destroy rsma info level %d since empty taskInfo", - SMA_VID(pInfo->pSma), pInfo->suid, i + 1); + smaDebug("vgId:%d, stb %" PRIi64 " no need to destroy rsma info level %d since empty taskInfo", SMA_VID(pSma), + pInfo->suid, i + 1); } } taosMemoryFree(pInfo->pTSchema); taosMemoryFree(pInfo); - } else { - smaDebug("vgId:%d, stb %" PRIi64 " no need to destroy rsma info since empty", SMA_VID(pInfo->pSma), pInfo->suid); } return NULL; @@ -255,6 +258,7 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaInfo if (param->qmsg[idx]) { SRSmaInfoItem *pItem = &(pRSmaInfo->items[idx]); + pItem->refId = RSMA_REF_ID(pRSmaInfo->pStat); pItem->pRsmaInfo = pRSmaInfo; pItem->taskInfo = qCreateStreamExecTaskInfo(param->qmsg[idx], pReadHandle); if (!pItem->taskInfo) { @@ -340,7 +344,7 @@ int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con goto _err; } pRSmaInfo->pTSchema = pTSchema; - pRSmaInfo->pSma = pSma; + pRSmaInfo->pStat = pStat; pRSmaInfo->suid = suid; if (tdSetRSmaInfoItemParams(pSma, param, pRSmaInfo, &handle, 0) < 0) { @@ -522,7 +526,7 @@ static void tdDestroySDataBlockArray(SArray *pArray) { static int32_t tdFetchAndSubmitRSmaResult(SRSmaInfoItem *pItem, int8_t blkType) { SArray *pResult = NULL; SRSmaInfo *pRSmaInfo = pItem->pRsmaInfo; - SSma *pSma = pRSmaInfo->pSma; + SSma *pSma = RSMA_INFO_SMA(pRSmaInfo); while (1) { SSDataBlock *output = NULL; @@ -585,21 +589,29 @@ _err: */ static void tdRSmaFetchTrigger(void *param, void *tmrId) { SRSmaInfoItem *pItem = param; - SSma *pSma = pItem->pRsmaInfo->pSma; - SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT((SSmaEnv *)pSma->pRSmaEnv); + SSma *pSma = NULL; + SRSmaStat *pStat = (SRSmaStat *)taosAcquireRef(smaMgmt.smaRef, pItem->refId); + if (!pStat) { + smaDebug("rsma fetch task not start since already destroyed"); + return; + } + pSma = RSMA_INFO_SMA(pItem->pRsmaInfo); + + // if rsma trigger stat in cancelled or finished, not start fetch task anymore int8_t rsmaTriggerStat = atomic_load_8(RSMA_TRIGGER_STAT(pStat)); if (rsmaTriggerStat == TASK_TRIGGER_STAT_CANCELLED || rsmaTriggerStat == TASK_TRIGGER_STAT_FINISHED) { - smaDebug("vgId:%d, level %" PRIi8 " not fetch since stat is cancelled for table suid:%" PRIi64, SMA_VID(pSma), - pItem->level, pItem->pRsmaInfo->suid); + taosReleaseRef(smaMgmt.smaRef, pItem->refId); + smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is cancelled", + SMA_VID(pSma), pItem->level, pItem->pRsmaInfo->suid); return; } int8_t fetchTriggerStat = atomic_val_compare_exchange_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE, TASK_TRIGGER_STAT_INACTIVE); if (fetchTriggerStat == TASK_TRIGGER_STAT_ACTIVE) { - smaDebug("vgId:%d, level %" PRIi8 " stat is active for table suid:%" PRIi64, SMA_VID(pSma), pItem->level, - pItem->pRsmaInfo->suid); + smaDebug("vgId:%d, fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is active", SMA_VID(pSma), + pItem->level, pItem->pRsmaInfo->suid); tdRefSmaStat(pSma, (SSmaStat *)pStat); @@ -610,9 +622,11 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { tdUnRefSmaStat(pSma, (SSmaStat *)pStat); } else { - smaDebug("vgId:%d, level %" PRIi8 " stat is inactive for table suid:%" PRIi64, SMA_VID(pSma), pItem->level, - pItem->pRsmaInfo->suid); + smaDebug("vgId:%d, not fetch rsma level %" PRIi8 " data for table:%" PRIi64 " since stat is inactive", + SMA_VID(pSma), pItem->level, pItem->pRsmaInfo->suid); } +_end: + taosReleaseRef(smaMgmt.smaRef, pItem->refId); } static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfoItem *pItem, tb_uid_t suid, @@ -632,7 +646,6 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType tdFetchAndSubmitRSmaResult(pItem, STREAM_DATA_TYPE_SUBMIT_BLOCK); atomic_store_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE); - smaDebug("vgId:%d, process rsma insert", SMA_VID(pSma)); SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); SRSmaStat *pStat = SMA_RSMA_STAT(pEnv->pStat); @@ -1036,7 +1049,7 @@ static void *tdRSmaPersistExec(void *param) { for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { qTaskInfo_t taskInfo = pRSmaInfo->items[i].taskInfo; if (!taskInfo) { - smaDebug("vgId:%d, table %" PRIi64 " level %d qTaskInfo is NULL", vid, pRSmaInfo->suid, i + 1); + smaDebug("vgId:%d, rsma, table %" PRIi64 " level %d qTaskInfo is NULL", vid, pRSmaInfo->suid, i + 1); continue; } @@ -1044,27 +1057,20 @@ static void *tdRSmaPersistExec(void *param) { int32_t len = 0; int8_t type = (int8_t)(i + 1); if (qSerializeTaskStatus(taskInfo, &pOutput, &len) < 0) { - smaError("vgId:%d, table %" PRIi64 " level %d serialize rsma task failed since %s", vid, pRSmaInfo->suid, i + 1, - terrstr(terrno)); + smaError("vgId:%d, rsma, table %" PRIi64 " level %d serialize qTaskInfo failed since %s", vid, pRSmaInfo->suid, + i + 1, terrstr(terrno)); goto _err; } if (!pOutput || len <= 0) { - smaDebug("vgId:%d, table %" PRIi64 " level %d serialize rsma task success but no output(len %d), not persist", + smaDebug("vgId:%d, rsma, table %" PRIi64 + " level %d serialize qTaskInfo success but no output(len %d), not persist", vid, pRSmaInfo->suid, i + 1, len); taosMemoryFreeClear(pOutput); continue; } - smaDebug("vgId:%d, table %" PRIi64 " level %d serialize rsma task success with len %d, need persist", vid, + smaDebug("vgId:%d, rsma, table %" PRIi64 " level %d serialize qTaskInfo success with len %d, need persist", vid, pRSmaInfo->suid, i + 1, len); -#if 0 - if (qDeserializeTaskStatus(taskInfo, pOutput, len) < 0) { - smaError("vgId:%d, table %" PRIi64 "level %d deserialize rsma task failed since %s", vid, pRSmaInfo->suid, - i + 1, terrstr(terrno)); - } else { - smaDebug("vgId:%d, table %" PRIi64 " level %d deserialize rsma task success", vid, pRSmaInfo->suid, i + 1); - } -#endif if (!isFileCreated) { char qTaskInfoFName[TSDB_FILENAME_LEN]; @@ -1084,11 +1090,11 @@ static void *tdRSmaPersistExec(void *param) { ASSERT(headLen <= RSMA_QTASKINFO_HEAD_LEN); tdAppendTFile(&tFile, (void *)&tmpBuf, headLen, &toffset); - smaDebug("vgId:%d, table %" PRIi64 " level %d head part(len:%d) appended to offset:%" PRIi64, vid, + smaDebug("vgId:%d, rsma, table %" PRIi64 " level %d head part(len:%d) appended to offset:%" PRIi64, vid, pRSmaInfo->suid, i + 1, headLen, toffset); tdAppendTFile(&tFile, pOutput, len, &toffset); - smaDebug("vgId:%d, table %" PRIi64 " level %d body part len:%d appended to offset:%" PRIi64, vid, pRSmaInfo->suid, - i + 1, len, toffset); + smaDebug("vgId:%d, rsma, table %" PRIi64 " level %d body part len:%d appended to offset:%" PRIi64, vid, + pRSmaInfo->suid, i + 1, len, toffset); taosMemoryFree(pOutput); } @@ -1098,13 +1104,13 @@ static void *tdRSmaPersistExec(void *param) { _normal: if (isFileCreated) { if (tdUpdateTFileHeader(&tFile) < 0) { - smaError("vgId:%d, failed to update tfile %s header since %s", vid, TD_TFILE_FULL_NAME(&tFile), + smaError("vgId:%d, rsma, failed to update tfile %s header since %s", vid, TD_TFILE_FULL_NAME(&tFile), tstrerror(terrno)); tdCloseTFile(&tFile); tdRemoveTFile(&tFile); goto _err; } else { - smaDebug("vgId:%d, succeed to update tfile %s header", vid, TD_TFILE_FULL_NAME(&tFile)); + smaDebug("vgId:%d, rsma, succeed to update tfile %s header", vid, TD_TFILE_FULL_NAME(&tFile)); } tdCloseTFile(&tFile); @@ -1114,10 +1120,10 @@ _normal: char *pos = strstr(newFName, tdQTaskInfoFname[TD_QTASK_TMP_F]); strncpy(pos, tdQTaskInfoFname[TD_QTASK_TMP_F], TSDB_FILENAME_LEN - POINTER_DISTANCE(pos, newFName)); if (taosRenameFile(TD_TFILE_FULL_NAME(&tFile), newFName) != 0) { - smaError("vgId:%d, failed to rename %s to %s", vid, TD_TFILE_FULL_NAME(&tFile), newFName); + smaError("vgId:%d, rsma, failed to rename %s to %s", vid, TD_TFILE_FULL_NAME(&tFile), newFName); goto _err; } else { - smaDebug("vgId:%d, succeed to rename %s to %s", vid, TD_TFILE_FULL_NAME(&tFile), newFName); + smaDebug("vgId:%d, rsma, succeed to rename %s to %s", vid, TD_TFILE_FULL_NAME(&tFile), newFName); } } goto _end; @@ -1129,13 +1135,13 @@ _end: if (TASK_TRIGGER_STAT_INACTIVE == atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_INACTIVE, TASK_TRIGGER_STAT_ACTIVE)) { - smaDebug("vgId:%d, persist task is active again", vid); + smaDebug("vgId:%d, rsma persist task is active again", vid); } else if (TASK_TRIGGER_STAT_CANCELLED == atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_CANCELLED, TASK_TRIGGER_STAT_FINISHED)) { - smaDebug("vgId:%d, persist task is cancelled", vid); + smaDebug("vgId:%d, rsma persist task is cancelled", vid); } else { - smaWarn("vgId:%d, persist task in abnormal stat %" PRIi8, vid, atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat))); + smaWarn("vgId:%d, rsma persist task in abnormal stat %" PRIi8, vid, atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat))); ASSERT(0); } atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 0); @@ -1179,9 +1185,8 @@ static void tdRSmaPersistTask(SRSmaStat *pRSmaStat) { */ static void tdRSmaPersistTrigger(void *param, void *tmrId) { SRSmaStat *rsmaStat = param; - int64_t refId = rsmaStat->refId; + SRSmaStat *pRSmaStat = (SRSmaStat *)taosAcquireRef(smaMgmt.smaRef, rsmaStat->refId); - SRSmaStat *pRSmaStat = (SRSmaStat *)taosAcquireRef(smaMgmt.smaRef, refId); if (!pRSmaStat) { smaDebug("rsma persistence task not start since already destroyed"); return; @@ -1221,5 +1226,5 @@ static void tdRSmaPersistTrigger(void *param, void *tmrId) { smaWarn("rsma persistence not start since unknown stat %" PRIi8, tmrStat); } break; } - taosReleaseRef(smaMgmt.smaRef, refId); + taosReleaseRef(smaMgmt.smaRef, rsmaStat->refId); } From 63d799feebe277a674818df76291984303d0dc01 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 30 Jun 2022 11:32:00 +0800 Subject: [PATCH 51/61] refactor(query): set stateduration function default time unit as db precision --- source/libs/function/src/builtinsimpl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index e40ab3784d..a75d5a2ecb 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -4283,7 +4283,7 @@ int32_t stateDurationFunction(SqlFunctionCtx* pCtx) { SColumnInfoData* pOutput = (SColumnInfoData*)pCtx->pOutput; // TODO: process timeUnit for different db precisions - int32_t timeUnit = 1000; + int32_t timeUnit = 1; if (pCtx->numOfParams == 5) { // TODO: param number incorrect timeUnit = pCtx->param[3].param.i; } From b0b5d417af60221264a34c9ea2d233b463414f9d Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 30 Jun 2022 11:33:06 +0800 Subject: [PATCH 52/61] modify test case --- tests/system-test/2-query/stateduration.py | 72 +++++++++++----------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/tests/system-test/2-query/stateduration.py b/tests/system-test/2-query/stateduration.py index fa71009ef2..23169553dc 100644 --- a/tests/system-test/2-query/stateduration.py +++ b/tests/system-test/2-query/stateduration.py @@ -38,15 +38,15 @@ class TDTestCase: tdSql.query(f"select stateduration(col{i},'{j}',5) from test") tdSql.checkRows(10) if j in ['LT' ,'lt','Lt','lT']: - tdSql.checkEqual(tdSql.queryResult,[(0,), (0,), (0,), (0,), (-1,), (-1,), (-1,), (-1,), (-1,), (-1,)]) + tdSql.checkEqual(tdSql.queryResult,[(0,), (1,), (2,), (3,), (-1,), (-1,), (-1,), (-1,), (-1,), (-1,)]) elif j in ['GT','gt', 'Gt','gT']: - tdSql.checkEqual(tdSql.queryResult,[(-1,), (-1,), (-1,), (-1,), (-1,), (0,), (0,), (0,), (0,), (0,)]) + tdSql.checkEqual(tdSql.queryResult,[(-1,), (-1,), (-1,), (-1,), (-1,), (0,), (1,), (2,), (3,), (4,)]) elif j in ['LE','le','Le','lE']: - tdSql.checkEqual(tdSql.queryResult,[(0,), (0,), (0,), (0,), (0,), (-1,), (-1,), (-1,), (-1,), (-1,)]) + tdSql.checkEqual(tdSql.queryResult,[(0,), (1,), (2,), (3,), (4,), (-1,), (-1,), (-1,), (-1,), (-1,)]) elif j in [ 'GE','ge','Ge','gE']: - tdSql.checkEqual(tdSql.queryResult,[(-1,), (-1,), (-1,), (-1,), (0,), (0,), (0,), (0,), (0,), (0,)]) + tdSql.checkEqual(tdSql.queryResult,[(-1,), (-1,), (-1,), (-1,), (0,), (1,), (2,), (3,), (4,), (5,)]) elif j in ['NE','ne','Ne','nE']: - tdSql.checkEqual(tdSql.queryResult,[(0,), (0,), (0,), (0,), (-1,), (0,), (0,), (0,), (0,), (0,)]) + tdSql.checkEqual(tdSql.queryResult,[(0,), (1,), (2,), (3,), (-1,), (0,), (1,), (2,), (3,), (4,)]) elif j in ['EQ','eq','Eq','eQ']: tdSql.checkEqual(tdSql.queryResult,[(-1,), (-1,), (-1,), (-1,), (0,), (-1,), (-1,), (-1,), (-1,), (-1,)]) for i in float_list: @@ -54,11 +54,11 @@ class TDTestCase: tdSql.query(f"select stateduration(col{i},'{j}',5) from test") tdSql.checkRows(10) if j in ['LT','lt','Lt','lT','LE','le','Le','lE']: - tdSql.checkEqual(tdSql.queryResult,[(0,), (0,), (0,), (0,), (0,), (-1,), (-1,), (-1,), (-1,), (-1,)]) + tdSql.checkEqual(tdSql.queryResult,[(0,), (1,), (2,), (3,), (4,), (-1,), (-1,), (-1,), (-1,), (-1,)]) elif j in ['GE','ge','Ge','gE','GT','gt','Gt','gT']: - tdSql.checkEqual(tdSql.queryResult,[(-1,), (-1,), (-1,), (-1,), (-1,), (0,), (0,), (0,), (0,), (0,)]) + tdSql.checkEqual(tdSql.queryResult,[(-1,), (-1,), (-1,), (-1,), (-1,), (0,), (1,), (2,), (3,), (4,)]) elif j in ['NE','ne','Ne','nE']: - tdSql.checkEqual(tdSql.queryResult,[(0,), (0,), (0,), (0,), (0,), (0,), (0,), (0,), (0,), (0,)]) + tdSql.checkEqual(tdSql.queryResult,[(0,), (1,), (2,), (3,), (4,), (5,), (6,), (7,), (8,), (9,)]) elif j in ['EQ','eq','Eq','eQ']: tdSql.checkEqual(tdSql.queryResult,[(-1,), (-1,), (-1,), (-1,), (-1,), (-1,), (-1,), (-1,), (-1,), (-1,)]) @@ -66,34 +66,34 @@ class TDTestCase: for i in error_column_list: for j in self.param_list: tdSql.error(f"select stateduration({i},{j},5) from test") - + error_param_list = ['a',1] for i in error_param_list: tdSql.error(f"select stateduration(col1,{i},5) from test") - + # timestamp = 1s, time_unit =1s tdSql.execute('''create table test1(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned)''') for i in range(self.row_num): tdSql.execute("insert into test1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" % (self.ts + i*1000, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) - + for i in integer_list: for j in self.param_list: tdSql.query(f"select stateduration(col{i},'{j}',5) from test1") tdSql.checkRows(10) # print(tdSql.queryResult) if j in ['LT' ,'lt','Lt','lT']: - tdSql.checkEqual(tdSql.queryResult,[(0,), (1,), (2,), (3,), (-1,), (-1,), (-1,), (-1,), (-1,), (-1,)]) + tdSql.checkEqual(tdSql.queryResult,[(0,), (1000,), (2000,), (3000,), (-1,), (-1,), (-1,), (-1,), (-1,), (-1,)]) elif j in ['GT','gt', 'Gt','gT']: - tdSql.checkEqual(tdSql.queryResult,[(-1,), (-1,), (-1,), (-1,), (-1,), (0,), (1,), (2,), (3,), (4,)]) + tdSql.checkEqual(tdSql.queryResult,[(-1,), (-1,), (-1,), (-1,), (-1,), (0,), (1000,), (2000,), (3000,), (4000,)]) elif j in ['LE','le','Le','lE']: - tdSql.checkEqual(tdSql.queryResult,[(0,), (1,), (2,), (3,), (4,), (-1,), (-1,), (-1,), (-1,), (-1,)]) + tdSql.checkEqual(tdSql.queryResult,[(0,), (1000,), (2000,), (3000,), (4000,), (-1,), (-1,), (-1,), (-1,), (-1,)]) elif j in [ 'GE','ge','Ge','gE']: - tdSql.checkEqual(tdSql.queryResult,[(-1,), (-1,), (-1,), (-1,), (0,), (1,), (2,), (3,), (4,), (5,)]) + tdSql.checkEqual(tdSql.queryResult,[(-1,), (-1,), (-1,), (-1,), (0,), (1000,), (2000,), (3000,), (4000,), (5000,)]) elif j in ['NE','ne','Ne','nE']: - tdSql.checkEqual(tdSql.queryResult,[(0,), (1,), (2,), (3,), (-1,), (0,), (1,), (2,), (3,), (4,)]) - elif j in ['EQ','eq','Eq','eQ']: + tdSql.checkEqual(tdSql.queryResult,[(0,), (1000,), (2000,), (3000,), (-1,), (0,), (1000,), (2000,), (3000,), (4000,)]) + elif j in ['EQ','eq','Eq','eQ']: tdSql.checkEqual(tdSql.queryResult,[(-1,), (-1,), (-1,), (-1,), (0,), (-1,), (-1,), (-1,), (-1,), (-1,)]) for i in float_list: for j in self.param_list: @@ -101,22 +101,22 @@ class TDTestCase: tdSql.checkRows(10) print(tdSql.queryResult) if j in ['LT','lt','Lt','lT','LE','le','Le','lE']: - tdSql.checkEqual(tdSql.queryResult,[(0,), (1,), (2,), (3,), (4,), (-1,), (-1,), (-1,), (-1,), (-1,)]) + tdSql.checkEqual(tdSql.queryResult,[(0,), (1000,), (2000,), (3000,), (4000,), (-1,), (-1,), (-1,), (-1,), (-1,)]) elif j in ['GE','ge','Ge','gE','GT','gt','Gt','gT']: - tdSql.checkEqual(tdSql.queryResult,[(-1,), (-1,), (-1,), (-1,), (-1,), (0,), (1,), (2,), (3,), (4,)]) + tdSql.checkEqual(tdSql.queryResult,[(-1,), (-1,), (-1,), (-1,), (-1,), (0,), (1000,), (2000,), (3000,), (4000,)]) elif j in ['NE','ne','Ne','nE']: - tdSql.checkEqual(tdSql.queryResult,[(0,), (1,), (2,), (3,), (4,), (5,), (6,), (7,), (8,), (9,)]) + tdSql.checkEqual(tdSql.queryResult,[(0,), (1000,), (2000,), (3000,), (4000,), (5000,), (6000,), (7000,), (8000,), (9000,)]) elif j in ['EQ','eq','Eq','eQ']: tdSql.checkEqual(tdSql.queryResult,[(-1,), (-1,), (-1,), (-1,), (-1,), (-1,), (-1,), (-1,), (-1,), (-1,)]) # timestamp = 1m, time_unit =1m - tdSql.execute('''create table test2(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + tdSql.execute('''create table test2(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned)''') for i in range(self.row_num): - tdSql.execute("insert into test2 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + tdSql.execute("insert into test2 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" % (self.ts + i*1000*60, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) - + for i in integer_list: for j in self.param_list: tdSql.query(f"select stateduration(col{i},'{j}',5,1m) from test2") @@ -132,7 +132,7 @@ class TDTestCase: tdSql.checkEqual(tdSql.queryResult,[(-1,), (-1,), (-1,), (-1,), (0,), (1,), (2,), (3,), (4,), (5,)]) elif j in ['NE','ne','Ne','nE']: tdSql.checkEqual(tdSql.queryResult,[(0,), (1,), (2,), (3,), (-1,), (0,), (1,), (2,), (3,), (4,)]) - elif j in ['EQ','eq','Eq','eQ']: + elif j in ['EQ','eq','Eq','eQ']: tdSql.checkEqual(tdSql.queryResult,[(-1,), (-1,), (-1,), (-1,), (0,), (-1,), (-1,), (-1,), (-1,), (-1,)]) for i in float_list: for j in self.param_list: @@ -147,14 +147,14 @@ class TDTestCase: tdSql.checkEqual(tdSql.queryResult,[(0,), (1,), (2,), (3,), (4,), (5,), (6,), (7,), (8,), (9,)]) elif j in ['EQ','eq','Eq','eQ']: tdSql.checkEqual(tdSql.queryResult,[(-1,), (-1,), (-1,), (-1,), (-1,), (-1,), (-1,), (-1,), (-1,), (-1,)]) - + # timestamp = 1h, time_unit =1h - tdSql.execute('''create table test3(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + tdSql.execute('''create table test3(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned)''') for i in range(self.row_num): - tdSql.execute("insert into test3 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + tdSql.execute("insert into test3 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" % (self.ts + i*1000*60*60, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) - + for i in integer_list: for j in self.param_list: tdSql.query(f"select stateduration(col{i},'{j}',5,1h) from test3") @@ -170,7 +170,7 @@ class TDTestCase: tdSql.checkEqual(tdSql.queryResult,[(-1,), (-1,), (-1,), (-1,), (0,), (1,), (2,), (3,), (4,), (5,)]) elif j in ['NE','ne','Ne','nE']: tdSql.checkEqual(tdSql.queryResult,[(0,), (1,), (2,), (3,), (-1,), (0,), (1,), (2,), (3,), (4,)]) - elif j in ['EQ','eq','Eq','eQ']: + elif j in ['EQ','eq','Eq','eQ']: tdSql.checkEqual(tdSql.queryResult,[(-1,), (-1,), (-1,), (-1,), (0,), (-1,), (-1,), (-1,), (-1,), (-1,)]) for i in float_list: for j in self.param_list: @@ -202,7 +202,7 @@ class TDTestCase: tdSql.checkEqual(tdSql.queryResult,[(-1,), (-1,), (-1,), (-1,), (0,), (60,), (120,), (180,), (240,), (300,)]) elif j in ['NE','ne','Ne','nE']: tdSql.checkEqual(tdSql.queryResult,[(0,), (60,), (120,), (180,), (-1,), (0,), (60,), (120,), (180,), (240,)]) - elif j in ['EQ','eq','Eq','eQ']: + elif j in ['EQ','eq','Eq','eQ']: tdSql.checkEqual(tdSql.queryResult,[(-1,), (-1,), (-1,), (-1,), (0,), (-1,), (-1,), (-1,), (-1,), (-1,)]) for i in float_list: for j in self.param_list: @@ -219,13 +219,13 @@ class TDTestCase: tdSql.checkEqual(tdSql.queryResult,[(-1,), (-1,), (-1,), (-1,), (-1,), (-1,), (-1,), (-1,), (-1,), (-1,)]) # for stb - tdSql.execute('''create table stb(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + tdSql.execute('''create table stb(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, col7 bool, col8 binary(20), col9 nchar(20), col11 tinyint unsigned, col12 smallint unsigned, col13 int unsigned, col14 bigint unsigned) tags(t0 int)''') tdSql.execute('create table stb_1 using stb tags(1)') for i in range(self.row_num): - tdSql.execute("insert into stb_1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + tdSql.execute("insert into stb_1 values(%d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" % (self.ts + i*1000*60*60, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) - + for i in integer_list: for j in self.param_list: tdSql.query(f"select stateduration(col{i},'{j}',5,1h) from stb") @@ -241,7 +241,7 @@ class TDTestCase: tdSql.checkEqual(tdSql.queryResult,[(-1,), (-1,), (-1,), (-1,), (0,), (1,), (2,), (3,), (4,), (5,)]) elif j in ['NE','ne','Ne','nE']: tdSql.checkEqual(tdSql.queryResult,[(0,), (1,), (2,), (3,), (-1,), (0,), (1,), (2,), (3,), (4,)]) - elif j in ['EQ','eq','Eq','eQ']: + elif j in ['EQ','eq','Eq','eQ']: tdSql.checkEqual(tdSql.queryResult,[(-1,), (-1,), (-1,), (-1,), (0,), (-1,), (-1,), (-1,), (-1,), (-1,)]) for i in float_list: for j in self.param_list: @@ -256,10 +256,10 @@ class TDTestCase: tdSql.checkEqual(tdSql.queryResult,[(0,), (1,), (2,), (3,), (4,), (5,), (6,), (7,), (8,), (9,)]) elif j in ['EQ','eq','Eq','eQ']: tdSql.checkEqual(tdSql.queryResult,[(-1,), (-1,), (-1,), (-1,), (-1,), (-1,), (-1,), (-1,), (-1,), (-1,)]) - + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) tdCases.addWindows(__file__, TDTestCase()) -tdCases.addLinux(__file__, TDTestCase()) \ No newline at end of file +tdCases.addLinux(__file__, TDTestCase()) From 5d694bc5d72544ba428868c6ee3063ace6a6f37a Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 30 Jun 2022 11:45:11 +0800 Subject: [PATCH 53/61] fix translate function for now/today --- source/libs/function/src/builtins.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 1f89538447..b61f95f9a0 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -234,7 +234,7 @@ static int32_t translateWduration(SFunctionNode* pFunc, char* pErrBuf, int32_t l return TSDB_CODE_SUCCESS; } -static int32_t translateTimePseudoColumn(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { +static int32_t translateNowToday(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { // pseudo column do not need to check parameters //add database precision as param @@ -245,6 +245,13 @@ static int32_t translateTimePseudoColumn(SFunctionNode* pFunc, char* pErrBuf, in return TSDB_CODE_SUCCESS; } +static int32_t translateTimePseudoColumn(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + // pseudo column do not need to check parameters + + pFunc->node.resType = (SDataType){.bytes = sizeof(int64_t), .type = TSDB_DATA_TYPE_TIMESTAMP}; + return TSDB_CODE_SUCCESS; +} + static int32_t translateTimezone(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { pFunc->node.resType = (SDataType){.bytes = TD_TIMEZONE_LEN, .type = TSDB_DATA_TYPE_BINARY}; return TSDB_CODE_SUCCESS; @@ -2471,7 +2478,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .name = "now", .type = FUNCTION_TYPE_NOW, .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_DATETIME_FUNC, - .translateFunc = translateTimePseudoColumn, + .translateFunc = translateNowToday, .getEnvFunc = NULL, .initFunc = NULL, .sprocessFunc = nowFunction, @@ -2481,7 +2488,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .name = "today", .type = FUNCTION_TYPE_TODAY, .classification = FUNC_MGT_SCALAR_FUNC | FUNC_MGT_DATETIME_FUNC, - .translateFunc = translateTimePseudoColumn, + .translateFunc = translateNowToday, .getEnvFunc = NULL, .initFunc = NULL, .sprocessFunc = todayFunction, From 80934e48220fde7fc3d8a4e629fbd94d79273762 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Thu, 30 Jun 2022 13:22:54 +0800 Subject: [PATCH 54/61] test: add test case for tmq --- tests/system-test/7-tmq/subscribeDb4.py | 47 ++++++++++++++----------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/tests/system-test/7-tmq/subscribeDb4.py b/tests/system-test/7-tmq/subscribeDb4.py index de475803fe..b30f99b542 100644 --- a/tests/system-test/7-tmq/subscribeDb4.py +++ b/tests/system-test/7-tmq/subscribeDb4.py @@ -14,19 +14,24 @@ sys.path.append("./7-tmq") from tmqCommon import * class TDTestCase: - paraDict = {'dbName': 'db12', - 'dropFlag':1, - 'vgroups': 4, - 'precision': 'ms', - 'stbName': 'stb0', - 'ctbNum': 10, + paraDict = {'dbName': 'db12', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'stbName': 'stb0', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':16, 'count':1}, {'type': 'timestamp','count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbStartIdx': 0, + 'ctbNum': 10, 'rowsPerTbl': 10000, - 'batchNum': 10, - 'startTs': 0, # 1640966400000 ----> 2022-01-01 00:00:00.000 - 'event':'', - 'columnDict': {'int':2}, - 'tagDict': {'int':1} - } + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 20, + 'showMsg': 1, + 'showRow': 1} cdbName = 'cdb' # some parameter to consumer processor @@ -57,17 +62,19 @@ class TDTestCase: tmqCom.initConsumerTable(self.cdbName) - tdCom.create_database(tdSql,self.paraDict["dbName"],self.paraDict["dropFlag"], self.paraDict['precision']) + tdCom.create_database(tdSql,self.paraDict["dbName"],self.paraDict["dropFlag"]) self.paraDict["stbName"] = 'stb1' - tdCom.create_stable(tdSql,self.paraDict["dbName"],self.paraDict["stbName"],self.paraDict["columnDict"],self.paraDict["tagDict"]) - tdCom.create_ctables(tdSql,self.paraDict["dbName"],self.paraDict["stbName"],self.paraDict["ctbNum"],self.paraDict["tagDict"]) - tdCom.insert_data(tdSql,self.paraDict["dbName"],self.paraDict["stbName"],self.paraDict["ctbNum"],self.paraDict["rowsPerTbl"],self.paraDict["batchNum"]) + tdCom.create_stable(tdSql,dbname=self.paraDict["dbName"],stbname=self.paraDict["stbName"],column_elm_list=self.paraDict["colSchema"],tag_elm_list=self.paraDict["tagSchema"],count=1, default_stbname_prefix=self.paraDict["stbName"]) + tdCom.create_ctable(tdSql,dbname=self.paraDict["dbName"],stbname=self.paraDict["stbName"],tag_elm_list=self.paraDict['tagSchema'],count=self.paraDict["ctbNum"],default_ctbname_prefix=self.paraDict["ctbPrefix"]) + tmqCom.insert_data_2(tdSql,self.paraDict["dbName"],self.paraDict["ctbPrefix"],self.paraDict["ctbNum"],self.paraDict["rowsPerTbl"],self.paraDict["batchNum"],self.paraDict["startTs"],self.paraDict["ctbStartIdx"]) + tdLog.info("22222222222222222") self.paraDict["stbName"] = 'stb2' - tdCom.create_stable(tdSql,self.paraDict["dbName"],self.paraDict["stbName"],self.paraDict["columnDict"],self.paraDict["tagDict"]) - tdCom.create_ctables(tdSql,self.paraDict["dbName"],self.paraDict["stbName"],self.paraDict["ctbNum"],self.paraDict["tagDict"]) - tdCom.insert_data(tdSql,self.paraDict["dbName"],self.paraDict["stbName"],self.paraDict["ctbNum"],self.paraDict["rowsPerTbl"],self.paraDict["batchNum"]) + self.paraDict["ctbPrefix"] = 'newctb' + tdCom.create_stable(tdSql,dbname=self.paraDict["dbName"],stbname=self.paraDict["stbName"],column_elm_list=self.paraDict["colSchema"],tag_elm_list=self.paraDict["tagSchema"],count=1, default_stbname_prefix=self.paraDict["stbName"]) + tdCom.create_ctable(tdSql,dbname=self.paraDict["dbName"],stbname=self.paraDict["stbName"],tag_elm_list=self.paraDict['tagSchema'],count=self.paraDict["ctbNum"],default_ctbname_prefix=self.paraDict["ctbPrefix"]) + tmqCom.insert_data_2(tdSql,self.paraDict["dbName"],self.paraDict["ctbPrefix"],self.paraDict["ctbNum"],self.paraDict["rowsPerTbl"],self.paraDict["batchNum"],self.paraDict["startTs"],self.paraDict["ctbStartIdx"]) tdLog.info("create topics from db") topicName1 = 'topic_%s'%(self.paraDict['dbName']) @@ -97,7 +104,7 @@ class TDTestCase: tdLog.info("act consume rows: %d, expect consume rows: between %d and %d"%(totalConsumeRows, self.expectrowcnt/2, self.expectrowcnt)) tdLog.exit("tmq consume rows error!") - time.sleep(15) + time.sleep(10) tdSql.query("drop topic %s"%topicName1) tdLog.printNoPrefix("======== test case 12 end ...... ") From 859d604022f53c1f67e72d6e9ff5fb9e4f7f49c4 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Thu, 30 Jun 2022 13:24:08 +0800 Subject: [PATCH 55/61] test:add test case --- tests/system-test/fulltest.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index d0fee5c49c..73bd21470e 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -135,6 +135,7 @@ python3 ./test.py -f 7-tmq/subscribeDb0.py python3 ./test.py -f 7-tmq/subscribeDb1.py python3 ./test.py -f 7-tmq/subscribeDb2.py python3 ./test.py -f 7-tmq/subscribeDb3.py +#python3 ./test.py -f 7-tmq/subscribeDb4.py python3 ./test.py -f 7-tmq/subscribeStb.py python3 ./test.py -f 7-tmq/subscribeStb0.py python3 ./test.py -f 7-tmq/subscribeStb1.py From ee47df9c2a01363816b9807eb06434d75beabac2 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Thu, 30 Jun 2022 13:46:28 +0800 Subject: [PATCH 56/61] fix(query): support irate function for ms/ns precision db --- source/libs/function/src/builtins.c | 5 +++++ source/libs/function/src/builtinsimpl.c | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index b61f95f9a0..fcaa695345 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -44,6 +44,7 @@ void static addDbPrecisonParam(SNodeList** pList, uint8_t precision) { pVal->literal = NULL; pVal->isDuration = false; pVal->translate = true; + pVal->notReserved = true; pVal->node.resType.type = TSDB_DATA_TYPE_TINYINT; pVal->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_TINYINT].bytes; pVal->node.resType.precision = precision; @@ -1016,6 +1017,10 @@ static int32_t translateIrate(SFunctionNode* pFunc, char* pErrBuf, int32_t len) return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } + //add database precision as param + uint8_t dbPrec = pFunc->node.resType.precision; + addDbPrecisonParam(&pFunc->pParameterList, dbPrec); + pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes, .type = TSDB_DATA_TYPE_DOUBLE}; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index a75d5a2ecb..07b2a2375f 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -5508,7 +5508,7 @@ int32_t irateFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { pResInfo->isNullRes = (pResInfo->numOfRes == 0) ? 1 : 0; SRateInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo); - double result = doCalcRate(pInfo, 1000); + double result = doCalcRate(pInfo, (double)TSDB_TICK_PER_SECOND(pCtx->param[1].param.i)); colDataAppend(pCol, pBlock->info.rows, (const char*)&result, pResInfo->isNullRes); return pResInfo->numOfRes; From 45e9b239015bde25882acd1d7b29a8d2a1889ac3 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 30 Jun 2022 13:53:11 +0800 Subject: [PATCH 57/61] fix: install script for3.0 (#14379) * fix: remove sudo from directory maniplation for 3.0 * fix: remove more sudo from directory maniplation * fix: packaging/release.sh --- packaging/release.sh | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/packaging/release.sh b/packaging/release.sh index 1d9f6dfc68..2ac52a0df9 100755 --- a/packaging/release.sh +++ b/packaging/release.sh @@ -111,9 +111,9 @@ else fi csudo="" -if command -v sudo > /dev/null; then - csudo="sudo " -fi +#if command -v sudo > /dev/null; then +# csudo="sudo " +#fi function is_valid_version() { [ -z $1 ] && return 1 || : @@ -181,7 +181,9 @@ cd "${curr_dir}" # 2. cmake executable file compile_dir="${top_dir}/debug" -${csudo}rm -rf ${compile_dir} +if [ -d ${compile_dir} ]; then + rm -rf ${compile_dir} +fi mkdir -p ${compile_dir} cd ${compile_dir} @@ -258,9 +260,9 @@ if [ "$osType" != "Darwin" ]; then if [[ "$pagMode" == "full" ]]; then if [ -d ${top_dir}/tools/taos-tools/packaging/deb ]; then cd ${top_dir}/tools/taos-tools/packaging/deb + taos_tools_ver=$(git describe --tags | sed -e 's/ver-//g' | awk -F '-' '{print $1}') [ -z "$taos_tools_ver" ] && taos_tools_ver="0.1.0" - taos_tools_ver=$(git describe --tags | sed -e 's/ver-//g' | awk -F '-' '{print $1}') ${csudo}./make-taos-tools-deb.sh ${top_dir} \ ${compile_dir} ${output_dir} ${taos_tools_ver} ${cpuType} ${osType} ${verMode} ${verType} fi @@ -283,9 +285,9 @@ if [ "$osType" != "Darwin" ]; then if [[ "$pagMode" == "full" ]]; then if [ -d ${top_dir}/tools/taos-tools/packaging/rpm ]; then cd ${top_dir}/tools/taos-tools/packaging/rpm + taos_tools_ver=$(git describe --tags | sed -e 's/ver-//g' | awk -F '-' '{print $1}' | sed -e 's/-/_/g') [ -z "$taos_tools_ver" ] && taos_tools_ver="0.1.0" - taos_tools_ver=$(git describe --tags | sed -e 's/ver-//g' | awk -F '-' '{print $1}' | sed -e 's/-/_/g') ${csudo}./make-taos-tools-rpm.sh ${top_dir} \ ${compile_dir} ${output_dir} ${taos_tools_ver} ${cpuType} ${osType} ${verMode} ${verType} fi @@ -300,7 +302,7 @@ if [ "$osType" != "Darwin" ]; then ${csudo}./makepkg.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${verNumberComp} ${dbName} ${csudo}./makeclient.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${dbName} - # ${csudo}./makearbi.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} + ${csudo}./makearbi.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} else # only make client for Darwin From e79fcc07f5593b3d7baf2499b9134abe42849631 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 30 Jun 2022 14:00:48 +0800 Subject: [PATCH 58/61] fix: fix stmt async issue --- source/client/src/clientImpl.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 09fb73cfba..4a537a3aeb 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1407,7 +1407,11 @@ void* doAsyncFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertU } SSyncQueryParam* pParam = pRequest->body.param; - + if (NULL == pParam) { + pParam = taosMemoryCalloc(1, sizeof(SSyncQueryParam)); + tsem_init(&pParam->sem, 0, 0); + } + // convert ucs4 to native multi-bytes string pResultInfo->convertUcs4 = convertUcs4; From 9a2af8580c7f971999ea549ba9be8b052a4531f2 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 30 Jun 2022 15:06:52 +0800 Subject: [PATCH 59/61] fix:compile error in sml --- source/client/src/clientSml.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 8a9905935a..e724637620 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -2450,6 +2450,7 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr return NULL; } + int batchs = 0; pTscObj->schemalessType = 1; SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; @@ -2497,8 +2498,8 @@ TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int pr goto end; } - int cnt = ceil(((double)numLines) / LINE_BATCH); - for (int i = 0; i < cnt; ++i) { + batchs = ceil(((double)numLines) / LINE_BATCH); + for (int i = 0; i < batchs; ++i) { SRequestObj* req = (SRequestObj*)createRequest(pTscObj, TSDB_SQL_INSERT); if(!req){ request->code = TSDB_CODE_OUT_OF_MEMORY; From 6305f6cf6497e7f55993a8b150a66e49dfc7db49 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Thu, 30 Jun 2022 15:27:31 +0800 Subject: [PATCH 60/61] test:modify case --- tests/system-test/7-tmq/subscribeDb4.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/system-test/7-tmq/subscribeDb4.py b/tests/system-test/7-tmq/subscribeDb4.py index b30f99b542..b99704b602 100644 --- a/tests/system-test/7-tmq/subscribeDb4.py +++ b/tests/system-test/7-tmq/subscribeDb4.py @@ -69,7 +69,6 @@ class TDTestCase: tdCom.create_ctable(tdSql,dbname=self.paraDict["dbName"],stbname=self.paraDict["stbName"],tag_elm_list=self.paraDict['tagSchema'],count=self.paraDict["ctbNum"],default_ctbname_prefix=self.paraDict["ctbPrefix"]) tmqCom.insert_data_2(tdSql,self.paraDict["dbName"],self.paraDict["ctbPrefix"],self.paraDict["ctbNum"],self.paraDict["rowsPerTbl"],self.paraDict["batchNum"],self.paraDict["startTs"],self.paraDict["ctbStartIdx"]) - tdLog.info("22222222222222222") self.paraDict["stbName"] = 'stb2' self.paraDict["ctbPrefix"] = 'newctb' tdCom.create_stable(tdSql,dbname=self.paraDict["dbName"],stbname=self.paraDict["stbName"],column_elm_list=self.paraDict["colSchema"],tag_elm_list=self.paraDict["tagSchema"],count=1, default_stbname_prefix=self.paraDict["stbName"]) From 00d342959ea26a3c7936ee966e007e338e3bfe57 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Thu, 30 Jun 2022 15:50:04 +0800 Subject: [PATCH 61/61] fix: remove arbitrator relatives from scripts (#14388) * fix: remove sudo from directory maniplation for 3.0 * fix: remove more sudo from directory maniplation * fix: packaging/release.sh * fix: remove arbitrator relatives from scripts --- packaging/cfg/tarbitratord.service | 20 -- packaging/deb/tarbitratord | 88 -------- packaging/release.sh | 1 - packaging/rpm/tarbitratord | 141 ------------ packaging/tools/install.sh | 14 -- packaging/tools/install_arbi.sh | 340 ----------------------------- packaging/tools/make_install.sh | 1 - packaging/tools/makearbi.sh | 71 ------ packaging/tools/makepkg.sh | 9 - packaging/tools/remove_arbi.sh | 132 ----------- 10 files changed, 817 deletions(-) delete mode 100644 packaging/cfg/tarbitratord.service delete mode 100644 packaging/deb/tarbitratord delete mode 100644 packaging/rpm/tarbitratord delete mode 100755 packaging/tools/install_arbi.sh delete mode 100755 packaging/tools/makearbi.sh delete mode 100755 packaging/tools/remove_arbi.sh diff --git a/packaging/cfg/tarbitratord.service b/packaging/cfg/tarbitratord.service deleted file mode 100644 index d60cb536b0..0000000000 --- a/packaging/cfg/tarbitratord.service +++ /dev/null @@ -1,20 +0,0 @@ -[Unit] -Description=TDengine arbitrator service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -ExecStart=/usr/bin/tarbitrator -TimeoutStopSec=1000000s -LimitNOFILE=infinity -LimitNPROC=infinity -LimitCORE=infinity -TimeoutStartSec=0 -StandardOutput=null -Restart=always -StartLimitBurst=3 -StartLimitInterval=60s - -[Install] -WantedBy=multi-user.target diff --git a/packaging/deb/tarbitratord b/packaging/deb/tarbitratord deleted file mode 100644 index 3f97c3c0c2..0000000000 --- a/packaging/deb/tarbitratord +++ /dev/null @@ -1,88 +0,0 @@ -#!/bin/bash -# -# Modified from original source: Elastic Search -# https://github.com/elasticsearch/elasticsearch -# Thank you to the Elastic Search authors -# -# chkconfig: 2345 99 01 -# -### BEGIN INIT INFO -# Provides: taoscluster -# Required-Start: $local_fs $network $syslog -# Required-Stop: $local_fs $network $syslog -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: Starts taoscluster tarbitrator -# Description: Starts taoscluster tarbitrator, a arbitrator -### END INIT INFO - -set -e - -PATH="/bin:/usr/bin:/sbin:/usr/sbin" -NAME="taoscluster" -USER="root" -GROUP="root" -DAEMON="/usr/local/taos/bin/tarbitrator" -DAEMON_OPTS="" -PID_FILE="/var/run/$NAME.pid" -APPARGS="" - -# Maximum number of open files -MAX_OPEN_FILES=65535 - -. /lib/lsb/init-functions - -case "$1" in - start) - - log_action_begin_msg "Starting tarbitrator..." - if start-stop-daemon --test --start --chuid "$USER:$GROUP" --background --make-pidfile --pidfile "$PID_FILE" --exec "$DAEMON" -- $APPARGS &> /dev/null; then - - touch "$PID_FILE" && chown "$USER":"$GROUP" "$PID_FILE" - - if [ -n "$MAX_OPEN_FILES" ]; then - ulimit -n $MAX_OPEN_FILES - fi - - start-stop-daemon --start --chuid "$USER:$GROUP" --background --make-pidfile --pidfile "$PID_FILE" --exec "$DAEMON" -- $APPARGS - - log_end_msg $? - fi - ;; - - stop) - log_action_begin_msg "Stopping tarbitrator..." - set +e - if [ -f "$PID_FILE" ]; then - start-stop-daemon --stop --pidfile "$PID_FILE" --user "$USER" --retry=TERM/120/KILL/5 > /dev/null - if [ $? -eq 1 ]; then - log_action_cont_msg "TSD is not running but pid file exists, cleaning up" - elif [ $? -eq 3 ]; then - PID="`cat $PID_FILE`" - log_failure_msg "Failed to stop tarbitrator (pid $PID)" - exit 1 - fi - rm -f "$PID_FILE" - else - log_action_cont_msg "tarbitrator was not running" - fi - log_action_end_msg 0 - set -e - ;; - - restart|force-reload) - if [ -f "$PID_FILE" ]; then - $0 stop - sleep 1 - fi - $0 start - ;; - status) - status_of_proc -p "$PID_FILE" "$DAEMON" "$NAME" - ;; - *) - exit 1 - ;; -esac - -exit 0 diff --git a/packaging/release.sh b/packaging/release.sh index 2ac52a0df9..3426c2856d 100755 --- a/packaging/release.sh +++ b/packaging/release.sh @@ -302,7 +302,6 @@ if [ "$osType" != "Darwin" ]; then ${csudo}./makepkg.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${verNumberComp} ${dbName} ${csudo}./makeclient.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} ${dbName} - ${csudo}./makearbi.sh ${compile_dir} ${verNumber} "${build_time}" ${cpuType} ${osType} ${verMode} ${verType} ${pagMode} else # only make client for Darwin diff --git a/packaging/rpm/tarbitratord b/packaging/rpm/tarbitratord deleted file mode 100644 index 68138f5c1d..0000000000 --- a/packaging/rpm/tarbitratord +++ /dev/null @@ -1,141 +0,0 @@ -#!/bin/bash -# -# tarbitratord This shell script takes care of starting and stopping tarbitrator. -# -# chkconfig: 2345 99 01 -# description: tarbitrator is a arbitrator used in TDengine cluster. -# -# -### BEGIN INIT INFO -# Provides: taoscluster -# Required-Start: $network $local_fs $remote_fs -# Required-Stop: $network $local_fs $remote_fs -# Short-Description: start and stop tarbitrator -# Description: tarbitrator is a arbitrator used in TDengine cluster. -### END INIT INFO - -# Source init functions -. /etc/init.d/functions - -# Maximum number of open files -MAX_OPEN_FILES=65535 - -# Default program options -NAME=tarbitrator -PROG=/usr/local/taos/bin/tarbitrator -USER=root -GROUP=root - -# Default directories -LOCK_DIR=/var/lock/subsys -PID_DIR=/var/run/$NAME - -# Set file names -LOCK_FILE=$LOCK_DIR/$NAME -PID_FILE=$PID_DIR/$NAME.pid - -[ -e $PID_DIR ] || mkdir -p $PID_DIR - -PROG_OPTS="" - -start() { - echo -n "Starting ${NAME}: " - # check identity - curid="`id -u -n`" - if [ "$curid" != root ] && [ "$curid" != "$USER" ] ; then - echo "Must be run as root or $USER, but was run as $curid" - return 1 - fi - # Sets the maximum number of open file descriptors allowed. - ulimit -n $MAX_OPEN_FILES - curulimit="`ulimit -n`" - if [ "$curulimit" -lt $MAX_OPEN_FILES ] ; then - echo "'ulimit -n' must be greater than or equal to $MAX_OPEN_FILES, is $curulimit" - return 1 - fi - - if [ "`id -u -n`" == root ] ; then - # Changes the owner of the lock, and the pid files to allow - # non-root OpenTSDB daemons to run /usr/share/opentsdb/bin/opentsdb_restart.py. - touch $LOCK_FILE && chown $USER:$GROUP $LOCK_FILE - touch $PID_FILE && chown $USER:$GROUP $PID_FILE - daemon --user $USER --pidfile $PID_FILE "$PROG $PROG_OPTS &> /dev/null &" - else - # Don't have to change user. - daemon --pidfile $PID_FILE "$PROG $PROG_OPTS &> /dev/null &" - fi - retval=$? - sleep 2 - echo - [ $retval -eq 0 ] && (findproc > $PID_FILE && touch $LOCK_FILE) - return $retval -} - -stop() { - echo -n "Stopping ${NAME}: " - killproc -p $PID_FILE $NAME - retval=$? - echo - # Non-root users don't have enough permission to remove pid and lock files. - # So, the opentsdb_restart.py cannot get rid of the files, and the command - # "service opentsdb status" will complain about the existing pid file. - # Makes the pid file empty. - echo > $PID_FILE - [ $retval -eq 0 ] && (rm -f $PID_FILE && rm -f $LOCK_FILE) - return $retval -} - -restart() { - stop - start -} - -reload() { - restart -} - -force_reload() { - restart -} - -rh_status() { - # run checks to determine if the service is running or use generic status - status -p $PID_FILE -l $LOCK_FILE $NAME -} - -rh_status_q() { - rh_status >/dev/null 2>&1 -} - -case "$1" in - start) - rh_status_q && exit 0 - $1 - ;; - stop) - rh_status_q || exit 0 - $1 - ;; - restart) - $1 - ;; - reload) - rh_status_q || exit 7 - $1 - ;; - force-reload) - force_reload - ;; - status) - rh_status - ;; - condrestart|try-restart) - rh_status_q || exit 0 - restart - ;; - *) - echo "Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload}" - exit 2 -esac - -exit $? diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 7c7d5477cf..2d17ef4812 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -194,7 +194,6 @@ function install_bin() { ${csudo}rm -f ${bin_link_dir}/${serverName} || : ${csudo}rm -f ${bin_link_dir}/${adapterName} || : ${csudo}rm -f ${bin_link_dir}/${uninstallScript} || : - ${csudo}rm -f ${bin_link_dir}/tarbitrator || : ${csudo}rm -f ${bin_link_dir}/set_core || : ${csudo}rm -f ${bin_link_dir}/TDinsight.sh || : @@ -210,7 +209,6 @@ function install_bin() { [ -x ${install_main_dir}/bin/TDinsight.sh ] && ${csudo}ln -s ${install_main_dir}/bin/TDinsight.sh ${bin_link_dir}/TDinsight.sh || : [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || : [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : - [ -x ${install_main_dir}/bin/tarbitrator ] && ${csudo}ln -s ${install_main_dir}/bin/tarbitrator ${bin_link_dir}/tarbitrator || : if [ "$verMode" == "cluster" ]; then ${csudo}cp -r ${script_dir}/nginxd/* ${nginx_dir} && ${csudo}chmod 0555 ${nginx_dir}/* @@ -606,28 +604,19 @@ function install_service_on_sysvinit() { if ((${os_type} == 1)); then # ${csudo}cp -f ${script_dir}/init.d/${serverName}.deb ${install_main_dir}/init.d/${serverName} ${csudo}cp ${script_dir}/init.d/${serverName}.deb ${service_config_dir}/${serverName} && ${csudo}chmod a+x ${service_config_dir}/${serverName} - # ${csudo}cp -f ${script_dir}/init.d/tarbitratord.deb ${install_main_dir}/init.d/tarbitratord - ${csudo}cp ${script_dir}/init.d/tarbitratord.deb ${service_config_dir}/tarbitratord && ${csudo}chmod a+x ${service_config_dir}/tarbitratord elif ((${os_type} == 2)); then # ${csudo}cp -f ${script_dir}/init.d/${serverName}.rpm ${install_main_dir}/init.d/${serverName} ${csudo}cp ${script_dir}/init.d/${serverName}.rpm ${service_config_dir}/${serverName} && ${csudo}chmod a+x ${service_config_dir}/${serverName} - # ${csudo}cp -f ${script_dir}/init.d/tarbitratord.rpm ${install_main_dir}/init.d/tarbitratord - ${csudo}cp ${script_dir}/init.d/tarbitratord.rpm ${service_config_dir}/tarbitratord && ${csudo}chmod a+x ${service_config_dir}/tarbitratord fi if ((${initd_mod} == 1)); then ${csudo}chkconfig --add ${serverName} || : ${csudo}chkconfig --level 2345 ${serverName} on || : - ${csudo}chkconfig --add tarbitratord || : - ${csudo}chkconfig --level 2345 tarbitratord on || : elif ((${initd_mod} == 2)); then ${csudo}insserv ${serverName} || : ${csudo}insserv -d ${serverName} || : - ${csudo}insserv tarbitratord || : - ${csudo}insserv -d tarbitratord || : elif ((${initd_mod} == 3)); then ${csudo}update-rc.d ${serverName} defaults || : - ${csudo}update-rc.d tarbitratord defaults || : fi } @@ -669,9 +658,6 @@ function install_service_on_systemd() { ${csudo}systemctl enable ${serverName} - [ -f ${script_dir}/cfg/tarbitratord.service ] && - ${csudo}cp ${script_dir}/cfg/tarbitratord.service \ - ${service_config_dir}/ || : ${csudo}systemctl daemon-reload if [ "$verMode" == "cluster" ]; then diff --git a/packaging/tools/install_arbi.sh b/packaging/tools/install_arbi.sh deleted file mode 100755 index 2863640153..0000000000 --- a/packaging/tools/install_arbi.sh +++ /dev/null @@ -1,340 +0,0 @@ -#!/bin/bash -# -# This file is used to install database on linux systems. The operating system -# is required to use systemd to manage services at boot - -set -e -#set -x - -# -----------------------Variables definition--------------------- -script_dir=$(dirname $(readlink -f "$0")) - -bin_link_dir="/usr/bin" -#inc_link_dir="/usr/include" - -#install main path -install_main_dir="/usr/local/tarbitrator" - -# old bin dir -bin_dir="/usr/local/tarbitrator/bin" - -service_config_dir="/etc/systemd/system" - -# Color setting -RED='\033[0;31m' -GREEN='\033[1;32m' -GREEN_DARK='\033[0;32m' -GREEN_UNDERLINE='\033[4;32m' -NC='\033[0m' - -csudo="" -if command -v sudo >/dev/null; then - csudo="sudo " -fi - -update_flag=0 - -initd_mod=0 -service_mod=2 -if pidof systemd &>/dev/null; then - service_mod=0 -elif $(which service &>/dev/null); then - service_mod=1 - service_config_dir="/etc/init.d" - if $(which chkconfig &>/dev/null); then - initd_mod=1 - elif $(which insserv &>/dev/null); then - initd_mod=2 - elif $(which update-rc.d &>/dev/null); then - initd_mod=3 - else - service_mod=2 - fi -else - service_mod=2 -fi - -# get the operating system type for using the corresponding init file -# ubuntu/debian(deb), centos/fedora(rpm), others: opensuse, redhat, ..., no verification -#osinfo=$(awk -F= '/^NAME/{print $2}' /etc/os-release) -if [[ -e /etc/os-release ]]; then - osinfo=$(cat /etc/os-release | grep "NAME" | cut -d '"' -f2) || : -else - osinfo="" -fi -#echo "osinfo: ${osinfo}" -os_type=0 -if echo $osinfo | grep -qwi "ubuntu"; then - # echo "This is ubuntu system" - os_type=1 -elif echo $osinfo | grep -qwi "debian"; then - # echo "This is debian system" - os_type=1 -elif echo $osinfo | grep -qwi "Kylin"; then - # echo "This is Kylin system" - os_type=1 -elif echo $osinfo | grep -qwi "centos"; then - # echo "This is centos system" - os_type=2 -elif echo $osinfo | grep -qwi "fedora"; then - # echo "This is fedora system" - os_type=2 -else - echo " osinfo: ${osinfo}" - echo " This is an officially unverified linux system," - echo " if there are any problems with the installation and operation, " - echo " please feel free to contact taosdata.com for support." - os_type=1 -fi - -function kill_tarbitrator() { - pid=$(ps -ef | grep "tarbitrator" | grep -v "grep" | awk '{print $2}') - if [ -n "$pid" ]; then - ${csudo}kill -9 $pid || : - fi -} - -function install_main_path() { - #create install main dir and all sub dir - ${csudo}rm -rf ${install_main_dir} || : - ${csudo}mkdir -p ${install_main_dir} - ${csudo}mkdir -p ${install_main_dir}/bin - #${csudo}mkdir -p ${install_main_dir}/include - ${csudo}mkdir -p ${install_main_dir}/init.d -} - -function install_bin() { - # Remove links - ${csudo}rm -f ${bin_link_dir}/rmtarbitrator || : - ${csudo}rm -f ${bin_link_dir}/tarbitrator || : - ${csudo}cp -r ${script_dir}/bin/* ${install_main_dir}/bin && ${csudo}chmod 0555 ${install_main_dir}/bin/* - - #Make link - [ -x ${install_main_dir}/bin/remove_arbi.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove_arbi.sh ${bin_link_dir}/rmtarbitrator || : - [ -x ${install_main_dir}/bin/tarbitrator ] && ${csudo}ln -s ${install_main_dir}/bin/tarbitrator ${bin_link_dir}/tarbitrator || : -} - -function install_header() { - ${csudo}rm -f ${inc_link_dir}/taos.h ${inc_link_dir}/taosdef.h ${inc_link_dir}/taoserror.h ${inc_link_dir}/taosudf.h || : - ${csudo}cp -f ${script_dir}/inc/* ${install_main_dir}/include && ${csudo}chmod 644 ${install_main_dir}/include/* - ${csudo}ln -s ${install_main_dir}/include/taos.h ${inc_link_dir}/taos.h - ${csudo}ln -s ${install_main_dir}/include/taosdef.h ${inc_link_dir}/taosdef.h - ${csudo}ln -s ${install_main_dir}/include/taoserror.h ${inc_link_dir}/taoserror.h - ${csudo}ln -s ${install_main_dir}/include/taosudf.h ${inc_link_dir}/taosudf.h -} - -function install_jemalloc() { - jemalloc_dir=${script_dir}/jemalloc - - if [ -d ${jemalloc_dir} ]; then - ${csudo}/usr/bin/install -c -d /usr/local/bin - - if [ -f ${jemalloc_dir}/bin/jemalloc-config ]; then - ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc-config /usr/local/bin - fi - if [ -f ${jemalloc_dir}/bin/jemalloc.sh ]; then - ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jemalloc.sh /usr/local/bin - fi - if [ -f ${jemalloc_dir}/bin/jeprof ]; then - ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/bin/jeprof /usr/local/bin - fi - if [ -f ${jemalloc_dir}/include/jemalloc/jemalloc.h ]; then - ${csudo}/usr/bin/install -c -d /usr/local/include/jemalloc - ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/include/jemalloc/jemalloc.h /usr/local/include/jemalloc - fi - if [ -f ${jemalloc_dir}/lib/libjemalloc.so.2 ]; then - ${csudo}/usr/bin/install -c -d /usr/local/lib - ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.so.2 /usr/local/lib - ${csudo}ln -sf libjemalloc.so.2 /usr/local/lib/libjemalloc.so - ${csudo}/usr/bin/install -c -d /usr/local/lib - if [ -f ${jemalloc_dir}/lib/libjemalloc.a ]; then - ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc.a /usr/local/lib - fi - if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then - ${csudo}/usr/bin/install -c -m 755 ${jemalloc_dir}/lib/libjemalloc_pic.a /usr/local/lib - fi - if [ -f ${jemalloc_dir}/lib/libjemalloc_pic.a ]; then - ${csudo}/usr/bin/install -c -d /usr/local/lib/pkgconfig - ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/lib/pkgconfig/jemalloc.pc /usr/local/lib/pkgconfig - fi - fi - if [ -f ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html ]; then - ${csudo}/usr/bin/install -c -d /usr/local/share/doc/jemalloc - ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/share/doc/jemalloc/jemalloc.html /usr/local/share/doc/jemalloc - fi - if [ -f ${jemalloc_dir}/share/man/man3/jemalloc.3 ]; then - ${csudo}/usr/bin/install -c -d /usr/local/share/man/man3 - ${csudo}/usr/bin/install -c -m 644 ${jemalloc_dir}/share/man/man3/jemalloc.3 /usr/local/share/man/man3 - fi - - if [ -d /etc/ld.so.conf.d ]; then - echo "/usr/local/lib" | ${csudo}tee /etc/ld.so.conf.d/jemalloc.conf >/dev/null || echo -e "failed to write /etc/ld.so.conf.d/jemalloc.conf" - ${csudo}ldconfig - else - echo "/etc/ld.so.conf.d not found!" - fi - fi -} - -function clean_service_on_sysvinit() { - if pidof tarbitrator &>/dev/null; then - ${csudo}service tarbitratord stop || : - fi - - if ((${initd_mod} == 1)); then - if [ -e ${service_config_dir}/tarbitratord ]; then - ${csudo}chkconfig --del tarbitratord || : - fi - elif ((${initd_mod} == 2)); then - if [ -e ${service_config_dir}/tarbitratord ]; then - ${csudo}insserv -r tarbitratord || : - fi - elif ((${initd_mod} == 3)); then - if [ -e ${service_config_dir}/tarbitratord ]; then - ${csudo}update-rc.d -f tarbitratord remove || : - fi - fi - - ${csudo}rm -f ${service_config_dir}/tarbitratord || : - - if $(which init &>/dev/null); then - ${csudo}init q || : - fi -} - -function install_service_on_sysvinit() { - clean_service_on_sysvinit - sleep 1 - - if ((${os_type} == 1)); then - ${csudo}cp -f ${script_dir}/init.d/tarbitratord.deb ${install_main_dir}/init.d/tarbitratord - ${csudo}cp ${script_dir}/init.d/tarbitratord.deb ${service_config_dir}/tarbitratord && ${csudo}chmod a+x ${service_config_dir}/tarbitratord - elif ((${os_type} == 2)); then - ${csudo}cp -f ${script_dir}/init.d/tarbitratord.rpm ${install_main_dir}/init.d/tarbitratord - ${csudo}cp ${script_dir}/init.d/tarbitratord.rpm ${service_config_dir}/tarbitratord && ${csudo}chmod a+x ${service_config_dir}/tarbitratord - fi - - if ((${initd_mod} == 1)); then - ${csudo}chkconfig --add tarbitratord || : - ${csudo}chkconfig --level 2345 tarbitratord on || : - elif ((${initd_mod} == 2)); then - ${csudo}insserv tarbitratord || : - ${csudo}insserv -d tarbitratord || : - elif ((${initd_mod} == 3)); then - ${csudo}update-rc.d tarbitratord defaults || : - fi -} - -function clean_service_on_systemd() { - tarbitratord_service_config="${service_config_dir}/tarbitratord.service" - if systemctl is-active --quiet tarbitratord; then - echo "tarbitrator is running, stopping it..." - ${csudo}systemctl stop tarbitratord &>/dev/null || echo &>/dev/null - fi - ${csudo}systemctl disable tarbitratord &>/dev/null || echo &>/dev/null - - ${csudo}rm -f ${tarbitratord_service_config} -} - -function install_service_on_systemd() { - clean_service_on_systemd - - tarbitratord_service_config="${service_config_dir}/tarbitratord.service" - - ${csudo}bash -c "echo '[Unit]' >> ${tarbitratord_service_config}" - ${csudo}bash -c "echo 'Description=TDengine arbitrator service' >> ${tarbitratord_service_config}" - ${csudo}bash -c "echo 'After=network-online.target' >> ${tarbitratord_service_config}" - ${csudo}bash -c "echo 'Wants=network-online.target' >> ${tarbitratord_service_config}" - ${csudo}bash -c "echo >> ${tarbitratord_service_config}" - ${csudo}bash -c "echo '[Service]' >> ${tarbitratord_service_config}" - ${csudo}bash -c "echo 'Type=simple' >> ${tarbitratord_service_config}" - ${csudo}bash -c "echo 'ExecStart=/usr/bin/tarbitrator' >> ${tarbitratord_service_config}" - ${csudo}bash -c "echo 'TimeoutStopSec=1000000s' >> ${tarbitratord_service_config}" - ${csudo}bash -c "echo 'LimitNOFILE=infinity' >> ${tarbitratord_service_config}" - ${csudo}bash -c "echo 'LimitNPROC=infinity' >> ${tarbitratord_service_config}" - ${csudo}bash -c "echo 'LimitCORE=infinity' >> ${tarbitratord_service_config}" - ${csudo}bash -c "echo 'TimeoutStartSec=0' >> ${tarbitratord_service_config}" - ${csudo}bash -c "echo 'StandardOutput=null' >> ${tarbitratord_service_config}" - ${csudo}bash -c "echo 'Restart=always' >> ${tarbitratord_service_config}" - ${csudo}bash -c "echo 'StartLimitBurst=3' >> ${tarbitratord_service_config}" - ${csudo}bash -c "echo 'StartLimitInterval=60s' >> ${tarbitratord_service_config}" - ${csudo}bash -c "echo >> ${tarbitratord_service_config}" - ${csudo}bash -c "echo '[Install]' >> ${tarbitratord_service_config}" - ${csudo}bash -c "echo 'WantedBy=multi-user.target' >> ${tarbitratord_service_config}" - ${csudo}systemctl enable tarbitratord -} - -function install_service() { - if ((${service_mod} == 0)); then - install_service_on_systemd - elif ((${service_mod} == 1)); then - install_service_on_sysvinit - else - kill_tarbitrator - fi -} - -function update_TDengine() { - # Start to update - echo -e "${GREEN}Start to update TDengine's arbitrator ...${NC}" - # Stop the service if running - if pidof tarbitrator &>/dev/null; then - if ((${service_mod} == 0)); then - ${csudo}systemctl stop tarbitratord || : - elif ((${service_mod} == 1)); then - ${csudo}service tarbitratord stop || : - else - kill_tarbitrator - fi - sleep 1 - fi - - install_main_path - #install_header - install_bin - install_service - install_jemalloc - - echo - if ((${service_mod} == 0)); then - echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo}systemctl start tarbitratord${NC}" - elif ((${service_mod} == 1)); then - echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo}service tarbitratord start${NC}" - else - echo -e "${GREEN_DARK}To start arbitrator ${NC}: ./tarbitrator${NC}" - fi - echo - echo -e "\033[44;32;1mTDengine's arbitrator is updated successfully!${NC}" -} - -function install_TDengine() { - # Start to install - echo -e "${GREEN}Start to install TDengine's arbitrator ...${NC}" - - install_main_path - #install_header - install_bin - install_service - install_jemalloc - - echo - if ((${service_mod} == 0)); then - echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo}systemctl start tarbitratord${NC}" - elif ((${service_mod} == 1)); then - echo -e "${GREEN_DARK}To start arbitrator ${NC}: ${csudo}service tarbitratord start${NC}" - else - echo -e "${GREEN_DARK}To start arbitrator ${NC}: tarbitrator${NC}" - fi - - echo -e "\033[44;32;1mTDengine's arbitrator is installed successfully!${NC}" - echo -} - -## ==============================Main program starts from here============================ -# Install server and client -if [ -x ${bin_dir}/tarbitrator ]; then - update_flag=1 - update_TDengine -else - install_TDengine -fi diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index 59be60f8fc..4e4ebb72c0 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -185,7 +185,6 @@ function install_bin() { [ -f ${binary_dir}/build/bin/taosadapter ] && ${csudo}cp -r ${binary_dir}/build/bin/taosadapter ${install_main_dir}/bin || : [ -f ${binary_dir}/build/bin/udfd ] && ${csudo}cp -r ${binary_dir}/build/bin/udfd ${install_main_dir}/bin || : ${csudo}cp -r ${binary_dir}/build/bin/${serverName} ${install_main_dir}/bin || : - # ${csudo}cp -r ${binary_dir}/build/bin/tarbitrator ${install_main_dir}/bin || : ${csudo}cp -r ${script_dir}/taosd-dump-cfg.gdb ${install_main_dir}/bin || : ${csudo}cp -r ${script_dir}/remove.sh ${install_main_dir}/bin || : diff --git a/packaging/tools/makearbi.sh b/packaging/tools/makearbi.sh deleted file mode 100755 index cfae2b4a46..0000000000 --- a/packaging/tools/makearbi.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/bin/bash -# -# Generate arbitrator's tar.gz setup package for all os system - -set -e -#set -x - -curr_dir=$(pwd) -compile_dir=$1 -version=$2 -build_time=$3 -cpuType=$4 -osType=$5 -verMode=$6 -verType=$7 -pagMode=$8 - -script_dir="$(dirname $(readlink -f $0))" -top_dir="$(readlink -f ${script_dir}/../..)" - -productName="TDengine" - -# create compressed install file. -build_dir="${compile_dir}/build" -code_dir="${top_dir}" -release_dir="${top_dir}/release" - -#package_name='linux' -if [ "$verMode" == "cluster" ]; then - install_dir="${release_dir}/${productName}-enterprise-arbitrator-${version}" -else - install_dir="${release_dir}/${productName}-arbitrator-${version}" -fi - -# Directories and files. -bin_files="${build_dir}/bin/tarbitrator ${script_dir}/remove_arbi.sh" -install_files="${script_dir}/install_arbi.sh" - -#header_files="${code_dir}/include/client/taos.h ${code_dir}/include/common/taosdef.h ${code_dir}/include/util/taoserror.h ${code_dir}/include/libs/function/taosudf.h" -init_file_tarbitrator_deb=${script_dir}/../deb/tarbitratord -init_file_tarbitrator_rpm=${script_dir}/../rpm/tarbitratord - -# make directories. -mkdir -p ${install_dir} && cp ${install_files} ${install_dir} && chmod a+x ${install_dir}/install_arbi.sh || : -#mkdir -p ${install_dir}/inc && cp ${header_files} ${install_dir}/inc || : -mkdir -p ${install_dir}/bin && cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || : -mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_deb} ${install_dir}/init.d/tarbitratord.deb || : -mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_rpm} ${install_dir}/init.d/tarbitratord.rpm || : - -cd ${release_dir} - -# install_dir has been distinguishes cluster from edege, so comments this code -pkg_name=${install_dir}-${osType}-${cpuType} - -if [[ "$verType" == "beta" ]] || [[ "$verType" == "preRelease" ]]; then - pkg_name=${install_dir}-${verType}-${osType}-${cpuType} -elif [ "$verType" == "stable" ]; then - pkg_name=${pkg_name} -else - echo "unknow verType, nor stabel or beta" - exit 1 -fi - -tar -zcv -f "$(basename ${pkg_name}).tar.gz" $(basename ${install_dir}) --remove-files || : -exitcode=$? -if [ "$exitcode" != "0" ]; then - echo "tar ${pkg_name}.tar.gz error !!!" - exit $exitcode -fi - -cd ${curr_dir} diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 0bc11b99b3..cd656fe6f2 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -85,7 +85,6 @@ else ${build_dir}/bin/${clientName} \ ${taostools_bin_files} \ ${build_dir}/bin/taosadapter \ - ${build_dir}/bin/tarbitrator\ ${script_dir}/remove.sh \ ${script_dir}/set_core.sh \ ${script_dir}/startPre.sh \ @@ -106,8 +105,6 @@ nginx_dir="${top_dir}/../enterprise/src/plugins/web" init_file_deb=${script_dir}/../deb/taosd init_file_rpm=${script_dir}/../rpm/taosd -init_file_tarbitrator_deb=${script_dir}/../deb/tarbitratord -init_file_tarbitrator_rpm=${script_dir}/../rpm/tarbitratord # make directories. mkdir -p ${install_dir} @@ -126,10 +123,6 @@ if [ -f "${cfg_dir}/${serverName}.service" ]; then cp ${cfg_dir}/${serverName}.service ${install_dir}/cfg || : fi -if [ -f "${top_dir}/packaging/cfg/tarbitratord.service" ]; then - cp ${top_dir}/packaging/cfg/tarbitratord.service ${install_dir}/cfg || : -fi - if [ -f "${top_dir}/packaging/cfg/nginxd.service" ]; then cp ${top_dir}/packaging/cfg/nginxd.service ${install_dir}/cfg || : fi @@ -137,8 +130,6 @@ fi mkdir -p ${install_dir}/bin && cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || : mkdir -p ${install_dir}/init.d && cp ${init_file_deb} ${install_dir}/init.d/${serverName}.deb mkdir -p ${install_dir}/init.d && cp ${init_file_rpm} ${install_dir}/init.d/${serverName}.rpm -mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_deb} ${install_dir}/init.d/tarbitratord.deb || : -mkdir -p ${install_dir}/init.d && cp ${init_file_tarbitrator_rpm} ${install_dir}/init.d/tarbitratord.rpm || : if [ $adapterName != "taosadapter" ]; then mv ${install_dir}/cfg/taosadapter.toml ${install_dir}/cfg/$adapterName.toml diff --git a/packaging/tools/remove_arbi.sh b/packaging/tools/remove_arbi.sh deleted file mode 100755 index c95c579d30..0000000000 --- a/packaging/tools/remove_arbi.sh +++ /dev/null @@ -1,132 +0,0 @@ -#!/bin/bash -# -# Script to stop the service and uninstall TDengine's arbitrator - -set -e -#set -x - -verMode=edge - -RED='\033[0;31m' -GREEN='\033[1;32m' -NC='\033[0m' - -#install main path -install_main_dir="/usr/local/tarbitrator" -bin_link_dir="/usr/bin" -#inc_link_dir="/usr/include" - -service_config_dir="/etc/systemd/system" -tarbitrator_service_name="tarbitratord" -csudo="" -if command -v sudo > /dev/null; then - csudo="sudo " -fi - -initd_mod=0 -service_mod=2 -if pidof systemd &> /dev/null; then - service_mod=0 -elif $(which service &> /dev/null); then - service_mod=1 - service_config_dir="/etc/init.d" - if $(which chkconfig &> /dev/null); then - initd_mod=1 - elif $(which insserv &> /dev/null); then - initd_mod=2 - elif $(which update-rc.d &> /dev/null); then - initd_mod=3 - else - service_mod=2 - fi -else - service_mod=2 -fi - -function kill_tarbitrator() { - pid=$(ps -ef | grep "tarbitrator" | grep -v "grep" | awk '{print $2}') - if [ -n "$pid" ]; then - ${csudo}kill -9 $pid || : - fi -} -function clean_bin() { - # Remove link - ${csudo}rm -f ${bin_link_dir}/tarbitrator || : -} - -function clean_header() { - # Remove link - ${csudo}rm -f ${inc_link_dir}/taos.h || : - ${csudo}rm -f ${inc_link_dir}/taosdef.h || : - ${csudo}rm -f ${inc_link_dir}/taoserror.h || : - ${csudo}rm -f ${inc_link_dir}/taosudf.h || : - -} - -function clean_log() { - # Remove link - ${csudo}rm -rf /arbitrator.log || : -} - -function clean_service_on_systemd() { - tarbitratord_service_config="${service_config_dir}/${tarbitrator_service_name}.service" - - if systemctl is-active --quiet ${tarbitrator_service_name}; then - echo "TDengine tarbitrator is running, stopping it..." - ${csudo}systemctl stop ${tarbitrator_service_name} &> /dev/null || echo &> /dev/null - fi - ${csudo}systemctl disable ${tarbitrator_service_name} &> /dev/null || echo &> /dev/null - - ${csudo}rm -f ${tarbitratord_service_config} -} - -function clean_service_on_sysvinit() { - if pidof tarbitrator &> /dev/null; then - echo "TDengine's tarbitrator is running, stopping it..." - ${csudo}service tarbitratord stop || : - fi - - if ((${initd_mod}==1)); then - if [ -e ${service_config_dir}/tarbitratord ]; then - ${csudo}chkconfig --del tarbitratord || : - fi - elif ((${initd_mod}==2)); then - if [ -e ${service_config_dir}/tarbitratord ]; then - ${csudo}insserv -r tarbitratord || : - fi - elif ((${initd_mod}==3)); then - if [ -e ${service_config_dir}/tarbitratord ]; then - ${csudo}update-rc.d -f tarbitratord remove || : - fi - fi - - ${csudo}rm -f ${service_config_dir}/tarbitratord || : - - if $(which init &> /dev/null); then - ${csudo}init q || : - fi -} - -function clean_service() { - if ((${service_mod}==0)); then - clean_service_on_systemd - elif ((${service_mod}==1)); then - clean_service_on_sysvinit - else - # must manual stop - kill_tarbitrator - fi -} - -# Stop service and disable booting start. -clean_service -# Remove binary file and links -clean_bin -# Remove header file. -##clean_header -# Remove log file -clean_log - -${csudo}rm -rf ${install_main_dir} - -echo -e "${GREEN}TDengine's arbitrator is removed successfully!${NC}"