From 3b1664d9e4542f903920a4af11f8b7c46bf19aae Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 22 Jun 2022 18:50:39 +0800 Subject: [PATCH 01/72] feat: refactor rpc code --- include/libs/qcom/query.h | 35 +++--- include/os/osSocket.h | 5 +- source/client/src/clientImpl.c | 104 ++++++++--------- source/libs/qcom/src/queryUtil.c | 19 +-- source/libs/transport/inc/transComm.h | 26 +++++ source/libs/transport/src/transCli.c | 159 +++++++++++++++++--------- source/libs/transport/src/transComm.c | 2 +- source/libs/transport/src/transSvr.c | 26 ----- source/os/src/osSocket.c | 8 +- 9 files changed, 222 insertions(+), 162 deletions(-) diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index d562d07d77..0780343c64 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -16,6 +16,7 @@ #ifndef _TD_QUERY_H_ #define _TD_QUERY_H_ +// clang-foramt off #ifdef __cplusplus extern "C" { #endif @@ -71,7 +72,7 @@ typedef struct SIndexMeta { } SIndexMeta; typedef struct STbVerInfo { - char tbFName[TSDB_TABLE_FNAME_LEN]; + char tbFName[TSDB_TABLE_FNAME_LEN]; int32_t sversion; int32_t tversion; } STbVerInfo; @@ -141,7 +142,7 @@ typedef struct SDataBuf { typedef struct STargetInfo { ETargetType type; - char* dbFName; // used to update db's vgroup epset + char* dbFName; // used to update db's vgroup epset int32_t vgId; } STargetInfo; @@ -149,15 +150,15 @@ typedef int32_t (*__async_send_cb_fn_t)(void* param, const SDataBuf* pMsg, int32 typedef int32_t (*__async_exec_fn_t)(void* param); typedef struct SRequestConnInfo { - void* pTrans; - uint64_t requestId; - int64_t requestObjRefId; - SEpSet mgmtEps; + void* pTrans; + uint64_t requestId; + int64_t requestObjRefId; + SEpSet mgmtEps; } SRequestConnInfo; typedef struct SMsgSendInfo { - __async_send_cb_fn_t fp; // async callback function - STargetInfo target; // for update epset + __async_send_cb_fn_t fp; // async callback function + STargetInfo target; // for update epset void* param; uint64_t requestId; uint64_t requestObjRefId; @@ -206,9 +207,10 @@ int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STabl char* jobTaskStatusStr(int32_t status); SSchema createSchema(int8_t type, int32_t bytes, col_id_t colId, const char* name); -void destroyQueryExecRes(SQueryExecRes* pRes); +void destroyQueryExecRes(SQueryExecRes* pRes); -extern int32_t (*queryBuildMsg[TDMT_MAX])(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void*(*mallocFp)(int32_t)); +extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char** msg, int32_t msgSize, int32_t* msgLen, + void* (*mallocFp)(int32_t)); extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t msgSize); #define SET_META_TYPE_NULL(t) (t) = META_TYPE_NULL_TABLE @@ -219,7 +221,7 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t #define NEED_CLIENT_RM_TBLMETA_ERROR(_code) \ ((_code) == TSDB_CODE_PAR_TABLE_NOT_EXIST || (_code) == TSDB_CODE_VND_TB_NOT_EXIST || \ (_code) == TSDB_CODE_PAR_INVALID_COLUMNS_NUM || (_code) == TSDB_CODE_PAR_INVALID_COLUMN || \ - (_code) == TSDB_CODE_PAR_TAGS_NOT_MATCHED || (_code) == TSDB_CODE_PAR_VALUE_TOO_LONG || \ + (_code) == TSDB_CODE_PAR_TAGS_NOT_MATCHED || (_code) == TSDB_CODE_PAR_VALUE_TOO_LONG || \ (_code) == TSDB_CODE_PAR_INVALID_DROP_COL || ((_code) == TSDB_CODE_TDB_INVALID_TABLE_ID)) #define NEED_CLIENT_REFRESH_VG_ERROR(_code) \ ((_code) == TSDB_CODE_VND_HASH_MISMATCH || (_code) == TSDB_CODE_VND_INVALID_VGROUP_ID) @@ -227,11 +229,13 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t #define NEED_CLIENT_HANDLE_ERROR(_code) \ (NEED_CLIENT_RM_TBLMETA_ERROR(_code) || NEED_CLIENT_REFRESH_VG_ERROR(_code) || \ NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code)) -#define NEED_CLIENT_RM_TBLMETA_REQ(_type) ((_type) == TDMT_VND_CREATE_TABLE || (_type) == TDMT_VND_CREATE_STB \ - || (_type) == TDMT_VND_DROP_TABLE || (_type) == TDMT_VND_DROP_STB) +#define NEED_CLIENT_RM_TBLMETA_REQ(_type) \ + ((_type) == TDMT_VND_CREATE_TABLE || (_type) == TDMT_VND_CREATE_STB || (_type) == TDMT_VND_DROP_TABLE || \ + (_type) == TDMT_VND_DROP_STB) -#define NEED_SCHEDULER_RETRY_ERROR(_code) \ - ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL || (_code) == TSDB_CODE_SCH_TIMEOUT_ERROR) +#define NEED_SCHEDULER_RETRY_ERROR(_code) \ + ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL || \ + (_code) == TSDB_CODE_SCH_TIMEOUT_ERROR) #define REQUEST_TOTAL_EXEC_TIMES 2 @@ -308,3 +312,4 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t #endif #endif /*_TD_QUERY_H_*/ + // clang-foramt on diff --git a/include/os/osSocket.h b/include/os/osSocket.h index 213a6930ee..9dd5b972fa 100644 --- a/include/os/osSocket.h +++ b/include/os/osSocket.h @@ -157,7 +157,10 @@ int32_t taosNonblockwrite(TdSocketPtr pSocket, char *ptr, int32_t nbytes); int64_t taosCopyFds(TdSocketPtr pSrcSocket, TdSocketPtr pDestSocket, int64_t len); void taosWinSocketInit(); -int taosCreateSocketWithTimeOutOpt(uint32_t conn_timeout_sec); +/* + * set timeout(ms) + */ +int32_t taosCreateSocketWithTimeout(uint32_t timeout); TdSocketPtr taosOpenUdpSocket(uint32_t localIp, uint16_t localPort); TdSocketPtr taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 8920922006..14649c9fd4 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -59,7 +59,7 @@ static STscObj* taosConnectImpl(const char* user, const char* auth, const char* SAppInstInfo* pAppInfo, int connType); STscObj* taos_connect_internal(const char* ip, const char* user, const char* pass, const char* auth, const char* db, - uint16_t port, int connType) { + uint16_t port, int connType) { if (taos_init() != TSDB_CODE_SUCCESS) { return NULL; } @@ -313,8 +313,8 @@ bool qnodeRequired(SRequestObj* pRequest) { } SAppInstInfo* pInfo = pRequest->pTscObj->pAppInfo; - bool required = false; - + bool required = false; + taosThreadMutexLock(&pInfo->qnodeMutex); required = (NULL == pInfo->pQnodeList); taosThreadMutexUnlock(&pInfo->qnodeMutex); @@ -419,11 +419,11 @@ int32_t buildVnodePolicyNodeList(SRequestObj* pRequest, SArray** pNodeList, SArr } for (int32_t j = 0; j < vgNum; ++j) { - SVgroupInfo* pInfo = taosArrayGet(pVg, j); + SVgroupInfo* pInfo = taosArrayGet(pVg, j); SQueryNodeLoad load = {0}; load.addr.nodeId = pInfo->vgId; load.addr.epSet = pInfo->epSet; - + taosArrayPush(nodeList, &load); } } @@ -481,17 +481,16 @@ _return: return TSDB_CODE_SUCCESS; } - -int32_t buildAsyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray* pMnodeList, SMetaData *pResultMeta) { +int32_t buildAsyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray* pMnodeList, SMetaData* pResultMeta) { SArray* pDbVgList = NULL; SArray* pQnodeList = NULL; int32_t code = 0; - + switch (tsQueryPolicy) { case QUERY_POLICY_VNODE: { if (pResultMeta) { pDbVgList = taosArrayInit(4, POINTER_BYTES); - + int32_t dbNum = taosArrayGetSize(pResultMeta->pDbVgroup); for (int32_t i = 0; i < dbNum; ++i) { SMetaRes* pRes = taosArrayGet(pResultMeta->pDbVgroup, i); @@ -500,9 +499,9 @@ int32_t buildAsyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray } taosArrayPush(pDbVgList, &pRes->pRes); - } + } } - + code = buildVnodePolicyNodeList(pRequest, pNodeList, pMnodeList, pDbVgList); break; } @@ -523,7 +522,7 @@ int32_t buildAsyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray } taosThreadMutexUnlock(&pInst->qnodeMutex); } - + code = buildQnodePolicyNodeList(pRequest, pNodeList, pMnodeList, pQnodeList); break; } @@ -534,7 +533,7 @@ int32_t buildAsyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray taosArrayDestroy(pDbVgList); taosArrayDestroy(pQnodeList); - + return code; } @@ -542,43 +541,43 @@ int32_t buildSyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray* SArray* pDbVgList = NULL; SArray* pQnodeList = NULL; int32_t code = 0; - + switch (tsQueryPolicy) { case QUERY_POLICY_VNODE: { int32_t dbNum = taosArrayGetSize(pRequest->dbList); if (dbNum > 0) { - SCatalog* pCtg = NULL; + SCatalog* pCtg = NULL; SAppInstInfo* pInst = pRequest->pTscObj->pAppInfo; code = catalogGetHandle(pInst->clusterId, &pCtg); if (code != TSDB_CODE_SUCCESS) { goto _return; } - pDbVgList = taosArrayInit(dbNum, POINTER_BYTES); + pDbVgList = taosArrayInit(dbNum, POINTER_BYTES); SArray* pVgList = NULL; for (int32_t i = 0; i < dbNum; ++i) { - char* dbFName = taosArrayGet(pRequest->dbList, i); + char* dbFName = taosArrayGet(pRequest->dbList, i); SRequestConnInfo conn = {.pTrans = pInst->pTransporter, .requestId = pRequest->requestId, .requestObjRefId = pRequest->self, - .mgmtEps = getEpSet_s(&pInst->mgmtEp)}; - + .mgmtEps = getEpSet_s(&pInst->mgmtEp)}; + code = catalogGetDBVgInfo(pCtg, &conn, dbFName, &pVgList); if (code) { goto _return; } - + taosArrayPush(pDbVgList, &pVgList); - } + } } - + code = buildVnodePolicyNodeList(pRequest, pNodeList, pMnodeList, pDbVgList); break; } case QUERY_POLICY_HYBRID: case QUERY_POLICY_QNODE: { getQnodeList(pRequest, &pQnodeList); - + code = buildQnodePolicyNodeList(pRequest, pNodeList, pMnodeList, pQnodeList); break; } @@ -591,11 +590,10 @@ _return: taosArrayDestroy(pDbVgList); taosArrayDestroy(pQnodeList); - + return code; } - int32_t scheduleAsyncQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList) { tsem_init(&schdRspSem, 0, 0); @@ -604,12 +602,12 @@ int32_t scheduleAsyncQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNod .requestId = pRequest->requestId, .requestObjRefId = pRequest->self}; SSchedulerReq req = {.pConn = &conn, - .pNodeList = pNodeList, - .pDag = pDag, - .sql = pRequest->sqlstr, - .startTs = pRequest->metric.start, - .fp = schdExecCallback, - .cbParam = &res}; + .pNodeList = pNodeList, + .pDag = pDag, + .sql = pRequest->sqlstr, + .startTs = pRequest->metric.start, + .fp = schdExecCallback, + .cbParam = &res}; int32_t code = schedulerAsyncExecJob(&req, &pRequest->body.queryJob); @@ -656,13 +654,13 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList .requestId = pRequest->requestId, .requestObjRefId = pRequest->self}; SSchedulerReq req = {.pConn = &conn, - .pNodeList = pNodeList, - .pDag = pDag, - .sql = pRequest->sqlstr, - .startTs = pRequest->metric.start, - .fp = NULL, - .cbParam = NULL, - .reqKilled = &pRequest->killed}; + .pNodeList = pNodeList, + .pDag = pDag, + .sql = pRequest->sqlstr, + .startTs = pRequest->metric.start, + .fp = NULL, + .cbParam = NULL, + .reqKilled = &pRequest->killed}; int32_t code = schedulerExecJob(&req, &pRequest->body.queryJob, &res); pRequest->body.resInfo.execRes = res.res; @@ -819,8 +817,8 @@ void schedulerExecCb(SQueryResult* pResult, void* param, int32_t code) { } } - tscDebug("0x%" PRIx64 " enter scheduler exec cb, code:%d - %s, reqId:0x%" PRIx64, - pRequest->self, code, tstrerror(code), pRequest->requestId); + tscDebug("0x%" PRIx64 " enter scheduler exec cb, code:%d - %s, reqId:0x%" PRIx64, pRequest->self, code, + tstrerror(code), pRequest->requestId); STscObj* pTscObj = pRequest->pTscObj; if (code != TSDB_CODE_SUCCESS && NEED_CLIENT_HANDLE_ERROR(code)) { @@ -862,7 +860,7 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQue if (TSDB_CODE_SUCCESS == code) { SArray* pNodeList = NULL; buildSyncExecNodeList(pRequest, &pNodeList, pMnodeList); - + code = scheduleQuery(pRequest, pRequest->body.pDag, pNodeList); taosArrayDestroy(pNodeList); } @@ -915,7 +913,7 @@ SRequestObj* launchQuery(STscObj* pTscObj, const char* sql, int sqlLen) { return launchQueryImpl(pRequest, pQuery, false, NULL); } -void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData *pResultMeta) { +void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultMeta) { int32_t code = 0; switch (pQuery->execMode) { @@ -948,7 +946,7 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData *pResultM if (TSDB_CODE_SUCCESS == code) { SArray* pNodeList = NULL; buildAsyncExecNodeList(pRequest, &pNodeList, pMnodeList, pResultMeta); - + SRequestConnInfo conn = { .pTrans = pAppInfo->pTransporter, .requestId = pRequest->requestId, .requestObjRefId = pRequest->self}; SSchedulerReq req = {.pConn = &conn, @@ -1308,7 +1306,7 @@ TAOS* taos_connect_auth(const char* ip, const char* user, const char* auth, cons if (pObj) { return pObj->id; } - + return NULL; } @@ -1554,10 +1552,10 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int return TSDB_CODE_SUCCESS; } -static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, int32_t numOfRows){ +static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, int32_t numOfRows) { char* p = (char*)pResultInfo->pData; - int32_t len = sizeof(int32_t) + sizeof(uint64_t) + numOfCols * (sizeof(int16_t) + sizeof(int32_t)); + int32_t len = sizeof(int32_t) + sizeof(uint64_t) + numOfCols * (sizeof(int16_t) + sizeof(int32_t)); int32_t* colLength = (int32_t*)(p + len); len += sizeof(int32_t) * numOfCols; @@ -1567,7 +1565,7 @@ static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, i if (pResultInfo->fields[i].type == TSDB_DATA_TYPE_JSON) { int32_t* offset = (int32_t*)pStart; - int32_t lenTmp = numOfRows * sizeof(int32_t); + int32_t lenTmp = numOfRows * sizeof(int32_t); len += lenTmp; pStart += lenTmp; @@ -1592,7 +1590,6 @@ static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, i } else { ASSERT(0); } - } } else if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) { int32_t lenTmp = numOfRows * sizeof(int32_t); @@ -1616,13 +1613,13 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int break; } } - if(!needConvert) return TSDB_CODE_SUCCESS; + if (!needConvert) return TSDB_CODE_SUCCESS; - char* p = (char*)pResultInfo->pData; + char* p = (char*)pResultInfo->pData; int32_t dataLen = estimateJsonLen(pResultInfo, numOfCols, numOfRows); pResultInfo->convertJson = taosMemoryCalloc(1, dataLen); - if(pResultInfo->convertJson == NULL) return TSDB_CODE_OUT_OF_MEMORY; + if (pResultInfo->convertJson == NULL) return TSDB_CODE_OUT_OF_MEMORY; char* p1 = pResultInfo->convertJson; int32_t len = sizeof(int32_t) + sizeof(uint64_t) + numOfCols * (sizeof(int16_t) + sizeof(int32_t)); @@ -1691,7 +1688,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int ASSERT(0); } - offset1[j]= len; + offset1[j] = len; memcpy(pStart1 + len, dst, varDataTLen(dst)); len += varDataTLen(dst); } @@ -1709,7 +1706,6 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int pStart += len; pStart1 += len; memcpy(pStart1, pStart, colLen); - } pStart += colLen; pStart1 += colLen1; @@ -1777,7 +1773,7 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32 pStart += colLength[i]; } - if(convertUcs4){ + if (convertUcs4) { code = doConvertUCS4(pResultInfo, numOfRows, numOfCols, colLength); } diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index 2120d24d26..c92ab9b008 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -19,7 +19,7 @@ #include "tmsg.h" #include "trpc.h" #include "tsched.h" - +// clang-format off #define VALIDNUMOFCOLS(x) ((x) >= TSDB_MIN_COLUMNS && (x) <= TSDB_MAX_COLUMNS) #define VALIDNUMOFTAGS(x) ((x) >= 0 && (x) <= TSDB_MAX_TAGS) @@ -146,13 +146,15 @@ int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTra } memcpy(pMsg, pInfo->msgInfo.pData, pInfo->msgInfo.len); - SRpcMsg rpcMsg = {.msgType = pInfo->msgType, - .pCont = pMsg, - .contLen = pInfo->msgInfo.len, - .info.ahandle = (void*)pInfo, - .info.handle = pInfo->msgInfo.handle, - .info.persistHandle = persistHandle, - .code = 0}; + SRpcMsg rpcMsg = { + .msgType = pInfo->msgType, + .pCont = pMsg, + .contLen = pInfo->msgInfo.len, + .info.ahandle = (void*)pInfo, + .info.handle = pInfo->msgInfo.handle, + .info.persistHandle = persistHandle, + .code = 0 + }; assert(pInfo->fp != NULL); TRACE_SET_ROOTID(&rpcMsg.info.traceId, pInfo->requestId); rpcSendRequestWithCtx(pTransporter, epSet, &rpcMsg, pTransporterId, rpcCtx); @@ -220,3 +222,4 @@ void destroyQueryExecRes(SQueryExecRes* pRes) { qError("invalid exec result for request type %d", pRes->msgType); } } +// clang-format on diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 327fe50814..158926c520 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -238,6 +238,32 @@ int transSendAsync(SAsyncPool* pool, queue* mq); } \ } \ } while (0) + +#define ASYNC_CHECK_HANDLE(exh1, refId) \ + do { \ + if (refId > 0) { \ + tTrace("handle step1"); \ + SExHandle* exh2 = transAcquireExHandle(refMgt, refId); \ + if (exh2 == NULL || refId != exh2->refId) { \ + tTrace("handle %p except, may already freed, ignore msg, ref1: %" PRIu64 ", ref2 : %" PRIu64 "", exh1, \ + exh2 ? exh2->refId : 0, refId); \ + goto _return1; \ + } \ + } else if (refId == 0) { \ + tTrace("handle step2"); \ + SExHandle* exh2 = transAcquireExHandle(refMgt, refId); \ + if (exh2 == NULL || refId != exh2->refId) { \ + tTrace("handle %p except, may already freed, ignore msg, ref1: %" PRIu64 ", ref2 : %" PRIu64 "", exh1, refId, \ + exh2 ? exh2->refId : 0); \ + goto _return1; \ + } else { \ + refId = exh1->refId; \ + } \ + } else if (refId < 0) { \ + tTrace("handle step3"); \ + goto _return2; \ + } \ + } while (0) int transInitBuffer(SConnBuffer* buf); int transClearBuffer(SConnBuffer* buf); int transDestroyBuffer(SConnBuffer* buf); diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 852ffc9a0e..402a26247a 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -25,7 +25,6 @@ typedef struct SCliConn { uv_write_t writeReq; void* hostThrd; - int hThrdIdx; SConnBuffer readBuf; STransQueue cliMsgs; @@ -36,6 +35,7 @@ typedef struct SCliConn { bool broken; // link broken or not ConnStatus status; // + int64_t refId; char* ip; uint32_t port; @@ -168,16 +168,24 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { snprintf(key, sizeof(key), "%s:%d", ip, (int)port); \ } while (0) -#define CONN_HOST_THREAD_IDX(conn) (conn ? ((SCliConn*)conn)->hThrdIdx : -1) +#define CONN_HOST_THREAD_IDX1(idx, exh, refId, pThrd) \ + do { \ + if (exh == NULL) { \ + idx = -1; \ + } else { \ + ASYNC_CHECK_HANDLE(exh, refId); \ + pThrd = (SCliThrdObj*)exh->pThrd; \ + } \ + } while (0) #define CONN_PERSIST_TIME(para) (para * 1000 * 10) #define CONN_GET_HOST_THREAD(conn) (conn ? ((SCliConn*)conn)->hostThrd : NULL) #define CONN_GET_INST_LABEL(conn) (((STrans*)(((SCliThrdObj*)(conn)->hostThrd)->pTransInst))->label) #define CONN_SHOULD_RELEASE(conn, head) \ do { \ if ((head)->release == 1 && (head->msgLen) == sizeof(*head)) { \ + int status = conn->status; \ uint64_t ahandle = head->ahandle; \ CONN_GET_MSGCTX_BY_AHANDLE(conn, ahandle); \ - conn->status = ConnRelease; \ transClearBuffer(&conn->readBuf); \ transFreeMsg(transContFromHead((char*)head)); \ tDebug("%s conn %p receive release request, ref: %d", CONN_GET_INST_LABEL(conn), conn, T_REF_VAL_GET(conn)); \ @@ -186,7 +194,9 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { } \ destroyCmsg(pMsg); \ cliReleaseUnfinishedMsg(conn); \ - addConnToPool(((SCliThrdObj*)conn->hostThrd)->pool, conn); \ + if (status != ConnInPool) { \ + addConnToPool(((SCliThrdObj*)conn->hostThrd)->pool, conn); \ + } \ return; \ } \ } while (0) @@ -323,23 +333,29 @@ void cliHandleResp(SCliConn* conn) { transClearBuffer(&conn->readBuf); if (!CONN_NO_PERSIST_BY_APP(conn)) { - transMsg.info.handle = conn; + SExHandle* exh = taosMemoryCalloc(1, sizeof(SExHandle)); + exh->handle = conn; + exh->pThrd = pThrd; + exh->refId = transAddExHandle(refMgt, exh); + + transMsg.info.handle = exh; + transMsg.info.refId = exh->refId; + conn->refId = exh->refId; tDebug("%s conn %p ref by app", CONN_GET_INST_LABEL(conn), conn); } - // char buf[64] = {0}; - // TRACE_TO_STR(&transMsg.info.traceId, buf); + STraceId* trace = &transMsg.info.traceId; - tGTrace("conn %p %s received from %s:%d, local info: %s:%d, msg size: %d, code: %d", conn, TMSG_INFO(pHead->msgType), - taosInetNtoa(conn->addr.sin_addr), ntohs(conn->addr.sin_port), taosInetNtoa(conn->localAddr.sin_addr), - ntohs(conn->localAddr.sin_port), transMsg.contLen, transMsg.code); + tGTrace("%s conn %p %s received from %s:%d, local info: %s:%d, msg size: %d, code: %d", CONN_GET_INST_LABEL(conn), + conn, TMSG_INFO(pHead->msgType), taosInetNtoa(conn->addr.sin_addr), ntohs(conn->addr.sin_port), + taosInetNtoa(conn->localAddr.sin_addr), ntohs(conn->localAddr.sin_port), transMsg.contLen, transMsg.code); if (pCtx == NULL && CONN_NO_PERSIST_BY_APP(conn)) { - tDebug("%s except, server continue send while cli ignore it", CONN_GET_INST_LABEL(conn)); + tDebug("%s except, conn %p read while cli ignore it", CONN_GET_INST_LABEL(conn), conn); // transUnrefCliHandle(conn); return; } if (CONN_RELEASE_BY_SERVER(conn) && transMsg.info.ahandle == NULL) { - tDebug("%s except, server continue send while cli ignore it", CONN_GET_INST_LABEL(conn)); + tDebug("%s except, conn %p read while cli ignore it", CONN_GET_INST_LABEL(conn), conn); // transUnrefCliHandle(conn); return; } @@ -476,10 +492,9 @@ static SCliConn* getConnFromPool(void* pool, char* ip, uint32_t port) { if (QUEUE_IS_EMPTY(&plist->conn)) { return NULL; } - queue* h = QUEUE_HEAD(&plist->conn); - // //QUEUE_REMOVE(h); + queue* h = QUEUE_HEAD(&plist->conn); SCliConn* conn = QUEUE_DATA(h, SCliConn, conn); - // conn->status = ConnNormal; + conn->status = ConnNormal; QUEUE_REMOVE(&conn->conn); QUEUE_INIT(&conn->conn); return conn; @@ -559,6 +574,13 @@ static SCliConn* cliCreateConn(SCliThrdObj* pThrd) { conn->status = ConnNormal; conn->broken = 0; transRefCliHandle(conn); + + SExHandle* exh = taosMemoryCalloc(1, sizeof(SExHandle)); + exh->handle = conn; + exh->pThrd = pThrd; + exh->refId = transAddExHandle(refMgt, exh); + conn->refId = exh->refId; + return conn; } static void cliDestroyConn(SCliConn* conn, bool clear) { @@ -566,6 +588,7 @@ static void cliDestroyConn(SCliConn* conn, bool clear) { QUEUE_REMOVE(&conn->conn); QUEUE_INIT(&conn->conn); + transRemoveExHandle(refMgt, conn->refId); if (clear) { uv_close((uv_handle_t*)conn->stream, cliDestroy); } @@ -650,12 +673,10 @@ void cliSend(SCliConn* pConn) { uv_buf_t wb = uv_buf_init((char*)pHead, msgLen); - // char buf[64] = {0}; - // TRACE_TO_STR(&pMsg->info.traceId, buf); STraceId* trace = &pMsg->info.traceId; - tGTrace("conn %p %s is sent to %s:%d, local info %s:%d", pConn, TMSG_INFO(pHead->msgType), - taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), taosInetNtoa(pConn->localAddr.sin_addr), - ntohs(pConn->localAddr.sin_port)); + tGTrace("%s conn %p %s is sent to %s:%d, local info %s:%d", CONN_GET_INST_LABEL(pConn), pConn, + TMSG_INFO(pHead->msgType), taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), + taosInetNtoa(pConn->localAddr.sin_addr), ntohs(pConn->localAddr.sin_port)); if (pHead->persist == 1) { CONN_SET_PERSIST_BY_APP(pConn); @@ -663,7 +684,6 @@ void cliSend(SCliConn* pConn) { pConn->writeReq.data = pConn; uv_write(&pConn->writeReq, (uv_stream_t*)pConn->stream, &wb, 1, cliSendCb); - return; _RETURN: return; @@ -723,20 +743,32 @@ static void cliHandleUpdate(SCliMsg* pMsg, SCliThrdObj* pThrd) { } SCliConn* cliGetConn(SCliMsg* pMsg, SCliThrdObj* pThrd) { - SCliConn* conn = NULL; - if (pMsg->msg.info.handle != NULL) { - conn = (SCliConn*)(pMsg->msg.info.handle); - if (conn != NULL) { - tTrace("%s conn %p reused", CONN_GET_INST_LABEL(conn), conn); + SCliConn* conn = NULL; + SRpcHandleInfo* pInfo = &pMsg->msg.info; + + SExHandle* exh = transAcquireExHandle(refMgt, pInfo->refId); + if (exh == NULL) { + if (pInfo->refId != 0) { + tTrace("%s conn %p ignore msg", CONN_GET_INST_LABEL(conn), conn); + assert(0); + return NULL; } } else { - STransConnCtx* pCtx = pMsg->ctx; - conn = getConnFromPool(pThrd->pool, EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet)); - if (conn != NULL) { - tTrace("%s conn %p get from conn pool", CONN_GET_INST_LABEL(conn), conn); - } else { - tTrace("%s not found conn in conn pool %p", ((STrans*)pThrd->pTransInst)->label, pThrd->pool); - } + transReleaseExHandle(refMgt, pInfo->refId); + return exh->handle; + } + + STransConnCtx* pCtx = pMsg->ctx; + conn = getConnFromPool(pThrd->pool, EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet)); + if (conn != NULL) { + exh = taosMemoryCalloc(1, sizeof(SExHandle)); + exh->handle = conn; + exh->pThrd = pThrd; + exh->refId = transAddExHandle(refMgt, exh); + conn->refId = exh->refId; + tTrace("%s conn %p get from conn pool", CONN_GET_INST_LABEL(conn), conn); + } else { + tTrace("%s not found conn in conn pool %p", ((STrans*)pThrd->pTransInst)->label, pThrd->pool); } return conn; } @@ -765,8 +797,6 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { SCliConn* conn = cliGetConn(pMsg, pThrd); if (conn != NULL) { - conn->hThrdIdx = pCtx->hThrdIdx; - transCtxMerge(&conn->ctx, &pCtx->appCtx); transQueuePush(&conn->cliMsgs, pMsg); cliSend(conn); @@ -775,7 +805,6 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { transCtxMerge(&conn->ctx, &pCtx->appCtx); transQueuePush(&conn->cliMsgs, pMsg); - conn->hThrdIdx = pCtx->hThrdIdx; conn->ip = strdup(EPSET_GET_INUSE_IP(&pCtx->epSet)); conn->port = EPSET_GET_INUSE_PORT(&pCtx->epSet); @@ -783,7 +812,7 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { if (ret) { tError("%s conn %p failed to set conn option, errmsg %s", transLabel(pTransInst), conn, uv_err_name(ret)); } - int fd = taosCreateSocketWithTimeOutOpt(TRANS_CONN_TIMEOUT); + int32_t fd = taosCreateSocketWithTimeout(TRANS_CONN_TIMEOUT); if (fd == -1) { tTrace("%s conn %p failed to create socket", transLabel(pTransInst), conn); cliHandleExcept(conn); @@ -1009,7 +1038,9 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { tTrace("%s use remote epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse, pCtx->retryCount + 1, TRANS_RETRY_COUNT_LIMIT); } - addConnToPool(pThrd->pool, pConn); + if (pConn->status != ConnInPool) { + addConnToPool(pThrd->pool, pConn); + } STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); arg->param1 = pMsg; @@ -1086,10 +1117,21 @@ void transReleaseCliHandle(void* handle) { } void transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransCtx* ctx) { - STrans* pTransInst = (STrans*)shandle; - int idx = CONN_HOST_THREAD_IDX((SCliConn*)pReq->info.handle); + STrans* pTransInst = (STrans*)shandle; + SRpcHandleInfo* info = &pReq->info; + + int idx = -1; + SCliThrdObj* pThrd = NULL; + SExHandle* exh = info->handle; + int64_t refId = -1; + if (exh != NULL) { + refId = exh->refId; + } + + CONN_HOST_THREAD_IDX1(idx, exh, refId, pThrd); if (idx == -1) { idx = cliRBChoseIdx(pTransInst); + pThrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[idx]; } TRACE_SET_MSGID(&pReq->info.traceId, tGenIdPI64()); @@ -1097,7 +1139,6 @@ void transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STra pCtx->epSet = *pEpSet; pCtx->ahandle = pReq->info.ahandle; pCtx->msgType = pReq->msgType; - pCtx->hThrdIdx = idx; if (ctx != NULL) { pCtx->appCtx = *ctx; @@ -1110,19 +1151,31 @@ void transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STra cliMsg->st = taosGetTimestampUs(); cliMsg->type = Normal; - SCliThrdObj* thrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[idx]; - STraceId* trace = &pReq->info.traceId; - tGTrace("%s send request at thread:%08" PRId64 ", dst: %s:%d, app:%p", transLabel(pTransInst), thrd->pid, + tGTrace("%s send request at thread:%08" PRId64 ", dst: %s:%d, app:%p", transLabel(pTransInst), pThrd->pid, EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet), pReq->info.ahandle); - ASSERT(transSendAsync(thrd->asyncPool, &(cliMsg->q)) == 0); + ASSERT(transSendAsync(pThrd->asyncPool, &(cliMsg->q)) == 0); +_return1: + return; +_return2: + return; } void transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp) { - STrans* pTransInst = (STrans*)shandle; - int idx = CONN_HOST_THREAD_IDX(pReq->info.handle); + STrans* pTransInst = (STrans*)shandle; + SRpcHandleInfo* info = &pReq->info; + SCliThrdObj* pThrd = NULL; + int idx = -1; + SExHandle* exh = info->handle; + int64_t refId = -1; + if (exh != NULL) { + refId = exh->refId; + } + + CONN_HOST_THREAD_IDX1(idx, exh, refId, pThrd); if (idx == -1) { idx = cliRBChoseIdx(pTransInst); + pThrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[idx]; } tsem_t* sem = taosMemoryCalloc(1, sizeof(tsem_t)); tsem_init(sem, 0, 0); @@ -1133,7 +1186,6 @@ void transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransM pCtx->epSet = *pEpSet; pCtx->ahandle = pReq->info.ahandle; pCtx->msgType = pReq->msgType; - pCtx->hThrdIdx = idx; pCtx->pSem = sem; pCtx->pRsp = pRsp; @@ -1143,16 +1195,18 @@ void transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransM cliMsg->st = taosGetTimestampUs(); cliMsg->type = Normal; - SCliThrdObj* thrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[idx]; - STraceId* trace = &pReq->info.traceId; - tGTrace("%s send request at thread:%08" PRId64 ", dst: %s:%d, app:%p", transLabel(pTransInst), thrd->pid, + tGTrace("%s send request at thread:%08" PRId64 ", dst: %s:%d, app:%p", transLabel(pTransInst), pThrd->pid, EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet), pReq->info.ahandle); - transSendAsync(thrd->asyncPool, &(cliMsg->q)); + transSendAsync(pThrd->asyncPool, &(cliMsg->q)); tsem_wait(sem); tsem_destroy(sem); taosMemoryFree(sem); +_return1: + return; +_return2: + return; } /* * @@ -1168,7 +1222,6 @@ void transSetDefaultAddr(void* ahandle, const char* ip, const char* fqdn) { } for (int i = 0; i < pTransInst->numOfThreads; i++) { STransConnCtx* pCtx = taosMemoryCalloc(1, sizeof(STransConnCtx)); - pCtx->hThrdIdx = i; pCtx->cvtAddr = cvtAddr; SCliMsg* cliMsg = taosMemoryCalloc(1, sizeof(SCliMsg)); diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 8cd7f9d827..5d342dd174 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -455,7 +455,7 @@ void transPrintEpSet(SEpSet* pEpSet) { return; } char buf[512] = {0}; - int len = snprintf(buf, sizeof(buf), "epset { "); + int len = snprintf(buf, sizeof(buf), "epset:{ "); for (int i = 0; i < pEpSet->numOfEps; i++) { if (i == pEpSet->numOfEps - 1) { len += snprintf(buf + len, sizeof(buf) - len, "%d. %s:%d ", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 593a790a21..121fddc99a 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -206,32 +206,6 @@ static bool addHandleToAcceptloop(void* arg); } \ } while (0) -#define ASYNC_CHECK_HANDLE(exh1, refId) \ - do { \ - if (refId > 0) { \ - tTrace("handle step1"); \ - SExHandle* exh2 = transAcquireExHandle(refMgt, refId); \ - if (exh2 == NULL || refId != exh2->refId) { \ - tTrace("handle %p except, may already freed, ignore msg, ref1: %" PRIu64 ", ref2 : %" PRIu64 "", exh1, \ - exh2 ? exh2->refId : 0, refId); \ - goto _return1; \ - } \ - } else if (refId == 0) { \ - tTrace("handle step2"); \ - SExHandle* exh2 = transAcquireExHandle(refMgt, refId); \ - if (exh2 == NULL || refId != exh2->refId) { \ - tTrace("handle %p except, may already freed, ignore msg, ref1: %" PRIu64 ", ref2 : %" PRIu64 "", exh1, refId, \ - exh2 ? exh2->refId : 0); \ - goto _return1; \ - } else { \ - refId = exh1->refId; \ - } \ - } else if (refId < 0) { \ - tTrace("handle step3"); \ - goto _return2; \ - } \ - } while (0) - void uvAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { SSvrConn* conn = handle->data; SConnBuffer* pBuf = &conn->readBuf; diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index 4d61e7036d..b0e07ff010 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -946,7 +946,7 @@ int32_t taosGetFqdn(char *fqdn) { #endif // __APPLE__ int32_t ret = getaddrinfo(hostname, NULL, &hints, &result); if (!result) { - fprintf(stderr,"failed to get fqdn, code:%d, reason:%s", ret, gai_strerror(ret)); + fprintf(stderr, "failed to get fqdn, code:%d, reason:%s", ret, gai_strerror(ret)); return -1; } @@ -1073,7 +1073,7 @@ int32_t taosCloseEpoll(TdEpollPtr *ppEpoll) { * Set TCP connection timeout per-socket level. * ref [https://github.com/libuv/help/issues/54] */ -int taosCreateSocketWithTimeOutOpt(uint32_t conn_timeout_sec) { +int32_t taosCreateSocketWithTimeout(uint32_t timeout) { #if defined(WINDOWS) SOCKET fd; #else @@ -1083,11 +1083,11 @@ int taosCreateSocketWithTimeOutOpt(uint32_t conn_timeout_sec) { return -1; } #if defined(WINDOWS) - if (0 != setsockopt(fd, IPPROTO_TCP, TCP_MAXRT, (char *)&conn_timeout_sec, sizeof(conn_timeout_sec))) { + if (0 != setsockopt(fd, IPPROTO_TCP, TCP_MAXRT, (char *)&timeout, sizeof(timeout))) { return -1; } #else // Linux like systems - uint32_t conn_timeout_ms = conn_timeout_sec * 1000; + uint32_t conn_timeout_ms = timeout * 1000; if (0 != setsockopt(fd, IPPROTO_TCP, TCP_USER_TIMEOUT, (char *)&conn_timeout_ms, sizeof(conn_timeout_ms))) { return -1; } From bfe5ab779b556372d418cd78f846e2e867c8ec9e Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 22 Jun 2022 19:20:35 +0800 Subject: [PATCH 02/72] fix: handle except --- source/libs/transport/src/transCli.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 402a26247a..3542333561 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -18,6 +18,10 @@ static int32_t transSCliInst = 0; static int32_t refMgt = 0; +typedef struct SExHandleWrap { + void* exhandle; + int64_t refId; +} SExHandleWrap; typedef struct SCliConn { T_REF_DECLARE() uv_connect_t connReq; @@ -338,9 +342,11 @@ void cliHandleResp(SCliConn* conn) { exh->pThrd = pThrd; exh->refId = transAddExHandle(refMgt, exh); - transMsg.info.handle = exh; - transMsg.info.refId = exh->refId; + SExHandleWrap* wrap = taosMemoryCalloc(1, sizeof(SExHandleWrap)); + wrap->exhandle = exh; + wrap->refId = exh->refId; conn->refId = exh->refId; + transMsg.info.handle = wrap; tDebug("%s conn %p ref by app", CONN_GET_INST_LABEL(conn), conn); } @@ -1007,6 +1013,8 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { pMsg->sent = 0; tTrace("try to send req to next node"); pMsg->st = taosGetTimestampUs(); + + taosMemoryFree(pResp->info.handle); pCtx->retryCount += 1; if (pResp->code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { if (pCtx->retryCount < pEpSet->numOfEps * 3) { From d5f5c33c9c547c75dd9b3d031f9abb7840817d8e Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 23 Jun 2022 15:55:47 +0800 Subject: [PATCH 03/72] refactor code --- include/util/ttrace.h | 8 +- source/libs/transport/inc/transComm.h | 63 +++++----- source/libs/transport/src/transCli.c | 162 ++++++++++++-------------- source/libs/transport/src/transSvr.c | 10 +- 4 files changed, 116 insertions(+), 127 deletions(-) diff --git a/include/util/ttrace.h b/include/util/ttrace.h index 206cbbf28d..579768228a 100644 --- a/include/util/ttrace.h +++ b/include/util/ttrace.h @@ -45,9 +45,11 @@ typedef struct STraceId { #define TRACE_GET_MSGID(traceId) (traceId)->msgId -#define TRACE_TO_STR(traceId, buf) \ - do { \ - sprintf(buf, "0x%" PRIx64 ":0x%" PRIx64 "", traceId->rootId, traceId->msgId); \ +#define TRACE_TO_STR(traceId, buf) \ + do { \ + int64_t rootId = (traceId) != NULL ? (traceId)->rootId : 0; \ + int64_t msgId = (traceId) != NULL ? (traceId)->msgId : 0; \ + sprintf(buf, "0x%" PRIx64 ":0x%" PRIx64 "", rootId, msgId); \ } while (0) #ifdef __cplusplus diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 158926c520..5367f6b49d 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -105,6 +105,13 @@ typedef SRpcCtxVal STransCtxVal; typedef SRpcInfo STrans; typedef SRpcConnInfo STransHandleInfo; +// ref mgt +// handle +typedef struct SExHandle { + void* handle; + int64_t refId; + void* pThrd; +} SExHandle; /*convet from fqdn to ip */ typedef struct SCvtAddr { char ip[TSDB_FQDN_LEN]; @@ -239,30 +246,30 @@ int transSendAsync(SAsyncPool* pool, queue* mq); } \ } while (0) -#define ASYNC_CHECK_HANDLE(exh1, refId) \ - do { \ - if (refId > 0) { \ - tTrace("handle step1"); \ - SExHandle* exh2 = transAcquireExHandle(refMgt, refId); \ - if (exh2 == NULL || refId != exh2->refId) { \ - tTrace("handle %p except, may already freed, ignore msg, ref1: %" PRIu64 ", ref2 : %" PRIu64 "", exh1, \ - exh2 ? exh2->refId : 0, refId); \ - goto _return1; \ - } \ - } else if (refId == 0) { \ - tTrace("handle step2"); \ - SExHandle* exh2 = transAcquireExHandle(refMgt, refId); \ - if (exh2 == NULL || refId != exh2->refId) { \ - tTrace("handle %p except, may already freed, ignore msg, ref1: %" PRIu64 ", ref2 : %" PRIu64 "", exh1, refId, \ - exh2 ? exh2->refId : 0); \ - goto _return1; \ - } else { \ - refId = exh1->refId; \ - } \ - } else if (refId < 0) { \ - tTrace("handle step3"); \ - goto _return2; \ - } \ +#define ASYNC_CHECK_HANDLE(exh1, id) \ + do { \ + if (id > 0) { \ + tTrace("handle step1"); \ + SExHandle* exh2 = transAcquireExHandle(refMgt, id); \ + if (exh2 == NULL || id != exh2->refId) { \ + tTrace("handle %p except, may already freed, ignore msg, ref1: %" PRIu64 ", ref2 : %" PRIu64 "", exh1, \ + exh2 ? exh2->refId : 0, id); \ + goto _return1; \ + } \ + } else if (id == 0) { \ + tTrace("handle step2"); \ + SExHandle* exh2 = transAcquireExHandle(refMgt, id); \ + if (exh2 == NULL || id == exh2->refId) { \ + tTrace("handle %p except, may already freed, ignore msg, ref1: %" PRIu64 ", ref2 : %" PRIu64 "", exh1, id, \ + exh2 ? exh2->refId : 0); \ + goto _return1; \ + } else { \ + id = exh1->refId; \ + } \ + } else if (id < 0) { \ + tTrace("handle step3"); \ + goto _return2; \ + } \ } while (0) int transInitBuffer(SConnBuffer* buf); int transClearBuffer(SConnBuffer* buf); @@ -381,14 +388,6 @@ bool transEpSetIsEqual(SEpSet* a, SEpSet* b); */ void transThreadOnce(); -// ref mgt -// handle -typedef struct SExHandle { - void* handle; - int64_t refId; - void* pThrd; -} SExHandle; - void transInitEnv(); int32_t transOpenExHandleMgt(int size); void transCloseExHandleMgt(int32_t mgt); diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 3542333561..e18723d976 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -18,10 +18,6 @@ static int32_t transSCliInst = 0; static int32_t refMgt = 0; -typedef struct SExHandleWrap { - void* exhandle; - int64_t refId; -} SExHandleWrap; typedef struct SCliConn { T_REF_DECLARE() uv_connect_t connReq; @@ -177,8 +173,8 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { if (exh == NULL) { \ idx = -1; \ } else { \ - ASYNC_CHECK_HANDLE(exh, refId); \ - pThrd = (SCliThrdObj*)exh->pThrd; \ + ASYNC_CHECK_HANDLE((exh), refId); \ + pThrd = (SCliThrdObj*)(exh)->pThrd; \ } \ } while (0) #define CONN_PERSIST_TIME(para) (para * 1000 * 10) @@ -201,6 +197,7 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { if (status != ConnInPool) { \ addConnToPool(((SCliThrdObj*)conn->hostThrd)->pool, conn); \ } \ + transRemoveExHandle(refMgt, conn->refId); \ return; \ } \ } while (0) @@ -335,18 +332,8 @@ void cliHandleResp(SCliConn* conn) { } // buf's mem alread translated to transMsg.pCont transClearBuffer(&conn->readBuf); - if (!CONN_NO_PERSIST_BY_APP(conn)) { - SExHandle* exh = taosMemoryCalloc(1, sizeof(SExHandle)); - exh->handle = conn; - exh->pThrd = pThrd; - exh->refId = transAddExHandle(refMgt, exh); - - SExHandleWrap* wrap = taosMemoryCalloc(1, sizeof(SExHandleWrap)); - wrap->exhandle = exh; - wrap->refId = exh->refId; - conn->refId = exh->refId; - transMsg.info.handle = wrap; + transMsg.info.handle = (void*)conn->refId; tDebug("%s conn %p ref by app", CONN_GET_INST_LABEL(conn), conn); } @@ -357,12 +344,10 @@ void cliHandleResp(SCliConn* conn) { if (pCtx == NULL && CONN_NO_PERSIST_BY_APP(conn)) { tDebug("%s except, conn %p read while cli ignore it", CONN_GET_INST_LABEL(conn), conn); - // transUnrefCliHandle(conn); return; } if (CONN_RELEASE_BY_SERVER(conn) && transMsg.info.ahandle == NULL) { tDebug("%s except, conn %p read while cli ignore it", CONN_GET_INST_LABEL(conn), conn); - // transUnrefCliHandle(conn); return; } @@ -433,7 +418,7 @@ void cliHandleExcept(SCliConn* pConn) { return; } destroyCmsg(pMsg); - tTrace("%s conn %p start to destroy", CONN_GET_INST_LABEL(pConn), pConn); + tTrace("%s conn %p start to destroy, ref:%d", CONN_GET_INST_LABEL(pConn), pConn, T_REF_VAL_GET(pConn)); } while (!transQueueEmpty(&pConn->cliMsgs)); transUnrefCliHandle(pConn); } @@ -505,10 +490,22 @@ static SCliConn* getConnFromPool(void* pool, char* ip, uint32_t port) { QUEUE_INIT(&conn->conn); return conn; } +static void allocConnRef(SCliConn* conn, bool update) { + if (update) { + transRemoveExHandle(refMgt, conn->refId); + } + SExHandle* exh = taosMemoryCalloc(1, sizeof(SExHandle)); + exh->handle = conn; + exh->pThrd = conn->hostThrd; + exh->refId = transAddExHandle(refMgt, exh); + conn->refId = exh->refId; +} static void addConnToPool(void* pool, SCliConn* conn) { SCliThrdObj* thrd = conn->hostThrd; CONN_HANDLE_THREAD_QUIT(thrd); + allocConnRef(conn, true); + STrans* pTransInst = thrd->pTransInst; conn->expireTime = taosGetTimestampMs() + CONN_PERSIST_TIME(pTransInst->idleTime); transQueueClear(&conn->cliMsgs); @@ -558,7 +555,8 @@ static void cliRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) { return; } if (nread < 0) { - tError("%s conn %p read error: %s", CONN_GET_INST_LABEL(conn), conn, uv_err_name(nread)); + tError("%s conn %p read error: %s, ref: %d", CONN_GET_INST_LABEL(conn), conn, uv_err_name(nread), + T_REF_VAL_GET(conn)); conn->broken = true; cliHandleExcept(conn); } @@ -581,11 +579,7 @@ static SCliConn* cliCreateConn(SCliThrdObj* pThrd) { conn->broken = 0; transRefCliHandle(conn); - SExHandle* exh = taosMemoryCalloc(1, sizeof(SExHandle)); - exh->handle = conn; - exh->pThrd = pThrd; - exh->refId = transAddExHandle(refMgt, exh); - conn->refId = exh->refId; + allocConnRef(conn, false); return conn; } @@ -749,25 +743,27 @@ static void cliHandleUpdate(SCliMsg* pMsg, SCliThrdObj* pThrd) { } SCliConn* cliGetConn(SCliMsg* pMsg, SCliThrdObj* pThrd) { - SCliConn* conn = NULL; - SRpcHandleInfo* pInfo = &pMsg->msg.info; + SCliConn* conn = NULL; + // SExHandleWrap* exWrap = &pMsg->msg.info.handle; + // if (exWrap != NULL) { + //} - SExHandle* exh = transAcquireExHandle(refMgt, pInfo->refId); - if (exh == NULL) { - if (pInfo->refId != 0) { - tTrace("%s conn %p ignore msg", CONN_GET_INST_LABEL(conn), conn); - assert(0); - return NULL; - } - } else { - transReleaseExHandle(refMgt, pInfo->refId); - return exh->handle; - } + // SExHandle* exh = transAcquireExHandle(refMgt, exWrap->refId); + // if (exh == NULL) { + // if (pInfo->refId != 0) { + // tTrace("%s conn %p ignore msg", CONN_GET_INST_LABEL(conn), conn); + // assert(0); + // return NULL; + // } + //} else { + // transReleaseExHandle(refMgt, pInfo->refId); + // return exh->handle; + //} STransConnCtx* pCtx = pMsg->ctx; conn = getConnFromPool(pThrd->pool, EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet)); if (conn != NULL) { - exh = taosMemoryCalloc(1, sizeof(SExHandle)); + SExHandle* exh = taosMemoryCalloc(1, sizeof(SExHandle)); exh->handle = conn; exh->pThrd = pThrd; exh->refId = transAddExHandle(refMgt, exh); @@ -790,10 +786,6 @@ void cliMayCvtFqdnToIp(SEpSet* pEpSet, SCvtAddr* pCvtAddr) { } } void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { - uint64_t et = taosGetTimestampUs(); - uint64_t el = et - pMsg->st; - // tTrace("%s cli msg tran time cost: %" PRIu64 "us", ((STrans*)pThrd->pTransInst)->label, el); - STransConnCtx* pCtx = pMsg->ctx; STrans* pTransInst = pThrd->pTransInst; @@ -1014,7 +1006,6 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { tTrace("try to send req to next node"); pMsg->st = taosGetTimestampUs(); - taosMemoryFree(pResp->info.handle); pCtx->retryCount += 1; if (pResp->code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { if (pCtx->retryCount < pEpSet->numOfEps * 3) { @@ -1060,16 +1051,16 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { STraceId* trace = &pResp->info.traceId; if (pCtx->pSem != NULL) { - tGTrace("conn %p(sync) handle resp", pConn); + tGTrace("%s conn %p(sync) handle resp", CONN_GET_INST_LABEL(pConn), pConn); if (pCtx->pRsp == NULL) { - tGTrace("conn %p(sync) failed to resp, ignore", pConn); + tGTrace("%s conn %p(sync) failed to resp, ignore", CONN_GET_INST_LABEL(pConn), pConn); } else { memcpy((char*)pCtx->pRsp, (char*)pResp, sizeof(*pResp)); } tsem_post(pCtx->pSem); pCtx->pRsp = NULL; } else { - tGTrace("conn %p handle resp", pConn); + tGTrace("%s conn %p handle resp", CONN_GET_INST_LABEL(pConn), pConn); if (pResp->code != 0 || pCtx->retryCount == 0 || transEpSetIsEqual(&pCtx->epSet, &pCtx->origEpSet)) { pTransInst->cfp(pTransInst->parent, pResp, NULL); } else { @@ -1105,14 +1096,33 @@ void transUnrefCliHandle(void* handle) { return; } int ref = T_REF_DEC((SCliConn*)handle); - tTrace("%s conn %p ref %d", CONN_GET_INST_LABEL((SCliConn*)handle), handle, ref); + tTrace("%s conn %p ref:%d", CONN_GET_INST_LABEL((SCliConn*)handle), handle, ref); if (ref == 0) { cliDestroyConn((SCliConn*)handle, true); } } +SCliThrdObj* transGetWorkThrdFromHandle(int64_t handle) { + SCliThrdObj* pThrd = NULL; + SExHandle* exh = transAcquireExHandle(refMgt, handle); + if (exh == NULL) { + return NULL; + } + pThrd = exh->pThrd; + transReleaseExHandle(refMgt, handle); + return pThrd; +} +SCliThrdObj* transGetWorkThrd(STrans* trans, int64_t handle) { + int idx = -1; + if (handle == 0) { + idx = cliRBChoseIdx(trans); + return ((SCliObj*)trans->tcphandle)->pThreadObj[idx]; + } + return transGetWorkThrdFromHandle(handle); +} void transReleaseCliHandle(void* handle) { - SCliThrdObj* thrd = CONN_GET_HOST_THREAD(handle); - if (thrd == NULL) { + int idx = -1; + SCliThrdObj* pThrd = transGetWorkThrdFromHandle((int64_t)handle); + if (pThrd == NULL) { return; } @@ -1121,26 +1131,18 @@ void transReleaseCliHandle(void* handle) { cmsg->msg = tmsg; cmsg->type = Release; - transSendAsync(thrd->asyncPool, &cmsg->q); + transSendAsync(pThrd->asyncPool, &cmsg->q); + return; } void transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransCtx* ctx) { - STrans* pTransInst = (STrans*)shandle; - SRpcHandleInfo* info = &pReq->info; - - int idx = -1; - SCliThrdObj* pThrd = NULL; - SExHandle* exh = info->handle; - int64_t refId = -1; - if (exh != NULL) { - refId = exh->refId; + STrans* pTransInst = (STrans*)shandle; + SCliThrdObj* pThrd = transGetWorkThrd(pTransInst, (int64_t)pReq->info.handle); + if (pThrd == NULL) { + transFreeMsg(pReq->pCont); + return; } - CONN_HOST_THREAD_IDX1(idx, exh, refId, pThrd); - if (idx == -1) { - idx = cliRBChoseIdx(pTransInst); - pThrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[idx]; - } TRACE_SET_MSGID(&pReq->info.traceId, tGenIdPI64()); STransConnCtx* pCtx = taosMemoryCalloc(1, sizeof(STransConnCtx)); @@ -1163,28 +1165,17 @@ void transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STra tGTrace("%s send request at thread:%08" PRId64 ", dst: %s:%d, app:%p", transLabel(pTransInst), pThrd->pid, EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet), pReq->info.ahandle); ASSERT(transSendAsync(pThrd->asyncPool, &(cliMsg->q)) == 0); -_return1: - return; -_return2: return; } void transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp) { - STrans* pTransInst = (STrans*)shandle; - SRpcHandleInfo* info = &pReq->info; - SCliThrdObj* pThrd = NULL; - int idx = -1; - SExHandle* exh = info->handle; - int64_t refId = -1; - if (exh != NULL) { - refId = exh->refId; + STrans* pTransInst = (STrans*)shandle; + SCliThrdObj* pThrd = transGetWorkThrd(pTransInst, (int64_t)pReq->info.handle); + if (pThrd == NULL) { + transFreeMsg(pReq->pCont); + return; } - CONN_HOST_THREAD_IDX1(idx, exh, refId, pThrd); - if (idx == -1) { - idx = cliRBChoseIdx(pTransInst); - pThrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[idx]; - } tsem_t* sem = taosMemoryCalloc(1, sizeof(tsem_t)); tsem_init(sem, 0, 0); @@ -1211,16 +1202,13 @@ void transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransM tsem_wait(sem); tsem_destroy(sem); taosMemoryFree(sem); -_return1: - return; -_return2: return; } /* * **/ -void transSetDefaultAddr(void* ahandle, const char* ip, const char* fqdn) { - STrans* pTransInst = ahandle; +void transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn) { + STrans* pTransInst = shandle; SCvtAddr cvtAddr = {0}; if (ip != NULL && fqdn != NULL) { diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 121fddc99a..4cc2a9c9b2 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -133,7 +133,7 @@ static SSvrConn* createConn(void* hThrd); static void destroyConn(SSvrConn* conn, bool clear /*clear handle or not*/); static void destroyConnRegArg(SSvrConn* conn); -static int reallocConnRefHandle(SSvrConn* conn); +static int reallocConnRef(SSvrConn* conn); static void uvHandleQuit(SSvrMsg* msg, SWorkThrdObj* thrd); static void uvHandleRelease(SSvrMsg* msg, SWorkThrdObj* thrd); @@ -176,7 +176,7 @@ static bool addHandleToAcceptloop(void* arg); srvMsg->msg = tmsg; \ srvMsg->type = Release; \ srvMsg->pConn = conn; \ - reallocConnRefHandle(conn); \ + reallocConnRef(conn); \ if (!transQueuePush(&conn->srvMsgs, srvMsg)) { \ return; \ } \ @@ -353,7 +353,7 @@ void uvOnSendCb(uv_write_t* req, int status) { // if (msg->type == Release && conn->status != ConnNormal) { // conn->status = ConnNormal; // transUnrefSrvHandle(conn); - // reallocConnRefHandle(conn); + // reallocConnRef(conn); // destroySmsg(msg); // transQueueClear(&conn->srvMsgs); // return; @@ -800,7 +800,7 @@ static void destroyConnRegArg(SSvrConn* conn) { conn->regArg.init = 0; } } -static int reallocConnRefHandle(SSvrConn* conn) { +static int reallocConnRef(SSvrConn* conn) { transReleaseExHandle(refMgt, conn->refId); transRemoveExHandle(refMgt, conn->refId); // avoid app continue to send msg on invalid handle @@ -945,7 +945,7 @@ void uvHandleQuit(SSvrMsg* msg, SWorkThrdObj* thrd) { void uvHandleRelease(SSvrMsg* msg, SWorkThrdObj* thrd) { SSvrConn* conn = msg->pConn; if (conn->status == ConnAcquire) { - reallocConnRefHandle(conn); + reallocConnRef(conn); if (!transQueuePush(&conn->srvMsgs, msg)) { return; } From 0d904d5aa3b27c69776572fbc04887d1db5ac9c5 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 23 Jun 2022 20:06:22 +0800 Subject: [PATCH 04/72] fix: handle except --- source/client/src/clientEnv.c | 31 ++-- source/libs/transport/src/transCli.c | 244 +++++++++++++-------------- source/libs/transport/src/transSvr.c | 72 ++++---- 3 files changed, 171 insertions(+), 176 deletions(-) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 9f04e89694..f1e4107e23 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -13,11 +13,11 @@ * along with this program. If not, see . */ -#include "os.h" #include "catalog.h" -#include "functionMgt.h" #include "clientInt.h" #include "clientLog.h" +#include "functionMgt.h" +#include "os.h" #include "query.h" #include "scheduler.h" #include "tcache.h" @@ -38,7 +38,7 @@ static TdThreadOnce tscinit = PTHREAD_ONCE_INIT; volatile int32_t tscInitRes = 0; static void registerRequest(SRequestObj *pRequest) { - STscObj *pTscObj = acquireTscObj(*(int64_t*)pRequest->pTscObj->id); + STscObj *pTscObj = acquireTscObj(*(int64_t *)pRequest->pTscObj->id); assert(pTscObj != NULL); @@ -54,14 +54,14 @@ static void registerRequest(SRequestObj *pRequest) { int32_t currentInst = atomic_add_fetch_64((int64_t *)&pSummary->currentRequests, 1); tscDebug("0x%" PRIx64 " new Request from connObj:0x%" PRIx64 ", current:%d, app current:%d, total:%d, reqId:0x%" PRIx64, - pRequest->self, *(int64_t*)pRequest->pTscObj->id, num, currentInst, total, pRequest->requestId); + pRequest->self, *(int64_t *)pRequest->pTscObj->id, num, currentInst, total, pRequest->requestId); } } static void deregisterRequest(SRequestObj *pRequest) { assert(pRequest != NULL); - STscObj *pTscObj = pRequest->pTscObj; + STscObj * pTscObj = pRequest->pTscObj; SAppClusterSummary *pActivity = &pTscObj->pAppInfo->summary; int32_t currentInst = atomic_sub_fetch_64((int64_t *)&pActivity->currentRequests, 1); @@ -70,8 +70,8 @@ static void deregisterRequest(SRequestObj *pRequest) { int64_t duration = taosGetTimestampUs() - pRequest->metric.start; tscDebug("0x%" PRIx64 " free Request from connObj: 0x%" PRIx64 ", reqId:0x%" PRIx64 " elapsed:%" PRIu64 " ms, current:%d, app current:%d", - pRequest->self, *(int64_t*)pTscObj->id, pRequest->requestId, duration / 1000, num, currentInst); - releaseTscObj(*(int64_t*)pTscObj->id); + pRequest->self, *(int64_t *)pTscObj->id, pRequest->requestId, duration / 1000, num, currentInst); + releaseTscObj(*(int64_t *)pTscObj->id); } // todo close the transporter properly @@ -80,7 +80,7 @@ void closeTransporter(STscObj *pTscObj) { return; } - tscDebug("free transporter:%p in connObj: 0x%" PRIx64, pTscObj->pAppInfo->pTransporter, *(int64_t*)pTscObj->id); + tscDebug("free transporter:%p in connObj: 0x%" PRIx64, pTscObj->pAppInfo->pTransporter, *(int64_t *)pTscObj->id); rpcClose(pTscObj->pAppInfo->pTransporter); } @@ -128,16 +128,17 @@ void closeAllRequests(SHashObj *pRequests) { void destroyTscObj(void *pObj) { STscObj *pTscObj = pObj; - SClientHbKey connKey = {.tscRid = *(int64_t*)pTscObj->id, .connType = pTscObj->connType}; + SClientHbKey connKey = {.tscRid = *(int64_t *)pTscObj->id, .connType = pTscObj->connType}; hbDeregisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey); int64_t connNum = atomic_sub_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); closeAllRequests(pTscObj->pRequests); schedulerStopQueryHb(pTscObj->pAppInfo->pTransporter); if (0 == connNum) { - // TODO - //closeTransporter(pTscObj); + // TODO + closeTransporter(pTscObj); } - tscDebug("connObj 0x%" PRIx64 " destroyed, totalConn:%" PRId64, *(int64_t*)pTscObj->id, pTscObj->pAppInfo->numOfConns); + tscDebug("connObj 0x%" PRIx64 " destroyed, totalConn:%" PRId64, *(int64_t *)pTscObj->id, + pTscObj->pAppInfo->numOfConns); taosThreadMutexDestroy(&pTscObj->mutex); taosMemoryFreeClear(pTscObj); } @@ -167,10 +168,10 @@ void *createTscObj(const char *user, const char *auth, const char *db, int32_t c taosThreadMutexInit(&pObj->mutex, NULL); pObj->id = taosMemoryMalloc(sizeof(int64_t)); - *(int64_t*)pObj->id = taosAddRef(clientConnRefPool, pObj); + *(int64_t *)pObj->id = taosAddRef(clientConnRefPool, pObj); pObj->schemalessType = 1; - tscDebug("connObj created, 0x%" PRIx64, *(int64_t*)pObj->id); + tscDebug("connObj created, 0x%" PRIx64, *(int64_t *)pObj->id); return pObj; } @@ -325,7 +326,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { return 0; } - SConfig *pCfg = taosGetCfg(); + SConfig * pCfg = taosGetCfg(); SConfigItem *pItem = NULL; switch (option) { diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index e18723d976..6abd230f31 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1,4 +1,4 @@ -/* * Copyright (c) 2019 TAOS Data, Inc. +/** Copyright (c) 2019 TAOS Data, Inc. * * This program is free software: you can use, redistribute, and/or modify * it under the terms of the GNU Affero General Public License, version 3 @@ -54,7 +54,7 @@ typedef struct SCliMsg { int sent; //(0: no send, 1: alread sent) } SCliMsg; -typedef struct SCliThrdObj { +typedef struct SCliThrd { TdThread thread; // tid int64_t pid; // pid uv_loop_t* loop; @@ -72,13 +72,13 @@ typedef struct SCliThrdObj { SCvtAddr cvtAddr; bool quit; -} SCliThrdObj; +} SCliThrd; typedef struct SCliObj { - char label[TSDB_LABEL_LEN]; - int32_t index; - int numOfThreads; - SCliThrdObj** pThreadObj; + char label[TSDB_LABEL_LEN]; + int32_t index; + int numOfThreads; + SCliThrd** pThreadObj; } SCliObj; typedef struct SConnList { @@ -106,7 +106,7 @@ static void cliAsyncCb(uv_async_t* handle); static int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg); -static SCliConn* cliCreateConn(SCliThrdObj* thrd); +static SCliConn* cliCreateConn(SCliThrd* thrd); static void cliDestroyConn(SCliConn* pConn, bool clear /*clear tcp handle or not*/); static void cliDestroy(uv_handle_t* handle); static void cliSend(SCliConn* pConn); @@ -122,14 +122,14 @@ static void cliHandleResp(SCliConn* conn); static void cliHandleExcept(SCliConn* conn); // handle req from app -static void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd); -static void cliHandleQuit(SCliMsg* pMsg, SCliThrdObj* pThrd); -static void cliHandleRelease(SCliMsg* pMsg, SCliThrdObj* pThrd); -static void cliHandleUpdate(SCliMsg* pMsg, SCliThrdObj* pThrd); -static void (*cliAsyncHandle[])(SCliMsg* pMsg, SCliThrdObj* pThrd) = {cliHandleReq, cliHandleQuit, cliHandleRelease, - NULL, cliHandleUpdate}; +static void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd); +static void cliHandleQuit(SCliMsg* pMsg, SCliThrd* pThrd); +static void cliHandleRelease(SCliMsg* pMsg, SCliThrd* pThrd); +static void cliHandleUpdate(SCliMsg* pMsg, SCliThrd* pThrd); +static void (*cliAsyncHandle[])(SCliMsg* pMsg, SCliThrd* pThrd) = {cliHandleReq, cliHandleQuit, cliHandleRelease, NULL, + cliHandleUpdate}; -static void cliSendQuit(SCliThrdObj* thrd); +static void cliSendQuit(SCliThrd* thrd); static void destroyUserdata(STransMsg* userdata); static int cliRBChoseIdx(STrans* pTransInst); @@ -137,8 +137,8 @@ static int cliRBChoseIdx(STrans* pTransInst); static void destroyCmsg(SCliMsg* cmsg); static void transDestroyConnCtx(STransConnCtx* ctx); // thread obj -static SCliThrdObj* createThrdObj(); -static void destroyThrdObj(SCliThrdObj* pThrd); +static SCliThrd* createThrdObj(); +static void destroyThrdObj(SCliThrd* pThrd); static void cliWalkCb(uv_handle_t* handle, void* arg); @@ -174,12 +174,12 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { idx = -1; \ } else { \ ASYNC_CHECK_HANDLE((exh), refId); \ - pThrd = (SCliThrdObj*)(exh)->pThrd; \ + pThrd = (SCliThrd*)(exh)->pThrd; \ } \ } while (0) #define CONN_PERSIST_TIME(para) (para * 1000 * 10) #define CONN_GET_HOST_THREAD(conn) (conn ? ((SCliConn*)conn)->hostThrd : NULL) -#define CONN_GET_INST_LABEL(conn) (((STrans*)(((SCliThrdObj*)(conn)->hostThrd)->pTransInst))->label) +#define CONN_GET_INST_LABEL(conn) (((STrans*)(((SCliThrd*)(conn)->hostThrd)->pTransInst))->label) #define CONN_SHOULD_RELEASE(conn, head) \ do { \ if ((head)->release == 1 && (head->msgLen) == sizeof(*head)) { \ @@ -195,7 +195,7 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { destroyCmsg(pMsg); \ cliReleaseUnfinishedMsg(conn); \ if (status != ConnInPool) { \ - addConnToPool(((SCliThrdObj*)conn->hostThrd)->pool, conn); \ + addConnToPool(((SCliThrd*)conn->hostThrd)->pool, conn); \ } \ transRemoveExHandle(refMgt, conn->refId); \ return; \ @@ -279,8 +279,8 @@ _RETURN: return false; } void cliHandleResp(SCliConn* conn) { - SCliThrdObj* pThrd = conn->hostThrd; - STrans* pTransInst = pThrd->pTransInst; + SCliThrd* pThrd = conn->hostThrd; + STrans* pTransInst = pThrd->pTransInst; STransMsgHead* pHead = (STransMsgHead*)(conn->readBuf.buf); pHead->code = htonl(pHead->code); @@ -379,9 +379,9 @@ void cliHandleExcept(SCliConn* pConn) { return; } } - SCliThrdObj* pThrd = pConn->hostThrd; - STrans* pTransInst = pThrd->pTransInst; - bool once = false; + SCliThrd* pThrd = pConn->hostThrd; + STrans* pTransInst = pThrd->pTransInst; + bool once = false; do { SCliMsg* pMsg = transQueuePop(&pConn->cliMsgs); if (pMsg == NULL && once) { @@ -424,9 +424,9 @@ void cliHandleExcept(SCliConn* pConn) { } void cliTimeoutCb(uv_timer_t* handle) { - SCliThrdObj* pThrd = handle->data; - STrans* pTransInst = pThrd->pTransInst; - int64_t currentTime = pThrd->nextTimeout; + SCliThrd* pThrd = handle->data; + STrans* pTransInst = pThrd->pTransInst; + int64_t currentTime = pThrd->nextTimeout; tTrace("%s conn timeout, try to remove expire conn from conn pool", pTransInst->label); SConnList* p = taosHashIterate((SHashObj*)pThrd->pool, NULL); @@ -501,7 +501,7 @@ static void allocConnRef(SCliConn* conn, bool update) { conn->refId = exh->refId; } static void addConnToPool(void* pool, SCliConn* conn) { - SCliThrdObj* thrd = conn->hostThrd; + SCliThrd* thrd = conn->hostThrd; CONN_HANDLE_THREAD_QUIT(thrd); allocConnRef(conn, true); @@ -562,7 +562,7 @@ static void cliRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) { } } -static SCliConn* cliCreateConn(SCliThrdObj* pThrd) { +static SCliConn* cliCreateConn(SCliThrd* pThrd) { SCliConn* conn = taosMemoryCalloc(1, sizeof(SCliConn)); // read/write stream handle conn->stream = (uv_stream_t*)taosMemoryMalloc(sizeof(uv_tcp_t)); @@ -615,7 +615,7 @@ static bool cliHandleNoResp(SCliConn* conn) { } if (res == true) { if (cliMaySendCachedMsg(conn) == false) { - SCliThrdObj* thrd = conn->hostThrd; + SCliThrd* thrd = conn->hostThrd; addConnToPool(thrd->pool, conn); } } @@ -651,8 +651,8 @@ void cliSend(SCliConn* pConn) { STransConnCtx* pCtx = pCliMsg->ctx; - SCliThrdObj* pThrd = pConn->hostThrd; - STrans* pTransInst = pThrd->pTransInst; + SCliThrd* pThrd = pConn->hostThrd; + STrans* pTransInst = pThrd->pTransInst; STransMsg* pMsg = (STransMsg*)(&pCliMsg->msg); if (pMsg->pCont == 0) { @@ -709,7 +709,7 @@ void cliConnCb(uv_connect_t* req, int status) { cliSend(pConn); } -static void cliHandleQuit(SCliMsg* pMsg, SCliThrdObj* pThrd) { +static void cliHandleQuit(SCliMsg* pMsg, SCliThrd* pThrd) { tDebug("cli work thread %p start to quit", pThrd); destroyCmsg(pMsg); destroyConnPool(pThrd->pool); @@ -720,7 +720,7 @@ static void cliHandleQuit(SCliMsg* pMsg, SCliThrdObj* pThrd) { // uv_stop(pThrd->loop); } -static void cliHandleRelease(SCliMsg* pMsg, SCliThrdObj* pThrd) { +static void cliHandleRelease(SCliMsg* pMsg, SCliThrd* pThrd) { SCliConn* conn = pMsg->msg.info.handle; tDebug("%s conn %p start to release to inst", CONN_GET_INST_LABEL(conn), conn); @@ -735,39 +735,30 @@ static void cliHandleRelease(SCliMsg* pMsg, SCliThrdObj* pThrd) { transUnrefCliHandle(conn); } } -static void cliHandleUpdate(SCliMsg* pMsg, SCliThrdObj* pThrd) { +static void cliHandleUpdate(SCliMsg* pMsg, SCliThrd* pThrd) { STransConnCtx* pCtx = pMsg->ctx; pThrd->cvtAddr = pCtx->cvtAddr; destroyCmsg(pMsg); } -SCliConn* cliGetConn(SCliMsg* pMsg, SCliThrdObj* pThrd) { +SCliConn* cliGetConn(SCliMsg* pMsg, SCliThrd* pThrd) { SCliConn* conn = NULL; - // SExHandleWrap* exWrap = &pMsg->msg.info.handle; - // if (exWrap != NULL) { - //} - - // SExHandle* exh = transAcquireExHandle(refMgt, exWrap->refId); - // if (exh == NULL) { - // if (pInfo->refId != 0) { - // tTrace("%s conn %p ignore msg", CONN_GET_INST_LABEL(conn), conn); - // assert(0); - // return NULL; - // } - //} else { - // transReleaseExHandle(refMgt, pInfo->refId); - // return exh->handle; - //} + int64_t refId = (int64_t)(pMsg->msg.info.handle); + if (refId != 0) { + SExHandle* exh = transAcquireExHandle(refMgt, refId); + if (exh == NULL) { + assert(0); + } else { + conn = exh->handle; + transReleaseExHandle(refMgt, refId); + } + return conn; + }; STransConnCtx* pCtx = pMsg->ctx; conn = getConnFromPool(pThrd->pool, EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet)); if (conn != NULL) { - SExHandle* exh = taosMemoryCalloc(1, sizeof(SExHandle)); - exh->handle = conn; - exh->pThrd = pThrd; - exh->refId = transAddExHandle(refMgt, exh); - conn->refId = exh->refId; tTrace("%s conn %p get from conn pool", CONN_GET_INST_LABEL(conn), conn); } else { tTrace("%s not found conn in conn pool %p", ((STrans*)pThrd->pTransInst)->label, pThrd->pool); @@ -785,7 +776,7 @@ void cliMayCvtFqdnToIp(SEpSet* pEpSet, SCvtAddr* pCvtAddr) { } } } -void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { +void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) { STransConnCtx* pCtx = pMsg->ctx; STrans* pTransInst = pThrd->pTransInst; @@ -833,9 +824,9 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { } } static void cliAsyncCb(uv_async_t* handle) { - SAsyncItem* item = handle->data; - SCliThrdObj* pThrd = item->pThrd; - SCliMsg* pMsg = NULL; + SAsyncItem* item = handle->data; + SCliThrd* pThrd = item->pThrd; + SCliMsg* pMsg = NULL; // batch process to avoid to lock/unlock frequently queue wq; @@ -861,7 +852,7 @@ static void cliAsyncCb(uv_async_t* handle) { } static void* cliWorkThread(void* arg) { - SCliThrdObj* pThrd = (SCliThrdObj*)arg; + SCliThrd* pThrd = (SCliThrd*)arg; pThrd->pid = taosGetSelfPthreadId(); setThreadName("trans-cli-work"); uv_run(pThrd->loop, UV_RUN_DEFAULT); @@ -874,10 +865,10 @@ void* transInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, STrans* pTransInst = shandle; memcpy(cli->label, label, strlen(label)); cli->numOfThreads = numOfThreads; - cli->pThreadObj = (SCliThrdObj**)taosMemoryCalloc(cli->numOfThreads, sizeof(SCliThrdObj*)); + cli->pThreadObj = (SCliThrd**)taosMemoryCalloc(cli->numOfThreads, sizeof(SCliThrd*)); for (int i = 0; i < cli->numOfThreads; i++) { - SCliThrdObj* pThrd = createThrdObj(); + SCliThrd* pThrd = createThrdObj(); pThrd->nextTimeout = taosGetTimestampMs() + CONN_PERSIST_TIME(pTransInst->idleTime); pThrd->pTransInst = shandle; @@ -911,8 +902,8 @@ static void destroyCmsg(SCliMsg* pMsg) { taosMemoryFree(pMsg); } -static SCliThrdObj* createThrdObj() { - SCliThrdObj* pThrd = (SCliThrdObj*)taosMemoryCalloc(1, sizeof(SCliThrdObj)); +static SCliThrd* createThrdObj() { + SCliThrd* pThrd = (SCliThrd*)taosMemoryCalloc(1, sizeof(SCliThrd)); QUEUE_INIT(&pThrd->msg); taosThreadMutexInit(&pThrd->msgMtx, NULL); @@ -930,7 +921,7 @@ static SCliThrdObj* createThrdObj() { pThrd->quit = false; return pThrd; } -static void destroyThrdObj(SCliThrdObj* pThrd) { +static void destroyThrdObj(SCliThrd* pThrd) { if (pThrd == NULL) { return; } @@ -951,7 +942,7 @@ static void transDestroyConnCtx(STransConnCtx* ctx) { taosMemoryFree(ctx); } -void cliSendQuit(SCliThrdObj* thrd) { +void cliSendQuit(SCliThrd* thrd) { // cli can stop gracefully SCliMsg* msg = taosMemoryCalloc(1, sizeof(SCliMsg)); msg->type = Quit; @@ -973,15 +964,16 @@ int cliRBChoseIdx(STrans* pTransInst) { static void doDelayTask(void* param) { STaskArg* arg = param; - SCliMsg* pMsg = arg->param1; - SCliThrdObj* pThrd = arg->param2; + SCliMsg* pMsg = arg->param1; + SCliThrd* pThrd = arg->param2; cliHandleReq(pMsg, pThrd); taosMemoryFree(arg); } + int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { - SCliThrdObj* pThrd = pConn->hostThrd; - STrans* pTransInst = pThrd->pTransInst; + SCliThrd* pThrd = pConn->hostThrd; + STrans* pTransInst = pThrd->pTransInst; if (pMsg == NULL || pMsg->ctx == NULL) { tTrace("%s conn %p handle resp", pTransInst->label, pConn); @@ -995,57 +987,60 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { if (pCtx->retryCount == 0) { pCtx->origEpSet = pCtx->epSet; } + /* * upper layer handle retry if code equal TSDB_CODE_RPC_NETWORK_UNAVAIL */ - tmsg_t msgType = pCtx->msgType; - if ((pTransInst->retry != NULL && pEpSet->numOfEps > 1 && (pTransInst->retry(pResp->code))) || - (pResp->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || pResp->code == TSDB_CODE_APP_NOT_READY || - pResp->code == TSDB_CODE_NODE_NOT_DEPLOYED || pResp->code == TSDB_CODE_SYN_NOT_LEADER)) { - pMsg->sent = 0; - tTrace("try to send req to next node"); - pMsg->st = taosGetTimestampUs(); + if (CONN_NO_PERSIST_BY_APP(pConn)) { + tmsg_t msgType = pCtx->msgType; + if ((pTransInst->retry != NULL && pEpSet->numOfEps > 1 && (pTransInst->retry(pResp->code))) || + (pResp->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || pResp->code == TSDB_CODE_APP_NOT_READY || + pResp->code == TSDB_CODE_NODE_NOT_DEPLOYED || pResp->code == TSDB_CODE_SYN_NOT_LEADER)) { + pMsg->sent = 0; + tTrace("try to send req to next node"); + pMsg->st = taosGetTimestampUs(); - pCtx->retryCount += 1; - if (pResp->code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { - if (pCtx->retryCount < pEpSet->numOfEps * 3) { - pEpSet->inUse = (++pEpSet->inUse) % pEpSet->numOfEps; + pCtx->retryCount += 1; + if (pResp->code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { + if (pCtx->retryCount < pEpSet->numOfEps * 3) { + pEpSet->inUse = (++pEpSet->inUse) % pEpSet->numOfEps; + + STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); + arg->param1 = pMsg; + arg->param2 = pThrd; + transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL); + transPrintEpSet(pEpSet); + tTrace("%s use local epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse, + pCtx->retryCount + 1, pEpSet->numOfEps * 3); + + transUnrefCliHandle(pConn); + return -1; + } + } else if (pCtx->retryCount < TRANS_RETRY_COUNT_LIMIT) { + if (pResp->contLen == 0) { + pEpSet->inUse = (++pEpSet->inUse) % pEpSet->numOfEps; + transPrintEpSet(&pCtx->epSet); + tTrace("%s use local epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse, + pCtx->retryCount + 1, TRANS_RETRY_COUNT_LIMIT); + } else { + SEpSet epSet = {0}; + tDeserializeSEpSet(pResp->pCont, pResp->contLen, &epSet); + pCtx->epSet = epSet; + + transPrintEpSet(&pCtx->epSet); + tTrace("%s use remote epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse, + pCtx->retryCount + 1, TRANS_RETRY_COUNT_LIMIT); + } + if (pConn->status != ConnInPool) { + addConnToPool(pThrd->pool, pConn); + } STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); arg->param1 = pMsg; arg->param2 = pThrd; transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL); - transPrintEpSet(pEpSet); - tTrace("%s use local epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse, - pCtx->retryCount + 1, pEpSet->numOfEps * 3); - - transUnrefCliHandle(pConn); return -1; } - } else if (pCtx->retryCount < TRANS_RETRY_COUNT_LIMIT) { - if (pResp->contLen == 0) { - pEpSet->inUse = (++pEpSet->inUse) % pEpSet->numOfEps; - transPrintEpSet(&pCtx->epSet); - tTrace("%s use local epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse, - pCtx->retryCount + 1, TRANS_RETRY_COUNT_LIMIT); - } else { - SEpSet epSet = {0}; - tDeserializeSEpSet(pResp->pCont, pResp->contLen, &epSet); - pCtx->epSet = epSet; - - transPrintEpSet(&pCtx->epSet); - tTrace("%s use remote epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse, - pCtx->retryCount + 1, TRANS_RETRY_COUNT_LIMIT); - } - if (pConn->status != ConnInPool) { - addConnToPool(pThrd->pool, pConn); - } - - STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); - arg->param1 = pMsg; - arg->param2 = pThrd; - transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL); - return -1; } } @@ -1101,9 +1096,9 @@ void transUnrefCliHandle(void* handle) { cliDestroyConn((SCliConn*)handle, true); } } -SCliThrdObj* transGetWorkThrdFromHandle(int64_t handle) { - SCliThrdObj* pThrd = NULL; - SExHandle* exh = transAcquireExHandle(refMgt, handle); +SCliThrd* transGetWorkThrdFromHandle(int64_t handle) { + SCliThrd* pThrd = NULL; + SExHandle* exh = transAcquireExHandle(refMgt, handle); if (exh == NULL) { return NULL; } @@ -1111,17 +1106,16 @@ SCliThrdObj* transGetWorkThrdFromHandle(int64_t handle) { transReleaseExHandle(refMgt, handle); return pThrd; } -SCliThrdObj* transGetWorkThrd(STrans* trans, int64_t handle) { - int idx = -1; +SCliThrd* transGetWorkThrd(STrans* trans, int64_t handle) { if (handle == 0) { - idx = cliRBChoseIdx(trans); + int idx = cliRBChoseIdx(trans); return ((SCliObj*)trans->tcphandle)->pThreadObj[idx]; } return transGetWorkThrdFromHandle(handle); } void transReleaseCliHandle(void* handle) { - int idx = -1; - SCliThrdObj* pThrd = transGetWorkThrdFromHandle((int64_t)handle); + int idx = -1; + SCliThrd* pThrd = transGetWorkThrdFromHandle((int64_t)handle); if (pThrd == NULL) { return; } @@ -1136,8 +1130,8 @@ void transReleaseCliHandle(void* handle) { } void transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransCtx* ctx) { - STrans* pTransInst = (STrans*)shandle; - SCliThrdObj* pThrd = transGetWorkThrd(pTransInst, (int64_t)pReq->info.handle); + STrans* pTransInst = (STrans*)shandle; + SCliThrd* pThrd = transGetWorkThrd(pTransInst, (int64_t)pReq->info.handle); if (pThrd == NULL) { transFreeMsg(pReq->pCont); return; @@ -1169,8 +1163,8 @@ void transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STra } void transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp) { - STrans* pTransInst = (STrans*)shandle; - SCliThrdObj* pThrd = transGetWorkThrd(pTransInst, (int64_t)pReq->info.handle); + STrans* pTransInst = (STrans*)shandle; + SCliThrd* pThrd = transGetWorkThrd(pTransInst, (int64_t)pReq->info.handle); if (pThrd == NULL) { transFreeMsg(pReq->pCont); return; @@ -1224,7 +1218,7 @@ void transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn) { cliMsg->ctx = pCtx; cliMsg->type = Update; - SCliThrdObj* thrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[i]; + SCliThrd* thrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[i]; tDebug("%s update epset at thread:%08" PRId64 "", pTransInst->label, thrd->pid); transSendAsync(thrd->asyncPool, &(cliMsg->q)); diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 4cc2a9c9b2..599d98a3e9 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -65,7 +65,7 @@ typedef struct SSvrMsg { STransMsgType type; } SSvrMsg; -typedef struct SWorkThrdObj { +typedef struct SWorkThrd { TdThread thread; uv_connect_t connect_req; uv_pipe_t* pipe; @@ -78,7 +78,7 @@ typedef struct SWorkThrdObj { queue conn; void* pTransInst; bool quit; -} SWorkThrdObj; +} SWorkThrd; typedef struct SServerObj { TdThread thread; @@ -86,10 +86,10 @@ typedef struct SServerObj { uv_loop_t* loop; // work thread info - int workerIdx; - int numOfThreads; - int numOfWorkerReady; - SWorkThrdObj** pThreadObj; + int workerIdx; + int numOfThreads; + int numOfWorkerReady; + SWorkThrd** pThreadObj; uv_pipe_t pipeListen; uv_pipe_t** pipe; @@ -135,12 +135,12 @@ static void destroyConnRegArg(SSvrConn* conn); static int reallocConnRef(SSvrConn* conn); -static void uvHandleQuit(SSvrMsg* msg, SWorkThrdObj* thrd); -static void uvHandleRelease(SSvrMsg* msg, SWorkThrdObj* thrd); -static void uvHandleResp(SSvrMsg* msg, SWorkThrdObj* thrd); -static void uvHandleRegister(SSvrMsg* msg, SWorkThrdObj* thrd); -static void (*transAsyncHandle[])(SSvrMsg* msg, SWorkThrdObj* thrd) = {uvHandleResp, uvHandleQuit, uvHandleRelease, - uvHandleRegister, NULL}; +static void uvHandleQuit(SSvrMsg* msg, SWorkThrd* thrd); +static void uvHandleRelease(SSvrMsg* msg, SWorkThrd* thrd); +static void uvHandleResp(SSvrMsg* msg, SWorkThrd* thrd); +static void uvHandleRegister(SSvrMsg* msg, SWorkThrd* thrd); +static void (*transAsyncHandle[])(SSvrMsg* msg, SWorkThrd* thrd) = {uvHandleResp, uvHandleQuit, uvHandleRelease, + uvHandleRegister, NULL}; static int32_t exHandlesMgt; @@ -160,7 +160,7 @@ static void* transWorkerThread(void* arg); static void* transAcceptThread(void* arg); // add handle loop -static bool addHandleToWorkloop(SWorkThrdObj* pThrd, char* pipeName); +static bool addHandleToWorkloop(SWorkThrd* pThrd, char* pipeName); static bool addHandleToAcceptloop(void* arg); #define CONN_SHOULD_RELEASE(conn, head) \ @@ -233,7 +233,7 @@ static void uvHandleReq(SSvrConn* pConn) { // wreq->data = pConn; // uv_read_stop((uv_stream_t*)pConn->pTcp); // transRefSrvHandle(pConn); - // uv_queue_work(((SWorkThrdObj*)pConn->hostThrd)->loop, wreq, uvWorkDoTask, uvWorkAfterTask); + // uv_queue_work(((SWorkThrd*)pConn->hostThrd)->loop, wreq, uvWorkDoTask, uvWorkAfterTask); CONN_SHOULD_RELEASE(pConn, pHead); @@ -478,7 +478,7 @@ static void destroySmsg(SSvrMsg* smsg) { transFreeMsg(smsg->msg.pCont); taosMemoryFree(smsg); } -static void destroyAllConn(SWorkThrdObj* pThrd) { +static void destroyAllConn(SWorkThrd* pThrd) { tTrace("thread %p destroy all conn ", pThrd); while (!QUEUE_IS_EMPTY(&pThrd->conn)) { queue* h = QUEUE_HEAD(&pThrd->conn); @@ -493,10 +493,10 @@ static void destroyAllConn(SWorkThrdObj* pThrd) { } } void uvWorkerAsyncCb(uv_async_t* handle) { - SAsyncItem* item = handle->data; - SWorkThrdObj* pThrd = item->pThrd; - SSvrConn* conn = NULL; - queue wq; + SAsyncItem* item = handle->data; + SWorkThrd* pThrd = item->pThrd; + SSvrConn* conn = NULL; + queue wq; // batch process to avoid to lock/unlock frequently taosThreadMutexLock(&item->mtx); @@ -624,7 +624,7 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { assert(buf->base[0] == notify[0]); taosMemoryFree(buf->base); - SWorkThrdObj* pThrd = q->data; + SWorkThrd* pThrd = q->data; uv_pipe_t* pipe = (uv_pipe_t*)q; if (!uv_pipe_pending_count(pipe)) { @@ -692,10 +692,10 @@ void uvOnPipeConnectionCb(uv_connect_t* connect, int status) { if (status != 0) { return; } - SWorkThrdObj* pThrd = container_of(connect, SWorkThrdObj, connect_req); + SWorkThrd* pThrd = container_of(connect, SWorkThrd, connect_req); uv_read_start((uv_stream_t*)pThrd->pipe, uvAllocConnBufferCb, uvOnConnectionCb); } -static bool addHandleToWorkloop(SWorkThrdObj* pThrd, char* pipeName) { +static bool addHandleToWorkloop(SWorkThrd* pThrd, char* pipeName) { pThrd->loop = (uv_loop_t*)taosMemoryMalloc(sizeof(uv_loop_t)); if (0 != uv_loop_init(pThrd->loop)) { return false; @@ -748,14 +748,14 @@ static bool addHandleToAcceptloop(void* arg) { } void* transWorkerThread(void* arg) { setThreadName("trans-worker"); - SWorkThrdObj* pThrd = (SWorkThrdObj*)arg; + SWorkThrd* pThrd = (SWorkThrd*)arg; uv_run(pThrd->loop, UV_RUN_DEFAULT); return NULL; } static SSvrConn* createConn(void* hThrd) { - SWorkThrdObj* pThrd = hThrd; + SWorkThrd* pThrd = hThrd; SSvrConn* pConn = (SSvrConn*)taosMemoryCalloc(1, sizeof(SSvrConn)); QUEUE_INIT(&pConn->queue); @@ -818,7 +818,7 @@ static void uvDestroyConn(uv_handle_t* handle) { if (conn == NULL) { return; } - SWorkThrdObj* thrd = conn->hostThrd; + SWorkThrd* thrd = conn->hostThrd; transReleaseExHandle(refMgt, conn->refId); transRemoveExHandle(refMgt, conn->refId); @@ -863,7 +863,7 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, srv->numOfThreads = numOfThreads; srv->workerIdx = 0; srv->numOfWorkerReady = 0; - srv->pThreadObj = (SWorkThrdObj**)taosMemoryCalloc(srv->numOfThreads, sizeof(SWorkThrdObj*)); + srv->pThreadObj = (SWorkThrd**)taosMemoryCalloc(srv->numOfThreads, sizeof(SWorkThrd*)); srv->pipe = (uv_pipe_t**)taosMemoryCalloc(srv->numOfThreads, sizeof(uv_pipe_t*)); srv->ip = ip; srv->port = port; @@ -888,7 +888,7 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, assert(0 == uv_listen((uv_stream_t*)&srv->pipeListen, SOMAXCONN, uvPipeListenCb)); for (int i = 0; i < srv->numOfThreads; i++) { - SWorkThrdObj* thrd = (SWorkThrdObj*)taosMemoryCalloc(1, sizeof(SWorkThrdObj)); + SWorkThrd* thrd = (SWorkThrd*)taosMemoryCalloc(1, sizeof(SWorkThrd)); thrd->pTransInst = shandle; thrd->quit = false; srv->pThreadObj[i] = thrd; @@ -933,7 +933,7 @@ End: return NULL; } -void uvHandleQuit(SSvrMsg* msg, SWorkThrdObj* thrd) { +void uvHandleQuit(SSvrMsg* msg, SWorkThrd* thrd) { thrd->quit = true; if (QUEUE_IS_EMPTY(&thrd->conn)) { uv_walk(thrd->loop, uvWalkCb, NULL); @@ -942,7 +942,7 @@ void uvHandleQuit(SSvrMsg* msg, SWorkThrdObj* thrd) { } taosMemoryFree(msg); } -void uvHandleRelease(SSvrMsg* msg, SWorkThrdObj* thrd) { +void uvHandleRelease(SSvrMsg* msg, SWorkThrd* thrd) { SSvrConn* conn = msg->pConn; if (conn->status == ConnAcquire) { reallocConnRef(conn); @@ -956,12 +956,12 @@ void uvHandleRelease(SSvrMsg* msg, SWorkThrdObj* thrd) { } destroySmsg(msg); } -void uvHandleResp(SSvrMsg* msg, SWorkThrdObj* thrd) { +void uvHandleResp(SSvrMsg* msg, SWorkThrd* thrd) { // send msg to client tDebug("%s conn %p start to send resp (2/2)", transLabel(thrd->pTransInst), msg->pConn); uvStartSendResp(msg); } -void uvHandleRegister(SSvrMsg* msg, SWorkThrdObj* thrd) { +void uvHandleRegister(SSvrMsg* msg, SWorkThrd* thrd) { SSvrConn* conn = msg->pConn; tDebug("%s conn %p register brokenlink callback", transLabel(thrd->pTransInst), conn); if (conn->status == ConnAcquire) { @@ -982,7 +982,7 @@ void uvHandleRegister(SSvrMsg* msg, SWorkThrdObj* thrd) { taosMemoryFree(msg); } } -void destroyWorkThrd(SWorkThrdObj* pThrd) { +void destroyWorkThrd(SWorkThrd* pThrd) { if (pThrd == NULL) { return; } @@ -993,7 +993,7 @@ void destroyWorkThrd(SWorkThrdObj* pThrd) { taosMemoryFree(pThrd->loop); taosMemoryFree(pThrd); } -void sendQuitToWorkThrd(SWorkThrdObj* pThrd) { +void sendQuitToWorkThrd(SWorkThrd* pThrd) { SSvrMsg* msg = taosMemoryCalloc(1, sizeof(SSvrMsg)); msg->type = Quit; tDebug("server send quit msg to work thread"); @@ -1060,7 +1060,7 @@ void transReleaseSrvHandle(void* handle) { ASYNC_CHECK_HANDLE(exh, refId); - SWorkThrdObj* pThrd = exh->pThrd; + SWorkThrd* pThrd = exh->pThrd; ASYNC_ERR_JRET(pThrd); STransMsg tmsg = {.code = 0, .info.handle = exh, .info.ahandle = NULL, .info.refId = refId}; @@ -1090,7 +1090,7 @@ void transSendResponse(const STransMsg* msg) { STransMsg tmsg = *msg; tmsg.info.refId = refId; - SWorkThrdObj* pThrd = exh->pThrd; + SWorkThrd* pThrd = exh->pThrd; ASYNC_ERR_JRET(pThrd); SSvrMsg* m = taosMemoryCalloc(1, sizeof(SSvrMsg)); @@ -1120,7 +1120,7 @@ void transRegisterMsg(const STransMsg* msg) { STransMsg tmsg = *msg; tmsg.info.refId = refId; - SWorkThrdObj* pThrd = exh->pThrd; + SWorkThrd* pThrd = exh->pThrd; ASYNC_ERR_JRET(pThrd); SSvrMsg* m = taosMemoryCalloc(1, sizeof(SSvrMsg)); From ffd105d0e037c612c632be406a962b042bc58f14 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 23 Jun 2022 20:51:09 +0800 Subject: [PATCH 05/72] handle except --- source/libs/transport/src/transCli.c | 55 ++++++++++++++------------- source/libs/transport/src/transComm.c | 6 +-- 2 files changed, 32 insertions(+), 29 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 5b100e3d56..0bf6b2778b 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -393,7 +393,6 @@ void cliHandleExcept(SCliConn* pConn) { transMsg.code = TSDB_CODE_RPC_NETWORK_UNAVAIL; transMsg.msgType = pMsg ? pMsg->msg.msgType + 1 : 0; transMsg.info.ahandle = NULL; - transMsg.info.handle = pConn; if (pMsg == NULL && !CONN_NO_PERSIST_BY_APP(pConn)) { transMsg.info.ahandle = transCtxDumpVal(&pConn->ctx, transMsg.msgType); @@ -987,10 +986,14 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { if (pCtx->retryCount == 0) { pCtx->origEpSet = pCtx->epSet; } - /* * upper layer handle retry if code equal TSDB_CODE_RPC_NETWORK_UNAVAIL */ + /* + * no retry + * 1. query conn 2. rpc thread already receive quit msg + * + */ if (CONN_NO_PERSIST_BY_APP(pConn) && pThrd->quit == false) { tmsg_t msgType = pCtx->msgType; if ((pTransInst->retry != NULL && pEpSet->numOfEps > 1 && (pTransInst->retry(pResp->code))) || @@ -1014,31 +1017,31 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { transUnrefCliHandle(pConn); return -1; - } else if (pCtx->retryCount < TRANS_RETRY_COUNT_LIMIT) { - if (pResp->contLen == 0) { - pEpSet->inUse = (++pEpSet->inUse) % pEpSet->numOfEps; - transPrintEpSet(&pCtx->epSet); - tTrace("%s use local epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse, - pCtx->retryCount + 1, TRANS_RETRY_COUNT_LIMIT); - } else { - SEpSet epSet = {0}; - tDeserializeSEpSet(pResp->pCont, pResp->contLen, &epSet); - pCtx->epSet = epSet; - - transPrintEpSet(&pCtx->epSet); - tTrace("%s use remote epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse, - pCtx->retryCount + 1, TRANS_RETRY_COUNT_LIMIT); - } - if (pConn->status != ConnInPool) { - addConnToPool(pThrd->pool, pConn); - } - - STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); - arg->param1 = pMsg; - arg->param2 = pThrd; - transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL); - return -1; } + } else if (pCtx->retryCount < TRANS_RETRY_COUNT_LIMIT) { + if (pResp->contLen == 0) { + pEpSet->inUse = (++pEpSet->inUse) % pEpSet->numOfEps; + transPrintEpSet(&pCtx->epSet); + tTrace("%s use local epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse, + pCtx->retryCount + 1, TRANS_RETRY_COUNT_LIMIT); + } else { + SEpSet epSet = {0}; + tDeserializeSEpSet(pResp->pCont, pResp->contLen, &epSet); + pCtx->epSet = epSet; + + transPrintEpSet(&pCtx->epSet); + tTrace("%s use remote epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse, + pCtx->retryCount + 1, TRANS_RETRY_COUNT_LIMIT); + } + if (pConn->status != ConnInPool) { + addConnToPool(pThrd->pool, pConn); + } + + STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); + arg->param1 = pMsg; + arg->param2 = pThrd; + transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL); + return -1; } } } diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 5d342dd174..bff7d79bd3 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -455,16 +455,16 @@ void transPrintEpSet(SEpSet* pEpSet) { return; } char buf[512] = {0}; - int len = snprintf(buf, sizeof(buf), "epset:{ "); + int len = snprintf(buf, sizeof(buf), "epset:{"); for (int i = 0; i < pEpSet->numOfEps; i++) { if (i == pEpSet->numOfEps - 1) { - len += snprintf(buf + len, sizeof(buf) - len, "%d. %s:%d ", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); + len += snprintf(buf + len, sizeof(buf) - len, "%d. %s:%d", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); } else { len += snprintf(buf + len, sizeof(buf) - len, "%d. %s:%d, ", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); } } len += snprintf(buf + len, sizeof(buf) - len, "}"); - tTrace("%s, inUse: %d", buf, pEpSet->inUse); + tTrace("%s, inUse:%d", buf, pEpSet->inUse); } bool transEpSetIsEqual(SEpSet* a, SEpSet* b) { if (a->numOfEps != b->numOfEps || a->inUse != b->inUse) { From 1d169ce22af04f958b5b635fa715118dcfe0e0fe Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 23 Jun 2022 21:16:32 +0800 Subject: [PATCH 06/72] handle except --- source/libs/transport/src/transCli.c | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 0bf6b2778b..96fbe3f6b7 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -300,17 +300,9 @@ void cliHandleResp(SCliConn* conn) { if (CONN_NO_PERSIST_BY_APP(conn)) { pMsg = transQueuePop(&conn->cliMsgs); - pCtx = pMsg ? pMsg->ctx : NULL; - if (pMsg == NULL && !CONN_NO_PERSIST_BY_APP(conn)) { - transMsg.info.ahandle = transCtxDumpVal(&conn->ctx, transMsg.msgType); - if (transMsg.info.ahandle == NULL) { - transMsg.info.ahandle = transCtxDumpBrokenlinkVal(&conn->ctx, (int32_t*)&(transMsg.msgType)); - } - tDebug("%s conn %p construct ahandle %p, persist: 0", CONN_GET_INST_LABEL(conn), conn, transMsg.info.ahandle); - } else { - transMsg.info.ahandle = pCtx ? pCtx->ahandle : NULL; - tDebug("%s conn %p get ahandle %p, persist: 0", CONN_GET_INST_LABEL(conn), conn, transMsg.info.ahandle); - } + pCtx = pMsg->ctx; + transMsg.info.ahandle = pCtx->ahandle; + tDebug("%s conn %p get ahandle %p, persist: 0", CONN_GET_INST_LABEL(conn), conn, transMsg.info.ahandle); } else { uint64_t ahandle = (uint64_t)pHead->ahandle; CONN_GET_MSGCTX_BY_AHANDLE(conn, ahandle); From d63eb11b70b60aa379e0c0ed011fc9851fdae6c5 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 23 Jun 2022 21:35:35 +0800 Subject: [PATCH 07/72] handle except --- source/libs/transport/src/transCli.c | 17 ++++++++--------- source/libs/transport/src/transSvr.c | 2 -- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 96fbe3f6b7..9ccbf3eaa0 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -712,7 +712,14 @@ static void cliHandleQuit(SCliMsg* pMsg, SCliThrd* pThrd) { // uv_stop(pThrd->loop); } static void cliHandleRelease(SCliMsg* pMsg, SCliThrd* pThrd) { - SCliConn* conn = pMsg->msg.info.handle; + int64_t refId = (int64_t)(pMsg->msg.info.handle); + SExHandle* exh = transAcquireExHandle(refMgt, refId); + if (exh == NULL) { + tDebug("%" PRid64 " already release", refId); + return NULL; + } + + SCliConn* conn = exh->handle; tDebug("%s conn %p start to release to inst", CONN_GET_INST_LABEL(conn), conn); if (T_REF_VAL_GET(conn) == 2) { @@ -721,14 +728,10 @@ static void cliHandleRelease(SCliMsg* pMsg, SCliThrd* pThrd) { return; } cliSend(conn); - } else { - // conn already broken down - transUnrefCliHandle(conn); } } static void cliHandleUpdate(SCliMsg* pMsg, SCliThrd* pThrd) { STransConnCtx* pCtx = pMsg->ctx; - pThrd->cvtAddr = pCtx->cvtAddr; destroyCmsg(pMsg); } @@ -772,9 +775,7 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) { STrans* pTransInst = pThrd->pTransInst; cliMayCvtFqdnToIp(&pCtx->epSet, &pThrd->cvtAddr); - transPrintEpSet(&pCtx->epSet); - SCliConn* conn = cliGetConn(pMsg, pThrd); if (conn != NULL) { transCtxMerge(&conn->ctx, &pCtx->appCtx); @@ -1112,7 +1113,6 @@ void transReleaseCliHandle(void* handle) { if (pThrd == NULL) { return; } - STransMsg tmsg = {.info.handle = handle}; SCliMsg* cmsg = taosMemoryCalloc(1, sizeof(SCliMsg)); cmsg->msg = tmsg; @@ -1162,7 +1162,6 @@ void transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransM transFreeMsg(pReq->pCont); return; } - tsem_t* sem = taosMemoryCalloc(1, sizeof(tsem_t)); tsem_init(sem, 0, 0); diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 599d98a3e9..215323f69d 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -1029,8 +1029,6 @@ void transCloseServer(void* arg) { int ref = atomic_sub_fetch_32(&tranSSvrInst, 1); if (ref == 0) { - // TdThreadOnce tmpInit = PTHREAD_ONCE_INIT; - // memcpy(&transModuleInit, &tmpInit, sizeof(TdThreadOnce)); transCloseExHandleMgt(refMgt); } } From 78be460ac31669b0373b76cc246f8684eec3800e Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 23 Jun 2022 21:40:12 +0800 Subject: [PATCH 08/72] handle except --- source/libs/transport/src/transCli.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 9ccbf3eaa0..a95afa8a27 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -715,8 +715,7 @@ static void cliHandleRelease(SCliMsg* pMsg, SCliThrd* pThrd) { int64_t refId = (int64_t)(pMsg->msg.info.handle); SExHandle* exh = transAcquireExHandle(refMgt, refId); if (exh == NULL) { - tDebug("%" PRid64 " already release", refId); - return NULL; + tDebug("%" PRId64 " already release", refId); } SCliConn* conn = exh->handle; From 3795ff37ad4955c5f7d5078e2ce028f34996a772 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 24 Jun 2022 10:38:18 +0800 Subject: [PATCH 09/72] handle except --- source/libs/transport/src/transCli.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index a95afa8a27..ce6c85bb57 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -197,7 +197,6 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { if (status != ConnInPool) { \ addConnToPool(((SCliThrd*)conn->hostThrd)->pool, conn); \ } \ - transRemoveExHandle(refMgt, conn->refId); \ return; \ } \ } while (0) From 87f4f5364f9df0d93ca5d40bda4d649f4cede862 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 24 Jun 2022 11:09:34 +0800 Subject: [PATCH 10/72] handle except --- source/libs/transport/src/transCli.c | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index ce6c85bb57..cb78d7f0df 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -953,7 +953,6 @@ int cliRBChoseIdx(STrans* pTransInst) { } static void doDelayTask(void* param) { STaskArg* arg = param; - SCliMsg* pMsg = arg->param1; SCliThrd* pThrd = arg->param2; cliHandleReq(pMsg, pThrd); @@ -977,13 +976,11 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { if (pCtx->retryCount == 0) { pCtx->origEpSet = pCtx->epSet; } + /* - * upper layer handle retry if code equal TSDB_CODE_RPC_NETWORK_UNAVAIL - */ - /* - * no retry - * 1. query conn 2. rpc thread already receive quit msg - * + * no retry + * 1. query conn + * 2. rpc thread already receive quit msg */ if (CONN_NO_PERSIST_BY_APP(pConn) && pThrd->quit == false) { tmsg_t msgType = pCtx->msgType; From ab23faed636ee498dd413e2d3e230804263c9577 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 24 Jun 2022 15:52:13 +0800 Subject: [PATCH 11/72] fix: planner memory leak --- source/libs/nodes/src/nodesUtilFuncs.c | 27 ++++++++++++++++++---- source/libs/parser/src/parTranslater.c | 5 +--- source/libs/planner/src/planLogicCreater.c | 10 ++++---- source/libs/planner/src/planOptimizer.c | 3 ++- source/libs/planner/src/planPhysiCreater.c | 12 ++++++++++ source/libs/planner/src/planSpliter.c | 9 +++++--- 6 files changed, 48 insertions(+), 18 deletions(-) diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 1d8baf5373..8fced1f2c5 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -346,9 +346,11 @@ static void destroyVgDataBlockArray(SArray* pArray) { } static void destroyLogicNode(SLogicNode* pNode) { - nodesDestroyList(pNode->pChildren); - nodesDestroyNode(pNode->pConditions); nodesDestroyList(pNode->pTargets); + nodesDestroyNode(pNode->pConditions); + nodesDestroyList(pNode->pChildren); + nodesDestroyNode(pNode->pLimit); + nodesDestroyNode(pNode->pSlimit); } static void destroyPhysiNode(SPhysiNode* pNode) { @@ -368,6 +370,7 @@ static void destroyWinodwPhysiNode(SWinodwPhysiNode* pNode) { static void destroyScanPhysiNode(SScanPhysiNode* pNode) { destroyPhysiNode((SPhysiNode*)pNode); nodesDestroyList(pNode->pScanCols); + nodesDestroyList(pNode->pScanPseudoCols); } static void destroyDataSinkNode(SDataSinkNode* pNode) { nodesDestroyNode((SNode*)pNode->pInputDataBlockDesc); } @@ -516,6 +519,9 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyNode(pStmt->pWindow); nodesDestroyList(pStmt->pGroupByList); nodesDestroyNode(pStmt->pHaving); + nodesDestroyNode(pStmt->pRange); + nodesDestroyNode(pStmt->pEvery); + nodesDestroyNode(pStmt->pFill); nodesDestroyList(pStmt->pOrderByList); nodesDestroyNode((SNode*)pStmt->pLimit); nodesDestroyNode((SNode*)pStmt->pSlimit); @@ -779,6 +785,8 @@ void nodesDestroyNode(SNode* pNode) { SInterpFuncLogicNode* pLogicNode = (SInterpFuncLogicNode*)pNode; destroyLogicNode((SLogicNode*)pLogicNode); nodesDestroyList(pLogicNode->pFuncs); + nodesDestroyNode(pLogicNode->pFillValues); + nodesDestroyNode(pLogicNode->pTimeSeries); break; } case QUERY_NODE_LOGIC_SUBPLAN: { @@ -793,14 +801,21 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyList(((SQueryLogicPlan*)pNode)->pTopSubplans); break; case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: - case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: - case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN: - case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN: case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN: case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN: case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN: destroyScanPhysiNode((SScanPhysiNode*)pNode); break; + case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN: { + STableScanPhysiNode* pPhyNode = (STableScanPhysiNode*)pNode; + destroyScanPhysiNode((SScanPhysiNode*)pNode); + nodesDestroyList(pPhyNode->pDynamicScanFuncs); + nodesDestroyList(pPhyNode->pPartitionTags); + break; + } case QUERY_NODE_PHYSICAL_PLAN_PROJECT: { SProjectPhysiNode* pPhyNode = (SProjectPhysiNode*)pNode; destroyPhysiNode((SPhysiNode*)pPhyNode); @@ -891,6 +906,8 @@ void nodesDestroyNode(SNode* pNode) { destroyPhysiNode((SPhysiNode*)pPhyNode); nodesDestroyList(pPhyNode->pExprs); nodesDestroyList(pPhyNode->pFuncs); + nodesDestroyNode(pPhyNode->pFillValues); + nodesDestroyNode(pPhyNode->pTimeSeries); break; } case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 424968ac80..5d464fdf27 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1840,10 +1840,7 @@ static int32_t createMultiResFuncsFromStar(STranslateContext* pCxt, SFunctionNod code = createMultiResFuncs(pSrcFunc, pExprs, pOutput); } - if (TSDB_CODE_SUCCESS != code) { - nodesDestroyList(pExprs); - } - + nodesDestroyList(pExprs); return code; } diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index ffb97bcc04..d4f18a663c 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -780,8 +780,8 @@ static int32_t createProjectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSel return TSDB_CODE_OUT_OF_MEMORY; } - pProject->node.pLimit = (SNode*)pSelect->pLimit; - pProject->node.pSlimit = (SNode*)pSelect->pSlimit; + TSWAP(pProject->node.pLimit, pSelect->pLimit); + TSWAP(pProject->node.pSlimit, pSelect->pSlimit); int32_t code = TSDB_CODE_SUCCESS; @@ -940,7 +940,7 @@ static int32_t createSetOpSortLogicNode(SLogicPlanContext* pCxt, SSetOperator* p return TSDB_CODE_OUT_OF_MEMORY; } - pSort->node.pLimit = pSetOperator->pLimit; + TSWAP(pSort->node.pLimit, pSetOperator->pLimit); int32_t code = TSDB_CODE_SUCCESS; @@ -973,7 +973,7 @@ static int32_t createSetOpProjectLogicNode(SLogicPlanContext* pCxt, SSetOperator } if (NULL == pSetOperator->pOrderByList) { - pProject->node.pLimit = pSetOperator->pLimit; + TSWAP(pProject->node.pLimit, pSetOperator->pLimit); } int32_t code = TSDB_CODE_SUCCESS; @@ -1004,7 +1004,7 @@ static int32_t createSetOpAggLogicNode(SLogicPlanContext* pCxt, SSetOperator* pS } if (NULL == pSetOperator->pOrderByList) { - pAgg->node.pLimit = pSetOperator->pLimit; + TSWAP(pAgg->node.pSlimit, pSetOperator->pLimit); } int32_t code = TSDB_CODE_SUCCESS; diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 7862ad5445..d4964d3cd4 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -682,7 +682,7 @@ static EOrder opkGetPrimaryKeyOrder(SSortLogicNode* pSort) { static SNode* opkRewriteDownNode(SSortLogicNode* pSort) { SNode* pDownNode = nodesListGetNode(pSort->node.pChildren, 0); // todo - pSort->node.pChildren = NULL; + NODES_CLEAR_LIST(pSort->node.pChildren); return pDownNode; } @@ -1061,6 +1061,7 @@ static EDealRes partTagsOptRebuildTbanmeImpl(SNode** pNode, void* pContext) { } strcpy(pFunc->functionName, "tbname"); pFunc->funcType = FUNCTION_TYPE_TBNAME; + pFunc->node.resType = ((SColumnNode*)*pNode)->node.resType; nodesDestroyNode(*pNode); *pNode = (SNode*)pFunc; return DEAL_RES_IGNORE_CHILD; diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index efb0f790e1..be9592507e 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -862,6 +862,9 @@ static int32_t createIndefRowsFuncPhysiNode(SPhysiPlanContext* pCxt, SNodeList* nodesDestroyNode((SNode*)pIdfRowsFunc); } + nodesDestroyList(pPrecalcExprs); + nodesDestroyList(pFuncs); + return code; } @@ -913,6 +916,9 @@ static int32_t createInterpFuncPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pCh nodesDestroyNode((SNode*)pInterpFunc); } + nodesDestroyList(pPrecalcExprs); + nodesDestroyList(pFuncs); + return code; } @@ -1048,6 +1054,9 @@ static int32_t createWindowPhysiNodeFinalize(SPhysiPlanContext* pCxt, SNodeList* nodesDestroyNode((SNode*)pWindow); } + nodesDestroyList(pPrecalcExprs); + nodesDestroyList(pFuncs); + return code; } @@ -1241,6 +1250,9 @@ static int32_t createPartitionPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChi nodesDestroyNode((SNode*)pPart); } + nodesDestroyList(pPrecalcExprs); + nodesDestroyList(pPartitionKeys); + return code; } diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 0e35239621..8e8559ba3f 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -179,7 +179,7 @@ static bool stbSplNeedSplitWindow(bool streamQuery, SLogicNode* pNode) { return !stbSplHasGatherExecFunc(pWindow->pFuncs) && stbSplHasMultiTbScan(streamQuery, pNode); } } - + if (WINDOW_TYPE_STATE == pWindow->winType) { if (!streamQuery) { return stbSplHasMultiTbScan(streamQuery, pNode); @@ -374,7 +374,7 @@ static int32_t stbSplCreateMergeNode(SSplitContext* pCxt, SLogicSubplan* pSubpla int32_t code = TSDB_CODE_SUCCESS; pMerge->pInputs = nodesCloneList(pPartChild->pTargets); - // NULL == pSubplan means 'merge node' replaces 'split node'. + // NULL != pSubplan means 'merge node' replaces 'split node'. if (NULL == pSubplan) { pMerge->node.pTargets = nodesCloneList(pPartChild->pTargets); } else { @@ -390,6 +390,9 @@ static int32_t stbSplCreateMergeNode(SSplitContext* pCxt, SLogicSubplan* pSubpla code = replaceLogicNode(pSubplan, pSplitNode, (SLogicNode*)pMerge); } } + if (TSDB_CODE_SUCCESS == code && NULL != pSubplan) { + nodesDestroyNode((SNode*)pSplitNode); + } if (TSDB_CODE_SUCCESS != code) { nodesDestroyNode((SNode*)pMerge); } @@ -512,7 +515,7 @@ static int32_t stbSplSplitSessionOrStateForBatch(SSplitContext* pCxt, SStableSpl SLogicNode* pChild = (SLogicNode*)nodesListGetNode(pWindow->pChildren, 0); SNodeList* pMergeKeys = NULL; - int32_t code = stbSplCreateMergeKeysByPrimaryKey(((SWindowLogicNode*)pWindow)->pTspk, &pMergeKeys); + int32_t code = stbSplCreateMergeKeysByPrimaryKey(((SWindowLogicNode*)pWindow)->pTspk, &pMergeKeys); if (TSDB_CODE_SUCCESS == code) { code = stbSplCreateMergeNode(pCxt, pInfo->pSubplan, pChild, pMergeKeys, (SLogicNode*)pChild); From 445d7f2d90bf61b5efdd0e29ffc936ed016f9caa Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 24 Jun 2022 16:21:01 +0800 Subject: [PATCH 12/72] feat: refactor rpc quit --- source/client/src/clientEnv.c | 3 +- source/libs/transport/src/transCli.c | 139 +++++++++++++++------------ 2 files changed, 82 insertions(+), 60 deletions(-) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index f1e4107e23..ff9003b8fc 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -85,7 +85,8 @@ void closeTransporter(STscObj *pTscObj) { } static bool clientRpcRfp(int32_t code) { - if (code == TSDB_CODE_RPC_REDIRECT) { + if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED || + code == TSDB_CODE_SYN_NOT_LEADER) { return true; } else { return false; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index cb78d7f0df..1cacc84d79 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -111,6 +111,13 @@ static void cliDestroyConn(SCliConn* pConn, bool clear /*clear tcp handle o static void cliDestroy(uv_handle_t* handle); static void cliSend(SCliConn* pConn); +static bool cliIsEpsetUpdated(int32_t code, STransConnCtx* pCtx) { + if (code != 0) return false; + if (pCtx->retryCnt == 0) return false; + if (transEpSetIsEqual(&pCtx->epSet, &pCtx->origEpSet)) return false; + return true; +} + void cliMayCvtFqdnToIp(SEpSet* pEpSet, SCvtAddr* pCvtAddr); /* * set TCP connection timeout per-socket level @@ -154,7 +161,6 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { destroyCmsg(pMsg); } } - #define CLI_RELEASE_UV(loop) \ do { \ uv_walk(loop, cliWalkCb, NULL); \ @@ -183,7 +189,6 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { #define CONN_SHOULD_RELEASE(conn, head) \ do { \ if ((head)->release == 1 && (head->msgLen) == sizeof(*head)) { \ - int status = conn->status; \ uint64_t ahandle = head->ahandle; \ CONN_GET_MSGCTX_BY_AHANDLE(conn, ahandle); \ transClearBuffer(&conn->readBuf); \ @@ -194,9 +199,7 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { } \ destroyCmsg(pMsg); \ cliReleaseUnfinishedMsg(conn); \ - if (status != ConnInPool) { \ - addConnToPool(((SCliThrd*)conn->hostThrd)->pool, conn); \ - } \ + addConnToPool(((SCliThrd*)conn->hostThrd)->pool, conn); \ return; \ } \ } while (0) @@ -262,8 +265,25 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { #define REQUEST_PERSIS_HANDLE(msg) ((msg)->info.persistHandle == 1) #define REQUEST_RELEASE_HANDLE(cmsg) ((cmsg)->type == Release) +#define EPSET_GET_SIZE(epSet) (epSet)->numOfEps #define EPSET_GET_INUSE_IP(epSet) ((epSet)->eps[(epSet)->inUse].fqdn) #define EPSET_GET_INUSE_PORT(epSet) ((epSet)->eps[(epSet)->inUse].port) +#define EPSET_FORWARD_INUSE(epSet) \ + do { \ + (epSet)->inUse = (++((epSet)->inUse)) % ((epSet)->numOfEps); \ + } while (0) +#define EPSET_DEBUG_STR(epSet, buf) \ + do { \ + int len = snprintf(buf, sizeof(buf), "epset:{"); \ + for (int i = 0; i < (epSet)->numOfEps; i++) { \ + if (i == (epSet)->numOfEps - 1) { \ + len += snprintf(buf + len, sizeof(buf) - len, "%d. %s:%d", i, (epSet)->eps[i].fqdn, (epSet)->eps[i].port); \ + } else { \ + len += snprintf(buf + len, sizeof(buf) - len, "%d. %s:%d, ", i, (epSet)->eps[i].fqdn, (epSet)->eps[i].port); \ + } \ + } \ + len += snprintf(buf + len, sizeof(buf) - len, "}"); \ + } while (0); static void* cliWorkThread(void* arg); @@ -492,6 +512,10 @@ static void allocConnRef(SCliConn* conn, bool update) { conn->refId = exh->refId; } static void addConnToPool(void* pool, SCliConn* conn) { + if (conn->status == ConnInPool) { + assert(0); + return; + } SCliThrd* thrd = conn->hostThrd; CONN_HANDLE_THREAD_QUIT(thrd); @@ -505,7 +529,7 @@ static void addConnToPool(void* pool, SCliConn* conn) { char key[128] = {0}; CONN_CONSTRUCT_HASH_KEY(key, conn->ip, conn->port); - tTrace("%s conn %p added to conn pool, read buf cap: %d", CONN_GET_INST_LABEL(conn), conn, conn->readBuf.cap); + tTrace("%s conn %p added to conn pool, read buf cap:%d", CONN_GET_INST_LABEL(conn), conn, conn->readBuf.cap); SConnList* plist = taosHashGet((SHashObj*)pool, key, strlen(key)); // list already create before @@ -751,9 +775,9 @@ SCliConn* cliGetConn(SCliMsg* pMsg, SCliThrd* pThrd) { STransConnCtx* pCtx = pMsg->ctx; conn = getConnFromPool(pThrd->pool, EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet)); if (conn != NULL) { - tTrace("%s conn %p get from conn pool", CONN_GET_INST_LABEL(conn), conn); + tTrace("%s conn %p get from conn pool:%p", CONN_GET_INST_LABEL(conn), conn, pThrd->pool); } else { - tTrace("%s not found conn in conn pool %p", ((STrans*)pThrd->pTransInst)->label, pThrd->pool); + tTrace("%s not found conn in conn pool:%p", ((STrans*)pThrd->pTransInst)->label, pThrd->pool); } return conn; } @@ -773,7 +797,8 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) { STrans* pTransInst = pThrd->pTransInst; cliMayCvtFqdnToIp(&pCtx->epSet, &pThrd->cvtAddr); - transPrintEpSet(&pCtx->epSet); + + // transPrintEpSet(&pCtx->epSet); SCliConn* conn = cliGetConn(pMsg, pThrd); if (conn != NULL) { transCtxMerge(&conn->ctx, &pCtx->appCtx); @@ -955,11 +980,30 @@ static void doDelayTask(void* param) { STaskArg* arg = param; SCliMsg* pMsg = arg->param1; SCliThrd* pThrd = arg->param2; - cliHandleReq(pMsg, pThrd); - taosMemoryFree(arg); + + cliHandleReq(pMsg, pThrd); } +static void cliSchedMsgToNextNode(SCliMsg* pMsg, SCliThrd* pThrd) { + STraceId* trace = &pMsg->msg.info.traceId; + STransConnCtx* pCtx = pMsg->ctx; + + char buf[256] = {0}; + EPSET_DEBUG_STR(&pCtx->epSet, buf); + tGTrace("%s %s, retryCnt:%d, limit:%d", transLabel(pThrd), buf, pCtx->retryCnt + 1, pCtx->retryLimit); + + STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); + arg->param1 = pMsg; + arg->param2 = pThrd; + transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL); +} + +void cliUpdateRetryLimit(int8_t* val, int8_t exp, int8_t newVal) { + if (*val != exp) { + *val = newVal; + } +} int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { SCliThrd* pThrd = pConn->hostThrd; STrans* pTransInst = pThrd->pTransInst; @@ -971,68 +1015,45 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { } STransConnCtx* pCtx = pMsg->ctx; - SEpSet* pEpSet = &pCtx->epSet; - - if (pCtx->retryCount == 0) { + if (pCtx->retryCnt == 0) { pCtx->origEpSet = pCtx->epSet; } - /* * no retry * 1. query conn * 2. rpc thread already receive quit msg */ - if (CONN_NO_PERSIST_BY_APP(pConn) && pThrd->quit == false) { - tmsg_t msgType = pCtx->msgType; - if ((pTransInst->retry != NULL && pEpSet->numOfEps > 1 && (pTransInst->retry(pResp->code))) || - (pResp->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || pResp->code == TSDB_CODE_APP_NOT_READY || - pResp->code == TSDB_CODE_NODE_NOT_DEPLOYED || pResp->code == TSDB_CODE_SYN_NOT_LEADER)) { + int32_t code = pResp->code; + if (CONN_NO_PERSIST_BY_APP(pConn)) { + if (pTransInst->retry != NULL && pTransInst->retry(code)) { pMsg->sent = 0; - tTrace("try to send req to next node"); - pMsg->st = taosGetTimestampUs(); + pCtx->retryCnt += 1; + if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { + transUnrefCliHandle(pConn); - pCtx->retryCount += 1; - if (pResp->code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { - if (pCtx->retryCount < pEpSet->numOfEps * 3) { - pEpSet->inUse = (++pEpSet->inUse) % pEpSet->numOfEps; - STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); - arg->param1 = pMsg; - arg->param2 = pThrd; - transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL); - transPrintEpSet(pEpSet); - tTrace("%s use local epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse, - pCtx->retryCount + 1, pEpSet->numOfEps * 3); - - transUnrefCliHandle(pConn); + cliUpdateRetryLimit(&pCtx->retryLimit, TRANS_RETRY_COUNT_LIMIT, EPSET_GET_SIZE(&pCtx->epSet) * 3); + if (pCtx->retryCnt < pCtx->retryLimit) { + EPSET_FORWARD_INUSE(&pCtx->epSet); + cliSchedMsgToNextNode(pMsg, pThrd); return -1; } - } else if (pCtx->retryCount < TRANS_RETRY_COUNT_LIMIT) { - if (pResp->contLen == 0) { - pEpSet->inUse = (++pEpSet->inUse) % pEpSet->numOfEps; - transPrintEpSet(&pCtx->epSet); - tTrace("%s use local epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse, - pCtx->retryCount + 1, TRANS_RETRY_COUNT_LIMIT); - } else { - SEpSet epSet = {0}; - tDeserializeSEpSet(pResp->pCont, pResp->contLen, &epSet); - pCtx->epSet = epSet; + } else { + addConnToPool(pThrd->pool, pConn); - transPrintEpSet(&pCtx->epSet); - tTrace("%s use remote epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse, - pCtx->retryCount + 1, TRANS_RETRY_COUNT_LIMIT); + cliUpdateRetryLimit(&pCtx->retryLimit, TRANS_RETRY_COUNT_LIMIT, TRANS_RETRY_COUNT_LIMIT); + if (pCtx->retryCnt < pCtx->retryLimit) { + if (pResp->contLen == 0) { + EPSET_FORWARD_INUSE(&pCtx->epSet); + } else { + tDeserializeSEpSet(pResp->pCont, pResp->contLen, &pCtx->epSet); + } + cliSchedMsgToNextNode(pMsg, pThrd); + return -1; } - if (pConn->status != ConnInPool) { - addConnToPool(pThrd->pool, pConn); - } - - STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); - arg->param1 = pMsg; - arg->param2 = pThrd; - transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL); - return -1; } } } + STraceId* trace = &pResp->info.traceId; if (pCtx->pSem != NULL) { tGTrace("%s conn %p(sync) handle resp", CONN_GET_INST_LABEL(pConn), pConn); @@ -1045,10 +1066,10 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { pCtx->pRsp = NULL; } else { tGTrace("%s conn %p handle resp", CONN_GET_INST_LABEL(pConn), pConn); - if (pResp->code != 0 || pCtx->retryCount == 0 || transEpSetIsEqual(&pCtx->epSet, &pCtx->origEpSet)) { + if (!cliIsEpsetUpdated(code, pCtx)) { pTransInst->cfp(pTransInst->parent, pResp, NULL); } else { - pTransInst->cfp(pTransInst->parent, pResp, pEpSet); + pTransInst->cfp(pTransInst->parent, pResp, &pCtx->epSet); } } return 0; From 4f629ec68bf46c38c6690a6ca680af8defb30b07 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 24 Jun 2022 16:53:56 +0800 Subject: [PATCH 13/72] enh(query): enhance cast function to support more types TD-15473 --- source/libs/function/src/builtins.c | 21 +++--- source/libs/scalar/src/sclfunc.c | 108 ++++++++++++++++++++++++++++ 2 files changed, 119 insertions(+), 10 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index f985cc324d..60107dee32 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1219,18 +1219,19 @@ static int32_t translateSubstr(SFunctionNode* pFunc, char* pErrBuf, int32_t len) static int32_t translateCast(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { // The number of parameters has been limited by the syntax definition - uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; + //uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; + // The function return type has been set during syntax parsing uint8_t para2Type = pFunc->node.resType.type; - if (para2Type != TSDB_DATA_TYPE_BIGINT && para2Type != TSDB_DATA_TYPE_UBIGINT && - para2Type != TSDB_DATA_TYPE_VARCHAR && para2Type != TSDB_DATA_TYPE_NCHAR && - para2Type != TSDB_DATA_TYPE_TIMESTAMP) { - return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); - } - if ((para2Type == TSDB_DATA_TYPE_TIMESTAMP && IS_VAR_DATA_TYPE(para1Type)) || - (para2Type == TSDB_DATA_TYPE_BINARY && para1Type == TSDB_DATA_TYPE_NCHAR)) { - return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); - } + //if (para2Type != TSDB_DATA_TYPE_BIGINT && para2Type != TSDB_DATA_TYPE_UBIGINT && + // para2Type != TSDB_DATA_TYPE_VARCHAR && para2Type != TSDB_DATA_TYPE_NCHAR && + // para2Type != TSDB_DATA_TYPE_TIMESTAMP) { + // return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + //} + //if ((para2Type == TSDB_DATA_TYPE_TIMESTAMP && IS_VAR_DATA_TYPE(para1Type)) || + // (para2Type == TSDB_DATA_TYPE_BINARY && para1Type == TSDB_DATA_TYPE_NCHAR)) { + // return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + //} int32_t para2Bytes = pFunc->node.resType.bytes; if (IS_VAR_DATA_TYPE(para2Type)) { diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 7ea0457592..721e38980e 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -730,6 +730,60 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp char *input = colDataGetData(pInput[0].columnData, i); switch(outputType) { + case TSDB_DATA_TYPE_TINYINT: { + if (inputType == TSDB_DATA_TYPE_BINARY) { + *(int8_t *)output = taosStr2Int8(varDataVal(input), NULL, 10); + } else if (inputType == TSDB_DATA_TYPE_NCHAR) { + char *newBuf = taosMemoryCalloc(1, inputLen); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + if (len < 0) { + taosMemoryFree(newBuf); + return TSDB_CODE_FAILED; + } + newBuf[len] = 0; + *(int8_t *)output = taosStr2Int8(newBuf, NULL, 10); + taosMemoryFree(newBuf); + } else { + GET_TYPED_DATA(*(int8_t *)output, int8_t, inputType, input); + } + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + if (inputType == TSDB_DATA_TYPE_BINARY) { + *(int16_t *)output = taosStr2Int16(varDataVal(input), NULL, 10); + } else if (inputType == TSDB_DATA_TYPE_NCHAR) { + char *newBuf = taosMemoryCalloc(1, inputLen); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + if (len < 0) { + taosMemoryFree(newBuf); + return TSDB_CODE_FAILED; + } + newBuf[len] = 0; + *(int16_t *)output = taosStr2Int16(newBuf, NULL, 10); + taosMemoryFree(newBuf); + } else { + GET_TYPED_DATA(*(int16_t *)output, int16_t, inputType, input); + } + break; + } + case TSDB_DATA_TYPE_INT: { + if (inputType == TSDB_DATA_TYPE_BINARY) { + *(int32_t *)output = taosStr2Int32(varDataVal(input), NULL, 10); + } else if (inputType == TSDB_DATA_TYPE_NCHAR) { + char *newBuf = taosMemoryCalloc(1, inputLen); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + if (len < 0) { + taosMemoryFree(newBuf); + return TSDB_CODE_FAILED; + } + newBuf[len] = 0; + *(int32_t *)output = taosStr2Int32(newBuf, NULL, 10); + taosMemoryFree(newBuf); + } else { + GET_TYPED_DATA(*(int32_t *)output, int32_t, inputType, input); + } + break; + } case TSDB_DATA_TYPE_BIGINT: { if (inputType == TSDB_DATA_TYPE_BINARY) { *(int64_t *)output = taosStr2Int64(varDataVal(input), NULL, 10); @@ -748,6 +802,60 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp } break; } + case TSDB_DATA_TYPE_UTINYINT: { + if (inputType == TSDB_DATA_TYPE_BINARY) { + *(uint8_t *)output = taosStr2UInt8(varDataVal(input), NULL, 10); + } else if (inputType == TSDB_DATA_TYPE_NCHAR) { + char *newBuf = taosMemoryCalloc(1, inputLen); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + if (len < 0) { + taosMemoryFree(newBuf); + return TSDB_CODE_FAILED; + } + newBuf[len] = 0; + *(uint8_t *)output = taosStr2UInt8(newBuf, NULL, 10); + taosMemoryFree(newBuf); + } else { + GET_TYPED_DATA(*(uint8_t *)output, uint8_t, inputType, input); + } + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + if (inputType == TSDB_DATA_TYPE_BINARY) { + *(uint16_t *)output = taosStr2UInt16(varDataVal(input), NULL, 10); + } else if (inputType == TSDB_DATA_TYPE_NCHAR) { + char *newBuf = taosMemoryCalloc(1, inputLen); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + if (len < 0) { + taosMemoryFree(newBuf); + return TSDB_CODE_FAILED; + } + newBuf[len] = 0; + *(uint16_t *)output = taosStr2UInt16(newBuf, NULL, 10); + taosMemoryFree(newBuf); + } else { + GET_TYPED_DATA(*(uint16_t *)output, uint16_t, inputType, input); + } + break; + } + case TSDB_DATA_TYPE_UINT: { + if (inputType == TSDB_DATA_TYPE_BINARY) { + *(uint32_t *)output = taosStr2UInt32(varDataVal(input), NULL, 10); + } else if (inputType == TSDB_DATA_TYPE_NCHAR) { + char *newBuf = taosMemoryCalloc(1, inputLen); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + if (len < 0) { + taosMemoryFree(newBuf); + return TSDB_CODE_FAILED; + } + newBuf[len] = 0; + *(uint32_t *)output = taosStr2UInt32(newBuf, NULL, 10); + taosMemoryFree(newBuf); + } else { + GET_TYPED_DATA(*(uint32_t *)output, uint32_t, inputType, input); + } + break; + } case TSDB_DATA_TYPE_UBIGINT: { if (inputType == TSDB_DATA_TYPE_BINARY) { *(uint64_t *)output = taosStr2UInt64(varDataVal(input), NULL, 10); From a215844467f8b5da581984aa01d04cda4c37ffc9 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 24 Jun 2022 17:02:53 +0800 Subject: [PATCH 14/72] handle except --- source/libs/transport/inc/transComm.h | 17 +++++++++-------- source/libs/transport/src/transCli.c | 2 +- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 5367f6b49d..db8de1c8c3 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -120,14 +120,15 @@ typedef struct SCvtAddr { } SCvtAddr; typedef struct { - SEpSet epSet; // ip list provided by app - SEpSet origEpSet; - void* ahandle; // handle provided by app - tmsg_t msgType; // message type - int8_t connType; // connection type cli/srv - int64_t rid; // refId returned by taosAddRef + SEpSet epSet; // ip list provided by app + SEpSet origEpSet; + void* ahandle; // handle provided by app + tmsg_t msgType; // message type + int8_t connType; // connection type cli/srv - int8_t retryCount; + int8_t retryCnt; + int8_t retryLimit; + // bool setMaxRetry; STransCtx appCtx; // STransMsg* pRsp; // for synchronous API tsem_t* pSem; // for synchronous API @@ -381,7 +382,7 @@ void transDQDestroy(SDelayQueue* queue); int transDQSched(SDelayQueue* queue, void (*func)(void* arg), void* arg, uint64_t timeoutMs); -void transPrintEpSet(SEpSet* pEpSet); +// void transPrintEpSet(SEpSet* pEpSet); bool transEpSetIsEqual(SEpSet* a, SEpSet* b); /* * init global func diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 1cacc84d79..b9eaae6cba 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1039,7 +1039,6 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { } } else { addConnToPool(pThrd->pool, pConn); - cliUpdateRetryLimit(&pCtx->retryLimit, TRANS_RETRY_COUNT_LIMIT, TRANS_RETRY_COUNT_LIMIT); if (pCtx->retryCnt < pCtx->retryLimit) { if (pResp->contLen == 0) { @@ -1047,6 +1046,7 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { } else { tDeserializeSEpSet(pResp->pCont, pResp->contLen, &pCtx->epSet); } + transFreeMsg(pResp->pCont); cliSchedMsgToNextNode(pMsg, pThrd); return -1; } From e54044de732badcbb4eda63731bbe07e7d3bce45 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 24 Jun 2022 19:24:14 +0800 Subject: [PATCH 15/72] update retry --- source/libs/transport/src/transCli.c | 34 ++++++++++++++-------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index b9eaae6cba..653666ee70 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -272,17 +272,17 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { do { \ (epSet)->inUse = (++((epSet)->inUse)) % ((epSet)->numOfEps); \ } while (0) -#define EPSET_DEBUG_STR(epSet, buf) \ - do { \ - int len = snprintf(buf, sizeof(buf), "epset:{"); \ - for (int i = 0; i < (epSet)->numOfEps; i++) { \ - if (i == (epSet)->numOfEps - 1) { \ - len += snprintf(buf + len, sizeof(buf) - len, "%d. %s:%d", i, (epSet)->eps[i].fqdn, (epSet)->eps[i].port); \ - } else { \ - len += snprintf(buf + len, sizeof(buf) - len, "%d. %s:%d, ", i, (epSet)->eps[i].fqdn, (epSet)->eps[i].port); \ - } \ - } \ - len += snprintf(buf + len, sizeof(buf) - len, "}"); \ +#define EPSET_DEBUG_STR(epSet, tbuf) \ + do { \ + int len = snprintf(tbuf, sizeof(tbuf), "epset:{"); \ + for (int i = 0; i < (epSet)->numOfEps; i++) { \ + if (i == (epSet)->numOfEps - 1) { \ + len += snprintf(tbuf + len, sizeof(tbuf) - len, "%d. %s:%d", i, (epSet)->eps[i].fqdn, (epSet)->eps[i].port); \ + } else { \ + len += snprintf(tbuf + len, sizeof(tbuf) - len, "%d. %s:%d, ", i, (epSet)->eps[i].fqdn, (epSet)->eps[i].port); \ + } \ + } \ + len += snprintf(tbuf + len, sizeof(tbuf) - len, "}, inUse:%d", (epSet)->inUse); \ } while (0); static void* cliWorkThread(void* arg); @@ -989,9 +989,10 @@ static void cliSchedMsgToNextNode(SCliMsg* pMsg, SCliThrd* pThrd) { STraceId* trace = &pMsg->msg.info.traceId; STransConnCtx* pCtx = pMsg->ctx; - char buf[256] = {0}; - EPSET_DEBUG_STR(&pCtx->epSet, buf); - tGTrace("%s %s, retryCnt:%d, limit:%d", transLabel(pThrd), buf, pCtx->retryCnt + 1, pCtx->retryLimit); + char tbuf[256] = {0}; + EPSET_DEBUG_STR(&pCtx->epSet, tbuf); + tGTrace("%s retry to send msg to next node %dms later , use %s, retryCnt:%d, limit:%d", transLabel(pThrd->pTransInst), + TRANS_RETRY_INTERVAL, tbuf, pCtx->retryCnt + 1, pCtx->retryLimit); STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); arg->param1 = pMsg; @@ -1029,18 +1030,17 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { pMsg->sent = 0; pCtx->retryCnt += 1; if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { - transUnrefCliHandle(pConn); - cliUpdateRetryLimit(&pCtx->retryLimit, TRANS_RETRY_COUNT_LIMIT, EPSET_GET_SIZE(&pCtx->epSet) * 3); if (pCtx->retryCnt < pCtx->retryLimit) { + transUnrefCliHandle(pConn); EPSET_FORWARD_INUSE(&pCtx->epSet); cliSchedMsgToNextNode(pMsg, pThrd); return -1; } } else { - addConnToPool(pThrd->pool, pConn); cliUpdateRetryLimit(&pCtx->retryLimit, TRANS_RETRY_COUNT_LIMIT, TRANS_RETRY_COUNT_LIMIT); if (pCtx->retryCnt < pCtx->retryLimit) { + addConnToPool(pThrd->pool, pConn); if (pResp->contLen == 0) { EPSET_FORWARD_INUSE(&pCtx->epSet); } else { From 720645800c43ec1ef7385ae4f612aac87a9d0af1 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 24 Jun 2022 20:51:00 +0800 Subject: [PATCH 16/72] feat: refactor rpc quit --- source/libs/transport/src/transCli.c | 53 +++++++++++++--------------- 1 file changed, 24 insertions(+), 29 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 653666ee70..9ca5e8e73e 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1014,42 +1014,36 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { pTransInst->cfp(pTransInst->parent, pResp, NULL); return 0; } - - STransConnCtx* pCtx = pMsg->ctx; - if (pCtx->retryCnt == 0) { - pCtx->origEpSet = pCtx->epSet; - } /* * no retry * 1. query conn * 2. rpc thread already receive quit msg */ - int32_t code = pResp->code; - if (CONN_NO_PERSIST_BY_APP(pConn)) { - if (pTransInst->retry != NULL && pTransInst->retry(code)) { - pMsg->sent = 0; - pCtx->retryCnt += 1; - if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { - cliUpdateRetryLimit(&pCtx->retryLimit, TRANS_RETRY_COUNT_LIMIT, EPSET_GET_SIZE(&pCtx->epSet) * 3); - if (pCtx->retryCnt < pCtx->retryLimit) { - transUnrefCliHandle(pConn); + STransConnCtx* pCtx = pMsg->ctx; + int32_t code = pResp->code; + if (pTransInst->retry != NULL && pTransInst->retry(code)) { + pMsg->sent = 0; + pCtx->retryCnt += 1; + if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { + cliUpdateRetryLimit(&pCtx->retryLimit, TRANS_RETRY_COUNT_LIMIT, EPSET_GET_SIZE(&pCtx->epSet) * 3); + if (pCtx->retryCnt < pCtx->retryLimit) { + transUnrefCliHandle(pConn); + EPSET_FORWARD_INUSE(&pCtx->epSet); + cliSchedMsgToNextNode(pMsg, pThrd); + return -1; + } + } else { + cliUpdateRetryLimit(&pCtx->retryLimit, TRANS_RETRY_COUNT_LIMIT, TRANS_RETRY_COUNT_LIMIT); + if (pCtx->retryCnt < pCtx->retryLimit) { + addConnToPool(pThrd->pool, pConn); + if (pResp->contLen == 0) { EPSET_FORWARD_INUSE(&pCtx->epSet); - cliSchedMsgToNextNode(pMsg, pThrd); - return -1; - } - } else { - cliUpdateRetryLimit(&pCtx->retryLimit, TRANS_RETRY_COUNT_LIMIT, TRANS_RETRY_COUNT_LIMIT); - if (pCtx->retryCnt < pCtx->retryLimit) { - addConnToPool(pThrd->pool, pConn); - if (pResp->contLen == 0) { - EPSET_FORWARD_INUSE(&pCtx->epSet); - } else { - tDeserializeSEpSet(pResp->pCont, pResp->contLen, &pCtx->epSet); - } - transFreeMsg(pResp->pCont); - cliSchedMsgToNextNode(pMsg, pThrd); - return -1; + } else { + tDeserializeSEpSet(pResp->pCont, pResp->contLen, &pCtx->epSet); } + transFreeMsg(pResp->pCont); + cliSchedMsgToNextNode(pMsg, pThrd); + return -1; } } } @@ -1185,6 +1179,7 @@ void transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransM STransConnCtx* pCtx = taosMemoryCalloc(1, sizeof(STransConnCtx)); pCtx->epSet = *pEpSet; + pCtx->origEpSet = *pEpSet; pCtx->ahandle = pReq->info.ahandle; pCtx->msgType = pReq->msgType; pCtx->pSem = sem; From 853f1df3abcfd071a6d584f36aea036edbd976a3 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 24 Jun 2022 21:07:12 +0800 Subject: [PATCH 17/72] feat: support partition by expression and aggregate function output together --- include/libs/nodes/plannodes.h | 1 + source/libs/function/src/builtins.c | 18 ++++++-- source/libs/parser/src/parTranslater.c | 44 ++++++++++++++----- source/libs/parser/test/parSelectTest.cpp | 23 ++++++++++ source/libs/planner/src/planLogicCreater.c | 8 ++-- source/libs/planner/src/planOptimizer.c | 15 ++++++- source/libs/planner/src/planSpliter.c | 20 ++++----- source/libs/planner/test/planOptimizeTest.cpp | 10 ++++- source/libs/planner/test/planPartByTest.cpp | 6 +++ 9 files changed, 115 insertions(+), 30 deletions(-) diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index f8cf58d8cb..a7b8c97740 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -75,6 +75,7 @@ typedef struct SScanLogicNode { double filesFactor; SArray* pSmaIndexes; SNodeList* pPartTags; + bool partSort; } SScanLogicNode; typedef struct SJoinLogicNode { diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 6516a10795..af5bebc601 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1274,13 +1274,12 @@ static bool validateTimestampDigits(const SValueNode* pVal) { } int64_t tsVal = pVal->datum.i; - char fraction[20] = {0}; + char fraction[20] = {0}; NUM_TO_STRING(pVal->node.resType.type, &tsVal, sizeof(fraction), fraction); int32_t tsDigits = (int32_t)strlen(fraction); if (tsDigits > TSDB_TIME_PRECISION_SEC_DIGITS) { - if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS || - tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS || + if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS || tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS || tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) { return true; } else { @@ -1510,6 +1509,11 @@ static int32_t translateBlockDistInfoFunc(SFunctionNode* pFunc, char* pErrBuf, i return TSDB_CODE_SUCCESS; } +static int32_t translateGroupKeyFunc(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + pFunc->node.resType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType; + return TSDB_CODE_SUCCESS; +} + static bool getBlockDistFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { pEnv->calcMemSize = sizeof(STableBlockDistInfo); return true; @@ -2499,6 +2503,14 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .type = FUNCTION_TYPE_BLOCK_DIST_INFO, .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_SCAN_PC_FUNC, .translateFunc = translateBlockDistInfoFunc, + }, + { + .name = "_group_key", + .type = FUNCTION_TYPE_BLOCK_DIST_INFO, + .classification = FUNC_MGT_AGG_FUNC, + .translateFunc = translateGroupKeyFunc, + .pPartialFunc = "_group_key", + .pMergeFunc = "_group_key" } }; // clang-format on diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 5d464fdf27..d94ec3460b 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1355,6 +1355,24 @@ static EDealRes rewriteColToSelectValFunc(STranslateContext* pCxt, SNode** pNode return TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR; } +static EDealRes rewriteExprToGroupKeyFunc(STranslateContext* pCxt, SNode** pNode) { + SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); + if (NULL == pFunc) { + pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; + return DEAL_RES_ERROR; + } + + strcpy(pFunc->functionName, "_group_key"); + strcpy(pFunc->node.aliasName, ((SExprNode*)*pNode)->aliasName); + pCxt->errCode = nodesListMakeAppend(&pFunc->pParameterList, *pNode); + if (TSDB_CODE_SUCCESS == pCxt->errCode) { + *pNode = (SNode*)pFunc; + pCxt->errCode = fmGetFuncInfo(pFunc, pCxt->msgBuf.buf, pCxt->msgBuf.len); + } + + return (TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR); +} + static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) { SCheckExprForGroupByCxt* pCxt = (SCheckExprForGroupByCxt*)pContext; if (!nodesIsExprNode(*pNode) || isAliasColumn(*pNode)) { @@ -1371,10 +1389,10 @@ static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) { if (isAggFunc(*pNode) && !isDistinctOrderBy(pCxt->pTranslateCxt)) { return DEAL_RES_IGNORE_CHILD; } - SNode* pGroupNode; + SNode* pGroupNode = NULL; FOREACH(pGroupNode, getGroupByList(pCxt->pTranslateCxt)) { if (nodesEqualNode(getGroupByNode(pGroupNode), *pNode)) { - return DEAL_RES_IGNORE_CHILD; + return rewriteExprToGroupKeyFunc(pCxt->pTranslateCxt, pNode); } } if (isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) { @@ -1441,22 +1459,28 @@ typedef struct CheckAggColCoexistCxt { bool existOtherAggFunc; } CheckAggColCoexistCxt; -static EDealRes doCheckAggColCoexist(SNode* pNode, void* pContext) { +static EDealRes doCheckAggColCoexist(SNode** pNode, void* pContext) { CheckAggColCoexistCxt* pCxt = (CheckAggColCoexistCxt*)pContext; - if (isSelectFunc(pNode)) { + if (isSelectFunc(*pNode)) { ++(pCxt->selectFuncNum); - } else if (isAggFunc(pNode)) { + } else if (isAggFunc(*pNode)) { pCxt->existOtherAggFunc = true; } - if (isAggFunc(pNode)) { + if (isAggFunc(*pNode)) { pCxt->existAggFunc = true; return DEAL_RES_IGNORE_CHILD; } - if (isIndefiniteRowsFunc(pNode)) { + if (isIndefiniteRowsFunc(*pNode)) { pCxt->existIndefiniteRowsFunc = true; return DEAL_RES_IGNORE_CHILD; } - if (isScanPseudoColumnFunc(pNode) || QUERY_NODE_COLUMN == nodeType(pNode)) { + SNode* pPartKey = NULL; + FOREACH(pPartKey, pCxt->pTranslateCxt->pCurrSelectStmt->pPartitionByList) { + if (nodesEqualNode(pPartKey, *pNode)) { + return rewriteExprToGroupKeyFunc(pCxt->pTranslateCxt, pNode); + } + } + if (isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) { pCxt->existCol = true; } return DEAL_RES_CONTINUE; @@ -1472,9 +1496,9 @@ static int32_t checkAggColCoexist(STranslateContext* pCxt, SSelectStmt* pSelect) .existIndefiniteRowsFunc = false, .selectFuncNum = 0, .existOtherAggFunc = false}; - nodesWalkExprs(pSelect->pProjectionList, doCheckAggColCoexist, &cxt); + nodesRewriteExprs(pSelect->pProjectionList, doCheckAggColCoexist, &cxt); if (!pSelect->isDistinct) { - nodesWalkExprs(pSelect->pOrderByList, doCheckAggColCoexist, &cxt); + nodesRewriteExprs(pSelect->pOrderByList, doCheckAggColCoexist, &cxt); } if (1 == cxt.selectFuncNum && !cxt.existOtherAggFunc) { return rewriteColsToSelectValFunc(pCxt, pSelect); diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index 5c8439f846..f8b8dc58fc 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -199,6 +199,20 @@ TEST_F(ParserSelectTest, tailFuncSemanticCheck) { run("SELECT TAIL(c1, 10) FROM t1 GROUP BY c2", TSDB_CODE_PAR_GROUP_BY_NOT_ALLOWED_FUNC); } +TEST_F(ParserSelectTest, partitionBy) { + useDb("root", "test"); + + run("SELECT c1, c2 FROM t1 PARTITION BY c2"); + + run("SELECT SUM(c1), c2 FROM t1 PARTITION BY c2"); +} + +TEST_F(ParserSelectTest, partitionBySemanticCheck) { + useDb("root", "test"); + + run("SELECT SUM(c1), c2, c3 FROM t1 PARTITION BY c2", TSDB_CODE_PAR_NOT_SINGLE_GROUP); +} + TEST_F(ParserSelectTest, groupBy) { useDb("root", "test"); @@ -213,6 +227,15 @@ TEST_F(ParserSelectTest, groupBy) { run("SELECT COUNT(*), c1 + 10, c2 cnt FROM t1 WHERE c1 > 0 GROUP BY c1 + 10, c2"); } +TEST_F(ParserSelectTest, groupBySemanticCheck) { + useDb("root", "test"); + + run("SELECT COUNT(*) cnt, c1 FROM t1 WHERE c1 > 0", TSDB_CODE_PAR_NOT_SINGLE_GROUP); + run("SELECT COUNT(*) cnt, c2 FROM t1 WHERE c1 > 0 GROUP BY c1", TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION); + run("SELECT COUNT(*) cnt, c2 FROM t1 WHERE c1 > 0 PARTITION BY c2 GROUP BY c1", + TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION); +} + TEST_F(ParserSelectTest, orderBy) { useDb("root", "test"); diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index d4f18a663c..6d4aa5ae23 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -457,15 +457,15 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, } } + if (TSDB_CODE_SUCCESS == code && pSelect->hasAggFuncs) { + code = nodesCollectFuncs(pSelect, SQL_CLAUSE_GROUP_BY, fmIsAggFunc, &pAgg->pAggFuncs); + } + // rewrite the expression in subsequent clauses if (TSDB_CODE_SUCCESS == code) { code = rewriteExprsForSelect(pAgg->pGroupKeys, pSelect, SQL_CLAUSE_GROUP_BY); } - if (TSDB_CODE_SUCCESS == code && pSelect->hasAggFuncs) { - code = nodesCollectFuncs(pSelect, SQL_CLAUSE_GROUP_BY, fmIsAggFunc, &pAgg->pAggFuncs); - } - // rewrite the expression in subsequent clauses if (TSDB_CODE_SUCCESS == code) { code = rewriteExprsForSelect(pAgg->pAggFuncs, pSelect, SQL_CLAUSE_GROUP_BY); diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index d4964d3cd4..7abfdb1f50 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1075,6 +1075,16 @@ static int32_t partTagsOptRebuildTbanme(SNodeList* pPartKeys) { return code; } +static int32_t partTagOptRewriteGroupKey(SAggLogicNode* pAgg, SScanLogicNode* pScan) { + // SNode* pNode = NULL; + // FOREACH(pNode, pScan->pPartTags) { + // if (QUERY_NODE_COLUMN != nodeType(pNode)) { + // createColumnByRewriteExpr(pNode, ); + // } + // } + return TSDB_CODE_SUCCESS; +} + static int32_t partTagsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { SLogicNode* pNode = optFindPossibleNode(pLogicSubplan->pNode, partTagsOptMayBeOptimized); if (NULL == pNode) { @@ -1100,6 +1110,9 @@ static int32_t partTagsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSub } } NODES_DESTORY_LIST(((SAggLogicNode*)pNode)->pGroupKeys); + if (TSDB_CODE_SUCCESS == code) { + code = partTagOptRewriteGroupKey((SAggLogicNode*)pNode, pScan); + } } if (TSDB_CODE_SUCCESS == code) { code = partTagsOptRebuildTbanme(pScan->pPartTags); @@ -1189,7 +1202,7 @@ static const SOptimizeRule optimizeRuleSet[] = { {.pName = "ConditionPushDown", .optimizeFunc = cpdOptimize}, {.pName = "OrderByPrimaryKey", .optimizeFunc = opkOptimize}, {.pName = "SmaIndex", .optimizeFunc = smaOptimize}, - // {.pName = "PartitionTags", .optimizeFunc = partTagsOptimize}, + {.pName = "PartitionTags", .optimizeFunc = partTagsOptimize}, {.pName = "EliminateProject", .optimizeFunc = eliminateProjOptimize} }; // clang-format on diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 8e8559ba3f..b4a966f206 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -390,9 +390,6 @@ static int32_t stbSplCreateMergeNode(SSplitContext* pCxt, SLogicSubplan* pSubpla code = replaceLogicNode(pSubplan, pSplitNode, (SLogicNode*)pMerge); } } - if (TSDB_CODE_SUCCESS == code && NULL != pSubplan) { - nodesDestroyNode((SNode*)pSplitNode); - } if (TSDB_CODE_SUCCESS != code) { nodesDestroyNode((SNode*)pMerge); } @@ -564,6 +561,8 @@ static int32_t stbSplSplitState(SSplitContext* pCxt, SStableSplitInfo* pInfo) { static SNodeList* stbSplGetPartKeys(SLogicNode* pNode) { if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) { return ((SScanLogicNode*)pNode)->pPartTags; + } else if (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode)) { + return ((SPartitionLogicNode*)pNode)->pPartitionKeys; } else { return NULL; } @@ -574,14 +573,15 @@ static bool stbSplIsPartTbanme(SNodeList* pPartKeys) { return false; } SNode* pPartKey = nodesListGetNode(pPartKeys, 0); - return QUERY_NODE_FUNCTION == nodeType(pPartKey) && FUNCTION_TYPE_TBNAME == ((SFunctionNode*)pPartKey)->funcType; + return (QUERY_NODE_FUNCTION == nodeType(pPartKey) && FUNCTION_TYPE_TBNAME == ((SFunctionNode*)pPartKey)->funcType) || + (QUERY_NODE_COLUMN == nodeType(pPartKey) && COLUMN_TYPE_TBNAME == ((SColumnNode*)pPartKey)->colType); } -static bool stbSplIsMultiTableWinodw(SWindowLogicNode* pWindow) { +static bool stbSplIsPartTableWinodw(SWindowLogicNode* pWindow) { return stbSplIsPartTbanme(stbSplGetPartKeys((SLogicNode*)nodesListGetNode(pWindow->node.pChildren, 0))); } -static int32_t stbSplSplitWindowForMergeTable(SSplitContext* pCxt, SStableSplitInfo* pInfo) { +static int32_t stbSplSplitWindowForCrossTable(SSplitContext* pCxt, SStableSplitInfo* pInfo) { switch (((SWindowLogicNode*)pInfo->pSplitNode)->winType) { case WINDOW_TYPE_INTERVAL: return stbSplSplitInterval(pCxt, pInfo); @@ -595,7 +595,7 @@ static int32_t stbSplSplitWindowForMergeTable(SSplitContext* pCxt, SStableSplitI return TSDB_CODE_PLAN_INTERNAL_ERROR; } -static int32_t stbSplSplitWindowForMultiTable(SSplitContext* pCxt, SStableSplitInfo* pInfo) { +static int32_t stbSplSplitWindowForPartTable(SSplitContext* pCxt, SStableSplitInfo* pInfo) { if (pCxt->pPlanCxt->streamQuery) { SPLIT_FLAG_SET_MASK(pInfo->pSubplan->splitFlag, SPLIT_FLAG_STABLE_SPLIT); return TSDB_CODE_SUCCESS; @@ -616,10 +616,10 @@ static int32_t stbSplSplitWindowForMultiTable(SSplitContext* pCxt, SStableSplitI } static int32_t stbSplSplitWindowNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) { - if (stbSplIsMultiTableWinodw((SWindowLogicNode*)pInfo->pSplitNode)) { - return stbSplSplitWindowForMultiTable(pCxt, pInfo); + if (stbSplIsPartTableWinodw((SWindowLogicNode*)pInfo->pSplitNode)) { + return stbSplSplitWindowForPartTable(pCxt, pInfo); } else { - return stbSplSplitWindowForMergeTable(pCxt, pInfo); + return stbSplSplitWindowForCrossTable(pCxt, pInfo); } } diff --git a/source/libs/planner/test/planOptimizeTest.cpp b/source/libs/planner/test/planOptimizeTest.cpp index 8b3d263d66..9bd95a79f9 100644 --- a/source/libs/planner/test/planOptimizeTest.cpp +++ b/source/libs/planner/test/planOptimizeTest.cpp @@ -57,9 +57,15 @@ TEST_F(PlanOptimizeTest, orderByPrimaryKey) { TEST_F(PlanOptimizeTest, PartitionTags) { useDb("root", "test"); - run("SELECT c1 FROM st1 PARTITION BY tag1"); + run("SELECT c1, tag1 FROM st1 PARTITION BY tag1"); - run("SELECT SUM(c1) FROM st1 GROUP BY tag1"); + run("SELECT SUM(c1), tag1 FROM st1 PARTITION BY tag1"); + + run("SELECT SUM(c1), tag1 + 10 FROM st1 PARTITION BY tag1 + 10"); + + run("SELECT SUM(c1), tag1 FROM st1 GROUP BY tag1"); + + run("SELECT SUM(c1), tag1 + 10 FROM st1 GROUP BY tag1 + 10"); } TEST_F(PlanOptimizeTest, eliminateProjection) { diff --git a/source/libs/planner/test/planPartByTest.cpp b/source/libs/planner/test/planPartByTest.cpp index 9437f6ad3f..2abcb44dfd 100644 --- a/source/libs/planner/test/planPartByTest.cpp +++ b/source/libs/planner/test/planPartByTest.cpp @@ -34,6 +34,8 @@ TEST_F(PlanPartitionByTest, withAggFunc) { useDb("root", "test"); run("select count(*) from t1 partition by c1"); + + run("select count(*), c1 from t1 partition by c1"); } TEST_F(PlanPartitionByTest, withInterval) { @@ -41,8 +43,12 @@ TEST_F(PlanPartitionByTest, withInterval) { // normal/child table run("select count(*) from t1 partition by c1 interval(10s)"); + + run("select count(*), c1 from t1 partition by c1 interval(10s)"); // super table run("select count(*) from st1 partition by tag1, tag2 interval(10s)"); + + run("select count(*), tag1 from st1 partition by tag1, tag2 interval(10s)"); } TEST_F(PlanPartitionByTest, withGroupBy) { From fb1e845256f278d0da3f9ac5a1363b4515fb6dea Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sat, 25 Jun 2022 08:44:45 +0800 Subject: [PATCH 18/72] feat: support partition by expression and aggregate function output together --- source/libs/parser/src/parTranslater.c | 44 ++++++++++++++++++------- source/libs/planner/src/planOptimizer.c | 15 +-------- 2 files changed, 34 insertions(+), 25 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 62cf2a125b..7a6c340474 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1450,6 +1450,25 @@ static int32_t rewriteColsToSelectValFunc(STranslateContext* pCxt, SSelectStmt* return pCxt->errCode; } +static EDealRes rewriteExprsToGroupKeyFuncImpl(SNode** pNode, void* pContext) { + STranslateContext* pCxt = pContext; + SNode* pPartKey = NULL; + FOREACH(pPartKey, pCxt->pCurrSelectStmt->pPartitionByList) { + if (nodesEqualNode(pPartKey, *pNode)) { + return rewriteExprToGroupKeyFunc(pCxt, pNode); + } + } + return DEAL_RES_CONTINUE; +} + +static int32_t rewriteExprsToGroupKeyFunc(STranslateContext* pCxt, SSelectStmt* pSelect) { + nodesRewriteExprs(pSelect->pProjectionList, rewriteExprsToGroupKeyFuncImpl, pCxt); + if (TSDB_CODE_SUCCESS == pCxt->errCode && !pSelect->isDistinct) { + nodesRewriteExprs(pSelect->pOrderByList, rewriteExprsToGroupKeyFuncImpl, pCxt); + } + return pCxt->errCode; +} + typedef struct CheckAggColCoexistCxt { STranslateContext* pTranslateCxt; bool existAggFunc; @@ -1459,28 +1478,28 @@ typedef struct CheckAggColCoexistCxt { bool existOtherAggFunc; } CheckAggColCoexistCxt; -static EDealRes doCheckAggColCoexist(SNode** pNode, void* pContext) { +static EDealRes doCheckAggColCoexist(SNode* pNode, void* pContext) { CheckAggColCoexistCxt* pCxt = (CheckAggColCoexistCxt*)pContext; - if (isSelectFunc(*pNode)) { + if (isSelectFunc(pNode)) { ++(pCxt->selectFuncNum); - } else if (isAggFunc(*pNode)) { + } else if (isAggFunc(pNode)) { pCxt->existOtherAggFunc = true; } - if (isAggFunc(*pNode)) { + if (isAggFunc(pNode)) { pCxt->existAggFunc = true; return DEAL_RES_IGNORE_CHILD; } - if (isIndefiniteRowsFunc(*pNode)) { + if (isIndefiniteRowsFunc(pNode)) { pCxt->existIndefiniteRowsFunc = true; return DEAL_RES_IGNORE_CHILD; } SNode* pPartKey = NULL; FOREACH(pPartKey, pCxt->pTranslateCxt->pCurrSelectStmt->pPartitionByList) { - if (nodesEqualNode(pPartKey, *pNode)) { - return rewriteExprToGroupKeyFunc(pCxt->pTranslateCxt, pNode); + if (nodesEqualNode(pPartKey, pNode)) { + return DEAL_RES_IGNORE_CHILD; } } - if (isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) { + if (isScanPseudoColumnFunc(pNode) || QUERY_NODE_COLUMN == nodeType(pNode)) { pCxt->existCol = true; } return DEAL_RES_CONTINUE; @@ -1496,9 +1515,9 @@ static int32_t checkAggColCoexist(STranslateContext* pCxt, SSelectStmt* pSelect) .existIndefiniteRowsFunc = false, .selectFuncNum = 0, .existOtherAggFunc = false}; - nodesRewriteExprs(pSelect->pProjectionList, doCheckAggColCoexist, &cxt); + nodesWalkExprs(pSelect->pProjectionList, doCheckAggColCoexist, &cxt); if (!pSelect->isDistinct) { - nodesRewriteExprs(pSelect->pOrderByList, doCheckAggColCoexist, &cxt); + nodesWalkExprs(pSelect->pOrderByList, doCheckAggColCoexist, &cxt); } if (1 == cxt.selectFuncNum && !cxt.existOtherAggFunc) { return rewriteColsToSelectValFunc(pCxt, pSelect); @@ -1509,6 +1528,9 @@ static int32_t checkAggColCoexist(STranslateContext* pCxt, SSelectStmt* pSelect) if (cxt.existIndefiniteRowsFunc && cxt.existCol) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC); } + if (cxt.existAggFunc && NULL != pSelect->pPartitionByList) { + return rewriteExprsToGroupKeyFunc(pCxt, pSelect); + } return TSDB_CODE_SUCCESS; } @@ -6039,7 +6061,7 @@ static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) { TSWAP(pQuery->pCmdMsg, pCxt->pCmdMsg); pQuery->msgType = pQuery->pCmdMsg->msgType; } - break; + break; default: pQuery->execMode = QUERY_EXEC_MODE_RPC; if (NULL != pCxt->pCmdMsg) { diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 7abfdb1f50..d4964d3cd4 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1075,16 +1075,6 @@ static int32_t partTagsOptRebuildTbanme(SNodeList* pPartKeys) { return code; } -static int32_t partTagOptRewriteGroupKey(SAggLogicNode* pAgg, SScanLogicNode* pScan) { - // SNode* pNode = NULL; - // FOREACH(pNode, pScan->pPartTags) { - // if (QUERY_NODE_COLUMN != nodeType(pNode)) { - // createColumnByRewriteExpr(pNode, ); - // } - // } - return TSDB_CODE_SUCCESS; -} - static int32_t partTagsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { SLogicNode* pNode = optFindPossibleNode(pLogicSubplan->pNode, partTagsOptMayBeOptimized); if (NULL == pNode) { @@ -1110,9 +1100,6 @@ static int32_t partTagsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSub } } NODES_DESTORY_LIST(((SAggLogicNode*)pNode)->pGroupKeys); - if (TSDB_CODE_SUCCESS == code) { - code = partTagOptRewriteGroupKey((SAggLogicNode*)pNode, pScan); - } } if (TSDB_CODE_SUCCESS == code) { code = partTagsOptRebuildTbanme(pScan->pPartTags); @@ -1202,7 +1189,7 @@ static const SOptimizeRule optimizeRuleSet[] = { {.pName = "ConditionPushDown", .optimizeFunc = cpdOptimize}, {.pName = "OrderByPrimaryKey", .optimizeFunc = opkOptimize}, {.pName = "SmaIndex", .optimizeFunc = smaOptimize}, - {.pName = "PartitionTags", .optimizeFunc = partTagsOptimize}, + // {.pName = "PartitionTags", .optimizeFunc = partTagsOptimize}, {.pName = "EliminateProject", .optimizeFunc = eliminateProjOptimize} }; // clang-format on From e187f4290236ecc4c9bc5ad93ba692cb235330c0 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sat, 25 Jun 2022 09:42:50 +0800 Subject: [PATCH 19/72] feat: support partition by expression and aggregate function output together --- source/libs/parser/src/parTranslater.c | 21 ++++++++++++++++++-- source/libs/planner/src/planOptimizer.c | 2 +- source/libs/planner/test/planGroupByTest.cpp | 10 ++-------- 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 7a6c340474..c81ecfc3e9 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1369,6 +1369,7 @@ static EDealRes rewriteExprToGroupKeyFunc(STranslateContext* pCxt, SNode** pNode *pNode = (SNode*)pFunc; pCxt->errCode = fmGetFuncInfo(pFunc, pCxt->msgBuf.buf, pCxt->msgBuf.len); } + pCxt->pCurrSelectStmt->hasAggFuncs = true; return (TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR); } @@ -2506,8 +2507,24 @@ static SNode* createOrderByExpr(STranslateContext* pCxt) { return (SNode*)pOrder; } -// from: select tail(expr, k, f) from t where_clause partition_by_clause order_by_clause ... -// to: select expr from t where_clause order by _rowts desc limit k offset f +/* case 1: + * in: select tail(expr, k, f) from t where_clause + * out: select expr from t where_clause order by _rowts desc limit k offset f + * + * case 2: + * in: select tail(expr, k, f) from t where_clause partition_by_clause + * out: select expr from t where_clause partition_by_clause sort by _rowts desc limit k offset f + * + * case 3: + * in: select tail(expr, k, f) from t where_clause order_by_clause limit_clause + * out: select expr from ( + * select expr from t where_clause order by _rowts desc limit k offset f + * ) order_by_clause limit_clause + * + * case 4: + * in: select tail(expr, k, f) from t where_clause partition_by_clause limit_clause + * out: + */ static int32_t rewriteTailStmt(STranslateContext* pCxt, SSelectStmt* pSelect) { if (!pSelect->hasTailFunc) { return TSDB_CODE_SUCCESS; diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index d4964d3cd4..c1b12d9d20 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1189,7 +1189,7 @@ static const SOptimizeRule optimizeRuleSet[] = { {.pName = "ConditionPushDown", .optimizeFunc = cpdOptimize}, {.pName = "OrderByPrimaryKey", .optimizeFunc = opkOptimize}, {.pName = "SmaIndex", .optimizeFunc = smaOptimize}, - // {.pName = "PartitionTags", .optimizeFunc = partTagsOptimize}, + {.pName = "PartitionTags", .optimizeFunc = partTagsOptimize}, {.pName = "EliminateProject", .optimizeFunc = eliminateProjOptimize} }; // clang-format on diff --git a/source/libs/planner/test/planGroupByTest.cpp b/source/libs/planner/test/planGroupByTest.cpp index a73c99bf68..657d8a6244 100644 --- a/source/libs/planner/test/planGroupByTest.cpp +++ b/source/libs/planner/test/planGroupByTest.cpp @@ -53,14 +53,6 @@ TEST_F(PlanGroupByTest, aggFunc) { run("SELECT SUM(10), COUNT(c1) FROM t1 GROUP BY c2"); } -TEST_F(PlanGroupByTest, rewriteFunc) { - useDb("root", "test"); - - run("SELECT AVG(c1) FROM t1"); - - run("SELECT AVG(c1) FROM t1 GROUP BY c2"); -} - TEST_F(PlanGroupByTest, selectFunc) { useDb("root", "test"); @@ -81,6 +73,8 @@ TEST_F(PlanGroupByTest, stable) { run("SELECT COUNT(*) FROM st1"); + run("SELECT c1 FROM st1 GROUP BY c1"); + run("SELECT COUNT(*) FROM st1 GROUP BY c1"); run("SELECT COUNT(*) FROM st1 PARTITION BY c2 GROUP BY c1"); From 96b3603bfda19844cd9f3e06438ddb402b1e731b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 25 Jun 2022 09:09:33 +0800 Subject: [PATCH 20/72] refactor: rename auth to privilege --- source/dnode/mnode/impl/inc/mndAuth.h | 13 ++++-- source/dnode/mnode/impl/src/mndAcct.c | 13 ++++++ source/dnode/mnode/impl/src/mndAuth.c | 10 ++-- source/dnode/mnode/impl/src/mndBnode.c | 4 +- source/dnode/mnode/impl/src/mndDb.c | 15 +++--- source/dnode/mnode/impl/src/mndDnode.c | 6 +-- source/dnode/mnode/impl/src/mndFunc.c | 4 +- source/dnode/mnode/impl/src/mndMnode.c | 4 +- source/dnode/mnode/impl/src/mndOffset.c | 16 ++++--- source/dnode/mnode/impl/src/mndProfile.c | 48 ++++++++------------ source/dnode/mnode/impl/src/mndQnode.c | 4 +- source/dnode/mnode/impl/src/mndShow.c | 2 +- source/dnode/mnode/impl/src/mndSma.c | 4 +- source/dnode/mnode/impl/src/mndSnode.c | 4 +- source/dnode/mnode/impl/src/mndStb.c | 6 +-- source/dnode/mnode/impl/src/mndStream.c | 4 +- source/dnode/mnode/impl/src/mndTopic.c | 2 +- source/dnode/mnode/impl/src/mndTrans.c | 3 +- source/dnode/mnode/impl/src/mndUser.c | 18 ++++---- source/dnode/mnode/impl/src/mndVgroup.c | 6 +-- tests/script/tsim/user/privilege_sysinfo.sim | 21 +++++++++ 21 files changed, 117 insertions(+), 90 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndAuth.h b/source/dnode/mnode/impl/inc/mndAuth.h index 81a776b652..c6f337b44e 100644 --- a/source/dnode/mnode/impl/inc/mndAuth.h +++ b/source/dnode/mnode/impl/inc/mndAuth.h @@ -24,6 +24,9 @@ extern "C" { typedef enum { MND_OPER_CONNECT = 1, + MND_OPER_CREATE_ACCT, + MND_OPER_DROP_ACCT, + MND_OPER_ALTER_ACCT, MND_OPER_CREATE_USER, MND_OPER_DROP_USER, MND_OPER_ALTER_USER, @@ -45,6 +48,8 @@ typedef enum { MND_OPER_CREATE_FUNC, MND_OPER_DROP_FUNC, MND_OPER_KILL_TRANS, + MND_OPER_KILL_CONN, + MND_OPER_KILL_QUERY, MND_OPER_CREATE_DB, MND_OPER_ALTER_DB, MND_OPER_DROP_DB, @@ -57,10 +62,10 @@ typedef enum { int32_t mndInitAuth(SMnode *pMnode); void mndCleanupAuth(SMnode *pMnode); -int32_t mndCheckOperAuth(SMnode *pMnode, const char *user, EOperType operType); -int32_t mndCheckDbAuth(SMnode *pMnode, const char *user, EOperType operType, SDbObj *pDb); -int32_t mndCheckShowAuth(SMnode *pMnode, const char *user, int32_t showType); -int32_t mndCheckAlterUserAuth(SUserObj *pOperUser, SUserObj *pUser, SAlterUserReq *pAlter); +int32_t mndCheckOperPrivilege(SMnode *pMnode, const char *user, EOperType operType); +int32_t mndCheckDbPrivilege(SMnode *pMnode, const char *user, EOperType operType, SDbObj *pDb); +int32_t mndCheckShowPrivilege(SMnode *pMnode, const char *user, int32_t showType); +int32_t mndCheckAlterUserPrivilege(SUserObj *pOperUser, SUserObj *pUser, SAlterUserReq *pAlter); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndAcct.c b/source/dnode/mnode/impl/src/mndAcct.c index 0ce4a8c76e..da78abf5c0 100644 --- a/source/dnode/mnode/impl/src/mndAcct.c +++ b/source/dnode/mnode/impl/src/mndAcct.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "mndAcct.h" +#include "mndAuth.h" #include "mndShow.h" #include "mndTrans.h" @@ -212,18 +213,30 @@ static int32_t mndAcctActionUpdate(SSdb *pSdb, SAcctObj *pOld, SAcctObj *pNew) { } static int32_t mndProcessCreateAcctReq(SRpcMsg *pReq) { + if (mndCheckOperPrivilege(pReq->info.node, pReq->info.conn.user, MND_OPER_CREATE_ACCT) != 0) { + return -1; + } + terrno = TSDB_CODE_MSG_NOT_PROCESSED; mError("failed to process create acct request since %s", terrstr()); return -1; } static int32_t mndProcessAlterAcctReq(SRpcMsg *pReq) { + if (mndCheckOperPrivilege(pReq->info.node, pReq->info.conn.user, MND_OPER_ALTER_ACCT) != 0) { + return -1; + } + terrno = TSDB_CODE_MSG_NOT_PROCESSED; mError("failed to process create acct request since %s", terrstr()); return -1; } static int32_t mndProcessDropAcctReq(SRpcMsg *pReq) { + if (mndCheckOperPrivilege(pReq->info.node, pReq->info.conn.user, MND_OPER_DROP_ACCT) != 0) { + return -1; + } + terrno = TSDB_CODE_MSG_NOT_PROCESSED; mError("failed to process create acct request since %s", terrstr()); return -1; diff --git a/source/dnode/mnode/impl/src/mndAuth.c b/source/dnode/mnode/impl/src/mndAuth.c index 4445e3b9f7..f1542f5d42 100644 --- a/source/dnode/mnode/impl/src/mndAuth.c +++ b/source/dnode/mnode/impl/src/mndAuth.c @@ -73,7 +73,7 @@ static int32_t mndProcessAuthReq(SRpcMsg *pReq) { return code; } -int32_t mndCheckOperAuth(SMnode *pMnode, const char *user, EOperType operType) { +int32_t mndCheckOperPrivilege(SMnode *pMnode, const char *user, EOperType operType) { int32_t code = 0; SUserObj *pUser = mndAcquireUser(pMnode, user); @@ -95,6 +95,8 @@ int32_t mndCheckOperAuth(SMnode *pMnode, const char *user, EOperType operType) { switch (operType) { case MND_OPER_CONNECT: + case MND_OPER_CREATE_FUNC: + case MND_OPER_DROP_FUNC: break; default: terrno = TSDB_CODE_MND_NO_RIGHTS; @@ -106,7 +108,7 @@ _OVER: return code; } -int32_t mndCheckAlterUserAuth(SUserObj *pOperUser, SUserObj *pUser, SAlterUserReq *pAlter) { +int32_t mndCheckAlterUserPrivilege(SUserObj *pOperUser, SUserObj *pUser, SAlterUserReq *pAlter) { if (pUser->superUser && pAlter->alterType != TSDB_ALTER_USER_PASSWD) { terrno = TSDB_CODE_MND_NO_RIGHTS; return -1; @@ -129,7 +131,7 @@ int32_t mndCheckAlterUserAuth(SUserObj *pOperUser, SUserObj *pUser, SAlterUserRe return -1; } -int32_t mndCheckShowAuth(SMnode *pMnode, const char *user, int32_t showType) { +int32_t mndCheckShowPrivilege(SMnode *pMnode, const char *user, int32_t showType) { int32_t code = 0; SUserObj *pUser = mndAcquireUser(pMnode, user); @@ -162,7 +164,7 @@ _OVER: return code; } -int32_t mndCheckDbAuth(SMnode *pMnode, const char *user, EOperType operType, SDbObj *pDb) { +int32_t mndCheckDbPrivilege(SMnode *pMnode, const char *user, EOperType operType, SDbObj *pDb) { int32_t code = 0; SUserObj *pUser = mndAcquireUser(pMnode, user); diff --git a/source/dnode/mnode/impl/src/mndBnode.c b/source/dnode/mnode/impl/src/mndBnode.c index e2b1aad008..debd7cbcc0 100644 --- a/source/dnode/mnode/impl/src/mndBnode.c +++ b/source/dnode/mnode/impl/src/mndBnode.c @@ -277,7 +277,7 @@ static int32_t mndProcessCreateBnodeReq(SRpcMsg *pReq) { } mDebug("bnode:%d, start to create", createReq.dnodeId); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_BNODE) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_BNODE) != 0) { goto _OVER; } @@ -382,7 +382,7 @@ static int32_t mndProcessDropBnodeReq(SRpcMsg *pReq) { } mDebug("bnode:%d, start to drop", dropReq.dnodeId); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_BNODE) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_BNODE) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 345464399e..00a5e609d7 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -506,6 +506,9 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) { } mDebug("db:%s, start to create, vgroups:%d", createReq.db, createReq.numOfVgroups); + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_DB, NULL) != 0) { + goto _OVER; + } pDb = mndAcquireDb(pMnode, createReq.db); if (pDb != NULL) { @@ -526,10 +529,6 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_DB, NULL) != 0) { - goto _OVER; - } - code = mndCreateDb(pMnode, pReq, &createReq, pUser); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; @@ -700,7 +699,7 @@ static int32_t mndProcessAlterDbReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_ALTER_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_ALTER_DB, pDb) != 0) { goto _OVER; } @@ -980,7 +979,7 @@ static int32_t mndProcessDropDbReq(SRpcMsg *pReq) { } } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_DB, pDb) != 0) { goto _OVER; } @@ -1127,7 +1126,7 @@ static int32_t mndProcessUseDbReq(SRpcMsg *pReq) { mError("db:%s, failed to process use db req since %s", usedbReq.db, terrstr()); } else { - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_USE_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_USE_DB, pDb) != 0) { goto _OVER; } @@ -1252,7 +1251,7 @@ static int32_t mndProcessCompactDbReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_COMPACT_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_COMPACT_DB, pDb) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 113777bc1f..6cf80ffe47 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -621,7 +621,7 @@ static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) { } mInfo("dnode:%s:%d, start to create", createReq.fqdn, createReq.port); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_DNODE) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_DNODE) != 0) { goto _OVER; } @@ -715,7 +715,7 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) { } mInfo("dnode:%d, start to drop, ep:%s:%d", dropReq.dnodeId, dropReq.fqdn, dropReq.port); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_MNODE) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_MNODE) != 0) { goto _OVER; } @@ -779,7 +779,7 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { } mInfo("dnode:%d, start to config, option:%s, value:%s", cfgReq.dnodeId, cfgReq.config, cfgReq.value); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CONFIG_DNODE) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONFIG_DNODE) != 0) { return -1; } diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index 37e0a719dd..7baa641aa7 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -283,7 +283,7 @@ static int32_t mndProcessCreateFuncReq(SRpcMsg *pReq) { } mDebug("func:%s, start to create", createReq.name); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_FUNC) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_FUNC) != 0) { goto _OVER; } @@ -346,7 +346,7 @@ static int32_t mndProcessDropFuncReq(SRpcMsg *pReq) { } mDebug("func:%s, start to drop", dropReq.name); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_FUNC) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_FUNC) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index bc3d23282c..1919340250 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -389,7 +389,7 @@ static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) { } mDebug("mnode:%d, start to create", createReq.dnodeId); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_MNODE) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_MNODE) != 0) { goto _OVER; } @@ -594,7 +594,7 @@ static int32_t mndProcessDropMnodeReq(SRpcMsg *pReq) { } mDebug("mnode:%d, start to drop", dropReq.dnodeId); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_MNODE) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_MNODE) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndOffset.c b/source/dnode/mnode/impl/src/mndOffset.c index 18f2e993b2..8b5a1401f0 100644 --- a/source/dnode/mnode/impl/src/mndOffset.c +++ b/source/dnode/mnode/impl/src/mndOffset.c @@ -36,13 +36,15 @@ static int32_t mndOffsetActionUpdate(SSdb *pSdb, SMqOffsetObj *pOffset, SMqOffse static int32_t mndProcessCommitOffsetReq(SRpcMsg *pReq); int32_t mndInitOffset(SMnode *pMnode) { - SSdbTable table = {.sdbType = SDB_OFFSET, - .keyType = SDB_KEY_BINARY, - .encodeFp = (SdbEncodeFp)mndOffsetActionEncode, - .decodeFp = (SdbDecodeFp)mndOffsetActionDecode, - .insertFp = (SdbInsertFp)mndOffsetActionInsert, - .updateFp = (SdbUpdateFp)mndOffsetActionUpdate, - .deleteFp = (SdbDeleteFp)mndOffsetActionDelete}; + SSdbTable table = { + .sdbType = SDB_OFFSET, + .keyType = SDB_KEY_BINARY, + .encodeFp = (SdbEncodeFp)mndOffsetActionEncode, + .decodeFp = (SdbDecodeFp)mndOffsetActionDecode, + .insertFp = (SdbInsertFp)mndOffsetActionInsert, + .updateFp = (SdbUpdateFp)mndOffsetActionUpdate, + .deleteFp = (SdbDeleteFp)mndOffsetActionDelete, + }; mndSetMsgHandle(pMnode, TDMT_MND_MQ_COMMIT_OFFSET, mndProcessCommitOffsetReq); diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index e9df4ae1d0..5e3fa3c1e3 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -227,6 +227,10 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { } taosIp2String(pReq->info.conn.clientIp, ip); + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONNECT) != 0) { + mGError("user:%s, failed to login from %s since %s", pReq->info.conn.user, ip, terrstr()); + goto _OVER; + } pUser = mndAcquireUser(pMnode, pReq->info.conn.user); if (pUser == NULL) { @@ -240,11 +244,6 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CONNECT) != 0) { - mGError("user:%s, failed to login from %s since %s", pReq->info.conn.user, ip, terrstr()); - goto _OVER; - } - if (connReq.db[0]) { char db[TSDB_DB_FNAME_LEN] = {0}; snprintf(db, TSDB_DB_FNAME_LEN, "%d%s%s", pUser->acctId, TS_PATH_DELIMITER, connReq.db); @@ -271,7 +270,7 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { connectRsp.connId = pConn->id; connectRsp.connType = connReq.connType; connectRsp.dnodeNum = mndGetDnodeSize(pMnode); - + strcpy(connectRsp.sVer, version); snprintf(connectRsp.sDetailVer, sizeof(connectRsp.sDetailVer), "ver:%s\nbuild:%s\ngitinfo:%s", version, buildinfo, gitinfo); @@ -475,16 +474,16 @@ static int32_t mndGetOnlineDnodeNum(SMnode *pMnode, int32_t *num) { SDnodeObj *pDnode = NULL; int64_t curMs = taosGetTimestampMs(); void *pIter = NULL; - + while (true) { pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pDnode); if (pIter == NULL) break; - + bool online = mndIsDnodeOnline(pDnode, curMs); if (online) { (*num)++; } - + sdbRelease(pSdb, pDnode); } @@ -652,15 +651,6 @@ static int32_t mndProcessKillQueryReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SProfileMgmt *pMgmt = &pMnode->profileMgmt; - SUserObj *pUser = mndAcquireUser(pMnode, pReq->info.conn.user); - if (pUser == NULL) return 0; - if (!pUser->superUser) { - mndReleaseUser(pMnode, pUser); - terrno = TSDB_CODE_MND_NO_RIGHTS; - return -1; - } - mndReleaseUser(pMnode, pUser); - SKillQueryReq killReq = {0}; if (tDeserializeSKillQueryReq(pReq->pCont, pReq->contLen, &killReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; @@ -668,6 +658,10 @@ static int32_t mndProcessKillQueryReq(SRpcMsg *pReq) { } mInfo("kill query msg is received, queryId:%s", killReq.queryStrId); + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_KILL_QUERY) != 0) { + return -1; + } + int32_t connId = 0; uint64_t queryId = 0; char *p = strchr(killReq.queryStrId, ':'); @@ -697,21 +691,16 @@ static int32_t mndProcessKillConnReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SProfileMgmt *pMgmt = &pMnode->profileMgmt; - SUserObj *pUser = mndAcquireUser(pMnode, pReq->info.conn.user); - if (pUser == NULL) return 0; - if (!pUser->superUser) { - mndReleaseUser(pMnode, pUser); - terrno = TSDB_CODE_MND_NO_RIGHTS; - return -1; - } - mndReleaseUser(pMnode, pUser); - SKillConnReq killReq = {0}; if (tDeserializeSKillConnReq(pReq->pCont, pReq->contLen, &killReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; return -1; } + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_KILL_CONN) != 0) { + return -1; + } + SConnObj *pConn = taosCacheAcquireByKey(pMgmt->connCache, &killReq.connId, sizeof(uint32_t)); if (pConn == NULL) { mError("connId:%u, failed to kill connection, conn not exist", killReq.connId); @@ -726,10 +715,10 @@ static int32_t mndProcessKillConnReq(SRpcMsg *pReq) { } static int32_t mndProcessSvrVerReq(SRpcMsg *pReq) { - int32_t code = -1; + int32_t code = -1; SServerVerRsp rsp = {0}; strcpy(rsp.ver, version); - + int32_t contLen = tSerializeSServerVerRsp(NULL, 0, &rsp); if (contLen < 0) goto _over; void *pRsp = rpcMallocCont(contLen); @@ -746,7 +735,6 @@ _over: return code; } - static int32_t mndRetrieveConns(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { SMnode *pMnode = pReq->info.node; SSdb *pSdb = pMnode->pSdb; diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index 9f1eb4ee24..4992a99c5c 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -279,7 +279,7 @@ static int32_t mndProcessCreateQnodeReq(SRpcMsg *pReq) { } mDebug("qnode:%d, start to create", createReq.dnodeId); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_QNODE) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_QNODE) != 0) { goto _OVER; } @@ -390,7 +390,7 @@ static int32_t mndProcessDropQnodeReq(SRpcMsg *pReq) { } mDebug("qnode:%d, start to drop", dropReq.dnodeId); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_QNODE) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_QNODE) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 27de3883e9..bd129a8045 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -231,7 +231,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { mDebug("show:0x%" PRIx64 ", start retrieve data, type:%d", pShow->id, pShow->type); - // if (mndCheckShowAuth(pMnode, pReq->info.conn.user, pShow->type) != 0) return -1; + // if (mndCheckShowPrivilege(pMnode, pReq->info.conn.user, pShow->type) != 0) return -1; int32_t numOfCols = pShow->pMeta->numOfColumns; SSDataBlock *pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 05603f8554..5445c1e09f 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -713,7 +713,7 @@ static int32_t mndProcessCreateSmaReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { goto _OVER; } @@ -974,7 +974,7 @@ static int32_t mndProcessDropSmaReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index a638bdf61f..ea5011a659 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -285,7 +285,7 @@ static int32_t mndProcessCreateSnodeReq(SRpcMsg *pReq) { } mDebug("snode:%d, start to create", createReq.dnodeId); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_SNODE) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_SNODE) != 0) { goto _OVER; } @@ -397,7 +397,7 @@ static int32_t mndProcessDropSnodeReq(SRpcMsg *pReq) { } mDebug("snode:%d, start to drop", dropReq.dnodeId); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_SNODE) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_SNODE) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index f1bae14c07..026ce97776 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -876,7 +876,7 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { goto _OVER; } @@ -1607,7 +1607,7 @@ static int32_t mndProcessAlterStbReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { goto _OVER; } @@ -1737,7 +1737,7 @@ static int32_t mndProcessDropStbReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 5e2f5bc2dd..8847344493 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -437,7 +437,7 @@ static int32_t mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStre goto _OVER; } - if (mndCheckDbAuth(pMnode, user, MND_OPER_WRITE_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, user, MND_OPER_WRITE_DB, pDb) != 0) { goto _OVER; } @@ -550,7 +550,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { goto _OVER; } #endif diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index a650ed29f1..d3c51d18fc 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -480,7 +480,7 @@ static int32_t mndProcessCreateTopicReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index d1d88fdc90..963efc9233 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -1384,8 +1384,7 @@ static int32_t mndProcessKillTransReq(SRpcMsg *pReq) { } mInfo("trans:%d, start to kill", killReq.transId); - - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_KILL_TRANS) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_KILL_TRANS) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 03c9647bfe..95e011db28 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -295,7 +295,7 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate tstrncpy(userObj.acct, acct, TSDB_USER_LEN); userObj.createdTime = taosGetTimestampMs(); userObj.updateTime = userObj.createdTime; - userObj.superUser = 0;//pCreate->superUser; + userObj.superUser = 0; // pCreate->superUser; userObj.sysInfo = pCreate->sysInfo; userObj.enable = pCreate->enable; @@ -337,6 +337,9 @@ static int32_t mndProcessCreateUserReq(SRpcMsg *pReq) { } mDebug("user:%s, start to create", createReq.user); + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_USER) != 0) { + goto _OVER; + } if (createReq.user[0] == 0) { terrno = TSDB_CODE_MND_INVALID_USER_FORMAT; @@ -360,10 +363,6 @@ static int32_t mndProcessCreateUserReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_USER) != 0) { - goto _OVER; - } - code = mndCreateUser(pMnode, pOperUser->acct, &createReq, pReq); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; @@ -466,7 +465,7 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckAlterUserAuth(pOperUser, pUser, &alterReq) != 0) { + if (mndCheckAlterUserPrivilege(pOperUser, pUser, &alterReq) != 0) { goto _OVER; } @@ -631,6 +630,9 @@ static int32_t mndProcessDropUserReq(SRpcMsg *pReq) { } mDebug("user:%s, start to drop", dropReq.user); + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_USER) != 0) { + goto _OVER; + } if (dropReq.user[0] == 0) { terrno = TSDB_CODE_MND_INVALID_USER_FORMAT; @@ -643,10 +645,6 @@ static int32_t mndProcessDropUserReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_USER) != 0) { - goto _OVER; - } - code = mndDropUser(pMnode, pReq, pUser); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 1c5d73031f..ccdeb2626e 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -1212,7 +1212,7 @@ static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq) { } mInfo("vgId:%d, start to redistribute vgroup to dnode %d:%d:%d", req.vgId, req.dnodeId1, req.dnodeId2, req.dnodeId3); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_REDISTRIBUTE_VGROUP) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_REDISTRIBUTE_VGROUP) != 0) { goto _OVER; } @@ -1507,7 +1507,7 @@ static int32_t mndProcessSplitVgroupMsg(SRpcMsg *pReq) { SDbObj *pDb = NULL; mDebug("vgId:%d, start to split", vgId); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_SPLIT_VGROUP) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_SPLIT_VGROUP) != 0) { goto _OVER; } @@ -1657,7 +1657,7 @@ static int32_t mndProcessBalanceVgroupMsg(SRpcMsg *pReq) { } mInfo("start to balance vgroup"); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_BALANCE_VGROUP) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_BALANCE_VGROUP) != 0) { goto _OVER; } diff --git a/tests/script/tsim/user/privilege_sysinfo.sim b/tests/script/tsim/user/privilege_sysinfo.sim index 9ddfce8a97..ea3294765c 100644 --- a/tests/script/tsim/user/privilege_sysinfo.sim +++ b/tests/script/tsim/user/privilege_sysinfo.sim @@ -22,5 +22,26 @@ sql_error drop user sysinfo1 sql_error alter user sysinfo1 pass '1' sql_error alter user sysinfo0 pass '1' +sql_error create dnode $hostname port 7200 +sql_error drop dnode 1 + +sql_error create qnode on dnode 1 +sql_error drop qnode on dnode 1 + +sql_error create mnode on dnode 1 +sql_error drop mnode on dnode 1 + +sql_error create snode on dnode 1 +sql_error drop snode on dnode 1 + +sql_error redistribute vgroup 2 dnode 1 dnode 2 +sql_error balance vgroup + +sql_error kill transaction 1 +sql_error kill connection 1 +sql_error kill query 1 + +print =============== check db +sql_error create database db system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file From 16ee8eeb2812d2eb134dcf9bf1dbffcfd70f07e2 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sat, 25 Jun 2022 10:29:44 +0800 Subject: [PATCH 21/72] feat: support partition by expression and aggregate function output together --- source/libs/planner/src/planPhysiCreater.c | 4 ++-- source/libs/planner/test/planDistinctTest.cpp | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index ec6181a7d4..4f46abd8da 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -348,8 +348,8 @@ static SPhysiNode* makePhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode return NULL; } - pPhysiNode->pLimit = pLogicNode->pLimit; - pPhysiNode->pSlimit = pLogicNode->pSlimit; + TSWAP(pPhysiNode->pLimit, pLogicNode->pLimit); + TSWAP(pPhysiNode->pSlimit, pLogicNode->pSlimit); int32_t code = createDataBlockDesc(pCxt, pLogicNode->pTargets, &pPhysiNode->pOutputDataBlockDesc); if (TSDB_CODE_SUCCESS != code) { diff --git a/source/libs/planner/test/planDistinctTest.cpp b/source/libs/planner/test/planDistinctTest.cpp index 58473dcff2..8caee01d35 100644 --- a/source/libs/planner/test/planDistinctTest.cpp +++ b/source/libs/planner/test/planDistinctTest.cpp @@ -40,3 +40,9 @@ TEST_F(PlanDistinctTest, withOrderBy) { run("select distinct c1 + 10 a from t1 order by a"); } + +TEST_F(PlanDistinctTest, withLimit) { + useDb("root", "test"); + + run("SELECT DISTINCT c1 FROM t1 LIMIT 3"); +} From aa8957bcc54b3a2d775cea36ddf3f19cfdc15ae3 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 25 Jun 2022 11:02:49 +0800 Subject: [PATCH 22/72] handle redirect --- source/libs/transport/src/transCli.c | 16 ++++++++-------- source/libs/transport/src/transSvr.c | 1 + 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 9ca5e8e73e..0671aa39d1 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -513,7 +513,7 @@ static void allocConnRef(SCliConn* conn, bool update) { } static void addConnToPool(void* pool, SCliConn* conn) { if (conn->status == ConnInPool) { - assert(0); + // assert(0); return; } SCliThrd* thrd = conn->hostThrd; @@ -986,13 +986,13 @@ static void doDelayTask(void* param) { } static void cliSchedMsgToNextNode(SCliMsg* pMsg, SCliThrd* pThrd) { - STraceId* trace = &pMsg->msg.info.traceId; STransConnCtx* pCtx = pMsg->ctx; - char tbuf[256] = {0}; + STraceId* trace = &pMsg->msg.info.traceId; + char tbuf[256] = {0}; EPSET_DEBUG_STR(&pCtx->epSet, tbuf); - tGTrace("%s retry to send msg to next node %dms later , use %s, retryCnt:%d, limit:%d", transLabel(pThrd->pTransInst), - TRANS_RETRY_INTERVAL, tbuf, pCtx->retryCnt + 1, pCtx->retryLimit); + tGTrace("%s retry on next node, use %s, retryCnt:%d, limit:%d", transLabel(pThrd->pTransInst), tbuf, + pCtx->retryCnt + 1, pCtx->retryLimit); STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); arg->param1 = pMsg; @@ -1000,7 +1000,7 @@ static void cliSchedMsgToNextNode(SCliMsg* pMsg, SCliThrd* pThrd) { transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL); } -void cliUpdateRetryLimit(int8_t* val, int8_t exp, int8_t newVal) { +void cliCompareAndSwap(int8_t* val, int8_t exp, int8_t newVal) { if (*val != exp) { *val = newVal; } @@ -1025,7 +1025,7 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { pMsg->sent = 0; pCtx->retryCnt += 1; if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { - cliUpdateRetryLimit(&pCtx->retryLimit, TRANS_RETRY_COUNT_LIMIT, EPSET_GET_SIZE(&pCtx->epSet) * 3); + cliCompareAndSwap(&pCtx->retryLimit, TRANS_RETRY_COUNT_LIMIT, EPSET_GET_SIZE(&pCtx->epSet) * 3); if (pCtx->retryCnt < pCtx->retryLimit) { transUnrefCliHandle(pConn); EPSET_FORWARD_INUSE(&pCtx->epSet); @@ -1033,7 +1033,7 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { return -1; } } else { - cliUpdateRetryLimit(&pCtx->retryLimit, TRANS_RETRY_COUNT_LIMIT, TRANS_RETRY_COUNT_LIMIT); + cliCompareAndSwap(&pCtx->retryLimit, TRANS_RETRY_COUNT_LIMIT, TRANS_RETRY_COUNT_LIMIT); if (pCtx->retryCnt < pCtx->retryLimit) { addConnToPool(pThrd->pool, pConn); if (pResp->contLen == 0) { diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 215323f69d..7651860224 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -422,6 +422,7 @@ static void uvPrepareSendData(SSvrMsg* smsg, uv_buf_t* wb) { transUnrefSrvHandle(pConn); } else { pHead->msgType = pMsg->msgType; + if (pHead->msgType == 0) pHead->msgType = pConn->inType + 1; } } From 23653fd8430f8221e642d8bf7a15178603376ff0 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 25 Jun 2022 11:22:34 +0800 Subject: [PATCH 23/72] enh: privilege for topic and stream --- source/dnode/mnode/impl/inc/mndPrivilege.h | 1 + source/dnode/mnode/impl/src/mndConsumer.c | 4 +++ source/dnode/mnode/impl/src/mndPrivilege.c | 20 ++++++----- source/dnode/mnode/impl/src/mndStream.c | 35 +++++++------------- source/dnode/mnode/impl/src/mndTopic.c | 8 +++-- tests/script/tsim/user/privilege_sysinfo.sim | 3 -- 6 files changed, 34 insertions(+), 37 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndPrivilege.h b/source/dnode/mnode/impl/inc/mndPrivilege.h index 15f9e4e6b5..a1bec69790 100644 --- a/source/dnode/mnode/impl/inc/mndPrivilege.h +++ b/source/dnode/mnode/impl/inc/mndPrivilege.h @@ -64,6 +64,7 @@ void mndCleanupPrivilege(SMnode *pMnode); int32_t mndCheckOperPrivilege(SMnode *pMnode, const char *user, EOperType operType); int32_t mndCheckDbPrivilege(SMnode *pMnode, const char *user, EOperType operType, SDbObj *pDb); +int32_t mndCheckDbPrivilegeByName(SMnode *pMnode, const char *user, EOperType operType, const char *name); int32_t mndCheckShowPrivilege(SMnode *pMnode, const char *user, int32_t showType); int32_t mndCheckAlterUserPrivilege(SUserObj *pOperUser, SUserObj *pUser, SAlterUserReq *pAlter); diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 7dc5ee1ea1..69f58f33cf 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -431,6 +431,10 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { goto SUBSCRIBE_OVER; } + if (mndCheckDbPrivilege(pMnode, pMsg->info.conn.user, MND_OPER_READ_DB, pTopic->db) != 0) { + goto SUBSCRIBE_OVER; + } + #if 0 // ref topic to prevent drop // TODO make topic complete diff --git a/source/dnode/mnode/impl/src/mndPrivilege.c b/source/dnode/mnode/impl/src/mndPrivilege.c index 478ba2bee4..752b11540d 100644 --- a/source/dnode/mnode/impl/src/mndPrivilege.c +++ b/source/dnode/mnode/impl/src/mndPrivilege.c @@ -16,6 +16,7 @@ #define _DEFAULT_SOURCE #include "mndPrivilege.h" #include "mndUser.h" +#include "mndDb.h" int32_t mndInitPrivilege(SMnode *pMnode) { return 0; } @@ -133,15 +134,7 @@ int32_t mndCheckDbPrivilege(SMnode *pMnode, const char *user, EOperType operType if (pUser->sysInfo) goto _OVER; } - if (operType == MND_OPER_ALTER_DB) { - if (strcmp(pUser->user, pDb->createUser) == 0 && pUser->sysInfo) goto _OVER; - } - - if (operType == MND_OPER_DROP_DB) { - if (strcmp(pUser->user, pDb->createUser) == 0 && pUser->sysInfo) goto _OVER; - } - - if (operType == MND_OPER_COMPACT_DB) { + if (operType == MND_OPER_ALTER_DB || operType == MND_OPER_DROP_DB || operType == MND_OPER_COMPACT_DB) { if (strcmp(pUser->user, pDb->createUser) == 0 && pUser->sysInfo) goto _OVER; } @@ -168,3 +161,12 @@ _OVER: mndReleaseUser(pMnode, pUser); return code; } + +int32_t mndCheckDbPrivilegeByName(SMnode *pMnode, const char *user, EOperType operType, const char *name) { + SDbObj *pDb = mndAcquireDb(pMnode, name); + if (pDb == NULL) return -1; + + int32_t code = mndCheckDbPrivilege(pMnode, user, operType, pDb); + mndReleaseDb(pMnode, pDb); + return code; +} \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index e49756c837..c45421baa9 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -437,10 +437,6 @@ static int32_t mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStre goto _OVER; } - if (mndCheckDbPrivilege(pMnode, user, MND_OPER_WRITE_DB, pDb) != 0) { - goto _OVER; - } - int32_t numOfStbs = -1; if (mndGetNumOfStbs(pMnode, pDb->name, &numOfStbs) != 0) { goto _OVER; @@ -542,19 +538,6 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { goto _OVER; } - // TODO check read auth for source and write auth for target -#if 0 - pDb = mndAcquireDb(pMnode, createStreamReq.sourceDB); - if (pDb == NULL) { - terrno = TSDB_CODE_MND_DB_NOT_SELECTED; - goto _OVER; - } - - if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { - goto _OVER; - } -#endif - // build stream obj from request SStreamObj streamObj = {0}; if (mndBuildStreamObjFromCreateReq(pMnode, &streamObj, &createStreamReq) < 0) { @@ -592,6 +575,16 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { goto _OVER; } + if (mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_READ_DB, streamObj.sourceDb) != 0) { + mndTransDrop(pTrans); + goto _OVER; + } + + if (mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, streamObj.targetDb) != 0) { + mndTransDrop(pTrans); + goto _OVER; + } + // execute creation if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); @@ -641,13 +634,9 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { } } -#if 0 - // todo check auth - pUser = mndAcquireUser(pMnode, pReq->info.conn.user); - if (pUser == NULL) { - goto DROP_STREAM_OVER; + if (mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pStream->targetDb) != 0) { + return -1; } -#endif STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq); if (pTrans == NULL) { diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index b8c17378c4..90c4e96517 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -480,7 +480,7 @@ static int32_t mndProcessCreateTopicReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_READ_DB, pDb) != 0) { goto _OVER; } @@ -571,6 +571,10 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { } #endif + if (mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_READ_DB, pTopic->db) != 0) { + return -1; + } + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pReq); mndTransSetDbName(pTrans, pTopic->db, NULL); if (pTrans == NULL) { @@ -579,7 +583,7 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { } mDebug("trans:%d, used to drop topic:%s", pTrans->id, pTopic->name); - + if (mndDropOffsetByTopic(pMnode, pTrans, dropReq.name) < 0) { ASSERT(0); return -1; diff --git a/tests/script/tsim/user/privilege_sysinfo.sim b/tests/script/tsim/user/privilege_sysinfo.sim index ea3294765c..35760d45fd 100644 --- a/tests/script/tsim/user/privilege_sysinfo.sim +++ b/tests/script/tsim/user/privilege_sysinfo.sim @@ -13,9 +13,6 @@ print user sysinfo0 login sql close sql connect sysinfo0 -system sh/exec.sh -n dnode1 -s stop -return - print =============== check oper sql_error create user u1 pass 'u1' sql_error drop user sysinfo1 From 43f1d411220ad559ab1c795893e24c3b3c10b504 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 25 Jun 2022 11:23:44 +0800 Subject: [PATCH 24/72] fix: compile error --- source/dnode/mnode/impl/src/mndConsumer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 69f58f33cf..5b5de10fba 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -431,7 +431,7 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { goto SUBSCRIBE_OVER; } - if (mndCheckDbPrivilege(pMnode, pMsg->info.conn.user, MND_OPER_READ_DB, pTopic->db) != 0) { + if (mndCheckDbPrivilegeByName(pMnode, pMsg->info.conn.user, MND_OPER_READ_DB, pTopic->db) != 0) { goto SUBSCRIBE_OVER; } From 0df1415caf617f63e55d255a06f777125cf2ad63 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sat, 25 Jun 2022 11:30:34 +0800 Subject: [PATCH 25/72] refactor(sync): add trace log --- source/libs/sync/src/syncAppendEntries.c | 276 ++++++++++++++++++ source/libs/sync/src/syncAppendEntriesReply.c | 73 +++++ source/libs/sync/src/syncRequestVote.c | 70 +++++ source/libs/sync/src/syncRequestVoteReply.c | 67 +++++ 4 files changed, 486 insertions(+) diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 47d0976aca..ba3dd66550 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -89,6 +89,280 @@ // /\ UNCHANGED <> // +int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { + int32_t ret = 0; + + // print log + syncAppendEntriesLog2("==syncNodeOnAppendEntriesCb==", pMsg); + + // if already drop replica, do not process + if (!syncNodeInRaftGroup(ths, &(pMsg->srcId)) && !ths->pRaftCfg->isStandBy) { + syncNodeEventLog(ths, "recv sync-append-entries, maybe replica already dropped"); + return ret; + } + + // maybe update term + if (pMsg->term > ths->pRaftStore->currentTerm) { + syncNodeUpdateTerm(ths, pMsg->term); + } + ASSERT(pMsg->term <= ths->pRaftStore->currentTerm); + + // reset elect timer + if (pMsg->term == ths->pRaftStore->currentTerm) { + ths->leaderCache = pMsg->srcId; + syncNodeResetElectTimer(ths); + } + ASSERT(pMsg->dataLen >= 0); + + do { + // return to follower state + if (pMsg->term == ths->pRaftStore->currentTerm && ths->state == TAOS_SYNC_STATE_CANDIDATE) { + syncNodeEventLog(ths, "recv sync-append-entries, candidate to follower"); + + syncNodeBecomeFollower(ths, "from candidate by append entries"); + + // ret or reply? + return ret; + } + } while (0); + + SyncTerm localPreLogTerm = 0; + if (pMsg->prevLogIndex >= SYNC_INDEX_BEGIN && pMsg->prevLogIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)) { + SSyncRaftEntry* pEntry = ths->pLogStore->getEntry(ths->pLogStore, pMsg->prevLogIndex); + if (pEntry == NULL) { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "getEntry error, index:%ld, since %s", pMsg->prevLogIndex, terrstr()); + syncNodeErrorLog(ths, logBuf); + return -1; + } + + localPreLogTerm = pEntry->term; + syncEntryDestory(pEntry); + } + + bool logOK = + (pMsg->prevLogIndex == SYNC_INDEX_INVALID) || + ((pMsg->prevLogIndex >= SYNC_INDEX_BEGIN) && + (pMsg->prevLogIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)) && (pMsg->prevLogTerm == localPreLogTerm)); + + // reject request + if ((pMsg->term < ths->pRaftStore->currentTerm) || + ((pMsg->term == ths->pRaftStore->currentTerm) && (ths->state == TAOS_SYNC_STATE_FOLLOWER) && !logOK)) { + do { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries, reject, pre-index:%ld, pre-term:%lu, datalen:%d", + pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen); + syncNodeEventLog(ths, logBuf); + } while (0); + + SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId); + pReply->srcId = ths->myRaftId; + pReply->destId = pMsg->srcId; + pReply->term = ths->pRaftStore->currentTerm; + pReply->success = false; + pReply->matchIndex = SYNC_INDEX_INVALID; + + SRpcMsg rpcMsg; + syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); + syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); + syncAppendEntriesReplyDestroy(pReply); + + return ret; + } + + // accept request + if (pMsg->term == ths->pRaftStore->currentTerm && ths->state == TAOS_SYNC_STATE_FOLLOWER && logOK) { + // preIndex = -1, or has preIndex entry in local log + ASSERT(pMsg->prevLogIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)); + + // has extra entries (> preIndex) in local log + bool hasExtraEntries = pMsg->prevLogIndex < ths->pLogStore->getLastIndex(ths->pLogStore); + + // has entries in SyncAppendEntries msg + bool hasAppendEntries = pMsg->dataLen > 0; + + do { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries, accept, pre-index:%ld, pre-term:%lu, datalen:%d", + pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen); + syncNodeEventLog(ths, logBuf); + } while (0); + + if (hasExtraEntries && hasAppendEntries) { + // not conflict by default + bool conflict = false; + + SyncIndex extraIndex = pMsg->prevLogIndex + 1; + SSyncRaftEntry* pExtraEntry = ths->pLogStore->getEntry(ths->pLogStore, extraIndex); + if (pExtraEntry == NULL) { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "getEntry error2, index:%ld, since %s", extraIndex, terrstr()); + syncNodeErrorLog(ths, logBuf); + return -1; + } + + SSyncRaftEntry* pAppendEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen); + if (pAppendEntry == NULL) { + syncNodeErrorLog(ths, "syncEntryDeserialize pAppendEntry error"); + return -1; + } + + // log not match, conflict + ASSERT(extraIndex == pAppendEntry->index); + if (pExtraEntry->term != pAppendEntry->term) { + conflict = true; + } + + if (conflict) { + // roll back + SyncIndex delBegin = ths->pLogStore->getLastIndex(ths->pLogStore); + SyncIndex delEnd = extraIndex; + + sTrace("syncNodeOnAppendEntriesCb --> conflict:%d, delBegin:%ld, delEnd:%ld", conflict, delBegin, delEnd); + + // notice! reverse roll back! + for (SyncIndex index = delEnd; index >= delBegin; --index) { + if (ths->pFsm->FpRollBackCb != NULL) { + SSyncRaftEntry* pRollBackEntry = ths->pLogStore->getEntry(ths->pLogStore, index); + if (pRollBackEntry == NULL) { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "getEntry error3, index:%ld, since %s", index, terrstr()); + syncNodeErrorLog(ths, logBuf); + return -1; + } + + // if (pRollBackEntry->msgType != TDMT_SYNC_NOOP) { + if (syncUtilUserRollback(pRollBackEntry->msgType)) { + SRpcMsg rpcMsg; + syncEntry2OriginalRpc(pRollBackEntry, &rpcMsg); + + SFsmCbMeta cbMeta = {0}; + cbMeta.index = pRollBackEntry->index; + cbMeta.lastConfigIndex = syncNodeGetSnapshotConfigIndex(ths, cbMeta.index); + cbMeta.isWeak = pRollBackEntry->isWeak; + cbMeta.code = 0; + cbMeta.state = ths->state; + cbMeta.seqNum = pRollBackEntry->seqNum; + ths->pFsm->FpRollBackCb(ths->pFsm, &rpcMsg, cbMeta); + rpcFreeCont(rpcMsg.pCont); + } + + syncEntryDestory(pRollBackEntry); + } + } + + // delete confict entries + ths->pLogStore->truncate(ths->pLogStore, extraIndex); + + // append new entries + ths->pLogStore->appendEntry(ths->pLogStore, pAppendEntry); + + // pre commit + SRpcMsg rpcMsg; + syncEntry2OriginalRpc(pAppendEntry, &rpcMsg); + if (ths->pFsm != NULL) { + // if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->originalRpcType != TDMT_SYNC_NOOP) { + if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pAppendEntry->originalRpcType)) { + SFsmCbMeta cbMeta = {0}; + cbMeta.index = pAppendEntry->index; + cbMeta.lastConfigIndex = syncNodeGetSnapshotConfigIndex(ths, cbMeta.index); + cbMeta.isWeak = pAppendEntry->isWeak; + cbMeta.code = 2; + cbMeta.state = ths->state; + cbMeta.seqNum = pAppendEntry->seqNum; + ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, cbMeta); + } + } + rpcFreeCont(rpcMsg.pCont); + } + + // free memory + syncEntryDestory(pExtraEntry); + syncEntryDestory(pAppendEntry); + + } else if (hasExtraEntries && !hasAppendEntries) { + // do nothing + + } else if (!hasExtraEntries && hasAppendEntries) { + SSyncRaftEntry* pAppendEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen); + if (pAppendEntry == NULL) { + syncNodeErrorLog(ths, "syncEntryDeserialize pAppendEntry2 error"); + return -1; + } + + // append new entries + ths->pLogStore->appendEntry(ths->pLogStore, pAppendEntry); + + // pre commit + SRpcMsg rpcMsg; + syncEntry2OriginalRpc(pAppendEntry, &rpcMsg); + if (ths->pFsm != NULL) { + // if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->originalRpcType != TDMT_SYNC_NOOP) { + if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pAppendEntry->originalRpcType)) { + SFsmCbMeta cbMeta = {0}; + cbMeta.index = pAppendEntry->index; + cbMeta.lastConfigIndex = syncNodeGetSnapshotConfigIndex(ths, cbMeta.index); + cbMeta.isWeak = pAppendEntry->isWeak; + cbMeta.code = 3; + cbMeta.state = ths->state; + cbMeta.seqNum = pAppendEntry->seqNum; + ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, cbMeta); + } + } + rpcFreeCont(rpcMsg.pCont); + + // free memory + syncEntryDestory(pAppendEntry); + + } else if (!hasExtraEntries && !hasAppendEntries) { + // do nothing + + } else { + syncNodeLog3("", ths); + ASSERT(0); + } + + SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId); + pReply->srcId = ths->myRaftId; + pReply->destId = pMsg->srcId; + pReply->term = ths->pRaftStore->currentTerm; + pReply->success = true; + + if (hasAppendEntries) { + pReply->matchIndex = pMsg->prevLogIndex + 1; + } else { + pReply->matchIndex = pMsg->prevLogIndex; + } + + SRpcMsg rpcMsg; + syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); + syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); + syncAppendEntriesReplyDestroy(pReply); + + // maybe update commit index from leader + if (pMsg->commitIndex > ths->commitIndex) { + // has commit entry in local + if (pMsg->commitIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)) { + SyncIndex beginIndex = ths->commitIndex + 1; + SyncIndex endIndex = pMsg->commitIndex; + + // update commit index + ths->commitIndex = pMsg->commitIndex; + + // call back Wal + ths->pLogStore->updateCommitIndex(ths->pLogStore, ths->commitIndex); + + int32_t code = syncNodeCommit(ths, beginIndex, endIndex, ths->state); + ASSERT(code == 0); + } + } + } + + return ret; +} + +#if 0 + int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { int32_t ret = 0; @@ -352,6 +626,8 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { return ret; } +#endif + static int32_t syncNodeMakeLogSame(SSyncNode* ths, SyncAppendEntries* pMsg) { int32_t code; diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index 6ca356b4f6..b7122b9c52 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -40,6 +40,78 @@ int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg) { int32_t ret = 0; + // print log + syncAppendEntriesReplyLog2("==syncNodeOnAppendEntriesReplyCb==", pMsg); + + // if already drop replica, do not process + if (!syncNodeInRaftGroup(ths, &(pMsg->srcId)) && !ths->pRaftCfg->isStandBy) { + syncNodeEventLog(ths, "recv sync-append-entries-reply, maybe replica already dropped"); + return 0; + } + + // drop stale response + if (pMsg->term < ths->pRaftStore->currentTerm) { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries-reply, recv-term:%lu, drop stale response", pMsg->term); + syncNodeEventLog(ths, logBuf); + return 0; + } + + if (gRaftDetailLog) { + syncNodeEventLog(ths, "recv sync-append-entries-reply, before"); + } + syncIndexMgrLog2("==syncNodeOnAppendEntriesReplyCb== before pNextIndex", ths->pNextIndex); + syncIndexMgrLog2("==syncNodeOnAppendEntriesReplyCb== before pMatchIndex", ths->pMatchIndex); + + // no need this code, because if I receive reply.term, then I must have sent for that term. + // if (pMsg->term > ths->pRaftStore->currentTerm) { + // syncNodeUpdateTerm(ths, pMsg->term); + // } + + if (pMsg->term > ths->pRaftStore->currentTerm) { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries-reply, error term, recv-term:%lu", pMsg->term); + syncNodeErrorLog(ths, logBuf); + return -1; + } + + ASSERT(pMsg->term == ths->pRaftStore->currentTerm); + + if (pMsg->success) { + // nextIndex' = [nextIndex EXCEPT ![i][j] = m.mmatchIndex + 1] + syncIndexMgrSetIndex(ths->pNextIndex, &(pMsg->srcId), pMsg->matchIndex + 1); + + // matchIndex' = [matchIndex EXCEPT ![i][j] = m.mmatchIndex] + syncIndexMgrSetIndex(ths->pMatchIndex, &(pMsg->srcId), pMsg->matchIndex); + + // maybe commit + syncMaybeAdvanceCommitIndex(ths); + + } else { + SyncIndex nextIndex = syncIndexMgrGetIndex(ths->pNextIndex, &(pMsg->srcId)); + + // notice! int64, uint64 + if (nextIndex > SYNC_INDEX_BEGIN) { + --nextIndex; + } else { + nextIndex = SYNC_INDEX_BEGIN; + } + syncIndexMgrSetIndex(ths->pNextIndex, &(pMsg->srcId), nextIndex); + } + + if (gRaftDetailLog) { + syncNodeEventLog(ths, "recv sync-append-entries-reply, after"); + } + syncIndexMgrLog2("==syncNodeOnAppendEntriesReplyCb== after pNextIndex", ths->pNextIndex); + syncIndexMgrLog2("==syncNodeOnAppendEntriesReplyCb== after pMatchIndex", ths->pMatchIndex); + + return ret; +} + +#if 0 +int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg) { + int32_t ret = 0; + char logBuf[128] = {0}; snprintf(logBuf, sizeof(logBuf), "==syncNodeOnAppendEntriesReplyCb== term:%lu", ths->pRaftStore->currentTerm); syncAppendEntriesReplyLog2(logBuf, pMsg); @@ -96,6 +168,7 @@ int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* p return ret; } +#endif int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg) { int32_t ret = 0; diff --git a/source/libs/sync/src/syncRequestVote.c b/source/libs/sync/src/syncRequestVote.c index 3270b5c0e4..95dec6cb83 100644 --- a/source/libs/sync/src/syncRequestVote.c +++ b/source/libs/sync/src/syncRequestVote.c @@ -45,6 +45,75 @@ int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg) { int32_t ret = 0; + syncRequestVoteLog2("==syncNodeOnRequestVoteCb==", pMsg); + + // if already drop replica, do not process + if (!syncNodeInRaftGroup(ths, &(pMsg->srcId)) && !ths->pRaftCfg->isStandBy) { + do { + char logBuf[256]; + char host[64]; + uint16_t port; + syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); + snprintf(logBuf, sizeof(logBuf), + "recv sync-request-vote from %s:%d, term:%lu, lindex:%ld, lterm:%lu, maybe replica already dropped", + host, port, pMsg->term, pMsg->lastLogIndex, pMsg->lastLogTerm); + syncNodeEventLog(ths, logBuf); + } while (0); + + return -1; + } + + // maybe update term + if (pMsg->term > ths->pRaftStore->currentTerm) { + syncNodeUpdateTerm(ths, pMsg->term); + } + ASSERT(pMsg->term <= ths->pRaftStore->currentTerm); + + bool logOK = (pMsg->lastLogTerm > ths->pLogStore->getLastTerm(ths->pLogStore)) || + ((pMsg->lastLogTerm == ths->pLogStore->getLastTerm(ths->pLogStore)) && + (pMsg->lastLogIndex >= ths->pLogStore->getLastIndex(ths->pLogStore))); + bool grant = (pMsg->term == ths->pRaftStore->currentTerm) && logOK && + ((!raftStoreHasVoted(ths->pRaftStore)) || (syncUtilSameId(&(ths->pRaftStore->voteFor), &(pMsg->srcId)))); + if (grant) { + // maybe has already voted for pMsg->srcId + // vote again, no harm + raftStoreVote(ths->pRaftStore, &(pMsg->srcId)); + + // forbid elect for this round + syncNodeResetElectTimer(ths); + } + + // send msg + SyncRequestVoteReply* pReply = syncRequestVoteReplyBuild(ths->vgId); + pReply->srcId = ths->myRaftId; + pReply->destId = pMsg->srcId; + pReply->term = ths->pRaftStore->currentTerm; + pReply->voteGranted = grant; + + // trace log + do { + char logBuf[256]; + char host[64]; + uint16_t port; + syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); + snprintf(logBuf, sizeof(logBuf), + "recv sync-request-vote from %s:%d, term:%lu, lindex:%ld, lterm:%lu, reply-grant:%d", host, port, + pMsg->term, pMsg->lastLogIndex, pMsg->lastLogTerm, pReply->voteGranted); + syncNodeEventLog(ths, logBuf); + } while (0); + + SRpcMsg rpcMsg; + syncRequestVoteReply2RpcMsg(pReply, &rpcMsg); + syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); + syncRequestVoteReplyDestroy(pReply); + + return ret; +} + +#if 0 +int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg) { + int32_t ret = 0; + char logBuf[128] = {0}; snprintf(logBuf, sizeof(logBuf), "==syncNodeOnRequestVoteCb== term:%lu", ths->pRaftStore->currentTerm); syncRequestVoteLog2(logBuf, pMsg); @@ -81,6 +150,7 @@ int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg) { return ret; } +#endif static bool syncNodeOnRequestVoteLogOK(SSyncNode* pSyncNode, SyncRequestVote* pMsg) { SyncTerm myLastTerm = syncNodeGetLastTerm(pSyncNode); diff --git a/source/libs/sync/src/syncRequestVoteReply.c b/source/libs/sync/src/syncRequestVoteReply.c index 60963d0dd3..4c205d16ec 100644 --- a/source/libs/sync/src/syncRequestVoteReply.c +++ b/source/libs/sync/src/syncRequestVoteReply.c @@ -40,6 +40,72 @@ int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg) { int32_t ret = 0; + // print log + char logBuf[128] = {0}; + snprintf(logBuf, sizeof(logBuf), "==syncNodeOnRequestVoteReplyCb== term:%lu", ths->pRaftStore->currentTerm); + syncRequestVoteReplyLog2(logBuf, pMsg); + + // if already drop replica, do not process + if (!syncNodeInRaftGroup(ths, &(pMsg->srcId)) && !ths->pRaftCfg->isStandBy) { + sInfo("recv SyncRequestVoteReply, maybe replica already dropped"); + return ret; + } + + // drop stale response + if (pMsg->term < ths->pRaftStore->currentTerm) { + sTrace("recv SyncRequestVoteReply, drop stale response, receive_term:%lu current_term:%lu", pMsg->term, + ths->pRaftStore->currentTerm); + return ret; + } + + // ASSERT(!(pMsg->term > ths->pRaftStore->currentTerm)); + // no need this code, because if I receive reply.term, then I must have sent for that term. + // if (pMsg->term > ths->pRaftStore->currentTerm) { + // syncNodeUpdateTerm(ths, pMsg->term); + // } + + if (pMsg->term > ths->pRaftStore->currentTerm) { + char logBuf[128] = {0}; + snprintf(logBuf, sizeof(logBuf), "syncNodeOnRequestVoteReplyCb error term, receive:%lu current:%lu", pMsg->term, + ths->pRaftStore->currentTerm); + syncNodePrint2(logBuf, ths); + sError("%s", logBuf); + return ret; + } + + ASSERT(pMsg->term == ths->pRaftStore->currentTerm); + + // This tallies votes even when the current state is not Candidate, + // but they won't be looked at, so it doesn't matter. + if (ths->state == TAOS_SYNC_STATE_CANDIDATE) { + votesRespondAdd(ths->pVotesRespond, pMsg); + if (pMsg->voteGranted) { + // add vote + voteGrantedVote(ths->pVotesGranted, pMsg); + + // maybe to leader + if (voteGrantedMajority(ths->pVotesGranted)) { + if (!ths->pVotesGranted->toLeader) { + syncNodeCandidate2Leader(ths); + + // prevent to leader again! + ths->pVotesGranted->toLeader = true; + } + } + } else { + ; + // do nothing + // UNCHANGED <> + } + } + + return ret; +} + +#if 0 +int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg) { + int32_t ret = 0; + char logBuf[128] = {0}; snprintf(logBuf, sizeof(logBuf), "==syncNodeOnRequestVoteReplyCb== term:%lu", ths->pRaftStore->currentTerm); syncRequestVoteReplyLog2(logBuf, pMsg); @@ -93,6 +159,7 @@ int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg) return ret; } +#endif int32_t syncNodeOnRequestVoteReplySnapshotCb(SSyncNode* ths, SyncRequestVoteReply* pMsg) { int32_t ret = 0; From d08835d51821a31eee196b8c16562a5201714580 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 25 Jun 2022 12:03:15 +0800 Subject: [PATCH 26/72] fix: privilege for create topic --- source/dnode/mnode/impl/src/mndTopic.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 078bbc1db7..f881f237c2 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -14,12 +14,12 @@ */ #include "mndTopic.h" -#include "mndPrivilege.h" #include "mndConsumer.h" #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" #include "mndOffset.h" +#include "mndPrivilege.h" #include "mndShow.h" #include "mndStb.h" #include "mndSubscribe.h" @@ -480,6 +480,11 @@ static int32_t mndProcessCreateTopicReq(SRpcMsg *pReq) { goto _OVER; } + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_READ_DB, pDb) != 0) { + goto _OVER; + } + + code = mndCreateTopic(pMnode, pReq, &createTopicReq, pDb); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: @@ -578,7 +583,7 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { } mDebug("trans:%d, used to drop topic:%s", pTrans->id, pTopic->name); - + if (mndDropOffsetByTopic(pMnode, pTrans, dropReq.name) < 0) { ASSERT(0); return -1; From 4a7938e9fffe1c8466a7c84dd059ef0c9c2ee043 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sat, 25 Jun 2022 12:05:17 +0800 Subject: [PATCH 27/72] cast support nchar->binary, binary/nchar->timestamp --- source/libs/scalar/src/sclfunc.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index a3cd1aba6d..e8f48cd0fa 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -881,8 +881,8 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp } case TSDB_DATA_TYPE_TIMESTAMP: { if (inputType == TSDB_DATA_TYPE_BINARY || inputType == TSDB_DATA_TYPE_NCHAR) { - //not support - return TSDB_CODE_FAILED; + //convert to 0 + *(int64_t *)output = 0; } else { GET_TYPED_DATA(*(int64_t *)output, int64_t, inputType, input); } @@ -897,8 +897,16 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp len = sprintf(varDataVal(output), "%.*s", len, varDataVal(input)); varDataSetLen(output, len); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { - //not support - return TSDB_CODE_FAILED; + char *newBuf = taosMemoryCalloc(1, inputLen); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + if (len < 0) { + taosMemoryFree(newBuf); + return TSDB_CODE_FAILED; + } + len = TMIN(len, outputLen - VARSTR_HEADER_SIZE); + memcpy(varDataVal(output), newBuf, len); + varDataSetLen(output, len); + taosMemoryFree(newBuf); } else { char tmp[400] = {0}; NUM_TO_STRING(inputType, input, sizeof(tmp), tmp); From cf4fcb739887ffdf0935277d53e9ffb876d48590 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sat, 25 Jun 2022 12:06:50 +0800 Subject: [PATCH 28/72] feat: support partition by expression and aggregate function output together --- source/libs/planner/src/planLogicCreater.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 6d4aa5ae23..a4cdcd35d3 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -450,6 +450,15 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, int32_t code = TSDB_CODE_SUCCESS; // set grouyp keys, agg funcs and having conditions + if (TSDB_CODE_SUCCESS == code && pSelect->hasAggFuncs) { + code = nodesCollectFuncs(pSelect, SQL_CLAUSE_GROUP_BY, fmIsAggFunc, &pAgg->pAggFuncs); + } + + // rewrite the expression in subsequent clauses + if (TSDB_CODE_SUCCESS == code) { + code = rewriteExprsForSelect(pAgg->pAggFuncs, pSelect, SQL_CLAUSE_GROUP_BY); + } + if (NULL != pSelect->pGroupByList) { pAgg->pGroupKeys = nodesCloneList(pSelect->pGroupByList); if (NULL == pAgg->pGroupKeys) { @@ -457,20 +466,11 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, } } - if (TSDB_CODE_SUCCESS == code && pSelect->hasAggFuncs) { - code = nodesCollectFuncs(pSelect, SQL_CLAUSE_GROUP_BY, fmIsAggFunc, &pAgg->pAggFuncs); - } - // rewrite the expression in subsequent clauses if (TSDB_CODE_SUCCESS == code) { code = rewriteExprsForSelect(pAgg->pGroupKeys, pSelect, SQL_CLAUSE_GROUP_BY); } - // rewrite the expression in subsequent clauses - if (TSDB_CODE_SUCCESS == code) { - code = rewriteExprsForSelect(pAgg->pAggFuncs, pSelect, SQL_CLAUSE_GROUP_BY); - } - if (TSDB_CODE_SUCCESS == code && NULL != pSelect->pHaving) { pAgg->node.pConditions = nodesCloneNode(pSelect->pHaving); if (NULL == pAgg->node.pConditions) { From 3b9953f483a8ec3794144b8649c3ac63e7986765 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sat, 25 Jun 2022 12:15:17 +0800 Subject: [PATCH 29/72] cast function support ->float, ->double --- source/libs/scalar/src/sclfunc.c | 36 ++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index e8f48cd0fa..4caccfc7f3 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -879,6 +879,42 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp } break; } + case TSDB_DATA_TYPE_FLOAT: { + if (inputType == TSDB_DATA_TYPE_BINARY) { + *(float *)output = taosStr2Float(varDataVal(input), NULL); + } else if (inputType == TSDB_DATA_TYPE_NCHAR) { + char *newBuf = taosMemoryCalloc(1, inputLen); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + if (len < 0) { + taosMemoryFree(newBuf); + return TSDB_CODE_FAILED; + } + newBuf[len] = 0; + *(float *)output = taosStr2Float(newBuf, NULL); + taosMemoryFree(newBuf); + } else { + GET_TYPED_DATA(*(float *)output, float, inputType, input); + } + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + if (inputType == TSDB_DATA_TYPE_BINARY) { + *(double *)output = taosStr2Double(varDataVal(input), NULL); + } else if (inputType == TSDB_DATA_TYPE_NCHAR) { + char *newBuf = taosMemoryCalloc(1, inputLen); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + if (len < 0) { + taosMemoryFree(newBuf); + return TSDB_CODE_FAILED; + } + newBuf[len] = 0; + *(double *)output = taosStr2Double(newBuf, NULL); + taosMemoryFree(newBuf); + } else { + GET_TYPED_DATA(*(double *)output, double, inputType, input); + } + break; + } case TSDB_DATA_TYPE_TIMESTAMP: { if (inputType == TSDB_DATA_TYPE_BINARY || inputType == TSDB_DATA_TYPE_NCHAR) { //convert to 0 From f7b1960725e5900c6551371878fb48a45bfea0da Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Sat, 25 Jun 2022 12:18:47 +0800 Subject: [PATCH 30/72] ci: drop sync windows test with linux --- Jenkinsfile2 | 36 +----------------------------------- 1 file changed, 1 insertion(+), 35 deletions(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index b65576deaf..6b48aabab8 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -4,11 +4,6 @@ import jenkins.model.CauseOfInterruption node { } -win_test_stage = 0 -linux_ready = 0 -linux_node_ip = "" -linux_node_pass = "" - def abortPreviousBuilds() { def currentJobName = env.JOB_NAME def currentBuildNumber = env.BUILD_NUMBER.toInteger() @@ -289,7 +284,6 @@ def run_win_ctest() { ''' } def run_win_test() { - echo "LINUX NODE: ${linux_node_ip} - ${linux_node_pass}" bat ''' echo "windows test ..." cd %WIN_CONNECTOR_ROOT% @@ -298,9 +292,8 @@ def run_win_test() { ls -l C:\\Windows\\System32\\taos.dll time /t cd %WIN_SYSTEM_TEST_ROOT% - echo "node: ''' + linux_node_ip + ''':''' + linux_node_pass + '''" echo "testing ..." - test-all.bat "{\\\"host\\\":\\\"''' + linux_node_ip + '''\\\",\\\"port\\\":22,\\\"user\\\":\\\"root\\\",\\\"password\\\":\\\"''' + linux_node_pass + '''\\\",\\\"path\\\":\\\"/var/lib/jenkins/workspace/TDinternal\\\"}" + test-all.bat time /t ''' } @@ -331,17 +324,9 @@ pipeline { pre_test_win() pre_test_build_win() run_win_ctest() - script { - while(linux_ready == 0) { - sleep(8) - } - } run_win_test() } } - script { - win_test_stage = 1 - } } } stage('linux test') { @@ -351,17 +336,6 @@ pipeline { changeRequest() } steps { - script { - linux_node_ip = sh ( - script: 'jq .ip /home/node_info.json | sed "s/\\\"//g"', - returnStdout: true - ).trim() - linux_node_pass = sh ( - script: 'jq .password /home/node_info.json | sed "s/\\\"//g" |sed "s/\\!/^^^^^^^^\\!/g"', - returnStdout: true - ).trim() - echo "${linux_node_ip}:${linux_node_pass}" - } catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { timeout(time: 120, unit: 'MINUTES'){ pre_test() @@ -442,14 +416,6 @@ pipeline { } } } - script { - linux_ready = 1 - } - script { - while(win_test_stage == 0){ - sleep(12) - } - } } } } From 7c57b03de71b99bdfa975d4b12e8c39998c29641 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 25 Jun 2022 12:19:52 +0800 Subject: [PATCH 31/72] handle rpc retry --- source/client/src/clientEnv.c | 2 +- source/dnode/mgmt/node_mgmt/src/dmTransport.c | 17 +++++--- source/libs/function/src/udfd.c | 40 ++++++++++--------- source/libs/transport/src/transCli.c | 13 ++++-- source/libs/transport/src/transSvr.c | 3 +- 5 files changed, 46 insertions(+), 29 deletions(-) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index ff9003b8fc..657c21c4f9 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -86,7 +86,7 @@ void closeTransporter(STscObj *pTscObj) { static bool clientRpcRfp(int32_t code) { if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED || - code == TSDB_CODE_SYN_NOT_LEADER) { + code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY) { return true; } else { return false; diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index 63d2a65df1..a4745abd5b 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -70,9 +70,9 @@ int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg) { } static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) { - SDnodeTrans *pTrans = &pDnode->trans; + SDnodeTrans * pTrans = &pDnode->trans; int32_t code = -1; - SRpcMsg *pMsg = NULL; + SRpcMsg * pMsg = NULL; SMgmtWrapper *pWrapper = NULL; SDnodeHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(pRpc->msgType)]; @@ -194,11 +194,11 @@ int32_t dmInitMsgHandle(SDnode *pDnode) { for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) { SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype]; - SArray *pArray = (*pWrapper->func.getHandlesFp)(); + SArray * pArray = (*pWrapper->func.getHandlesFp)(); if (pArray == NULL) return -1; for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { - SMgmtHandle *pMgmt = taosArrayGet(pArray, i); + SMgmtHandle * pMgmt = taosArrayGet(pArray, i); SDnodeHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(pMgmt->msgType)]; if (pMgmt->needCheckVgId) { pHandle->needCheckVgId = pMgmt->needCheckVgId; @@ -248,7 +248,14 @@ static inline void dmReleaseHandle(SRpcHandleInfo *pHandle, int8_t type) { } } -static bool rpcRfp(int32_t code) { return code == TSDB_CODE_RPC_REDIRECT; } +static bool rpcRfp(int32_t code) { + if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED || + code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY) { + return true; + } else { + return false; + } +} int32_t dmInitClient(SDnode *pDnode) { SDnodeTrans *pTrans = &pDnode->trans; diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 838071dbf1..983cffe9dc 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -12,6 +12,8 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ + +// clang-format off #include "uv.h" #include "os.h" #include "fnLog.h" @@ -25,6 +27,7 @@ #include "tglobal.h" #include "tmsg.h" #include "trpc.h" +// clang-foramt on typedef struct SUdfdContext { uv_loop_t * loop; @@ -103,12 +106,12 @@ typedef struct SUdfdRpcSendRecvInfo { uv_sem_t resultSem; } SUdfdRpcSendRecvInfo; -static void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); +static void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); static int32_t udfdFillUdfInfoFromMNode(void *clientRpc, char *udfName, SUdf *udf); static int32_t udfdConnectToMnode(); static int32_t udfdLoadUdf(char *udfName, SUdf *udf); -static bool udfdRpcRfp(int32_t code); -static int initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet); +static bool udfdRpcRfp(int32_t code); +static int initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet); static int32_t udfdOpenClientRpc(); static int32_t udfdCloseClientRpc(); @@ -126,19 +129,19 @@ static void udfdUvHandleError(SUdfdUvConn *conn) { uv_close((uv_handle_t *)conn- static void udfdPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf); static void udfdOnNewConnection(uv_stream_t *server, int status); -static void udfdIntrSignalHandler(uv_signal_t *handle, int signum); +static void udfdIntrSignalHandler(uv_signal_t *handle, int signum); static int32_t removeListeningPipe(); -static void udfdPrintVersion(); +static void udfdPrintVersion(); static int32_t udfdParseArgs(int32_t argc, char *argv[]); static int32_t udfdInitLog(); -static void udfdCtrlAllocBufCb(uv_handle_t *handle, size_t suggested_size, uv_buf_t *buf); -static void udfdCtrlReadCb(uv_stream_t *q, ssize_t nread, const uv_buf_t *buf); +static void udfdCtrlAllocBufCb(uv_handle_t *handle, size_t suggested_size, uv_buf_t *buf); +static void udfdCtrlReadCb(uv_stream_t *q, ssize_t nread, const uv_buf_t *buf); static int32_t udfdUvInit(); -static void udfdCloseWalkCb(uv_handle_t *handle, void *arg); +static void udfdCloseWalkCb(uv_handle_t *handle, void *arg); static int32_t udfdRun(); -static void udfdConnectMnodeThreadFunc(void* args); +static void udfdConnectMnodeThreadFunc(void *args); void udfdProcessRequest(uv_work_t *req) { SUvUdfWork *uvUdf = (SUvUdfWork *)(req->data); @@ -401,11 +404,11 @@ void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { udf->bufSize = pFuncInfo->bufSize; char path[PATH_MAX] = {0}; - #ifdef WINDOWS +#ifdef WINDOWS snprintf(path, sizeof(path), "%s%s.dll", TD_TMP_DIR_PATH, pFuncInfo->name); - #else +#else snprintf(path, sizeof(path), "%s/lib%s.so", TD_TMP_DIR_PATH, pFuncInfo->name); - #endif +#endif TdFilePtr file = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC | TD_FILE_AUTO_DEL); if (file == NULL) { @@ -544,7 +547,8 @@ int32_t udfdLoadUdf(char *udfName, SUdf *udf) { return 0; } static bool udfdRpcRfp(int32_t code) { - if (code == TSDB_CODE_RPC_REDIRECT) { + if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED || + code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY) { return true; } else { return false; @@ -652,8 +656,7 @@ void udfdAllocBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) { buf->base = ctx->inputBuf; buf->len = ctx->inputCap; } else { - fnError("udfd can not allocate enough memory") - buf->base = NULL; + fnError("udfd can not allocate enough memory") buf->base = NULL; buf->len = 0; } } else { @@ -664,8 +667,7 @@ void udfdAllocBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) { buf->base = ctx->inputBuf + ctx->inputLen; buf->len = ctx->inputCap - ctx->inputLen; } else { - fnError("udfd can not allocate enough memory") - buf->base = NULL; + fnError("udfd can not allocate enough memory") buf->base = NULL; buf->len = 0; } } @@ -881,7 +883,7 @@ static int32_t udfdRun() { return 0; } -void udfdConnectMnodeThreadFunc(void* args) { +void udfdConnectMnodeThreadFunc(void *args) { int32_t retryMnodeTimes = 0; int32_t code = 0; while (retryMnodeTimes++ <= TSDB_MAX_REPLICA) { @@ -939,7 +941,7 @@ int main(int argc, char *argv[]) { uv_thread_create(&mnodeConnectThread, udfdConnectMnodeThreadFunc, NULL); udfdRun(); - + removeListeningPipe(); udfdCloseClientRpc(); diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 0671aa39d1..a21b753d6e 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -758,13 +758,16 @@ static void cliHandleUpdate(SCliMsg* pMsg, SCliThrd* pThrd) { destroyCmsg(pMsg); } -SCliConn* cliGetConn(SCliMsg* pMsg, SCliThrd* pThrd) { +SCliConn* cliGetConn(SCliMsg* pMsg, SCliThrd* pThrd, bool* ignore) { SCliConn* conn = NULL; int64_t refId = (int64_t)(pMsg->msg.info.handle); if (refId != 0) { SExHandle* exh = transAcquireExHandle(refMgt, refId); if (exh == NULL) { - assert(0); + *ignore = true; + destroyCmsg(pMsg); + return NULL; + // assert(0); } else { conn = exh->handle; transReleaseExHandle(refMgt, refId); @@ -799,7 +802,11 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) { cliMayCvtFqdnToIp(&pCtx->epSet, &pThrd->cvtAddr); // transPrintEpSet(&pCtx->epSet); - SCliConn* conn = cliGetConn(pMsg, pThrd); + bool ignore = false; + SCliConn* conn = cliGetConn(pMsg, pThrd, &ignore); + if (ignore == true) { + return; + } if (conn != NULL) { transCtxMerge(&conn->ctx, &pCtx->appCtx); transQueuePush(&conn->cliMsgs, pMsg); diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 7651860224..892d32696e 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -422,7 +422,8 @@ static void uvPrepareSendData(SSvrMsg* smsg, uv_buf_t* wb) { transUnrefSrvHandle(pConn); } else { pHead->msgType = pMsg->msgType; - if (pHead->msgType == 0) pHead->msgType = pConn->inType + 1; + if (pHead->msgType == 0 && transMsgLenFromCont(pMsg->contLen) == sizeof(STransMsgHead)) + pHead->msgType = pConn->inType + 1; } } From b139daf08bbcdd958e9e930413f1ea87fb691a41 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sat, 25 Jun 2022 13:32:14 +0800 Subject: [PATCH 32/72] add cast function support ->bool --- source/libs/scalar/src/sclfunc.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 4caccfc7f3..80bebafef2 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -915,6 +915,24 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp } break; } + case TSDB_DATA_TYPE_BOOL: { + if (inputType == TSDB_DATA_TYPE_BINARY) { + *(bool *)output = taosStr2Int8(varDataVal(input), NULL, 10); + } else if (inputType == TSDB_DATA_TYPE_NCHAR) { + char *newBuf = taosMemoryCalloc(1, inputLen); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + if (len < 0) { + taosMemoryFree(newBuf); + return TSDB_CODE_FAILED; + } + newBuf[len] = 0; + *(bool *)output = taosStr2Int8(newBuf, NULL, 10); + taosMemoryFree(newBuf); + } else { + GET_TYPED_DATA(*(bool *)output, bool, inputType, input); + } + break; + } case TSDB_DATA_TYPE_TIMESTAMP: { if (inputType == TSDB_DATA_TYPE_BINARY || inputType == TSDB_DATA_TYPE_NCHAR) { //convert to 0 From 27a67652824dc3ea3903d16819e40b01caf5500a Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Sat, 25 Jun 2022 13:46:53 +0800 Subject: [PATCH 33/72] fix: add a default parameter to windows python test script --- Jenkinsfile2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 6b48aabab8..515667d8cc 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -293,7 +293,7 @@ def run_win_test() { time /t cd %WIN_SYSTEM_TEST_ROOT% echo "testing ..." - test-all.bat + test-all.bat ci time /t ''' } From 7acf01225391e02a7eedc0665aff907a1c5654ab Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Sat, 25 Jun 2022 13:51:43 +0800 Subject: [PATCH 34/72] fix:memory error --- source/dnode/vnode/src/meta/metaTable.c | 2 +- source/libs/parser/src/parTranslater.c | 3 +++ source/libs/scalar/src/sclvector.c | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 4a3feca8d0..7109bf1dfc 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -388,7 +388,7 @@ int metaTtlDropTable(SMeta *pMeta, int64_t ttl, SArray *tbUids) { } static void metaBuildTtlIdxKey(STtlIdxKey *ttlKey, const SMetaEntry *pME){ - int32_t ttlDays; + int64_t ttlDays; int64_t ctime; if (pME->type == TSDB_CHILD_TABLE) { ctime = pME->ctbEntry.ctime; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index d4bde5d624..fe562fc936 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -4963,6 +4963,7 @@ typedef struct SVgroupCreateTableBatch { static void destroyCreateTbReq(SVCreateTbReq* pReq) { taosMemoryFreeClear(pReq->name); + taosMemoryFreeClear(pReq->comment); taosMemoryFreeClear(pReq->ntb.schemaRow.pSchema); } @@ -4980,6 +4981,7 @@ static int32_t buildNormalTableBatchReq(int32_t acctId, const SCreateTableStmt* if (pStmt->pOptions->commentNull == false) { req.comment = strdup(pStmt->pOptions->comment); if (NULL == req.comment) { + destroyCreateTbReq(&req); return TSDB_CODE_OUT_OF_MEMORY; } req.commentLen = strlen(pStmt->pOptions->comment); @@ -5051,6 +5053,7 @@ static void destroyCreateTbReqBatch(SVgroupCreateTableBatch* pTbBatch) { for (int32_t i = 0; i < size; ++i) { SVCreateTbReq* pTableReq = taosArrayGet(pTbBatch->req.pArray, i); taosMemoryFreeClear(pTableReq->name); + taosMemoryFreeClear(pTableReq->comment); if (pTableReq->type == TSDB_NORMAL_TABLE) { taosMemoryFreeClear(pTableReq->ntb.schemaRow.pSchema); diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index f9a9ec0f72..292db3db1f 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -468,7 +468,7 @@ double getVectorDoubleValue_JSON(void *src, int32_t index){ } void* ncharTobinary(void *buf){ // todo need to remove , if tobinary is nchar - int32_t inputLen = varDataLen(buf); + int32_t inputLen = varDataTLen(buf); void* t = taosMemoryCalloc(1, inputLen); int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(buf), varDataLen(buf), varDataVal(t)); From a9f094613a47047acab797feec6d8eb58798e718 Mon Sep 17 00:00:00 2001 From: slzhou Date: Sat, 25 Jun 2022 13:52:52 +0800 Subject: [PATCH 35/72] fix: restore table merge scan operator --- source/dnode/vnode/inc/vnode.h | 1 + source/dnode/vnode/src/tsdb/tsdbRead.c | 2 +- source/libs/executor/src/executil.c | 4 +- source/libs/executor/src/executorimpl.c | 3 +- source/libs/executor/src/scanoperator.c | 122 ++++++++++++++---------- 5 files changed, 77 insertions(+), 55 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index a32bf0ecdb..b97d4605e7 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -196,6 +196,7 @@ struct SVnodeCfg { typedef struct { TSKEY lastKey; uint64_t uid; + uint64_t groupId; } STableKeyInfo; struct SMetaEntry { diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 540810f876..ab31d65d68 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2852,7 +2852,7 @@ int32_t tsdbGetAllTableList(SMeta* pMeta, uint64_t uid, SArray* list) { break; } - STableKeyInfo info = {.lastKey = TSKEY_INITIAL_VAL, uid = id}; + STableKeyInfo info = {.lastKey = TSKEY_INITIAL_VAL, uid = id, .groupId = 0}; taosArrayPush(list, &info); } diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 5ac5957f2b..374a3a736d 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -315,7 +315,7 @@ int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo } for (int i = 0; i < taosArrayGetSize(res); i++) { - STableKeyInfo info = {.lastKey = TSKEY_INITIAL_VAL, .uid = *(uint64_t*)taosArrayGet(res, i)}; + STableKeyInfo info = {.lastKey = TSKEY_INITIAL_VAL, .uid = *(uint64_t*)taosArrayGet(res, i), .groupId = 0}; taosArrayPush(pListInfo->pTableList, &info); } taosArrayDestroy(res); @@ -336,7 +336,7 @@ int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo } } }else { // Create one table group. - STableKeyInfo info = {.lastKey = 0, .uid = tableUid}; + STableKeyInfo info = {.lastKey = 0, .uid = tableUid, .groupId = 0}; taosArrayPush(pListInfo->pTableList, &info); } pListInfo->pGroupList = taosArrayInit(4, POINTER_BYTES); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index c23d9a5040..b4d87a53b1 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4028,6 +4028,7 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, int32_t len = (int32_t)(pStart - (char*)keyBuf); uint64_t groupId = calcGroupId(keyBuf, len); taosHashPut(pTableListInfo->map, &(info->uid), sizeof(uint64_t), &groupId, sizeof(uint64_t)); + info->groupId = groupId; groupNum++; nodesClearList(groupNew); @@ -4127,7 +4128,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo return NULL; } } else { // Create one table group. - STableKeyInfo info = {.lastKey = 0, .uid = pBlockNode->uid}; + STableKeyInfo info = {.lastKey = 0, .uid = pBlockNode->uid, .groupId = 0}; taosArrayPush(pTableListInfo->pTableList, &info); } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 9c0ed40c30..dedf6c0707 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1965,7 +1965,10 @@ _error: typedef struct STableMergeScanInfo { STableListInfo* tableListInfo; - int32_t currentGroupId; + int32_t tableStartIndex; + int32_t tableEndIndex; + bool hasGroupId; + uint64_t groupId; SArray* dataReaders; // array of tsdbReaderT* SReadHandle readHandle; @@ -2006,7 +2009,7 @@ typedef struct STableMergeScanInfo { int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan int32_t dataBlockLoadFlag; SInterval interval; // if the upstream is an interval operator, the interval info is also kept here to get the time - // window to check if current data block needs to be loaded. + // window to check if current data block needs to be loaded. SSampleExecInfo sample; // sample execution info } STableMergeScanInfo; @@ -2030,6 +2033,22 @@ int32_t createScanTableListInfo(STableScanPhysiNode* pTableScanNode, SReadHandle return TSDB_CODE_SUCCESS; } +int32_t createMultipleDataReaders(SQueryTableDataCond* pQueryCond, SReadHandle* pHandle, STableListInfo* pTableListInfo, + int32_t tableStartIdx, int32_t tableEndIdx, SArray* arrayReader, uint64_t queryId, + uint64_t taskId) { + for (int32_t i = tableStartIdx; i <= tableEndIdx; ++i) { + SArray* subTableList = taosArrayInit(1, sizeof(STableKeyInfo)); + taosArrayPush(subTableList, taosArrayGet(pTableListInfo->pTableList, i)); + + tsdbReaderT* pReader = tsdbReaderOpen(pHandle->vnode, pQueryCond, subTableList, queryId, taskId); + taosArrayPush(arrayReader, &pReader); + + taosArrayDestroy(subTableList); + } + + return TSDB_CODE_SUCCESS; +} + // todo refactor static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeScanInfo* pTableScanInfo, int32_t readerIdx, SSDataBlock* pBlock, uint32_t* status) { @@ -2216,34 +2235,32 @@ SArray* generateSortByTsInfo(int32_t order) { return pList; } -static int32_t createMultipleDataReaders(SQueryTableDataCond* pQueryCond, SReadHandle* pHandle, SArray* tableList, SArray* arrayReader, uint64_t queryId, - uint64_t taskId) { - for (int32_t i = 0; i < taosArrayGetSize(tableList); ++i) { - SArray* tmp = taosArrayInit(1, sizeof(STableKeyInfo)); - taosArrayPush(tmp, taosArrayGet(tableList, i)); - - tsdbReaderT* pReader = tsdbReaderOpen(pHandle->vnode, pQueryCond, tmp, queryId, taskId); - taosArrayPush(arrayReader, &pReader); - - taosArrayDestroy(tmp); - } - - return TSDB_CODE_SUCCESS; -} - int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) { STableMergeScanInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SArray* tableList = taosArrayGetP(pInfo->tableListInfo->pGroupList, pInfo->currentGroupId); + { + size_t tableListSize = taosArrayGetSize(pInfo->tableListInfo->pTableList); + int32_t i = pInfo->tableStartIndex + 1; + for (; i < tableListSize; ++i) { + STableKeyInfo* tableKeyInfo = taosArrayGet(pInfo->tableListInfo->pTableList, i); + if (tableKeyInfo->groupId != pInfo->groupId) { + break; + } + } + pInfo->tableEndIndex = i - 1; + } - createMultipleDataReaders(&pInfo->cond, &pInfo->readHandle, tableList, + int32_t tableStartIdx = pInfo->tableStartIndex; + int32_t tableEndIdx = pInfo->tableEndIndex; + + STableListInfo* tableListInfo = pInfo->tableListInfo; + createMultipleDataReaders(&pInfo->cond, &pInfo->readHandle, tableListInfo, tableStartIdx, tableEndIdx, pInfo->dataReaders, pInfo->queryId, pInfo->taskId); // todo the total available buffer should be determined by total capacity of buffer of this task. // the additional one is reserved for merge result - int32_t tableLen = taosArrayGetSize(tableList); - pInfo->sortBufSize = pInfo->bufPageSize * ((tableLen==0?1:tableLen) + 1); + pInfo->sortBufSize = pInfo->bufPageSize * (tableEndIdx - tableStartIdx + 1 + 1); int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_MULTISOURCE_MERGE, pInfo->bufPageSize, numOfBufPage, pInfo->pSortInputBlock, pTaskInfo->id.str); @@ -2330,43 +2347,38 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) { if (code != TSDB_CODE_SUCCESS) { longjmp(pTaskInfo->env, code); } + size_t tableListSize = taosArrayGetSize(pInfo->tableListInfo->pTableList); + if (!pInfo->hasGroupId) { + pInfo->hasGroupId = true; - if (pInfo->currentGroupId == -1) { - pInfo->currentGroupId++; - if (pInfo->currentGroupId >= taosArrayGetSize(pInfo->tableListInfo->pGroupList)) { + if (tableListSize == 0) { doSetOperatorCompleted(pOperator); return NULL; } + pInfo->tableStartIndex = 0; + pInfo->groupId = ((STableKeyInfo*)taosArrayGet(pInfo->tableListInfo->pTableList, pInfo->tableStartIndex))->groupId; startGroupTableMergeScan(pOperator); } - SSDataBlock* pBlock = getSortedTableMergeScanBlockData(pInfo->pSortHandle, pOperator->resultInfo.capacity, pOperator); - if (pBlock != NULL) { - uint64_t* groupId = taosHashGet(pInfo->tableListInfo->map, &(pBlock->info.uid), sizeof(uint64_t)); - if(groupId) pBlock->info.groupId = *groupId; - - pOperator->resultInfo.totalRows += pBlock->info.rows; - return pBlock; + SSDataBlock* pBlock = NULL; + while (pInfo->tableStartIndex < tableListSize) { + pBlock = getSortedTableMergeScanBlockData(pInfo->pSortHandle, pOperator->resultInfo.capacity, pOperator); + if (pBlock != NULL) { + pBlock->info.groupId = pInfo->groupId; + pOperator->resultInfo.totalRows += pBlock->info.rows; + return pBlock; + } else { + stopGroupTableMergeScan(pOperator); + if (pInfo->tableEndIndex >= tableListSize - 1) { + doSetOperatorCompleted(pOperator); + break; + } + pInfo->tableStartIndex = pInfo->tableEndIndex + 1; + pInfo->groupId = + ((STableKeyInfo*)taosArrayGet(pInfo->tableListInfo->pTableList, pInfo->tableStartIndex))->groupId; + startGroupTableMergeScan(pOperator); + } } - stopGroupTableMergeScan(pOperator); - pInfo->currentGroupId++; - if (pInfo->currentGroupId >= taosArrayGetSize(pInfo->tableListInfo->pGroupList)) { - doSetOperatorCompleted(pOperator); - return NULL; - } - startGroupTableMergeScan(pOperator); - - pBlock = getSortedTableMergeScanBlockData(pInfo->pSortHandle, pOperator->resultInfo.capacity, pOperator); - if (pBlock != NULL) { - uint64_t* groupId = taosHashGet(pInfo->tableListInfo->map, &(pBlock->info.uid), sizeof(uint64_t)); - if(groupId) pBlock->info.groupId = *groupId; - - pOperator->resultInfo.totalRows += pBlock->info.rows; - return pBlock; - } - - doSetOperatorCompleted(pOperator); - return pBlock; } @@ -2403,6 +2415,12 @@ int32_t getTableMergeScanExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExpla return TSDB_CODE_SUCCESS; } +int32_t compareTableKeyInfoByGid(const void* p1, const void* p2) { + const STableKeyInfo* info1 = p1; + const STableKeyInfo* info2 = p2; + return info1->groupId - info2->groupId; +} + SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, STableListInfo* pTableListInfo, SReadHandle* readHandle, SExecTaskInfo* pTaskInfo, uint64_t queryId, uint64_t taskId) { @@ -2411,6 +2429,9 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN if (pInfo == NULL || pOperator == NULL) { goto _error; } + if (pTableScanNode->pPartitionTags) { + taosArraySort(pTableListInfo->pTableList, compareTableKeyInfoByGid); + } SDataBlockDescNode* pDescNode = pTableScanNode->scan.node.pOutputDataBlockDesc; @@ -2443,7 +2464,6 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN pInfo->dataReaders = taosArrayInit(64, POINTER_BYTES); pInfo->queryId = queryId; pInfo->taskId = taskId; - pInfo->currentGroupId = -1; pInfo->sortSourceParams = taosArrayInit(64, sizeof(STableMergeScanSortSourceParam)); From 81ed902d7f1e67b22c258e2f998cd4641c89ee2d Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Sat, 25 Jun 2022 13:37:11 +0800 Subject: [PATCH 36/72] fix(stream): stream rsp to retrieve msg if data is empty --- include/common/tcommon.h | 3 ++- include/libs/stream/tstream.h | 1 + source/libs/stream/src/stream.c | 6 +++--- source/libs/stream/src/streamDispatch.c | 2 +- source/libs/stream/src/streamExec.c | 14 ++++++++++++-- source/libs/wal/src/walWrite.c | 11 +++++++---- 6 files changed, 26 insertions(+), 11 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 9c84d0dd99..a14f7eff8a 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -47,6 +47,7 @@ typedef enum EStreamType { STREAM_GET_ALL, STREAM_DELETE, STREAM_RETRIEVE, + STREAM_PUSH_DATA, } EStreamType; typedef struct { @@ -71,7 +72,7 @@ typedef struct SColumnDataAgg { typedef struct SDataBlockInfo { STimeWindow window; - int32_t rows; // todo hide this attribute + int32_t rows; // todo hide this attribute int32_t rowSize; uint64_t uid; // the uid of table, from which current data block comes uint16_t blockId; // block id, generated by physical planner diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index a47810e7c1..594344ba8a 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -58,6 +58,7 @@ enum { enum { STREAM_INPUT__DATA_SUBMIT = 1, STREAM_INPUT__DATA_BLOCK, + STREAM_INPUT__DATA_RETRIEVE, STREAM_INPUT__TRIGGER, STREAM_INPUT__CHECKPOINT, STREAM_INPUT__DROP, diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 9d92865e47..56d063ae51 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -111,7 +111,7 @@ int32_t streamTaskEnqueue(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* // enqueue if (pData != NULL) { - pData->type = STREAM_DATA_TYPE_SSDATA_BLOCK; + pData->type = STREAM_INPUT__DATA_BLOCK; pData->srcVgId = pReq->dataSrcVgId; // decode /*pData->blocks = pReq->data;*/ @@ -146,7 +146,7 @@ int32_t streamTaskEnqueueRetrieve(SStreamTask* pTask, SStreamRetrieveReq* pReq, // enqueue if (pData != NULL) { - pData->type = STREAM_DATA_TYPE_SSDATA_BLOCK; + pData->type = STREAM_INPUT__DATA_RETRIEVE; pData->srcVgId = 0; // decode /*pData->blocks = pReq->data;*/ @@ -170,7 +170,7 @@ int32_t streamTaskEnqueueRetrieve(SStreamTask* pTask, SStreamRetrieveReq* pReq, pCont->rspToTaskId = pReq->srcTaskId; pCont->rspFromTaskId = pReq->dstTaskId; pRsp->pCont = buf; - pRsp->contLen = sizeof(SMsgHead) + sizeof(SStreamDispatchRsp); + pRsp->contLen = sizeof(SMsgHead) + sizeof(SStreamRetrieveRsp); tmsgSendRsp(pRsp); return status == TASK_INPUT_STATUS__NORMAL ? 0 : -1; } diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 1f51a927e7..0c9d46f055 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -300,7 +300,7 @@ int32_t streamDispatch(SStreamTask* pTask, SMsgCb* pMsgCb) { atomic_store_8(&pTask->outputStatus, TASK_OUTPUT_STATUS__NORMAL); return 0; } - ASSERT(pBlock->type == STREAM_DATA_TYPE_SSDATA_BLOCK); + ASSERT(pBlock->type == STREAM_INPUT__DATA_BLOCK); qInfo("stream continue dispatching: task %d", pTask->taskId); diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index cb7dd241c1..8e7cac03a2 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -17,6 +17,7 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) { void* exec = pTask->exec.executor; + bool hasData = false; // set input SStreamQueueItem* pItem = (SStreamQueueItem*)data; @@ -27,7 +28,7 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) ASSERT(pTask->isDataScan); SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)data; qSetStreamInput(exec, pSubmit->data, STREAM_DATA_TYPE_SUBMIT_BLOCK, false); - } else if (pItem->type == STREAM_INPUT__DATA_BLOCK) { + } else if (pItem->type == STREAM_INPUT__DATA_BLOCK || pItem->type == STREAM_INPUT__DATA_RETRIEVE) { SStreamDataBlock* pBlock = (SStreamDataBlock*)data; SArray* blocks = pBlock->blocks; qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_DATA_TYPE_SSDATA_BLOCK, false); @@ -43,7 +44,16 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) if (qExecTask(exec, &output, &ts) < 0) { ASSERT(false); } - if (output == NULL) break; + if (output == NULL) { + if (pItem->type == STREAM_INPUT__DATA_RETRIEVE && !hasData) { + SSDataBlock block = {0}; + block.info.type = STREAM_PUSH_DATA; + block.info.childId = pTask->selfChildId; + taosArrayPush(pRes, &block); + } + break; + } + hasData = true; if (output->info.type == STREAM_RETRIEVE) { if (streamBroadcastToChildren(pTask, output) < 0) { diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 1d169a0891..d1295667af 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -42,6 +42,9 @@ int32_t walRestoreFromSnapshot(SWal *pWal, int64_t ver) { char fnameStr[WAL_FILE_LEN]; walBuildLogName(pWal, pFileInfo->firstVer, fnameStr); taosRemoveFile(fnameStr); + + walBuildIdxName(pWal, pFileInfo->firstVer, fnameStr); + taosRemoveFile(fnameStr); } } walRemoveMeta(pWal); @@ -105,7 +108,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) { } walBuildIdxName(pWal, walGetCurFileFirstVer(pWal), fnameStr); - TdFilePtr pIdxTFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ); + TdFilePtr pIdxTFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ | TD_FILE_APPEND); if (pIdxTFile == NULL) { taosThreadMutexUnlock(&pWal->mutex); @@ -126,7 +129,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) { ASSERT(entry.ver == ver); walBuildLogName(pWal, walGetCurFileFirstVer(pWal), fnameStr); - TdFilePtr pLogTFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ); + TdFilePtr pLogTFile = taosOpenFile(fnameStr, TD_FILE_WRITE | TD_FILE_READ | TD_FILE_APPEND); if (pLogTFile == NULL) { // TODO taosThreadMutexUnlock(&pWal->mutex); @@ -307,8 +310,8 @@ int walRoll(SWal *pWal) { static int walWriteIndex(SWal *pWal, int64_t ver, int64_t offset) { SWalIdxEntry entry = {.ver = ver, .offset = offset}; - /*int64_t idxOffset = taosLSeekFile(pWal->pWriteIdxTFile, 0, SEEK_CUR);*/ - /*wDebug("write index: ver: %ld, offset: %ld, at %ld", ver, offset, idxOffset);*/ + int64_t idxOffset = taosLSeekFile(pWal->pWriteIdxTFile, 0, SEEK_END); + wDebug("write index: ver: %ld, offset: %ld, at %ld", ver, offset, idxOffset); int64_t size = taosWriteFile(pWal->pWriteIdxTFile, &entry, sizeof(SWalIdxEntry)); if (size != sizeof(SWalIdxEntry)) { terrno = TAOS_SYSTEM_ERROR(errno); From 25a12d96b05f6234c069de516b3c632d2a700d68 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 25 Jun 2022 14:06:33 +0800 Subject: [PATCH 37/72] handle rpc retry --- source/client/src/clientEnv.c | 2 +- source/libs/transport/src/trans.c | 1 + source/libs/transport/src/transCli.c | 9 ++++++--- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 657c21c4f9..d7bf4b60f1 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -136,7 +136,7 @@ void destroyTscObj(void *pObj) { schedulerStopQueryHb(pTscObj->pAppInfo->pTransporter); if (0 == connNum) { // TODO - closeTransporter(pTscObj); + // closeTransporter(pTscObj); } tscDebug("connObj 0x%" PRIx64 " destroyed, totalConn:%" PRId64, *(int64_t *)pTscObj->id, pTscObj->pAppInfo->numOfConns); diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 1ec96f4a7a..4f7b19b539 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -79,6 +79,7 @@ void* rpcOpen(const SRpcInit* pInit) { return pRpc; } void rpcClose(void* arg) { + tInfo("start to close rpc"); SRpcInfo* pRpc = (SRpcInfo*)arg; (*taosCloseHandle[pRpc->connType])(pRpc->tcphandle); transCloseExHandleMgt(pRpc->refMgt); diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index a21b753d6e..7374d1fffc 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -724,14 +724,13 @@ void cliConnCb(uv_connect_t* req, int status) { } static void cliHandleQuit(SCliMsg* pMsg, SCliThrd* pThrd) { + pThrd->quit = true; tDebug("cli work thread %p start to quit", pThrd); destroyCmsg(pMsg); destroyConnPool(pThrd->pool); uv_timer_stop(&pThrd->timer); uv_walk(pThrd->loop, cliWalkCb, NULL); - pThrd->quit = true; - // uv_stop(pThrd->loop); } static void cliHandleRelease(SCliMsg* pMsg, SCliThrd* pThrd) { @@ -977,7 +976,10 @@ void cliWalkCb(uv_handle_t* handle, void* arg) { } int cliRBChoseIdx(STrans* pTransInst) { - int64_t index = pTransInst->index; + int8_t index = pTransInst->index; + if (pTransInst->numOfThreads == 0) { + return -1; + } if (pTransInst->index++ >= pTransInst->numOfThreads) { pTransInst->index = 0; } @@ -1120,6 +1122,7 @@ SCliThrd* transGetWorkThrdFromHandle(int64_t handle) { SCliThrd* transGetWorkThrd(STrans* trans, int64_t handle) { if (handle == 0) { int idx = cliRBChoseIdx(trans); + if (idx < 0) return NULL; return ((SCliObj*)trans->tcphandle)->pThreadObj[idx]; } return transGetWorkThrdFromHandle(handle); From d05ef645077a3401154f037806550bfc7f8eeaf7 Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Sat, 25 Jun 2022 14:10:34 +0800 Subject: [PATCH 38/72] enh: skip install package --- Jenkinsfile2 | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 515667d8cc..861478160f 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -392,27 +392,6 @@ pipeline { cd ${WKC}/packaging ./release.sh -v cluster -n 3.0.0.100 -s static ''' - sh ''' - echo "install ..." - cd ${WKC}/release - tar xzf TDengine-enterprise-server-3.0.0.100-Linux-x64.tar.gz - cd TDengine-enterprise-server-3.0.0.100 - service taosd stop || : - rm -rf /var/lib/taos - ./install.sh -e no - ''' - sh ''' - echo "checking ..." - which taos - which taosd - rm -rf ${WK}/debug - mv ${WKC}/debug ${WK}/ - ''' - sh ''' - echo "install taospy ..." - cd ${WKPY} - pip3 install . - ''' } } } From 7752a7e4fd067a7452b977bf09f4bb94e7304a7b Mon Sep 17 00:00:00 2001 From: jiacy-jcy Date: Sat, 25 Jun 2022 14:14:01 +0800 Subject: [PATCH 39/72] add test case for param ttl --- tests/system-test/1-insert/table_param_ttl.py | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 tests/system-test/1-insert/table_param_ttl.py diff --git a/tests/system-test/1-insert/table_param_ttl.py b/tests/system-test/1-insert/table_param_ttl.py new file mode 100644 index 0000000000..f93ebb98cb --- /dev/null +++ b/tests/system-test/1-insert/table_param_ttl.py @@ -0,0 +1,61 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * + +class TDTestCase: + updatecfgDict = {'ttlUnit':10,'ttlPushInterval':5} + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.ntbname = 'ntb' + self.stbname = 'stb' + self.tbnum = 10 + self.ttl_param = 1 + def ttl_check(self): + tdSql.prepare() + for i in range(self.tbnum): + tdSql.execute(f'create table {self.ntbname}_{i} (ts timestamp,c0 int) ttl {self.ttl_param}') + tdSql.query(f'show tables') + tdSql.checkRows(self.tbnum) + tdSql.execute('reset query cache') + sleep(self.updatecfgDict['ttlUnit']*self.ttl_param+5) + tdSql.query(f'show tables') + tdSql.checkRows(0) + + tdSql.execute(f'create table {self.stbname} (ts timestamp,c0 int) tags(t0 int)') + for i in range(self.tbnum): + tdSql.execute(f'create table {self.stbname}_{i} using {self.stbname} tags({i}) ttl {self.ttl_param}') + tdSql.query(f'show tables') + tdSql.checkRows(self.tbnum) + sleep(self.updatecfgDict['ttlUnit']*self.ttl_param+5) + tdSql.query(f'show tables') + tdSql.checkRows(0) + + def run(self): + + self.ttl_check() + + + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From c0564bd6395595e0ebf9ff87ccfb58005d90d4e7 Mon Sep 17 00:00:00 2001 From: jiacy-jcy Date: Sat, 25 Jun 2022 14:16:07 +0800 Subject: [PATCH 40/72] update --- tests/system-test/1-insert/table_param_ttl.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/system-test/1-insert/table_param_ttl.py b/tests/system-test/1-insert/table_param_ttl.py index f93ebb98cb..65be3cdcf3 100644 --- a/tests/system-test/1-insert/table_param_ttl.py +++ b/tests/system-test/1-insert/table_param_ttl.py @@ -46,12 +46,7 @@ class TDTestCase: tdSql.checkRows(0) def run(self): - self.ttl_check() - - - - def stop(self): tdSql.close() From 52bfe2e74cac305bebb3adaf8a2a49dcedce1a6a Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sat, 25 Jun 2022 14:19:17 +0800 Subject: [PATCH 41/72] modify test cases for cast.py --- tests/system-test/2-query/cast.py | 38 +++++++++++++++---------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/tests/system-test/2-query/cast.py b/tests/system-test/2-query/cast.py index 387170991f..934bbbd7b4 100644 --- a/tests/system-test/2-query/cast.py +++ b/tests/system-test/2-query/cast.py @@ -630,27 +630,27 @@ class TDTestCase: ( tdSql.checkData(i, 0, '12') for i in range(tdSql.queryRows) ) tdLog.printNoPrefix("==========step40: error cast condition, should return error ") - tdSql.error("select cast(c1 as int) as b from ct4") - tdSql.error("select cast(c1 as bool) as b from ct4") - tdSql.error("select cast(c1 as tinyint) as b from ct4") - tdSql.error("select cast(c1 as smallint) as b from ct4") - tdSql.error("select cast(c1 as float) as b from ct4") - tdSql.error("select cast(c1 as double) as b from ct4") - tdSql.error("select cast(c1 as tinyint unsigned) as b from ct4") - tdSql.error("select cast(c1 as smallint unsigned) as b from ct4") - tdSql.error("select cast(c1 as int unsigned) as b from ct4") + #tdSql.error("select cast(c1 as int) as b from ct4") + #tdSql.error("select cast(c1 as bool) as b from ct4") + #tdSql.error("select cast(c1 as tinyint) as b from ct4") + #tdSql.error("select cast(c1 as smallint) as b from ct4") + #tdSql.error("select cast(c1 as float) as b from ct4") + #tdSql.error("select cast(c1 as double) as b from ct4") + #tdSql.error("select cast(c1 as tinyint unsigned) as b from ct4") + #tdSql.error("select cast(c1 as smallint unsigned) as b from ct4") + #tdSql.error("select cast(c1 as int unsigned) as b from ct4") - tdSql.error("select cast(c2 as int) as b from ct4") - tdSql.error("select cast(c3 as bool) as b from ct4") - tdSql.error("select cast(c4 as tinyint) as b from ct4") - tdSql.error("select cast(c5 as smallint) as b from ct4") - tdSql.error("select cast(c6 as float) as b from ct4") - tdSql.error("select cast(c7 as double) as b from ct4") - tdSql.error("select cast(c8 as tinyint unsigned) as b from ct4") + #tdSql.error("select cast(c2 as int) as b from ct4") + #tdSql.error("select cast(c3 as bool) as b from ct4") + #tdSql.error("select cast(c4 as tinyint) as b from ct4") + #tdSql.error("select cast(c5 as smallint) as b from ct4") + #tdSql.error("select cast(c6 as float) as b from ct4") + #tdSql.error("select cast(c7 as double) as b from ct4") + #tdSql.error("select cast(c8 as tinyint unsigned) as b from ct4") - tdSql.error("select cast(c8 as timestamp ) as b from ct4") - tdSql.error("select cast(c9 as timestamp ) as b from ct4") - tdSql.error("select cast(c9 as binary(64) ) as b from ct4") + #tdSql.error("select cast(c8 as timestamp ) as b from ct4") + #tdSql.error("select cast(c9 as timestamp ) as b from ct4") + #tdSql.error("select cast(c9 as binary(64) ) as b from ct4") pass def run(self): From fa84d68f9846f5f3449e1e99ebf598683c82abf1 Mon Sep 17 00:00:00 2001 From: jiacy-jcy Date: Sat, 25 Jun 2022 14:20:57 +0800 Subject: [PATCH 42/72] add case into ci --- tests/system-test/fulltest.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index ef217b828f..22dc829a23 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -23,6 +23,7 @@ python3 ./test.py -f 1-insert/alter_stable.py python3 ./test.py -f 1-insert/alter_table.py python3 ./test.py -f 1-insert/insertWithMoreVgroup.py python3 ./test.py -f 1-insert/table_comment.py +python3 ./test.py -f 1-insert/table_param_ttl.py python3 ./test.py -f 2-query/between.py python3 ./test.py -f 2-query/distinct.py python3 ./test.py -f 2-query/varchar.py From 4eae779c85b4407fb70d4d5ef1a7aa7d88c58af6 Mon Sep 17 00:00:00 2001 From: jiacy-jcy Date: Sat, 25 Jun 2022 14:24:57 +0800 Subject: [PATCH 43/72] update test case --- tests/system-test/1-insert/table_param_ttl.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/system-test/1-insert/table_param_ttl.py b/tests/system-test/1-insert/table_param_ttl.py index 65be3cdcf3..850c323be0 100644 --- a/tests/system-test/1-insert/table_param_ttl.py +++ b/tests/system-test/1-insert/table_param_ttl.py @@ -17,7 +17,7 @@ from util.sql import * from util.common import * class TDTestCase: - updatecfgDict = {'ttlUnit':10,'ttlPushInterval':5} + updatecfgDict = {'ttlUnit':5,'ttlPushInterval':3} def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) @@ -32,7 +32,7 @@ class TDTestCase: tdSql.query(f'show tables') tdSql.checkRows(self.tbnum) tdSql.execute('reset query cache') - sleep(self.updatecfgDict['ttlUnit']*self.ttl_param+5) + sleep(self.updatecfgDict['ttlUnit']*self.ttl_param+self.updatecfgDict['ttlPushInterval']) tdSql.query(f'show tables') tdSql.checkRows(0) @@ -41,7 +41,7 @@ class TDTestCase: tdSql.execute(f'create table {self.stbname}_{i} using {self.stbname} tags({i}) ttl {self.ttl_param}') tdSql.query(f'show tables') tdSql.checkRows(self.tbnum) - sleep(self.updatecfgDict['ttlUnit']*self.ttl_param+5) + sleep(self.updatecfgDict['ttlUnit']*self.ttl_param+self.updatecfgDict['ttlPushInterval']) tdSql.query(f'show tables') tdSql.checkRows(0) From b0f16e105566fbb149dfba4ddaaf5e7a835124f0 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 25 Jun 2022 14:27:19 +0800 Subject: [PATCH 44/72] feat: taos-tools update for3.0 (#14163) * feat: update taos-tools for 3.0 prepare for 3.0 [TD-13052] * feat: update taos-tools fix -I -s [TD-13052] --- tools/taos-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/taos-tools b/tools/taos-tools index 28a49b447f..a875a057d1 160000 --- a/tools/taos-tools +++ b/tools/taos-tools @@ -1 +1 @@ -Subproject commit 28a49b447f71c4f014ebbac858b7215b897d57fd +Subproject commit a875a057d1225d85c6323b9edaccc2b1a9641987 From e70ec32c118b0eb2ff77698631a140009e0836b5 Mon Sep 17 00:00:00 2001 From: jiacy-jcy Date: Sat, 25 Jun 2022 14:41:25 +0800 Subject: [PATCH 45/72] update --- tests/system-test/1-insert/table_param_ttl.py | 31 ++++++++++++++++--- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/tests/system-test/1-insert/table_param_ttl.py b/tests/system-test/1-insert/table_param_ttl.py index 850c323be0..49d6476d9c 100644 --- a/tests/system-test/1-insert/table_param_ttl.py +++ b/tests/system-test/1-insert/table_param_ttl.py @@ -25,18 +25,30 @@ class TDTestCase: self.stbname = 'stb' self.tbnum = 10 self.ttl_param = 1 - def ttl_check(self): + self.default_ttl = 100 + self.modify_ttl = 1 + def ttl_check_ntb(self): tdSql.prepare() + for i in range(self.tbnum): tdSql.execute(f'create table {self.ntbname}_{i} (ts timestamp,c0 int) ttl {self.ttl_param}') tdSql.query(f'show tables') tdSql.checkRows(self.tbnum) - tdSql.execute('reset query cache') sleep(self.updatecfgDict['ttlUnit']*self.ttl_param+self.updatecfgDict['ttlPushInterval']) tdSql.query(f'show tables') tdSql.checkRows(0) - + for i in range(self.tbnum): + tdSql.execute(f'create table {self.ntbname}_{i} (ts timestamp,c0 int) ttl {self.default_ttl}') + for i in range(int(self.tbnum/2)): + tdSql.execute(f'alter table {self.ntbname}_{i} ttl {self.modify_ttl}') + sleep(self.updatecfgDict['ttlUnit']*self.modify_ttl+self.updatecfgDict['ttlPushInterval']) + tdSql.query(f'show tables') + tdSql.checkRows(self.tbnum - int(self.tbnum/2)) + tdSql.execute('drop database db') + def ttl_check_ctb(self): + tdSql.prepare() tdSql.execute(f'create table {self.stbname} (ts timestamp,c0 int) tags(t0 int)') + for i in range(self.tbnum): tdSql.execute(f'create table {self.stbname}_{i} using {self.stbname} tags({i}) ttl {self.ttl_param}') tdSql.query(f'show tables') @@ -44,9 +56,20 @@ class TDTestCase: sleep(self.updatecfgDict['ttlUnit']*self.ttl_param+self.updatecfgDict['ttlPushInterval']) tdSql.query(f'show tables') tdSql.checkRows(0) + for i in range(self.tbnum): + tdSql.execute(f'create table {self.stbname}_{i} using {self.stbname} tags({i}) ttl {self.default_ttl}') + tdSql.query(f'show tables') + tdSql.checkRows(self.tbnum) + for i in range(int(self.tbnum/2)): + tdSql.execute(f'alter table {self.stbname}_{i} ttl {self.modify_ttl}') + sleep(self.updatecfgDict['ttlUnit']*self.modify_ttl+self.updatecfgDict['ttlPushInterval']) + tdSql.query(f'show tables') + tdSql.checkRows(self.tbnum - int(self.tbnum/2)) + tdSql.execute('drop database db') def run(self): - self.ttl_check() + self.ttl_check_ntb() + self.ttl_check_ctb() def stop(self): tdSql.close() From 91735042b74cc415be1d60aa264f045ffe79e914 Mon Sep 17 00:00:00 2001 From: jiacy-jcy Date: Sat, 25 Jun 2022 15:17:25 +0800 Subject: [PATCH 46/72] update bottom.py --- tests/system-test/2-query/bottom.py | 180 +++++++++++++++++----------- 1 file changed, 110 insertions(+), 70 deletions(-) diff --git a/tests/system-test/2-query/bottom.py b/tests/system-test/2-query/bottom.py index 1037b0a8f3..a95daf22f4 100644 --- a/tests/system-test/2-query/bottom.py +++ b/tests/system-test/2-query/bottom.py @@ -17,100 +17,140 @@ from util.log import * from util.cases import * from util.sql import * from util.common import * - +from util.sqlset import * class TDTestCase: def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) - + self.dbname = 'db_test' + self.setsql = TDSetSql() + self.ntbname = 'ntb' self.rowNum = 10 self.tbnum = 20 self.ts = 1537146000000 self.binary_str = 'taosdata' self.nchar_str = '涛思数据' - def bottom_check_base(self): - tdSql.prepare() - tdSql.execute('''create table stb(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 tinyint unsigned, col6 smallint unsigned, - col7 int unsigned, col8 bigint unsigned, col9 float, col10 double, col11 bool, col12 binary(20), col13 nchar(20)) tags(loc nchar(20))''') - tdSql.execute("create table stb_1 using stb tags('beijing')") - column_list = ['col1','col2','col3','col4','col5','col6','col7','col8'] - error_column_list = ['col11','col12','col13'] - error_param_list = [0,101] - for i in range(self.rowNum): - tdSql.execute(f"insert into stb_1 values(%d, %d, %d, %d, %d, %d, %d, %d, %d, %f, %f, %d, '{self.binary_str}%d', '{self.nchar_str}%d')" - % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) + self.column_dict = { + 'ts' : 'timestamp', + 'col1': 'tinyint', + 'col2': 'smallint', + 'col3': 'int', + 'col4': 'bigint', + 'col5': 'tinyint unsigned', + 'col6': 'smallint unsigned', + 'col7': 'int unsigned', + 'col8': 'bigint unsigned', + 'col9': 'float', + 'col10': 'double', + 'col11': 'bool', + 'col12': 'binary(20)', + 'col13': 'nchar(20)' + } - for i in column_list: - tdSql.query(f'select bottom({i},2) from stb_1') - tdSql.checkRows(2) - tdSql.checkEqual(tdSql.queryResult,[(2,),(1,)]) - for j in error_param_list: - tdSql.error(f'select bottom({i},{j}) from stb_1') - for i in error_column_list: - tdSql.error(f'select bottom({i},10) from stb_1') - tdSql.query("select ts,bottom(col1, 2),ts from stb_1 group by tbname") - tdSql.checkRows(2) - tdSql.query('select bottom(col2,1) from stb_1 interval(1y) order by col2') - tdSql.checkData(0,0,1) - - tdSql.error('select * from stb_1 where bottom(col2,1)=1') - tdSql.execute('drop database db') - def bottom_check_distribute(self): - # prepare data for vgroup 4 - dbname = tdCom.getLongName(5, "letters") + self.param_list = [1,100] + def insert_data(self,column_dict,tbname,row_num): + sql = '' + for k, v in column_dict.items(): + if v.lower() == 'timestamp' or v.lower() == 'tinyint' or v.lower() == 'smallint' or v.lower() == 'int' or v.lower() == 'bigint' or \ + v.lower() == 'tinyint unsigned' or v.lower() == 'smallint unsigned' or v.lower() == 'int unsigned' or v.lower() == 'bigint unsigned' or v.lower() == 'bool': + sql += '%d,' + elif v.lower() == 'float' or v.lower() == 'double': + sql += '%f,' + elif 'binary' in v.lower(): + sql += f'"{self.binary_str}%d",' + elif 'nchar' in v.lower(): + sql += f'"{self.nchar_str}%d",' + insert_sql = f'insert into {tbname} values({sql[:-1]})' + for i in range(row_num): + insert_list = [] + for k, v in column_dict.items(): + if v.lower() in[ 'tinyint' , 'smallint' , 'int', 'bigint' , 'tinyint unsigned' , 'smallint unsigned' , 'int unsigned' , 'bigint unsigned'] or\ + 'binary' in v.lower() or 'nchar' in v.lower(): + insert_list.append(0 + i) + elif v.lower() == 'float' or v.lower() == 'double': + insert_list.append(0.1 + i) + elif v.lower() == 'bool': + insert_list.append(i % 2) + elif v.lower() == 'timestamp': + insert_list.append(self.ts + i) + tdSql.execute(insert_sql%(tuple(insert_list))) + def bottom_check_data(self,tbname,tb_type): + new_column_dict = {} + for param in self.param_list: + for k,v in self.column_dict.items(): + if v.lower() in ['tinyint','smallint','int','bigint','tinyint unsigned','smallint unsigned','int unsigned','bigint unsigned']: + tdSql.query(f'select bottom({k},{param}) from {tbname} order by {k}') + if param >= self.rowNum: + if tb_type in ['normal_table','child_table']: + tdSql.checkRows(self.rowNum) + values_list = [] + for i in range(self.rowNum): + tp = (i,) + values_list.append(tp) + tdSql.checkEqual(tdSql.queryResult,values_list) + elif tb_type == 'stable': + tdSql.checkRows(param) + elif param < self.rowNum: + if tb_type in ['normal_table','child_table']: + tdSql.checkRows(param) + values_list = [] + for i in range(param): + tp = (i,) + values_list.append(tp) + tdSql.checkEqual(tdSql.queryResult,values_list) + elif tb_type == 'stable': + tdSql.checkRows(param) + for i in [self.param_list[0]-1,self.param_list[-1]+1]: + tdSql.error(f'select top({k},{i}) from {tbname}') + new_column_dict.update({k:v}) + elif v.lower() == 'bool' or 'binary' in v.lower() or 'nchar' in v.lower(): + tdSql.error(f'select top({k},{param}) from {tbname}') + tdSql.error(f'select * from {tbname} where top({k},{param})=1') + pass + def bottom_check_ntb(self): + tdSql.execute(f'create database if not exists {self.dbname} vgroups 1') + tdSql.execute(f'use {self.dbname}') + tdSql.execute(self.setsql.set_create_normaltable_sql(self.ntbname,self.column_dict)) + self.insert_data(self.column_dict,self.ntbname,self.rowNum) + self.bottom_check_data(self.ntbname,'normal_table') + tdSql.execute(f'drop database {self.dbname}') + def bottom_check_stb(self): stbname = tdCom.getLongName(5, "letters") - vgroup_num = 2 - child_table_num = 20 - tdSql.execute(f"create database if not exists {dbname} vgroups {vgroup_num}") - tdSql.execute(f'use {dbname}') - # build 20 child tables,every table insert 10 rows - tdSql.execute(f'''create table {stbname}(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 tinyint unsigned, col6 smallint unsigned, - col7 int unsigned, col8 bigint unsigned, col9 float, col10 double, col11 bool, col12 binary(20), col13 nchar(20)) tags(loc nchar(20))''') - for i in range(child_table_num): - tdSql.execute(f"create table {stbname}_{i} using {stbname} tags('beijing')") - tdSql.execute(f"insert into {stbname}_{i}(ts) values(%d)" % (self.ts - 1-i)) - column_list = ['col1','col2','col3','col4','col5','col6','col7','col8'] - error_column_list = ['col11','col12','col13'] - error_param_list = [0,101] - for i in [f'{stbname}', f'{dbname}.{stbname}']: - for j in column_list: - tdSql.query(f"select bottom({j},1) from {i}") - tdSql.checkRows(0) + tag_dict = { + 't0':'int' + } + tag_values = [ + f'1' + ] + tdSql.execute(f"create database if not exists {self.dbname} vgroups 2") + tdSql.execute(f'use {self.dbname}') + tdSql.execute(self.setsql.set_create_stable_sql(stbname,self.column_dict,tag_dict)) + for i in range(self.tbnum): + tdSql.execute(f"create table {stbname}_{i} using {stbname} tags({tag_values[0]})") + tdSql.execute(self.insert_data(self.column_dict,f'{stbname}_{i}',self.rowNum)) tdSql.query('show tables') vgroup_list = [] for i in range(len(tdSql.queryResult)): vgroup_list.append(tdSql.queryResult[i][6]) vgroup_list_set = set(vgroup_list) - for i in vgroup_list_set: vgroups_num = vgroup_list.count(i) - if vgroups_num >=2: + if vgroups_num >= 2: tdLog.info(f'This scene with {vgroups_num} vgroups is ok!') - continue else: - tdLog.exit(f'This scene does not meet the requirements with {vgroups_num} vgroup!\n') - for i in range(self.rowNum): - for j in range(child_table_num): - tdSql.execute(f"insert into {stbname}_{j} values(%d, %d, %d, %d, %d, %d, %d, %d, %d, %f, %f, %d, '{self.binary_str}%d', '{self.nchar_str}%d')" - % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) - for i in column_list: - tdSql.query(f'select bottom({i},2) from {stbname}') - tdSql.checkRows(2) - tdSql.checkEqual(tdSql.queryResult,[(1,),(1,)]) - for j in error_param_list: - tdSql.error(f'select bottom({i},{j}) from {stbname}') - for i in error_column_list: - tdSql.error(f'select bottom({i},10) from {stbname}') - - tdSql.execute(f'drop database {dbname}') + tdLog.exit( + 'This scene does not meet the requirements with {vgroups_num} vgroup!\n') + for i in range(self.tbnum): + self.bottom_check_data(f'{stbname}_{i}','child_table') + self.bottom_check_data(f'{stbname}','stable') + tdSql.execute(f'drop database {self.dbname}') + def run(self): - - self.bottom_check_base() - self.bottom_check_distribute() + self.bottom_check_ntb() + self.bottom_check_stb() - def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) From 6adc19d4437705552669c7459e80b763c7344de5 Mon Sep 17 00:00:00 2001 From: slzhou Date: Sat, 25 Jun 2022 15:20:11 +0800 Subject: [PATCH 47/72] fix: overlapping intervals problem --- source/libs/executor/src/timewindowoperator.c | 51 +++++++++++-------- 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 48b0b1c071..0ba898e4f7 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -3900,18 +3900,22 @@ _error: // merge interval operator typedef struct SMergeIntervalAggOperatorInfo { SIntervalAggOperatorInfo intervalAggOperatorInfo; - - SHashObj* groupIntervalHash; - void* groupIntervalIter; + SList* groupIntervals; + SListIter groupIntervalsIter; bool hasGroupId; uint64_t groupId; SSDataBlock* prefetchedBlock; bool inputBlocksFinished; } SMergeIntervalAggOperatorInfo; +typedef struct SGroupTimeWindow { + uint64_t groupId; + STimeWindow window; +} SGroupTimeWindow; + void destroyMergeIntervalOperatorInfo(void* param, int32_t numOfOutput) { SMergeIntervalAggOperatorInfo* miaInfo = (SMergeIntervalAggOperatorInfo*)param; - taosHashCleanup(miaInfo->groupIntervalHash); + tdListFree(miaInfo->groupIntervals); destroyIntervalOperatorInfo(&miaInfo->intervalAggOperatorInfo, numOfOutput); } @@ -3940,15 +3944,22 @@ static int32_t outputPrevIntervalResult(SOperatorInfo* pOperatorInfo, uint64_t t bool ascScan = (iaInfo->order == TSDB_ORDER_ASC); SExprSupp* pExprSup = &pOperatorInfo->exprSupp; - STimeWindow* prevWin = taosHashGet(miaInfo->groupIntervalHash, &tableGroupId, sizeof(tableGroupId)); - if (prevWin == NULL) { - taosHashPut(miaInfo->groupIntervalHash, &tableGroupId, sizeof(tableGroupId), newWin, sizeof(STimeWindow)); - return 0; - } + SGroupTimeWindow groupTimeWindow = {.groupId = tableGroupId, .window = *newWin}; + tdListAppend(miaInfo->groupIntervals, &groupTimeWindow); - if ((ascScan && newWin->skey > prevWin->skey || (!ascScan) && newWin->skey < prevWin->skey)) { - finalizeWindowResult(pOperatorInfo, tableGroupId, prevWin, pResultBlock); - taosHashPut(miaInfo->groupIntervalHash, &tableGroupId, sizeof(tableGroupId), newWin, sizeof(STimeWindow)); + SListIter iter = {0}; + tdListInitIter(miaInfo->groupIntervals, &iter, TD_LIST_FORWARD); + SListNode* listNode = NULL; + while ((listNode = tdListNext(&iter)) != NULL) { + SGroupTimeWindow* prevGrpWin = (SGroupTimeWindow*)listNode->data; + if (prevGrpWin->groupId != tableGroupId ) { + continue; + } + STimeWindow* prevWin = &prevGrpWin->window; + if ((ascScan && newWin->skey > prevWin->ekey || (!ascScan) && newWin->skey < prevWin->ekey)) { + finalizeWindowResult(pOperatorInfo, tableGroupId, prevWin, pResultBlock); + tdListPopNode(miaInfo->groupIntervals, listNode); + } } return 0; @@ -4075,6 +4086,7 @@ static SSDataBlock* doMergeIntervalAgg(SOperatorInfo* pOperator) { } if (pBlock == NULL) { + tdListInitIter(miaInfo->groupIntervals, &miaInfo->groupIntervalsIter, TD_LIST_FORWARD); miaInfo->inputBlocksFinished = true; break; } @@ -4100,14 +4112,12 @@ static SSDataBlock* doMergeIntervalAgg(SOperatorInfo* pOperator) { } if (miaInfo->inputBlocksFinished) { - void* win = taosHashIterate(miaInfo->groupIntervalHash, miaInfo->groupIntervalIter); - if (win != NULL) { - miaInfo->groupIntervalIter = win; + SListNode* listNode = tdListNext(&miaInfo->groupIntervalsIter); - size_t len = 0; - uint64_t* pTableGroupId = taosHashGetKey(win, &len); - finalizeWindowResult(pOperator, *pTableGroupId, win, pRes); - pRes->info.groupId = *pTableGroupId; + if (listNode != NULL) { + SGroupTimeWindow* grpWin = (SGroupTimeWindow*)(listNode->data); + finalizeWindowResult(pOperator, grpWin->groupId, &grpWin->window, pRes); + pRes->info.groupId = grpWin->groupId; } } @@ -4129,8 +4139,7 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SExprI goto _error; } - miaInfo->groupIntervalHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_NO_LOCK); - miaInfo->groupIntervalIter = NULL; + miaInfo->groupIntervals = tdListNew(sizeof(SGroupTimeWindow)); SIntervalAggOperatorInfo* iaInfo = &miaInfo->intervalAggOperatorInfo; From 90121ae43982ccb2214e742c5b8922723c0ad5ad Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Sat, 25 Jun 2022 15:26:29 +0800 Subject: [PATCH 48/72] fix(tmq): check stb existence when subscribing stb --- source/dnode/mnode/impl/src/mndTopic.c | 4 ++++ source/libs/stream/src/streamExec.c | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index a650ed29f1..8fcd345544 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -401,6 +401,10 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq * } } else if (pCreate->subType == TOPIC_SUB_TYPE__TABLE) { SStbObj *pStb = mndAcquireStb(pMnode, pCreate->subStbName); + if (pStb == NULL) { + terrno = TSDB_CODE_MND_STB_NOT_EXIST; + return -1; + } topicObj.stbUid = pStb->uid; } /*} else if (pCreate->subType == TOPIC_SUB_TYPE__DB) {*/ diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 8e7cac03a2..fe0f406f8d 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -109,7 +109,7 @@ static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) { if (type == STREAM_INPUT__TRIGGER) { blockDataDestroy(((SStreamTrigger*)data)->pBlock); taosFreeQitem(data); - } else if (type == STREAM_INPUT__DATA_BLOCK) { + } else if (type == STREAM_INPUT__DATA_BLOCK || type == STREAM_INPUT__DATA_RETRIEVE) { taosArrayDestroyEx(((SStreamDataBlock*)data)->blocks, (FDelete)tDeleteSSDataBlock); taosFreeQitem(data); } else if (type == STREAM_INPUT__DATA_SUBMIT) { From 1ba41e03ff12133ea1a60dd2e65013d37a026bd2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chappyguoxy=E2=80=9D?= <“happy_guoxy@163.com”> Date: Sat, 25 Jun 2022 15:48:25 +0800 Subject: [PATCH 49/72] test: refine query cases --- tests/system-test/2-query/json_tag.py | 102 ++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/tests/system-test/2-query/json_tag.py b/tests/system-test/2-query/json_tag.py index 2ef1b8dad2..9b96b6ebd0 100644 --- a/tests/system-test/2-query/json_tag.py +++ b/tests/system-test/2-query/json_tag.py @@ -11,12 +11,14 @@ # -*- coding: utf-8 -*- +import imp import sys import taos from util.log import tdLog from util.cases import tdCases from util.sql import tdSql import json +import os class TDTestCase: @@ -29,6 +31,9 @@ class TDTestCase: return def init(self, conn, logSql): + self.testcasePath = os.path.split(__file__)[0] + self.testcaseFilename = os.path.split(__file__)[-1] + os.system("rm -rf %s/%s.sql" % (self.testcasePath,self.testcaseFilename)) tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) @@ -557,6 +562,103 @@ class TDTestCase: tdSql.checkRows(3) tdSql.query("select round(dataint) from jsons1 where jtag->'tag1'>1") tdSql.checkRows(3) + + #math function + tdSql.query("select sin(dataint) from jsons1 where jtag->'tag1'>1;") + tdSql.checkRows(3) + tdSql.query("select cos(dataint) from jsons1 where jtag->'tag1'>1;") + tdSql.checkRows(3) + tdSql.query("select tan(dataint) from jsons1 where jtag->'tag1'>1;") + tdSql.checkRows(3) + tdSql.query("select asin(dataint) from jsons1 where jtag->'tag1'>1;") + tdSql.checkRows(3) + tdSql.query("select acos(dataint) from jsons1 where jtag->'tag1'>1;") + tdSql.checkRows(3) + tdSql.query("select atan(dataint) from jsons1 where jtag->'tag1'>1;") + tdSql.checkRows(3) + tdSql.query("select ceil(dataint) from jsons1 where jtag->'tag1'>1;") + tdSql.checkRows(3) + tdSql.query("select floor(dataint) from jsons1 where jtag->'tag1'>1;") + tdSql.checkRows(3) + tdSql.query("select round(dataint) from jsons1 where jtag->'tag1'>1;") + tdSql.checkRows(3) + tdSql.query("select abs(dataint) from jsons1 where jtag->'tag1'>1;") + tdSql.checkRows(3) + tdSql.query("select pow(dataint,5) from jsons1 where jtag->'tag1'>1;") + tdSql.checkRows(3) + tdSql.query("select log(dataint,10) from jsons1 where jtag->'tag1'>1;") + tdSql.checkRows(3) + tdSql.query("select sqrt(dataint) from jsons1 where jtag->'tag1'>1;") + tdSql.checkRows(3) + tdSql.query("select HISTOGRAM(dataint,'user_input','[1, 33, 555, 7777]',1) from jsons1 where jtag->'tag1'>1;") + tdSql.checkRows(3) + tdSql.query("select csum(dataint) from jsons1 where jtag->'tag1'>1;") + tdSql.checkRows(3) + tdSql.query("select mavg(dataint,1) from jsons1 where jtag->'tag1'>1;") + tdSql.checkRows(3) + tdSql.query("select statecount(dataint,'GE',10) from jsons1 where jtag->'tag1'>1;") + tdSql.checkRows(3) + tdSql.query("select stateduration(dataint,'GE',0) from jsons1 where jtag->'tag1'>1;") + tdSql.checkRows(3) + tdSql.query("select sample(dataint,3) from jsons1 where jtag->'tag1'>1;") + tdSql.checkRows(3) + tdSql.query("select HYPERLOGLOG(dataint) from jsons1 where jtag->'tag1'>1;") + tdSql.checkRows(1) + tdSql.query("select twa(dataint) from jsons1 where jtag->'tag1'>1;") + tdSql.checkRows(1) + + # function not ready + # tdSql.query("select tail(dataint,1) from jsons1 where jtag->'tag1'>1;") + # tdSql.checkRows(3) + # tdSql.query("select unique(dataint) from jsons1 where jtag->'tag1'>1;") + # tdSql.checkRows(3) + # tdSql.query("select mode(dataint) from jsons1 where jtag->'tag1'>1;") + # tdSql.checkRows(3) + # tdSql.query("select irate(dataint) from jsons1 where jtag->'tag1'>1;") + # tdSql.checkRows(1) + + #str function + tdSql.query("select upper(dataStr) from jsons1 where jtag->'tag1'>1;") + tdSql.checkRows(3) + tdSql.query("select ltrim(dataStr) from jsons1 where jtag->'tag1'>1;") + tdSql.checkRows(3) + tdSql.query("select lower(dataStr) from jsons1 where jtag->'tag1'>1;") + tdSql.checkRows(3) + tdSql.query("select rtrim(dataStr) from jsons1 where jtag->'tag1'>1;") + tdSql.checkRows(3) + tdSql.query("select LENGTH(dataStr) from jsons1 where jtag->'tag1'>1;") + tdSql.checkRows(3) + tdSql.query("select CHAR_LENGTH(dataStr) from jsons1 where jtag->'tag1'>1;") + tdSql.checkRows(3) + tdSql.query("select SUBSTR(dataStr,5) from jsons1 where jtag->'tag1'>1;") + tdSql.checkRows(3) + tdSql.query("select CONCAT(dataStr,dataStrBin) from jsons1 where jtag->'tag1'>1;") + tdSql.checkRows(3) + tdSql.query("select CONCAT_ws('adad!@!@%$^$%$^$%^a',dataStr,dataStrBin) from jsons1 where jtag->'tag1'>1;") + tdSql.checkRows(3) + tdSql.query("select CAST(dataStr as bigint) from jsons1 where jtag->'tag1'>1;") + tdSql.checkRows(3) + + #time function + tdSql.query("select now() from jsons1 where jtag->'tag1'>1;") + tdSql.checkRows(3) + tdSql.query("select today() from jsons1 where jtag->'tag1'>1;") + tdSql.checkRows(3) + tdSql.query("select TIMEZONE() from jsons1 where jtag->'tag1'>1;") + tdSql.checkRows(3) + tdSql.query("select TO_ISO8601(ts) from jsons1 where jtag->'tag1'>1;") + tdSql.checkRows(3) + tdSql.query("select TO_UNIXTIMESTAMP(datastr) from jsons1 where jtag->'tag1'>1;") + tdSql.checkRows(3) + tdSql.query("select TIMETRUNCATE(ts,1u) from jsons1 where jtag->'tag1'>1;") + tdSql.checkRows(3) + tdSql.query("select TIMEDIFF(ts,_c0) from jsons1 where jtag->'tag1'>1;") + tdSql.checkRows(3) + tdSql.query("select TIMEDIFF(ts,1u) from jsons1 where jtag->'tag1'>1;") + tdSql.checkRows(3) + tdSql.query("select ELAPSED(ts,1h) from jsons1 where jtag->'tag1'>1;") + tdSql.checkRows(1) + # # #test TD-12077 tdSql.execute("insert into jsons1_16 using jsons1 tags('{\"tag1\":\"收到货\",\"tag2\":\"\",\"tag3\":-2.111}') values(1591062628000, 2, NULL, '你就会', 'dws')") From 592d8e488f5c877f41b7ea3bed67956f3d94a4c4 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Sat, 25 Jun 2022 16:14:54 +0800 Subject: [PATCH 50/72] test: add sim full test --- tests/pytest/util/dnodes.py | 24 +++++---- tests/script/test-all.bat | 58 +++++++++++++++++++++ tests/system-test/6-cluster/5dnode1mnode.py | 6 +-- tests/system-test/6-cluster/5dnode2mnode.py | 5 +- tests/system-test/7-tmq/tmqCommon.py | 8 ++- tests/system-test/7-tmq/tmqError.py | 5 +- tests/system-test/test-all.bat | 2 +- tests/system-test/test.py | 46 +++++++++++++--- tools/shell/src/shellEngine.c | 1 + 9 files changed, 128 insertions(+), 27 deletions(-) create mode 100644 tests/script/test-all.bat diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index be3454f78f..a38b14a52d 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -508,7 +508,6 @@ class TDDnode: def stoptaosd(self): if (not self.remoteIP == ""): - print("123") self.remoteExec(self.cfgDict, "tdDnodes.dnodes[%d].running=1\ntdDnodes.dnodes[%d].stop()"%(self.index-1,self.index-1)) tdLog.info("stop dnode%d"%self.index) return @@ -518,18 +517,21 @@ class TDDnode: toBeKilled = "valgrind.bin" if self.running != 0: - psCmd = "ps -ef|grep -w %s| grep dnode%d|grep -v grep | awk '{print $2}'" % (toBeKilled,self.index) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - while(processID): - killCmd = "kill -INT %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) + if platform.system().lower() == 'windows': + os.system("wmic process where \"name='taosd.exe' and CommandLine like '%%dnode%d%%'\" get processId | xargs echo | awk '{print $2}' | xargs taskkill -f -pid"%self.index) + else: + psCmd = "ps -ef|grep -w %s| grep dnode%d|grep -v grep | awk '{print $2}'" % (toBeKilled,self.index) processID = subprocess.check_output( psCmd, shell=True).decode("utf-8") - if self.valgrind: - time.sleep(2) + + while(processID): + killCmd = "kill -INT %s > /dev/null 2>&1" % processID + os.system(killCmd) + time.sleep(1) + processID = subprocess.check_output( + psCmd, shell=True).decode("utf-8") + if self.valgrind: + time.sleep(2) self.running = 0 tdLog.debug("dnode:%d is stopped by kill -INT" % (self.index)) diff --git a/tests/script/test-all.bat b/tests/script/test-all.bat new file mode 100644 index 0000000000..7a1a4bc7fa --- /dev/null +++ b/tests/script/test-all.bat @@ -0,0 +1,58 @@ +@echo off +SETLOCAL EnableDelayedExpansion +for /F "tokens=1,2 delims=#" %%a in ('"prompt #$H#$E# & echo on & for %%b in (1) do rem"') do ( set "DEL=%%a") +set /a a=0 +echo Windows Taosd Full Test +set /a exitNum=0 +rm -rf failed.txt +set caseFile="jenkins\\basic.txt" +if not "%2" == "" ( + set caseFile="%2" +) +for /F "usebackq tokens=*" %%i in (!caseFile!) do ( + set line=%%i + if "!line:~,9!" == "./test.sh" ( + set /a a+=1 + echo !a! Processing %%i + call :GetTimeSeconds !time! + set time1=!_timeTemp! + echo Start at !time! + call !line:./test.sh=wtest.bat! > result_!a!.txt 2>error_!a!.txt + if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && set /a exitNum=8 && echo %%i >>failed.txt ) else ( call :colorEcho 0a "Success" &echo. ) + ) +) +exit !exitNum! + +:colorEcho +set timeNow=%time% +call :GetTimeSeconds %timeNow% +set time2=%_timeTemp% +set /a interTime=%time2% - %time1% +echo End at %timeNow% , cast %interTime%s +echo off + "%~2" +findstr /v /a:%1 /R "^$" "%~2" nul +del "%~2" > nul 2>&1i +goto :eof + +:GetTimeSeconds +set tt=%1 +set tt=%tt:.= % +set tt=%tt::= % +set tt=%tt: 0= % +set /a index=1 +for %%a in (%tt%) do ( + if !index! EQU 1 ( + set /a hh=%%a + )^ + else if !index! EQU 2 ( + set /a mm=%%a + + )^ + else if !index! EQU 3 ( + set /a ss=%%a + ) + set /a index=index+1 +) +set /a _timeTemp=(%hh%*60+%mm%)*60+%ss% +goto :eof diff --git a/tests/system-test/6-cluster/5dnode1mnode.py b/tests/system-test/6-cluster/5dnode1mnode.py index 75134224db..5f4ab7357b 100644 --- a/tests/system-test/6-cluster/5dnode1mnode.py +++ b/tests/system-test/6-cluster/5dnode1mnode.py @@ -20,7 +20,7 @@ class MyDnodes(TDDnodes): self.simDeployed = False class TDTestCase: - + noConn = True def init(self,conn ,logSql): tdLog.debug(f"start to excute {__file__}") self.TDDnodes = None @@ -40,7 +40,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] @@ -81,7 +81,7 @@ class TDTestCase: dnode_id = dnode.cfgDict["fqdn"] + ":" +dnode.cfgDict["serverPort"] dnode_first_host = dnode.cfgDict["firstEp"].split(":")[0] dnode_first_port = dnode.cfgDict["firstEp"].split(":")[-1] - cmd = f" taos -h {dnode_first_host} -P {dnode_first_port} -s ' create dnode \"{dnode_id} \" ' ;" + cmd = f"{self.getBuildPath()}/build/bin/taos -h {dnode_first_host} -P {dnode_first_port} -s \"create dnode \\\"{dnode_id}\\\"\"" print(cmd) os.system(cmd) diff --git a/tests/system-test/6-cluster/5dnode2mnode.py b/tests/system-test/6-cluster/5dnode2mnode.py index d3cde987c6..e08e738be6 100644 --- a/tests/system-test/6-cluster/5dnode2mnode.py +++ b/tests/system-test/6-cluster/5dnode2mnode.py @@ -20,6 +20,7 @@ class MyDnodes(TDDnodes): self.simDeployed = False class TDTestCase: + noConn = True def init(self,conn ,logSql): tdLog.debug(f"start to excute {__file__}") @@ -40,7 +41,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] @@ -85,7 +86,7 @@ class TDTestCase: dnode_id = dnode.cfgDict["fqdn"] + ":" +dnode.cfgDict["serverPort"] dnode_first_host = dnode.cfgDict["firstEp"].split(":")[0] dnode_first_port = dnode.cfgDict["firstEp"].split(":")[-1] - cmd = f" taos -h {dnode_first_host} -P {dnode_first_port} -s ' create dnode \"{dnode_id} \" ' ;" + cmd = f"{self.getBuildPath()}/build/bin/taos -h {dnode_first_host} -P {dnode_first_port} -s \"create dnode \\\"{dnode_id}\\\"\"" print(cmd) os.system(cmd) diff --git a/tests/system-test/7-tmq/tmqCommon.py b/tests/system-test/7-tmq/tmqCommon.py index b8aa78e3ac..788ae3474c 100644 --- a/tests/system-test/7-tmq/tmqCommon.py +++ b/tests/system-test/7-tmq/tmqCommon.py @@ -86,7 +86,13 @@ class TMQCom: shellCmd += '--tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all --num-callers=20 -v --workaround-gcc296-bugs=yes ' if (platform.system().lower() == 'windows'): - shellCmd = 'mintty -h never -w hide ' + buildPath + '\\build\\bin\\tmq_sim.exe -c ' + cfgPath + processorName = buildPath + '\\build\\bin\\tmq_sim.exe' + if alias != 0: + processorNameNew = buildPath + '\\build\\bin\\tmq_sim_new.exe' + shellCmd = 'cp %s %s'%(processorName, processorNameNew) + os.system(shellCmd) + processorName = processorNameNew + shellCmd = 'mintty -h never ' + processorName + ' -c ' + cfgPath shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) shellCmd += "> nul 2>&1 &" else: diff --git a/tests/system-test/7-tmq/tmqError.py b/tests/system-test/7-tmq/tmqError.py index 5b5658d528..bd8ec565d8 100644 --- a/tests/system-test/7-tmq/tmqError.py +++ b/tests/system-test/7-tmq/tmqError.py @@ -288,7 +288,10 @@ class TDTestCase: tdLog.exit("tmq consume rows error!") tdSql.query("drop topic %s"%topicFromStb1) - os.system('pkill tmq_sim') + if (platform.system().lower() == 'windows'): + os.system("TASKKILL /F /IM tmq_sim.exe") + else: + os.system('pkill tmq_sim') tdLog.printNoPrefix("======== test case 1 end ...... ") diff --git a/tests/system-test/test-all.bat b/tests/system-test/test-all.bat index 275cbeebbb..adc9e0ce28 100644 --- a/tests/system-test/test-all.bat +++ b/tests/system-test/test-all.bat @@ -2,7 +2,7 @@ SETLOCAL EnableDelayedExpansion for /F "tokens=1,2 delims=#" %%a in ('"prompt #$H#$E# & echo on & for %%b in (1) do rem"') do ( set "DEL=%%a") set /a a=0 -if %1 == full ( +if "%1" == "full" ( echo Windows Taosd Full Test set /a exitNum=0 del /Q /F failed.txt diff --git a/tests/system-test/test.py b/tests/system-test/test.py index 8a8356449c..35f8ea953c 100644 --- a/tests/system-test/test.py +++ b/tests/system-test/test.py @@ -23,6 +23,7 @@ import platform import socket import threading from distutils.log import warn as printf +from tkinter import N from fabric2 import Connection sys.path.append("../pytest") from util.log import * @@ -187,9 +188,9 @@ if __name__ == "__main__": tdLog.info("Procedures for tdengine deployed in %s" % (host)) if platform.system().lower() == 'windows': + fileName = fileName.replace("/", os.sep) if (masterIp == "" and not fileName[0:12] == "0-others\\udf"): threading.Thread(target=checkRunTimeError,daemon=True).start() - tdCases.logSql(logSql) tdLog.info("Procedures for testing self-deployment") tdDnodes.init(deployPath, masterIp) tdDnodes.setTestCluster(testCluster) @@ -208,18 +209,46 @@ if __name__ == "__main__": uModule = importlib.import_module(moduleName) try: ucase = uModule.TDTestCase() - if ((json.dumps(updateCfgDict) == '{}') and (ucase.updatecfgDict is not None)): + if ((json.dumps(updateCfgDict) == '{}') and hasattr(ucase, 'updatecfgDict')): updateCfgDict = ucase.updatecfgDict updateCfgDictStr = "-d %s"%base64.b64encode(json.dumps(updateCfgDict).encode()).decode() except Exception as r: print(r) else: pass - tdDnodes.deploy(1,updateCfgDict) - tdDnodes.start(1) - conn = taos.connect( - host="%s"%(host), - config=tdDnodes.sim.getCfgDir()) + if dnodeNums == 1 : + tdDnodes.deploy(1,updateCfgDict) + tdDnodes.start(1) + tdCases.logSql(logSql) + else : + tdLog.debug("create an cluster with %s nodes and make %s dnode as independent mnode"%(dnodeNums,mnodeNums)) + dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums,mnodeNums=mnodeNums) + tdDnodes = ClusterDnodes(dnodeslist) + tdDnodes.init(deployPath, masterIp) + tdDnodes.setTestCluster(testCluster) + tdDnodes.setValgrind(valgrind) + tdDnodes.stopAll() + for dnode in tdDnodes.dnodes: + tdDnodes.deploy(dnode.index,{}) + for dnode in tdDnodes.dnodes: + tdDnodes.starttaosd(dnode.index) + tdCases.logSql(logSql) + conn = taos.connect( + host, + config=tdDnodes.getSimCfgPath()) + print(tdDnodes.getSimCfgPath(),host) + cluster.create_dnode(conn) + try: + if cluster.check_dnode(conn) : + print("check dnode ready") + except Exception as r: + print(r) + if ucase is not None and hasattr(ucase, 'noConn') and ucase.noConn == True: + conn = None + else: + conn = taos.connect( + host="%s"%(host), + config=tdDnodes.sim.getCfgDir()) if is_test_framework: tdCases.runOneWindows(conn, fileName) else: @@ -307,4 +336,5 @@ if __name__ == "__main__": tdCases.runOneLinux(conn, sp[0] + "_" + "restart.py") else: tdLog.info("not need to query") - conn.close() + if conn is not None: + conn.close() diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 8a017d378d..8bc99a2665 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -156,6 +156,7 @@ void shellRunSingleCommandImp(char *command) { } fname = sptr + 2; + while (*fname == ' ') fname++; *sptr = '\0'; } From 77b365f0ccd6cf68c58e24d2e8afecc4c6a63678 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sat, 25 Jun 2022 16:27:05 +0800 Subject: [PATCH 51/72] refactor(sync): do not replicate when one replica --- source/libs/sync/src/syncMain.c | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index c5af72c971..3100d0525c 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -914,6 +914,9 @@ void syncNodeStart(SSyncNode* pSyncNode) { syncNodeBecomeLeader(pSyncNode, "one replica start"); // Raft 3.6.2 Committing entries from previous terms + syncNodeAppendNoop(pSyncNode); + syncMaybeAdvanceCommitIndex(pSyncNode); + return; } @@ -1662,6 +1665,12 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde // change isStandBy to normal (election timeout) if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { syncNodeBecomeLeader(pSyncNode, tmpbuf); + + // Raft 3.6.2 Committing entries from previous terms + syncNodeReplicate(pSyncNode); + syncNodeAppendNoop(pSyncNode); + syncMaybeAdvanceCommitIndex(pSyncNode); + } else { syncNodeBecomeFollower(pSyncNode, tmpbuf); } @@ -1807,16 +1816,9 @@ void syncNodeBecomeLeader(SSyncNode* pSyncNode, const char* debugStr) { // stop elect timer syncNodeStopElectTimer(pSyncNode); - // start replicate right now! - syncNodeReplicate(pSyncNode); - // start heartbeat timer syncNodeStartHeartbeatTimer(pSyncNode); - // append noop - syncNodeAppendNoop(pSyncNode); - syncMaybeAdvanceCommitIndex(pSyncNode); // maybe only one replica - // trace log do { int32_t debugStrLen = strlen(debugStr); @@ -1841,9 +1843,9 @@ void syncNodeCandidate2Leader(SSyncNode* pSyncNode) { syncNodeLog2("==state change syncNodeCandidate2Leader==", pSyncNode); // Raft 3.6.2 Committing entries from previous terms - - // do not use this - // syncNodeEqNoop(pSyncNode); + syncNodeReplicate(pSyncNode); + syncNodeAppendNoop(pSyncNode); + syncMaybeAdvanceCommitIndex(pSyncNode); } void syncNodeFollower2Candidate(SSyncNode* pSyncNode) { From 10e90ce973830815b616e0ba28b626766a1d4e96 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Sat, 25 Jun 2022 16:12:00 +0800 Subject: [PATCH 52/72] feat(stream): auto pull data --- include/common/tcommon.h | 3 +- include/common/tdatablock.h | 3 + include/libs/stream/tstream.h | 2 +- include/libs/stream/tstreamUpdate.h | 3 + source/common/src/tdatablock.c | 78 +++- source/dnode/snode/src/snode.c | 2 +- source/dnode/vnode/src/tq/tqRead.c | 6 +- source/libs/executor/inc/executorimpl.h | 11 +- source/libs/executor/src/scanoperator.c | 59 ++- source/libs/executor/src/timewindowoperator.c | 382 ++++++++++++++---- source/libs/stream/src/streamDispatch.c | 3 +- source/libs/stream/src/streamExec.c | 15 +- source/libs/stream/src/streamUpdate.c | 20 + .../tsim/stream/distributeInterval0.sim | 8 +- tests/script/tsim/stream/partitionby.sim | 40 +- tests/script/tsim/stream/schedSnode.sim | 6 +- 16 files changed, 519 insertions(+), 122 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index a14f7eff8a..928fe0aa0e 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -42,12 +42,13 @@ enum { typedef enum EStreamType { STREAM_NORMAL = 1, STREAM_INVERT, - STREAM_REPROCESS, + STREAM_CLEAR, STREAM_INVALID, STREAM_GET_ALL, STREAM_DELETE, STREAM_RETRIEVE, STREAM_PUSH_DATA, + STREAM_PUSH_EMPTY, } EStreamType; typedef struct { diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 2a0d4e7ff6..2e2c7d1700 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -224,6 +224,7 @@ size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize); int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n); int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src); +int32_t copyDataBlock(SSDataBlock* dst, const SSDataBlock* src); SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData); SSDataBlock* createDataBlock(); int32_t blockDataAppendColInfo(SSDataBlock* pBlock, SColumnInfoData* pColInfoData); @@ -236,6 +237,8 @@ void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData); void blockDebugShowData(const SArray* dataBlocks, const char* flag); +// for debug +char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** dumpBuf); int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, STSchema* pTSchema, int32_t vgId, tb_uid_t suid); diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 594344ba8a..db928f194c 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -319,7 +319,7 @@ static FORCE_INLINE int32_t streamTaskInput(SStreamTask* pTask, SStreamQueueItem return -1; } taosWriteQitem(pTask->inputQueue->queue, pSubmitClone); - } else if (pItem->type == STREAM_INPUT__DATA_BLOCK) { + } else if (pItem->type == STREAM_INPUT__DATA_BLOCK || pItem->type == STREAM_INPUT__DATA_RETRIEVE) { taosWriteQitem(pTask->inputQueue->queue, pItem); } else if (pItem->type == STREAM_INPUT__CHECKPOINT) { taosWriteQitem(pTask->inputQueue->queue, pItem); diff --git a/include/libs/stream/tstreamUpdate.h b/include/libs/stream/tstreamUpdate.h index 398851a09f..21a1515d8f 100644 --- a/include/libs/stream/tstreamUpdate.h +++ b/include/libs/stream/tstreamUpdate.h @@ -32,12 +32,15 @@ typedef struct SUpdateInfo { int64_t interval; int64_t watermark; TSKEY minTS; + SScalableBf* pCloseWinSBF; } SUpdateInfo; SUpdateInfo *updateInfoInitP(SInterval* pInterval, int64_t watermark); SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t watermark); bool updateInfoIsUpdated(SUpdateInfo *pInfo, tb_uid_t tableId, TSKEY ts); void updateInfoDestroy(SUpdateInfo *pInfo); +void updateInfoAddCloseWindowSBF(SUpdateInfo *pInfo); +void updateInfoDestoryColseWinSBF(SUpdateInfo *pInfo); #ifdef __cplusplus } diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 593f8c5c0b..cc995c4d64 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1164,7 +1164,7 @@ int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows) int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) { int32_t code = 0; - ASSERT(numOfRows > 0); + //ASSERT(numOfRows > 0); if (numOfRows == 0) { return TSDB_CODE_SUCCESS; @@ -1230,6 +1230,32 @@ int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src) { return 0; } +int32_t copyDataBlock(SSDataBlock* dst, const SSDataBlock* src) { + ASSERT(src != NULL && dst != NULL); + + blockDataCleanup(dst); + int32_t code = blockDataEnsureCapacity(dst, src->info.rows); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + return code; + } + + size_t numOfCols = taosArrayGetSize(src->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pDst = taosArrayGet(dst->pDataBlock, i); + SColumnInfoData* pSrc = taosArrayGet(src->pDataBlock, i); + if (pSrc->pData == NULL) { + continue; + } + + colDataAssign(pDst, pSrc, src->info.rows, &src->info); + } + + dst->info.rows = src->info.rows; + dst->info.window = src->info.window; + return TSDB_CODE_SUCCESS; +} + SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) { if (pDataBlock == NULL) { return NULL; @@ -1627,6 +1653,56 @@ void blockDebugShowData(const SArray* dataBlocks, const char* flag) { } } +// for debug +char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) { + int32_t size = 2048; + *pDataBuf = taosMemoryCalloc(size, 1); + char* dumpBuf = *pDataBuf; + char pBuf[128] = {0}; + int32_t colNum = taosArrayGetSize(pDataBlock->pDataBlock); + int32_t rows = pDataBlock->info.rows; + int32_t len = 0; + len += snprintf(dumpBuf + len, size - len, "\n%s |block type %d |child id %d|\n", flag, (int32_t)pDataBlock->info.type, pDataBlock->info.childId); + for (int32_t j = 0; j < rows; j++) { + len += snprintf(dumpBuf + len, size - len, "%s |", flag); + for (int32_t k = 0; k < colNum; k++) { + SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); + void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); + if (colDataIsNull(pColInfoData, rows, j, NULL)) { + len += snprintf(dumpBuf + len, size - len, " %15s |", "NULL"); + continue; + } + switch (pColInfoData->info.type) { + case TSDB_DATA_TYPE_TIMESTAMP: + formatTimestamp(pBuf, *(uint64_t*)var, TSDB_TIME_PRECISION_MILLI); + len += snprintf(dumpBuf + len, size - len, " %25s |", pBuf); + break; + case TSDB_DATA_TYPE_INT: + len += snprintf(dumpBuf + len, size - len, " %15d |", *(int32_t*)var); + break; + case TSDB_DATA_TYPE_UINT: + len += snprintf(dumpBuf + len, size - len, " %15u |", *(uint32_t*)var); + break; + case TSDB_DATA_TYPE_BIGINT: + len += snprintf(dumpBuf + len, size - len, " %15ld |", *(int64_t*)var); + break; + case TSDB_DATA_TYPE_UBIGINT: + len += snprintf(dumpBuf + len, size - len, " %15lu |", *(uint64_t*)var); + break; + case TSDB_DATA_TYPE_FLOAT: + len += snprintf(dumpBuf + len, size - len, " %15f |", *(float*)var); + break; + case TSDB_DATA_TYPE_DOUBLE: + len += snprintf(dumpBuf + len, size - len, " %15lf |", *(double*)var); + break; + } + } + len += snprintf(dumpBuf + len, size - len, "\n"); + } + len += snprintf(dumpBuf + len, size - len, "%s |end\n", flag); + return dumpBuf; +} + /** * @brief TODO: Assume that the final generated result it less than 3M * diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 3a92cba773..b13e654caf 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -257,7 +257,7 @@ int32_t sndProcessSMsg(SSnode *pSnode, SRpcMsg *pMsg) { case TDMT_STREAM_TASK_RECOVER_RSP: return sndProcessTaskRecoverRsp(pSnode, pMsg); case TDMT_STREAM_RETRIEVE_RSP: - return sndProcessTaskRecoverRsp(pSnode, pMsg); + return sndProcessTaskRetrieveRsp(pSnode, pMsg); default: ASSERT(0); } diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 6184d8f810..96f4eb3fd9 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -109,11 +109,15 @@ int32_t tqReadHandleSetMsg(STqReadHandle* pReadHandle, SSubmitReq* pMsg, int64_t } bool tqNextDataBlock(STqReadHandle* pHandle) { + if (pHandle->pMsg == NULL) return false; while (1) { if (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) < 0) { return false; } - if (pHandle->pBlock == NULL) return false; + if (pHandle->pBlock == NULL) { + pHandle->pMsg = NULL; + return false; + } if (pHandle->tbIdHash == NULL) { return true; diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index d8d231e952..36f81e86ff 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -293,6 +293,7 @@ typedef enum EStreamScanMode { STREAM_SCAN_FROM_RES, STREAM_SCAN_FROM_UPDATERES, STREAM_SCAN_FROM_DATAREADER, + STREAM_SCAN_FROM_DATAREADER_RETRIEVE, } EStreamScanMode; typedef struct SCatchSupporter { @@ -348,7 +349,9 @@ typedef struct SStreamBlockScanInfo { SArray* childIds; SessionWindowSupporter sessionSup; bool assignBlockUid; // assign block uid to groupId, temporarily used for generating rollup SMA. - int32_t scanWinIndex; + int32_t scanWinIndex; // for state operator + int32_t pullDataResIndex; + SSDataBlock* pPullDataRes; // pull data SSDataBlock } SStreamBlockScanInfo; typedef struct SSysTableScanInfo { @@ -427,8 +430,13 @@ typedef struct SStreamFinalIntervalOperatorInfo { STimeWindowAggSupp twAggSup; SArray* pChildren; SSDataBlock* pUpdateRes; + bool returnUpdate; SPhysiNode* pPhyNode; // create new child bool isFinal; + SHashObj* pPullDataMap; + SArray* pPullWins; // SPullWindowInfo + int32_t pullIndex; + SSDataBlock* pPullDataRes; } SStreamFinalIntervalOperatorInfo; typedef struct SAggOperatorInfo { @@ -851,6 +859,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex); int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* groupKey); +SSDataBlock* createPullDataBlock(); #ifdef __cplusplus } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 7affac76d2..edf98deed4 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -776,15 +776,14 @@ static bool isStateWindow(SStreamBlockScanInfo* pInfo) { return pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE; } -static bool prepareDataScan(SStreamBlockScanInfo* pInfo) { - SSDataBlock* pSDB = pInfo->pUpdateRes; +static bool prepareDataScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pSDB, int32_t tsColIndex, int32_t* pRowIndex) { STimeWindow win = { .skey = INT64_MIN, .ekey = INT64_MAX, }; bool needRead = false; - if (!isStateWindow(pInfo) && pInfo->updateResIndex < pSDB->info.rows) { - SColumnInfoData* pColDataInfo = taosArrayGet(pSDB->pDataBlock, pInfo->primaryTsIndex); + if (!isStateWindow(pInfo) && (*pRowIndex) < pSDB->info.rows) { + SColumnInfoData* pColDataInfo = taosArrayGet(pSDB->pDataBlock, tsColIndex); TSKEY* tsCols = (TSKEY*)pColDataInfo->pData; SResultRowInfo dumyInfo; dumyInfo.cur.pageId = -1; @@ -793,14 +792,14 @@ static bool prepareDataScan(SStreamBlockScanInfo* pInfo) { int64_t gap = pInfo->sessionSup.gap; int32_t winIndex = 0; SResultWindowInfo* pCurWin = - getSessionTimeWindow(pAggSup, tsCols[pInfo->updateResIndex], INT64_MIN, pSDB->info.groupId, gap, &winIndex); + getSessionTimeWindow(pAggSup, tsCols[(*pRowIndex)], INT64_MIN, pSDB->info.groupId, gap, &winIndex); win = pCurWin->win; - pInfo->updateResIndex += - updateSessionWindowInfo(pCurWin, tsCols, NULL, pSDB->info.rows, pInfo->updateResIndex, gap, NULL); + (*pRowIndex) += + updateSessionWindowInfo(pCurWin, tsCols, NULL, pSDB->info.rows, (*pRowIndex), gap, NULL); } else { - win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[pInfo->updateResIndex], &pInfo->interval, + win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[(*pRowIndex)], &pInfo->interval, pInfo->interval.precision, NULL); - pInfo->updateResIndex += getNumOfRowsInTimeWindow(&pSDB->info, tsCols, pInfo->updateResIndex, win.ekey, + (*pRowIndex) += getNumOfRowsInTimeWindow(&pSDB->info, tsCols, (*pRowIndex), win.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); } needRead = true; @@ -823,6 +822,9 @@ static bool prepareDataScan(SStreamBlockScanInfo* pInfo) { pTableScanInfo->cond.twindows[0] = win; pTableScanInfo->curTWinIdx = 0; // tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0); + // if (!pTableScanInfo->dataReader) { + // return false; + // } pTableScanInfo->scanTimes = 0; pTableScanInfo->currentGroupId = -1; return true; @@ -862,12 +864,12 @@ static uint64_t getGroupId(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_ */ } -static SSDataBlock* doDataScan(SStreamBlockScanInfo* pInfo) { +static SSDataBlock* doDataScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pSDB, int32_t tsColIndex, int32_t* pRowIndex) { while (1) { SSDataBlock* pResult = NULL; pResult = doTableScan(pInfo->pSnapshotReadOp); if (pResult == NULL) { - if (prepareDataScan(pInfo)) { + if (prepareDataScan(pInfo, pSDB, tsColIndex, pRowIndex)) { // scan next window data pResult = doTableScan(pInfo->pSnapshotReadOp); } @@ -916,7 +918,7 @@ static void setUpdateData(SStreamBlockScanInfo* pInfo, SSDataBlock* pBlock, SSDa pUpdateBlock->info.rows = i; pInfo->tsArrayIndex += i; pUpdateBlock->info.groupId = pInfo->groupId; - pUpdateBlock->info.type = STREAM_REPROCESS; + pUpdateBlock->info.type = STREAM_CLEAR; blockDataUpdateTsWindow(pUpdateBlock, 0); } // all rows have same group id @@ -970,6 +972,14 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { int32_t current = pInfo->validBlockIndex++; SSDataBlock* pBlock = taosArrayGetP(pInfo->pBlockLists, current); blockDataUpdateTsWindow(pBlock, 0); + if (pBlock->info.type == STREAM_RETRIEVE) { + pInfo->blockType = STREAM_DATA_TYPE_SUBMIT_BLOCK; + pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RETRIEVE; + copyDataBlock(pInfo->pPullDataRes, pBlock); + pInfo->pullDataResIndex = 0; + prepareDataScan(pInfo, pInfo->pPullDataRes, 0, &pInfo->pullDataResIndex); + updateInfoAddCloseWindowSBF(pInfo->pUpdateInfo); + } return pBlock; } else if (pInfo->blockType == STREAM_DATA_TYPE_SUBMIT_BLOCK) { if (pInfo->scanMode == STREAM_SCAN_FROM_RES) { @@ -979,28 +989,39 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { } else if (pInfo->scanMode == STREAM_SCAN_FROM_UPDATERES) { pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER; if (!isStateWindow(pInfo)) { - prepareDataScan(pInfo); + prepareDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex); } return pInfo->pUpdateRes; + } else if (pInfo->scanMode == STREAM_SCAN_FROM_DATAREADER_RETRIEVE) { + SSDataBlock* pSDB = doDataScan(pInfo, pInfo->pPullDataRes, 0, &pInfo->pullDataResIndex); + if (pSDB != NULL) { + getUpdateDataBlock(pInfo, true, pSDB, NULL); + pSDB->info.type = STREAM_PUSH_DATA; + return pSDB; + } + pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER; } else { if (isStateWindow(pInfo) && taosArrayGetSize(pInfo->sessionSup.pStreamAggSup->pScanWindow) > 0) { pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER; pInfo->updateResIndex = pInfo->pUpdateRes->info.rows; - prepareDataScan(pInfo); + prepareDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex); } if (pInfo->scanMode == STREAM_SCAN_FROM_DATAREADER) { - SSDataBlock* pSDB = doDataScan(pInfo); + SSDataBlock* pSDB = doDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex); if (pSDB == NULL) { setUpdateData(pInfo, pInfo->pRes, pInfo->pUpdateRes); if (pInfo->pUpdateRes->info.rows > 0) { if (!isStateWindow(pInfo)) { - prepareDataScan(pInfo); + // Todo(liuyao) mybe can delete this. + bool test = prepareDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex); + ASSERT(test == false); } return pInfo->pUpdateRes; } else { pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; } } else { + pSDB->info.type = STREAM_NORMAL; getUpdateDataBlock(pInfo, true, pSDB, NULL); return pSDB; } @@ -1070,10 +1091,12 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { taosArrayDestroy(block.pDataBlock); if (pInfo->pRes->pDataBlock == NULL) { // TODO add log + updateInfoDestoryColseWinSBF(pInfo->pUpdateInfo); pOperator->status = OP_EXEC_DONE; pTaskInfo->code = terrno; return NULL; } + // currently only the tbname pseudo column if (pInfo->numOfPseudoExpr > 0) { addTagPseudoColumnData(&pInfo->readHandle, pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, pInfo->pRes); @@ -1091,12 +1114,13 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { pOperator->resultInfo.totalRows += pBlockInfo->rows; if (pBlockInfo->rows == 0) { + updateInfoDestoryColseWinSBF(pInfo->pUpdateInfo); pOperator->status = OP_EXEC_DONE; } else if (pInfo->pUpdateInfo) { pInfo->tsArrayIndex = 0; getUpdateDataBlock(pInfo, true, pInfo->pRes, pInfo->pUpdateRes); if (pInfo->pUpdateRes->info.rows > 0) { - if (pInfo->pUpdateRes->info.type == STREAM_REPROCESS) { + if (pInfo->pUpdateRes->info.type == STREAM_CLEAR) { pInfo->updateResIndex = 0; pInfo->scanMode = STREAM_SCAN_FROM_UPDATERES; } else if (pInfo->pUpdateRes->info.type == STREAM_INVERT) { @@ -1209,6 +1233,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; pInfo->sessionSup = (SessionWindowSupporter){.pStreamAggSup = NULL, .gap = -1}; pInfo->groupId = 0; + pInfo->pPullDataRes = createPullDataBlock(); pOperator->name = "StreamBlockScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 0ba898e4f7..b7d39207d8 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -15,6 +15,7 @@ #include "executorimpl.h" #include "function.h" #include "functionMgt.h" +#include "tcompare.h" #include "tdatablock.h" #include "tfill.h" #include "ttime.h" @@ -26,6 +27,16 @@ typedef enum SResultTsInterpType { #define IS_FINAL_OP(op) ((op)->isFinal) +typedef struct SWinRes { + TSKEY ts; + uint64_t groupId; +} SWinRes; + +typedef struct SPullWindowInfo { + STimeWindow window; + uint64_t groupId; +} SPullWindowInfo; + static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator); static int64_t* extractTsCol(SSDataBlock* pBlock, const SIntervalAggOperatorInfo* pInfo); @@ -684,11 +695,13 @@ static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t num } void printDataBlock(SSDataBlock* pBlock, const char* flag) { - if (pBlock == NULL) return; - SArray* blocks = taosArrayInit(1, sizeof(SSDataBlock)); - taosArrayPush(blocks, pBlock); - blockDebugShowData(blocks, flag); - taosArrayDestroy(blocks); + if (pBlock == NULL){ + qDebug("======printDataBlock Block is Null"); + return; + } + char *pBuf = NULL; + qDebug("%s", dumpBlockData(pBlock, flag, &pBuf)); + taosMemoryFree(pBuf); } typedef int64_t (*__get_value_fn_t)(void* data, int32_t index); @@ -1217,30 +1230,40 @@ void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf, SExprS } } -void doClearWindow(SAggSupporter* pAggSup, SExprSupp* pSup, char* pData, int16_t bytes, uint64_t groupId, +bool doClearWindow(SAggSupporter* pAggSup, SExprSupp* pSup, char* pData, int16_t bytes, uint64_t groupId, int32_t numOfOutput) { SET_RES_WINDOW_KEY(pAggSup->keyBuf, pData, bytes, groupId); SResultRowPosition* p1 = (SResultRowPosition*)taosHashGet(pAggSup->pResultRowHashTable, pAggSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); if (!p1) { // window has been closed - return; + return false; } doClearWindowImpl(p1, pAggSup->pResultBuf, pSup, numOfOutput); + return true; } static void doClearWindows(SAggSupporter* pAggSup, SExprSupp* pSup1, SInterval* pInterval, int32_t tsIndex, int32_t numOfOutput, SSDataBlock* pBlock, SArray* pUpWins) { - SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, tsIndex); - TSKEY* tsCols = (TSKEY*)pColDataInfo->pData; + SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, tsIndex); + TSKEY* tsCols = (TSKEY*)pTsCol->pData; + uint64_t* pGpDatas = NULL; + if (pBlock->info.type == STREAM_RETRIEVE) { + SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, 2); + pGpDatas = (uint64_t*)pGpCol->pData; + } int32_t step = 0; for (int32_t i = 0; i < pBlock->info.rows; i += step) { SResultRowInfo dumyInfo; dumyInfo.cur.pageId = -1; STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[i], pInterval, pInterval->precision, NULL); step = getNumOfRowsInTimeWindow(&pBlock->info, tsCols, i, win.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); - doClearWindow(pAggSup, pSup1, (char*)&win.skey, sizeof(TKEY), pBlock->info.groupId, numOfOutput); - if (pUpWins) { + uint64_t groupId = pBlock->info.groupId; + if (pGpDatas) { + groupId = pGpDatas[i]; + } + bool res = doClearWindow(pAggSup, pSup1, (char*)&win.skey, sizeof(TKEY), groupId, numOfOutput); + if (pUpWins && res) { taosArrayPush(pUpWins, &win); } } @@ -1268,8 +1291,8 @@ bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup) { return pSup->maxTs != INT64_MIN && pWin->ekey < pSup->maxTs - pSup->waterMark; } -static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, SInterval* pInterval, - SArray* closeWins) { +static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, + SInterval* pInterval, SHashObj* pPullDataMap, SArray* closeWins) { void* pIte = NULL; size_t keyLen = 0; while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) { @@ -1280,10 +1303,20 @@ static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, SResultRowInfo dumyInfo; dumyInfo.cur.pageId = -1; STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, ts, pInterval, pInterval->precision, NULL); + SWinRes winRe = {.ts = win.skey, .groupId = groupId,}; + void* chIds = taosHashGet(pPullDataMap, &winRe, sizeof(SWinRes)); if (isCloseWindow(&win, pSup)) { - char keyBuf[GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY))]; - SET_RES_WINDOW_KEY(keyBuf, &ts, sizeof(TSKEY), groupId); - taosHashRemove(pHashMap, keyBuf, keyLen); + if (chIds && pPullDataMap) { + SArray* chAy = *(SArray**) chIds; + int32_t size = taosArrayGetSize(chAy); + qInfo("======window %ld wait child size:%d", win.skey ,size); + for (int32_t i = 0; i < size; i++) { + qInfo("======window %ld wait chid id:%d", win.skey ,*(int32_t*)taosArrayGet(chAy, i)); + } + continue; + } else if (pPullDataMap) { + qInfo("======close window %ld", win.skey); + } SResultRowPosition* pPos = (SResultRowPosition*)pIte; if (pSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { int32_t code = saveResult(ts, pPos->pageId, pPos->offset, groupId, closeWins); @@ -1291,11 +1324,25 @@ static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, return code; } } + char keyBuf[GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY))]; + SET_RES_WINDOW_KEY(keyBuf, &ts, sizeof(TSKEY), groupId); + taosHashRemove(pHashMap, keyBuf, keyLen); } } return TSDB_CODE_SUCCESS; } +static void closeChildIntervalWindow(SArray* pChildren, TSKEY maxTs) { + int32_t size = taosArrayGetSize(pChildren); + for (int32_t i = 0; i < size; i++) { + SOperatorInfo* pChildOp = taosArrayGetP(pChildren, i); + SStreamFinalIntervalOperatorInfo* pChInfo = pChildOp->info; + ASSERT(pChInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE); + pChInfo->twAggSup.maxTs = TMAX(pChInfo->twAggSup.maxTs, maxTs); + closeIntervalWindow(pChInfo->aggSup.pResultRowHashTable, &pChInfo->twAggSup, &pChInfo->interval, NULL, NULL); + } +} + static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { SIntervalAggOperatorInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -1324,7 +1371,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { break; } - if (pBlock->info.type == STREAM_REPROCESS) { + if (pBlock->info.type == STREAM_CLEAR) { doClearWindows(&pInfo->aggSup, &pOperator->exprSupp, &pInfo->interval, 0, pOperator->exprSupp.numOfExprs, pBlock, NULL); qDebug("%s clear existed time window results for updates checked", GET_TASKID(pTaskInfo)); @@ -1345,7 +1392,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey); hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, MAIN_SCAN, pUpdated); } - closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, pUpdated); + closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, NULL, pUpdated); finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset); initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); @@ -1373,6 +1420,11 @@ void destroyStreamFinalIntervalOperatorInfo(void* param, int32_t numOfOutput) { SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)param; cleanupBasicInfo(&pInfo->binfo); cleanupAggSup(&pInfo->aggSup); + //it should be empty. + taosHashCleanup(pInfo->pPullDataMap); + taosArrayDestroy(pInfo->pPullWins); + blockDataDestroy(pInfo->pPullDataRes); + if (pInfo->pChildren) { int32_t size = taosArrayGetSize(pInfo->pChildren); for (int32_t i = 0; i < size; i++) { @@ -2164,6 +2216,24 @@ bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup) { return p1 == NULL; } +int32_t getNexWindowPos(SInterval* pInterval, SDataBlockInfo* pBlockInfo, TSKEY* tsCols, + int32_t startPos, TSKEY eKey, STimeWindow* pNextWin) { + int32_t forwardRows = getNumOfRowsInTimeWindow(pBlockInfo, tsCols, startPos, + eKey, binarySearchForKey, NULL, TSDB_ORDER_ASC); + int32_t prevEndPos = forwardRows - 1 + startPos; + return getNextQualifiedWindow(pInterval, pNextWin, pBlockInfo, tsCols, prevEndPos, TSDB_ORDER_ASC); +} + +void addPullWindow(SHashObj* pMap, SWinRes* pWinRes, int32_t size) { + SArray* childIds = taosArrayInit(8, sizeof(int32_t)); + for (int32_t i = 0; i < size; i++) { + taosArrayPush(childIds, &i); + } + taosHashPut(pMap, pWinRes, sizeof(SWinRes), &childIds, sizeof(void*)); +} + +static int32_t getChildIndex(SSDataBlock* pBlock) { return pBlock->info.childId; } + static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, uint64_t tableGroupId, SArray* pUpdated) { SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)pOperatorInfo->info; @@ -2177,35 +2247,59 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc SResultRow* pResult = NULL; int32_t forwardRows = 0; - if (pSDataBlock->pDataBlock != NULL) { - SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); - tsCols = (int64_t*)pColDataInfo->pData; - } else { - return; - } + ASSERT(pSDataBlock->pDataBlock != NULL); + SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); + tsCols = (int64_t*)pColDataInfo->pData; int32_t startPos = ascScan ? 0 : (pSDataBlock->info.rows - 1); TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols); STimeWindow nextWin = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, pInfo->interval.precision, NULL); while (1) { - if (IS_FINAL_OP(pInfo) && isCloseWindow(&nextWin, &pInfo->twAggSup) && - isDeletedWindow(&nextWin, tableGroupId, &pInfo->aggSup)) { - SArray* pUpWins = taosArrayInit(8, sizeof(STimeWindow)); - taosArrayPush(pUpWins, &nextWin); - rebuildIntervalWindow(pInfo, pSup, pUpWins, pInfo->binfo.pRes->info.groupId, pSup->numOfExprs, - pOperatorInfo->pTaskInfo); - taosArrayDestroy(pUpWins); + if (IS_FINAL_OP(pInfo) && isCloseWindow(&nextWin, &pInfo->twAggSup) && pInfo->pChildren) { + bool ignore = true; + SWinRes winRes = {.ts = nextWin.skey, .groupId = tableGroupId,}; + void* chIds = taosHashGet(pInfo->pPullDataMap, &winRes, sizeof(SWinRes)); + if (isDeletedWindow(&nextWin, tableGroupId, &pInfo->aggSup) && !chIds) { + SPullWindowInfo pull = {.window = nextWin, .groupId = tableGroupId}; + // add pull data request + taosArrayPush(pInfo->pPullWins, &pull); + addPullWindow(pInfo->pPullDataMap, &winRes, taosArrayGetSize(pInfo->pChildren)); + } else { + int32_t index = -1; + SArray* chArray = NULL; + if (chIds) { + chArray = *(void**) chIds; + int32_t chId = getChildIndex(pSDataBlock); + index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ); + } + if (index != -1 && pSDataBlock->info.type == STREAM_PUSH_DATA) { + taosArrayRemove(chArray, index); + if (taosArrayGetSize(chArray) == 0) { + // pull data is over + taosHashRemove(pInfo->pPullDataMap, &winRes, sizeof(SWinRes)); + } + } + if ( index == -1 || pSDataBlock->info.type == STREAM_PUSH_DATA) { + ignore = false; + } + } + + if (ignore) { + startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin); + if (startPos < 0) { + break; + } + continue; + } } + int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, true, &pResult, tableGroupId, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); if (code != TSDB_CODE_SUCCESS || pResult == NULL) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } - SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t)); - pos->groupId = tableGroupId; - pos->pos = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset}; - *(int64_t*)pos->key = pResult->win.skey; + forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdated) { @@ -2230,14 +2324,17 @@ static void clearStreamIntervalOperator(SStreamFinalIntervalOperatorInfo* pInfo) initResultRowInfo(&pInfo->binfo.resultRowInfo); } -static void clearUpdateDataBlock(SSDataBlock* pBlock) { +static void clearSpecialDataBlock(SSDataBlock* pBlock) { + if (pBlock->info.rows <= 0) { + return; + } blockDataCleanup(pBlock); } void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex) { // ASSERT(pDest->info.capacity >= pSource->info.rows); blockDataEnsureCapacity(pDest, pSource->info.rows); - clearUpdateDataBlock(pDest); + clearSpecialDataBlock(pDest); SColumnInfoData* pDestCol = taosArrayGet(pDest->pDataBlock, 0); SColumnInfoData* pSourceCol = taosArrayGet(pSource->pDataBlock, tsColIndex); @@ -2254,7 +2351,63 @@ void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsCol blockDataUpdateTsWindow(pDest, 0); } -static int32_t getChildIndex(SSDataBlock* pBlock) { return pBlock->info.childId; } +static bool needBreak(SStreamFinalIntervalOperatorInfo* pInfo) { + int32_t size = taosArrayGetSize(pInfo->pPullWins); + if (pInfo->pullIndex < size) { + return true; + } + return false; +} + +static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pBlock) { + clearSpecialDataBlock(pBlock); + int32_t size = taosArrayGetSize(array); + if (size - (*pIndex) == 0) { + return; + } + blockDataEnsureCapacity(pBlock, size - (*pIndex) ); + ASSERT(3 <= taosArrayGetSize(pBlock->pDataBlock)); + for (; (*pIndex) < size; (*pIndex)++) { + SPullWindowInfo* pWin = taosArrayGet(array, (*pIndex) ); + SColumnInfoData* pStartTs = (SColumnInfoData*) taosArrayGet(pBlock->pDataBlock, 0); + colDataAppend(pStartTs, pBlock->info.rows, (const char*)&pWin->window.skey, false); + + SColumnInfoData* pEndTs = (SColumnInfoData*) taosArrayGet(pBlock->pDataBlock, 1); + colDataAppend(pEndTs, pBlock->info.rows, (const char*)&pWin->window.ekey, false); + + SColumnInfoData* pGroupId = (SColumnInfoData*) taosArrayGet(pBlock->pDataBlock, 2); + colDataAppend(pGroupId, pBlock->info.rows, (const char*)&pWin->groupId, false); + pBlock->info.rows++; + } + if ((*pIndex) == size) { + *pIndex = 0; + taosArrayClear(array); + } + blockDataUpdateTsWindow(pBlock, 0); +} + +void processPushEmpty(SSDataBlock* pBlock, SHashObj* pMap) { + SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, 0); + TSKEY* tsData = (TSKEY*)pStartCol->pData; + SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, 2); + uint64_t* groupIdData = (uint64_t*)pGroupCol->pData; + int32_t chId = getChildIndex(pBlock); + for (int32_t i = 0; i < pBlock->info.rows; i++) { + SWinRes winRes = {.ts = tsData[i], .groupId = groupIdData[i]}; + void* chIds = taosHashGet(pMap, &winRes, sizeof(SWinRes)); + if (chIds) { + SArray* chArray = *(SArray**) chIds; + int32_t index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ); + if (index != -1) { + taosArrayRemove(chArray, index); + if (taosArrayGetSize(chArray) == 0) { + // pull data is over + taosHashRemove(pMap, &winRes, sizeof(SWinRes)); + } + } + } + } +} static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { SStreamFinalIntervalOperatorInfo* pInfo = pOperator->info; @@ -2270,28 +2423,50 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); if (pInfo->binfo.pRes->info.rows == 0) { pOperator->status = OP_EXEC_DONE; - if (IS_FINAL_OP(pInfo) || pInfo->pUpdateRes->info.rows == 0) { - if (!IS_FINAL_OP(pInfo)) { - // semi interval operator clear disk buffer - clearStreamIntervalOperator(pInfo); - } - return NULL; + if (!IS_FINAL_OP(pInfo)) { + // semi interval operator clear disk buffer + clearStreamIntervalOperator(pInfo); } + return NULL; + } + printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); + return pInfo->binfo.pRes; + } else { + doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); + if (pInfo->binfo.pRes->info.rows != 0) { + printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); + return pInfo->binfo.pRes; + } + if (pInfo->pUpdateRes->info.rows != 0 && pInfo->returnUpdate) { + pInfo->returnUpdate = false; + ASSERT(!IS_FINAL_OP(pInfo)); + printDataBlock(pInfo->pUpdateRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); // process the rest of the data - pOperator->status = OP_OPENED; return pInfo->pUpdateRes; } - return pInfo->binfo.pRes; + doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes); + if (pInfo->pPullDataRes->info.rows != 0) { + // process the rest of the data + ASSERT(IS_FINAL_OP(pInfo)); + printDataBlock(pInfo->pPullDataRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); + return pInfo->pPullDataRes; + } } while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { - clearUpdateDataBlock(pInfo->pUpdateRes); + clearSpecialDataBlock(pInfo->pUpdateRes); + pOperator->status = OP_RES_TO_RETURN; + qInfo("Stream Final Interval return data"); break; } + printDataBlock(pBlock, IS_FINAL_OP(pInfo) ? "interval Final recv" : "interval Semi recv"); + maxTs = TMAX(maxTs, pBlock->info.window.ekey); - if (pBlock->info.type == STREAM_REPROCESS) { + if (pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_PUSH_DATA || pBlock->info.type == STREAM_INVALID) { + pInfo->binfo.pRes->info.type = pBlock->info.type; + } else if (pBlock->info.type == STREAM_CLEAR) { SArray* pUpWins = taosArrayInit(8, sizeof(STimeWindow)); doClearWindows(&pInfo->aggSup, pSup, &pInfo->interval, pInfo->primaryTsIndex, pOperator->exprSupp.numOfExprs, pBlock, pUpWins); @@ -2310,11 +2485,25 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { } removeResults(pUpWins, pUpdated); copyUpdateDataBlock(pInfo->pUpdateRes, pBlock, pInfo->primaryTsIndex); + pInfo->returnUpdate = true; taosArrayDestroy(pUpWins); break; } else if (pBlock->info.type == STREAM_GET_ALL && IS_FINAL_OP(pInfo)) { getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdated); continue; + } else if (pBlock->info.type == STREAM_RETRIEVE && !IS_FINAL_OP(pInfo)) { + SArray* pUpWins = taosArrayInit(8, sizeof(STimeWindow)); + doClearWindows(&pInfo->aggSup, pSup, &pInfo->interval, 0, pOperator->exprSupp.numOfExprs, + pBlock, pUpWins); + removeResults(pUpWins, pUpdated); + taosArrayDestroy(pUpWins); + if (taosArrayGetSize(pUpdated) > 0) { + break; + } + continue; + } else if (pBlock->info.type == STREAM_PUSH_EMPTY && IS_FINAL_OP(pInfo)) { + processPushEmpty(pBlock, pInfo->pPullDataMap); + continue; } setInputDataBlock(pOperator, pSup->pCtx, pBlock, pInfo->order, MAIN_SCAN, true); @@ -2334,31 +2523,70 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { SStreamFinalIntervalOperatorInfo* pChInfo = pChildOp->info; setInputDataBlock(pChildOp, pChildOp->exprSupp.pCtx, pBlock, pChInfo->order, MAIN_SCAN, true); doHashInterval(pChildOp, pBlock, pBlock->info.groupId, NULL); - pChInfo->twAggSup.maxTs = TMAX(pChInfo->twAggSup.maxTs, pBlock->info.window.ekey); + + if (needBreak(pInfo)) { + break; + } } - maxTs = TMAX(maxTs, pBlock->info.window.ekey); } pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); if (IS_FINAL_OP(pInfo)) { - closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, pUpdated); + closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, + &pInfo->interval, pInfo->pPullDataMap, pUpdated); + closeChildIntervalWindow(pInfo->pChildren, pInfo->twAggSup.maxTs); } finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset); initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - pOperator->status = OP_RES_TO_RETURN; - if (pInfo->binfo.pRes->info.rows == 0) { - pOperator->status = OP_EXEC_DONE; - if (pInfo->pUpdateRes->info.rows == 0) { - return NULL; - } + if (pInfo->binfo.pRes->info.rows != 0) { + printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); + return pInfo->binfo.pRes; + } + + if (pInfo->pUpdateRes->info.rows != 0 && pInfo->returnUpdate) { + pInfo->returnUpdate = false; + ASSERT(!IS_FINAL_OP(pInfo)); + printDataBlock(pInfo->pUpdateRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); // process the rest of the data - pOperator->status = OP_OPENED; return pInfo->pUpdateRes; } - return pInfo->binfo.pRes; + + doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes); + if (pInfo->pPullDataRes->info.rows != 0) { + // process the rest of the data + ASSERT(IS_FINAL_OP(pInfo)); + printDataBlock(pInfo->pPullDataRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); + return pInfo->pPullDataRes; + } + // ASSERT(false); + return NULL; +} + +SSDataBlock* createPullDataBlock() { + SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); + pBlock->info.hasVarCol = false; + pBlock->info.groupId = 0; + pBlock->info.rows = 0; + pBlock->info.type = STREAM_RETRIEVE; + pBlock->info.rowSize = sizeof(TSKEY) + sizeof(TSKEY) + sizeof(uint64_t); + + pBlock->pDataBlock = taosArrayInit(3, sizeof(SColumnInfoData)); + SColumnInfoData infoData = {0}; + infoData.info.type = TSDB_DATA_TYPE_TIMESTAMP; + infoData.info.bytes = sizeof(TSKEY); + // window start ts + taosArrayPush(pBlock->pDataBlock, &infoData); + // window end ts + taosArrayPush(pBlock->pDataBlock, &infoData); + + infoData.info.type = TSDB_DATA_TYPE_UBIGINT; + infoData.info.bytes = sizeof(uint64_t); + taosArrayPush(pBlock->pDataBlock, &infoData); + + return pBlock; } SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, @@ -2412,23 +2640,30 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, goto _error; } } - // semi interval operator does not catch result pInfo->pUpdateRes = createResDataBlock(pPhyNode->pOutputDataBlockDesc); - pInfo->pUpdateRes->info.type = STREAM_REPROCESS; + pInfo->pUpdateRes->info.type = STREAM_CLEAR; blockDataEnsureCapacity(pInfo->pUpdateRes, 128); + pInfo->returnUpdate = false; + pInfo->pPhyNode = (SPhysiNode*)nodesCloneNode((SNode*)pPhyNode); if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) { pInfo->isFinal = true; pOperator->name = "StreamFinalIntervalOperator"; } else { + // semi interval operator does not catch result pInfo->isFinal = false; pOperator->name = "StreamSemiIntervalOperator"; } - if (!IS_FINAL_OP(pInfo)) { + if (!IS_FINAL_OP(pInfo) || numOfChild == 0) { pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE; } + pInfo->pPullWins = taosArrayInit(8, sizeof(SPullWindowInfo)); + pInfo->pullIndex = 0; + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + pInfo->pPullDataMap = taosHashInit(64, hashFn, false, HASH_NO_LOCK); + pInfo->pPullDataRes = createPullDataBlock(); pOperator->operatorType = pPhyNode->type; pOperator->blocking = true; @@ -2811,11 +3046,6 @@ void compactTimeWindow(SStreamSessionAggOperatorInfo* pInfo, int32_t startIndex, } } -typedef struct SWinRes { - TSKEY ts; - uint64_t groupId; -} SWinRes; - static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SHashObj* pStUpdated, SHashObj* pStDeleted, bool hasEndTs) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -3027,14 +3257,14 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { } else if (pOperator->status == OP_RES_TO_RETURN) { doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator); if (pInfo->pDelRes->info.rows > 0) { - /*printDataBlock(pInfo->pDelRes, "session del");*/ + printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo)? "Final Session" : "Single Session"); return pInfo->pDelRes; } doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); if (pBInfo->pRes->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); } - /*printDataBlock(pBInfo->pRes, "session insert");*/ + printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo)? "Final Session" : "Single Session"); return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; } @@ -3048,7 +3278,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { break; } - if (pBlock->info.type == STREAM_REPROCESS) { + if (pBlock->info.type == STREAM_CLEAR) { SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo)); doClearSessionWindows(&pInfo->streamAggSup, &pOperator->exprSupp, pBlock, 0, pOperator->exprSupp.numOfExprs, pInfo->gap, pWins); @@ -3102,11 +3332,11 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator); if (pInfo->pDelRes->info.rows > 0) { - /*printDataBlock(pInfo->pDelRes, "session del");*/ + printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo)? "Final Session" : "Single Session"); return pInfo->pDelRes; } doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); - /*printDataBlock(pBInfo->pRes, "session insert");*/ + printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo)? "Final Session" : "Single Session"); return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; } @@ -3169,11 +3399,11 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { - clearUpdateDataBlock(pInfo->pUpdateRes); + clearSpecialDataBlock(pInfo->pUpdateRes); break; } - if (pBlock->info.type == STREAM_REPROCESS) { + if (pBlock->info.type == STREAM_CLEAR) { SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo)); doClearSessionWindows(&pInfo->streamAggSup, pSup, pBlock, 0, pSup->numOfExprs, pInfo->gap, pWins); removeSessionResults(pStUpdated, pWins); @@ -3236,7 +3466,7 @@ SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream } else { pInfo->isFinal = false; pInfo->pUpdateRes = createResDataBlock(pPhyNode->pOutputDataBlockDesc); - pInfo->pUpdateRes->info.type = STREAM_REPROCESS; + pInfo->pUpdateRes->info.type = STREAM_CLEAR; blockDataEnsureCapacity(pInfo->pUpdateRes, 128); pOperator->name = "StreamSessionSemiAggOperator"; pOperator->fpSet = @@ -3554,7 +3784,7 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { break; } - if (pBlock->info.type == STREAM_REPROCESS) { + if (pBlock->info.type == STREAM_CLEAR) { doClearStateWindows(&pInfo->streamAggSup, pBlock, pInfo->primaryTsIndex, &pInfo->stateCol, pInfo->stateCol.slotId, pSeUpdated, pInfo->pSeDeleted); continue; diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 0c9d46f055..a5e9b8edd9 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -115,6 +115,7 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) .srcNodeId = pTask->nodeId, .srcTaskId = pTask->taskId, .pRetrieve = pRetrieve, + .retrieveLen = dataStrLen, }; int32_t sz = taosArrayGetSize(pTask->childEpInfo); @@ -146,7 +147,7 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) .code = 0, .msgType = TDMT_STREAM_RETRIEVE, .pCont = buf, - .contLen = len, + .contLen = sizeof(SMsgHead) + len, }; if (tmsgSendReq(&pEpInfo->epSet, &rpcMsg) < 0) { diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index fe0f406f8d..c75e6c004a 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -45,11 +45,16 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) ASSERT(false); } if (output == NULL) { - if (pItem->type == STREAM_INPUT__DATA_RETRIEVE && !hasData) { - SSDataBlock block = {0}; - block.info.type = STREAM_PUSH_DATA; - block.info.childId = pTask->selfChildId; - taosArrayPush(pRes, &block); + if (pItem->type == STREAM_INPUT__DATA_RETRIEVE) { + //SSDataBlock block = {0}; + //block.info.type = STREAM_PUSH_EMPTY; + //block.info.childId = pTask->selfChildId; + SStreamDataBlock* pRetrieveBlock = (SStreamDataBlock*)data; + ASSERT(taosArrayGetSize(pRetrieveBlock->blocks) == 1); + SSDataBlock* pBlock = createOneDataBlock(taosArrayGet(pRetrieveBlock->blocks, 0), true); + pBlock->info.type = STREAM_PUSH_EMPTY; + pBlock->info.childId = pTask->selfChildId; + taosArrayPush(pRes, pBlock); } break; } diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index ada391b40a..3d64cec8d8 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -119,6 +119,7 @@ SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t waterma taosArrayPush(pInfo->pTsBuckets, &dumy); } pInfo->numBuckets = DEFAULT_BUCKET_SIZE; + pInfo->pCloseWinSBF = NULL; return pInfo; } @@ -154,6 +155,9 @@ bool updateInfoIsUpdated(SUpdateInfo *pInfo, tb_uid_t tableId, TSKEY ts) { TSKEY maxTs = *(TSKEY *)taosArrayGet(pInfo->pTsBuckets, index); if (ts < maxTs - pInfo->watermark) { // this window has been closed. + if (pInfo->pCloseWinSBF) { + return tScalableBfPut(pInfo->pCloseWinSBF, &ts, sizeof(TSKEY)); + } return true; } @@ -193,3 +197,19 @@ void updateInfoDestroy(SUpdateInfo *pInfo) { taosArrayDestroy(pInfo->pTsSBFs); taosMemoryFree(pInfo); } + +void updateInfoAddCloseWindowSBF(SUpdateInfo *pInfo) { + if (pInfo->pCloseWinSBF) { + return; + } + int64_t rows = adjustExpEntries(pInfo->interval * ROWS_PER_MILLISECOND); + pInfo->pCloseWinSBF = tScalableBfInit(rows, DEFAULT_FALSE_POSITIVE); +} + +void updateInfoDestoryColseWinSBF(SUpdateInfo *pInfo) { + if (!pInfo || !pInfo->pCloseWinSBF) { + return; + } + tScalableBfDestroy(pInfo->pCloseWinSBF); + pInfo->pCloseWinSBF = NULL; +} diff --git a/tests/script/tsim/stream/distributeInterval0.sim b/tests/script/tsim/stream/distributeInterval0.sim index 91ce49bc8c..3e38df2c89 100644 --- a/tests/script/tsim/stream/distributeInterval0.sim +++ b/tests/script/tsim/stream/distributeInterval0.sim @@ -80,17 +80,17 @@ endi if $data03 != 4 then print ======$data03 - return -1 + goto loop1 endi if $data04 != 52 then print ======$data04 - return -1 + goto loop1 endi if $data05 != 13 then print ======$data05 - return -1 + goto loop1 endi # row 1 @@ -179,7 +179,7 @@ sql use test1; sql create stable st(ts timestamp, a int, b int , c int) tags(ta int,tb int,tc int); sql create table ts1 using st tags(1,1,1); sql create table ts2 using st tags(2,2,2); -sql create stream stream_t2 trigger at_once into streamtST1 as select _wstartts, count(*) c1, count(a) c2 , sum(a) c3 , max(b) c5, min(c) c6 from st interval(10s) ; +sql create stream stream_t2 trigger at_once watermark 20s into streamtST1 as select _wstartts, count(*) c1, count(a) c2 , sum(a) c3 , max(b) c5, min(c) c6 from st interval(10s) ; sql insert into ts1 values(1648791211000,1,2,3); sql insert into ts1 values(1648791222001,2,2,3); diff --git a/tests/script/tsim/stream/partitionby.sim b/tests/script/tsim/stream/partitionby.sim index b84a01eb4a..c634ad85ee 100644 --- a/tests/script/tsim/stream/partitionby.sim +++ b/tests/script/tsim/stream/partitionby.sim @@ -74,7 +74,7 @@ sql create stable st(ts timestamp,a int,b int,c int,id int) tags(ta int,tb int,t sql create table ts1 using st tags(1,1,1); sql create table ts2 using st tags(2,2,2); -sql create stream stream_t2 trigger at_once into streamtST as select _wstartts, count(*) c1, count(a) c2 , sum(a) c3 , max(b) c5, min(c) c6, max(id) c7 from st partition by ta interval(10s) ; +sql create stream stream_t2 trigger at_once watermark 20s into streamtST as select _wstartts, count(*) c1, count(a) c2 , sum(a) c3 , max(b) c5, min(c) c6, max(id) c7 from st partition by ta interval(10s) ; sql insert into ts1 values(1648791211000,1,2,3,1); sql insert into ts1 values(1648791222001,2,2,3,2); sql insert into ts2 values(1648791211000,1,2,3,3); @@ -83,16 +83,16 @@ sql insert into ts2 values(1648791222001,2,2,3,4); sql insert into ts2 values(1648791222002,2,2,3,5); sql insert into ts2 values(1648791222002,2,2,3,6); -sql insert into ts1 values(1648791211000,1,2,3,1); -sql insert into ts1 values(1648791222001,2,2,3,2); -sql insert into ts2 values(1648791211000,1,2,3,3); -sql insert into ts2 values(1648791222001,2,2,3,4); +sql insert into ts1 values(1648791211000,1,2,3,7); +sql insert into ts1 values(1648791222001,2,2,3,8); +sql insert into ts2 values(1648791211000,1,2,3,9); +sql insert into ts2 values(1648791222001,2,2,3,10); $loop_count = 0 loop2: sleep 300 -sql select * from streamtST; +sql select * from streamtST order by c7 asc; $loop_count = $loop_count + 1 if $loop_count == 10 then @@ -104,8 +104,18 @@ print =====data01=$data01 goto loop2 endi -if $data02 != 1 then -print =====data02=$data02 +if $data11 != 1 then +print =====data11=$data11 +goto loop2 +endi + +if $data21 != 1 then +print =====data21=$data21 +goto loop2 +endi + +if $data31 != 2 then +print =====data31=$data31 goto loop2 endi @@ -114,8 +124,18 @@ print =====data03=$data03 goto loop2 endi -if $data04 != 2 then -print =====data04=$data04 +if $data13 != 2 then +print =====data13=$data13 +goto loop2 +endi + +if $data23 != 1 then +print =====data23=$data23 +goto loop2 +endi + +if $data33 != 4 then +print =====data33=$data33 goto loop2 endi diff --git a/tests/script/tsim/stream/schedSnode.sim b/tests/script/tsim/stream/schedSnode.sim index dbf714a96f..dbdaaf65d0 100644 --- a/tests/script/tsim/stream/schedSnode.sim +++ b/tests/script/tsim/stream/schedSnode.sim @@ -79,17 +79,17 @@ endi if $data03 != 4 then print ======$data03 - return -1 + goto loop1 endi if $data04 != 52 then print ======$data04 - return -1 + goto loop1 endi if $data05 != 13 then print ======$data05 - return -1 + goto loop1 endi # row 1 From 814b3caabf3af2ce115e59398529e07025729b35 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Sat, 25 Jun 2022 17:14:50 +0800 Subject: [PATCH 53/72] enh(stream): generate schema only once --- source/dnode/vnode/src/tq/tqSink.c | 133 ++++++++++++++--------------- source/os/src/osFile.c | 2 +- 2 files changed, 63 insertions(+), 72 deletions(-) diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index ef3b205b3e..9abc2f639b 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -18,57 +18,87 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, bool createTb, int64_t suid, const char* stbFullName, int32_t vgId) { SSubmitReq* ret = NULL; + SArray* schemaReqs = NULL; + SArray* schemaReqSz = NULL; SArray* tagArray = taosArrayInit(1, sizeof(STagVal)); if (!tagArray) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - // cal size - int32_t cap = sizeof(SSubmitReq); int32_t sz = taosArrayGetSize(pBlocks); - for (int32_t i = 0; i < sz; i++) { - SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); - int32_t rows = pDataBlock->info.rows; - // TODO min - int32_t rowSize = pDataBlock->info.rowSize; - int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSchema); - int32_t schemaLen = 0; - if (createTb) { - SVCreateTbReq createTbReq = {0}; - char* cname = buildCtbNameByGroupId(stbFullName, pDataBlock->info.groupId); - createTbReq.name = cname; - createTbReq.flags = 0; - createTbReq.type = TSDB_CHILD_TABLE; - createTbReq.ctb.suid = suid; - - STagVal tagVal = { - .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1, - .type = TSDB_DATA_TYPE_UBIGINT, - .i64 = (int64_t)pDataBlock->info.groupId, + if (createTb) { + schemaReqs = taosArrayInit(sz, sizeof(void*)); + schemaReqSz = taosArrayInit(sz, sizeof(int32_t)); + for (int32_t i = 0; i < sz; i++) { + SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); + STagVal tagVal = { + .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1, + .type = TSDB_DATA_TYPE_UBIGINT, + .i64 = (int64_t)pDataBlock->info.groupId, }; STag* pTag = NULL; taosArrayClear(tagArray); taosArrayPush(tagArray, &tagVal); tTagNew(tagArray, 1, false, &pTag); if (pTag == NULL) { - tdDestroySVCreateTbReq(&createTbReq); + taosArrayDestroy(schemaReqs); taosArrayDestroy(tagArray); return NULL; } + + SVCreateTbReq createTbReq = {0}; + createTbReq.name = buildCtbNameByGroupId(stbFullName, pDataBlock->info.groupId); + createTbReq.flags = 0; + createTbReq.type = TSDB_CHILD_TABLE; + createTbReq.ctb.suid = suid; createTbReq.ctb.pTag = (uint8_t*)pTag; int32_t code; + int32_t schemaLen; tEncodeSize(tEncodeSVCreateTbReq, &createTbReq, schemaLen, code); - - tdDestroySVCreateTbReq(&createTbReq); if (code < 0) { + tdDestroySVCreateTbReq(&createTbReq); taosArrayDestroy(tagArray); + taosMemoryFreeClear(ret); return NULL; } - } + void* schemaStr = taosMemoryMalloc(schemaLen); + if (schemaStr == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + taosArrayPush(schemaReqs, &schemaStr); + taosArrayPush(schemaReqSz, &schemaLen); + + SEncoder encoder = {0}; + tEncoderInit(&encoder, schemaStr, schemaLen); + code = tEncodeSVCreateTbReq(&encoder, &createTbReq); + if (code < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + tEncoderClear(&encoder); + tdDestroySVCreateTbReq(&createTbReq); + } + } + taosArrayDestroy(tagArray); + + // cal size + int32_t cap = sizeof(SSubmitReq); + for (int32_t i = 0; i < sz; i++) { + SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); + int32_t rows = pDataBlock->info.rows; + // TODO min + int32_t rowSize = pDataBlock->info.rowSize; + int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSchema); + + int32_t schemaLen = 0; + if (createTb) { + schemaLen = *(int32_t*)taosArrayGet(schemaReqSz, i); + } cap += sizeof(SSubmitBlk) + schemaLen + rows * maxLen; } @@ -99,55 +129,13 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo int32_t schemaLen = 0; if (createTb) { - SVCreateTbReq createTbReq = {0}; - char* cname = buildCtbNameByGroupId(stbFullName, pDataBlock->info.groupId); - createTbReq.name = cname; - createTbReq.flags = 0; - createTbReq.type = TSDB_CHILD_TABLE; - createTbReq.ctb.suid = suid; - - STagVal tagVal = { - .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1, - .type = TSDB_DATA_TYPE_UBIGINT, - .i64 = (int64_t)pDataBlock->info.groupId, - }; - taosArrayClear(tagArray); - taosArrayPush(tagArray, &tagVal); - STag* pTag = NULL; - tTagNew(tagArray, 1, false, &pTag); - if (pTag == NULL) { - tdDestroySVCreateTbReq(&createTbReq); - taosArrayDestroy(tagArray); - taosMemoryFreeClear(ret); - return NULL; - } - createTbReq.ctb.pTag = (uint8_t*)pTag; - - int32_t code; - tEncodeSize(tEncodeSVCreateTbReq, &createTbReq, schemaLen, code); - if (code < 0) { - tdDestroySVCreateTbReq(&createTbReq); - taosArrayDestroy(tagArray); - taosMemoryFreeClear(ret); - return NULL; - } - - SEncoder encoder = {0}; - tEncoderInit(&encoder, blkSchema, schemaLen); - code = tEncodeSVCreateTbReq(&encoder, &createTbReq); - tEncoderClear(&encoder); - tdDestroySVCreateTbReq(&createTbReq); - - if (code < 0) { - taosArrayDestroy(tagArray); - taosMemoryFreeClear(ret); - return NULL; - } + schemaLen = *(int32_t*)taosArrayGet(schemaReqSz, i); + void* schemaStr = taosArrayGetP(schemaReqs, i); + memcpy(blkSchema, schemaStr, schemaLen); } blkHead->schemaLen = htonl(schemaLen); STSRow* rowData = POINTER_SHIFT(blkSchema, schemaLen); - for (int32_t j = 0; j < rows; j++) { SRowBuilder rb = {0}; tdSRowInit(&rb, pTSchema->version); @@ -175,7 +163,10 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo } ret->length = htonl(ret->length); - taosArrayDestroy(tagArray); + + if (schemaReqs) taosArrayDestroyP(schemaReqs, taosMemoryFree); + taosArrayDestroy(schemaReqSz); + return ret; } diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 05b7498cc0..0c6cd80f44 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -671,7 +671,7 @@ void taosFprintfFile(TdFilePtr pFile, const char *format, ...) { fflush(pFile->fp); } -bool taosValidFile(TdFilePtr pFile) { return pFile != NULL; } +bool taosValidFile(TdFilePtr pFile) { return pFile != NULL && pFile->fd > 0; } int32_t taosUmaskFile(int32_t maskVal) { #ifdef WINDOWS From 35336d7964015ef19aa3f50a2e65a38f43aa4863 Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Sat, 25 Jun 2022 17:33:24 +0800 Subject: [PATCH 54/72] ci: keep change branch in log --- Jenkinsfile2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 861478160f..f582461fb2 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -79,6 +79,7 @@ def pre_test(){ git pull >/dev/null git log -5 echo "`date "+%Y%m%d-%H%M%S"` ${JOB_NAME}:${BRANCH_NAME}:${BUILD_ID}:${CHANGE_TARGET}" >>${WKDIR}/jenkins.log + echo "CHANGE_BRANCH:${CHANGE_BRANCH}" >>${WKDIR}/jenkins.log echo "community log: `git log -5`" >>${WKDIR}/jenkins.log git fetch origin +refs/pull/${CHANGE_ID}/merge git checkout -qf FETCH_HEAD @@ -95,6 +96,7 @@ def pre_test(){ git pull >/dev/null git log -5 echo "`date "+%Y%m%d-%H%M%S"` ${JOB_NAME}:${BRANCH_NAME}:${BUILD_ID}:${CHANGE_TARGET}" >>${WKDIR}/jenkins.log + echo "CHANGE_BRANCH:${CHANGE_BRANCH}" >>${WKDIR}/jenkins.log echo "tdinternal log: `git log -5`" >>${WKDIR}/jenkins.log git fetch origin +refs/pull/${CHANGE_ID}/merge git checkout -qf FETCH_HEAD From 8526b2ff89b7e7fee235cbdd7a122bbedab846ea Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Sat, 25 Jun 2022 17:43:26 +0800 Subject: [PATCH 55/72] test: add test case for tmq --- tests/system-test/7-tmq/tmqShow.py | 158 +++++++++++++++++++++++++++++ tests/test/c/tmqSim.c | 3 +- 2 files changed, 160 insertions(+), 1 deletion(-) create mode 100644 tests/system-test/7-tmq/tmqShow.py diff --git a/tests/system-test/7-tmq/tmqShow.py b/tests/system-test/7-tmq/tmqShow.py new file mode 100644 index 0000000000..6b7e7375ff --- /dev/null +++ b/tests/system-test/7-tmq/tmqShow.py @@ -0,0 +1,158 @@ + +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + #tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def tmqCase1(self): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'db1', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':20, 'count':1},{'type': 'TIMESTAMP', 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbNum': 10, + 'rowsPerTbl': 4000, + 'batchNum': 15, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 20, + 'showMsg': 1, + 'showRow': 1} + + topicNameList = ['topic1', 'topic2', 'topic3', 'topic4'] + consumeGroupIdList = ['cgrp1', 'cgrp1', 'cgrp3', 'cgrp4'] + consumerIdList = [0, 1, 2, 3] + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=paraDict['vgroups'],replica=1) + tdLog.info("create stb") + tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) + tdLog.info("create ctb") + tdCom.create_ctable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix']) + # tdLog.info("insert data") + # tmqCom.insert_data(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"]) + + tdLog.info("create 4 topics") + sqlString = "create topic %s as database %s" %(topicNameList[0], paraDict['dbName']) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + sqlString = "create topic %s as stable %s.%s" %(topicNameList[1], paraDict['dbName'], paraDict['stbName']) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + queryString = "select * from %s.%s where c1 %% 7 == 0" %(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[2], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + queryString = "select ts, log(c1), ceil(pow(c1,3)) from %s.%s where c1 %% 7 == 0" %(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as %s " %(topicNameList[3], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + + tdSql.query("show topics") + tdLog.debug(tdSql.queryResult) + rows = tdSql.getRows() + if rows != len(consumerIdList): + tdLog.exit("topic rows error") + + for i in range (rows): + topicName = tdSql.getData(i,0) + matchFlag = 0 + while matchFlag == 0: + for j in range(len(topicNameList)): + if topicName == topicNameList[j]: + matchFlag = 1 + break + if matchFlag == 0: + tdLog.exit("topic name: %s is error", topicName) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] + topicList = topicNameList[0] + ifcheckdata = 0 + ifManualCommit = 0 + keyList = 'group.id:%s, enable.auto.commit:false, auto.commit.interval.ms:6000, auto.offset.reset:earliest'%consumeGroupIdList[0] + tmqCom.insertConsumerInfo(consumerIdList[0], expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + topicList = topicNameList[1] + keyList = 'group.id:%s, enable.auto.commit:false, auto.commit.interval.ms:6000, auto.offset.reset:earliest'%consumeGroupIdList[1] + tmqCom.insertConsumerInfo(consumerIdList[1], expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + topicList = topicNameList[2] + keyList = 'group.id:%s, enable.auto.commit:false, auto.commit.interval.ms:6000, auto.offset.reset:earliest'%consumeGroupIdList[2] + tmqCom.insertConsumerInfo(consumerIdList[2], expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + topicList = topicNameList[3] + keyList = 'group.id:%s, enable.auto.commit:false, auto.commit.interval.ms:6000, auto.offset.reset:earliest'%consumeGroupIdList[3] + tmqCom.insertConsumerInfo(consumerIdList[3], expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow']) + + tdLog.info("async insert data") + pThread = tmqCom.asyncInsertData(paraDict) + + time.sleep(5) + tdLog.info("check show consumers") + tdSql.query("show consumers") + # tdLog.info(tdSql.queryResult) + rows = tdSql.getRows() + tdLog.info("show consumers rows: %d"%rows) + if rows != len(topicNameList): + tdLog.exit("show consumers rows error") + + tdLog.info("check show subscriptions") + tdSql.query("show subscriptions") + # tdLog.debug(tdSql.queryResult) + rows = tdSql.getRows() + tdLog.info("show subscriptions rows: %d"%rows) + if rows != paraDict['vgroups'] * len(topicNameList): + tdLog.exit("show subscriptions rows error") + + pThread.join() + + tdLog.info("insert process end, and start to check consume result") + expectRows = len(consumerIdList) + _ = tmqCom.selectConsumeResult(expectRows) + + time.sleep(10) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def run(self): + tdSql.prepare() + self.tmqCase1() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index c2fba52e6e..81fa72d15a 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -635,8 +635,9 @@ void loop_consume(SThreadInfo* pInfo) { } } + int32_t consumeDelay = g_stConfInfo.consumeDelay == -1 ? -1 : (g_stConfInfo.consumeDelay * 1000); while (running) { - TAOS_RES* tmqMsg = tmq_consumer_poll(pInfo->tmq, g_stConfInfo.consumeDelay * 1000); + TAOS_RES* tmqMsg = tmq_consumer_poll(pInfo->tmq, consumeDelay); if (tmqMsg) { if (0 != g_stConfInfo.showMsgFlag) { totalRows += msg_process(tmqMsg, pInfo, totalMsgs); From 40a21701f9afa399b307d5aa9a5081e0e407fd87 Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Sat, 25 Jun 2022 17:44:43 +0800 Subject: [PATCH 56/72] test: add test case into CI --- tests/system-test/fulltest.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index ef217b828f..81195ca1b1 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -139,3 +139,4 @@ python3 ./test.py -f 7-tmq/tmqCheckData.py python3 ./test.py -f 7-tmq/tmqUdf.py #python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 5 python3 ./test.py -f 7-tmq/tmqConsumerGroup.py +python3 ./test.py -f 7-tmq/tmqShow.py From 47ae534c0f23e8a5223f0b904e0d7e8b003bce9c Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Sat, 25 Jun 2022 17:44:54 +0800 Subject: [PATCH 57/72] fix(sma): drop stream when drop sma --- source/dnode/mnode/impl/inc/mndStream.h | 5 +++ source/dnode/mnode/impl/src/mndSma.c | 18 +++++++++ source/dnode/mnode/impl/src/mndStream.c | 2 +- source/dnode/vnode/src/tq/tqSink.c | 2 +- source/libs/executor/inc/executorimpl.h | 1 - source/libs/executor/src/executor.c | 11 ++--- source/libs/executor/src/scanoperator.c | 53 +++++++++++-------------- source/libs/wal/src/walRead.c | 2 +- 8 files changed, 53 insertions(+), 41 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndStream.h b/source/dnode/mnode/impl/inc/mndStream.h index 69385c3a46..5e9089cec9 100644 --- a/source/dnode/mnode/impl/inc/mndStream.h +++ b/source/dnode/mnode/impl/inc/mndStream.h @@ -34,6 +34,11 @@ SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw); int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb); int32_t mndPersistStream(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); +// for sma +// TODO refactor +int32_t mndDropStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); +int32_t mndPersistDropStreamLog(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 05603f8554..3dadcf77f0 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -857,6 +857,24 @@ static int32_t mndDropSma(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SSmaObj *p mDebug("trans:%d, used to drop sma:%s", pTrans->id, pSma->name); mndTransSetDbName(pTrans, pDb->name, NULL); + SStreamObj *pStream = mndAcquireStream(pMnode, pSma->name); + if (pStream == NULL || pStream->smaId != pSma->uid) { + sdbRelease(pMnode->pSdb, pStream); + goto _OVER; + } else { + if (mndDropStreamTasks(pMnode, pTrans, pStream) < 0) { + mError("stream:%s, failed to drop task since %s", pStream->name, terrstr()); + sdbRelease(pMnode->pSdb, pStream); + goto _OVER; + } + + // drop stream + if (mndPersistDropStreamLog(pMnode, pTrans, pStream) < 0) { + sdbRelease(pMnode->pSdb, pStream); + goto _OVER; + } + } + if (mndSetDropSmaRedoLogs(pMnode, pTrans, pSma) != 0) goto _OVER; if (mndSetDropSmaVgroupRedoLogs(pMnode, pTrans, pVgroup) != 0) goto _OVER; if (mndSetDropSmaCommitLogs(pMnode, pTrans, pSma) != 0) goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 21158bb0a2..29c6819163 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -494,7 +494,7 @@ static int32_t mndPersistTaskDropReq(STrans *pTrans, SStreamTask *pTask) { return 0; } -static int32_t mndDropStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) { +int32_t mndDropStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) { int32_t lv = taosArrayGetSize(pStream->tasks); for (int32_t i = 0; i < lv; i++) { SArray *pTasks = taosArrayGetP(pStream->tasks, i); diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 9abc2f639b..0bb9918488 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -43,7 +43,7 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo taosArrayPush(tagArray, &tagVal); tTagNew(tagArray, 1, false, &pTag); if (pTag == NULL) { - taosArrayDestroy(schemaReqs); + terrno = TSDB_CODE_OUT_OF_MEMORY; taosArrayDestroy(tagArray); return NULL; } diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index eb7beb19db..0f3251515e 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -347,7 +347,6 @@ typedef struct SStreamBlockScanInfo { SInterval interval; // if the upstream is an interval operator, the interval info is also kept here. SArray* childIds; SessionWindowSupporter sessionSup; - bool assignBlockUid; // assign block uid to groupId, temporarily used for generating rollup SMA. int32_t scanWinIndex; } SStreamBlockScanInfo; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 6de364e63a..3fd491885f 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -19,8 +19,7 @@ #include "tdatablock.h" #include "vnode.h" -static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t numOfBlocks, int32_t type, bool assignUid, - char* id) { +static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t numOfBlocks, int32_t type, char* id) { ASSERT(pOperator != NULL); if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { if (pOperator->numOfDownstream == 0) { @@ -33,12 +32,11 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu return TSDB_CODE_QRY_APP_ERROR; } pOperator->status = OP_NOT_OPENED; - return doSetStreamBlock(pOperator->pDownstream[0], input, numOfBlocks, type, assignUid, id); + return doSetStreamBlock(pOperator->pDownstream[0], input, numOfBlocks, type, id); } else { pOperator->status = OP_NOT_OPENED; SStreamBlockScanInfo* pInfo = pOperator->info; - pInfo->assignBlockUid = assignUid; // TODO: if a block was set but not consumed, // prevent setting a different type of block @@ -76,7 +74,7 @@ int32_t qStreamScanSnapshot(qTaskInfo_t tinfo) { return TSDB_CODE_QRY_APP_ERROR; } SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - return doSetStreamBlock(pTaskInfo->pRoot, NULL, 0, STREAM_DATA_TYPE_FROM_SNAPSHOT, 0, NULL); + return doSetStreamBlock(pTaskInfo->pRoot, NULL, 0, STREAM_DATA_TYPE_FROM_SNAPSHOT, NULL); } int32_t qSetStreamInput(qTaskInfo_t tinfo, const void* input, int32_t type, bool assignUid) { @@ -94,8 +92,7 @@ int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numO SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - int32_t code = - doSetStreamBlock(pTaskInfo->pRoot, (void**)pBlocks, numOfBlocks, type, assignUid, GET_TASKID(pTaskInfo)); + int32_t code = doSetStreamBlock(pTaskInfo->pRoot, (void**)pBlocks, numOfBlocks, type, GET_TASKID(pTaskInfo)); if (code != TSDB_CODE_SUCCESS) { qError("%s failed to set the stream block data", GET_TASKID(pTaskInfo)); } else { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 9c0ed40c30..0c73ec72a5 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -507,20 +507,21 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { STableScanInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - if(pInfo->currentGroupId == -1){ + if (pInfo->currentGroupId == -1) { pInfo->currentGroupId++; if (pInfo->currentGroupId >= taosArrayGetSize(pTaskInfo->tableqinfoList.pGroupList)) { setTaskStatus(pTaskInfo, TASK_COMPLETED); return NULL; } - SArray *tableList = taosArrayGetP(pTaskInfo->tableqinfoList.pGroupList, pInfo->currentGroupId); + SArray* tableList = taosArrayGetP(pTaskInfo->tableqinfoList.pGroupList, pInfo->currentGroupId); tsdbCleanupReadHandle(pInfo->dataReader); - tsdbReaderT* pReader = tsdbReaderOpen(pInfo->readHandle.vnode, &pInfo->cond, tableList, pInfo->queryId, pInfo->taskId); + tsdbReaderT* pReader = + tsdbReaderOpen(pInfo->readHandle.vnode, &pInfo->cond, tableList, pInfo->queryId, pInfo->taskId); pInfo->dataReader = pReader; } SSDataBlock* result = doTableScanGroup(pOperator); - if(result){ + if (result) { return result; } @@ -530,7 +531,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { return NULL; } - SArray *tableList = taosArrayGetP(pTaskInfo->tableqinfoList.pGroupList, pInfo->currentGroupId); + SArray* tableList = taosArrayGetP(pTaskInfo->tableqinfoList.pGroupList, pInfo->currentGroupId); tsdbSetTableList(pInfo->dataReader, tableList); tsdbResetReadHandle(pInfo->dataReader, &pInfo->cond, 0); @@ -538,7 +539,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { pInfo->scanTimes = 0; result = doTableScanGroup(pOperator); - if(result){ + if (result) { return result; } @@ -822,7 +823,7 @@ static bool prepareDataScan(SStreamBlockScanInfo* pInfo) { STableScanInfo* pTableScanInfo = pInfo->pSnapshotReadOp->info; pTableScanInfo->cond.twindows[0] = win; pTableScanInfo->curTWinIdx = 0; -// tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0); + // tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0); pTableScanInfo->scanTimes = 0; pTableScanInfo->currentGroupId = -1; return true; @@ -1030,14 +1031,6 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { pInfo->pRes->info.type = STREAM_NORMAL; pInfo->pRes->info.capacity = numOfRows; - // for generating rollup SMA result, each time is an independent time serie. - // TODO temporarily used, when the statement of "partition by tbname" is ready, remove this - if (pInfo->assignBlockUid) { - pInfo->pRes->info.groupId = uid; - } else { - pInfo->pRes->info.groupId = groupId; - } - uint64_t* groupIdPre = taosHashGet(pOperator->pTaskInfo->tableqinfoList.map, &uid, sizeof(int64_t)); if (groupIdPre) { pInfo->pRes->info.groupId = *groupIdPre; @@ -1132,9 +1125,9 @@ static SArray* extractTableIdList(const STableListInfo* pTableGroupInfo) { return tableIdList; } -SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, - STableScanPhysiNode* pTableScanNode, SExecTaskInfo* pTaskInfo, - STimeWindowAggSupp* pTwSup, uint64_t queryId, uint64_t taskId) { +SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, + SExecTaskInfo* pTaskInfo, STimeWindowAggSupp* pTwSup, uint64_t queryId, + uint64_t taskId) { SStreamBlockScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamBlockScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); @@ -1934,11 +1927,11 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi goto _error; } - pInfo->pTableList = pTableListInfo; - pInfo->pColMatchInfo = colList; - pInfo->pRes = createResDataBlock(pDescNode); - pInfo->readHandle = *pReadHandle; - pInfo->curPos = 0; + pInfo->pTableList = pTableListInfo; + pInfo->pColMatchInfo = colList; + pInfo->pRes = createResDataBlock(pDescNode); + pInfo->readHandle = *pReadHandle; + pInfo->curPos = 0; pOperator->name = "TagScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN; @@ -2216,8 +2209,8 @@ SArray* generateSortByTsInfo(int32_t order) { return pList; } -static int32_t createMultipleDataReaders(SQueryTableDataCond* pQueryCond, SReadHandle* pHandle, SArray* tableList, SArray* arrayReader, uint64_t queryId, - uint64_t taskId) { +static int32_t createMultipleDataReaders(SQueryTableDataCond* pQueryCond, SReadHandle* pHandle, SArray* tableList, + SArray* arrayReader, uint64_t queryId, uint64_t taskId) { for (int32_t i = 0; i < taosArrayGetSize(tableList); ++i) { SArray* tmp = taosArrayInit(1, sizeof(STableKeyInfo)); taosArrayPush(tmp, taosArrayGet(tableList, i)); @@ -2237,13 +2230,13 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) { SArray* tableList = taosArrayGetP(pInfo->tableListInfo->pGroupList, pInfo->currentGroupId); - createMultipleDataReaders(&pInfo->cond, &pInfo->readHandle, tableList, - pInfo->dataReaders, pInfo->queryId, pInfo->taskId); + createMultipleDataReaders(&pInfo->cond, &pInfo->readHandle, tableList, pInfo->dataReaders, pInfo->queryId, + pInfo->taskId); // todo the total available buffer should be determined by total capacity of buffer of this task. // the additional one is reserved for merge result int32_t tableLen = taosArrayGetSize(tableList); - pInfo->sortBufSize = pInfo->bufPageSize * ((tableLen==0?1:tableLen) + 1); + pInfo->sortBufSize = pInfo->bufPageSize * ((tableLen == 0 ? 1 : tableLen) + 1); int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_MULTISOURCE_MERGE, pInfo->bufPageSize, numOfBufPage, pInfo->pSortInputBlock, pTaskInfo->id.str); @@ -2342,7 +2335,7 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) { SSDataBlock* pBlock = getSortedTableMergeScanBlockData(pInfo->pSortHandle, pOperator->resultInfo.capacity, pOperator); if (pBlock != NULL) { uint64_t* groupId = taosHashGet(pInfo->tableListInfo->map, &(pBlock->info.uid), sizeof(uint64_t)); - if(groupId) pBlock->info.groupId = *groupId; + if (groupId) pBlock->info.groupId = *groupId; pOperator->resultInfo.totalRows += pBlock->info.rows; return pBlock; @@ -2359,7 +2352,7 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) { pBlock = getSortedTableMergeScanBlockData(pInfo->pSortHandle, pOperator->resultInfo.capacity, pOperator); if (pBlock != NULL) { uint64_t* groupId = taosHashGet(pInfo->tableListInfo->map, &(pBlock->info.uid), sizeof(uint64_t)); - if(groupId) pBlock->info.groupId = *groupId; + if (groupId) pBlock->info.groupId = *groupId; pOperator->resultInfo.totalRows += pBlock->info.rows; return pBlock; diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 682afbb785..20fa5f1f2b 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -103,6 +103,7 @@ static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) { wError("cannot open file %s, since %s", fnameStr, terrstr()); return -1; } + pRead->pReadLogTFile = pLogTFile; walBuildIdxName(pRead->pWal, fileFirstVer, fnameStr); TdFilePtr pIdxTFile = taosOpenFile(fnameStr, TD_FILE_READ); @@ -112,7 +113,6 @@ static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) { return -1; } - pRead->pReadLogTFile = pLogTFile; pRead->pReadIdxTFile = pIdxTFile; return 0; } From 297c848e61e471159b9cfe9cae1ac1de18d57f66 Mon Sep 17 00:00:00 2001 From: slzhou Date: Sat, 25 Jun 2022 17:50:42 +0800 Subject: [PATCH 58/72] feat: add set operator optimization --- source/libs/planner/src/planOptimizer.c | 92 +++++++++++++++++++------ 1 file changed, 72 insertions(+), 20 deletions(-) diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index c1b12d9d20..34ab7b72b7 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1107,6 +1107,26 @@ static int32_t partTagsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSub return code; } +//===================================================================================================================== +// eliminate project optimization +static bool eliminateProjOptCheckProjColumnNames(SProjectLogicNode* pProjectNode) { + SHashObj* pProjColNameHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + SNode* pProjection; + FOREACH(pProjection, pProjectNode->pProjections) { + char* projColumnName = ((SColumnNode*)pProjection)->colName; + int32_t* pExist = taosHashGet(pProjColNameHash, projColumnName, strlen(projColumnName)); + if (NULL != pExist) { + taosHashCleanup(pProjColNameHash); + return false; + } else { + int32_t exist = 1; + taosHashPut(pProjColNameHash, projColumnName, strlen(projColumnName), &exist, sizeof(exist)); + } + } + taosHashCleanup(pProjColNameHash); + return true; +} + static bool eliminateProjOptMayBeOptimized(SLogicNode* pNode) { // TODO: enable this optimization after new mechanising that map projection and targets of project node if (NULL != pNode->pParent) { @@ -1122,28 +1142,15 @@ static bool eliminateProjOptMayBeOptimized(SLogicNode* pNode) { return false; } - SHashObj* pProjColNameHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); SNode* pProjection; FOREACH(pProjection, pProjectNode->pProjections) { SExprNode* pExprNode = (SExprNode*)pProjection; if (QUERY_NODE_COLUMN != nodeType(pExprNode)) { - taosHashCleanup(pProjColNameHash); return false; } - - char* projColumnName = ((SColumnNode*)pProjection)->colName; - int32_t* pExist = taosHashGet(pProjColNameHash, projColumnName, strlen(projColumnName)); - if (NULL != pExist) { - taosHashCleanup(pProjColNameHash); - return false; - } else { - int32_t exist = 1; - taosHashPut(pProjColNameHash, projColumnName, strlen(projColumnName), &exist, sizeof(exist)); - } } - taosHashCleanup(pProjColNameHash); - return true; + return eliminateProjOptCheckProjColumnNames(pProjectNode); } static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, @@ -1183,14 +1190,59 @@ static int32_t eliminateProjOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLog return eliminateProjOptimizeImpl(pCxt, pLogicSubplan, pProjectNode); } +//===================================================================================================================== +// eliminate Set Operator optimization + +static bool eliminateSetOpMayBeOptimized(SLogicNode* pNode) { + SLogicNode* pParent = pNode->pParent; + if (NULL == pParent || + QUERY_NODE_LOGIC_PLAN_AGG != nodeType(pParent) && QUERY_NODE_LOGIC_PLAN_PROJECT != nodeType(pParent) || + LIST_LENGTH(pParent->pChildren) < 2) { + return false; + } + if (nodeType(pNode) != nodeType(pNode->pParent) || + LIST_LENGTH(pNode->pChildren) < 2) { + return false; + } + return true; +} + +static int32_t eliminateSetOpOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, SLogicNode* pSetOpNode) { + SNode* pSibling; + FOREACH(pSibling, pSetOpNode->pParent->pChildren) { + if (nodesEqualNode(pSibling, (SNode*)pSetOpNode)) { + SNode* pChild; + FOREACH(pChild, pSetOpNode->pChildren) { + ((SLogicNode*)pChild)->pParent = pSetOpNode->pParent; + } + nodesListInsertList(pSetOpNode->pParent->pChildren, cell, pSetOpNode->pChildren); + pSetOpNode->pChildren = NULL; + ERASE_NODE(pSetOpNode->pParent->pChildren); + return TSDB_CODE_SUCCESS; + } + } + + return TSDB_CODE_PLAN_INTERNAL_ERROR; +} + +static int32_t eliminateSetOpOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { + SLogicNode* pSetOpNode = optFindPossibleNode(pLogicSubplan->pNode, eliminateSetOpMayBeOptimized); + if (NULL == pSetOpNode) { + return TSDB_CODE_SUCCESS; + } + + return eliminateSetOpOptimizeImpl(pCxt, pLogicSubplan, pSetOpNode); +} + // clang-format off static const SOptimizeRule optimizeRuleSet[] = { - {.pName = "OptimizeScanData", .optimizeFunc = osdOptimize}, - {.pName = "ConditionPushDown", .optimizeFunc = cpdOptimize}, - {.pName = "OrderByPrimaryKey", .optimizeFunc = opkOptimize}, - {.pName = "SmaIndex", .optimizeFunc = smaOptimize}, - {.pName = "PartitionTags", .optimizeFunc = partTagsOptimize}, - {.pName = "EliminateProject", .optimizeFunc = eliminateProjOptimize} + {.pName = "OptimizeScanData", .optimizeFunc = osdOptimize}, + {.pName = "ConditionPushDown", .optimizeFunc = cpdOptimize}, + {.pName = "OrderByPrimaryKey", .optimizeFunc = opkOptimize}, + {.pName = "SmaIndex", .optimizeFunc = smaOptimize}, + {.pName = "PartitionTags", .optimizeFunc = partTagsOptimize}, + {.pName = "EliminateProject", .optimizeFunc = eliminateProjOptimize}, + {.pName = "EliminateSetOperator", .optimizeFunc = eliminateSetOpOptimize} }; // clang-format on From 89d6ca78233ff3a7c48968645612b3f022b4c99b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 25 Jun 2022 18:03:12 +0800 Subject: [PATCH 59/72] feat: privilege for sysinfo user --- include/libs/transport/trpc.h | 6 +-- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 5 ++ source/dnode/mnode/impl/inc/mndPrivilege.h | 6 ++- source/dnode/mnode/impl/src/mndDb.c | 27 ++++++---- source/dnode/mnode/impl/src/mndDnode.c | 6 ++- source/dnode/mnode/impl/src/mndPrivilege.c | 41 ++++++++++---- source/dnode/mnode/impl/src/mndProfile.c | 4 ++ source/dnode/mnode/impl/src/mndShow.c | 6 ++- source/dnode/mnode/impl/src/mndStb.c | 56 +++++++++----------- tests/script/tsim/user/privilege_sysinfo.sim | 21 +++++++- tools/shell/src/shellEngine.c | 4 +- 11 files changed, 119 insertions(+), 63 deletions(-) diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index 77de5f4756..c2c1a3534d 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -34,10 +34,8 @@ extern int32_t tsRpcHeadSize; typedef struct { uint32_t clientIp; uint16_t clientPort; - union { - char user[TSDB_USER_LEN]; - int64_t applyIndex; - }; + int64_t applyIndex; + char user[TSDB_USER_LEN]; } SRpcConnInfo; typedef struct SRpcHandleInfo { diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 3d1ece0b64..363f438b80 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -253,6 +253,11 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t size = 0; int32_t rowsRead = 0; + if (strcmp(pMsg->info.conn.user, TSDB_DEFAULT_USER) != 0) { + terrno = TSDB_CODE_MND_NO_RIGHTS; + return -1; + } + SRetrieveTableReq retrieveReq = {0}; if (tDeserializeSRetrieveTableReq(pMsg->pCont, pMsg->contLen, &retrieveReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; diff --git a/source/dnode/mnode/impl/inc/mndPrivilege.h b/source/dnode/mnode/impl/inc/mndPrivilege.h index a1bec69790..0c56f6f155 100644 --- a/source/dnode/mnode/impl/inc/mndPrivilege.h +++ b/source/dnode/mnode/impl/inc/mndPrivilege.h @@ -57,6 +57,8 @@ typedef enum { MND_OPER_USE_DB, MND_OPER_WRITE_DB, MND_OPER_READ_DB, + MND_OPER_READ_OR_WRITE_DB, + MND_OPER_SHOW_VARIBALES, } EOperType; int32_t mndInitPrivilege(SMnode *pMnode); @@ -64,8 +66,8 @@ void mndCleanupPrivilege(SMnode *pMnode); int32_t mndCheckOperPrivilege(SMnode *pMnode, const char *user, EOperType operType); int32_t mndCheckDbPrivilege(SMnode *pMnode, const char *user, EOperType operType, SDbObj *pDb); -int32_t mndCheckDbPrivilegeByName(SMnode *pMnode, const char *user, EOperType operType, const char *name); -int32_t mndCheckShowPrivilege(SMnode *pMnode, const char *user, int32_t showType); +int32_t mndCheckDbPrivilegeByName(SMnode *pMnode, const char *user, EOperType operType, const char *dbname); +int32_t mndCheckShowPrivilege(SMnode *pMnode, const char *user, EShowType showType, const char *dbname); int32_t mndCheckAlterUserPrivilege(SUserObj *pOperUser, SUserObj *pUser, SAlterUserReq *pAlter); #ifdef __cplusplus diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 0345f1b345..abd1492d5e 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -15,9 +15,9 @@ #define _DEFAULT_SOURCE #include "mndDb.h" -#include "mndPrivilege.h" #include "mndDnode.h" #include "mndOffset.h" +#include "mndPrivilege.h" #include "mndShow.h" #include "mndSma.h" #include "mndStb.h" @@ -1336,7 +1336,7 @@ char *buildRetension(SArray *pRetension) { } static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, int32_t rows, int64_t numOfTables, - bool sysDb, ESdbStatus objStatus) { + bool sysDb, ESdbStatus objStatus, bool sysinfo) { int32_t cols = 0; int32_t bytes = pShow->pMeta->pSchemas[cols].bytes; @@ -1354,7 +1354,7 @@ static void dumpDbInfoData(SSDataBlock *pBlock, SDbObj *pDb, SShowObj *pShow, in char statusB[24] = {0}; STR_WITH_SIZE_TO_VARSTR(statusB, status, strlen(status)); - if (sysDb) { + if (sysDb || !sysinfo) { for (int32_t i = 0; i < pShow->numOfColumns; ++i) { SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, i); if (i == 0) { @@ -1528,17 +1528,21 @@ static int32_t mndRetrieveDbs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc SDbObj *pDb = NULL; ESdbStatus objStatus = 0; + SUserObj *pUser = mndAcquireUser(pMnode, pReq->info.conn.user); + if (pUser == NULL) return 0; + bool sysinfo = pUser->sysInfo; + // Append the information_schema database into the result. if (!pShow->sysDbRsp) { SDbObj infoschemaDb = {0}; setInformationSchemaDbCfg(&infoschemaDb); - dumpDbInfoData(pBlock, &infoschemaDb, pShow, numOfRows, 14, true, 0); + dumpDbInfoData(pBlock, &infoschemaDb, pShow, numOfRows, 14, true, 0, 1); numOfRows += 1; SDbObj perfschemaDb = {0}; setPerfSchemaDbCfg(&perfschemaDb); - dumpDbInfoData(pBlock, &perfschemaDb, pShow, numOfRows, 3, true, 0); + dumpDbInfoData(pBlock, &perfschemaDb, pShow, numOfRows, 3, true, 0, 1); numOfRows += 1; pShow->sysDbRsp = true; @@ -1550,16 +1554,19 @@ static int32_t mndRetrieveDbs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc break; } - int32_t numOfTables = 0; - sdbTraverse(pSdb, SDB_VGROUP, mndGetTablesOfDbFp, &numOfTables, NULL, NULL); + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_READ_OR_WRITE_DB, pDb) == 0) { + int32_t numOfTables = 0; + sdbTraverse(pSdb, SDB_VGROUP, mndGetTablesOfDbFp, &numOfTables, NULL, NULL); + + dumpDbInfoData(pBlock, pDb, pShow, numOfRows, numOfTables, false, objStatus, sysinfo); + numOfRows++; + } - dumpDbInfoData(pBlock, pDb, pShow, numOfRows, numOfTables, false, objStatus); - numOfRows++; sdbRelease(pSdb, pDb); } pShow->numOfRows += numOfRows; - + mndReleaseUser(pMnode, pUser); return numOfRows; } diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index af1d641ebf..5e78d0f434 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -558,7 +558,11 @@ _OVER: static int32_t mndProcessShowVariablesReq(SRpcMsg *pReq) { SShowVariablesRsp rsp = {0}; - int32_t code = -1; + int32_t code = -1; + + if (mndCheckOperPrivilege(pReq->info.node, pReq->info.conn.user, MND_OPER_SHOW_VARIBALES) != 0) { + goto _OVER; + } rsp.variables = taosArrayInit(4, sizeof(SVariablesInfo)); if (NULL == rsp.variables) { diff --git a/source/dnode/mnode/impl/src/mndPrivilege.c b/source/dnode/mnode/impl/src/mndPrivilege.c index 752b11540d..dc321e38d1 100644 --- a/source/dnode/mnode/impl/src/mndPrivilege.c +++ b/source/dnode/mnode/impl/src/mndPrivilege.c @@ -15,8 +15,8 @@ #define _DEFAULT_SOURCE #include "mndPrivilege.h" -#include "mndUser.h" #include "mndDb.h" +#include "mndUser.h" int32_t mndInitPrivilege(SMnode *pMnode) { return 0; } @@ -46,6 +46,7 @@ int32_t mndCheckOperPrivilege(SMnode *pMnode, const char *user, EOperType operTy case MND_OPER_CONNECT: case MND_OPER_CREATE_FUNC: case MND_OPER_DROP_FUNC: + case MND_OPER_SHOW_VARIBALES: break; default: terrno = TSDB_CODE_MND_NO_RIGHTS; @@ -80,7 +81,7 @@ int32_t mndCheckAlterUserPrivilege(SUserObj *pOperUser, SUserObj *pUser, SAlterU return -1; } -int32_t mndCheckShowPrivilege(SMnode *pMnode, const char *user, int32_t showType) { +int32_t mndCheckShowPrivilege(SMnode *pMnode, const char *user, EShowType showType, const char *dbname) { int32_t code = 0; SUserObj *pUser = mndAcquireUser(pMnode, user); @@ -99,14 +100,34 @@ int32_t mndCheckShowPrivilege(SMnode *pMnode, const char *user, int32_t showType goto _OVER; } - if (!pUser->sysInfo) { - terrno = TSDB_CODE_MND_NO_RIGHTS; - code = -1; + if (pUser->sysInfo) { goto _OVER; } - terrno = TSDB_CODE_MND_NO_RIGHTS; - code = -1; + switch (showType) { + case TSDB_MGMT_TABLE_DB: + case TSDB_MGMT_TABLE_STB: + case TSDB_MGMT_TABLE_INDEX: + case TSDB_MGMT_TABLE_STREAMS: + case TSDB_MGMT_TABLE_CONSUMERS: + case TSDB_MGMT_TABLE_TOPICS: + case TSDB_MGMT_TABLE_SUBSCRIPTIONS: + case TSDB_MGMT_TABLE_FUNC: + case TSDB_MGMT_TABLE_QUERIES: + case TSDB_MGMT_TABLE_CONNS: + case TSDB_MGMT_TABLE_APPS: + case TSDB_MGMT_TABLE_TRANS: + code = 0; + break; + default: + terrno = TSDB_CODE_MND_NO_RIGHTS; + code = -1; + goto _OVER; + } + + if (showType == TSDB_MGMT_TABLE_STB || showType == TSDB_MGMT_TABLE_VGROUP || showType == TSDB_MGMT_TABLE_INDEX) { + code = mndCheckDbPrivilegeByName(pMnode, user, MND_OPER_READ_OR_WRITE_DB, dbname); + } _OVER: mndReleaseUser(pMnode, pUser); @@ -138,7 +159,7 @@ int32_t mndCheckDbPrivilege(SMnode *pMnode, const char *user, EOperType operType if (strcmp(pUser->user, pDb->createUser) == 0 && pUser->sysInfo) goto _OVER; } - if (operType == MND_OPER_USE_DB) { + if (operType == MND_OPER_USE_DB || operType == MND_OPER_READ_OR_WRITE_DB) { if (strcmp(pUser->user, pDb->createUser) == 0) goto _OVER; if (taosHashGet(pUser->readDbs, pDb->name, strlen(pDb->name) + 1) != NULL) goto _OVER; if (taosHashGet(pUser->writeDbs, pDb->name, strlen(pDb->name) + 1) != NULL) goto _OVER; @@ -162,8 +183,8 @@ _OVER: return code; } -int32_t mndCheckDbPrivilegeByName(SMnode *pMnode, const char *user, EOperType operType, const char *name) { - SDbObj *pDb = mndAcquireDb(pMnode, name); +int32_t mndCheckDbPrivilegeByName(SMnode *pMnode, const char *user, EOperType operType, const char *dbname) { + SDbObj *pDb = mndAcquireDb(pMnode, dbname); if (pDb == NULL) return -1; int32_t code = mndCheckDbPrivilege(pMnode, user, operType, pDb); diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index f2e599b073..38368b4ece 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -254,6 +254,10 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { terrstr()); goto _OVER; } + + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_READ_OR_WRITE_DB, pDb) != 0) { + goto _OVER; + } } pConn = mndCreateConn(pMnode, pReq->info.conn.user, connReq.connType, pReq->info.conn.clientIp, diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 3351cff3a3..671adba046 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -231,8 +231,10 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { } mDebug("show:0x%" PRIx64 ", start retrieve data, type:%d", pShow->id, pShow->type); - - // if (mndCheckShowPrivilege(pMnode, pReq->info.conn.user, pShow->type) != 0) return -1; + if (mndCheckShowPrivilege(pMnode, retrieveReq.user, pShow->type, retrieveReq.db) != 0) { + return -1; + } + memcpy(pReq->info.conn.user, retrieveReq.user, TSDB_USER_LEN); int32_t numOfCols = pShow->pMeta->numOfColumns; SSDataBlock *pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 77b13cd82d..92f85ecd04 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -15,12 +15,12 @@ #define _DEFAULT_SOURCE #include "mndStb.h" -#include "mndPrivilege.h" #include "mndDb.h" #include "mndDnode.h" #include "mndInfoSchema.h" #include "mndMnode.h" #include "mndPerfSchema.h" +#include "mndPrivilege.h" #include "mndScheduler.h" #include "mndShow.h" #include "mndSma.h" @@ -107,7 +107,7 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) { int32_t funcNum = taosArrayGetSize(pStb->pFuncs); SDB_SET_INT32(pRaw, dataPos, funcNum, _OVER) for (int32_t i = 0; i < funcNum; ++i) { - char* func = taosArrayGet(pStb->pFuncs, i); + char *func = taosArrayGet(pStb->pFuncs, i); SDB_SET_BINARY(pRaw, dataPos, func, TSDB_FUNC_NAME_LEN, _OVER) } @@ -708,7 +708,7 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat pDst->commentLen = pCreate->commentLen; pDst->pFuncs = pCreate->pFuncs; pCreate->pFuncs = NULL; - + if (pDst->commentLen > 0) { pDst->comment = taosMemoryCalloc(pDst->commentLen + 1, 1); if (pDst->comment == NULL) { @@ -1391,12 +1391,11 @@ static int32_t mndBuildStbCfgImp(SDbObj *pDb, SStbObj *pStb, const char *tbName, if (pStb->pFuncs) { pRsp->pFuncs = taosArrayDup(pStb->pFuncs); } - + taosRUnLockLatch(&pStb->lock); return 0; } - static int32_t mndBuildStbSchema(SMnode *pMnode, const char *dbFName, const char *tbName, STableMetaRsp *pRsp, int32_t *smaVer) { char tbFName[TSDB_TABLE_FNAME_LEN] = {0}; @@ -1426,31 +1425,29 @@ static int32_t mndBuildStbSchema(SMnode *pMnode, const char *dbFName, const char } static int32_t mndBuildStbCfg(SMnode *pMnode, const char *dbFName, const char *tbName, STableCfgRsp *pRsp) { - char tbFName[TSDB_TABLE_FNAME_LEN] = {0}; - snprintf(tbFName, sizeof(tbFName), "%s.%s", dbFName, tbName); + char tbFName[TSDB_TABLE_FNAME_LEN] = {0}; + snprintf(tbFName, sizeof(tbFName), "%s.%s", dbFName, tbName); - SDbObj *pDb = mndAcquireDb(pMnode, dbFName); - if (pDb == NULL) { - terrno = TSDB_CODE_MND_DB_NOT_SELECTED; - return -1; - } + SDbObj *pDb = mndAcquireDb(pMnode, dbFName); + if (pDb == NULL) { + terrno = TSDB_CODE_MND_DB_NOT_SELECTED; + return -1; + } - SStbObj *pStb = mndAcquireStb(pMnode, tbFName); - if (pStb == NULL) { - mndReleaseDb(pMnode, pDb); - terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; - return -1; - } + SStbObj *pStb = mndAcquireStb(pMnode, tbFName); + if (pStb == NULL) { + mndReleaseDb(pMnode, pDb); + terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; + return -1; + } - int32_t code = mndBuildStbCfgImp(pDb, pStb, tbName, pRsp); - - mndReleaseDb(pMnode, pDb); - mndReleaseStb(pMnode, pStb); - return code; + int32_t code = mndBuildStbCfgImp(pDb, pStb, tbName, pRsp); + + mndReleaseDb(pMnode, pDb); + mndReleaseStb(pMnode, pStb); + return code; } - - static int32_t mndBuildSMAlterStbRsp(SDbObj *pDb, const SMAlterStbReq *pAlter, SStbObj *pObj, void **pCont, int32_t *pLen) { int32_t ret; @@ -1811,10 +1808,10 @@ _OVER: } static int32_t mndProcessTableCfgReq(SRpcMsg *pReq) { - SMnode *pMnode = pReq->info.node; - int32_t code = -1; - STableCfgReq cfgReq = {0}; - STableCfgRsp cfgRsp = {0}; + SMnode *pMnode = pReq->info.node; + int32_t code = -1; + STableCfgReq cfgReq = {0}; + STableCfgRsp cfgRsp = {0}; if (tDeserializeSTableCfgReq(pReq->pCont, pReq->contLen, &cfgReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; @@ -1866,7 +1863,6 @@ _OVER: return code; } - int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbVersions, int32_t numOfStbs, void **ppRsp, int32_t *pRspLen) { SSTbHbRsp hbRsp = {0}; diff --git a/tests/script/tsim/user/privilege_sysinfo.sim b/tests/script/tsim/user/privilege_sysinfo.sim index 35760d45fd..10c31dc288 100644 --- a/tests/script/tsim/user/privilege_sysinfo.sim +++ b/tests/script/tsim/user/privilege_sysinfo.sim @@ -8,6 +8,7 @@ sql create user sysinfo0 pass 'taosdata' sql create user sysinfo1 pass 'taosdata' sql alter user sysinfo0 sysinfo 0 sql alter user sysinfo1 sysinfo 1 +sql create database db print user sysinfo0 login sql close @@ -39,6 +40,24 @@ sql_error kill connection 1 sql_error kill query 1 print =============== check db -sql_error create database db +sql_error create database d1 +sql_error drop database db +sql_error use db +sql_error alter database db replica 1; +sql_error show db.vgroups +sql_error show db.stables +sql_error show db.tables + +print =============== check show +sql_error show users +sql_error show cluster +sql_error show dnodes +sql_error show mnodes +sql_error show snodes +sql_error show qnodes +sql_error show bnodes +sql_error show grants +sql_error show dnode 1 variables; +sql show variables; system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 8a017d378d..2296d573cb 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -858,9 +858,7 @@ void shellGetGrantInfo() { int32_t code = taos_errno(tres); if (code != TSDB_CODE_SUCCESS) { - if (code == TSDB_CODE_OPS_NOT_SUPPORT) { - fprintf(stdout, "Server is Community Edition, %s\n\n", sinfo); - } else { + if (code != TSDB_CODE_OPS_NOT_SUPPORT && code != TSDB_CODE_MND_NO_RIGHTS) { fprintf(stderr, "Failed to check Server Edition, Reason:0x%04x:%s\n\n", code, taos_errstr(tres)); } return; From beae5e3a82d8a2f45a47f67e2e209d493d95b985 Mon Sep 17 00:00:00 2001 From: tomchon Date: Sat, 25 Jun 2022 18:07:05 +0800 Subject: [PATCH 60/72] test: modify testcase of muti-mnodes --- tests/pytest/util/cluster.py | 6 +- .../5dnode3mnodeSep1VnodeStopCreateDb.py | 181 +++++++++++ .../5dnode3mnodeSep1VnodeStopMnodeCreateDb.py | 181 +++++++++++ .../5dnode3mnodeSep1VnodeStopVnodeCreateDb.py | 181 +++++++++++ .../5dnode3mnodeSeperate1VnodeStopInsert.py | 85 +---- .../system-test/6-cluster/5dnode3mnodeStop.py | 82 +++-- .../6-cluster/clusterCommonCheck.py | 211 +++++++++++++ .../6-cluster/clusterCommonCreate.py | 298 ++++++++++++++++++ tests/system-test/fulltest.sh | 4 + 9 files changed, 1130 insertions(+), 99 deletions(-) create mode 100644 tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopCreateDb.py create mode 100644 tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py create mode 100644 tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py create mode 100644 tests/system-test/6-cluster/clusterCommonCheck.py create mode 100644 tests/system-test/6-cluster/clusterCommonCreate.py diff --git a/tests/pytest/util/cluster.py b/tests/pytest/util/cluster.py index 6eb78f0771..21cad4c5bb 100644 --- a/tests/pytest/util/cluster.py +++ b/tests/pytest/util/cluster.py @@ -24,7 +24,7 @@ class ClusterDnodes(TDDnodes): class ConfigureyCluster: - """This will create defined number of dnodes and create a cluset. + """This will create defined number of dnodes and create a cluster. at the same time, it will return TDDnodes list: dnodes, """ hostname= socket.gethostname() @@ -85,8 +85,8 @@ class ConfigureyCluster: count+=1 time.sleep(1) else: - tdLog.debug("create cluster with %d dnode but check dnode not ready within 5s ! "%self.dnodeNums) - return -1 + tdLog.exit("create cluster with %d dnode but check dnode not ready within 5s ! "%self.dnodeNums) + cluster = ConfigureyCluster() \ No newline at end of file diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopCreateDb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopCreateDb.py new file mode 100644 index 0000000000..59fe1c0b16 --- /dev/null +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopCreateDb.py @@ -0,0 +1,181 @@ +from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE +import taos +import sys +import time +import os + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import TDDnodes +from util.dnodes import TDDnode +from util.cluster import * +sys.path.append("./6-cluster") +from clusterCommonCreate import * +from clusterCommonCheck import clusterComCheck + +import time +import socket +import subprocess +from multiprocessing import Process +import threading +import time +import inspect +import ctypes + +class TDTestCase: + + def init(self,conn ,logSql): + tdLog.debug(f"start to excute {__file__}") + self.TDDnodes = None + tdSql.init(conn.cursor()) + self.host = socket.gethostname() + + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def _async_raise(self, tid, exctype): + """raises the exception, performs cleanup if needed""" + if not inspect.isclass(exctype): + exctype = type(exctype) + res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(exctype)) + if res == 0: + raise ValueError("invalid thread id") + elif res != 1: + # """if it returns a number greater than one, you're in trouble, + # and you should call it again with exc=NULL to revert the effect""" + ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None) + raise SystemError("PyThreadState_SetAsyncExc failed") + + def stopThread(self,thread): + self._async_raise(thread.ident, SystemExit) + + + def insertData(self,countstart,countstop): + # fisrt add data : db\stable\childtable\general table + + for couti in range(countstart,countstop): + tdLog.debug("drop database if exists db%d" %couti) + tdSql.execute("drop database if exists db%d" %couti) + print("create database if not exists db%d replica 1 duration 300" %couti) + tdSql.execute("create database if not exists db%d replica 1 duration 300" %couti) + tdSql.execute("use db%d" %couti) + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + + def fiveDnodeThreeMnode(self,dnodenumbers,mnodeNums,restartNumber): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'db', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'replica': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbNum': 1, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 10, + 'showMsg': 1, + 'showRow': 1} + dnodenumbers=int(dnodenumbers) + mnodeNums=int(mnodeNums) + dbNumbers = int(dnodenumbers * restartNumber) + + tdLog.info("first check dnode and mnode") + tdSql.query("show dnodes;") + tdSql.checkData(0,1,'%s:6030'%self.host) + tdSql.checkData(4,1,'%s:6430'%self.host) + clusterComCheck.checkDnodes(dnodenumbers) + clusterComCheck.checkMnodeStatus(1) + + # fisr add three mnodes; + tdLog.info("fisr add three mnodes and check mnode status") + tdSql.execute("create mnode on dnode 2") + clusterComCheck.checkMnodeStatus(2) + tdSql.execute("create mnode on dnode 3") + clusterComCheck.checkMnodeStatus(3) + + # add some error operations and + tdLog.info("Confirm the status of the dnode again") + tdSql.error("create mnode on dnode 2") + tdSql.query("show dnodes;") + print(tdSql.queryResult) + clusterComCheck.checkDnodes(dnodenumbers) + + tdLog.info("Take turns stopping all dnodes ") + # seperate vnode and mnode in different dnodes. + # create database and stable + tdDnodes=cluster.dnodes + stopcount =0 + while stopcount < restartNumber: + for i in range(dnodenumbers): + # threads=[] + # threads = MyThreadFunc(self.insert_data(i*2,i*2+2)) + paraDict["dbName"]= 'db%d%d'%(stopcount,i) + threads=threading.Thread(target=clusterComCreate.create_database, args=(tdSql, paraDict["dbName"],paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica'])) + threads.start() + tdDnodes[i].stoptaosd() + # sleep(10) + tdDnodes[i].starttaosd() + # sleep(10) + + if clusterComCheck.checkDnodes(dnodenumbers): + # threads.join() + tdLog.info("first restart loop") + else: + print("456") + threads.join() + self.stopThread(threads) + tdLog.exit("one or more of dnodes failed to start ") + # self.check3mnode() + stopcount+=1 + threads.join() + clusterComCheck.checkDnodes(dnodenumbers) + clusterComCheck.checkDbRows(dbNumbers) + for i in range(restartNumber): + clusterComCheck.checkDb(dnodenumbers,'db%d'%i) + + + def run(self): + # print(self.master_dnode.cfgDict) + self.fiveDnodeThreeMnode(5,3,1) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py new file mode 100644 index 0000000000..cf608f6480 --- /dev/null +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py @@ -0,0 +1,181 @@ +from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE +import taos +import sys +import time +import os + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import TDDnodes +from util.dnodes import TDDnode +from util.cluster import * +sys.path.append("./6-cluster") +from clusterCommonCreate import * +from clusterCommonCheck import clusterComCheck + +import time +import socket +import subprocess +from multiprocessing import Process +import threading +import time +import inspect +import ctypes + +class TDTestCase: + + def init(self,conn ,logSql): + tdLog.debug(f"start to excute {__file__}") + self.TDDnodes = None + tdSql.init(conn.cursor()) + self.host = socket.gethostname() + + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def _async_raise(self, tid, exctype): + """raises the exception, performs cleanup if needed""" + if not inspect.isclass(exctype): + exctype = type(exctype) + res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(exctype)) + if res == 0: + raise ValueError("invalid thread id") + elif res != 1: + # """if it returns a number greater than one, you're in trouble, + # and you should call it again with exc=NULL to revert the effect""" + ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None) + raise SystemError("PyThreadState_SetAsyncExc failed") + + def stopThread(self,thread): + self._async_raise(thread.ident, SystemExit) + + + def insertData(self,countstart,countstop): + # fisrt add data : db\stable\childtable\general table + + for couti in range(countstart,countstop): + tdLog.debug("drop database if exists db%d" %couti) + tdSql.execute("drop database if exists db%d" %couti) + print("create database if not exists db%d replica 1 duration 300" %couti) + tdSql.execute("create database if not exists db%d replica 1 duration 300" %couti) + tdSql.execute("use db%d" %couti) + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + + def fiveDnodeThreeMnode(self,dnodenumbers,mnodeNums,restartNumber): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'db', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'replica': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbNum': 1, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 10, + 'showMsg': 1, + 'showRow': 1} + dnodenumbers=int(dnodenumbers) + mnodeNums=int(mnodeNums) + dbNumbers = int(mnodeNums * restartNumber) + tdLog.info("first check dnode and mnode") + tdSql.query("show dnodes;") + tdSql.checkData(0,1,'%s:6030'%self.host) + tdSql.checkData(4,1,'%s:6430'%self.host) + clusterComCheck.checkDnodes(dnodenumbers) + clusterComCheck.checkMnodeStatus(1) + + # fisr add three mnodes; + tdLog.info("fisr add three mnodes and check mnode status") + tdSql.execute("create mnode on dnode 2") + clusterComCheck.checkMnodeStatus(2) + tdSql.execute("create mnode on dnode 3") + clusterComCheck.checkMnodeStatus(3) + + # add some error operations and + tdLog.info("Confirm the status of the dnode again") + tdSql.error("create mnode on dnode 2") + tdSql.query("show dnodes;") + print(tdSql.queryResult) + clusterComCheck.checkDnodes(dnodenumbers) + + tdLog.info("Take turns stopping Mnodes ") + # seperate vnode and mnode in different dnodes. + # create database and stable + tdDnodes=cluster.dnodes + stopcount =0 + while stopcount < restartNumber: + tdLog.info("first restart loop") + for i in range(mnodeNums): + # threads=[] + # threads = MyThreadFunc(self.insert_data(i*2,i*2+2)) + paraDict["dbName"]= 'db%d%d'%(stopcount,i) + threads=threading.Thread(target=clusterComCreate.create_database, args=(tdSql, paraDict["dbName"],paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica'])) + threads.start() + tdDnodes[i].stoptaosd() + # sleep(10) + tdDnodes[i].starttaosd() + # sleep(10) + + if clusterComCheck.checkDnodes(dnodenumbers): + # threads.join() + tdLog.info("123") + else: + print("456") + threads.join() + self.stopThread(threads) + tdLog.exit("one or more of dnodes failed to start ") + # self.check3mnode() + stopcount+=1 + threads.join() + clusterComCheck.checkDnodes(dnodenumbers) + clusterComCheck.checkDbRows(dbNumbers) + for i in range(restartNumber): + clusterComCheck.checkDb(mnodeNums,'db%d'%i) + + + def run(self): + # print(self.master_dnode.cfgDict) + self.fiveDnodeThreeMnode(5,3,1) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py new file mode 100644 index 0000000000..2d2322fada --- /dev/null +++ b/tests/system-test/6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py @@ -0,0 +1,181 @@ +from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE +import taos +import sys +import time +import os + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import TDDnodes +from util.dnodes import TDDnode +from util.cluster import * +sys.path.append("./6-cluster") +from clusterCommonCreate import * +from clusterCommonCheck import clusterComCheck + +import time +import socket +import subprocess +from multiprocessing import Process +import threading +import time +import inspect +import ctypes + +class TDTestCase: + + def init(self,conn ,logSql): + tdLog.debug(f"start to excute {__file__}") + self.TDDnodes = None + tdSql.init(conn.cursor()) + self.host = socket.gethostname() + + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root) - len("/build/bin")] + break + return buildPath + + def _async_raise(self, tid, exctype): + """raises the exception, performs cleanup if needed""" + if not inspect.isclass(exctype): + exctype = type(exctype) + res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(exctype)) + if res == 0: + raise ValueError("invalid thread id") + elif res != 1: + # """if it returns a number greater than one, you're in trouble, + # and you should call it again with exc=NULL to revert the effect""" + ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None) + raise SystemError("PyThreadState_SetAsyncExc failed") + + def stopThread(self,thread): + self._async_raise(thread.ident, SystemExit) + + + def insertData(self,countstart,countstop): + # fisrt add data : db\stable\childtable\general table + + for couti in range(countstart,countstop): + tdLog.debug("drop database if exists db%d" %couti) + tdSql.execute("drop database if exists db%d" %couti) + print("create database if not exists db%d replica 1 duration 300" %couti) + tdSql.execute("create database if not exists db%d replica 1 duration 300" %couti) + tdSql.execute("use db%d" %couti) + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t1 int) + ''' + ) + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + + def fiveDnodeThreeMnode(self,dnodenumbers,mnodeNums,restartNumber): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'db', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'replica': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbNum': 1, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 10, + 'showMsg': 1, + 'showRow': 1} + dnodenumbers=int(dnodenumbers) + mnodeNums=int(mnodeNums) + vnodeNumbers = int(dnodenumbers-mnodeNums) + dbNumbers = int(vnodeNumbers * restartNumber) + tdLog.info("first check dnode and mnode") + tdSql.query("show dnodes;") + tdSql.checkData(0,1,'%s:6030'%self.host) + tdSql.checkData(4,1,'%s:6430'%self.host) + clusterComCheck.checkDnodes(dnodenumbers) + clusterComCheck.checkMnodeStatus(1) + + # fisr add three mnodes; + tdLog.info("fisr add three mnodes and check mnode status") + tdSql.execute("create mnode on dnode 2") + clusterComCheck.checkMnodeStatus(2) + tdSql.execute("create mnode on dnode 3") + clusterComCheck.checkMnodeStatus(3) + + # add some error operations and + tdLog.info("Confirm the status of the dnode again") + tdSql.error("create mnode on dnode 2") + tdSql.query("show dnodes;") + print(tdSql.queryResult) + clusterComCheck.checkDnodes(dnodenumbers) + + tdLog.info("Take turns stopping Vnodes ") + # seperate vnode and mnode in different dnodes. + # create database and stable + tdDnodes=cluster.dnodes + stopcount =0 + while stopcount < restartNumber: + for i in range(vnodeNumbers): + # threads=[] + # threads = MyThreadFunc(self.insert_data(i*2,i*2+2)) + paraDict["dbName"]= 'db%d%d'%(stopcount,i) + threads=threading.Thread(target=clusterComCreate.create_database, args=(tdSql, paraDict["dbName"],paraDict["dropFlag"], paraDict["vgroups"],paraDict['replica'])) + threads.start() + tdDnodes[mnodeNums+i].stoptaosd() + # sleep(10) + tdDnodes[mnodeNums+i].starttaosd() + # sleep(10) + + if clusterComCheck.checkDnodes(vnodeNumbers): + # threads.join() + tdLog.info("first restart loop") + else: + print("456") + threads.join() + self.stopThread(threads) + tdLog.exit("one or more of dnodes failed to start ") + # self.check3mnode() + stopcount+=1 + threads.join() + clusterComCheck.checkDnodes(dnodenumbers) + clusterComCheck.checkDbRows(dbNumbers) + for i in range(restartNumber): + clusterComCheck.checkDb(vnodeNumbers,'db%d'%i) + + + def run(self): + # print(self.master_dnode.cfgDict) + self.fiveDnodeThreeMnode(5,3,1) + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file diff --git a/tests/system-test/6-cluster/5dnode3mnodeSeperate1VnodeStopInsert.py b/tests/system-test/6-cluster/5dnode3mnodeSeperate1VnodeStopInsert.py index 1739db09af..aa1d7ecc29 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeSeperate1VnodeStopInsert.py +++ b/tests/system-test/6-cluster/5dnode3mnodeSeperate1VnodeStopInsert.py @@ -9,6 +9,11 @@ from util.sql import * from util.cases import * from util.dnodes import TDDnodes from util.dnodes import TDDnode +from util.cluster import * +from util.common import * +sys.path.append("./7-tmq") +from tmqCommon import * + import time import socket import subprocess @@ -17,26 +22,13 @@ import threading import time import inspect import ctypes -class MyDnodes(TDDnodes): - def __init__(self ,dnodes_lists): - super(MyDnodes,self).__init__() - self.dnodes = dnodes_lists # dnode must be TDDnode instance - self.simDeployed = False - class TDTestCase: def init(self,conn ,logSql): tdLog.debug(f"start to excute {__file__}") - self.TDDnodes = None - - def buildcluster(self,dnodenumber): - self.depoly_cluster(dnodenumber) - self.master_dnode = self.TDDnodes.dnodes[0] - self.host=self.master_dnode.cfgDict["fqdn"] - conn1 = taos.connect(self.master_dnode.cfgDict["fqdn"] , config=self.master_dnode.cfgDir) - tdSql.init(conn1.cursor()) - + # tdSql.init(conn.cursor()) + # self.host = socket.gethostname() def getBuildPath(self): selfPath = os.path.dirname(os.path.realpath(__file__)) @@ -106,52 +98,6 @@ class TDTestCase: tdSql.checkData(0,0,rowsPerSTable) return - def depoly_cluster(self ,dnodes_nums=5,independent=True): - - testCluster = False - valgrind = 0 - hostname = socket.gethostname() - dnodes = [] - start_port = 6030 - start_port_sec = 6130 - for num in range(1, dnodes_nums+1): - dnode = TDDnode(num) - dnode.addExtraCfg("firstEp", f"{hostname}:{start_port}") - dnode.addExtraCfg("fqdn", f"{hostname}") - dnode.addExtraCfg("serverPort", f"{start_port + (num-1)*100}") - dnode.addExtraCfg("monitorFqdn", hostname) - dnode.addExtraCfg("monitorPort", 7043) - dnode.addExtraCfg("secondEp", f"{hostname}:{start_port_sec}") - # configure three dnoe don't support vnodes - if independent and (num < 4): - dnode.addExtraCfg("supportVnodes", 0) - - dnodes.append(dnode) - - self.TDDnodes = MyDnodes(dnodes) - self.TDDnodes.init("") - self.TDDnodes.setTestCluster(testCluster) - self.TDDnodes.setValgrind(valgrind) - self.TDDnodes.stopAll() - for dnode in self.TDDnodes.dnodes: - self.TDDnodes.deploy(dnode.index,{}) - - for dnode in self.TDDnodes.dnodes: - self.TDDnodes.starttaosd(dnode.index) - - # create cluster - for dnode in self.TDDnodes.dnodes[1:]: - # print(dnode.cfgDict) - dnode_id = dnode.cfgDict["fqdn"] + ":" +dnode.cfgDict["serverPort"] - dnode_first_host = dnode.cfgDict["firstEp"].split(":")[0] - dnode_first_port = dnode.cfgDict["firstEp"].split(":")[-1] - cmd = f" taos -h {dnode_first_host} -P {dnode_first_port} -s ' create dnode \"{dnode_id} \" ' ;" - print(cmd) - os.system(cmd) - - time.sleep(2) - tdLog.info(" create cluster with %d dnode done! " %dnodes_nums) - def checkdnodes(self,dnodenumber): count=0 while count < 100: @@ -305,6 +251,14 @@ class TDTestCase: tdSql.checkData(2,2,'offline') tdSql.checkData(2,3,'ready') + + def check5dnode(self): + tdSql.query("show dnodes;") + tdSql.checkData(0,1,'%s:6030'%self.host) + tdSql.checkData(4,1,'%s:6430'%self.host) + tdSql.checkData(0,4,'ready') + tdSql.checkData(4,4,'ready') + def five_dnode_three_mnode(self,dnodenumber): tdSql.query("show dnodes;") tdSql.checkData(0,1,'%s:6030'%self.host) @@ -346,6 +300,7 @@ class TDTestCase: threads.join() else: print("456") + threads.join() self.stop_thread(threads) assert 1 == 2 ,"some dnode started failed" return False @@ -357,16 +312,8 @@ class TDTestCase: self.check3mnode() - def getConnection(self, dnode): - host = dnode.cfgDict["fqdn"] - port = dnode.cfgDict["serverPort"] - config_dir = dnode.cfgDir - return taos.connect(host=host, port=int(port), config=config_dir) - - def run(self): # print(self.master_dnode.cfgDict) - self.buildcluster(5) self.five_dnode_three_mnode(5) def stop(self): diff --git a/tests/system-test/6-cluster/5dnode3mnodeStop.py b/tests/system-test/6-cluster/5dnode3mnodeStop.py index 69b9c3d879..6c32da853a 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeStop.py +++ b/tests/system-test/6-cluster/5dnode3mnodeStop.py @@ -12,7 +12,10 @@ from util.dnodes import TDDnodes from util.dnodes import TDDnode from util.cluster import * from test import tdDnodes +sys.path.append("./6-cluster") +from clusterCommonCreate import * +from clusterCommonCheck import * import time import socket import subprocess @@ -216,59 +219,84 @@ class TDTestCase: else: tdLog.exit("create cluster with %d dnode but check dnode not ready within 5s ! "%dnodeNumbers) - def five_dnode_three_mnode(self,dnodenumber): - self.check_dnodes_status(5) - tdSql.query("show mnodes;") - tdLog.debug(self.host) - tdSql.checkRows(1) + def fiveDnodeThreeMnode(self,dnodenumbers,mnodeNums,restartNumber): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'db', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'replica': 1, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbNum': 1, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 10, + 'showMsg': 1, + 'showRow': 1} + dnodenumbers=int(dnodenumbers) + mnodeNums=int(mnodeNums) + dbNumbers = int(dnodenumbers * restartNumber) + + tdLog.info("first check dnode and mnode") + tdSql.query("show dnodes;") tdSql.checkData(0,1,'%s:6030'%self.host) - tdSql.checkData(0,2,'leader') - tdSql.checkData(0,3,'ready') + tdSql.checkData(4,1,'%s:6430'%self.host) + clusterComCheck.checkDnodes(dnodenumbers) + clusterComCheck.checkMnodeStatus(1) # fisr add three mnodes; + tdLog.info("fisr add three mnodes and check mnode status") tdSql.execute("create mnode on dnode 2") + clusterComCheck.checkMnodeStatus(2) tdSql.execute("create mnode on dnode 3") + clusterComCheck.checkMnodeStatus(3) - # fisrt check statut ready - self.check3mnode() - - + # add some error operations and + tdLog.info("Confirm the status of the dnode again") tdSql.error("create mnode on dnode 2") - tdSql.query("show dnodes;") - # tdLog.debug(tdSql.queryResult) - - tdLog.debug("stop and follower of mnode") + print(tdSql.queryResult) + clusterComCheck.checkDnodes(dnodenumbers) + # restart all taosd tdDnodes=cluster.dnodes - # tdLog.debug(tdDnodes[0]) tdDnodes[1].stoptaosd() - self.check3mnode2off() + clusterComCheck.check3mnodeoff(2,3) tdDnodes[1].starttaosd() - self.check3mnode() + clusterComCheck.checkMnodeStatus(3) tdDnodes[2].stoptaosd() - self.check3mnode3off() + clusterComCheck.check3mnodeoff(3,3) tdDnodes[2].starttaosd() - self.check3mnode() + clusterComCheck.checkMnodeStatus(3) tdDnodes[0].stoptaosd() - self.check3mnode1off() + clusterComCheck.check3mnodeoff(1,3) tdDnodes[0].starttaosd() - self.check3mnode() + clusterComCheck.checkMnodeStatus(3) - self.check3mnode() + tdLog.info("Take turns stopping all dnodes ") + # seperate vnode and mnode in different dnodes. + # create database and stable stopcount =0 while stopcount <= 2: - for i in range(dnodenumber): + tdLog.info("first restart loop") + for i in range(dnodenumbers): tdDnodes[i].stoptaosd() tdDnodes[i].starttaosd() - # self.check3mnode() stopcount+=1 - self.check3mnode() + clusterComCheck.checkDnodes(dnodenumbers) + clusterComCheck.checkMnodeStatus(3) def run(self): - self.five_dnode_three_mnode(5) + # print(self.master_dnode.cfgDict) + self.fiveDnodeThreeMnode(5,3,1) def stop(self): tdSql.close() diff --git a/tests/system-test/6-cluster/clusterCommonCheck.py b/tests/system-test/6-cluster/clusterCommonCheck.py new file mode 100644 index 0000000000..d030008963 --- /dev/null +++ b/tests/system-test/6-cluster/clusterCommonCheck.py @@ -0,0 +1,211 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +from collections import defaultdict +import random +import string +import threading +import requests +import time +# import socketfrom + +import taos +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * + +# class actionType(Enum): +# CREATE_DATABASE = 0 +# CREATE_STABLE = 1 +# CREATE_CTABLE = 2 +# INSERT_DATA = 3 + +class ClusterComCheck: + def init(self, conn, logSql): + tdSql.init(conn.cursor()) + # tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def checkDnodes(self,dnodeNumbers): + count=0 + while count < 5: + tdSql.query("show dnodes") + # tdLog.debug(tdSql.queryResult) + status=0 + for i in range(dnodeNumbers): + if tdSql.queryResult[i][4] == "ready": + status+=1 + tdLog.info(status) + + if status == dnodeNumbers: + tdLog.success("it find cluster with %d dnodes and check that all cluster dnodes are ready within 5s! " %dnodeNumbers) + return True + count+=1 + time.sleep(1) + else: + tdLog.debug(tdSql.queryResult) + tdLog.exit("it find cluster with %d dnodes but check that there dnodes are not ready within 5s ! "%dnodeNumbers) + + def checkDbRows(self,dbNumbers): + dbNumbers=int(dbNumbers) + count=0 + while count < 5: + tdSql.query("show databases;") + if tdSql.checkRows(dbNumbers+2): + tdLog.success("we find %d databases and expect %d in clusters! " %(tdSql.queryRows,dbNumbers+2)) + return True + else: + continue + else : + tdLog.debug(tdSql.queryResult) + tdLog.exit("we find %d databases but expect %d in clusters! " %(tdSql.queryRows,dbNumbers)) + + def checkDb(self,dbNumbers,dbindex): + count=0 + while count < 5: + query_status=0 + for i in range(dbNumbers): + for j in range(dbNumbers): + tdSql.query("show databases;") + if "%s%d"%(dbindex,j) == tdSql.queryResult[i+2][0] : + if tdSql.queryResult[i+2][19] == "ready": + query_status+=1 + else: + continue + # print(query_status) + count+=1 + if query_status == dbNumbers: + tdLog.success("we find cluster with %d dnode and check all databases are ready within 5s! " %dbNumbers) + return True + else: + tdLog.debug(tdSql.queryResult) + tdLog.exit("database is not ready within 5s") + + def checkData(self,dbname,stbname,stableCount,CtableCount,rowsPerSTable,): + tdSql.execute("use %s"%dbname) + tdSql.query("show stables") + tdSql.checkRows(stableCount) + tdSql.query("show tables") + tdSql.checkRows(CtableCount) + for i in range(stableCount): + tdSql.query("select count(*) from %s%d"%(stbname,i)) + tdSql.checkData(0,0,rowsPerSTable) + return + + def checkMnodeStatus(self,mnodeNums): + self.mnodeNums=int(mnodeNums) + # self.leaderDnode=int(leaderDnode) + + count=0 + + while count < 10: + time.sleep(1) + tdSql.query("show mnodes;") + if tdSql.checkRows(self.mnodeNums) : + tdLog.success("cluster has %d mnodes" %self.mnodeNums ) + + if self.mnodeNums == 1: + if tdSql.queryResult[0][2]== 'leader' and tdSql.queryResult[0][3]== 'ready' : + tdLog.success("%d mnodes is ready in 10s"%self.mnodeNums) + return True + count+=1 + elif self.mnodeNums == 3 : + if tdSql.queryResult[0][2]=='leader' and tdSql.queryResult[0][3]== 'ready' : + if tdSql.queryResult[1][2]=='follower' and tdSql.queryResult[1][3]== 'ready' : + if tdSql.queryResult[2][2]=='follower' and tdSql.queryResult[2][3]== 'ready' : + tdLog.success("%d mnodes is ready in 10s"%self.mnodeNums) + return True + elif tdSql.queryResult[1][2]=='leader' and tdSql.queryResult[1][3]== 'ready' : + if tdSql.queryResult[0][2]=='follower' and tdSql.queryResult[0][3]== 'ready' : + if tdSql.queryResult[2][2]=='follower' and tdSql.queryResult[2][3]== 'ready' : + tdLog.success("%d mnodes is ready in 10s"%self.mnodeNums) + return True + elif tdSql.queryResult[2][2]=='leader' and tdSql.queryResult[2][3]== 'ready' : + if tdSql.queryResult[0][2]=='follower' and tdSql.queryResult[0][3]== 'ready' : + if tdSql.queryResult[1][2]=='follower' and tdSql.queryResult[1][3]== 'ready' : + tdLog.success("%d mnodes is ready in 10s"%self.mnodeNums) + return True + count+=1 + elif self.mnodeNums == 2 : + if tdSql.queryResult[0][2]=='leader' and tdSql.queryResult[0][3]== 'ready' : + if tdSql.queryResult[1][2]=='follower' and tdSql.queryResult[1][3]== 'ready' : + tdLog.success("%d mnodes is ready in 10s"%self.mnodeNums) + return True + elif tdSql.queryResult[1][2]=='leader' and tdSql.queryResult[1][3]== 'ready' : + if tdSql.queryResult[0][2]=='follower' and tdSql.queryResult[0][3]== 'ready' : + tdLog.success("%d mnodes is ready in 10s"%self.mnodeNums) + return True + count+=1 + else: + tdLog.debug(tdSql.queryResult) + tdLog.exit("cluster of %d mnodes is not ready in 10s " %self.mnodeNums) + + + + + def check3mnodeoff(self,offlineDnodeNo,mnodeNums=3): + count=0 + while count < 10: + time.sleep(1) + tdSql.query("show mnodes;") + if tdSql.checkRows(mnodeNums) : + tdLog.success("cluster has %d mnodes" %self.mnodeNums ) + else: + tdLog.exit("mnode number is correct") + if offlineDnodeNo == 1: + if tdSql.queryResult[0][2]=='offline' : + if tdSql.queryResult[1][2]=='leader' and tdSql.queryResult[1][3]== 'ready' : + if tdSql.queryResult[2][2]=='follower' and tdSql.queryResult[2][3]== 'ready' : + tdLog.success("stop mnodes on dnode %d successfully in 10s" %offlineDnodeNo) + return True + elif tdSql.queryResult[1][2]=='follower' and tdSql.queryResult[1][3]== 'ready' : + if tdSql.queryResult[2][2]=='leader' and tdSql.queryResult[2][3]== 'ready' : + tdLog.debug("stop mnodes on dnode %d successfully in 10s" %offlineDnodeNo) + return True + count+=1 + elif offlineDnodeNo == 2: + if tdSql.queryResult[1][2]=='offline' : + if tdSql.queryResult[0][2]=='leader' and tdSql.queryResult[0][3]== 'ready' : + if tdSql.queryResult[2][2]=='follower' and tdSql.queryResult[2][3]== 'ready' : + tdLog.debug("stop mnodes on dnode %d successfully in 10s" %offlineDnodeNo) + return True + elif tdSql.queryResult[0][2]=='follower' and tdSql.queryResult[0][3]== 'ready' : + if tdSql.queryResult[2][2]=='leader' and tdSql.queryResult[2][3]== 'ready' : + tdLog.debug("stop mnodes on dnode %d successfully in 10s" %offlineDnodeNo) + return True + count+=1 + elif offlineDnodeNo == 3: + if tdSql.queryResult[2][2]=='offline' : + if tdSql.queryResult[0][2]=='leader' and tdSql.queryResult[0][3]== 'ready' : + if tdSql.queryResult[1][2]=='follower' and tdSql.queryResult[1][3]== 'ready' : + tdLog.debug("stop mnodes on dnode %d successfully in 10s" %offlineDnodeNo) + return True + elif tdSql.queryResult[0][2]=='follower' and tdSql.queryResult[0][3]== 'ready' : + if tdSql.queryResult[1][2]=='leader' and tdSql.queryResult[1][3]== 'ready' : + tdLog.debug("stop mnodes on dnode %d successfully in 10s" %offlineDnodeNo) + return True + count+=1 + else: + tdLog.debug(tdSql.queryResult) + tdLog.exit("stop mnodes on dnode %d failed in 10s ") + + + + + + + def close(self): + self.cursor.close() + +clusterComCheck = ClusterComCheck() diff --git a/tests/system-test/6-cluster/clusterCommonCreate.py b/tests/system-test/6-cluster/clusterCommonCreate.py new file mode 100644 index 0000000000..b3107d8537 --- /dev/null +++ b/tests/system-test/6-cluster/clusterCommonCreate.py @@ -0,0 +1,298 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +from collections import defaultdict +import random +import string +import threading +import requests +import time +# import socketfrom + +import taos +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * + +# class actionType(Enum): +# CREATE_DATABASE = 0 +# CREATE_STABLE = 1 +# CREATE_CTABLE = 2 +# INSERT_DATA = 3 + +class ClusterComCreate: + def init(self, conn, logSql): + tdSql.init(conn.cursor()) + # tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def initConsumerTable(self,cdbName='cdb'): + tdLog.info("create consume database, and consume info table, and consume result table") + tdSql.query("create database if not exists %s vgroups 1"%(cdbName)) + tdSql.query("drop table if exists %s.consumeinfo "%(cdbName)) + tdSql.query("drop table if exists %s.consumeresult "%(cdbName)) + tdSql.query("drop table if exists %s.notifyinfo "%(cdbName)) + + tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)"%cdbName) + tdSql.query("create table %s.consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)"%cdbName) + tdSql.query("create table %s.notifyinfo (ts timestamp, cmdid int, consumerid int)"%cdbName) + + def initConsumerInfoTable(self,cdbName='cdb'): + tdLog.info("drop consumeinfo table") + tdSql.query("drop table if exists %s.consumeinfo "%(cdbName)) + tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)"%cdbName) + + def insertConsumerInfo(self,consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifmanualcommit,cdbName='cdb'): + sql = "insert into %s.consumeinfo values "%cdbName + sql += "(now, %d, '%s', '%s', %d, %d, %d)"%(consumerId, topicList, keyList, expectrowcnt, ifcheckdata, ifmanualcommit) + tdLog.info("consume info sql: %s"%sql) + tdSql.query(sql) + + def selectConsumeResult(self,expectRows,cdbName='cdb'): + resultList=[] + while 1: + tdSql.query("select * from %s.consumeresult"%cdbName) + #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) + if tdSql.getRows() == expectRows: + break + else: + time.sleep(5) + + for i in range(expectRows): + tdLog.info ("consume id: %d, consume msgs: %d, consume rows: %d"%(tdSql.getData(i , 1), tdSql.getData(i , 2), tdSql.getData(i , 3))) + resultList.append(tdSql.getData(i , 3)) + + return resultList + + def startTmqSimProcess(self,pollDelay,dbName,showMsg=1,showRow=1,cdbName='cdb',valgrind=0): + buildPath = tdCom.getBuildPath() + cfgPath = tdCom.getClientCfgPath() + if valgrind == 1: + logFile = cfgPath + '/../log/valgrind-tmq.log' + shellCmd = 'nohup valgrind --log-file=' + logFile + shellCmd += '--tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all --num-callers=20 -v --workaround-gcc296-bugs=yes ' + + if (platform.system().lower() == 'windows'): + shellCmd = 'mintty -h never -w hide ' + buildPath + '\\build\\bin\\tmq_sim.exe -c ' + cfgPath + shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) + shellCmd += "> nul 2>&1 &" + else: + shellCmd = 'nohup ' + buildPath + '/build/bin/tmq_sim -c ' + cfgPath + shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) + shellCmd += "> /dev/null 2>&1 &" + tdLog.info(shellCmd) + os.system(shellCmd) + + def getStartConsumeNotifyFromTmqsim(self,cdbName='cdb'): + while 1: + tdSql.query("select * from %s.notifyinfo"%cdbName) + #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) + if (tdSql.getRows() == 1) and (tdSql.getData(0, 1) == 0): + break + else: + time.sleep(0.1) + return + + def getStartCommitNotifyFromTmqsim(self,cdbName='cdb'): + while 1: + tdSql.query("select * from %s.notifyinfo"%cdbName) + #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) + if tdSql.getRows() == 2 : + print(tdSql.getData(0, 1), tdSql.getData(1, 1)) + if tdSql.getData(1, 1) == 1: + break + time.sleep(0.1) + return + + def create_database(self,tsql, dbName,dropFlag=1,vgroups=4,replica=1): + if dropFlag == 1: + tsql.execute("drop database if exists %s"%(dbName)) + + tsql.execute("create database if not exists %s vgroups %d replica %d"%(dbName, vgroups, replica)) + tdLog.debug("complete to create database %s"%(dbName)) + return + + def create_stable(self,tsql, dbName,stbName): + tsql.execute("create table if not exists %s.%s (ts timestamp, c1 int, c2 int, c3 binary(16)) tags(t1 int, t2 binary(32))"%(dbName, stbName)) + tdLog.debug("complete to create %s.%s" %(dbName, stbName)) + return + + def create_ctable(self,tsql=None, dbName='dbx',stbName='stb',ctbPrefix='ctb',ctbNum=1): + tsql.execute("use %s" %dbName) + pre_create = "create table" + sql = pre_create + #tdLog.debug("doing create one stable %s and %d child table in %s ..." %(stbname, count ,dbname)) + for i in range(ctbNum): + tagValue = 'beijing' + if (i % 2 == 0): + tagValue = 'shanghai' + + sql += " %s%d using %s tags(%d, '%s')"%(ctbPrefix,i,stbName,i+1, tagValue) + if (i > 0) and (i%100 == 0): + tsql.execute(sql) + sql = pre_create + if sql != pre_create: + tsql.execute(sql) + + tdLog.debug("complete to create %d child tables in %s.%s" %(ctbNum, dbName, stbName)) + return + + def insert_data(self,tsql,dbName,stbName,ctbNum,rowsPerTbl,batchNum,startTs=None): + tdLog.debug("start to insert data ............") + tsql.execute("use %s" %dbName) + pre_insert = "insert into " + sql = pre_insert + + if startTs is None: + t = time.time() + startTs = int(round(t * 1000)) + #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows)) + for i in range(ctbNum): + sql += " %s%d values "%(stbName,i) + for j in range(rowsPerTbl): + sql += "(%d, %d, 'tmqrow_%d') "%(startTs + j, j, j) + if (j > 0) and ((j%batchNum == 0) or (j == rowsPerTbl - 1)): + tsql.execute(sql) + if j < rowsPerTbl - 1: + sql = "insert into %s%d values " %(stbName,i) + else: + sql = "insert into " + #end sql + if sql != pre_insert: + #print("insert sql:%s"%sql) + tsql.execute(sql) + tdLog.debug("insert data ............ [OK]") + return + + def insert_data_1(self,tsql,dbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs): + tdLog.debug("start to insert data ............") + tsql.execute("use %s" %dbName) + pre_insert = "insert into " + sql = pre_insert + + t = time.time() + startTs = int(round(t * 1000)) + #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows)) + for i in range(ctbNum): + sql += " %s%d values "%(ctbPrefix,i) + for j in range(rowsPerTbl): + if (j % 2 == 0): + sql += "(%d, %d, %d, 'tmqrow_%d') "%(startTs + j, j, j, j) + else: + sql += "(%d, %d, %d, 'tmqrow_%d') "%(startTs + j, j, -j, j) + if (j > 0) and ((j%batchNum == 0) or (j == rowsPerTbl - 1)): + tsql.execute(sql) + if j < rowsPerTbl - 1: + sql = "insert into %s%d values " %(ctbPrefix,i) + else: + sql = "insert into " + #end sql + if sql != pre_insert: + #print("insert sql:%s"%sql) + tsql.execute(sql) + tdLog.debug("insert data ............ [OK]") + return + + def insert_data_interlaceByMultiTbl(self,tsql,dbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs=0): + tdLog.debug("start to insert data ............") + tsql.execute("use %s" %dbName) + pre_insert = "insert into " + sql = pre_insert + + if startTs == 0: + t = time.time() + startTs = int(round(t * 1000)) + + ctbDict = {} + for i in range(ctbNum): + ctbDict[i] = 0 + + #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows)) + rowsOfCtb = 0 + while rowsOfCtb < rowsPerTbl: + for i in range(ctbNum): + sql += " %s.%s_%d values "%(dbName,ctbPrefix,i) + for k in range(batchNum): + sql += "(%d, %d, 'tmqrow_%d') "%(startTs + ctbDict[i], ctbDict[i], ctbDict[i]) + ctbDict[i] += 1 + if (0 == ctbDict[i]%batchNum) or (ctbDict[i] == rowsPerTbl): + tsql.execute(sql) + sql = "insert into " + break + rowsOfCtb = ctbDict[0] + + tdLog.debug("insert data ............ [OK]") + return + + def insert_data_with_autoCreateTbl(self,tsql,dbName,stbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs=0): + tdLog.debug("start to insert data wiht auto create child table ............") + tsql.execute("use %s" %dbName) + pre_insert = "insert into " + sql = pre_insert + + if startTs == 0: + t = time.time() + startTs = int(round(t * 1000)) + + #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows)) + rowsOfSql = 0 + for i in range(ctbNum): + sql += " %s.%s_%d using %s.%s tags (%d) values "%(dbName,ctbPrefix,i,dbName,stbName,i) + for j in range(rowsPerTbl): + sql += "(%d, %d, 'tmqrow_%d') "%(startTs + j, j, j) + rowsOfSql += 1 + if (j > 0) and ((rowsOfSql == batchNum) or (j == rowsPerTbl - 1)): + tsql.execute(sql) + rowsOfSql = 0 + if j < rowsPerTbl - 1: + sql = "insert into %s.%s_%d using %s.%s tags (%d) values " %(dbName,ctbPrefix,i,dbName,stbName,i) + else: + sql = "insert into " + #end sql + if sql != pre_insert: + #print("insert sql:%s"%sql) + tsql.execute(sql) + tdLog.debug("insert data ............ [OK]") + return + + def syncCreateDbStbCtbInsertData(self, tsql, paraDict): + tdCom.create_database(tsql, paraDict["dbName"],paraDict["dropFlag"]) + tdCom.create_stable(tsql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) + tdCom.create_ctable(tsql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix']) + if "event" in paraDict and type(paraDict['event']) == type(threading.Event()): + paraDict["event"].set() + + ctbPrefix = paraDict['ctbPrefix'] + ctbNum = paraDict["ctbNum"] + for i in range(ctbNum): + tbName = '%s%s'%(ctbPrefix,i) + tdCom.insert_rows(tsql,dbname=paraDict["dbName"],tbname=tbName,start_ts_value=paraDict['startTs'],count=paraDict['rowsPerTbl']) + return + + def threadFunction(self, **paraDict): + # create new connector for new tdSql instance in my thread + newTdSql = tdCom.newTdSql() + self.syncCreateDbStbCtbInsertData(self, newTdSql, paraDict) + return + + def asyncCreateDbStbCtbInsertData(self, paraDict): + pThread = threading.Thread(target=self.threadFunction, kwargs=paraDict) + pThread.start() + return pThread + + + def close(self): + self.cursor.close() + +clusterComCreate = ClusterComCreate() diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 73e6716cad..825b6ea26b 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -113,6 +113,10 @@ python3 ./test.py -f 2-query/twa.py python3 ./test.py -f 6-cluster/5dnode1mnode.py python3 ./test.py -f 6-cluster/5dnode2mnode.py python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 5 -M 3 +# BUG python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopCreateDb.py -N 5 -M 3 +# BUG python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 5 -M 3 + # python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5 # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeStopInsert.py From bdd4134fc1d2b368316e8fe6c6063d671c3d2ac5 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Sat, 25 Jun 2022 18:11:11 +0800 Subject: [PATCH 61/72] Revert "fix(sma): drop stream when drop sma" --- source/dnode/mnode/impl/inc/mndStream.h | 5 ----- source/dnode/mnode/impl/src/mndSma.c | 18 ------------------ source/dnode/mnode/impl/src/mndStream.c | 2 +- source/dnode/vnode/src/tq/tqSink.c | 2 +- source/libs/executor/inc/executorimpl.h | 1 + source/libs/executor/src/executor.c | 11 +++++++---- source/libs/executor/src/scanoperator.c | 8 ++++++++ source/libs/wal/src/walRead.c | 2 +- 8 files changed, 19 insertions(+), 30 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndStream.h b/source/dnode/mnode/impl/inc/mndStream.h index 5e9089cec9..69385c3a46 100644 --- a/source/dnode/mnode/impl/inc/mndStream.h +++ b/source/dnode/mnode/impl/inc/mndStream.h @@ -34,11 +34,6 @@ SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw); int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb); int32_t mndPersistStream(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); -// for sma -// TODO refactor -int32_t mndDropStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); -int32_t mndPersistDropStreamLog(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); - #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index d880fb28d4..ef24cd0ba4 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -857,24 +857,6 @@ static int32_t mndDropSma(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SSmaObj *p mDebug("trans:%d, used to drop sma:%s", pTrans->id, pSma->name); mndTransSetDbName(pTrans, pDb->name, NULL); - SStreamObj *pStream = mndAcquireStream(pMnode, pSma->name); - if (pStream == NULL || pStream->smaId != pSma->uid) { - sdbRelease(pMnode->pSdb, pStream); - goto _OVER; - } else { - if (mndDropStreamTasks(pMnode, pTrans, pStream) < 0) { - mError("stream:%s, failed to drop task since %s", pStream->name, terrstr()); - sdbRelease(pMnode->pSdb, pStream); - goto _OVER; - } - - // drop stream - if (mndPersistDropStreamLog(pMnode, pTrans, pStream) < 0) { - sdbRelease(pMnode->pSdb, pStream); - goto _OVER; - } - } - if (mndSetDropSmaRedoLogs(pMnode, pTrans, pSma) != 0) goto _OVER; if (mndSetDropSmaVgroupRedoLogs(pMnode, pTrans, pVgroup) != 0) goto _OVER; if (mndSetDropSmaCommitLogs(pMnode, pTrans, pSma) != 0) goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 444a8864ef..b78756d8b8 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -490,7 +490,7 @@ static int32_t mndPersistTaskDropReq(STrans *pTrans, SStreamTask *pTask) { return 0; } -int32_t mndDropStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) { +static int32_t mndDropStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) { int32_t lv = taosArrayGetSize(pStream->tasks); for (int32_t i = 0; i < lv; i++) { SArray *pTasks = taosArrayGetP(pStream->tasks, i); diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 0bb9918488..9abc2f639b 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -43,7 +43,7 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo taosArrayPush(tagArray, &tagVal); tTagNew(tagArray, 1, false, &pTag); if (pTag == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; + taosArrayDestroy(schemaReqs); taosArrayDestroy(tagArray); return NULL; } diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index eb2c6f5102..36f81e86ff 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -348,6 +348,7 @@ typedef struct SStreamBlockScanInfo { SInterval interval; // if the upstream is an interval operator, the interval info is also kept here. SArray* childIds; SessionWindowSupporter sessionSup; + bool assignBlockUid; // assign block uid to groupId, temporarily used for generating rollup SMA. int32_t scanWinIndex; // for state operator int32_t pullDataResIndex; SSDataBlock* pPullDataRes; // pull data SSDataBlock diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 3fd491885f..6de364e63a 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -19,7 +19,8 @@ #include "tdatablock.h" #include "vnode.h" -static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t numOfBlocks, int32_t type, char* id) { +static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t numOfBlocks, int32_t type, bool assignUid, + char* id) { ASSERT(pOperator != NULL); if (pOperator->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { if (pOperator->numOfDownstream == 0) { @@ -32,11 +33,12 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu return TSDB_CODE_QRY_APP_ERROR; } pOperator->status = OP_NOT_OPENED; - return doSetStreamBlock(pOperator->pDownstream[0], input, numOfBlocks, type, id); + return doSetStreamBlock(pOperator->pDownstream[0], input, numOfBlocks, type, assignUid, id); } else { pOperator->status = OP_NOT_OPENED; SStreamBlockScanInfo* pInfo = pOperator->info; + pInfo->assignBlockUid = assignUid; // TODO: if a block was set but not consumed, // prevent setting a different type of block @@ -74,7 +76,7 @@ int32_t qStreamScanSnapshot(qTaskInfo_t tinfo) { return TSDB_CODE_QRY_APP_ERROR; } SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - return doSetStreamBlock(pTaskInfo->pRoot, NULL, 0, STREAM_DATA_TYPE_FROM_SNAPSHOT, NULL); + return doSetStreamBlock(pTaskInfo->pRoot, NULL, 0, STREAM_DATA_TYPE_FROM_SNAPSHOT, 0, NULL); } int32_t qSetStreamInput(qTaskInfo_t tinfo, const void* input, int32_t type, bool assignUid) { @@ -92,7 +94,8 @@ int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numO SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - int32_t code = doSetStreamBlock(pTaskInfo->pRoot, (void**)pBlocks, numOfBlocks, type, GET_TASKID(pTaskInfo)); + int32_t code = + doSetStreamBlock(pTaskInfo->pRoot, (void**)pBlocks, numOfBlocks, type, assignUid, GET_TASKID(pTaskInfo)); if (code != TSDB_CODE_SUCCESS) { qError("%s failed to set the stream block data", GET_TASKID(pTaskInfo)); } else { diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index aadf7b3b7e..15803a4b73 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1051,6 +1051,14 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { pInfo->pRes->info.type = STREAM_NORMAL; pInfo->pRes->info.capacity = numOfRows; + // for generating rollup SMA result, each time is an independent time serie. + // TODO temporarily used, when the statement of "partition by tbname" is ready, remove this + if (pInfo->assignBlockUid) { + pInfo->pRes->info.groupId = uid; + } else { + pInfo->pRes->info.groupId = groupId; + } + uint64_t* groupIdPre = taosHashGet(pOperator->pTaskInfo->tableqinfoList.map, &uid, sizeof(int64_t)); if (groupIdPre) { pInfo->pRes->info.groupId = *groupIdPre; diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 20fa5f1f2b..682afbb785 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -103,7 +103,6 @@ static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) { wError("cannot open file %s, since %s", fnameStr, terrstr()); return -1; } - pRead->pReadLogTFile = pLogTFile; walBuildIdxName(pRead->pWal, fileFirstVer, fnameStr); TdFilePtr pIdxTFile = taosOpenFile(fnameStr, TD_FILE_READ); @@ -113,6 +112,7 @@ static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) { return -1; } + pRead->pReadLogTFile = pLogTFile; pRead->pReadIdxTFile = pIdxTFile; return 0; } From 996f995644fbf8bc2566b99a3c1cd8d979acee4e Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Sat, 25 Jun 2022 18:14:42 +0800 Subject: [PATCH 62/72] fix(sma): drop stream when drop sma --- source/dnode/mnode/impl/inc/mndStream.h | 4 ++++ source/dnode/mnode/impl/src/mndSma.c | 19 ++++++++++++++++++- source/dnode/mnode/impl/src/mndStream.c | 4 ++-- source/dnode/vnode/src/tq/tqSink.c | 2 +- source/libs/wal/src/walRead.c | 3 ++- 5 files changed, 27 insertions(+), 5 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndStream.h b/source/dnode/mnode/impl/inc/mndStream.h index 69385c3a46..0901e77287 100644 --- a/source/dnode/mnode/impl/inc/mndStream.h +++ b/source/dnode/mnode/impl/inc/mndStream.h @@ -33,6 +33,10 @@ SSdbRow *mndStreamActionDecode(SSdbRaw *pRaw); int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb); int32_t mndPersistStream(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); +// for sma +// TODO refactor +int32_t mndDropStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); +int32_t mndPersistDropStreamLog(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index ef24cd0ba4..10eeaba982 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -15,11 +15,11 @@ #define _DEFAULT_SOURCE #include "mndSma.h" -#include "mndPrivilege.h" #include "mndDb.h" #include "mndDnode.h" #include "mndInfoSchema.h" #include "mndMnode.h" +#include "mndPrivilege.h" #include "mndScheduler.h" #include "mndShow.h" #include "mndStb.h" @@ -857,6 +857,23 @@ static int32_t mndDropSma(SMnode *pMnode, SRpcMsg *pReq, SDbObj *pDb, SSmaObj *p mDebug("trans:%d, used to drop sma:%s", pTrans->id, pSma->name); mndTransSetDbName(pTrans, pDb->name, NULL); + SStreamObj *pStream = mndAcquireStream(pMnode, pSma->name); + if (pStream == NULL || pStream->smaId != pSma->uid) { + sdbRelease(pMnode->pSdb, pStream); + goto _OVER; + } else { + if (mndDropStreamTasks(pMnode, pTrans, pStream) < 0) { + mError("stream:%s, failed to drop task since %s", pStream->name, terrstr()); + sdbRelease(pMnode->pSdb, pStream); + goto _OVER; + } + + // drop stream + if (mndPersistDropStreamLog(pMnode, pTrans, pStream) < 0) { + sdbRelease(pMnode->pSdb, pStream); + goto _OVER; + } + } if (mndSetDropSmaRedoLogs(pMnode, pTrans, pSma) != 0) goto _OVER; if (mndSetDropSmaVgroupRedoLogs(pMnode, pTrans, pVgroup) != 0) goto _OVER; if (mndSetDropSmaCommitLogs(pMnode, pTrans, pSma) != 0) goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index b78756d8b8..cabbac14f1 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -14,10 +14,10 @@ */ #include "mndStream.h" -#include "mndPrivilege.h" #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" +#include "mndPrivilege.h" #include "mndScheduler.h" #include "mndShow.h" #include "mndStb.h" @@ -490,7 +490,7 @@ static int32_t mndPersistTaskDropReq(STrans *pTrans, SStreamTask *pTask) { return 0; } -static int32_t mndDropStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) { +int32_t mndDropStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStream) { int32_t lv = taosArrayGetSize(pStream->tasks); for (int32_t i = 0; i < lv; i++) { SArray *pTasks = taosArrayGetP(pStream->tasks, i); diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 9abc2f639b..0bb9918488 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -43,7 +43,7 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo taosArrayPush(tagArray, &tagVal); tTagNew(tagArray, 1, false, &pTag); if (pTag == NULL) { - taosArrayDestroy(schemaReqs); + terrno = TSDB_CODE_OUT_OF_MEMORY; taosArrayDestroy(tagArray); return NULL; } diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 682afbb785..2de0fea9ac 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -104,6 +104,8 @@ static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) { return -1; } + pRead->pReadLogTFile = pLogTFile; + walBuildIdxName(pRead->pWal, fileFirstVer, fnameStr); TdFilePtr pIdxTFile = taosOpenFile(fnameStr, TD_FILE_READ); if (pIdxTFile == NULL) { @@ -112,7 +114,6 @@ static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) { return -1; } - pRead->pReadLogTFile = pLogTFile; pRead->pReadIdxTFile = pIdxTFile; return 0; } From 565ee08895f427c4701f8f6caedc86d904e979ef Mon Sep 17 00:00:00 2001 From: plum-lihui Date: Sat, 25 Jun 2022 18:32:56 +0800 Subject: [PATCH 63/72] test: add tmq test case --- tests/system-test/99-TDcase/TD-16821.py | 186 ++++++++++++++++++++++++ 1 file changed, 186 insertions(+) create mode 100644 tests/system-test/99-TDcase/TD-16821.py diff --git a/tests/system-test/99-TDcase/TD-16821.py b/tests/system-test/99-TDcase/TD-16821.py new file mode 100644 index 0000000000..ef74515792 --- /dev/null +++ b/tests/system-test/99-TDcase/TD-16821.py @@ -0,0 +1,186 @@ + +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + #tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def checkFileContent(self, consumerId, queryString): + buildPath = tdCom.getBuildPath() + cfgPath = tdCom.getClientCfgPath() + dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId) + cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile) + tdLog.info(cmdStr) + os.system(cmdStr) + + consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId) + tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile)) + + consumeFile = open(consumeRowsFile, mode='r') + queryFile = open(dstFile, mode='r') + + # skip first line for it is schema + queryFile.readline() + + while True: + dst = queryFile.readline() + src = consumeFile.readline() + + if dst: + if dst != src: + tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId) + else: + break + return + + def tmqCase1(self): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'db1', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbNum': 1, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 10, + 'showMsg': 1, + 'showRow': 1} + + topicNameList = ['topic1', 'topic2', 'topic3'] + expectRowsList = [] + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) + tdLog.info("create stb") + tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) + tdLog.info("create ctb") + tdCom.create_ctable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix']) + tdLog.info("insert data") + tmqCom.insert_data(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"]) + + tdLog.info("create topics from stb with filter") + # queryString = "select ts, log(c1), ceil(pow(c1,3)) from %s.%s where c1 %% 7 == 0" %(paraDict['dbName'], paraDict['stbName']) + # queryString = "select ts, c1, c2 from %s.%s" %(paraDict['dbName'], paraDict['stbName']) + queryString = "select * from %s.%s" %(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as stable %s.%s" %(topicNameList[0], paraDict["dbName"],paraDict["stbName"]) + # sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + # queryString = 'select * from %s.%s'%(paraDict["dbName"],paraDict["stbName"]) + tdSql.query(queryString) + expectRowsList.append(tdSql.getRows()) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:false, auto.commit.interval.ms:6000, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow']) + + tdLog.info("wait the consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + if expectRowsList[0] != resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) + tdLog.exit("0 tmq consume rows error!") + + self.checkFileContent(consumerId, queryString) + + # reinit consume info, and start tmq_sim, then check consume result + tmqCom.initConsumerTable() + + queryString = "select ts, log(c1), cos(c1) from %s.%s where c1 > 3169" %(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[1], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + tdSql.query(queryString) + expectRowsList.append(tdSql.getRows()) + + consumerId = 1 + topicList = topicNameList[1] + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow']) + + tdLog.info("wait the consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + if expectRowsList[1] != resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[1], resultList[0])) + tdLog.exit("1 tmq consume rows error!") + + self.checkFileContent(consumerId, queryString) + + # reinit consume info, and start tmq_sim, then check consume result + tmqCom.initConsumerTable() + + queryString = "select ts, log(c1), atan(c1) from %s.%s where ts >= %d" %(paraDict['dbName'], paraDict['stbName'], paraDict["startTs"]+6137) + sqlString = "create topic %s as %s" %(topicNameList[2], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + tdSql.query(queryString) + expectRowsList.append(tdSql.getRows()) + + consumerId = 2 + topicList = topicNameList[2] + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow']) + + tdLog.info("wait the consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + # if expectRowsList[2] != resultList[0]: + # tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[2], resultList[0])) + # tdLog.exit("2 tmq consume rows error!") + + # self.checkFileContent(consumerId, queryString) + + time.sleep(10) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def run(self): + tdSql.prepare() + self.tmqCase1() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) From e8563be0501d22e00d39aca58fda4168a1da2679 Mon Sep 17 00:00:00 2001 From: haoranchen Date: Sat, 25 Jun 2022 18:44:46 +0800 Subject: [PATCH 64/72] Update 5dnode3mnodeStop.py --- tests/system-test/6-cluster/5dnode3mnodeStop.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/system-test/6-cluster/5dnode3mnodeStop.py b/tests/system-test/6-cluster/5dnode3mnodeStop.py index c1d5a99af2..5311d29846 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeStop.py +++ b/tests/system-test/6-cluster/5dnode3mnodeStop.py @@ -253,11 +253,7 @@ class TDTestCase: # fisr add three mnodes; tdLog.info("fisr add three mnodes and check mnode status") tdSql.execute("create mnode on dnode 2") -<<<<<<< HEAD clusterComCheck.checkMnodeStatus(2) -======= - time.sleep(10) ->>>>>>> e571567ec2ae0c7668ccf1b8aaaf949fc84bb44c tdSql.execute("create mnode on dnode 3") clusterComCheck.checkMnodeStatus(3) @@ -307,4 +303,4 @@ class TDTestCase: tdLog.success(f"{__file__} successfully executed") tdCases.addLinux(__file__, TDTestCase()) -tdCases.addWindows(__file__, TDTestCase()) \ No newline at end of file +tdCases.addWindows(__file__, TDTestCase()) From 1bc2f81b2092d548a9a1fbeca9cc2893775a8e10 Mon Sep 17 00:00:00 2001 From: slzhou Date: Sat, 25 Jun 2022 18:46:54 +0800 Subject: [PATCH 65/72] feat: add set op elimination --- source/libs/planner/src/planOptimizer.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 34ab7b72b7..f1cad6010c 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1215,7 +1215,8 @@ static int32_t eliminateSetOpOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* FOREACH(pChild, pSetOpNode->pChildren) { ((SLogicNode*)pChild)->pParent = pSetOpNode->pParent; } - nodesListInsertList(pSetOpNode->pParent->pChildren, cell, pSetOpNode->pChildren); + INSERT_LIST(pSetOpNode->pParent->pChildren, pSetOpNode->pChildren); + pSetOpNode->pChildren = NULL; ERASE_NODE(pSetOpNode->pParent->pChildren); return TSDB_CODE_SUCCESS; From 1975c981dbf0675f05a754e6f140c6e3143b0852 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Sat, 25 Jun 2022 18:47:18 +0800 Subject: [PATCH 66/72] test: del test tkinter --- tests/system-test/test.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/system-test/test.py b/tests/system-test/test.py index 35f8ea953c..76b83da348 100644 --- a/tests/system-test/test.py +++ b/tests/system-test/test.py @@ -22,9 +22,6 @@ import json import platform import socket import threading -from distutils.log import warn as printf -from tkinter import N -from fabric2 import Connection sys.path.append("../pytest") from util.log import * from util.dnodes import * From 86f82af8214d1650262bccab4ada4161ced7fbca Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 25 Jun 2022 19:21:35 +0800 Subject: [PATCH 67/72] fix: check privilege while show dnode variables --- source/dnode/mgmt/mgmt_dnode/src/dmHandle.c | 17 ++++++------- .../tsim/stream/distributeInterval0.sim | 24 +++++++++++++++++++ 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 363f438b80..980749ca70 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -247,16 +247,9 @@ int32_t dmAppendVariablesToBlock(SSDataBlock* pBlock, int32_t dnodeId) { return TSDB_CODE_SUCCESS; } - - int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { - int32_t size = 0; - int32_t rowsRead = 0; - - if (strcmp(pMsg->info.conn.user, TSDB_DEFAULT_USER) != 0) { - terrno = TSDB_CODE_MND_NO_RIGHTS; - return -1; - } + int32_t size = 0; + int32_t rowsRead = 0; SRetrieveTableReq retrieveReq = {0}; if (tDeserializeSRetrieveTableReq(pMsg->pCont, pMsg->contLen, &retrieveReq) != 0) { @@ -264,6 +257,11 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { return -1; } + if (strcmp(retrieveReq.user, TSDB_DEFAULT_USER) != 0) { + terrno = TSDB_CODE_MND_NO_RIGHTS; + return -1; + } + if (strcasecmp(retrieveReq.tb, TSDB_INS_TABLE_DNODE_VARIABLES)) { terrno = TSDB_CODE_INVALID_MSG; return -1; @@ -314,7 +312,6 @@ int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { return TSDB_CODE_SUCCESS; } - SArray *dmGetMsgHandles() { int32_t code = -1; SArray *pArray = taosArrayInit(16, sizeof(SMgmtHandle)); diff --git a/tests/script/tsim/stream/distributeInterval0.sim b/tests/script/tsim/stream/distributeInterval0.sim index 3e38df2c89..8936655974 100644 --- a/tests/script/tsim/stream/distributeInterval0.sim +++ b/tests/script/tsim/stream/distributeInterval0.sim @@ -10,6 +10,30 @@ sql create dnode $hostname2 port 7200 system sh/exec.sh -n dnode2 -s start +print ===== step1 +$x = 0 +step1: + $x = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not ready! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +if $rows != 2 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi +if $data(2)[4] != ready then + goto step1 +endi + +print ===== step2 + sql create database test vgroups 4; sql use test; sql create stable st(ts timestamp, a int, b int , c int, d double) tags(ta int,tb int,tc int); From 0c1a51bad63e88cd2f6f84dd08ce0295dd8fc88f Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Sat, 25 Jun 2022 20:02:57 +0800 Subject: [PATCH 68/72] enh(stream): refine tqRetrieveDataBlock api --- source/client/src/clientImpl.c | 26 ++++---- source/client/src/clientMain.c | 31 ++++------ source/common/src/tdatablock.c | 9 +-- source/dnode/vnode/inc/vnode.h | 27 ++++----- source/dnode/vnode/src/tq/tqExec.c | 4 +- source/dnode/vnode/src/tq/tqRead.c | 16 ++--- source/libs/executor/src/scanoperator.c | 81 +++++++++++++------------ 7 files changed, 93 insertions(+), 101 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index ac9daa5119..489966b636 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -617,12 +617,12 @@ int32_t scheduleAsyncQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNod .requestId = pRequest->requestId, .requestObjRefId = pRequest->self}; SSchedulerReq req = {.pConn = &conn, - .pNodeList = pNodeList, - .pDag = pDag, - .sql = pRequest->sqlstr, - .startTs = pRequest->metric.start, - .fp = schdExecCallback, - .cbParam = &res}; + .pNodeList = pNodeList, + .pDag = pDag, + .sql = pRequest->sqlstr, + .startTs = pRequest->metric.start, + .fp = schdExecCallback, + .cbParam = &res}; int32_t code = schedulerAsyncExecJob(&req, &pRequest->body.queryJob); @@ -669,13 +669,13 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList .requestId = pRequest->requestId, .requestObjRefId = pRequest->self}; SSchedulerReq req = {.pConn = &conn, - .pNodeList = pNodeList, - .pDag = pDag, - .sql = pRequest->sqlstr, - .startTs = pRequest->metric.start, - .fp = NULL, - .cbParam = NULL, - .reqKilled = &pRequest->killed}; + .pNodeList = pNodeList, + .pDag = pDag, + .sql = pRequest->sqlstr, + .startTs = pRequest->metric.start, + .fp = NULL, + .cbParam = NULL, + .reqKilled = &pRequest->killed}; int32_t code = schedulerExecJob(&req, &pRequest->body.queryJob, &res); pRequest->body.resInfo.execRes = res.res; diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index f5dfe2de36..bbd477fa3b 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -199,10 +199,7 @@ TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) { return pResInfo->userFields; } - -TAOS_RES *taos_query(TAOS *taos, const char *sql) { - return taosQueryImpl(taos, sql, false); -} +TAOS_RES *taos_query(TAOS *taos, const char *sql) { return taosQueryImpl(taos, sql, false); } TAOS_ROW taos_fetch_row(TAOS_RES *res) { if (res == NULL) { @@ -593,11 +590,11 @@ int *taos_get_column_data_offset(TAOS_RES *res, int columnIndex) { return pResInfo->pCol[columnIndex].offset; } -int taos_validate_sql(TAOS *taos, const char *sql) { - TAOS_RES* pObj = taosQueryImpl(taos, sql, true); +int taos_validate_sql(TAOS *taos, const char *sql) { + TAOS_RES *pObj = taosQueryImpl(taos, sql, true); int code = taos_errno(pObj); - + taos_free_result(pObj); return code; } @@ -884,10 +881,10 @@ void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress) { int taos_load_table_info(TAOS *taos, const char *tableNameList) { const int32_t MAX_TABLE_NAME_LENGTH = 12 * 1024 * 1024; // 12MB list - int32_t code = 0; - SRequestObj *pRequest = NULL; - SCatalogReq catalogReq = {0}; - + int32_t code = 0; + SRequestObj *pRequest = NULL; + SCatalogReq catalogReq = {0}; + if (NULL == tableNameList) { return TSDB_CODE_SUCCESS; } @@ -911,26 +908,25 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) { goto _return; } - SCatalog* pCtg = NULL; + SCatalog *pCtg = NULL; code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCtg); if (code != TSDB_CODE_SUCCESS) { goto _return; } - char* sql = "taos_load_table_info"; + char *sql = "taos_load_table_info"; code = buildRequest(pTscObj, sql, strlen(sql), &pRequest); if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } - + SSyncQueryParam param = {0}; tsem_init(¶m.sem, 0, 0); param.pRequest = pRequest; - SRequestConnInfo conn = {.pTrans = pTscObj->pAppInfo->pTransporter, - .requestId = pRequest->requestId, - .requestObjRefId = pRequest->self}; + SRequestConnInfo conn = { + .pTrans = pTscObj->pAppInfo->pTransporter, .requestId = pRequest->requestId, .requestObjRefId = pRequest->self}; conn.mgmtEps = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); @@ -951,7 +947,6 @@ _return: return code; } - TAOS_STMT *taos_stmt_init(TAOS *taos) { STscObj *pObj = acquireTscObj(*(int64_t *)taos); if (NULL == pObj) { diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index cc995c4d64..9f89d72172 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1164,7 +1164,7 @@ int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows) int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) { int32_t code = 0; - //ASSERT(numOfRows > 0); + // ASSERT(numOfRows > 0); if (numOfRows == 0) { return TSDB_CODE_SUCCESS; @@ -1657,12 +1657,13 @@ void blockDebugShowData(const SArray* dataBlocks, const char* flag) { char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) { int32_t size = 2048; *pDataBuf = taosMemoryCalloc(size, 1); - char* dumpBuf = *pDataBuf; - char pBuf[128] = {0}; + char* dumpBuf = *pDataBuf; + char pBuf[128] = {0}; int32_t colNum = taosArrayGetSize(pDataBlock->pDataBlock); int32_t rows = pDataBlock->info.rows; int32_t len = 0; - len += snprintf(dumpBuf + len, size - len, "\n%s |block type %d |child id %d|\n", flag, (int32_t)pDataBlock->info.type, pDataBlock->info.childId); + len += snprintf(dumpBuf + len, size - len, "\n%s |block type %d |child id %d|\n", flag, + (int32_t)pDataBlock->info.type, pDataBlock->info.childId); for (int32_t j = 0; j < rows; j++) { len += snprintf(dumpBuf + len, size - len, "%s |", flag); for (int32_t k = 0; k < colNum; k++) { diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index b97d4605e7..858376519d 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -116,18 +116,18 @@ typedef void *tsdbReaderT; #define BLOCK_LOAD_TABLE_SEQ_ORDER 2 #define BLOCK_LOAD_TABLE_RR_ORDER 3 -int32_t tsdbSetTableList(tsdbReaderT reader, SArray* tableList); -tsdbReaderT tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, SArray *tableList, uint64_t qId, - uint64_t taskId); -tsdbReaderT tsdbQueryCacheLast(SVnode *pVnode, SQueryTableDataCond *pCond, STableListInfo *groupList, uint64_t qId, - void *pMemRef); -int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT *pReader, STableBlockDistInfo *pTableBlockInfo); -bool isTsdbCacheLastRow(tsdbReaderT *pReader); -int32_t tsdbGetAllTableList(SMeta *pMeta, uint64_t uid, SArray *list); -int32_t tsdbGetCtbIdList(SMeta *pMeta, int64_t suid, SArray *list); -void *tsdbGetIdx(SMeta *pMeta); -void *tsdbGetIvtIdx(SMeta *pMeta); -int64_t tsdbGetNumOfRowsInMemTable(tsdbReaderT *pHandle); +int32_t tsdbSetTableList(tsdbReaderT reader, SArray *tableList); +tsdbReaderT tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, SArray *tableList, uint64_t qId, + uint64_t taskId); +tsdbReaderT tsdbQueryCacheLast(SVnode *pVnode, SQueryTableDataCond *pCond, STableListInfo *groupList, uint64_t qId, + void *pMemRef); +int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT *pReader, STableBlockDistInfo *pTableBlockInfo); +bool isTsdbCacheLastRow(tsdbReaderT *pReader); +int32_t tsdbGetAllTableList(SMeta *pMeta, uint64_t uid, SArray *list); +int32_t tsdbGetCtbIdList(SMeta *pMeta, int64_t suid, SArray *list); +void *tsdbGetIdx(SMeta *pMeta); +void *tsdbGetIvtIdx(SMeta *pMeta); +int64_t tsdbGetNumOfRowsInMemTable(tsdbReaderT *pHandle); bool tsdbNextDataBlock(tsdbReaderT pTsdbReadHandle); void tsdbRetrieveDataBlockInfo(tsdbReaderT *pTsdbReadHandle, SDataBlockInfo *pBlockInfo); @@ -150,8 +150,7 @@ int32_t tqReadHandleRemoveTbUidList(STqReadHandle *pHandle, const SArray *tbUidL int32_t tqReadHandleSetMsg(STqReadHandle *pHandle, SSubmitReq *pMsg, int64_t ver); bool tqNextDataBlock(STqReadHandle *pHandle); bool tqNextDataBlockFilterOut(STqReadHandle *pHandle, SHashObj *filterOutUids); -int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle *pHandle, uint64_t *pGroupId, uint64_t *pUid, - int32_t *pNumOfRows); +int32_t tqRetrieveDataBlock(SSDataBlock *pBlock, STqReadHandle *pHandle); // sma int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days); diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index be8fef1249..afeeeab500 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -112,7 +112,7 @@ int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkR tqReadHandleSetMsg(pReader, pReq, 0); while (tqNextDataBlock(pReader)) { SSDataBlock block = {0}; - if (tqRetrieveDataBlock(&block, pReader, &block.info.groupId, &block.info.uid, &block.info.rows) < 0) { + if (tqRetrieveDataBlock(&block, pReader) < 0) { if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue; ASSERT(0); } @@ -129,7 +129,7 @@ int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkR tqReadHandleSetMsg(pReader, pReq, 0); while (tqNextDataBlockFilterOut(pReader, pExec->execDb.pFilterOutTbUid)) { SSDataBlock block = {0}; - if (tqRetrieveDataBlock(&block, pReader, &block.info.groupId, &block.info.uid, &block.info.rows) < 0) { + if (tqRetrieveDataBlock(&block, pReader) < 0) { if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue; ASSERT(0); } diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 96f4eb3fd9..cbee639911 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -146,10 +146,7 @@ bool tqNextDataBlockFilterOut(STqReadHandle* pHandle, SHashObj* filterOutUids) { return false; } -int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle* pHandle, uint64_t* pGroupId, uint64_t* pUid, - int32_t* pNumOfRows) { - *pUid = 0; - +int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle* pHandle) { // TODO: cache multiple schema int32_t sversion = htonl(pHandle->pBlock->sversion); if (pHandle->cachedSchemaSuid == 0 || pHandle->cachedSchemaVer != sversion || @@ -180,7 +177,6 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle* pHandle, uint64_ STSchema* pTschema = pHandle->pSchema; SSchemaWrapper* pSchemaWrapper = pHandle->pSchemaWrapper; - *pNumOfRows = pHandle->msgIter.numOfRows; int32_t colNumNeed = taosArrayGetSize(pHandle->pColIdList); if (colNumNeed == 0) { @@ -221,22 +217,22 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle* pHandle, uint64_ } } - if (blockDataEnsureCapacity(pBlock, *pNumOfRows) < 0) { + if (blockDataEnsureCapacity(pBlock, pHandle->msgIter.numOfRows) < 0) { goto FAIL; } int32_t colActual = blockDataGetNumOfCols(pBlock); - // TODO in stream shuffle case, fetch groupId - *pGroupId = 0; - STSRowIter iter = {0}; tdSTSRowIterInit(&iter, pTschema); STSRow* row; int32_t curRow = 0; tInitSubmitBlkIter(&pHandle->msgIter, pHandle->pBlock, &pHandle->blkIter); - *pUid = pHandle->msgIter.uid; // set the uid of table for submit block + + pBlock->info.groupId = 0; + pBlock->info.uid = pHandle->msgIter.uid; // set the uid of table for submit block + pBlock->info.rows = pHandle->msgIter.numOfRows; while ((row = tGetSubmitBlkNext(&pHandle->blkIter)) != NULL) { tdSTSRowIterReset(&iter, row); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 15803a4b73..ac57d3a9da 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -507,20 +507,21 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { STableScanInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - if(pInfo->currentGroupId == -1){ + if (pInfo->currentGroupId == -1) { pInfo->currentGroupId++; if (pInfo->currentGroupId >= taosArrayGetSize(pTaskInfo->tableqinfoList.pGroupList)) { setTaskStatus(pTaskInfo, TASK_COMPLETED); return NULL; } - SArray *tableList = taosArrayGetP(pTaskInfo->tableqinfoList.pGroupList, pInfo->currentGroupId); + SArray* tableList = taosArrayGetP(pTaskInfo->tableqinfoList.pGroupList, pInfo->currentGroupId); tsdbCleanupReadHandle(pInfo->dataReader); - tsdbReaderT* pReader = tsdbReaderOpen(pInfo->readHandle.vnode, &pInfo->cond, tableList, pInfo->queryId, pInfo->taskId); + tsdbReaderT* pReader = + tsdbReaderOpen(pInfo->readHandle.vnode, &pInfo->cond, tableList, pInfo->queryId, pInfo->taskId); pInfo->dataReader = pReader; } SSDataBlock* result = doTableScanGroup(pOperator); - if(result){ + if (result) { return result; } @@ -530,7 +531,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { return NULL; } - SArray *tableList = taosArrayGetP(pTaskInfo->tableqinfoList.pGroupList, pInfo->currentGroupId); + SArray* tableList = taosArrayGetP(pTaskInfo->tableqinfoList.pGroupList, pInfo->currentGroupId); tsdbSetTableList(pInfo->dataReader, tableList); tsdbResetReadHandle(pInfo->dataReader, &pInfo->cond, 0); @@ -538,7 +539,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { pInfo->scanTimes = 0; result = doTableScanGroup(pOperator); - if(result){ + if (result) { return result; } @@ -777,9 +778,9 @@ static bool isStateWindow(SStreamBlockScanInfo* pInfo) { } static bool prepareDataScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pSDB, int32_t tsColIndex, int32_t* pRowIndex) { - STimeWindow win = { - .skey = INT64_MIN, - .ekey = INT64_MAX, + STimeWindow win = { + .skey = INT64_MIN, + .ekey = INT64_MAX, }; bool needRead = false; if (!isStateWindow(pInfo) && (*pRowIndex) < pSDB->info.rows) { @@ -794,13 +795,12 @@ static bool prepareDataScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pSDB, int3 SResultWindowInfo* pCurWin = getSessionTimeWindow(pAggSup, tsCols[(*pRowIndex)], INT64_MIN, pSDB->info.groupId, gap, &winIndex); win = pCurWin->win; - (*pRowIndex) += - updateSessionWindowInfo(pCurWin, tsCols, NULL, pSDB->info.rows, (*pRowIndex), gap, NULL); + (*pRowIndex) += updateSessionWindowInfo(pCurWin, tsCols, NULL, pSDB->info.rows, (*pRowIndex), gap, NULL); } else { - win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[(*pRowIndex)], &pInfo->interval, - pInfo->interval.precision, NULL); - (*pRowIndex) += getNumOfRowsInTimeWindow(&pSDB->info, tsCols, (*pRowIndex), win.ekey, - binarySearchForKey, NULL, TSDB_ORDER_ASC); + win = + getActiveTimeWindow(NULL, &dumyInfo, tsCols[(*pRowIndex)], &pInfo->interval, pInfo->interval.precision, NULL); + (*pRowIndex) += getNumOfRowsInTimeWindow(&pSDB->info, tsCols, (*pRowIndex), win.ekey, binarySearchForKey, NULL, + TSDB_ORDER_ASC); } needRead = true; } else if (isStateWindow(pInfo)) { @@ -821,7 +821,7 @@ static bool prepareDataScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pSDB, int3 STableScanInfo* pTableScanInfo = pInfo->pSnapshotReadOp->info; pTableScanInfo->cond.twindows[0] = win; pTableScanInfo->curTWinIdx = 0; -// tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0); + // tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0); // if (!pTableScanInfo->dataReader) { // return false; // } @@ -1033,12 +1033,13 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { while (tqNextDataBlock(pInfo->streamBlockReader)) { SSDataBlock block = {0}; - uint64_t groupId = 0; - uint64_t uid = 0; - int32_t numOfRows = 0; // todo refactor - int32_t code = tqRetrieveDataBlock(&block, pInfo->streamBlockReader, &groupId, &uid, &numOfRows); + int32_t code = tqRetrieveDataBlock(&block, pInfo->streamBlockReader); + + uint64_t groupId = block.info.groupId; + uint64_t uid = block.info.uid; + int32_t numOfRows = block.info.rows; if (code != TSDB_CODE_SUCCESS || numOfRows == 0) { pTaskInfo->code = code; @@ -1154,9 +1155,9 @@ static SArray* extractTableIdList(const STableListInfo* pTableGroupInfo) { return tableIdList; } -SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, - STableScanPhysiNode* pTableScanNode, SExecTaskInfo* pTaskInfo, - STimeWindowAggSupp* pTwSup, uint64_t queryId, uint64_t taskId) { +SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, + SExecTaskInfo* pTaskInfo, STimeWindowAggSupp* pTwSup, uint64_t queryId, + uint64_t taskId) { SStreamBlockScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamBlockScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); @@ -1743,14 +1744,14 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan SSDataBlock* pResBlock = createResDataBlock(pDescNode); int32_t num = 0; - SArray* colList = extractColMatchInfo(pScanNode->pScanCols, pDescNode, &num, COL_MATCH_FROM_COL_ID); + SArray* colList = extractColMatchInfo(pScanNode->pScanCols, pDescNode, &num, COL_MATCH_FROM_COL_ID); - pInfo->accountId = pScanPhyNode->accountId; - pInfo->pUser = taosMemoryStrDup((void*) pUser); + pInfo->accountId = pScanPhyNode->accountId; + pInfo->pUser = taosMemoryStrDup((void*)pUser); pInfo->showRewrite = pScanPhyNode->showRewrite; - pInfo->pRes = pResBlock; - pInfo->pCondition = pScanNode->node.pConditions; - pInfo->scanCols = colList; + pInfo->pRes = pResBlock; + pInfo->pCondition = pScanNode->node.pConditions; + pInfo->scanCols = colList; initResultSizeInfo(pOperator, 4096); @@ -1766,13 +1767,13 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan pInfo->readHandle = *(SReadHandle*)readHandle; } - pOperator->name = "SysTableScanOperator"; + pOperator->name = "SysTableScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; pOperator->exprSupp.numOfExprs = taosArrayGetSize(pResBlock->pDataBlock); - pOperator->pTaskInfo = pTaskInfo; + pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSysTableScan, NULL, NULL, destroySysScanOperator, NULL, NULL, NULL); @@ -1959,11 +1960,11 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi goto _error; } - pInfo->pTableList = pTableListInfo; - pInfo->pColMatchInfo = colList; - pInfo->pRes = createResDataBlock(pDescNode); - pInfo->readHandle = *pReadHandle; - pInfo->curPos = 0; + pInfo->pTableList = pTableListInfo; + pInfo->pColMatchInfo = colList; + pInfo->pRes = createResDataBlock(pDescNode); + pInfo->readHandle = *pReadHandle; + pInfo->curPos = 0; pOperator->name = "TagScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN; @@ -2034,7 +2035,7 @@ typedef struct STableMergeScanInfo { int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan int32_t dataBlockLoadFlag; SInterval interval; // if the upstream is an interval operator, the interval info is also kept here to get the time - // window to check if current data block needs to be loaded. + // window to check if current data block needs to be loaded. SSampleExecInfo sample; // sample execution info } STableMergeScanInfo; From 225f283738b6fd8d8c2f7499c85755a1e3c8bdcf Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 25 Jun 2022 20:05:27 +0800 Subject: [PATCH 69/72] fix: set user name to root in unitest --- source/dnode/mnode/impl/src/mndShow.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 671adba046..029a1e6f8c 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -231,10 +231,14 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { } mDebug("show:0x%" PRIx64 ", start retrieve data, type:%d", pShow->id, pShow->type); - if (mndCheckShowPrivilege(pMnode, retrieveReq.user, pShow->type, retrieveReq.db) != 0) { + if (retrieveReq.user[0] != 0) { + memcpy(pReq->info.conn.user, retrieveReq.user, TSDB_USER_LEN); + } else { + memcpy(pReq->info.conn.user, TSDB_DEFAULT_USER, TSDB_USER_LEN); + } + if (mndCheckShowPrivilege(pMnode, pReq->info.conn.user, pShow->type, retrieveReq.db) != 0) { return -1; } - memcpy(pReq->info.conn.user, retrieveReq.user, TSDB_USER_LEN); int32_t numOfCols = pShow->pMeta->numOfColumns; SSDataBlock *pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); From 0597f3b9cdbb98fb087315fb9d8d111f510afc60 Mon Sep 17 00:00:00 2001 From: Cary Xu Date: Sun, 26 Jun 2022 18:44:49 +0800 Subject: [PATCH 70/72] refactor: rsma resource release --- source/common/src/tmsg.c | 10 +- source/dnode/vnode/inc/vnode.h | 1 + source/dnode/vnode/src/inc/meta.h | 1 + source/dnode/vnode/src/inc/sma.h | 30 +- source/dnode/vnode/src/inc/vnodeInt.h | 6 + source/dnode/vnode/src/meta/metaOpen.c | 9 + source/dnode/vnode/src/meta/metaQuery.c | 64 ++++ source/dnode/vnode/src/meta/metaTable.c | 12 + source/dnode/vnode/src/sma/smaEnv.c | 139 +++++---- source/dnode/vnode/src/sma/smaOpen.c | 18 +- source/dnode/vnode/src/sma/smaRollup.c | 371 ++++++++++++++++------- source/dnode/vnode/src/sma/smaUtil.c | 11 +- source/dnode/vnode/src/tsdb/tsdbRead.c | 24 ++ source/dnode/vnode/src/vnd/vnodeCommit.c | 7 +- source/dnode/vnode/src/vnd/vnodeOpen.c | 6 +- source/libs/executor/inc/executorimpl.h | 1 + source/libs/executor/src/executorimpl.c | 10 +- source/os/src/osFile.c | 4 +- 18 files changed, 526 insertions(+), 198 deletions(-) diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 9db5019512..8a052026f2 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -3696,7 +3696,7 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq * if (tDecodeI8(&decoder, &pReq->isTsma) < 0) return -1; if (pReq->isTsma) { - if (tDecodeBinaryAlloc(&decoder, &pReq->pTsma, NULL) < 0) return -1; + if (tDecodeBinary(&decoder, (uint8_t **)&pReq->pTsma, NULL) < 0) return -1; } tEndDecode(&decoder); @@ -3707,9 +3707,6 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq * int32_t tFreeSCreateVnodeReq(SCreateVnodeReq *pReq) { taosArrayDestroy(pReq->pRetensions); pReq->pRetensions = NULL; - if (pReq->isTsma) { - taosMemoryFreeClear(pReq->pTsma); - } return 0; } @@ -4747,9 +4744,8 @@ int32_t tDecodeSRSmaParam(SDecoder *pCoder, SRSmaParam *pRSmaParam) { if (tDecodeI64v(pCoder, &pRSmaParam->watermark[i]) < 0) return -1; if (tDecodeI32v(pCoder, &pRSmaParam->qmsgLen[i]) < 0) return -1; if (pRSmaParam->qmsgLen[i] > 0) { - uint64_t len; - if (tDecodeBinaryAlloc(pCoder, (void **)&pRSmaParam->qmsg[i], &len) < 0) - return -1; // qmsgLen contains len of '\0' + tDecoderMalloc(pCoder, pRSmaParam->qmsgLen[i]); + if (tDecodeBinary(pCoder, (uint8_t **)&pRSmaParam->qmsg[i], NULL) < 0) return -1; // qmsgLen contains len of '\0' } else { pRSmaParam->qmsg[i] = NULL; } diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 858376519d..c49b33beb2 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -125,6 +125,7 @@ int32_t tsdbGetFileBlocksDistInfo(tsdbReaderT *pReader, STableBlockDistInfo bool isTsdbCacheLastRow(tsdbReaderT *pReader); int32_t tsdbGetAllTableList(SMeta *pMeta, uint64_t uid, SArray *list); int32_t tsdbGetCtbIdList(SMeta *pMeta, int64_t suid, SArray *list); +int32_t tsdbGetStbIdList(SMeta *pMeta, int64_t suid, SArray *list); void *tsdbGetIdx(SMeta *pMeta); void *tsdbGetIvtIdx(SMeta *pMeta); int64_t tsdbGetNumOfRowsInMemTable(tsdbReaderT *pHandle); diff --git a/source/dnode/vnode/src/inc/meta.h b/source/dnode/vnode/src/inc/meta.h index 63a8e7ec68..66d1689d57 100644 --- a/source/dnode/vnode/src/inc/meta.h +++ b/source/dnode/vnode/src/inc/meta.h @@ -69,6 +69,7 @@ struct SMeta { TTB* pUidIdx; TTB* pNameIdx; TTB* pCtbIdx; + TTB* pSuidIdx; // ivt idx and idx void* pTagIvtIdx; TTB* pTagIdx; diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index 1489b02492..956e451b58 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -60,8 +60,9 @@ struct SRSmaStat { SSma *pSma; void *tmrHandle; tmr_h tmrId; - int8_t tmrStat; int32_t tmrSeconds; + int8_t triggerStat; + int8_t runningStat; SHashObj *rsmaInfoHash; // key: stbUid, value: SRSmaInfo; }; @@ -72,13 +73,21 @@ struct SSmaStat { }; T_REF_DECLARE() }; -#define SMA_TSMA_STAT(s) (&(s)->tsmaStat) -#define SMA_RSMA_STAT(s) (&(s)->rsmaStat) -#define SMA_RSMA_INFO_HASH(s) ((s)->rsmaStat.rsmaInfoHash) -#define SMA_RSMA_TMR_HANDLE(s) ((s)->rsmaStat.tmrHandle) -#define SMA_RSMA_TMR_STAT(s) ((s)->rsmaStat.tmrStat) -#define RSMA_INFO_HASH(r) ((r)->rsmaInfoHash) +#define SMA_TSMA_STAT(s) (&(s)->tsmaStat) +#define SMA_RSMA_STAT(s) (&(s)->rsmaStat) +#define RSMA_INFO_HASH(r) ((r)->rsmaInfoHash) +#define RSMA_TMR_ID(r) ((r)->tmrId) +#define RSMA_TMR_HANDLE(r) ((r)->tmrHandle) +#define RSMA_TRIGGER_STAT(r) (&(r)->triggerStat) +#define RSMA_RUNNING_STAT(r) (&(r)->runningStat) +enum { + TASK_TRIGGER_STAT_INIT = 0, + TASK_TRIGGER_STAT_ACTIVE = 1, + TASK_TRIGGER_STAT_INACTIVE = 2, + TASK_TRIGGER_STAT_CANCELLED = 3, + TASK_TRIGGER_STAT_FINISHED = 4, +}; void tdDestroySmaEnv(SSmaEnv *pSmaEnv); void *tdFreeSmaEnv(SSmaEnv *pSmaEnv); @@ -174,6 +183,8 @@ int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *pMsg); int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg); int32_t tdProcessTSmaGetDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days); +// smaFileUtil ================ + typedef struct STFInfo STFInfo; typedef struct STFile STFile; @@ -181,7 +192,7 @@ struct STFInfo { uint32_t magic; uint32_t ftype; uint32_t fver; - uint64_t fsize; + int64_t fsize; }; struct STFile { @@ -199,11 +210,8 @@ struct STFile { #define TD_FILE_OPENED(tf) (TD_FILE_PFILE(tf) != NULL) #define TD_FILE_CLOSED(tf) (!TD_FILE_OPENED(tf)) #define TD_FILE_SET_CLOSED(f) (TD_FILE_PFILE(f) = NULL) -#define TD_FILE_STATE(tf) ((tf)->state) #define TD_FILE_SET_STATE(tf, s) ((tf)->state = (s)) #define TD_FILE_DID(tf) (TD_FILE_F(tf)->did) -#define TD_FILE_IS_OK(tf) (TD_FILE_STATE(tf) == TD_FILE_STATE_OK) -#define TD_FILE_IS_BAD(tf) (TD_FILE_STATE(tf) == TD_FILE_STATE_BAD) int32_t tdInitTFile(STFile *pTFile, STfs *pTfs, const char *fname); int32_t tdCreateTFile(STFile *pTFile, STfs *pTfs, bool updateHeader, int8_t fType); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 3b30224e17..68ed6dde51 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -76,6 +76,7 @@ void vnodeFree(void* p); // meta typedef struct SMCtbCursor SMCtbCursor; +typedef struct SMStbCursor SMStbCursor; typedef struct STbUidStore STbUidStore; int metaOpen(SVnode* pVnode, SMeta** ppMeta); @@ -97,6 +98,9 @@ int metaGetTbNum(SMeta* pMeta); SMCtbCursor* metaOpenCtbCursor(SMeta* pMeta, tb_uid_t uid); void metaCloseCtbCursor(SMCtbCursor* pCtbCur); tb_uid_t metaCtbCursorNext(SMCtbCursor* pCtbCur); +SMStbCursor* metaOpenStbCursor(SMeta* pMeta, tb_uid_t uid); +void metaCloseStbCursor(SMStbCursor* pStbCur); +tb_uid_t metaStbCursorNext(SMStbCursor* pStbCur); STSma* metaGetSmaInfoByIndex(SMeta* pMeta, int64_t indexUid); STSmaWrapper* metaGetSmaInfoByTable(SMeta* pMeta, tb_uid_t uid, bool deepCopy); SArray* metaGetSmaIdsByTable(SMeta* pMeta, tb_uid_t uid); @@ -158,6 +162,8 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pSchema, bool // sma int32_t smaOpen(SVnode* pVnode); int32_t smaClose(SSma* pSma); +int32_t smaCloseEnv(SSma* pSma); +int32_t smaCloseEx(SSma* pSma); int32_t tdProcessTSmaCreate(SSma* pSma, int64_t version, const char* msg); int32_t tdProcessTSmaInsert(SSma* pSma, int64_t indexUid, const char* msg); diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index 95510c25e5..614ccc5b5e 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -92,6 +92,13 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) { goto _err; } + // open pSuidIdx + ret = tdbTbOpen("suid.idx", sizeof(tb_uid_t), 0, uidIdxKeyCmpr, pMeta->pEnv, &pMeta->pSuidIdx); + if (ret < 0) { + metaError("vgId:%d, failed to open meta super table index since %s", TD_VID(pVnode), tstrerror(terrno)); + goto _err; + } + // open pTagIdx // TODO(yihaoDeng), refactor later char indexFullPath[128] = {0}; @@ -141,6 +148,7 @@ _err: if (pMeta->pTagIvtIdx) indexClose(pMeta->pTagIvtIdx); if (pMeta->pTagIdx) tdbTbClose(pMeta->pTagIdx); if (pMeta->pCtbIdx) tdbTbClose(pMeta->pCtbIdx); + if (pMeta->pSuidIdx) tdbTbClose(pMeta->pSuidIdx); if (pMeta->pNameIdx) tdbTbClose(pMeta->pNameIdx); if (pMeta->pUidIdx) tdbTbClose(pMeta->pUidIdx); if (pMeta->pSkmDb) tdbTbClose(pMeta->pSkmDb); @@ -162,6 +170,7 @@ int metaClose(SMeta *pMeta) { if (pMeta->pTagIdx) tdbTbClose(pMeta->pTagIdx); #endif if (pMeta->pCtbIdx) tdbTbClose(pMeta->pCtbIdx); + if (pMeta->pSuidIdx) tdbTbClose(pMeta->pSuidIdx); if (pMeta->pNameIdx) tdbTbClose(pMeta->pNameIdx); if (pMeta->pUidIdx) tdbTbClose(pMeta->pUidIdx); if (pMeta->pSkmDb) tdbTbClose(pMeta->pSkmDb); diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 910f4bba51..f57ee54400 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -325,6 +325,70 @@ tb_uid_t metaCtbCursorNext(SMCtbCursor *pCtbCur) { return pCtbIdxKey->uid; } +struct SMStbCursor { + SMeta *pMeta; + TBC *pCur; + tb_uid_t suid; + void *pKey; + void *pVal; + int kLen; + int vLen; +}; + +SMStbCursor *metaOpenStbCursor(SMeta *pMeta, tb_uid_t suid) { + SMStbCursor *pStbCur = NULL; + int ret = 0; + int c = 0; + + pStbCur = (SMStbCursor *)taosMemoryCalloc(1, sizeof(*pStbCur)); + if (pStbCur == NULL) { + return NULL; + } + + pStbCur->pMeta = pMeta; + pStbCur->suid = suid; + metaRLock(pMeta); + + ret = tdbTbcOpen(pMeta->pSuidIdx, &pStbCur->pCur, NULL); + if (ret < 0) { + metaULock(pMeta); + taosMemoryFree(pStbCur); + return NULL; + } + + // move to the suid + tdbTbcMoveTo(pStbCur->pCur, &suid, sizeof(suid), &c); + if (c > 0) { + tdbTbcMoveToNext(pStbCur->pCur); + } + + return pStbCur; +} + +void metaCloseStbCursor(SMStbCursor *pStbCur) { + if (pStbCur) { + if (pStbCur->pMeta) metaULock(pStbCur->pMeta); + if (pStbCur->pCur) { + tdbTbcClose(pStbCur->pCur); + + tdbFree(pStbCur->pKey); + tdbFree(pStbCur->pVal); + } + + taosMemoryFree(pStbCur); + } +} + +tb_uid_t metaStbCursorNext(SMStbCursor *pStbCur) { + int ret; + + ret = tdbTbcNext(pStbCur->pCur, &pStbCur->pKey, &pStbCur->kLen, &pStbCur->pVal, &pStbCur->vLen); + if (ret < 0) { + return 0; + } + return *(tb_uid_t*)pStbCur->pKey; +} + STSchema *metaGetTbTSchema(SMeta *pMeta, tb_uid_t uid, int32_t sver) { // SMetaReader mr = {0}; STSchema * pTSchema = NULL; diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 7109bf1dfc..ea425ca7de 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -23,6 +23,7 @@ static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaSaveToSkmDb(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME); +static int metaUpdateSuidIdx(SMeta *pMeta, const SMetaEntry *pME); static int metaUpdateTagIdx(SMeta *pMeta, const SMetaEntry *pCtbEntry); static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type); @@ -209,6 +210,7 @@ _drop_super_table: &pMeta->txn); tdbTbDelete(pMeta->pNameIdx, pReq->name, strlen(pReq->name) + 1, &pMeta->txn); tdbTbDelete(pMeta->pUidIdx, &pReq->suid, sizeof(tb_uid_t), &pMeta->txn); + tdbTbDelete(pMeta->pSuidIdx, &pReq->suid, sizeof(tb_uid_t), &pMeta->txn); metaULock(pMeta); @@ -436,11 +438,13 @@ static int metaDropTableByUid(SMeta *pMeta, tb_uid_t uid, int *type) { tdbTbDelete(pMeta->pUidIdx, &uid, sizeof(uid), &pMeta->txn); if(e.type != TSDB_SUPER_TABLE) metaDeleteTtlIdx(pMeta, &e); + if (e.type == TSDB_CHILD_TABLE) { tdbTbDelete(pMeta->pCtbIdx, &(SCtbIdxKey){.suid = e.ctbEntry.suid, .uid = uid}, sizeof(SCtbIdxKey), &pMeta->txn); } else if (e.type == TSDB_NORMAL_TABLE) { // drop schema.db (todo) } else if (e.type == TSDB_SUPER_TABLE) { + tdbTbDelete(pMeta->pSuidIdx, &e.uid, sizeof(tb_uid_t), &pMeta->txn); // drop schema.db (todo) } @@ -911,6 +915,10 @@ static int metaUpdateUidIdx(SMeta *pMeta, const SMetaEntry *pME) { return tdbTbInsert(pMeta->pUidIdx, &pME->uid, sizeof(tb_uid_t), &pME->version, sizeof(int64_t), &pMeta->txn); } +static int metaUpdateSuidIdx(SMeta *pMeta, const SMetaEntry *pME) { + return tdbTbInsert(pMeta->pSuidIdx, &pME->uid, sizeof(tb_uid_t), NULL, 0, &pMeta->txn); +} + static int metaUpdateNameIdx(SMeta *pMeta, const SMetaEntry *pME) { return tdbTbInsert(pMeta->pNameIdx, pME->name, strlen(pME->name) + 1, &pME->uid, sizeof(tb_uid_t), &pMeta->txn); } @@ -1081,6 +1089,10 @@ static int metaHandleEntry(SMeta *pMeta, const SMetaEntry *pME) { } else { // update schema.db if (metaSaveToSkmDb(pMeta, pME) < 0) goto _err; + + if (pME->type == TSDB_SUPER_TABLE) { + if (metaUpdateSuidIdx(pMeta, pME) < 0) goto _err; + } } if (pME->type != TSDB_SUPER_TABLE) { diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c index 4e907e93ca..1e8832615e 100644 --- a/source/dnode/vnode/src/sma/smaEnv.c +++ b/source/dnode/vnode/src/sma/smaEnv.c @@ -126,22 +126,21 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS } if (smaType == TSDB_SMA_TYPE_ROLLUP) { - SMA_RSMA_STAT(*pSmaStat)->pSma = (SSma*)pSma; + SRSmaStat *pRSmaStat = (SRSmaStat *)(*pSmaStat); + pRSmaStat->pSma = (SSma *)pSma; // init timer - SMA_RSMA_TMR_HANDLE(*pSmaStat) = taosTmrInit(10000, 100, 10000, "RSMA_G"); - if (!SMA_RSMA_TMR_HANDLE(*pSmaStat)) { + RSMA_TMR_HANDLE(pRSmaStat) = taosTmrInit(10000, 100, 10000, "RSMA"); + if (!RSMA_TMR_HANDLE(pRSmaStat)) { taosMemoryFreeClear(*pSmaStat); return TSDB_CODE_FAILED; } - - atomic_store_8(&SMA_RSMA_TMR_STAT(*pSmaStat), TASK_TRIGGER_STATUS__ACTIVE); // init hash - SMA_RSMA_INFO_HASH(*pSmaStat) = taosHashInit( + RSMA_INFO_HASH(pRSmaStat) = taosHashInit( RSMA_TASK_INFO_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); - if (!SMA_RSMA_INFO_HASH(*pSmaStat)) { - if (SMA_RSMA_TMR_HANDLE(*pSmaStat)) { - taosTmrCleanUp(SMA_RSMA_TMR_HANDLE(*pSmaStat)); + if (!RSMA_INFO_HASH(pRSmaStat)) { + if (RSMA_TMR_HANDLE(pRSmaStat)) { + taosTmrCleanUp(RSMA_TMR_HANDLE(pRSmaStat)); } taosMemoryFreeClear(*pSmaStat); return TSDB_CODE_FAILED; @@ -155,12 +154,79 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS return TSDB_CODE_SUCCESS; } -static void *tdFreeTSmaStat(STSmaStat *pStat) { +static void tdDestroyTSmaStat(STSmaStat *pStat) { if (pStat) { tDestroyTSma(pStat->pTSma); taosMemoryFreeClear(pStat->pTSma); - taosMemoryFreeClear(pStat); + taosMemoryFreeClear(pStat->pTSchema); } +} + +static void *tdFreeTSmaStat(STSmaStat *pStat) { + tdDestroyTSmaStat(pStat); + taosMemoryFreeClear(pStat); + return NULL; +} + +static void tdDestroyRSmaStat(SRSmaStat *pStat) { + if (pStat) { + smaDebug("vgId:%d, %s:%d free rsma stat", SMA_VID(pStat->pSma), __func__, __LINE__); + // step 1: set persistence task cancelled + atomic_store_8(RSMA_TRIGGER_STAT(pStat), TASK_TRIGGER_STAT_CANCELLED); + + // step 2: clean timer + taosTmrStopA(&RSMA_TMR_ID(pStat)); + if (RSMA_TMR_HANDLE(pStat)) { + taosTmrCleanUp(RSMA_TMR_HANDLE(pStat)); + } + + // step 3: wait the persistence thread to finish + int32_t nLoops = 0; + if (atomic_load_8(RSMA_RUNNING_STAT(pStat)) == 1) { + while (1) { + if (atomic_load_8(RSMA_TRIGGER_STAT(pStat)) == TASK_TRIGGER_STAT_FINISHED) { + break; + } else { + smaDebug("not destroyed since rsma stat in %" PRIi8, atomic_load_8(RSMA_TRIGGER_STAT(pStat))); + } + ++nLoops; + if (nLoops > 1000) { + sched_yield(); + nLoops = 0; + } + taosMsleep(1000); // TODO: remove this line when release + } + } + + // step 4: destroy the rsma info and associated fetch tasks + // TODO: use taosHashSetFreeFp when taosHashSetFreeFp is ready. + void *infoHash = taosHashIterate(RSMA_INFO_HASH(pStat), NULL); + while (infoHash) { + SRSmaInfo *pSmaInfo = *(SRSmaInfo **)infoHash; + tdFreeRSmaInfo(pSmaInfo); + infoHash = taosHashIterate(RSMA_INFO_HASH(pStat), infoHash); + } + taosHashCleanup(RSMA_INFO_HASH(pStat)); + + // step 5: wait all triggered fetch tasks finished + nLoops = 0; + while (1) { + if (T_REF_VAL_GET((SSmaStat *)pStat) == 0) { + break; + } + ++nLoops; + if (nLoops > 1000) { + sched_yield(); + nLoops = 0; + } + taosMsleep(1000); // TODO: remove this line when release + } + } +} + +static void *tdFreeRSmaStat(SRSmaStat *pStat) { + tdDestroyRSmaStat(pStat); + taosMemoryFreeClear(pStat); return NULL; } @@ -179,22 +245,16 @@ void *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType) { int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType) { if (pSmaStat) { if (smaType == TSDB_SMA_TYPE_TIME_RANGE) { - tdFreeTSmaStat(&pSmaStat->tsmaStat); + smaDebug("%s:%d destroy tsma stat", __func__, __LINE__); + tdDestroyTSmaStat(SMA_TSMA_STAT(pSmaStat)); } else if (smaType == TSDB_SMA_TYPE_ROLLUP) { - if (SMA_RSMA_TMR_HANDLE(pSmaStat)) { - taosTmrCleanUp(SMA_RSMA_TMR_HANDLE(pSmaStat)); - } - // TODO: use taosHashSetFreeFp when taosHashSetFreeFp is ready. - void *infoHash = taosHashIterate(SMA_RSMA_INFO_HASH(pSmaStat), NULL); - while (infoHash) { - SRSmaInfo *pInfoHash = *(SRSmaInfo **)infoHash; - tdFreeRSmaInfo(pInfoHash); - infoHash = taosHashIterate(SMA_RSMA_INFO_HASH(pSmaStat), infoHash); - } - taosHashCleanup(SMA_RSMA_INFO_HASH(pSmaStat)); + smaDebug("%s:%d destroy rsma stat", __func__, __LINE__); + tdDestroyRSmaStat(SMA_RSMA_STAT(pSmaStat)); } else { ASSERT(0); } + } else { + smaDebug("%s:%d no need to destroy rsma stat", __func__, __LINE__); } return TSDB_CODE_SUCCESS; } @@ -259,35 +319,4 @@ int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType) { tdUnLockSma(pSma); return TSDB_CODE_SUCCESS; -}; - -int32_t smaTimerInit(void **timer, int8_t *initFlag, const char *label) { - int8_t old; - while (1) { - old = atomic_val_compare_exchange_8(initFlag, 0, 2); - if (old != 2) break; - } - - if (old == 0) { - *timer = taosTmrInit(10000, 100, 10000, label); - if (!(*timer)) { - atomic_store_8(initFlag, 0); - return -1; - } - atomic_store_8(initFlag, 1); - } - return 0; -} - -void smaTimerCleanUp(void *timer, int8_t *initFlag) { - int8_t old; - while (1) { - old = atomic_val_compare_exchange_8(initFlag, 1, 2); - if (old != 2) break; - } - - if (old == 1) { - taosTmrCleanUp(timer); - atomic_store_8(initFlag, 0); - } -} \ No newline at end of file +}; \ No newline at end of file diff --git a/source/dnode/vnode/src/sma/smaOpen.c b/source/dnode/vnode/src/sma/smaOpen.c index 383c264e7b..2f40df8b45 100644 --- a/source/dnode/vnode/src/sma/smaOpen.c +++ b/source/dnode/vnode/src/sma/smaOpen.c @@ -126,19 +126,31 @@ _err: return -1; } -int32_t smaClose(SSma *pSma) { +int32_t smaCloseEnv(SSma *pSma) { + if(pSma) { + SMA_TSMA_ENV(pSma) = tdFreeSmaEnv(SMA_TSMA_ENV(pSma)); + SMA_RSMA_ENV(pSma) = tdFreeSmaEnv(SMA_RSMA_ENV(pSma)); + } + return 0; +} + +int32_t smaCloseEx(SSma *pSma) { if (pSma) { taosThreadMutexDestroy(&pSma->mutex); if SMA_RSMA_TSDB0 (pSma) tsdbClose(&SMA_RSMA_TSDB0(pSma)); if SMA_RSMA_TSDB1 (pSma) tsdbClose(&SMA_RSMA_TSDB1(pSma)); if SMA_RSMA_TSDB2 (pSma) tsdbClose(&SMA_RSMA_TSDB2(pSma)); - // SMA_TSMA_ENV(pSma) = tdFreeSmaEnv(SMA_TSMA_ENV(pSma)); - // SMA_RSMA_ENV(pSma) = tdFreeSmaEnv(SMA_RSMA_ENV(pSma)); taosMemoryFreeClear(pSma); } return 0; } +int32_t smaClose(SSma *pSma) { + smaCloseEnv(pSma); + smaCloseEx(pSma); + return 0; +} + /** * @brief rsma env restore * diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 3c2f710fa7..020ee38db9 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -14,8 +14,8 @@ */ #include "sma.h" -#include "tstream.h" +#define RSMA_QTASK_PERSIST_MS 7200000 typedef enum { TD_QTASK_TMP_FILE = 0, TD_QTASK_CUR_FILE } TD_QTASK_FILE_T; static const char *tdQTaskInfoFname[] = {"qtaskinfo.t", "qtaskinfo"}; @@ -31,13 +31,13 @@ static void tdRSmaPersistTrigger(void *param, void *tmrId); struct SRSmaInfoItem { SRSmaInfo *pRsmaInfo; void *taskInfo; // qTaskInfo_t - void *tmrHandle; tmr_h tmrId; int8_t level; int8_t tmrInitFlag; - int8_t triggerStatus; // TASK_TRIGGER_STATUS__IN_ACTIVE/TASK_TRIGGER_STATUS__ACTIVE + int8_t triggerStat; int32_t maxDelay; }; + struct SRSmaInfo { STSchema *pTSchema; SSma *pSma; @@ -45,11 +45,14 @@ struct SRSmaInfo { SRSmaInfoItem items[TSDB_RETENTION_L2]; }; -static FORCE_INLINE void tdFreeTaskHandle(qTaskInfo_t *taskHandle) { +static FORCE_INLINE void tdFreeTaskHandle(qTaskInfo_t *taskHandle, int32_t vgId, int32_t level) { // Note: free/kill may in RC qTaskInfo_t otaskHandle = atomic_load_ptr(taskHandle); if (otaskHandle && atomic_val_compare_exchange_ptr(taskHandle, otaskHandle, NULL)) { + smaDebug("vgId:%d, %s:%d free qTaskInfo_t %p of level %d", vgId, __func__, __LINE__, otaskHandle, level); qDestroyTask(otaskHandle); + } else { + smaDebug("vgId:%d, %s:%d not free qTaskInfo_t %p of level %d", vgId, __func__, __LINE__, otaskHandle, level); } } @@ -58,14 +61,19 @@ void *tdFreeRSmaInfo(SRSmaInfo *pInfo) { for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { SRSmaInfoItem *pItem = &pInfo->items[i]; if (pItem->taskInfo) { - tdFreeTaskHandle(pItem->taskInfo); - } - if (pItem->tmrHandle) { - taosTmrCleanUp(pItem->tmrHandle); + smaDebug("vgId:%d, stb %" PRIi64 " stop fetch-timer %p level %d", SMA_VID(pInfo->pSma), pInfo->suid, + pItem->tmrId, i + 1); + taosTmrStopA(&pItem->tmrId); + tdFreeTaskHandle(&pItem->taskInfo, SMA_VID(pInfo->pSma), i + 1); + } else { + smaDebug("vgId:%d, stb %" PRIi64 " no need to destroy rsma info level %d since empty taskInfo", + SMA_VID(pInfo->pSma), pInfo->suid, i + 1); } } taosMemoryFree(pInfo->pTSchema); taosMemoryFree(pInfo); + } else { + smaDebug("vgId:%d, stb %" PRIi64 " no need to destroy rsma info since empty", SMA_VID(pInfo->pSma), pInfo->suid); } return NULL; @@ -83,7 +91,7 @@ static FORCE_INLINE int32_t tdUidStoreInit(STbUidStore **pStore) { static FORCE_INLINE int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids) { SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); - SSmaStat *pStat = SMA_ENV_STAT(pEnv); + SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); SRSmaInfo *pRSmaInfo = NULL; if (!suid || !tbUids) { @@ -92,27 +100,31 @@ static FORCE_INLINE int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SA return TSDB_CODE_FAILED; } - pRSmaInfo = taosHashGet(SMA_RSMA_INFO_HASH(pStat), suid, sizeof(tb_uid_t)); + pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), suid, sizeof(tb_uid_t)); if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) { smaError("vgId:%d, failed to get rsma info for uid:%" PRIi64, SMA_VID(pSma), *suid); terrno = TSDB_CODE_RSMA_INVALID_STAT; return TSDB_CODE_FAILED; } - if (pRSmaInfo->items[0].taskInfo && (qUpdateQualifiedTableId(pRSmaInfo->items[0].taskInfo, tbUids, true) < 0)) { - smaError("vgId:%d, update tbUidList failed for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr(terrno)); - return TSDB_CODE_FAILED; - } else { - smaDebug("vgId:%d, update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 ", uid:%" PRIi64, SMA_VID(pSma), - pRSmaInfo->items[0].taskInfo, *suid, *(int64_t *)taosArrayGet(tbUids, 0)); + if (pRSmaInfo->items[0].taskInfo) { + if ((qUpdateQualifiedTableId(pRSmaInfo->items[0].taskInfo, tbUids, true) < 0)) { + smaError("vgId:%d, update tbUidList failed for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr(terrno)); + return TSDB_CODE_FAILED; + } else { + smaDebug("vgId:%d, update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 ", uid:%" PRIi64, SMA_VID(pSma), + pRSmaInfo->items[0].taskInfo, *suid, *(int64_t *)taosArrayGet(tbUids, 0)); + } } - if (pRSmaInfo->items[1].taskInfo && (qUpdateQualifiedTableId(pRSmaInfo->items[1].taskInfo, tbUids, true) < 0)) { - smaError("vgId:%d, update tbUidList failed for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr(terrno)); - return TSDB_CODE_FAILED; - } else { - smaDebug("vgId:%d, update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 ", uid:%" PRIi64, SMA_VID(pSma), - pRSmaInfo->items[1].taskInfo, *suid, *(int64_t *)taosArrayGet(tbUids, 0)); + if (pRSmaInfo->items[1].taskInfo) { + if ((qUpdateQualifiedTableId(pRSmaInfo->items[1].taskInfo, tbUids, true) < 0)) { + smaError("vgId:%d, update tbUidList failed for uid:%" PRIi64 " since %s", SMA_VID(pSma), *suid, terrstr(terrno)); + return TSDB_CODE_FAILED; + } else { + smaDebug("vgId:%d, update tbUidList succeed for qTaskInfo:%p with suid:%" PRIi64 ", uid:%" PRIi64, SMA_VID(pSma), + pRSmaInfo->items[1].taskInfo, *suid, *(int64_t *)taosArrayGet(tbUids, 0)); + } } return TSDB_CODE_SUCCESS; @@ -159,9 +171,9 @@ int32_t tdFetchTbUidList(SSma *pSma, STbUidStore **ppStore, tb_uid_t suid, tb_ui return TSDB_CODE_SUCCESS; } - SSmaStat *pStat = SMA_ENV_STAT(pEnv); - SHashObj *infoHash = NULL; - if (!pStat || !(infoHash = SMA_RSMA_INFO_HASH(pStat))) { + SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); + SHashObj *infoHash = NULL; + if (!pStat || !(infoHash = RSMA_INFO_HASH(pStat))) { terrno = TSDB_CODE_RSMA_INVALID_STAT; return TSDB_CODE_FAILED; } @@ -195,11 +207,11 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaInfo if (param->qmsg[idx]) { SRSmaInfoItem *pItem = &(pRSmaInfo->items[idx]); pItem->pRsmaInfo = pRSmaInfo; - pItem->taskInfo = qCreateStreamExecTaskInfo(param->qmsg[0], pReadHandle); + pItem->taskInfo = qCreateStreamExecTaskInfo(param->qmsg[idx], pReadHandle); if (!pItem->taskInfo) { goto _err; } - pItem->triggerStatus = TASK_TRIGGER_STATUS__IN_ACTIVE; + pItem->triggerStat = TASK_TRIGGER_STAT_INACTIVE; if (param->maxdelay[idx] < TSDB_MIN_ROLLUP_MAX_DELAY) { int64_t msInterval = convertTimeFromPrecisionToUnit(pRetention[idx + 1].freq, pTsdbCfg->precision, TIME_UNIT_MILLISECOND); @@ -211,10 +223,6 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaInfo pItem->maxDelay = TSDB_MAX_ROLLUP_MAX_DELAY; } pItem->level = (idx == 0 ? TSDB_RETENTION_L1 : TSDB_RETENTION_L2); - pItem->tmrHandle = taosTmrInit(10000, 100, 10000, "RSMA"); - if (!pItem->tmrHandle) { - goto _err; - } } return TSDB_CODE_SUCCESS; _err: @@ -251,10 +259,10 @@ int32_t tdProcessRSmaCreate(SVnode *pVnode, SVCreateStbReq *pReq) { } SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); - SSmaStat *pStat = SMA_ENV_STAT(pEnv); + SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); SRSmaInfo *pRSmaInfo = NULL; - pRSmaInfo = taosHashGet(SMA_RSMA_INFO_HASH(pStat), &pReq->suid, sizeof(tb_uid_t)); + pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &pReq->suid, sizeof(tb_uid_t)); if (pRSmaInfo) { ASSERT(0); // TODO: free original pRSmaInfo is exists abnormally smaWarn("vgId:%d, rsma info already exists for stb: %s, %" PRIi64, SMA_VID(pSma), pReq->name, pReq->suid); @@ -293,16 +301,23 @@ int32_t tdProcessRSmaCreate(SVnode *pVnode, SVCreateStbReq *pReq) { if (tdSetRSmaInfoItemParams(pSma, param, pRSmaInfo, &handle, 0) < 0) { goto _err; } + if (tdSetRSmaInfoItemParams(pSma, param, pRSmaInfo, &handle, 1) < 0) { goto _err; } - if (taosHashPut(SMA_RSMA_INFO_HASH(pStat), &pReq->suid, sizeof(tb_uid_t), &pRSmaInfo, sizeof(pRSmaInfo)) < 0) { + if (taosHashPut(RSMA_INFO_HASH(pStat), &pReq->suid, sizeof(tb_uid_t), &pRSmaInfo, sizeof(pRSmaInfo)) < 0) { goto _err; } else { smaDebug("vgId:%d, register rsma info succeed for suid:%" PRIi64, SMA_VID(pSma), pReq->suid); } + // start the persist timer + if (TASK_TRIGGER_STAT_INIT == + atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pStat), TASK_TRIGGER_STAT_INIT, TASK_TRIGGER_STAT_ACTIVE)) { + taosTmrStart(tdRSmaPersistTrigger, RSMA_QTASK_PERSIST_MS, pStat, RSMA_TMR_HANDLE(pStat)); + } + return TSDB_CODE_SUCCESS; _err: tdFreeRSmaInfo(pRSmaInfo); @@ -432,6 +447,16 @@ static int32_t tdFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) { return 0; } +static void tdDestroySDataBlockArray(SArray *pArray) { +#if 0 + for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { + SSDataBlock *pDataBlock = taosArrayGet(pArray, i); + blockDestroyInner(pDataBlock); + } +#endif + taosArrayDestroy(pArray); +} + static int32_t tdFetchAndSubmitRSmaResult(SRSmaInfoItem *pItem, int8_t blkType) { SArray *pResult = NULL; SRSmaInfo *pRSmaInfo = pItem->pRsmaInfo; @@ -466,6 +491,7 @@ static int32_t tdFetchAndSubmitRSmaResult(SRSmaInfoItem *pItem, int8_t blkType) #endif STsdb *sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb1 : pSma->pRSmaTsdb2); SSubmitReq *pReq = NULL; + // TODO: the schema update should be handled if (buildSubmitReqFromDataBlock(&pReq, pResult, pRSmaInfo->pTSchema, SMA_VID(pSma), pRSmaInfo->suid) < 0) { goto _err; } @@ -477,17 +503,13 @@ static int32_t tdFetchAndSubmitRSmaResult(SRSmaInfoItem *pItem, int8_t blkType) taosMemoryFreeClear(pReq); } else { - smaDebug("vgId:%d, no rsma % " PRIi8 " data generated since %s", SMA_VID(pSma), pItem->level, tstrerror(terrno)); + smaDebug("vgId:%d, no rsma %" PRIi8 " data generated since %s", SMA_VID(pSma), pItem->level, tstrerror(terrno)); } - if (blkType == STREAM_DATA_TYPE_SUBMIT_BLOCK) { - atomic_store_8(&pItem->triggerStatus, TASK_TRIGGER_STATUS__ACTIVE); - } - - taosArrayDestroy(pResult); + tdDestroySDataBlockArray(pResult); return TSDB_CODE_SUCCESS; _err: - taosArrayDestroy(pResult); + tdDestroySDataBlockArray(pResult); return TSDB_CODE_FAILED; } @@ -499,22 +521,34 @@ _err: */ static void tdRSmaFetchTrigger(void *param, void *tmrId) { SRSmaInfoItem *pItem = param; + SSma *pSma = pItem->pRsmaInfo->pSma; + SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT((SSmaEnv *)pSma->pRSmaEnv); - if (atomic_load_8(&pItem->triggerStatus) == TASK_TRIGGER_STATUS__ACTIVE) { - smaWarn("%s:%d THREAD:%" PRIi64 " level %" PRIi8 " status is active for tb suid:%" PRIi64, __func__, __LINE__, - taosGetSelfPthreadId(), pItem->level, pItem->pRsmaInfo->suid); - SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL}; - - atomic_store_8(&pItem->triggerStatus, TASK_TRIGGER_STATUS__IN_ACTIVE); - qSetStreamInput(pItem->taskInfo, &dataBlock, STREAM_DATA_TYPE_SSDATA_BLOCK, false); - - tdFetchAndSubmitRSmaResult(pItem, STREAM_DATA_TYPE_SSDATA_BLOCK); - } else { - smaWarn("%s:%d THREAD:%" PRIi64 " level %" PRIi8 " status is inactive for tb suid:%" PRIi64, __func__, __LINE__, - taosGetSelfPthreadId(), pItem->level, pItem->pRsmaInfo->suid); + int8_t rsmaTriggerStat = atomic_load_8(RSMA_TRIGGER_STAT(pStat)); + if (rsmaTriggerStat == TASK_TRIGGER_STAT_CANCELLED || rsmaTriggerStat == TASK_TRIGGER_STAT_FINISHED) { + smaDebug("vgId:%d, %s:%d level %" PRIi8 " not fetch since stat is cancelled for table suid:%" PRIi64, SMA_VID(pSma), + __func__, __LINE__, pItem->level, pItem->pRsmaInfo->suid); + return; } - // taosTmrReset(tdRSmaFetchTrigger, pItem->maxDelay, pItem, pItem->tmrHandle, &pItem->tmrId); + int8_t fetchTriggerStat = + atomic_val_compare_exchange_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE, TASK_TRIGGER_STAT_INACTIVE); + if (fetchTriggerStat == TASK_TRIGGER_STAT_ACTIVE) { + smaDebug("vgId:%d, %s:%d level %" PRIi8 " stat is active for table suid:%" PRIi64, SMA_VID(pSma), __func__, + __LINE__, pItem->level, pItem->pRsmaInfo->suid); + + tdRefSmaStat(pSma, (SSmaStat *)pStat); + + SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL}; + qSetStreamInput(pItem->taskInfo, &dataBlock, STREAM_DATA_TYPE_SSDATA_BLOCK, false); + tdFetchAndSubmitRSmaResult(pItem, STREAM_DATA_TYPE_SSDATA_BLOCK); + + tdUnRefSmaStat(pSma, (SSmaStat *)pStat); + + } else { + smaDebug("vgId:%d, %s:%d level %" PRIi8 " stat is inactive for table suid:%" PRIi64, SMA_VID(pSma), __func__, + __LINE__, pItem->level, pItem->pRsmaInfo->suid); + } } static FORCE_INLINE int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfoItem *pItem, @@ -533,14 +567,17 @@ static FORCE_INLINE int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int3 } tdFetchAndSubmitRSmaResult(pItem, STREAM_DATA_TYPE_SUBMIT_BLOCK); - atomic_store_8(&pItem->triggerStatus, TASK_TRIGGER_STATUS__ACTIVE); - smaWarn("%s:%d THREAD:%" PRIi64 " process rsma insert", __func__, __LINE__, taosGetSelfPthreadId()); + atomic_store_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE); + smaDebug("vgId:%d, %s:%d process rsma insert", SMA_VID(pSma), __func__, __LINE__); SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); SRSmaStat *pStat = SMA_RSMA_STAT(pEnv->pStat); - taosTmrStart(tdRSmaPersistTrigger, 5000, pStat, pStat->tmrHandle); - taosTmrReset(tdRSmaFetchTrigger, pItem->maxDelay, pItem, pItem->tmrHandle, &pItem->tmrId); + if (pStat->tmrHandle) { + taosTmrReset(tdRSmaFetchTrigger, pItem->maxDelay, pItem, pStat->tmrHandle, &pItem->tmrId); + } else { + ASSERT(0); + } return TSDB_CODE_SUCCESS; } @@ -552,10 +589,10 @@ static int32_t tdExecuteRSma(SSma *pSma, const void *pMsg, int32_t inputType, tb return TSDB_CODE_SUCCESS; } - SSmaStat *pStat = SMA_ENV_STAT(pEnv); + SRSmaStat *pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); SRSmaInfo *pRSmaInfo = NULL; - pRSmaInfo = taosHashGet(SMA_RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); + pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) { smaDebug("vgId:%d, return as no rsma info for suid:%" PRIu64, SMA_VID(pSma), suid); @@ -608,7 +645,7 @@ int32_t tdProcessRSmaSubmit(SSma *pSma, void *pMsg, int32_t inputType) { return TSDB_CODE_SUCCESS; } -void tdRSmaQTaskGetFName(int32_t vid, int8_t ftype, char* outputName) { +void tdRSmaQTaskGetFName(int32_t vid, int8_t ftype, char *outputName) { tdGetVndFileName(vid, "rsma", tdQTaskInfoFname[ftype], outputName); } @@ -618,6 +655,23 @@ static void *tdRSmaPersistExec(void *param) { SSma *pSma = pRSmaStat->pSma; STfs *pTfs = pSma->pVnode->pTfs; int64_t toffset = 0; + bool isFileCreated = false; + + if (TASK_TRIGGER_STAT_CANCELLED == atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat))) { + goto _end; + } + +#if 0 + SArray *suidList = taosArrayInit(1, sizeof(tb_uid_t)); + if (tsdbGetStbIdList(SMA_META(pSma), 0, suidList) < 0) { + ASSERT(0); + } else { + for (int32_t i = 0; i < taosArrayGetSize(suidList); ++i) { + tb_uid_t suid = *(tb_uid_t *)taosArrayGet(suidList, i); + smaDebug("suid [%d] is %" PRIi64, i, suid); + } + } +#endif void *infoHash = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), NULL); if (!infoHash) { @@ -625,67 +679,142 @@ static void *tdRSmaPersistExec(void *param) { } STFile tFile = {0}; - int32_t vid = 2; - char qTaskInfoFName[TSDB_FILENAME_LEN]; - tdRSmaQTaskGetFName(vid, TD_QTASK_TMP_FILE, qTaskInfoFName); - tdInitTFile(&tFile, pTfs, qTaskInfoFName); - tdCreateTFile(&tFile, pTfs, true, -1); + int32_t vid = SMA_VID(pSma); while (infoHash) { SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)infoHash; - char *pOutput = NULL; - int32_t len = 0; - if (qSerializeTaskStatus(pRSmaInfo->items[0].taskInfo, &pOutput, &len) < 0) { - smaError("serialize rsma task for table %" PRIi64 " failed since %s", pRSmaInfo->items[0].pRsmaInfo->suid, - terrstr(terrno)); - } else { - smaWarn("serialize rsma task for table %" PRIi64 " success and len is %d", pRSmaInfo->items[0].pRsmaInfo->suid, - len); - } - tdAppendTFile(&tFile, &len, sizeof(len), &toffset); - tdAppendTFile(&tFile, pOutput, len, &toffset); - taosMemoryFree(pOutput); +#if 0 + smaDebug("table %" PRIi64 " sleep 15s start ...", pRSmaInfo->items[0].pRsmaInfo->suid); + for (int32_t i = 15; i > 0; --i) { + taosSsleep(1); + smaDebug("table %" PRIi64 " countdown %d", pRSmaInfo->items[0].pRsmaInfo->suid, i); + } + smaDebug("table %" PRIi64 " sleep 15s end ...", pRSmaInfo->items[0].pRsmaInfo->suid); +#endif + for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { + qTaskInfo_t taskInfo = pRSmaInfo->items[i].taskInfo; + if (!taskInfo) { + smaDebug("vgId:%d, table %" PRIi64 " level %d qTaskInfo is NULL", vid, pRSmaInfo->suid, i + 1); + continue; + } + char *pOutput = NULL; + int32_t len = 0; + int8_t type = 0; + if (qSerializeTaskStatus(taskInfo, &pOutput, &len) < 0) { + smaError("vgId:%d, table %" PRIi64 " level %d serialize rsma task failed since %s", vid, pRSmaInfo->suid, i + 1, + terrstr(terrno)); + goto _err; + } else { + if (!pOutput) { + smaDebug("vgId:%d, table %" PRIi64 + " level %d serialize rsma task success but no output(len %d) and no need to persist", + vid, pRSmaInfo->suid, i + 1, len); + continue; + } else if (len <= 0) { + smaDebug("vgId:%d, table %" PRIi64 " level %d serialize rsma task success with len %d and no need to persist", + vid, pRSmaInfo->suid, i + 1, len); + taosMemoryFree(pOutput); + } + smaDebug("vgId:%d, table %" PRIi64 " level %d serialize rsma task success with len %d and need persist", vid, + pRSmaInfo->suid, i + 1, len); +#if 1 + if (qDeserializeTaskStatus(taskInfo, pOutput, len) < 0) { + smaError("vgId:%d, table %" PRIi64 "level %d deserialize rsma task failed since %s", vid, pRSmaInfo->suid, + i + 1, terrstr(terrno)); + } else { + smaDebug("vgId:%d, table %" PRIi64 " level %d deserialize rsma task success", vid, pRSmaInfo->suid, i + 1); + } +#endif + } + + if (!isFileCreated) { + char qTaskInfoFName[TSDB_FILENAME_LEN]; + tdRSmaQTaskGetFName(vid, TD_QTASK_TMP_FILE, qTaskInfoFName); + tdInitTFile(&tFile, pTfs, qTaskInfoFName); + tdCreateTFile(&tFile, pTfs, true, -1); + + isFileCreated = true; + } + len += (sizeof(len) + sizeof(pRSmaInfo->suid)); + tdAppendTFile(&tFile, &len, sizeof(len), &toffset); + tdAppendTFile(&tFile, &pRSmaInfo->suid, sizeof(pRSmaInfo->suid), &toffset); + tdAppendTFile(&tFile, pOutput, len, &toffset); + + taosMemoryFree(pOutput); + } infoHash = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), infoHash); } -_end: +_normal: + if (isFileCreated) { + if (tdUpdateTFileHeader(&tFile) < 0) { + smaError("vgId:%d, failed to update tfile %s header since %s", vid, TD_FILE_FULL_NAME(&tFile), tstrerror(terrno)); + tdCloseTFile(&tFile); + tdRemoveTFile(&tFile); + goto _err; + } else { + smaDebug("vgId:%d, succeed to update tfile %s header", vid, TD_FILE_FULL_NAME(&tFile)); + } - if (tdUpdateTFileHeader(&tFile) < 0) { - smaError("vgId:%d, failed to update tfile %s header since %s", vid, TD_FILE_FULL_NAME(&tFile), tstrerror(terrno)); tdCloseTFile(&tFile); - tdRemoveTFile(&tFile); - return NULL; + + char newFName[TSDB_FILENAME_LEN]; + strncpy(newFName, TD_FILE_FULL_NAME(&tFile), TSDB_FILENAME_LEN); + char *pos = strstr(newFName, tdQTaskInfoFname[TD_QTASK_TMP_FILE]); + strncpy(pos, tdQTaskInfoFname[TD_QTASK_CUR_FILE], TSDB_FILENAME_LEN - POINTER_DISTANCE(pos, newFName)); + if (taosRenameFile(TD_FILE_FULL_NAME(&tFile), newFName) != 0) { + smaError("vgId:%d, failed to rename %s to %s", vid, TD_FILE_FULL_NAME(&tFile), newFName); + goto _err; + } else { + smaDebug("vgId:%d, succeed to rename %s to %s", vid, TD_FILE_FULL_NAME(&tFile), newFName); + } } - - tdCloseTFile(&tFile); - - char newFName[TSDB_FILENAME_LEN]; - strncpy(newFName, TD_FILE_FULL_NAME(&tFile), TSDB_FILENAME_LEN); - char *pos = strstr(newFName, tdQTaskInfoFname[TD_QTASK_TMP_FILE]); - strncpy(pos, tdQTaskInfoFname[TD_QTASK_CUR_FILE], TSDB_FILENAME_LEN - POINTER_DISTANCE(pos, newFName)); - taosRenameFile(TD_FILE_FULL_NAME(&tFile), newFName); - - atomic_store_8(&pRSmaStat->tmrStat, TASK_TRIGGER_STATUS__ACTIVE); - return NULL; + goto _end; _err: - atomic_store_8(&pRSmaStat->tmrStat, TASK_TRIGGER_STATUS__ACTIVE); - // remove the .tmp file + if (isFileCreated) { + tdRemoveTFile(&tFile); + } +_end: + if (TASK_TRIGGER_STAT_INACTIVE == atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat), + TASK_TRIGGER_STAT_INACTIVE, + TASK_TRIGGER_STAT_ACTIVE)) { + smaDebug("vgId:%d, persist task is active again", vid); + } else if (TASK_TRIGGER_STAT_CANCELLED == atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat), + TASK_TRIGGER_STAT_CANCELLED, + TASK_TRIGGER_STAT_FINISHED)) { + smaDebug("vgId:%d, persist task is cancelled", vid); + } else { + smaWarn("vgId:%d, persist task in abnormal stat %" PRIi8, vid, atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat))); + ASSERT(0); + } + atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 0); + taosThreadExit(NULL); return NULL; } static void tdRSmaPersistTask(SRSmaStat *pRSmaStat) { - smaWarn("%s:%d entry ", __func__, __LINE__); - TdThread threadId; TdThreadAttr thAttr; taosThreadAttrInit(&thAttr); taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_DETACHED); + TdThread tid; - if (taosThreadCreate(&threadId, &thAttr, tdRSmaPersistExec, pRSmaStat) != 0) { - smaError("failed to create thread to persist rsma qTaskInfo since %s", strerror(errno)); + if (taosThreadCreate(&tid, &thAttr, tdRSmaPersistExec, pRSmaStat) != 0) { + if (TASK_TRIGGER_STAT_INACTIVE == atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat), + TASK_TRIGGER_STAT_INACTIVE, + TASK_TRIGGER_STAT_ACTIVE)) { + smaDebug("persist task is active again"); + } else if (TASK_TRIGGER_STAT_CANCELLED == atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat), + TASK_TRIGGER_STAT_CANCELLED, + TASK_TRIGGER_STAT_FINISHED)) { + smaDebug(" persist task is cancelled and set finished"); + } else { + smaWarn("persist task in abnormal stat %" PRIi8, atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat))); + ASSERT(0); + } + atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 0); } taosThreadAttrDestroy(&thAttr); - smaWarn("%s:%d end ", __func__, __LINE__); } /** @@ -696,17 +825,33 @@ static void tdRSmaPersistTask(SRSmaStat *pRSmaStat) { */ static void tdRSmaPersistTrigger(void *param, void *tmrId) { SRSmaStat *pRSmaStat = param; - - if (atomic_load_8(&pRSmaStat->tmrStat) == TASK_TRIGGER_STATUS__ACTIVE) { - smaWarn("%s:%d THREAD:%" PRIi64 " rsma persistence start since active", __func__, __LINE__, taosGetSelfPthreadId()); - atomic_store_8(&pRSmaStat->tmrStat, TASK_TRIGGER_STATUS__IN_ACTIVE); - - // execution - tdRSmaPersistTask(pRSmaStat); - } else { - smaWarn("%s:%d THREAD:%" PRIi64 " rsma persistence not start since inactive", __func__, __LINE__, - taosGetSelfPthreadId()); + int8_t tmrStat = + atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_ACTIVE, TASK_TRIGGER_STAT_INACTIVE); + switch (tmrStat) { + case TASK_TRIGGER_STAT_ACTIVE: { + atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 1); + if (TASK_TRIGGER_STAT_CANCELLED != atomic_val_compare_exchange_8(RSMA_TRIGGER_STAT(pRSmaStat), + TASK_TRIGGER_STAT_CANCELLED, + TASK_TRIGGER_STAT_FINISHED)) { + smaDebug("%s:%d rsma persistence start since active", __func__, __LINE__); + tdRSmaPersistTask(pRSmaStat); + taosTmrReset(tdRSmaPersistTrigger, RSMA_QTASK_PERSIST_MS, pRSmaStat, pRSmaStat->tmrHandle, &pRSmaStat->tmrId); + } else { + atomic_store_8(RSMA_RUNNING_STAT(pRSmaStat), 0); + } + } break; + case TASK_TRIGGER_STAT_CANCELLED: { + atomic_store_8(RSMA_TRIGGER_STAT(pRSmaStat), TASK_TRIGGER_STAT_FINISHED); + smaDebug("%s:%d rsma persistence not start since cancelled and finished", __func__, __LINE__); + } break; + case TASK_TRIGGER_STAT_INACTIVE: { + smaDebug("%s:%d rsma persistence not start since inactive", __func__, __LINE__); + } break; + case TASK_TRIGGER_STAT_INIT: { + smaDebug("%s:%d rsma persistence not start since init", __func__, __LINE__); + } break; + default: { + smaWarn("%s:%d rsma persistence not start since unknown stat %" PRIi8, __func__, __LINE__, tmrStat); + } break; } - - taosTmrReset(tdRSmaPersistTrigger, 3600000, pRSmaStat, pRSmaStat->tmrHandle, &pRSmaStat->tmrId); } \ No newline at end of file diff --git a/source/dnode/vnode/src/sma/smaUtil.c b/source/dnode/vnode/src/sma/smaUtil.c index 76690a3bde..85fddebf55 100644 --- a/source/dnode/vnode/src/sma/smaUtil.c +++ b/source/dnode/vnode/src/sma/smaUtil.c @@ -15,6 +15,8 @@ #include "sma.h" +// smaFileUtil ================ + #define TD_FILE_HEAD_SIZE 512 #define TD_FILE_STATE_OK 0 @@ -32,7 +34,7 @@ static int32_t tdEncodeTFInfo(void **buf, STFInfo *pInfo) { tlen += taosEncodeFixedU32(buf, pInfo->magic); tlen += taosEncodeFixedU32(buf, pInfo->ftype); tlen += taosEncodeFixedU32(buf, pInfo->fver); - tlen += taosEncodeFixedU64(buf, pInfo->fsize); + tlen += taosEncodeFixedI64(buf, pInfo->fsize); return tlen; } @@ -41,7 +43,7 @@ static void *tdDecodeTFInfo(void *buf, STFInfo *pInfo) { buf = taosDecodeFixedU32(buf, &(pInfo->magic)); buf = taosDecodeFixedU32(buf, &(pInfo->ftype)); buf = taosDecodeFixedU32(buf, &(pInfo->fver)); - buf = taosDecodeFixedU64(buf, &(pInfo->fsize)); + buf = taosDecodeFixedI64(buf, &(pInfo->fsize)); return buf; } @@ -235,4 +237,7 @@ int32_t tdCreateTFile(STFile *pTFile, STfs *pTfs, bool updateHeader, int8_t fTyp return 0; } -int32_t tdRemoveTFile(STFile *pTFile) { return tfsRemoveFile(TD_FILE_F(pTFile)); } \ No newline at end of file +int32_t tdRemoveTFile(STFile *pTFile) { return tfsRemoveFile(TD_FILE_F(pTFile)); } + +// smaXXXUtil ================ +// ... \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index ab31d65d68..bd40fd4a1f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2876,6 +2876,30 @@ int32_t tsdbGetCtbIdList(SMeta* pMeta, int64_t suid, SArray* list) { return TSDB_CODE_SUCCESS; } +/** + * @brief Get all suids since suid + * + * @param pMeta + * @param suid return all suids in one vnode if suid is 0 + * @param list + * @return int32_t + */ +int32_t tsdbGetStbIdList(SMeta* pMeta, int64_t suid, SArray* list) { + SMStbCursor* pCur = metaOpenStbCursor(pMeta, suid); + + while (1) { + tb_uid_t id = metaStbCursorNext(pCur); + if (id == 0) { + break; + } + + taosArrayPush(list, &id); + } + + metaCloseStbCursor(pCur); + return TSDB_CODE_SUCCESS; +} + static void destroyHelper(void* param) { if (param == NULL) { return; diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 3715866bb8..dbbe3d3750 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -72,7 +72,12 @@ int vnodeBegin(SVnode *pVnode) { return 0; } -int vnodeShouldCommit(SVnode *pVnode) { return pVnode->inUse->size > pVnode->config.szBuf / 3; } +int vnodeShouldCommit(SVnode *pVnode) { + if (pVnode->inUse) { + return pVnode->inUse->size > pVnode->config.szBuf / 3; + } + return false; +} int vnodeSaveInfo(const char *dir, const SVnodeInfo *pInfo) { char fname[TSDB_FILENAME_LEN]; diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index dfc258b42b..2834e0a2e7 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -152,12 +152,13 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { return pVnode; _err: + if (pVnode->pSma) smaClose(pVnode->pSma); if (pVnode->pQuery) vnodeQueryClose(pVnode); if (pVnode->pTq) tqClose(pVnode->pTq); if (pVnode->pWal) walClose(pVnode->pWal); if (pVnode->pTsdb) tsdbClose(&pVnode->pTsdb); if (pVnode->pMeta) metaClose(pVnode->pMeta); - if (pVnode->pSma) smaClose(pVnode->pSma); + tsem_destroy(&(pVnode->canCommit)); taosMemoryFree(pVnode); @@ -166,13 +167,14 @@ _err: void vnodeClose(SVnode *pVnode) { if (pVnode) { + smaCloseEnv(pVnode->pSma); vnodeCommit(pVnode); vnodeSyncClose(pVnode); vnodeQueryClose(pVnode); walClose(pVnode->pWal); + smaCloseEx(pVnode->pSma); tqClose(pVnode->pTq); if (pVnode->pTsdb) tsdbClose(&pVnode->pTsdb); - smaClose(pVnode->pSma); metaClose(pVnode->pMeta); vnodeCloseBufPool(pVnode); // destroy handle diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 36f81e86ff..5b0dae00cf 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -808,6 +808,7 @@ int32_t getMaximumIdleDurationSec(); * ops: root operator * data: *data save the result of encode, need to be freed by caller * length: *length save the length of *data + * nOptrWithVal: *nOptrWithVal save the number of optr with value * return: result code, 0 means success */ int32_t encodeOperator(SOperatorInfo* ops, char** data, int32_t *length); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index f18da72f58..56cf6a5a72 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -2929,6 +2929,13 @@ int32_t aggEncodeResultRow(SOperatorInfo* pOperator, char** result, int32_t* len int32_t totalSize = sizeof(int32_t) + sizeof(int32_t) + size * (sizeof(int32_t) + keyLen + sizeof(int32_t) + pSup->resultRowSize); + // no result + if (getTotalBufSize(pSup->pResultBuf) == 0) { + *result = NULL; + *length = 0; + return TSDB_CODE_SUCCESS; + } + *result = (char*)taosMemoryCalloc(1, totalSize); if (*result == NULL) { return TSDB_CODE_OUT_OF_MEMORY; @@ -4482,6 +4489,8 @@ int32_t encodeOperator(SOperatorInfo* ops, char** result, int32_t* length) { } return code; } + + ASSERT(currLength >= 0); if (*result == NULL) { *result = (char*)taosMemoryCalloc(1, currLength + sizeof(int32_t)); @@ -4507,7 +4516,6 @@ int32_t encodeOperator(SOperatorInfo* ops, char** result, int32_t* length) { taosMemoryFree(pCurrent); *length = *(int32_t*)(*result); } - for (int32_t i = 0; i < ops->numOfDownstream; ++i) { code = encodeOperator(ops->pDownstream[i], result, length); if (code != TDB_CODE_SUCCESS) { diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 0c6cd80f44..291d38bdca 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -157,14 +157,14 @@ int32_t taosRenameFile(const char *oldName, const char *newName) { #ifdef WINDOWS bool code = MoveFileEx(oldName, newName, MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED); if (!code) { - printf("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno)); + printf("failed to rename file %s to %s, reason:%s\n", oldName, newName, strerror(errno)); } return !code; #else int32_t code = rename(oldName, newName); if (code < 0) { - printf("failed to rename file %s to %s, reason:%s", oldName, newName, strerror(errno)); + printf("failed to rename file %s to %s, reason:%s\n", oldName, newName, strerror(errno)); } return code; From 393dbc5d2089abc6a8c602fe9eb8c09b153a3292 Mon Sep 17 00:00:00 2001 From: Yu Chen <74105241+yu285@users.noreply.github.com> Date: Sun, 26 Jun 2022 12:32:51 +0800 Subject: [PATCH 71/72] docs: update mnode description --- docs/en/21-tdinternal/01-arch.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/21-tdinternal/01-arch.md b/docs/en/21-tdinternal/01-arch.md index d7d472eb98..44651c0496 100644 --- a/docs/en/21-tdinternal/01-arch.md +++ b/docs/en/21-tdinternal/01-arch.md @@ -22,7 +22,7 @@ A complete TDengine system runs on one or more physical nodes. Logically, it inc **Virtual node (vnode)**: To better support data sharding, load balancing and prevent data from overheating or skewing, data nodes are virtualized into multiple virtual nodes (vnode, V2, V3, V4, etc. in the figure). Each vnode is a relatively independent work unit, which is the basic unit of time-series data storage and has independent running threads, memory space and persistent storage path. A vnode contains a certain number of tables (data collection points). When a new table is created, the system checks whether a new vnode needs to be created. The number of vnodes that can be created on a data node depends on the capacity of the hardware of the physical node where the data node is located. A vnode belongs to only one DB, but a DB can have multiple vnodes. In addition to the stored time-series data, a vnode also stores the schema and tag values of the included tables. A virtual node is uniquely identified in the system by the EP of the data node and the VGroup ID to which it belongs and is created and managed by the management node. -**Management node (mnode)**: A virtual logical unit responsible for monitoring and maintaining the running status of all data nodes and load balancing among nodes (M in the figure). At the same time, the management node is also responsible for the storage and management of metadata (including users, databases, tables, static tags, etc.), so it is also called Meta Node. Multiple (up to 5) mnodes can be configured in a TDengine cluster, and they are automatically constructed into a virtual management node group (M0, M1, M2 in the figure). The leader/follower mechanism is adopted for the mnode group and the data synchronization is carried out in a strongly consistent way. Any data update operation can only be executed on the leader. The creation of mnode cluster is completed automatically by the system without manual intervention. There is at most one mnode on each dnode, which is uniquely identified by the EP of the data node to which it belongs. Each dnode automatically obtains the EP of the dnode where all mnodes in the whole cluster are located, through internal messaging interaction. +**Management node (mnode)**: A virtual logical unit responsible for monitoring and maintaining the running status of all data nodes and load balancing among nodes (M in the figure). At the same time, the management node is also responsible for the storage and management of metadata (including users, databases, tables, static tags, etc.), so it is also called Meta Node. Multiple (up to 3) mnodes can be configured in a TDengine cluster, and they are automatically constructed into a virtual management node group (M0, M1, M2 in the figure). The leader/follower mechanism is adopted for the mnode group and the data synchronization is carried out in a strongly consistent way. Any data update operation can only be executed on the leader. The creation of mnode cluster is completed automatically by the system without manual intervention. There is at most one mnode on each dnode, which is uniquely identified by the EP of the data node to which it belongs. Each dnode automatically obtains the EP of the dnode where all mnodes in the whole cluster are located, through internal messaging interaction. **Virtual node group (VGroup)**: Vnodes on different data nodes can form a virtual node group to ensure the high availability of the system. The virtual node group is managed in a leader/follower mechanism. Write operations can only be performed on the leader vnode, and then replicated to follower vnodes, thus ensuring that one single replica of data is copied on multiple physical nodes. The number of virtual nodes in a vgroup equals the number of data replicas. If the number of replicas of a DB is N, the system must have at least N data nodes. The number of replicas can be specified by the parameter `“replica”` when creating a DB, and the default is 1. Using the multi-replication feature of TDengine, the same high data reliability can be achieved without the need for expensive storage devices such as disk arrays. Virtual node groups are created and managed by the management node, and the management node assigns a system unique ID, aka VGroup ID. If two virtual nodes have the same vnode group ID, it means that they belong to the same group and the data is backed up to each other. The number of virtual nodes in a virtual node group can be dynamically changed, allowing only one, that is, no data replication. VGroup ID is never changed. Even if a virtual node group is deleted, its ID will not be reused. From 56997d20575e45fc763c4a5a092b0d643219458c Mon Sep 17 00:00:00 2001 From: Yu Chen <74105241+yu285@users.noreply.github.com> Date: Sun, 26 Jun 2022 12:17:52 +0800 Subject: [PATCH 72/72] docs: update docs --- docs/zh/21-tdinternal/01-arch.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/21-tdinternal/01-arch.md b/docs/zh/21-tdinternal/01-arch.md index 433cb4808b..507ccea629 100644 --- a/docs/zh/21-tdinternal/01-arch.md +++ b/docs/zh/21-tdinternal/01-arch.md @@ -23,7 +23,7 @@ TDengine 分布式架构的逻辑结构图如下： **虚拟节点（vnode）：** 为更好的支持数据分片、负载均衡，防止数据过热或倾斜，数据节点被虚拟化成多个虚拟节点（vnode，图中 V2，V3，V4 等）。每个 vnode 都是一个相对独立的工作单元，是时序数据存储的基本单元，具有独立的运行线程、内存空间与持久化存储的路径。一个 vnode 包含一定数量的表（数据采集点）。当创建一张新表时，系统会检查是否需要创建新的 vnode。一个数据节点上能创建的 vnode 的数量取决于该数据节点所在物理节点的硬件资源。一个 vnode 只属于一个 DB，但一个 DB 可以有多个 vnode。一个 vnode 除存储的时序数据外，也保存有所包含的表的 schema、标签值等。一个虚拟节点由所属的数据节点的 EP，以及所属的 VGroup ID 在系统内唯一标识，由管理节点创建并管理。 -**管理节点（mnode）：** 一个虚拟的逻辑单元，负责所有数据节点运行状态的监控和维护，以及节点之间的负载均衡（图中 M）。同时，管理节点也负责元数据（包括用户、数据库、表、静态标签等）的存储和管理，因此也称为 Meta Node。TDengine 集群中可配置多个（开源版最多不超过 3 个）mnode，它们自动构建成为一个虚拟管理节点组（图中 M0，M1，M2）。mnode 间采用 master/slave 的机制进行管理，而且采取强一致方式进行数据同步，任何数据更新操作只能在 Master 上进行。mnode 集群的创建由系统自动完成，无需人工干预。每个 dnode 上至多有一个 mnode，由所属的数据节点的 EP 来唯一标识。每个 dnode 通过内部消息交互自动获取整个集群中所有 mnode 所在的 dnode 的 EP。 +**管理节点（mnode）：** 一个虚拟的逻辑单元，负责所有数据节点运行状态的监控和维护，以及节点之间的负载均衡（图中 M）。同时，管理节点也负责元数据（包括用户、数据库、表、静态标签等）的存储和管理，因此也称为 Meta Node。TDengine 集群中可配置多个（最多不超过 3 个）mnode，它们自动构建成为一个虚拟管理节点组（图中 M0，M1，M2）。mnode 间采用 master/slave 的机制进行管理，而且采取强一致方式进行数据同步，任何数据更新操作只能在 Master 上进行。mnode 集群的创建由系统自动完成，无需人工干预。每个 dnode 上至多有一个 mnode，由所属的数据节点的 EP 来唯一标识。每个 dnode 通过内部消息交互自动获取整个集群中所有 mnode 所在的 dnode 的 EP。 **虚拟节点组（VGroup）：** 不同数据节点上的 vnode 可以组成一个虚拟节点组（vgroup）来保证系统的高可靠。虚拟节点组内采取 master/slave 的方式进行管理。写操作只能在 master vnode 上进行，系统采用异步复制的方式将数据同步到 slave vnode，这样确保了一份数据在多个物理节点上有拷贝。一个 vgroup 里虚拟节点个数就是数据的副本数。如果一个 DB 的副本数为 N，系统必须有至少 N 数据节点。副本数在创建 DB 时通过参数 replica 可以指定，缺省为 1。使用 TDengine 的多副本特性，可以不再需要昂贵的磁盘阵列等存储设备，就可以获得同样的数据高可靠性。虚拟节点组由管理节点创建、管理，并且由管理节点分配一个系统唯一的 ID，VGroup ID。如果两个虚拟节点的 VGroup ID 相同，说明他们属于同一个组，数据互为备份。虚拟节点组里虚拟节点的个数是可以动态改变的，容许只有一个，也就是没有数据复制。VGroup ID 是永远不变的，即使一个虚拟节点组被删除，它的 ID 也不会被收回重复利用。