From 3b1664d9e4542f903920a4af11f8b7c46bf19aae Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 22 Jun 2022 18:50:39 +0800 Subject: [PATCH 01/49] feat: refactor rpc code --- include/libs/qcom/query.h | 35 +++--- include/os/osSocket.h | 5 +- source/client/src/clientImpl.c | 104 ++++++++--------- source/libs/qcom/src/queryUtil.c | 19 +-- source/libs/transport/inc/transComm.h | 26 +++++ source/libs/transport/src/transCli.c | 159 +++++++++++++++++--------- source/libs/transport/src/transComm.c | 2 +- source/libs/transport/src/transSvr.c | 26 ----- source/os/src/osSocket.c | 8 +- 9 files changed, 222 insertions(+), 162 deletions(-) diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index d562d07d77..0780343c64 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -16,6 +16,7 @@ #ifndef _TD_QUERY_H_ #define _TD_QUERY_H_ +// clang-foramt off #ifdef __cplusplus extern "C" { #endif @@ -71,7 +72,7 @@ typedef struct SIndexMeta { } SIndexMeta; typedef struct STbVerInfo { - char tbFName[TSDB_TABLE_FNAME_LEN]; + char tbFName[TSDB_TABLE_FNAME_LEN]; int32_t sversion; int32_t tversion; } STbVerInfo; @@ -141,7 +142,7 @@ typedef struct SDataBuf { typedef struct STargetInfo { ETargetType type; - char* dbFName; // used to update db's vgroup epset + char* dbFName; // used to update db's vgroup epset int32_t vgId; } STargetInfo; @@ -149,15 +150,15 @@ typedef int32_t (*__async_send_cb_fn_t)(void* param, const SDataBuf* pMsg, int32 typedef int32_t (*__async_exec_fn_t)(void* param); typedef struct SRequestConnInfo { - void* pTrans; - uint64_t requestId; - int64_t requestObjRefId; - SEpSet mgmtEps; + void* pTrans; + uint64_t requestId; + int64_t requestObjRefId; + SEpSet mgmtEps; } SRequestConnInfo; typedef struct SMsgSendInfo { - __async_send_cb_fn_t fp; // async callback function - STargetInfo target; // for update epset + __async_send_cb_fn_t fp; // async callback function + STargetInfo target; // for update epset void* param; uint64_t requestId; uint64_t requestObjRefId; @@ -206,9 +207,10 @@ int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STabl char* jobTaskStatusStr(int32_t status); SSchema createSchema(int8_t type, int32_t bytes, col_id_t colId, const char* name); -void destroyQueryExecRes(SQueryExecRes* pRes); +void destroyQueryExecRes(SQueryExecRes* pRes); -extern int32_t (*queryBuildMsg[TDMT_MAX])(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void*(*mallocFp)(int32_t)); +extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char** msg, int32_t msgSize, int32_t* msgLen, + void* (*mallocFp)(int32_t)); extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t msgSize); #define SET_META_TYPE_NULL(t) (t) = META_TYPE_NULL_TABLE @@ -219,7 +221,7 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t #define NEED_CLIENT_RM_TBLMETA_ERROR(_code) \ ((_code) == TSDB_CODE_PAR_TABLE_NOT_EXIST || (_code) == TSDB_CODE_VND_TB_NOT_EXIST || \ (_code) == TSDB_CODE_PAR_INVALID_COLUMNS_NUM || (_code) == TSDB_CODE_PAR_INVALID_COLUMN || \ - (_code) == TSDB_CODE_PAR_TAGS_NOT_MATCHED || (_code) == TSDB_CODE_PAR_VALUE_TOO_LONG || \ + (_code) == TSDB_CODE_PAR_TAGS_NOT_MATCHED || (_code) == TSDB_CODE_PAR_VALUE_TOO_LONG || \ (_code) == TSDB_CODE_PAR_INVALID_DROP_COL || ((_code) == TSDB_CODE_TDB_INVALID_TABLE_ID)) #define NEED_CLIENT_REFRESH_VG_ERROR(_code) \ ((_code) == TSDB_CODE_VND_HASH_MISMATCH || (_code) == TSDB_CODE_VND_INVALID_VGROUP_ID) @@ -227,11 +229,13 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t #define NEED_CLIENT_HANDLE_ERROR(_code) \ (NEED_CLIENT_RM_TBLMETA_ERROR(_code) || NEED_CLIENT_REFRESH_VG_ERROR(_code) || \ NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code)) -#define NEED_CLIENT_RM_TBLMETA_REQ(_type) ((_type) == TDMT_VND_CREATE_TABLE || (_type) == TDMT_VND_CREATE_STB \ - || (_type) == TDMT_VND_DROP_TABLE || (_type) == TDMT_VND_DROP_STB) +#define NEED_CLIENT_RM_TBLMETA_REQ(_type) \ + ((_type) == TDMT_VND_CREATE_TABLE || (_type) == TDMT_VND_CREATE_STB || (_type) == TDMT_VND_DROP_TABLE || \ + (_type) == TDMT_VND_DROP_STB) -#define NEED_SCHEDULER_RETRY_ERROR(_code) \ - ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL || (_code) == TSDB_CODE_SCH_TIMEOUT_ERROR) +#define NEED_SCHEDULER_RETRY_ERROR(_code) \ + ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL || \ + (_code) == TSDB_CODE_SCH_TIMEOUT_ERROR) #define REQUEST_TOTAL_EXEC_TIMES 2 @@ -308,3 +312,4 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t #endif #endif /*_TD_QUERY_H_*/ + // clang-foramt on diff --git a/include/os/osSocket.h b/include/os/osSocket.h index 213a6930ee..9dd5b972fa 100644 --- a/include/os/osSocket.h +++ b/include/os/osSocket.h @@ -157,7 +157,10 @@ int32_t taosNonblockwrite(TdSocketPtr pSocket, char *ptr, int32_t nbytes); int64_t taosCopyFds(TdSocketPtr pSrcSocket, TdSocketPtr pDestSocket, int64_t len); void taosWinSocketInit(); -int taosCreateSocketWithTimeOutOpt(uint32_t conn_timeout_sec); +/* + * set timeout(ms) + */ +int32_t taosCreateSocketWithTimeout(uint32_t timeout); TdSocketPtr taosOpenUdpSocket(uint32_t localIp, uint16_t localPort); TdSocketPtr taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp); diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 8920922006..14649c9fd4 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -59,7 +59,7 @@ static STscObj* taosConnectImpl(const char* user, const char* auth, const char* SAppInstInfo* pAppInfo, int connType); STscObj* taos_connect_internal(const char* ip, const char* user, const char* pass, const char* auth, const char* db, - uint16_t port, int connType) { + uint16_t port, int connType) { if (taos_init() != TSDB_CODE_SUCCESS) { return NULL; } @@ -313,8 +313,8 @@ bool qnodeRequired(SRequestObj* pRequest) { } SAppInstInfo* pInfo = pRequest->pTscObj->pAppInfo; - bool required = false; - + bool required = false; + taosThreadMutexLock(&pInfo->qnodeMutex); required = (NULL == pInfo->pQnodeList); taosThreadMutexUnlock(&pInfo->qnodeMutex); @@ -419,11 +419,11 @@ int32_t buildVnodePolicyNodeList(SRequestObj* pRequest, SArray** pNodeList, SArr } for (int32_t j = 0; j < vgNum; ++j) { - SVgroupInfo* pInfo = taosArrayGet(pVg, j); + SVgroupInfo* pInfo = taosArrayGet(pVg, j); SQueryNodeLoad load = {0}; load.addr.nodeId = pInfo->vgId; load.addr.epSet = pInfo->epSet; - + taosArrayPush(nodeList, &load); } } @@ -481,17 +481,16 @@ _return: return TSDB_CODE_SUCCESS; } - -int32_t buildAsyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray* pMnodeList, SMetaData *pResultMeta) { +int32_t buildAsyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray* pMnodeList, SMetaData* pResultMeta) { SArray* pDbVgList = NULL; SArray* pQnodeList = NULL; int32_t code = 0; - + switch (tsQueryPolicy) { case QUERY_POLICY_VNODE: { if (pResultMeta) { pDbVgList = taosArrayInit(4, POINTER_BYTES); - + int32_t dbNum = taosArrayGetSize(pResultMeta->pDbVgroup); for (int32_t i = 0; i < dbNum; ++i) { SMetaRes* pRes = taosArrayGet(pResultMeta->pDbVgroup, i); @@ -500,9 +499,9 @@ int32_t buildAsyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray } taosArrayPush(pDbVgList, &pRes->pRes); - } + } } - + code = buildVnodePolicyNodeList(pRequest, pNodeList, pMnodeList, pDbVgList); break; } @@ -523,7 +522,7 @@ int32_t buildAsyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray } taosThreadMutexUnlock(&pInst->qnodeMutex); } - + code = buildQnodePolicyNodeList(pRequest, pNodeList, pMnodeList, pQnodeList); break; } @@ -534,7 +533,7 @@ int32_t buildAsyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray taosArrayDestroy(pDbVgList); taosArrayDestroy(pQnodeList); - + return code; } @@ -542,43 +541,43 @@ int32_t buildSyncExecNodeList(SRequestObj* pRequest, SArray** pNodeList, SArray* SArray* pDbVgList = NULL; SArray* pQnodeList = NULL; int32_t code = 0; - + switch (tsQueryPolicy) { case QUERY_POLICY_VNODE: { int32_t dbNum = taosArrayGetSize(pRequest->dbList); if (dbNum > 0) { - SCatalog* pCtg = NULL; + SCatalog* pCtg = NULL; SAppInstInfo* pInst = pRequest->pTscObj->pAppInfo; code = catalogGetHandle(pInst->clusterId, &pCtg); if (code != TSDB_CODE_SUCCESS) { goto _return; } - pDbVgList = taosArrayInit(dbNum, POINTER_BYTES); + pDbVgList = taosArrayInit(dbNum, POINTER_BYTES); SArray* pVgList = NULL; for (int32_t i = 0; i < dbNum; ++i) { - char* dbFName = taosArrayGet(pRequest->dbList, i); + char* dbFName = taosArrayGet(pRequest->dbList, i); SRequestConnInfo conn = {.pTrans = pInst->pTransporter, .requestId = pRequest->requestId, .requestObjRefId = pRequest->self, - .mgmtEps = getEpSet_s(&pInst->mgmtEp)}; - + .mgmtEps = getEpSet_s(&pInst->mgmtEp)}; + code = catalogGetDBVgInfo(pCtg, &conn, dbFName, &pVgList); if (code) { goto _return; } - + taosArrayPush(pDbVgList, &pVgList); - } + } } - + code = buildVnodePolicyNodeList(pRequest, pNodeList, pMnodeList, pDbVgList); break; } case QUERY_POLICY_HYBRID: case QUERY_POLICY_QNODE: { getQnodeList(pRequest, &pQnodeList); - + code = buildQnodePolicyNodeList(pRequest, pNodeList, pMnodeList, pQnodeList); break; } @@ -591,11 +590,10 @@ _return: taosArrayDestroy(pDbVgList); taosArrayDestroy(pQnodeList); - + return code; } - int32_t scheduleAsyncQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList) { tsem_init(&schdRspSem, 0, 0); @@ -604,12 +602,12 @@ int32_t scheduleAsyncQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNod .requestId = pRequest->requestId, .requestObjRefId = pRequest->self}; SSchedulerReq req = {.pConn = &conn, - .pNodeList = pNodeList, - .pDag = pDag, - .sql = pRequest->sqlstr, - .startTs = pRequest->metric.start, - .fp = schdExecCallback, - .cbParam = &res}; + .pNodeList = pNodeList, + .pDag = pDag, + .sql = pRequest->sqlstr, + .startTs = pRequest->metric.start, + .fp = schdExecCallback, + .cbParam = &res}; int32_t code = schedulerAsyncExecJob(&req, &pRequest->body.queryJob); @@ -656,13 +654,13 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList .requestId = pRequest->requestId, .requestObjRefId = pRequest->self}; SSchedulerReq req = {.pConn = &conn, - .pNodeList = pNodeList, - .pDag = pDag, - .sql = pRequest->sqlstr, - .startTs = pRequest->metric.start, - .fp = NULL, - .cbParam = NULL, - .reqKilled = &pRequest->killed}; + .pNodeList = pNodeList, + .pDag = pDag, + .sql = pRequest->sqlstr, + .startTs = pRequest->metric.start, + .fp = NULL, + .cbParam = NULL, + .reqKilled = &pRequest->killed}; int32_t code = schedulerExecJob(&req, &pRequest->body.queryJob, &res); pRequest->body.resInfo.execRes = res.res; @@ -819,8 +817,8 @@ void schedulerExecCb(SQueryResult* pResult, void* param, int32_t code) { } } - tscDebug("0x%" PRIx64 " enter scheduler exec cb, code:%d - %s, reqId:0x%" PRIx64, - pRequest->self, code, tstrerror(code), pRequest->requestId); + tscDebug("0x%" PRIx64 " enter scheduler exec cb, code:%d - %s, reqId:0x%" PRIx64, pRequest->self, code, + tstrerror(code), pRequest->requestId); STscObj* pTscObj = pRequest->pTscObj; if (code != TSDB_CODE_SUCCESS && NEED_CLIENT_HANDLE_ERROR(code)) { @@ -862,7 +860,7 @@ SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQue if (TSDB_CODE_SUCCESS == code) { SArray* pNodeList = NULL; buildSyncExecNodeList(pRequest, &pNodeList, pMnodeList); - + code = scheduleQuery(pRequest, pRequest->body.pDag, pNodeList); taosArrayDestroy(pNodeList); } @@ -915,7 +913,7 @@ SRequestObj* launchQuery(STscObj* pTscObj, const char* sql, int sqlLen) { return launchQueryImpl(pRequest, pQuery, false, NULL); } -void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData *pResultMeta) { +void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultMeta) { int32_t code = 0; switch (pQuery->execMode) { @@ -948,7 +946,7 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData *pResultM if (TSDB_CODE_SUCCESS == code) { SArray* pNodeList = NULL; buildAsyncExecNodeList(pRequest, &pNodeList, pMnodeList, pResultMeta); - + SRequestConnInfo conn = { .pTrans = pAppInfo->pTransporter, .requestId = pRequest->requestId, .requestObjRefId = pRequest->self}; SSchedulerReq req = {.pConn = &conn, @@ -1308,7 +1306,7 @@ TAOS* taos_connect_auth(const char* ip, const char* user, const char* auth, cons if (pObj) { return pObj->id; } - + return NULL; } @@ -1554,10 +1552,10 @@ static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int return TSDB_CODE_SUCCESS; } -static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, int32_t numOfRows){ +static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, int32_t numOfRows) { char* p = (char*)pResultInfo->pData; - int32_t len = sizeof(int32_t) + sizeof(uint64_t) + numOfCols * (sizeof(int16_t) + sizeof(int32_t)); + int32_t len = sizeof(int32_t) + sizeof(uint64_t) + numOfCols * (sizeof(int16_t) + sizeof(int32_t)); int32_t* colLength = (int32_t*)(p + len); len += sizeof(int32_t) * numOfCols; @@ -1567,7 +1565,7 @@ static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, i if (pResultInfo->fields[i].type == TSDB_DATA_TYPE_JSON) { int32_t* offset = (int32_t*)pStart; - int32_t lenTmp = numOfRows * sizeof(int32_t); + int32_t lenTmp = numOfRows * sizeof(int32_t); len += lenTmp; pStart += lenTmp; @@ -1592,7 +1590,6 @@ static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, i } else { ASSERT(0); } - } } else if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) { int32_t lenTmp = numOfRows * sizeof(int32_t); @@ -1616,13 +1613,13 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int break; } } - if(!needConvert) return TSDB_CODE_SUCCESS; + if (!needConvert) return TSDB_CODE_SUCCESS; - char* p = (char*)pResultInfo->pData; + char* p = (char*)pResultInfo->pData; int32_t dataLen = estimateJsonLen(pResultInfo, numOfCols, numOfRows); pResultInfo->convertJson = taosMemoryCalloc(1, dataLen); - if(pResultInfo->convertJson == NULL) return TSDB_CODE_OUT_OF_MEMORY; + if (pResultInfo->convertJson == NULL) return TSDB_CODE_OUT_OF_MEMORY; char* p1 = pResultInfo->convertJson; int32_t len = sizeof(int32_t) + sizeof(uint64_t) + numOfCols * (sizeof(int16_t) + sizeof(int32_t)); @@ -1691,7 +1688,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int ASSERT(0); } - offset1[j]= len; + offset1[j] = len; memcpy(pStart1 + len, dst, varDataTLen(dst)); len += varDataTLen(dst); } @@ -1709,7 +1706,6 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int pStart += len; pStart1 += len; memcpy(pStart1, pStart, colLen); - } pStart += colLen; pStart1 += colLen1; @@ -1777,7 +1773,7 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32 pStart += colLength[i]; } - if(convertUcs4){ + if (convertUcs4) { code = doConvertUCS4(pResultInfo, numOfRows, numOfCols, colLength); } diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index 2120d24d26..c92ab9b008 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -19,7 +19,7 @@ #include "tmsg.h" #include "trpc.h" #include "tsched.h" - +// clang-format off #define VALIDNUMOFCOLS(x) ((x) >= TSDB_MIN_COLUMNS && (x) <= TSDB_MAX_COLUMNS) #define VALIDNUMOFTAGS(x) ((x) >= 0 && (x) <= TSDB_MAX_TAGS) @@ -146,13 +146,15 @@ int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTra } memcpy(pMsg, pInfo->msgInfo.pData, pInfo->msgInfo.len); - SRpcMsg rpcMsg = {.msgType = pInfo->msgType, - .pCont = pMsg, - .contLen = pInfo->msgInfo.len, - .info.ahandle = (void*)pInfo, - .info.handle = pInfo->msgInfo.handle, - .info.persistHandle = persistHandle, - .code = 0}; + SRpcMsg rpcMsg = { + .msgType = pInfo->msgType, + .pCont = pMsg, + .contLen = pInfo->msgInfo.len, + .info.ahandle = (void*)pInfo, + .info.handle = pInfo->msgInfo.handle, + .info.persistHandle = persistHandle, + .code = 0 + }; assert(pInfo->fp != NULL); TRACE_SET_ROOTID(&rpcMsg.info.traceId, pInfo->requestId); rpcSendRequestWithCtx(pTransporter, epSet, &rpcMsg, pTransporterId, rpcCtx); @@ -220,3 +222,4 @@ void destroyQueryExecRes(SQueryExecRes* pRes) { qError("invalid exec result for request type %d", pRes->msgType); } } +// clang-format on diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 327fe50814..158926c520 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -238,6 +238,32 @@ int transSendAsync(SAsyncPool* pool, queue* mq); } \ } \ } while (0) + +#define ASYNC_CHECK_HANDLE(exh1, refId) \ + do { \ + if (refId > 0) { \ + tTrace("handle step1"); \ + SExHandle* exh2 = transAcquireExHandle(refMgt, refId); \ + if (exh2 == NULL || refId != exh2->refId) { \ + tTrace("handle %p except, may already freed, ignore msg, ref1: %" PRIu64 ", ref2 : %" PRIu64 "", exh1, \ + exh2 ? exh2->refId : 0, refId); \ + goto _return1; \ + } \ + } else if (refId == 0) { \ + tTrace("handle step2"); \ + SExHandle* exh2 = transAcquireExHandle(refMgt, refId); \ + if (exh2 == NULL || refId != exh2->refId) { \ + tTrace("handle %p except, may already freed, ignore msg, ref1: %" PRIu64 ", ref2 : %" PRIu64 "", exh1, refId, \ + exh2 ? exh2->refId : 0); \ + goto _return1; \ + } else { \ + refId = exh1->refId; \ + } \ + } else if (refId < 0) { \ + tTrace("handle step3"); \ + goto _return2; \ + } \ + } while (0) int transInitBuffer(SConnBuffer* buf); int transClearBuffer(SConnBuffer* buf); int transDestroyBuffer(SConnBuffer* buf); diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 852ffc9a0e..402a26247a 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -25,7 +25,6 @@ typedef struct SCliConn { uv_write_t writeReq; void* hostThrd; - int hThrdIdx; SConnBuffer readBuf; STransQueue cliMsgs; @@ -36,6 +35,7 @@ typedef struct SCliConn { bool broken; // link broken or not ConnStatus status; // + int64_t refId; char* ip; uint32_t port; @@ -168,16 +168,24 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { snprintf(key, sizeof(key), "%s:%d", ip, (int)port); \ } while (0) -#define CONN_HOST_THREAD_IDX(conn) (conn ? ((SCliConn*)conn)->hThrdIdx : -1) +#define CONN_HOST_THREAD_IDX1(idx, exh, refId, pThrd) \ + do { \ + if (exh == NULL) { \ + idx = -1; \ + } else { \ + ASYNC_CHECK_HANDLE(exh, refId); \ + pThrd = (SCliThrdObj*)exh->pThrd; \ + } \ + } while (0) #define CONN_PERSIST_TIME(para) (para * 1000 * 10) #define CONN_GET_HOST_THREAD(conn) (conn ? ((SCliConn*)conn)->hostThrd : NULL) #define CONN_GET_INST_LABEL(conn) (((STrans*)(((SCliThrdObj*)(conn)->hostThrd)->pTransInst))->label) #define CONN_SHOULD_RELEASE(conn, head) \ do { \ if ((head)->release == 1 && (head->msgLen) == sizeof(*head)) { \ + int status = conn->status; \ uint64_t ahandle = head->ahandle; \ CONN_GET_MSGCTX_BY_AHANDLE(conn, ahandle); \ - conn->status = ConnRelease; \ transClearBuffer(&conn->readBuf); \ transFreeMsg(transContFromHead((char*)head)); \ tDebug("%s conn %p receive release request, ref: %d", CONN_GET_INST_LABEL(conn), conn, T_REF_VAL_GET(conn)); \ @@ -186,7 +194,9 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { } \ destroyCmsg(pMsg); \ cliReleaseUnfinishedMsg(conn); \ - addConnToPool(((SCliThrdObj*)conn->hostThrd)->pool, conn); \ + if (status != ConnInPool) { \ + addConnToPool(((SCliThrdObj*)conn->hostThrd)->pool, conn); \ + } \ return; \ } \ } while (0) @@ -323,23 +333,29 @@ void cliHandleResp(SCliConn* conn) { transClearBuffer(&conn->readBuf); if (!CONN_NO_PERSIST_BY_APP(conn)) { - transMsg.info.handle = conn; + SExHandle* exh = taosMemoryCalloc(1, sizeof(SExHandle)); + exh->handle = conn; + exh->pThrd = pThrd; + exh->refId = transAddExHandle(refMgt, exh); + + transMsg.info.handle = exh; + transMsg.info.refId = exh->refId; + conn->refId = exh->refId; tDebug("%s conn %p ref by app", CONN_GET_INST_LABEL(conn), conn); } - // char buf[64] = {0}; - // TRACE_TO_STR(&transMsg.info.traceId, buf); + STraceId* trace = &transMsg.info.traceId; - tGTrace("conn %p %s received from %s:%d, local info: %s:%d, msg size: %d, code: %d", conn, TMSG_INFO(pHead->msgType), - taosInetNtoa(conn->addr.sin_addr), ntohs(conn->addr.sin_port), taosInetNtoa(conn->localAddr.sin_addr), - ntohs(conn->localAddr.sin_port), transMsg.contLen, transMsg.code); + tGTrace("%s conn %p %s received from %s:%d, local info: %s:%d, msg size: %d, code: %d", CONN_GET_INST_LABEL(conn), + conn, TMSG_INFO(pHead->msgType), taosInetNtoa(conn->addr.sin_addr), ntohs(conn->addr.sin_port), + taosInetNtoa(conn->localAddr.sin_addr), ntohs(conn->localAddr.sin_port), transMsg.contLen, transMsg.code); if (pCtx == NULL && CONN_NO_PERSIST_BY_APP(conn)) { - tDebug("%s except, server continue send while cli ignore it", CONN_GET_INST_LABEL(conn)); + tDebug("%s except, conn %p read while cli ignore it", CONN_GET_INST_LABEL(conn), conn); // transUnrefCliHandle(conn); return; } if (CONN_RELEASE_BY_SERVER(conn) && transMsg.info.ahandle == NULL) { - tDebug("%s except, server continue send while cli ignore it", CONN_GET_INST_LABEL(conn)); + tDebug("%s except, conn %p read while cli ignore it", CONN_GET_INST_LABEL(conn), conn); // transUnrefCliHandle(conn); return; } @@ -476,10 +492,9 @@ static SCliConn* getConnFromPool(void* pool, char* ip, uint32_t port) { if (QUEUE_IS_EMPTY(&plist->conn)) { return NULL; } - queue* h = QUEUE_HEAD(&plist->conn); - // //QUEUE_REMOVE(h); + queue* h = QUEUE_HEAD(&plist->conn); SCliConn* conn = QUEUE_DATA(h, SCliConn, conn); - // conn->status = ConnNormal; + conn->status = ConnNormal; QUEUE_REMOVE(&conn->conn); QUEUE_INIT(&conn->conn); return conn; @@ -559,6 +574,13 @@ static SCliConn* cliCreateConn(SCliThrdObj* pThrd) { conn->status = ConnNormal; conn->broken = 0; transRefCliHandle(conn); + + SExHandle* exh = taosMemoryCalloc(1, sizeof(SExHandle)); + exh->handle = conn; + exh->pThrd = pThrd; + exh->refId = transAddExHandle(refMgt, exh); + conn->refId = exh->refId; + return conn; } static void cliDestroyConn(SCliConn* conn, bool clear) { @@ -566,6 +588,7 @@ static void cliDestroyConn(SCliConn* conn, bool clear) { QUEUE_REMOVE(&conn->conn); QUEUE_INIT(&conn->conn); + transRemoveExHandle(refMgt, conn->refId); if (clear) { uv_close((uv_handle_t*)conn->stream, cliDestroy); } @@ -650,12 +673,10 @@ void cliSend(SCliConn* pConn) { uv_buf_t wb = uv_buf_init((char*)pHead, msgLen); - // char buf[64] = {0}; - // TRACE_TO_STR(&pMsg->info.traceId, buf); STraceId* trace = &pMsg->info.traceId; - tGTrace("conn %p %s is sent to %s:%d, local info %s:%d", pConn, TMSG_INFO(pHead->msgType), - taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), taosInetNtoa(pConn->localAddr.sin_addr), - ntohs(pConn->localAddr.sin_port)); + tGTrace("%s conn %p %s is sent to %s:%d, local info %s:%d", CONN_GET_INST_LABEL(pConn), pConn, + TMSG_INFO(pHead->msgType), taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), + taosInetNtoa(pConn->localAddr.sin_addr), ntohs(pConn->localAddr.sin_port)); if (pHead->persist == 1) { CONN_SET_PERSIST_BY_APP(pConn); @@ -663,7 +684,6 @@ void cliSend(SCliConn* pConn) { pConn->writeReq.data = pConn; uv_write(&pConn->writeReq, (uv_stream_t*)pConn->stream, &wb, 1, cliSendCb); - return; _RETURN: return; @@ -723,20 +743,32 @@ static void cliHandleUpdate(SCliMsg* pMsg, SCliThrdObj* pThrd) { } SCliConn* cliGetConn(SCliMsg* pMsg, SCliThrdObj* pThrd) { - SCliConn* conn = NULL; - if (pMsg->msg.info.handle != NULL) { - conn = (SCliConn*)(pMsg->msg.info.handle); - if (conn != NULL) { - tTrace("%s conn %p reused", CONN_GET_INST_LABEL(conn), conn); + SCliConn* conn = NULL; + SRpcHandleInfo* pInfo = &pMsg->msg.info; + + SExHandle* exh = transAcquireExHandle(refMgt, pInfo->refId); + if (exh == NULL) { + if (pInfo->refId != 0) { + tTrace("%s conn %p ignore msg", CONN_GET_INST_LABEL(conn), conn); + assert(0); + return NULL; } } else { - STransConnCtx* pCtx = pMsg->ctx; - conn = getConnFromPool(pThrd->pool, EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet)); - if (conn != NULL) { - tTrace("%s conn %p get from conn pool", CONN_GET_INST_LABEL(conn), conn); - } else { - tTrace("%s not found conn in conn pool %p", ((STrans*)pThrd->pTransInst)->label, pThrd->pool); - } + transReleaseExHandle(refMgt, pInfo->refId); + return exh->handle; + } + + STransConnCtx* pCtx = pMsg->ctx; + conn = getConnFromPool(pThrd->pool, EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet)); + if (conn != NULL) { + exh = taosMemoryCalloc(1, sizeof(SExHandle)); + exh->handle = conn; + exh->pThrd = pThrd; + exh->refId = transAddExHandle(refMgt, exh); + conn->refId = exh->refId; + tTrace("%s conn %p get from conn pool", CONN_GET_INST_LABEL(conn), conn); + } else { + tTrace("%s not found conn in conn pool %p", ((STrans*)pThrd->pTransInst)->label, pThrd->pool); } return conn; } @@ -765,8 +797,6 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { SCliConn* conn = cliGetConn(pMsg, pThrd); if (conn != NULL) { - conn->hThrdIdx = pCtx->hThrdIdx; - transCtxMerge(&conn->ctx, &pCtx->appCtx); transQueuePush(&conn->cliMsgs, pMsg); cliSend(conn); @@ -775,7 +805,6 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { transCtxMerge(&conn->ctx, &pCtx->appCtx); transQueuePush(&conn->cliMsgs, pMsg); - conn->hThrdIdx = pCtx->hThrdIdx; conn->ip = strdup(EPSET_GET_INUSE_IP(&pCtx->epSet)); conn->port = EPSET_GET_INUSE_PORT(&pCtx->epSet); @@ -783,7 +812,7 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { if (ret) { tError("%s conn %p failed to set conn option, errmsg %s", transLabel(pTransInst), conn, uv_err_name(ret)); } - int fd = taosCreateSocketWithTimeOutOpt(TRANS_CONN_TIMEOUT); + int32_t fd = taosCreateSocketWithTimeout(TRANS_CONN_TIMEOUT); if (fd == -1) { tTrace("%s conn %p failed to create socket", transLabel(pTransInst), conn); cliHandleExcept(conn); @@ -1009,7 +1038,9 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { tTrace("%s use remote epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse, pCtx->retryCount + 1, TRANS_RETRY_COUNT_LIMIT); } - addConnToPool(pThrd->pool, pConn); + if (pConn->status != ConnInPool) { + addConnToPool(pThrd->pool, pConn); + } STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); arg->param1 = pMsg; @@ -1086,10 +1117,21 @@ void transReleaseCliHandle(void* handle) { } void transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransCtx* ctx) { - STrans* pTransInst = (STrans*)shandle; - int idx = CONN_HOST_THREAD_IDX((SCliConn*)pReq->info.handle); + STrans* pTransInst = (STrans*)shandle; + SRpcHandleInfo* info = &pReq->info; + + int idx = -1; + SCliThrdObj* pThrd = NULL; + SExHandle* exh = info->handle; + int64_t refId = -1; + if (exh != NULL) { + refId = exh->refId; + } + + CONN_HOST_THREAD_IDX1(idx, exh, refId, pThrd); if (idx == -1) { idx = cliRBChoseIdx(pTransInst); + pThrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[idx]; } TRACE_SET_MSGID(&pReq->info.traceId, tGenIdPI64()); @@ -1097,7 +1139,6 @@ void transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STra pCtx->epSet = *pEpSet; pCtx->ahandle = pReq->info.ahandle; pCtx->msgType = pReq->msgType; - pCtx->hThrdIdx = idx; if (ctx != NULL) { pCtx->appCtx = *ctx; @@ -1110,19 +1151,31 @@ void transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STra cliMsg->st = taosGetTimestampUs(); cliMsg->type = Normal; - SCliThrdObj* thrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[idx]; - STraceId* trace = &pReq->info.traceId; - tGTrace("%s send request at thread:%08" PRId64 ", dst: %s:%d, app:%p", transLabel(pTransInst), thrd->pid, + tGTrace("%s send request at thread:%08" PRId64 ", dst: %s:%d, app:%p", transLabel(pTransInst), pThrd->pid, EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet), pReq->info.ahandle); - ASSERT(transSendAsync(thrd->asyncPool, &(cliMsg->q)) == 0); + ASSERT(transSendAsync(pThrd->asyncPool, &(cliMsg->q)) == 0); +_return1: + return; +_return2: + return; } void transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp) { - STrans* pTransInst = (STrans*)shandle; - int idx = CONN_HOST_THREAD_IDX(pReq->info.handle); + STrans* pTransInst = (STrans*)shandle; + SRpcHandleInfo* info = &pReq->info; + SCliThrdObj* pThrd = NULL; + int idx = -1; + SExHandle* exh = info->handle; + int64_t refId = -1; + if (exh != NULL) { + refId = exh->refId; + } + + CONN_HOST_THREAD_IDX1(idx, exh, refId, pThrd); if (idx == -1) { idx = cliRBChoseIdx(pTransInst); + pThrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[idx]; } tsem_t* sem = taosMemoryCalloc(1, sizeof(tsem_t)); tsem_init(sem, 0, 0); @@ -1133,7 +1186,6 @@ void transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransM pCtx->epSet = *pEpSet; pCtx->ahandle = pReq->info.ahandle; pCtx->msgType = pReq->msgType; - pCtx->hThrdIdx = idx; pCtx->pSem = sem; pCtx->pRsp = pRsp; @@ -1143,16 +1195,18 @@ void transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransM cliMsg->st = taosGetTimestampUs(); cliMsg->type = Normal; - SCliThrdObj* thrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[idx]; - STraceId* trace = &pReq->info.traceId; - tGTrace("%s send request at thread:%08" PRId64 ", dst: %s:%d, app:%p", transLabel(pTransInst), thrd->pid, + tGTrace("%s send request at thread:%08" PRId64 ", dst: %s:%d, app:%p", transLabel(pTransInst), pThrd->pid, EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet), pReq->info.ahandle); - transSendAsync(thrd->asyncPool, &(cliMsg->q)); + transSendAsync(pThrd->asyncPool, &(cliMsg->q)); tsem_wait(sem); tsem_destroy(sem); taosMemoryFree(sem); +_return1: + return; +_return2: + return; } /* * @@ -1168,7 +1222,6 @@ void transSetDefaultAddr(void* ahandle, const char* ip, const char* fqdn) { } for (int i = 0; i < pTransInst->numOfThreads; i++) { STransConnCtx* pCtx = taosMemoryCalloc(1, sizeof(STransConnCtx)); - pCtx->hThrdIdx = i; pCtx->cvtAddr = cvtAddr; SCliMsg* cliMsg = taosMemoryCalloc(1, sizeof(SCliMsg)); diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 8cd7f9d827..5d342dd174 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -455,7 +455,7 @@ void transPrintEpSet(SEpSet* pEpSet) { return; } char buf[512] = {0}; - int len = snprintf(buf, sizeof(buf), "epset { "); + int len = snprintf(buf, sizeof(buf), "epset:{ "); for (int i = 0; i < pEpSet->numOfEps; i++) { if (i == pEpSet->numOfEps - 1) { len += snprintf(buf + len, sizeof(buf) - len, "%d. %s:%d ", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 593a790a21..121fddc99a 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -206,32 +206,6 @@ static bool addHandleToAcceptloop(void* arg); } \ } while (0) -#define ASYNC_CHECK_HANDLE(exh1, refId) \ - do { \ - if (refId > 0) { \ - tTrace("handle step1"); \ - SExHandle* exh2 = transAcquireExHandle(refMgt, refId); \ - if (exh2 == NULL || refId != exh2->refId) { \ - tTrace("handle %p except, may already freed, ignore msg, ref1: %" PRIu64 ", ref2 : %" PRIu64 "", exh1, \ - exh2 ? exh2->refId : 0, refId); \ - goto _return1; \ - } \ - } else if (refId == 0) { \ - tTrace("handle step2"); \ - SExHandle* exh2 = transAcquireExHandle(refMgt, refId); \ - if (exh2 == NULL || refId != exh2->refId) { \ - tTrace("handle %p except, may already freed, ignore msg, ref1: %" PRIu64 ", ref2 : %" PRIu64 "", exh1, refId, \ - exh2 ? exh2->refId : 0); \ - goto _return1; \ - } else { \ - refId = exh1->refId; \ - } \ - } else if (refId < 0) { \ - tTrace("handle step3"); \ - goto _return2; \ - } \ - } while (0) - void uvAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { SSvrConn* conn = handle->data; SConnBuffer* pBuf = &conn->readBuf; diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index 4d61e7036d..b0e07ff010 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -946,7 +946,7 @@ int32_t taosGetFqdn(char *fqdn) { #endif // __APPLE__ int32_t ret = getaddrinfo(hostname, NULL, &hints, &result); if (!result) { - fprintf(stderr,"failed to get fqdn, code:%d, reason:%s", ret, gai_strerror(ret)); + fprintf(stderr, "failed to get fqdn, code:%d, reason:%s", ret, gai_strerror(ret)); return -1; } @@ -1073,7 +1073,7 @@ int32_t taosCloseEpoll(TdEpollPtr *ppEpoll) { * Set TCP connection timeout per-socket level. * ref [https://github.com/libuv/help/issues/54] */ -int taosCreateSocketWithTimeOutOpt(uint32_t conn_timeout_sec) { +int32_t taosCreateSocketWithTimeout(uint32_t timeout) { #if defined(WINDOWS) SOCKET fd; #else @@ -1083,11 +1083,11 @@ int taosCreateSocketWithTimeOutOpt(uint32_t conn_timeout_sec) { return -1; } #if defined(WINDOWS) - if (0 != setsockopt(fd, IPPROTO_TCP, TCP_MAXRT, (char *)&conn_timeout_sec, sizeof(conn_timeout_sec))) { + if (0 != setsockopt(fd, IPPROTO_TCP, TCP_MAXRT, (char *)&timeout, sizeof(timeout))) { return -1; } #else // Linux like systems - uint32_t conn_timeout_ms = conn_timeout_sec * 1000; + uint32_t conn_timeout_ms = timeout * 1000; if (0 != setsockopt(fd, IPPROTO_TCP, TCP_USER_TIMEOUT, (char *)&conn_timeout_ms, sizeof(conn_timeout_ms))) { return -1; } From bfe5ab779b556372d418cd78f846e2e867c8ec9e Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 22 Jun 2022 19:20:35 +0800 Subject: [PATCH 02/49] fix: handle except --- source/libs/transport/src/transCli.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 402a26247a..3542333561 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -18,6 +18,10 @@ static int32_t transSCliInst = 0; static int32_t refMgt = 0; +typedef struct SExHandleWrap { + void* exhandle; + int64_t refId; +} SExHandleWrap; typedef struct SCliConn { T_REF_DECLARE() uv_connect_t connReq; @@ -338,9 +342,11 @@ void cliHandleResp(SCliConn* conn) { exh->pThrd = pThrd; exh->refId = transAddExHandle(refMgt, exh); - transMsg.info.handle = exh; - transMsg.info.refId = exh->refId; + SExHandleWrap* wrap = taosMemoryCalloc(1, sizeof(SExHandleWrap)); + wrap->exhandle = exh; + wrap->refId = exh->refId; conn->refId = exh->refId; + transMsg.info.handle = wrap; tDebug("%s conn %p ref by app", CONN_GET_INST_LABEL(conn), conn); } @@ -1007,6 +1013,8 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { pMsg->sent = 0; tTrace("try to send req to next node"); pMsg->st = taosGetTimestampUs(); + + taosMemoryFree(pResp->info.handle); pCtx->retryCount += 1; if (pResp->code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { if (pCtx->retryCount < pEpSet->numOfEps * 3) { From d5f5c33c9c547c75dd9b3d031f9abb7840817d8e Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 23 Jun 2022 15:55:47 +0800 Subject: [PATCH 03/49] refactor code --- include/util/ttrace.h | 8 +- source/libs/transport/inc/transComm.h | 63 +++++----- source/libs/transport/src/transCli.c | 162 ++++++++++++-------------- source/libs/transport/src/transSvr.c | 10 +- 4 files changed, 116 insertions(+), 127 deletions(-) diff --git a/include/util/ttrace.h b/include/util/ttrace.h index 206cbbf28d..579768228a 100644 --- a/include/util/ttrace.h +++ b/include/util/ttrace.h @@ -45,9 +45,11 @@ typedef struct STraceId { #define TRACE_GET_MSGID(traceId) (traceId)->msgId -#define TRACE_TO_STR(traceId, buf) \ - do { \ - sprintf(buf, "0x%" PRIx64 ":0x%" PRIx64 "", traceId->rootId, traceId->msgId); \ +#define TRACE_TO_STR(traceId, buf) \ + do { \ + int64_t rootId = (traceId) != NULL ? (traceId)->rootId : 0; \ + int64_t msgId = (traceId) != NULL ? (traceId)->msgId : 0; \ + sprintf(buf, "0x%" PRIx64 ":0x%" PRIx64 "", rootId, msgId); \ } while (0) #ifdef __cplusplus diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 158926c520..5367f6b49d 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -105,6 +105,13 @@ typedef SRpcCtxVal STransCtxVal; typedef SRpcInfo STrans; typedef SRpcConnInfo STransHandleInfo; +// ref mgt +// handle +typedef struct SExHandle { + void* handle; + int64_t refId; + void* pThrd; +} SExHandle; /*convet from fqdn to ip */ typedef struct SCvtAddr { char ip[TSDB_FQDN_LEN]; @@ -239,30 +246,30 @@ int transSendAsync(SAsyncPool* pool, queue* mq); } \ } while (0) -#define ASYNC_CHECK_HANDLE(exh1, refId) \ - do { \ - if (refId > 0) { \ - tTrace("handle step1"); \ - SExHandle* exh2 = transAcquireExHandle(refMgt, refId); \ - if (exh2 == NULL || refId != exh2->refId) { \ - tTrace("handle %p except, may already freed, ignore msg, ref1: %" PRIu64 ", ref2 : %" PRIu64 "", exh1, \ - exh2 ? exh2->refId : 0, refId); \ - goto _return1; \ - } \ - } else if (refId == 0) { \ - tTrace("handle step2"); \ - SExHandle* exh2 = transAcquireExHandle(refMgt, refId); \ - if (exh2 == NULL || refId != exh2->refId) { \ - tTrace("handle %p except, may already freed, ignore msg, ref1: %" PRIu64 ", ref2 : %" PRIu64 "", exh1, refId, \ - exh2 ? exh2->refId : 0); \ - goto _return1; \ - } else { \ - refId = exh1->refId; \ - } \ - } else if (refId < 0) { \ - tTrace("handle step3"); \ - goto _return2; \ - } \ +#define ASYNC_CHECK_HANDLE(exh1, id) \ + do { \ + if (id > 0) { \ + tTrace("handle step1"); \ + SExHandle* exh2 = transAcquireExHandle(refMgt, id); \ + if (exh2 == NULL || id != exh2->refId) { \ + tTrace("handle %p except, may already freed, ignore msg, ref1: %" PRIu64 ", ref2 : %" PRIu64 "", exh1, \ + exh2 ? exh2->refId : 0, id); \ + goto _return1; \ + } \ + } else if (id == 0) { \ + tTrace("handle step2"); \ + SExHandle* exh2 = transAcquireExHandle(refMgt, id); \ + if (exh2 == NULL || id == exh2->refId) { \ + tTrace("handle %p except, may already freed, ignore msg, ref1: %" PRIu64 ", ref2 : %" PRIu64 "", exh1, id, \ + exh2 ? exh2->refId : 0); \ + goto _return1; \ + } else { \ + id = exh1->refId; \ + } \ + } else if (id < 0) { \ + tTrace("handle step3"); \ + goto _return2; \ + } \ } while (0) int transInitBuffer(SConnBuffer* buf); int transClearBuffer(SConnBuffer* buf); @@ -381,14 +388,6 @@ bool transEpSetIsEqual(SEpSet* a, SEpSet* b); */ void transThreadOnce(); -// ref mgt -// handle -typedef struct SExHandle { - void* handle; - int64_t refId; - void* pThrd; -} SExHandle; - void transInitEnv(); int32_t transOpenExHandleMgt(int size); void transCloseExHandleMgt(int32_t mgt); diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 3542333561..e18723d976 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -18,10 +18,6 @@ static int32_t transSCliInst = 0; static int32_t refMgt = 0; -typedef struct SExHandleWrap { - void* exhandle; - int64_t refId; -} SExHandleWrap; typedef struct SCliConn { T_REF_DECLARE() uv_connect_t connReq; @@ -177,8 +173,8 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { if (exh == NULL) { \ idx = -1; \ } else { \ - ASYNC_CHECK_HANDLE(exh, refId); \ - pThrd = (SCliThrdObj*)exh->pThrd; \ + ASYNC_CHECK_HANDLE((exh), refId); \ + pThrd = (SCliThrdObj*)(exh)->pThrd; \ } \ } while (0) #define CONN_PERSIST_TIME(para) (para * 1000 * 10) @@ -201,6 +197,7 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { if (status != ConnInPool) { \ addConnToPool(((SCliThrdObj*)conn->hostThrd)->pool, conn); \ } \ + transRemoveExHandle(refMgt, conn->refId); \ return; \ } \ } while (0) @@ -335,18 +332,8 @@ void cliHandleResp(SCliConn* conn) { } // buf's mem alread translated to transMsg.pCont transClearBuffer(&conn->readBuf); - if (!CONN_NO_PERSIST_BY_APP(conn)) { - SExHandle* exh = taosMemoryCalloc(1, sizeof(SExHandle)); - exh->handle = conn; - exh->pThrd = pThrd; - exh->refId = transAddExHandle(refMgt, exh); - - SExHandleWrap* wrap = taosMemoryCalloc(1, sizeof(SExHandleWrap)); - wrap->exhandle = exh; - wrap->refId = exh->refId; - conn->refId = exh->refId; - transMsg.info.handle = wrap; + transMsg.info.handle = (void*)conn->refId; tDebug("%s conn %p ref by app", CONN_GET_INST_LABEL(conn), conn); } @@ -357,12 +344,10 @@ void cliHandleResp(SCliConn* conn) { if (pCtx == NULL && CONN_NO_PERSIST_BY_APP(conn)) { tDebug("%s except, conn %p read while cli ignore it", CONN_GET_INST_LABEL(conn), conn); - // transUnrefCliHandle(conn); return; } if (CONN_RELEASE_BY_SERVER(conn) && transMsg.info.ahandle == NULL) { tDebug("%s except, conn %p read while cli ignore it", CONN_GET_INST_LABEL(conn), conn); - // transUnrefCliHandle(conn); return; } @@ -433,7 +418,7 @@ void cliHandleExcept(SCliConn* pConn) { return; } destroyCmsg(pMsg); - tTrace("%s conn %p start to destroy", CONN_GET_INST_LABEL(pConn), pConn); + tTrace("%s conn %p start to destroy, ref:%d", CONN_GET_INST_LABEL(pConn), pConn, T_REF_VAL_GET(pConn)); } while (!transQueueEmpty(&pConn->cliMsgs)); transUnrefCliHandle(pConn); } @@ -505,10 +490,22 @@ static SCliConn* getConnFromPool(void* pool, char* ip, uint32_t port) { QUEUE_INIT(&conn->conn); return conn; } +static void allocConnRef(SCliConn* conn, bool update) { + if (update) { + transRemoveExHandle(refMgt, conn->refId); + } + SExHandle* exh = taosMemoryCalloc(1, sizeof(SExHandle)); + exh->handle = conn; + exh->pThrd = conn->hostThrd; + exh->refId = transAddExHandle(refMgt, exh); + conn->refId = exh->refId; +} static void addConnToPool(void* pool, SCliConn* conn) { SCliThrdObj* thrd = conn->hostThrd; CONN_HANDLE_THREAD_QUIT(thrd); + allocConnRef(conn, true); + STrans* pTransInst = thrd->pTransInst; conn->expireTime = taosGetTimestampMs() + CONN_PERSIST_TIME(pTransInst->idleTime); transQueueClear(&conn->cliMsgs); @@ -558,7 +555,8 @@ static void cliRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) { return; } if (nread < 0) { - tError("%s conn %p read error: %s", CONN_GET_INST_LABEL(conn), conn, uv_err_name(nread)); + tError("%s conn %p read error: %s, ref: %d", CONN_GET_INST_LABEL(conn), conn, uv_err_name(nread), + T_REF_VAL_GET(conn)); conn->broken = true; cliHandleExcept(conn); } @@ -581,11 +579,7 @@ static SCliConn* cliCreateConn(SCliThrdObj* pThrd) { conn->broken = 0; transRefCliHandle(conn); - SExHandle* exh = taosMemoryCalloc(1, sizeof(SExHandle)); - exh->handle = conn; - exh->pThrd = pThrd; - exh->refId = transAddExHandle(refMgt, exh); - conn->refId = exh->refId; + allocConnRef(conn, false); return conn; } @@ -749,25 +743,27 @@ static void cliHandleUpdate(SCliMsg* pMsg, SCliThrdObj* pThrd) { } SCliConn* cliGetConn(SCliMsg* pMsg, SCliThrdObj* pThrd) { - SCliConn* conn = NULL; - SRpcHandleInfo* pInfo = &pMsg->msg.info; + SCliConn* conn = NULL; + // SExHandleWrap* exWrap = &pMsg->msg.info.handle; + // if (exWrap != NULL) { + //} - SExHandle* exh = transAcquireExHandle(refMgt, pInfo->refId); - if (exh == NULL) { - if (pInfo->refId != 0) { - tTrace("%s conn %p ignore msg", CONN_GET_INST_LABEL(conn), conn); - assert(0); - return NULL; - } - } else { - transReleaseExHandle(refMgt, pInfo->refId); - return exh->handle; - } + // SExHandle* exh = transAcquireExHandle(refMgt, exWrap->refId); + // if (exh == NULL) { + // if (pInfo->refId != 0) { + // tTrace("%s conn %p ignore msg", CONN_GET_INST_LABEL(conn), conn); + // assert(0); + // return NULL; + // } + //} else { + // transReleaseExHandle(refMgt, pInfo->refId); + // return exh->handle; + //} STransConnCtx* pCtx = pMsg->ctx; conn = getConnFromPool(pThrd->pool, EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet)); if (conn != NULL) { - exh = taosMemoryCalloc(1, sizeof(SExHandle)); + SExHandle* exh = taosMemoryCalloc(1, sizeof(SExHandle)); exh->handle = conn; exh->pThrd = pThrd; exh->refId = transAddExHandle(refMgt, exh); @@ -790,10 +786,6 @@ void cliMayCvtFqdnToIp(SEpSet* pEpSet, SCvtAddr* pCvtAddr) { } } void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { - uint64_t et = taosGetTimestampUs(); - uint64_t el = et - pMsg->st; - // tTrace("%s cli msg tran time cost: %" PRIu64 "us", ((STrans*)pThrd->pTransInst)->label, el); - STransConnCtx* pCtx = pMsg->ctx; STrans* pTransInst = pThrd->pTransInst; @@ -1014,7 +1006,6 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { tTrace("try to send req to next node"); pMsg->st = taosGetTimestampUs(); - taosMemoryFree(pResp->info.handle); pCtx->retryCount += 1; if (pResp->code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { if (pCtx->retryCount < pEpSet->numOfEps * 3) { @@ -1060,16 +1051,16 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { STraceId* trace = &pResp->info.traceId; if (pCtx->pSem != NULL) { - tGTrace("conn %p(sync) handle resp", pConn); + tGTrace("%s conn %p(sync) handle resp", CONN_GET_INST_LABEL(pConn), pConn); if (pCtx->pRsp == NULL) { - tGTrace("conn %p(sync) failed to resp, ignore", pConn); + tGTrace("%s conn %p(sync) failed to resp, ignore", CONN_GET_INST_LABEL(pConn), pConn); } else { memcpy((char*)pCtx->pRsp, (char*)pResp, sizeof(*pResp)); } tsem_post(pCtx->pSem); pCtx->pRsp = NULL; } else { - tGTrace("conn %p handle resp", pConn); + tGTrace("%s conn %p handle resp", CONN_GET_INST_LABEL(pConn), pConn); if (pResp->code != 0 || pCtx->retryCount == 0 || transEpSetIsEqual(&pCtx->epSet, &pCtx->origEpSet)) { pTransInst->cfp(pTransInst->parent, pResp, NULL); } else { @@ -1105,14 +1096,33 @@ void transUnrefCliHandle(void* handle) { return; } int ref = T_REF_DEC((SCliConn*)handle); - tTrace("%s conn %p ref %d", CONN_GET_INST_LABEL((SCliConn*)handle), handle, ref); + tTrace("%s conn %p ref:%d", CONN_GET_INST_LABEL((SCliConn*)handle), handle, ref); if (ref == 0) { cliDestroyConn((SCliConn*)handle, true); } } +SCliThrdObj* transGetWorkThrdFromHandle(int64_t handle) { + SCliThrdObj* pThrd = NULL; + SExHandle* exh = transAcquireExHandle(refMgt, handle); + if (exh == NULL) { + return NULL; + } + pThrd = exh->pThrd; + transReleaseExHandle(refMgt, handle); + return pThrd; +} +SCliThrdObj* transGetWorkThrd(STrans* trans, int64_t handle) { + int idx = -1; + if (handle == 0) { + idx = cliRBChoseIdx(trans); + return ((SCliObj*)trans->tcphandle)->pThreadObj[idx]; + } + return transGetWorkThrdFromHandle(handle); +} void transReleaseCliHandle(void* handle) { - SCliThrdObj* thrd = CONN_GET_HOST_THREAD(handle); - if (thrd == NULL) { + int idx = -1; + SCliThrdObj* pThrd = transGetWorkThrdFromHandle((int64_t)handle); + if (pThrd == NULL) { return; } @@ -1121,26 +1131,18 @@ void transReleaseCliHandle(void* handle) { cmsg->msg = tmsg; cmsg->type = Release; - transSendAsync(thrd->asyncPool, &cmsg->q); + transSendAsync(pThrd->asyncPool, &cmsg->q); + return; } void transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransCtx* ctx) { - STrans* pTransInst = (STrans*)shandle; - SRpcHandleInfo* info = &pReq->info; - - int idx = -1; - SCliThrdObj* pThrd = NULL; - SExHandle* exh = info->handle; - int64_t refId = -1; - if (exh != NULL) { - refId = exh->refId; + STrans* pTransInst = (STrans*)shandle; + SCliThrdObj* pThrd = transGetWorkThrd(pTransInst, (int64_t)pReq->info.handle); + if (pThrd == NULL) { + transFreeMsg(pReq->pCont); + return; } - CONN_HOST_THREAD_IDX1(idx, exh, refId, pThrd); - if (idx == -1) { - idx = cliRBChoseIdx(pTransInst); - pThrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[idx]; - } TRACE_SET_MSGID(&pReq->info.traceId, tGenIdPI64()); STransConnCtx* pCtx = taosMemoryCalloc(1, sizeof(STransConnCtx)); @@ -1163,28 +1165,17 @@ void transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STra tGTrace("%s send request at thread:%08" PRId64 ", dst: %s:%d, app:%p", transLabel(pTransInst), pThrd->pid, EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet), pReq->info.ahandle); ASSERT(transSendAsync(pThrd->asyncPool, &(cliMsg->q)) == 0); -_return1: - return; -_return2: return; } void transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp) { - STrans* pTransInst = (STrans*)shandle; - SRpcHandleInfo* info = &pReq->info; - SCliThrdObj* pThrd = NULL; - int idx = -1; - SExHandle* exh = info->handle; - int64_t refId = -1; - if (exh != NULL) { - refId = exh->refId; + STrans* pTransInst = (STrans*)shandle; + SCliThrdObj* pThrd = transGetWorkThrd(pTransInst, (int64_t)pReq->info.handle); + if (pThrd == NULL) { + transFreeMsg(pReq->pCont); + return; } - CONN_HOST_THREAD_IDX1(idx, exh, refId, pThrd); - if (idx == -1) { - idx = cliRBChoseIdx(pTransInst); - pThrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[idx]; - } tsem_t* sem = taosMemoryCalloc(1, sizeof(tsem_t)); tsem_init(sem, 0, 0); @@ -1211,16 +1202,13 @@ void transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransM tsem_wait(sem); tsem_destroy(sem); taosMemoryFree(sem); -_return1: - return; -_return2: return; } /* * **/ -void transSetDefaultAddr(void* ahandle, const char* ip, const char* fqdn) { - STrans* pTransInst = ahandle; +void transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn) { + STrans* pTransInst = shandle; SCvtAddr cvtAddr = {0}; if (ip != NULL && fqdn != NULL) { diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 121fddc99a..4cc2a9c9b2 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -133,7 +133,7 @@ static SSvrConn* createConn(void* hThrd); static void destroyConn(SSvrConn* conn, bool clear /*clear handle or not*/); static void destroyConnRegArg(SSvrConn* conn); -static int reallocConnRefHandle(SSvrConn* conn); +static int reallocConnRef(SSvrConn* conn); static void uvHandleQuit(SSvrMsg* msg, SWorkThrdObj* thrd); static void uvHandleRelease(SSvrMsg* msg, SWorkThrdObj* thrd); @@ -176,7 +176,7 @@ static bool addHandleToAcceptloop(void* arg); srvMsg->msg = tmsg; \ srvMsg->type = Release; \ srvMsg->pConn = conn; \ - reallocConnRefHandle(conn); \ + reallocConnRef(conn); \ if (!transQueuePush(&conn->srvMsgs, srvMsg)) { \ return; \ } \ @@ -353,7 +353,7 @@ void uvOnSendCb(uv_write_t* req, int status) { // if (msg->type == Release && conn->status != ConnNormal) { // conn->status = ConnNormal; // transUnrefSrvHandle(conn); - // reallocConnRefHandle(conn); + // reallocConnRef(conn); // destroySmsg(msg); // transQueueClear(&conn->srvMsgs); // return; @@ -800,7 +800,7 @@ static void destroyConnRegArg(SSvrConn* conn) { conn->regArg.init = 0; } } -static int reallocConnRefHandle(SSvrConn* conn) { +static int reallocConnRef(SSvrConn* conn) { transReleaseExHandle(refMgt, conn->refId); transRemoveExHandle(refMgt, conn->refId); // avoid app continue to send msg on invalid handle @@ -945,7 +945,7 @@ void uvHandleQuit(SSvrMsg* msg, SWorkThrdObj* thrd) { void uvHandleRelease(SSvrMsg* msg, SWorkThrdObj* thrd) { SSvrConn* conn = msg->pConn; if (conn->status == ConnAcquire) { - reallocConnRefHandle(conn); + reallocConnRef(conn); if (!transQueuePush(&conn->srvMsgs, msg)) { return; } From 0d904d5aa3b27c69776572fbc04887d1db5ac9c5 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 23 Jun 2022 20:06:22 +0800 Subject: [PATCH 04/49] fix: handle except --- source/client/src/clientEnv.c | 31 ++-- source/libs/transport/src/transCli.c | 244 +++++++++++++-------------- source/libs/transport/src/transSvr.c | 72 ++++---- 3 files changed, 171 insertions(+), 176 deletions(-) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 9f04e89694..f1e4107e23 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -13,11 +13,11 @@ * along with this program. If not, see . */ -#include "os.h" #include "catalog.h" -#include "functionMgt.h" #include "clientInt.h" #include "clientLog.h" +#include "functionMgt.h" +#include "os.h" #include "query.h" #include "scheduler.h" #include "tcache.h" @@ -38,7 +38,7 @@ static TdThreadOnce tscinit = PTHREAD_ONCE_INIT; volatile int32_t tscInitRes = 0; static void registerRequest(SRequestObj *pRequest) { - STscObj *pTscObj = acquireTscObj(*(int64_t*)pRequest->pTscObj->id); + STscObj *pTscObj = acquireTscObj(*(int64_t *)pRequest->pTscObj->id); assert(pTscObj != NULL); @@ -54,14 +54,14 @@ static void registerRequest(SRequestObj *pRequest) { int32_t currentInst = atomic_add_fetch_64((int64_t *)&pSummary->currentRequests, 1); tscDebug("0x%" PRIx64 " new Request from connObj:0x%" PRIx64 ", current:%d, app current:%d, total:%d, reqId:0x%" PRIx64, - pRequest->self, *(int64_t*)pRequest->pTscObj->id, num, currentInst, total, pRequest->requestId); + pRequest->self, *(int64_t *)pRequest->pTscObj->id, num, currentInst, total, pRequest->requestId); } } static void deregisterRequest(SRequestObj *pRequest) { assert(pRequest != NULL); - STscObj *pTscObj = pRequest->pTscObj; + STscObj * pTscObj = pRequest->pTscObj; SAppClusterSummary *pActivity = &pTscObj->pAppInfo->summary; int32_t currentInst = atomic_sub_fetch_64((int64_t *)&pActivity->currentRequests, 1); @@ -70,8 +70,8 @@ static void deregisterRequest(SRequestObj *pRequest) { int64_t duration = taosGetTimestampUs() - pRequest->metric.start; tscDebug("0x%" PRIx64 " free Request from connObj: 0x%" PRIx64 ", reqId:0x%" PRIx64 " elapsed:%" PRIu64 " ms, current:%d, app current:%d", - pRequest->self, *(int64_t*)pTscObj->id, pRequest->requestId, duration / 1000, num, currentInst); - releaseTscObj(*(int64_t*)pTscObj->id); + pRequest->self, *(int64_t *)pTscObj->id, pRequest->requestId, duration / 1000, num, currentInst); + releaseTscObj(*(int64_t *)pTscObj->id); } // todo close the transporter properly @@ -80,7 +80,7 @@ void closeTransporter(STscObj *pTscObj) { return; } - tscDebug("free transporter:%p in connObj: 0x%" PRIx64, pTscObj->pAppInfo->pTransporter, *(int64_t*)pTscObj->id); + tscDebug("free transporter:%p in connObj: 0x%" PRIx64, pTscObj->pAppInfo->pTransporter, *(int64_t *)pTscObj->id); rpcClose(pTscObj->pAppInfo->pTransporter); } @@ -128,16 +128,17 @@ void closeAllRequests(SHashObj *pRequests) { void destroyTscObj(void *pObj) { STscObj *pTscObj = pObj; - SClientHbKey connKey = {.tscRid = *(int64_t*)pTscObj->id, .connType = pTscObj->connType}; + SClientHbKey connKey = {.tscRid = *(int64_t *)pTscObj->id, .connType = pTscObj->connType}; hbDeregisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey); int64_t connNum = atomic_sub_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); closeAllRequests(pTscObj->pRequests); schedulerStopQueryHb(pTscObj->pAppInfo->pTransporter); if (0 == connNum) { - // TODO - //closeTransporter(pTscObj); + // TODO + closeTransporter(pTscObj); } - tscDebug("connObj 0x%" PRIx64 " destroyed, totalConn:%" PRId64, *(int64_t*)pTscObj->id, pTscObj->pAppInfo->numOfConns); + tscDebug("connObj 0x%" PRIx64 " destroyed, totalConn:%" PRId64, *(int64_t *)pTscObj->id, + pTscObj->pAppInfo->numOfConns); taosThreadMutexDestroy(&pTscObj->mutex); taosMemoryFreeClear(pTscObj); } @@ -167,10 +168,10 @@ void *createTscObj(const char *user, const char *auth, const char *db, int32_t c taosThreadMutexInit(&pObj->mutex, NULL); pObj->id = taosMemoryMalloc(sizeof(int64_t)); - *(int64_t*)pObj->id = taosAddRef(clientConnRefPool, pObj); + *(int64_t *)pObj->id = taosAddRef(clientConnRefPool, pObj); pObj->schemalessType = 1; - tscDebug("connObj created, 0x%" PRIx64, *(int64_t*)pObj->id); + tscDebug("connObj created, 0x%" PRIx64, *(int64_t *)pObj->id); return pObj; } @@ -325,7 +326,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { return 0; } - SConfig *pCfg = taosGetCfg(); + SConfig * pCfg = taosGetCfg(); SConfigItem *pItem = NULL; switch (option) { diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index e18723d976..6abd230f31 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1,4 +1,4 @@ -/* * Copyright (c) 2019 TAOS Data, Inc. +/** Copyright (c) 2019 TAOS Data, Inc. * * This program is free software: you can use, redistribute, and/or modify * it under the terms of the GNU Affero General Public License, version 3 @@ -54,7 +54,7 @@ typedef struct SCliMsg { int sent; //(0: no send, 1: alread sent) } SCliMsg; -typedef struct SCliThrdObj { +typedef struct SCliThrd { TdThread thread; // tid int64_t pid; // pid uv_loop_t* loop; @@ -72,13 +72,13 @@ typedef struct SCliThrdObj { SCvtAddr cvtAddr; bool quit; -} SCliThrdObj; +} SCliThrd; typedef struct SCliObj { - char label[TSDB_LABEL_LEN]; - int32_t index; - int numOfThreads; - SCliThrdObj** pThreadObj; + char label[TSDB_LABEL_LEN]; + int32_t index; + int numOfThreads; + SCliThrd** pThreadObj; } SCliObj; typedef struct SConnList { @@ -106,7 +106,7 @@ static void cliAsyncCb(uv_async_t* handle); static int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg); -static SCliConn* cliCreateConn(SCliThrdObj* thrd); +static SCliConn* cliCreateConn(SCliThrd* thrd); static void cliDestroyConn(SCliConn* pConn, bool clear /*clear tcp handle or not*/); static void cliDestroy(uv_handle_t* handle); static void cliSend(SCliConn* pConn); @@ -122,14 +122,14 @@ static void cliHandleResp(SCliConn* conn); static void cliHandleExcept(SCliConn* conn); // handle req from app -static void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd); -static void cliHandleQuit(SCliMsg* pMsg, SCliThrdObj* pThrd); -static void cliHandleRelease(SCliMsg* pMsg, SCliThrdObj* pThrd); -static void cliHandleUpdate(SCliMsg* pMsg, SCliThrdObj* pThrd); -static void (*cliAsyncHandle[])(SCliMsg* pMsg, SCliThrdObj* pThrd) = {cliHandleReq, cliHandleQuit, cliHandleRelease, - NULL, cliHandleUpdate}; +static void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd); +static void cliHandleQuit(SCliMsg* pMsg, SCliThrd* pThrd); +static void cliHandleRelease(SCliMsg* pMsg, SCliThrd* pThrd); +static void cliHandleUpdate(SCliMsg* pMsg, SCliThrd* pThrd); +static void (*cliAsyncHandle[])(SCliMsg* pMsg, SCliThrd* pThrd) = {cliHandleReq, cliHandleQuit, cliHandleRelease, NULL, + cliHandleUpdate}; -static void cliSendQuit(SCliThrdObj* thrd); +static void cliSendQuit(SCliThrd* thrd); static void destroyUserdata(STransMsg* userdata); static int cliRBChoseIdx(STrans* pTransInst); @@ -137,8 +137,8 @@ static int cliRBChoseIdx(STrans* pTransInst); static void destroyCmsg(SCliMsg* cmsg); static void transDestroyConnCtx(STransConnCtx* ctx); // thread obj -static SCliThrdObj* createThrdObj(); -static void destroyThrdObj(SCliThrdObj* pThrd); +static SCliThrd* createThrdObj(); +static void destroyThrdObj(SCliThrd* pThrd); static void cliWalkCb(uv_handle_t* handle, void* arg); @@ -174,12 +174,12 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { idx = -1; \ } else { \ ASYNC_CHECK_HANDLE((exh), refId); \ - pThrd = (SCliThrdObj*)(exh)->pThrd; \ + pThrd = (SCliThrd*)(exh)->pThrd; \ } \ } while (0) #define CONN_PERSIST_TIME(para) (para * 1000 * 10) #define CONN_GET_HOST_THREAD(conn) (conn ? ((SCliConn*)conn)->hostThrd : NULL) -#define CONN_GET_INST_LABEL(conn) (((STrans*)(((SCliThrdObj*)(conn)->hostThrd)->pTransInst))->label) +#define CONN_GET_INST_LABEL(conn) (((STrans*)(((SCliThrd*)(conn)->hostThrd)->pTransInst))->label) #define CONN_SHOULD_RELEASE(conn, head) \ do { \ if ((head)->release == 1 && (head->msgLen) == sizeof(*head)) { \ @@ -195,7 +195,7 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { destroyCmsg(pMsg); \ cliReleaseUnfinishedMsg(conn); \ if (status != ConnInPool) { \ - addConnToPool(((SCliThrdObj*)conn->hostThrd)->pool, conn); \ + addConnToPool(((SCliThrd*)conn->hostThrd)->pool, conn); \ } \ transRemoveExHandle(refMgt, conn->refId); \ return; \ @@ -279,8 +279,8 @@ _RETURN: return false; } void cliHandleResp(SCliConn* conn) { - SCliThrdObj* pThrd = conn->hostThrd; - STrans* pTransInst = pThrd->pTransInst; + SCliThrd* pThrd = conn->hostThrd; + STrans* pTransInst = pThrd->pTransInst; STransMsgHead* pHead = (STransMsgHead*)(conn->readBuf.buf); pHead->code = htonl(pHead->code); @@ -379,9 +379,9 @@ void cliHandleExcept(SCliConn* pConn) { return; } } - SCliThrdObj* pThrd = pConn->hostThrd; - STrans* pTransInst = pThrd->pTransInst; - bool once = false; + SCliThrd* pThrd = pConn->hostThrd; + STrans* pTransInst = pThrd->pTransInst; + bool once = false; do { SCliMsg* pMsg = transQueuePop(&pConn->cliMsgs); if (pMsg == NULL && once) { @@ -424,9 +424,9 @@ void cliHandleExcept(SCliConn* pConn) { } void cliTimeoutCb(uv_timer_t* handle) { - SCliThrdObj* pThrd = handle->data; - STrans* pTransInst = pThrd->pTransInst; - int64_t currentTime = pThrd->nextTimeout; + SCliThrd* pThrd = handle->data; + STrans* pTransInst = pThrd->pTransInst; + int64_t currentTime = pThrd->nextTimeout; tTrace("%s conn timeout, try to remove expire conn from conn pool", pTransInst->label); SConnList* p = taosHashIterate((SHashObj*)pThrd->pool, NULL); @@ -501,7 +501,7 @@ static void allocConnRef(SCliConn* conn, bool update) { conn->refId = exh->refId; } static void addConnToPool(void* pool, SCliConn* conn) { - SCliThrdObj* thrd = conn->hostThrd; + SCliThrd* thrd = conn->hostThrd; CONN_HANDLE_THREAD_QUIT(thrd); allocConnRef(conn, true); @@ -562,7 +562,7 @@ static void cliRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) { } } -static SCliConn* cliCreateConn(SCliThrdObj* pThrd) { +static SCliConn* cliCreateConn(SCliThrd* pThrd) { SCliConn* conn = taosMemoryCalloc(1, sizeof(SCliConn)); // read/write stream handle conn->stream = (uv_stream_t*)taosMemoryMalloc(sizeof(uv_tcp_t)); @@ -615,7 +615,7 @@ static bool cliHandleNoResp(SCliConn* conn) { } if (res == true) { if (cliMaySendCachedMsg(conn) == false) { - SCliThrdObj* thrd = conn->hostThrd; + SCliThrd* thrd = conn->hostThrd; addConnToPool(thrd->pool, conn); } } @@ -651,8 +651,8 @@ void cliSend(SCliConn* pConn) { STransConnCtx* pCtx = pCliMsg->ctx; - SCliThrdObj* pThrd = pConn->hostThrd; - STrans* pTransInst = pThrd->pTransInst; + SCliThrd* pThrd = pConn->hostThrd; + STrans* pTransInst = pThrd->pTransInst; STransMsg* pMsg = (STransMsg*)(&pCliMsg->msg); if (pMsg->pCont == 0) { @@ -709,7 +709,7 @@ void cliConnCb(uv_connect_t* req, int status) { cliSend(pConn); } -static void cliHandleQuit(SCliMsg* pMsg, SCliThrdObj* pThrd) { +static void cliHandleQuit(SCliMsg* pMsg, SCliThrd* pThrd) { tDebug("cli work thread %p start to quit", pThrd); destroyCmsg(pMsg); destroyConnPool(pThrd->pool); @@ -720,7 +720,7 @@ static void cliHandleQuit(SCliMsg* pMsg, SCliThrdObj* pThrd) { // uv_stop(pThrd->loop); } -static void cliHandleRelease(SCliMsg* pMsg, SCliThrdObj* pThrd) { +static void cliHandleRelease(SCliMsg* pMsg, SCliThrd* pThrd) { SCliConn* conn = pMsg->msg.info.handle; tDebug("%s conn %p start to release to inst", CONN_GET_INST_LABEL(conn), conn); @@ -735,39 +735,30 @@ static void cliHandleRelease(SCliMsg* pMsg, SCliThrdObj* pThrd) { transUnrefCliHandle(conn); } } -static void cliHandleUpdate(SCliMsg* pMsg, SCliThrdObj* pThrd) { +static void cliHandleUpdate(SCliMsg* pMsg, SCliThrd* pThrd) { STransConnCtx* pCtx = pMsg->ctx; pThrd->cvtAddr = pCtx->cvtAddr; destroyCmsg(pMsg); } -SCliConn* cliGetConn(SCliMsg* pMsg, SCliThrdObj* pThrd) { +SCliConn* cliGetConn(SCliMsg* pMsg, SCliThrd* pThrd) { SCliConn* conn = NULL; - // SExHandleWrap* exWrap = &pMsg->msg.info.handle; - // if (exWrap != NULL) { - //} - - // SExHandle* exh = transAcquireExHandle(refMgt, exWrap->refId); - // if (exh == NULL) { - // if (pInfo->refId != 0) { - // tTrace("%s conn %p ignore msg", CONN_GET_INST_LABEL(conn), conn); - // assert(0); - // return NULL; - // } - //} else { - // transReleaseExHandle(refMgt, pInfo->refId); - // return exh->handle; - //} + int64_t refId = (int64_t)(pMsg->msg.info.handle); + if (refId != 0) { + SExHandle* exh = transAcquireExHandle(refMgt, refId); + if (exh == NULL) { + assert(0); + } else { + conn = exh->handle; + transReleaseExHandle(refMgt, refId); + } + return conn; + }; STransConnCtx* pCtx = pMsg->ctx; conn = getConnFromPool(pThrd->pool, EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet)); if (conn != NULL) { - SExHandle* exh = taosMemoryCalloc(1, sizeof(SExHandle)); - exh->handle = conn; - exh->pThrd = pThrd; - exh->refId = transAddExHandle(refMgt, exh); - conn->refId = exh->refId; tTrace("%s conn %p get from conn pool", CONN_GET_INST_LABEL(conn), conn); } else { tTrace("%s not found conn in conn pool %p", ((STrans*)pThrd->pTransInst)->label, pThrd->pool); @@ -785,7 +776,7 @@ void cliMayCvtFqdnToIp(SEpSet* pEpSet, SCvtAddr* pCvtAddr) { } } } -void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { +void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) { STransConnCtx* pCtx = pMsg->ctx; STrans* pTransInst = pThrd->pTransInst; @@ -833,9 +824,9 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { } } static void cliAsyncCb(uv_async_t* handle) { - SAsyncItem* item = handle->data; - SCliThrdObj* pThrd = item->pThrd; - SCliMsg* pMsg = NULL; + SAsyncItem* item = handle->data; + SCliThrd* pThrd = item->pThrd; + SCliMsg* pMsg = NULL; // batch process to avoid to lock/unlock frequently queue wq; @@ -861,7 +852,7 @@ static void cliAsyncCb(uv_async_t* handle) { } static void* cliWorkThread(void* arg) { - SCliThrdObj* pThrd = (SCliThrdObj*)arg; + SCliThrd* pThrd = (SCliThrd*)arg; pThrd->pid = taosGetSelfPthreadId(); setThreadName("trans-cli-work"); uv_run(pThrd->loop, UV_RUN_DEFAULT); @@ -874,10 +865,10 @@ void* transInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, STrans* pTransInst = shandle; memcpy(cli->label, label, strlen(label)); cli->numOfThreads = numOfThreads; - cli->pThreadObj = (SCliThrdObj**)taosMemoryCalloc(cli->numOfThreads, sizeof(SCliThrdObj*)); + cli->pThreadObj = (SCliThrd**)taosMemoryCalloc(cli->numOfThreads, sizeof(SCliThrd*)); for (int i = 0; i < cli->numOfThreads; i++) { - SCliThrdObj* pThrd = createThrdObj(); + SCliThrd* pThrd = createThrdObj(); pThrd->nextTimeout = taosGetTimestampMs() + CONN_PERSIST_TIME(pTransInst->idleTime); pThrd->pTransInst = shandle; @@ -911,8 +902,8 @@ static void destroyCmsg(SCliMsg* pMsg) { taosMemoryFree(pMsg); } -static SCliThrdObj* createThrdObj() { - SCliThrdObj* pThrd = (SCliThrdObj*)taosMemoryCalloc(1, sizeof(SCliThrdObj)); +static SCliThrd* createThrdObj() { + SCliThrd* pThrd = (SCliThrd*)taosMemoryCalloc(1, sizeof(SCliThrd)); QUEUE_INIT(&pThrd->msg); taosThreadMutexInit(&pThrd->msgMtx, NULL); @@ -930,7 +921,7 @@ static SCliThrdObj* createThrdObj() { pThrd->quit = false; return pThrd; } -static void destroyThrdObj(SCliThrdObj* pThrd) { +static void destroyThrdObj(SCliThrd* pThrd) { if (pThrd == NULL) { return; } @@ -951,7 +942,7 @@ static void transDestroyConnCtx(STransConnCtx* ctx) { taosMemoryFree(ctx); } -void cliSendQuit(SCliThrdObj* thrd) { +void cliSendQuit(SCliThrd* thrd) { // cli can stop gracefully SCliMsg* msg = taosMemoryCalloc(1, sizeof(SCliMsg)); msg->type = Quit; @@ -973,15 +964,16 @@ int cliRBChoseIdx(STrans* pTransInst) { static void doDelayTask(void* param) { STaskArg* arg = param; - SCliMsg* pMsg = arg->param1; - SCliThrdObj* pThrd = arg->param2; + SCliMsg* pMsg = arg->param1; + SCliThrd* pThrd = arg->param2; cliHandleReq(pMsg, pThrd); taosMemoryFree(arg); } + int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { - SCliThrdObj* pThrd = pConn->hostThrd; - STrans* pTransInst = pThrd->pTransInst; + SCliThrd* pThrd = pConn->hostThrd; + STrans* pTransInst = pThrd->pTransInst; if (pMsg == NULL || pMsg->ctx == NULL) { tTrace("%s conn %p handle resp", pTransInst->label, pConn); @@ -995,57 +987,60 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { if (pCtx->retryCount == 0) { pCtx->origEpSet = pCtx->epSet; } + /* * upper layer handle retry if code equal TSDB_CODE_RPC_NETWORK_UNAVAIL */ - tmsg_t msgType = pCtx->msgType; - if ((pTransInst->retry != NULL && pEpSet->numOfEps > 1 && (pTransInst->retry(pResp->code))) || - (pResp->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || pResp->code == TSDB_CODE_APP_NOT_READY || - pResp->code == TSDB_CODE_NODE_NOT_DEPLOYED || pResp->code == TSDB_CODE_SYN_NOT_LEADER)) { - pMsg->sent = 0; - tTrace("try to send req to next node"); - pMsg->st = taosGetTimestampUs(); + if (CONN_NO_PERSIST_BY_APP(pConn)) { + tmsg_t msgType = pCtx->msgType; + if ((pTransInst->retry != NULL && pEpSet->numOfEps > 1 && (pTransInst->retry(pResp->code))) || + (pResp->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || pResp->code == TSDB_CODE_APP_NOT_READY || + pResp->code == TSDB_CODE_NODE_NOT_DEPLOYED || pResp->code == TSDB_CODE_SYN_NOT_LEADER)) { + pMsg->sent = 0; + tTrace("try to send req to next node"); + pMsg->st = taosGetTimestampUs(); - pCtx->retryCount += 1; - if (pResp->code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { - if (pCtx->retryCount < pEpSet->numOfEps * 3) { - pEpSet->inUse = (++pEpSet->inUse) % pEpSet->numOfEps; + pCtx->retryCount += 1; + if (pResp->code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { + if (pCtx->retryCount < pEpSet->numOfEps * 3) { + pEpSet->inUse = (++pEpSet->inUse) % pEpSet->numOfEps; + + STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); + arg->param1 = pMsg; + arg->param2 = pThrd; + transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL); + transPrintEpSet(pEpSet); + tTrace("%s use local epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse, + pCtx->retryCount + 1, pEpSet->numOfEps * 3); + + transUnrefCliHandle(pConn); + return -1; + } + } else if (pCtx->retryCount < TRANS_RETRY_COUNT_LIMIT) { + if (pResp->contLen == 0) { + pEpSet->inUse = (++pEpSet->inUse) % pEpSet->numOfEps; + transPrintEpSet(&pCtx->epSet); + tTrace("%s use local epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse, + pCtx->retryCount + 1, TRANS_RETRY_COUNT_LIMIT); + } else { + SEpSet epSet = {0}; + tDeserializeSEpSet(pResp->pCont, pResp->contLen, &epSet); + pCtx->epSet = epSet; + + transPrintEpSet(&pCtx->epSet); + tTrace("%s use remote epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse, + pCtx->retryCount + 1, TRANS_RETRY_COUNT_LIMIT); + } + if (pConn->status != ConnInPool) { + addConnToPool(pThrd->pool, pConn); + } STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); arg->param1 = pMsg; arg->param2 = pThrd; transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL); - transPrintEpSet(pEpSet); - tTrace("%s use local epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse, - pCtx->retryCount + 1, pEpSet->numOfEps * 3); - - transUnrefCliHandle(pConn); return -1; } - } else if (pCtx->retryCount < TRANS_RETRY_COUNT_LIMIT) { - if (pResp->contLen == 0) { - pEpSet->inUse = (++pEpSet->inUse) % pEpSet->numOfEps; - transPrintEpSet(&pCtx->epSet); - tTrace("%s use local epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse, - pCtx->retryCount + 1, TRANS_RETRY_COUNT_LIMIT); - } else { - SEpSet epSet = {0}; - tDeserializeSEpSet(pResp->pCont, pResp->contLen, &epSet); - pCtx->epSet = epSet; - - transPrintEpSet(&pCtx->epSet); - tTrace("%s use remote epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse, - pCtx->retryCount + 1, TRANS_RETRY_COUNT_LIMIT); - } - if (pConn->status != ConnInPool) { - addConnToPool(pThrd->pool, pConn); - } - - STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); - arg->param1 = pMsg; - arg->param2 = pThrd; - transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL); - return -1; } } @@ -1101,9 +1096,9 @@ void transUnrefCliHandle(void* handle) { cliDestroyConn((SCliConn*)handle, true); } } -SCliThrdObj* transGetWorkThrdFromHandle(int64_t handle) { - SCliThrdObj* pThrd = NULL; - SExHandle* exh = transAcquireExHandle(refMgt, handle); +SCliThrd* transGetWorkThrdFromHandle(int64_t handle) { + SCliThrd* pThrd = NULL; + SExHandle* exh = transAcquireExHandle(refMgt, handle); if (exh == NULL) { return NULL; } @@ -1111,17 +1106,16 @@ SCliThrdObj* transGetWorkThrdFromHandle(int64_t handle) { transReleaseExHandle(refMgt, handle); return pThrd; } -SCliThrdObj* transGetWorkThrd(STrans* trans, int64_t handle) { - int idx = -1; +SCliThrd* transGetWorkThrd(STrans* trans, int64_t handle) { if (handle == 0) { - idx = cliRBChoseIdx(trans); + int idx = cliRBChoseIdx(trans); return ((SCliObj*)trans->tcphandle)->pThreadObj[idx]; } return transGetWorkThrdFromHandle(handle); } void transReleaseCliHandle(void* handle) { - int idx = -1; - SCliThrdObj* pThrd = transGetWorkThrdFromHandle((int64_t)handle); + int idx = -1; + SCliThrd* pThrd = transGetWorkThrdFromHandle((int64_t)handle); if (pThrd == NULL) { return; } @@ -1136,8 +1130,8 @@ void transReleaseCliHandle(void* handle) { } void transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransCtx* ctx) { - STrans* pTransInst = (STrans*)shandle; - SCliThrdObj* pThrd = transGetWorkThrd(pTransInst, (int64_t)pReq->info.handle); + STrans* pTransInst = (STrans*)shandle; + SCliThrd* pThrd = transGetWorkThrd(pTransInst, (int64_t)pReq->info.handle); if (pThrd == NULL) { transFreeMsg(pReq->pCont); return; @@ -1169,8 +1163,8 @@ void transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STra } void transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp) { - STrans* pTransInst = (STrans*)shandle; - SCliThrdObj* pThrd = transGetWorkThrd(pTransInst, (int64_t)pReq->info.handle); + STrans* pTransInst = (STrans*)shandle; + SCliThrd* pThrd = transGetWorkThrd(pTransInst, (int64_t)pReq->info.handle); if (pThrd == NULL) { transFreeMsg(pReq->pCont); return; @@ -1224,7 +1218,7 @@ void transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn) { cliMsg->ctx = pCtx; cliMsg->type = Update; - SCliThrdObj* thrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[i]; + SCliThrd* thrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[i]; tDebug("%s update epset at thread:%08" PRId64 "", pTransInst->label, thrd->pid); transSendAsync(thrd->asyncPool, &(cliMsg->q)); diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 4cc2a9c9b2..599d98a3e9 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -65,7 +65,7 @@ typedef struct SSvrMsg { STransMsgType type; } SSvrMsg; -typedef struct SWorkThrdObj { +typedef struct SWorkThrd { TdThread thread; uv_connect_t connect_req; uv_pipe_t* pipe; @@ -78,7 +78,7 @@ typedef struct SWorkThrdObj { queue conn; void* pTransInst; bool quit; -} SWorkThrdObj; +} SWorkThrd; typedef struct SServerObj { TdThread thread; @@ -86,10 +86,10 @@ typedef struct SServerObj { uv_loop_t* loop; // work thread info - int workerIdx; - int numOfThreads; - int numOfWorkerReady; - SWorkThrdObj** pThreadObj; + int workerIdx; + int numOfThreads; + int numOfWorkerReady; + SWorkThrd** pThreadObj; uv_pipe_t pipeListen; uv_pipe_t** pipe; @@ -135,12 +135,12 @@ static void destroyConnRegArg(SSvrConn* conn); static int reallocConnRef(SSvrConn* conn); -static void uvHandleQuit(SSvrMsg* msg, SWorkThrdObj* thrd); -static void uvHandleRelease(SSvrMsg* msg, SWorkThrdObj* thrd); -static void uvHandleResp(SSvrMsg* msg, SWorkThrdObj* thrd); -static void uvHandleRegister(SSvrMsg* msg, SWorkThrdObj* thrd); -static void (*transAsyncHandle[])(SSvrMsg* msg, SWorkThrdObj* thrd) = {uvHandleResp, uvHandleQuit, uvHandleRelease, - uvHandleRegister, NULL}; +static void uvHandleQuit(SSvrMsg* msg, SWorkThrd* thrd); +static void uvHandleRelease(SSvrMsg* msg, SWorkThrd* thrd); +static void uvHandleResp(SSvrMsg* msg, SWorkThrd* thrd); +static void uvHandleRegister(SSvrMsg* msg, SWorkThrd* thrd); +static void (*transAsyncHandle[])(SSvrMsg* msg, SWorkThrd* thrd) = {uvHandleResp, uvHandleQuit, uvHandleRelease, + uvHandleRegister, NULL}; static int32_t exHandlesMgt; @@ -160,7 +160,7 @@ static void* transWorkerThread(void* arg); static void* transAcceptThread(void* arg); // add handle loop -static bool addHandleToWorkloop(SWorkThrdObj* pThrd, char* pipeName); +static bool addHandleToWorkloop(SWorkThrd* pThrd, char* pipeName); static bool addHandleToAcceptloop(void* arg); #define CONN_SHOULD_RELEASE(conn, head) \ @@ -233,7 +233,7 @@ static void uvHandleReq(SSvrConn* pConn) { // wreq->data = pConn; // uv_read_stop((uv_stream_t*)pConn->pTcp); // transRefSrvHandle(pConn); - // uv_queue_work(((SWorkThrdObj*)pConn->hostThrd)->loop, wreq, uvWorkDoTask, uvWorkAfterTask); + // uv_queue_work(((SWorkThrd*)pConn->hostThrd)->loop, wreq, uvWorkDoTask, uvWorkAfterTask); CONN_SHOULD_RELEASE(pConn, pHead); @@ -478,7 +478,7 @@ static void destroySmsg(SSvrMsg* smsg) { transFreeMsg(smsg->msg.pCont); taosMemoryFree(smsg); } -static void destroyAllConn(SWorkThrdObj* pThrd) { +static void destroyAllConn(SWorkThrd* pThrd) { tTrace("thread %p destroy all conn ", pThrd); while (!QUEUE_IS_EMPTY(&pThrd->conn)) { queue* h = QUEUE_HEAD(&pThrd->conn); @@ -493,10 +493,10 @@ static void destroyAllConn(SWorkThrdObj* pThrd) { } } void uvWorkerAsyncCb(uv_async_t* handle) { - SAsyncItem* item = handle->data; - SWorkThrdObj* pThrd = item->pThrd; - SSvrConn* conn = NULL; - queue wq; + SAsyncItem* item = handle->data; + SWorkThrd* pThrd = item->pThrd; + SSvrConn* conn = NULL; + queue wq; // batch process to avoid to lock/unlock frequently taosThreadMutexLock(&item->mtx); @@ -624,7 +624,7 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { assert(buf->base[0] == notify[0]); taosMemoryFree(buf->base); - SWorkThrdObj* pThrd = q->data; + SWorkThrd* pThrd = q->data; uv_pipe_t* pipe = (uv_pipe_t*)q; if (!uv_pipe_pending_count(pipe)) { @@ -692,10 +692,10 @@ void uvOnPipeConnectionCb(uv_connect_t* connect, int status) { if (status != 0) { return; } - SWorkThrdObj* pThrd = container_of(connect, SWorkThrdObj, connect_req); + SWorkThrd* pThrd = container_of(connect, SWorkThrd, connect_req); uv_read_start((uv_stream_t*)pThrd->pipe, uvAllocConnBufferCb, uvOnConnectionCb); } -static bool addHandleToWorkloop(SWorkThrdObj* pThrd, char* pipeName) { +static bool addHandleToWorkloop(SWorkThrd* pThrd, char* pipeName) { pThrd->loop = (uv_loop_t*)taosMemoryMalloc(sizeof(uv_loop_t)); if (0 != uv_loop_init(pThrd->loop)) { return false; @@ -748,14 +748,14 @@ static bool addHandleToAcceptloop(void* arg) { } void* transWorkerThread(void* arg) { setThreadName("trans-worker"); - SWorkThrdObj* pThrd = (SWorkThrdObj*)arg; + SWorkThrd* pThrd = (SWorkThrd*)arg; uv_run(pThrd->loop, UV_RUN_DEFAULT); return NULL; } static SSvrConn* createConn(void* hThrd) { - SWorkThrdObj* pThrd = hThrd; + SWorkThrd* pThrd = hThrd; SSvrConn* pConn = (SSvrConn*)taosMemoryCalloc(1, sizeof(SSvrConn)); QUEUE_INIT(&pConn->queue); @@ -818,7 +818,7 @@ static void uvDestroyConn(uv_handle_t* handle) { if (conn == NULL) { return; } - SWorkThrdObj* thrd = conn->hostThrd; + SWorkThrd* thrd = conn->hostThrd; transReleaseExHandle(refMgt, conn->refId); transRemoveExHandle(refMgt, conn->refId); @@ -863,7 +863,7 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, srv->numOfThreads = numOfThreads; srv->workerIdx = 0; srv->numOfWorkerReady = 0; - srv->pThreadObj = (SWorkThrdObj**)taosMemoryCalloc(srv->numOfThreads, sizeof(SWorkThrdObj*)); + srv->pThreadObj = (SWorkThrd**)taosMemoryCalloc(srv->numOfThreads, sizeof(SWorkThrd*)); srv->pipe = (uv_pipe_t**)taosMemoryCalloc(srv->numOfThreads, sizeof(uv_pipe_t*)); srv->ip = ip; srv->port = port; @@ -888,7 +888,7 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, assert(0 == uv_listen((uv_stream_t*)&srv->pipeListen, SOMAXCONN, uvPipeListenCb)); for (int i = 0; i < srv->numOfThreads; i++) { - SWorkThrdObj* thrd = (SWorkThrdObj*)taosMemoryCalloc(1, sizeof(SWorkThrdObj)); + SWorkThrd* thrd = (SWorkThrd*)taosMemoryCalloc(1, sizeof(SWorkThrd)); thrd->pTransInst = shandle; thrd->quit = false; srv->pThreadObj[i] = thrd; @@ -933,7 +933,7 @@ End: return NULL; } -void uvHandleQuit(SSvrMsg* msg, SWorkThrdObj* thrd) { +void uvHandleQuit(SSvrMsg* msg, SWorkThrd* thrd) { thrd->quit = true; if (QUEUE_IS_EMPTY(&thrd->conn)) { uv_walk(thrd->loop, uvWalkCb, NULL); @@ -942,7 +942,7 @@ void uvHandleQuit(SSvrMsg* msg, SWorkThrdObj* thrd) { } taosMemoryFree(msg); } -void uvHandleRelease(SSvrMsg* msg, SWorkThrdObj* thrd) { +void uvHandleRelease(SSvrMsg* msg, SWorkThrd* thrd) { SSvrConn* conn = msg->pConn; if (conn->status == ConnAcquire) { reallocConnRef(conn); @@ -956,12 +956,12 @@ void uvHandleRelease(SSvrMsg* msg, SWorkThrdObj* thrd) { } destroySmsg(msg); } -void uvHandleResp(SSvrMsg* msg, SWorkThrdObj* thrd) { +void uvHandleResp(SSvrMsg* msg, SWorkThrd* thrd) { // send msg to client tDebug("%s conn %p start to send resp (2/2)", transLabel(thrd->pTransInst), msg->pConn); uvStartSendResp(msg); } -void uvHandleRegister(SSvrMsg* msg, SWorkThrdObj* thrd) { +void uvHandleRegister(SSvrMsg* msg, SWorkThrd* thrd) { SSvrConn* conn = msg->pConn; tDebug("%s conn %p register brokenlink callback", transLabel(thrd->pTransInst), conn); if (conn->status == ConnAcquire) { @@ -982,7 +982,7 @@ void uvHandleRegister(SSvrMsg* msg, SWorkThrdObj* thrd) { taosMemoryFree(msg); } } -void destroyWorkThrd(SWorkThrdObj* pThrd) { +void destroyWorkThrd(SWorkThrd* pThrd) { if (pThrd == NULL) { return; } @@ -993,7 +993,7 @@ void destroyWorkThrd(SWorkThrdObj* pThrd) { taosMemoryFree(pThrd->loop); taosMemoryFree(pThrd); } -void sendQuitToWorkThrd(SWorkThrdObj* pThrd) { +void sendQuitToWorkThrd(SWorkThrd* pThrd) { SSvrMsg* msg = taosMemoryCalloc(1, sizeof(SSvrMsg)); msg->type = Quit; tDebug("server send quit msg to work thread"); @@ -1060,7 +1060,7 @@ void transReleaseSrvHandle(void* handle) { ASYNC_CHECK_HANDLE(exh, refId); - SWorkThrdObj* pThrd = exh->pThrd; + SWorkThrd* pThrd = exh->pThrd; ASYNC_ERR_JRET(pThrd); STransMsg tmsg = {.code = 0, .info.handle = exh, .info.ahandle = NULL, .info.refId = refId}; @@ -1090,7 +1090,7 @@ void transSendResponse(const STransMsg* msg) { STransMsg tmsg = *msg; tmsg.info.refId = refId; - SWorkThrdObj* pThrd = exh->pThrd; + SWorkThrd* pThrd = exh->pThrd; ASYNC_ERR_JRET(pThrd); SSvrMsg* m = taosMemoryCalloc(1, sizeof(SSvrMsg)); @@ -1120,7 +1120,7 @@ void transRegisterMsg(const STransMsg* msg) { STransMsg tmsg = *msg; tmsg.info.refId = refId; - SWorkThrdObj* pThrd = exh->pThrd; + SWorkThrd* pThrd = exh->pThrd; ASYNC_ERR_JRET(pThrd); SSvrMsg* m = taosMemoryCalloc(1, sizeof(SSvrMsg)); From cebecb9ae64c2148d8dbd856e6b3c510eebd82a5 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 23 Jun 2022 20:49:00 +0800 Subject: [PATCH 05/49] test: disable unstable test case. --- tests/script/jenkins/basic.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index cc28b19de9..d6039eb311 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -70,7 +70,7 @@ ./test.sh -f tsim/mnode/basic2.sim ./test.sh -f tsim/mnode/basic3.sim ./test.sh -f tsim/mnode/basic4.sim -./test.sh -f tsim/mnode/basic5.sim +#./test.sh -f tsim/mnode/basic5.sim # ---- show ./test.sh -f tsim/show/basic.sim From ffd105d0e037c612c632be406a962b042bc58f14 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 23 Jun 2022 20:51:09 +0800 Subject: [PATCH 06/49] handle except --- source/libs/transport/src/transCli.c | 55 ++++++++++++++------------- source/libs/transport/src/transComm.c | 6 +-- 2 files changed, 32 insertions(+), 29 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 5b100e3d56..0bf6b2778b 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -393,7 +393,6 @@ void cliHandleExcept(SCliConn* pConn) { transMsg.code = TSDB_CODE_RPC_NETWORK_UNAVAIL; transMsg.msgType = pMsg ? pMsg->msg.msgType + 1 : 0; transMsg.info.ahandle = NULL; - transMsg.info.handle = pConn; if (pMsg == NULL && !CONN_NO_PERSIST_BY_APP(pConn)) { transMsg.info.ahandle = transCtxDumpVal(&pConn->ctx, transMsg.msgType); @@ -987,10 +986,14 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { if (pCtx->retryCount == 0) { pCtx->origEpSet = pCtx->epSet; } - /* * upper layer handle retry if code equal TSDB_CODE_RPC_NETWORK_UNAVAIL */ + /* + * no retry + * 1. query conn 2. rpc thread already receive quit msg + * + */ if (CONN_NO_PERSIST_BY_APP(pConn) && pThrd->quit == false) { tmsg_t msgType = pCtx->msgType; if ((pTransInst->retry != NULL && pEpSet->numOfEps > 1 && (pTransInst->retry(pResp->code))) || @@ -1014,31 +1017,31 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { transUnrefCliHandle(pConn); return -1; - } else if (pCtx->retryCount < TRANS_RETRY_COUNT_LIMIT) { - if (pResp->contLen == 0) { - pEpSet->inUse = (++pEpSet->inUse) % pEpSet->numOfEps; - transPrintEpSet(&pCtx->epSet); - tTrace("%s use local epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse, - pCtx->retryCount + 1, TRANS_RETRY_COUNT_LIMIT); - } else { - SEpSet epSet = {0}; - tDeserializeSEpSet(pResp->pCont, pResp->contLen, &epSet); - pCtx->epSet = epSet; - - transPrintEpSet(&pCtx->epSet); - tTrace("%s use remote epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse, - pCtx->retryCount + 1, TRANS_RETRY_COUNT_LIMIT); - } - if (pConn->status != ConnInPool) { - addConnToPool(pThrd->pool, pConn); - } - - STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); - arg->param1 = pMsg; - arg->param2 = pThrd; - transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL); - return -1; } + } else if (pCtx->retryCount < TRANS_RETRY_COUNT_LIMIT) { + if (pResp->contLen == 0) { + pEpSet->inUse = (++pEpSet->inUse) % pEpSet->numOfEps; + transPrintEpSet(&pCtx->epSet); + tTrace("%s use local epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse, + pCtx->retryCount + 1, TRANS_RETRY_COUNT_LIMIT); + } else { + SEpSet epSet = {0}; + tDeserializeSEpSet(pResp->pCont, pResp->contLen, &epSet); + pCtx->epSet = epSet; + + transPrintEpSet(&pCtx->epSet); + tTrace("%s use remote epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse, + pCtx->retryCount + 1, TRANS_RETRY_COUNT_LIMIT); + } + if (pConn->status != ConnInPool) { + addConnToPool(pThrd->pool, pConn); + } + + STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); + arg->param1 = pMsg; + arg->param2 = pThrd; + transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL); + return -1; } } } diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 5d342dd174..bff7d79bd3 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -455,16 +455,16 @@ void transPrintEpSet(SEpSet* pEpSet) { return; } char buf[512] = {0}; - int len = snprintf(buf, sizeof(buf), "epset:{ "); + int len = snprintf(buf, sizeof(buf), "epset:{"); for (int i = 0; i < pEpSet->numOfEps; i++) { if (i == pEpSet->numOfEps - 1) { - len += snprintf(buf + len, sizeof(buf) - len, "%d. %s:%d ", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); + len += snprintf(buf + len, sizeof(buf) - len, "%d. %s:%d", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); } else { len += snprintf(buf + len, sizeof(buf) - len, "%d. %s:%d, ", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); } } len += snprintf(buf + len, sizeof(buf) - len, "}"); - tTrace("%s, inUse: %d", buf, pEpSet->inUse); + tTrace("%s, inUse:%d", buf, pEpSet->inUse); } bool transEpSetIsEqual(SEpSet* a, SEpSet* b) { if (a->numOfEps != b->numOfEps || a->inUse != b->inUse) { From 1d169ce22af04f958b5b635fa715118dcfe0e0fe Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 23 Jun 2022 21:16:32 +0800 Subject: [PATCH 07/49] handle except --- source/libs/transport/src/transCli.c | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 0bf6b2778b..96fbe3f6b7 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -300,17 +300,9 @@ void cliHandleResp(SCliConn* conn) { if (CONN_NO_PERSIST_BY_APP(conn)) { pMsg = transQueuePop(&conn->cliMsgs); - pCtx = pMsg ? pMsg->ctx : NULL; - if (pMsg == NULL && !CONN_NO_PERSIST_BY_APP(conn)) { - transMsg.info.ahandle = transCtxDumpVal(&conn->ctx, transMsg.msgType); - if (transMsg.info.ahandle == NULL) { - transMsg.info.ahandle = transCtxDumpBrokenlinkVal(&conn->ctx, (int32_t*)&(transMsg.msgType)); - } - tDebug("%s conn %p construct ahandle %p, persist: 0", CONN_GET_INST_LABEL(conn), conn, transMsg.info.ahandle); - } else { - transMsg.info.ahandle = pCtx ? pCtx->ahandle : NULL; - tDebug("%s conn %p get ahandle %p, persist: 0", CONN_GET_INST_LABEL(conn), conn, transMsg.info.ahandle); - } + pCtx = pMsg->ctx; + transMsg.info.ahandle = pCtx->ahandle; + tDebug("%s conn %p get ahandle %p, persist: 0", CONN_GET_INST_LABEL(conn), conn, transMsg.info.ahandle); } else { uint64_t ahandle = (uint64_t)pHead->ahandle; CONN_GET_MSGCTX_BY_AHANDLE(conn, ahandle); From d63eb11b70b60aa379e0c0ed011fc9851fdae6c5 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 23 Jun 2022 21:35:35 +0800 Subject: [PATCH 08/49] handle except --- source/libs/transport/src/transCli.c | 17 ++++++++--------- source/libs/transport/src/transSvr.c | 2 -- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 96fbe3f6b7..9ccbf3eaa0 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -712,7 +712,14 @@ static void cliHandleQuit(SCliMsg* pMsg, SCliThrd* pThrd) { // uv_stop(pThrd->loop); } static void cliHandleRelease(SCliMsg* pMsg, SCliThrd* pThrd) { - SCliConn* conn = pMsg->msg.info.handle; + int64_t refId = (int64_t)(pMsg->msg.info.handle); + SExHandle* exh = transAcquireExHandle(refMgt, refId); + if (exh == NULL) { + tDebug("%" PRid64 " already release", refId); + return NULL; + } + + SCliConn* conn = exh->handle; tDebug("%s conn %p start to release to inst", CONN_GET_INST_LABEL(conn), conn); if (T_REF_VAL_GET(conn) == 2) { @@ -721,14 +728,10 @@ static void cliHandleRelease(SCliMsg* pMsg, SCliThrd* pThrd) { return; } cliSend(conn); - } else { - // conn already broken down - transUnrefCliHandle(conn); } } static void cliHandleUpdate(SCliMsg* pMsg, SCliThrd* pThrd) { STransConnCtx* pCtx = pMsg->ctx; - pThrd->cvtAddr = pCtx->cvtAddr; destroyCmsg(pMsg); } @@ -772,9 +775,7 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) { STrans* pTransInst = pThrd->pTransInst; cliMayCvtFqdnToIp(&pCtx->epSet, &pThrd->cvtAddr); - transPrintEpSet(&pCtx->epSet); - SCliConn* conn = cliGetConn(pMsg, pThrd); if (conn != NULL) { transCtxMerge(&conn->ctx, &pCtx->appCtx); @@ -1112,7 +1113,6 @@ void transReleaseCliHandle(void* handle) { if (pThrd == NULL) { return; } - STransMsg tmsg = {.info.handle = handle}; SCliMsg* cmsg = taosMemoryCalloc(1, sizeof(SCliMsg)); cmsg->msg = tmsg; @@ -1162,7 +1162,6 @@ void transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransM transFreeMsg(pReq->pCont); return; } - tsem_t* sem = taosMemoryCalloc(1, sizeof(tsem_t)); tsem_init(sem, 0, 0); diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 599d98a3e9..215323f69d 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -1029,8 +1029,6 @@ void transCloseServer(void* arg) { int ref = atomic_sub_fetch_32(&tranSSvrInst, 1); if (ref == 0) { - // TdThreadOnce tmpInit = PTHREAD_ONCE_INIT; - // memcpy(&transModuleInit, &tmpInit, sizeof(TdThreadOnce)); transCloseExHandleMgt(refMgt); } } From 78be460ac31669b0373b76cc246f8684eec3800e Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 23 Jun 2022 21:40:12 +0800 Subject: [PATCH 09/49] handle except --- source/libs/transport/src/transCli.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 9ccbf3eaa0..a95afa8a27 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -715,8 +715,7 @@ static void cliHandleRelease(SCliMsg* pMsg, SCliThrd* pThrd) { int64_t refId = (int64_t)(pMsg->msg.info.handle); SExHandle* exh = transAcquireExHandle(refMgt, refId); if (exh == NULL) { - tDebug("%" PRid64 " already release", refId); - return NULL; + tDebug("%" PRId64 " already release", refId); } SCliConn* conn = exh->handle; From 3795ff37ad4955c5f7d5078e2ce028f34996a772 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 24 Jun 2022 10:38:18 +0800 Subject: [PATCH 10/49] handle except --- source/libs/transport/src/transCli.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index a95afa8a27..ce6c85bb57 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -197,7 +197,6 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { if (status != ConnInPool) { \ addConnToPool(((SCliThrd*)conn->hostThrd)->pool, conn); \ } \ - transRemoveExHandle(refMgt, conn->refId); \ return; \ } \ } while (0) From 87f4f5364f9df0d93ca5d40bda4d649f4cede862 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 24 Jun 2022 11:09:34 +0800 Subject: [PATCH 11/49] handle except --- source/libs/transport/src/transCli.c | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index ce6c85bb57..cb78d7f0df 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -953,7 +953,6 @@ int cliRBChoseIdx(STrans* pTransInst) { } static void doDelayTask(void* param) { STaskArg* arg = param; - SCliMsg* pMsg = arg->param1; SCliThrd* pThrd = arg->param2; cliHandleReq(pMsg, pThrd); @@ -977,13 +976,11 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { if (pCtx->retryCount == 0) { pCtx->origEpSet = pCtx->epSet; } + /* - * upper layer handle retry if code equal TSDB_CODE_RPC_NETWORK_UNAVAIL - */ - /* - * no retry - * 1. query conn 2. rpc thread already receive quit msg - * + * no retry + * 1. query conn + * 2. rpc thread already receive quit msg */ if (CONN_NO_PERSIST_BY_APP(pConn) && pThrd->quit == false) { tmsg_t msgType = pCtx->msgType; From ab23faed636ee498dd413e2d3e230804263c9577 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 24 Jun 2022 15:52:13 +0800 Subject: [PATCH 12/49] fix: planner memory leak --- source/libs/nodes/src/nodesUtilFuncs.c | 27 ++++++++++++++++++---- source/libs/parser/src/parTranslater.c | 5 +--- source/libs/planner/src/planLogicCreater.c | 10 ++++---- source/libs/planner/src/planOptimizer.c | 3 ++- source/libs/planner/src/planPhysiCreater.c | 12 ++++++++++ source/libs/planner/src/planSpliter.c | 9 +++++--- 6 files changed, 48 insertions(+), 18 deletions(-) diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 1d8baf5373..8fced1f2c5 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -346,9 +346,11 @@ static void destroyVgDataBlockArray(SArray* pArray) { } static void destroyLogicNode(SLogicNode* pNode) { - nodesDestroyList(pNode->pChildren); - nodesDestroyNode(pNode->pConditions); nodesDestroyList(pNode->pTargets); + nodesDestroyNode(pNode->pConditions); + nodesDestroyList(pNode->pChildren); + nodesDestroyNode(pNode->pLimit); + nodesDestroyNode(pNode->pSlimit); } static void destroyPhysiNode(SPhysiNode* pNode) { @@ -368,6 +370,7 @@ static void destroyWinodwPhysiNode(SWinodwPhysiNode* pNode) { static void destroyScanPhysiNode(SScanPhysiNode* pNode) { destroyPhysiNode((SPhysiNode*)pNode); nodesDestroyList(pNode->pScanCols); + nodesDestroyList(pNode->pScanPseudoCols); } static void destroyDataSinkNode(SDataSinkNode* pNode) { nodesDestroyNode((SNode*)pNode->pInputDataBlockDesc); } @@ -516,6 +519,9 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyNode(pStmt->pWindow); nodesDestroyList(pStmt->pGroupByList); nodesDestroyNode(pStmt->pHaving); + nodesDestroyNode(pStmt->pRange); + nodesDestroyNode(pStmt->pEvery); + nodesDestroyNode(pStmt->pFill); nodesDestroyList(pStmt->pOrderByList); nodesDestroyNode((SNode*)pStmt->pLimit); nodesDestroyNode((SNode*)pStmt->pSlimit); @@ -779,6 +785,8 @@ void nodesDestroyNode(SNode* pNode) { SInterpFuncLogicNode* pLogicNode = (SInterpFuncLogicNode*)pNode; destroyLogicNode((SLogicNode*)pLogicNode); nodesDestroyList(pLogicNode->pFuncs); + nodesDestroyNode(pLogicNode->pFillValues); + nodesDestroyNode(pLogicNode->pTimeSeries); break; } case QUERY_NODE_LOGIC_SUBPLAN: { @@ -793,14 +801,21 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyList(((SQueryLogicPlan*)pNode)->pTopSubplans); break; case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: - case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: - case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN: - case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN: case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN: case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN: case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN: destroyScanPhysiNode((SScanPhysiNode*)pNode); break; + case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN: { + STableScanPhysiNode* pPhyNode = (STableScanPhysiNode*)pNode; + destroyScanPhysiNode((SScanPhysiNode*)pNode); + nodesDestroyList(pPhyNode->pDynamicScanFuncs); + nodesDestroyList(pPhyNode->pPartitionTags); + break; + } case QUERY_NODE_PHYSICAL_PLAN_PROJECT: { SProjectPhysiNode* pPhyNode = (SProjectPhysiNode*)pNode; destroyPhysiNode((SPhysiNode*)pPhyNode); @@ -891,6 +906,8 @@ void nodesDestroyNode(SNode* pNode) { destroyPhysiNode((SPhysiNode*)pPhyNode); nodesDestroyList(pPhyNode->pExprs); nodesDestroyList(pPhyNode->pFuncs); + nodesDestroyNode(pPhyNode->pFillValues); + nodesDestroyNode(pPhyNode->pTimeSeries); break; } case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 424968ac80..5d464fdf27 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1840,10 +1840,7 @@ static int32_t createMultiResFuncsFromStar(STranslateContext* pCxt, SFunctionNod code = createMultiResFuncs(pSrcFunc, pExprs, pOutput); } - if (TSDB_CODE_SUCCESS != code) { - nodesDestroyList(pExprs); - } - + nodesDestroyList(pExprs); return code; } diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index ffb97bcc04..d4f18a663c 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -780,8 +780,8 @@ static int32_t createProjectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSel return TSDB_CODE_OUT_OF_MEMORY; } - pProject->node.pLimit = (SNode*)pSelect->pLimit; - pProject->node.pSlimit = (SNode*)pSelect->pSlimit; + TSWAP(pProject->node.pLimit, pSelect->pLimit); + TSWAP(pProject->node.pSlimit, pSelect->pSlimit); int32_t code = TSDB_CODE_SUCCESS; @@ -940,7 +940,7 @@ static int32_t createSetOpSortLogicNode(SLogicPlanContext* pCxt, SSetOperator* p return TSDB_CODE_OUT_OF_MEMORY; } - pSort->node.pLimit = pSetOperator->pLimit; + TSWAP(pSort->node.pLimit, pSetOperator->pLimit); int32_t code = TSDB_CODE_SUCCESS; @@ -973,7 +973,7 @@ static int32_t createSetOpProjectLogicNode(SLogicPlanContext* pCxt, SSetOperator } if (NULL == pSetOperator->pOrderByList) { - pProject->node.pLimit = pSetOperator->pLimit; + TSWAP(pProject->node.pLimit, pSetOperator->pLimit); } int32_t code = TSDB_CODE_SUCCESS; @@ -1004,7 +1004,7 @@ static int32_t createSetOpAggLogicNode(SLogicPlanContext* pCxt, SSetOperator* pS } if (NULL == pSetOperator->pOrderByList) { - pAgg->node.pLimit = pSetOperator->pLimit; + TSWAP(pAgg->node.pSlimit, pSetOperator->pLimit); } int32_t code = TSDB_CODE_SUCCESS; diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 7862ad5445..d4964d3cd4 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -682,7 +682,7 @@ static EOrder opkGetPrimaryKeyOrder(SSortLogicNode* pSort) { static SNode* opkRewriteDownNode(SSortLogicNode* pSort) { SNode* pDownNode = nodesListGetNode(pSort->node.pChildren, 0); // todo - pSort->node.pChildren = NULL; + NODES_CLEAR_LIST(pSort->node.pChildren); return pDownNode; } @@ -1061,6 +1061,7 @@ static EDealRes partTagsOptRebuildTbanmeImpl(SNode** pNode, void* pContext) { } strcpy(pFunc->functionName, "tbname"); pFunc->funcType = FUNCTION_TYPE_TBNAME; + pFunc->node.resType = ((SColumnNode*)*pNode)->node.resType; nodesDestroyNode(*pNode); *pNode = (SNode*)pFunc; return DEAL_RES_IGNORE_CHILD; diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index efb0f790e1..be9592507e 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -862,6 +862,9 @@ static int32_t createIndefRowsFuncPhysiNode(SPhysiPlanContext* pCxt, SNodeList* nodesDestroyNode((SNode*)pIdfRowsFunc); } + nodesDestroyList(pPrecalcExprs); + nodesDestroyList(pFuncs); + return code; } @@ -913,6 +916,9 @@ static int32_t createInterpFuncPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pCh nodesDestroyNode((SNode*)pInterpFunc); } + nodesDestroyList(pPrecalcExprs); + nodesDestroyList(pFuncs); + return code; } @@ -1048,6 +1054,9 @@ static int32_t createWindowPhysiNodeFinalize(SPhysiPlanContext* pCxt, SNodeList* nodesDestroyNode((SNode*)pWindow); } + nodesDestroyList(pPrecalcExprs); + nodesDestroyList(pFuncs); + return code; } @@ -1241,6 +1250,9 @@ static int32_t createPartitionPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChi nodesDestroyNode((SNode*)pPart); } + nodesDestroyList(pPrecalcExprs); + nodesDestroyList(pPartitionKeys); + return code; } diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 0e35239621..8e8559ba3f 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -179,7 +179,7 @@ static bool stbSplNeedSplitWindow(bool streamQuery, SLogicNode* pNode) { return !stbSplHasGatherExecFunc(pWindow->pFuncs) && stbSplHasMultiTbScan(streamQuery, pNode); } } - + if (WINDOW_TYPE_STATE == pWindow->winType) { if (!streamQuery) { return stbSplHasMultiTbScan(streamQuery, pNode); @@ -374,7 +374,7 @@ static int32_t stbSplCreateMergeNode(SSplitContext* pCxt, SLogicSubplan* pSubpla int32_t code = TSDB_CODE_SUCCESS; pMerge->pInputs = nodesCloneList(pPartChild->pTargets); - // NULL == pSubplan means 'merge node' replaces 'split node'. + // NULL != pSubplan means 'merge node' replaces 'split node'. if (NULL == pSubplan) { pMerge->node.pTargets = nodesCloneList(pPartChild->pTargets); } else { @@ -390,6 +390,9 @@ static int32_t stbSplCreateMergeNode(SSplitContext* pCxt, SLogicSubplan* pSubpla code = replaceLogicNode(pSubplan, pSplitNode, (SLogicNode*)pMerge); } } + if (TSDB_CODE_SUCCESS == code && NULL != pSubplan) { + nodesDestroyNode((SNode*)pSplitNode); + } if (TSDB_CODE_SUCCESS != code) { nodesDestroyNode((SNode*)pMerge); } @@ -512,7 +515,7 @@ static int32_t stbSplSplitSessionOrStateForBatch(SSplitContext* pCxt, SStableSpl SLogicNode* pChild = (SLogicNode*)nodesListGetNode(pWindow->pChildren, 0); SNodeList* pMergeKeys = NULL; - int32_t code = stbSplCreateMergeKeysByPrimaryKey(((SWindowLogicNode*)pWindow)->pTspk, &pMergeKeys); + int32_t code = stbSplCreateMergeKeysByPrimaryKey(((SWindowLogicNode*)pWindow)->pTspk, &pMergeKeys); if (TSDB_CODE_SUCCESS == code) { code = stbSplCreateMergeNode(pCxt, pInfo->pSubplan, pChild, pMergeKeys, (SLogicNode*)pChild); From 445d7f2d90bf61b5efdd0e29ffc936ed016f9caa Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 24 Jun 2022 16:21:01 +0800 Subject: [PATCH 13/49] feat: refactor rpc quit --- source/client/src/clientEnv.c | 3 +- source/libs/transport/src/transCli.c | 139 +++++++++++++++------------ 2 files changed, 82 insertions(+), 60 deletions(-) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index f1e4107e23..ff9003b8fc 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -85,7 +85,8 @@ void closeTransporter(STscObj *pTscObj) { } static bool clientRpcRfp(int32_t code) { - if (code == TSDB_CODE_RPC_REDIRECT) { + if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED || + code == TSDB_CODE_SYN_NOT_LEADER) { return true; } else { return false; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index cb78d7f0df..1cacc84d79 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -111,6 +111,13 @@ static void cliDestroyConn(SCliConn* pConn, bool clear /*clear tcp handle o static void cliDestroy(uv_handle_t* handle); static void cliSend(SCliConn* pConn); +static bool cliIsEpsetUpdated(int32_t code, STransConnCtx* pCtx) { + if (code != 0) return false; + if (pCtx->retryCnt == 0) return false; + if (transEpSetIsEqual(&pCtx->epSet, &pCtx->origEpSet)) return false; + return true; +} + void cliMayCvtFqdnToIp(SEpSet* pEpSet, SCvtAddr* pCvtAddr); /* * set TCP connection timeout per-socket level @@ -154,7 +161,6 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { destroyCmsg(pMsg); } } - #define CLI_RELEASE_UV(loop) \ do { \ uv_walk(loop, cliWalkCb, NULL); \ @@ -183,7 +189,6 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { #define CONN_SHOULD_RELEASE(conn, head) \ do { \ if ((head)->release == 1 && (head->msgLen) == sizeof(*head)) { \ - int status = conn->status; \ uint64_t ahandle = head->ahandle; \ CONN_GET_MSGCTX_BY_AHANDLE(conn, ahandle); \ transClearBuffer(&conn->readBuf); \ @@ -194,9 +199,7 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { } \ destroyCmsg(pMsg); \ cliReleaseUnfinishedMsg(conn); \ - if (status != ConnInPool) { \ - addConnToPool(((SCliThrd*)conn->hostThrd)->pool, conn); \ - } \ + addConnToPool(((SCliThrd*)conn->hostThrd)->pool, conn); \ return; \ } \ } while (0) @@ -262,8 +265,25 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { #define REQUEST_PERSIS_HANDLE(msg) ((msg)->info.persistHandle == 1) #define REQUEST_RELEASE_HANDLE(cmsg) ((cmsg)->type == Release) +#define EPSET_GET_SIZE(epSet) (epSet)->numOfEps #define EPSET_GET_INUSE_IP(epSet) ((epSet)->eps[(epSet)->inUse].fqdn) #define EPSET_GET_INUSE_PORT(epSet) ((epSet)->eps[(epSet)->inUse].port) +#define EPSET_FORWARD_INUSE(epSet) \ + do { \ + (epSet)->inUse = (++((epSet)->inUse)) % ((epSet)->numOfEps); \ + } while (0) +#define EPSET_DEBUG_STR(epSet, buf) \ + do { \ + int len = snprintf(buf, sizeof(buf), "epset:{"); \ + for (int i = 0; i < (epSet)->numOfEps; i++) { \ + if (i == (epSet)->numOfEps - 1) { \ + len += snprintf(buf + len, sizeof(buf) - len, "%d. %s:%d", i, (epSet)->eps[i].fqdn, (epSet)->eps[i].port); \ + } else { \ + len += snprintf(buf + len, sizeof(buf) - len, "%d. %s:%d, ", i, (epSet)->eps[i].fqdn, (epSet)->eps[i].port); \ + } \ + } \ + len += snprintf(buf + len, sizeof(buf) - len, "}"); \ + } while (0); static void* cliWorkThread(void* arg); @@ -492,6 +512,10 @@ static void allocConnRef(SCliConn* conn, bool update) { conn->refId = exh->refId; } static void addConnToPool(void* pool, SCliConn* conn) { + if (conn->status == ConnInPool) { + assert(0); + return; + } SCliThrd* thrd = conn->hostThrd; CONN_HANDLE_THREAD_QUIT(thrd); @@ -505,7 +529,7 @@ static void addConnToPool(void* pool, SCliConn* conn) { char key[128] = {0}; CONN_CONSTRUCT_HASH_KEY(key, conn->ip, conn->port); - tTrace("%s conn %p added to conn pool, read buf cap: %d", CONN_GET_INST_LABEL(conn), conn, conn->readBuf.cap); + tTrace("%s conn %p added to conn pool, read buf cap:%d", CONN_GET_INST_LABEL(conn), conn, conn->readBuf.cap); SConnList* plist = taosHashGet((SHashObj*)pool, key, strlen(key)); // list already create before @@ -751,9 +775,9 @@ SCliConn* cliGetConn(SCliMsg* pMsg, SCliThrd* pThrd) { STransConnCtx* pCtx = pMsg->ctx; conn = getConnFromPool(pThrd->pool, EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet)); if (conn != NULL) { - tTrace("%s conn %p get from conn pool", CONN_GET_INST_LABEL(conn), conn); + tTrace("%s conn %p get from conn pool:%p", CONN_GET_INST_LABEL(conn), conn, pThrd->pool); } else { - tTrace("%s not found conn in conn pool %p", ((STrans*)pThrd->pTransInst)->label, pThrd->pool); + tTrace("%s not found conn in conn pool:%p", ((STrans*)pThrd->pTransInst)->label, pThrd->pool); } return conn; } @@ -773,7 +797,8 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) { STrans* pTransInst = pThrd->pTransInst; cliMayCvtFqdnToIp(&pCtx->epSet, &pThrd->cvtAddr); - transPrintEpSet(&pCtx->epSet); + + // transPrintEpSet(&pCtx->epSet); SCliConn* conn = cliGetConn(pMsg, pThrd); if (conn != NULL) { transCtxMerge(&conn->ctx, &pCtx->appCtx); @@ -955,11 +980,30 @@ static void doDelayTask(void* param) { STaskArg* arg = param; SCliMsg* pMsg = arg->param1; SCliThrd* pThrd = arg->param2; - cliHandleReq(pMsg, pThrd); - taosMemoryFree(arg); + + cliHandleReq(pMsg, pThrd); } +static void cliSchedMsgToNextNode(SCliMsg* pMsg, SCliThrd* pThrd) { + STraceId* trace = &pMsg->msg.info.traceId; + STransConnCtx* pCtx = pMsg->ctx; + + char buf[256] = {0}; + EPSET_DEBUG_STR(&pCtx->epSet, buf); + tGTrace("%s %s, retryCnt:%d, limit:%d", transLabel(pThrd), buf, pCtx->retryCnt + 1, pCtx->retryLimit); + + STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); + arg->param1 = pMsg; + arg->param2 = pThrd; + transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL); +} + +void cliUpdateRetryLimit(int8_t* val, int8_t exp, int8_t newVal) { + if (*val != exp) { + *val = newVal; + } +} int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { SCliThrd* pThrd = pConn->hostThrd; STrans* pTransInst = pThrd->pTransInst; @@ -971,68 +1015,45 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { } STransConnCtx* pCtx = pMsg->ctx; - SEpSet* pEpSet = &pCtx->epSet; - - if (pCtx->retryCount == 0) { + if (pCtx->retryCnt == 0) { pCtx->origEpSet = pCtx->epSet; } - /* * no retry * 1. query conn * 2. rpc thread already receive quit msg */ - if (CONN_NO_PERSIST_BY_APP(pConn) && pThrd->quit == false) { - tmsg_t msgType = pCtx->msgType; - if ((pTransInst->retry != NULL && pEpSet->numOfEps > 1 && (pTransInst->retry(pResp->code))) || - (pResp->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || pResp->code == TSDB_CODE_APP_NOT_READY || - pResp->code == TSDB_CODE_NODE_NOT_DEPLOYED || pResp->code == TSDB_CODE_SYN_NOT_LEADER)) { + int32_t code = pResp->code; + if (CONN_NO_PERSIST_BY_APP(pConn)) { + if (pTransInst->retry != NULL && pTransInst->retry(code)) { pMsg->sent = 0; - tTrace("try to send req to next node"); - pMsg->st = taosGetTimestampUs(); + pCtx->retryCnt += 1; + if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { + transUnrefCliHandle(pConn); - pCtx->retryCount += 1; - if (pResp->code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { - if (pCtx->retryCount < pEpSet->numOfEps * 3) { - pEpSet->inUse = (++pEpSet->inUse) % pEpSet->numOfEps; - STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); - arg->param1 = pMsg; - arg->param2 = pThrd; - transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL); - transPrintEpSet(pEpSet); - tTrace("%s use local epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse, - pCtx->retryCount + 1, pEpSet->numOfEps * 3); - - transUnrefCliHandle(pConn); + cliUpdateRetryLimit(&pCtx->retryLimit, TRANS_RETRY_COUNT_LIMIT, EPSET_GET_SIZE(&pCtx->epSet) * 3); + if (pCtx->retryCnt < pCtx->retryLimit) { + EPSET_FORWARD_INUSE(&pCtx->epSet); + cliSchedMsgToNextNode(pMsg, pThrd); return -1; } - } else if (pCtx->retryCount < TRANS_RETRY_COUNT_LIMIT) { - if (pResp->contLen == 0) { - pEpSet->inUse = (++pEpSet->inUse) % pEpSet->numOfEps; - transPrintEpSet(&pCtx->epSet); - tTrace("%s use local epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse, - pCtx->retryCount + 1, TRANS_RETRY_COUNT_LIMIT); - } else { - SEpSet epSet = {0}; - tDeserializeSEpSet(pResp->pCont, pResp->contLen, &epSet); - pCtx->epSet = epSet; + } else { + addConnToPool(pThrd->pool, pConn); - transPrintEpSet(&pCtx->epSet); - tTrace("%s use remote epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse, - pCtx->retryCount + 1, TRANS_RETRY_COUNT_LIMIT); + cliUpdateRetryLimit(&pCtx->retryLimit, TRANS_RETRY_COUNT_LIMIT, TRANS_RETRY_COUNT_LIMIT); + if (pCtx->retryCnt < pCtx->retryLimit) { + if (pResp->contLen == 0) { + EPSET_FORWARD_INUSE(&pCtx->epSet); + } else { + tDeserializeSEpSet(pResp->pCont, pResp->contLen, &pCtx->epSet); + } + cliSchedMsgToNextNode(pMsg, pThrd); + return -1; } - if (pConn->status != ConnInPool) { - addConnToPool(pThrd->pool, pConn); - } - - STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); - arg->param1 = pMsg; - arg->param2 = pThrd; - transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL); - return -1; } } } + STraceId* trace = &pResp->info.traceId; if (pCtx->pSem != NULL) { tGTrace("%s conn %p(sync) handle resp", CONN_GET_INST_LABEL(pConn), pConn); @@ -1045,10 +1066,10 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { pCtx->pRsp = NULL; } else { tGTrace("%s conn %p handle resp", CONN_GET_INST_LABEL(pConn), pConn); - if (pResp->code != 0 || pCtx->retryCount == 0 || transEpSetIsEqual(&pCtx->epSet, &pCtx->origEpSet)) { + if (!cliIsEpsetUpdated(code, pCtx)) { pTransInst->cfp(pTransInst->parent, pResp, NULL); } else { - pTransInst->cfp(pTransInst->parent, pResp, pEpSet); + pTransInst->cfp(pTransInst->parent, pResp, &pCtx->epSet); } } return 0; From 4f629ec68bf46c38c6690a6ca680af8defb30b07 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Fri, 24 Jun 2022 16:53:56 +0800 Subject: [PATCH 14/49] enh(query): enhance cast function to support more types TD-15473 --- source/libs/function/src/builtins.c | 21 +++--- source/libs/scalar/src/sclfunc.c | 108 ++++++++++++++++++++++++++++ 2 files changed, 119 insertions(+), 10 deletions(-) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index f985cc324d..60107dee32 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1219,18 +1219,19 @@ static int32_t translateSubstr(SFunctionNode* pFunc, char* pErrBuf, int32_t len) static int32_t translateCast(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { // The number of parameters has been limited by the syntax definition - uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; + //uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; + // The function return type has been set during syntax parsing uint8_t para2Type = pFunc->node.resType.type; - if (para2Type != TSDB_DATA_TYPE_BIGINT && para2Type != TSDB_DATA_TYPE_UBIGINT && - para2Type != TSDB_DATA_TYPE_VARCHAR && para2Type != TSDB_DATA_TYPE_NCHAR && - para2Type != TSDB_DATA_TYPE_TIMESTAMP) { - return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); - } - if ((para2Type == TSDB_DATA_TYPE_TIMESTAMP && IS_VAR_DATA_TYPE(para1Type)) || - (para2Type == TSDB_DATA_TYPE_BINARY && para1Type == TSDB_DATA_TYPE_NCHAR)) { - return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); - } + //if (para2Type != TSDB_DATA_TYPE_BIGINT && para2Type != TSDB_DATA_TYPE_UBIGINT && + // para2Type != TSDB_DATA_TYPE_VARCHAR && para2Type != TSDB_DATA_TYPE_NCHAR && + // para2Type != TSDB_DATA_TYPE_TIMESTAMP) { + // return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + //} + //if ((para2Type == TSDB_DATA_TYPE_TIMESTAMP && IS_VAR_DATA_TYPE(para1Type)) || + // (para2Type == TSDB_DATA_TYPE_BINARY && para1Type == TSDB_DATA_TYPE_NCHAR)) { + // return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + //} int32_t para2Bytes = pFunc->node.resType.bytes; if (IS_VAR_DATA_TYPE(para2Type)) { diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 7ea0457592..721e38980e 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -730,6 +730,60 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp char *input = colDataGetData(pInput[0].columnData, i); switch(outputType) { + case TSDB_DATA_TYPE_TINYINT: { + if (inputType == TSDB_DATA_TYPE_BINARY) { + *(int8_t *)output = taosStr2Int8(varDataVal(input), NULL, 10); + } else if (inputType == TSDB_DATA_TYPE_NCHAR) { + char *newBuf = taosMemoryCalloc(1, inputLen); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + if (len < 0) { + taosMemoryFree(newBuf); + return TSDB_CODE_FAILED; + } + newBuf[len] = 0; + *(int8_t *)output = taosStr2Int8(newBuf, NULL, 10); + taosMemoryFree(newBuf); + } else { + GET_TYPED_DATA(*(int8_t *)output, int8_t, inputType, input); + } + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + if (inputType == TSDB_DATA_TYPE_BINARY) { + *(int16_t *)output = taosStr2Int16(varDataVal(input), NULL, 10); + } else if (inputType == TSDB_DATA_TYPE_NCHAR) { + char *newBuf = taosMemoryCalloc(1, inputLen); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + if (len < 0) { + taosMemoryFree(newBuf); + return TSDB_CODE_FAILED; + } + newBuf[len] = 0; + *(int16_t *)output = taosStr2Int16(newBuf, NULL, 10); + taosMemoryFree(newBuf); + } else { + GET_TYPED_DATA(*(int16_t *)output, int16_t, inputType, input); + } + break; + } + case TSDB_DATA_TYPE_INT: { + if (inputType == TSDB_DATA_TYPE_BINARY) { + *(int32_t *)output = taosStr2Int32(varDataVal(input), NULL, 10); + } else if (inputType == TSDB_DATA_TYPE_NCHAR) { + char *newBuf = taosMemoryCalloc(1, inputLen); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + if (len < 0) { + taosMemoryFree(newBuf); + return TSDB_CODE_FAILED; + } + newBuf[len] = 0; + *(int32_t *)output = taosStr2Int32(newBuf, NULL, 10); + taosMemoryFree(newBuf); + } else { + GET_TYPED_DATA(*(int32_t *)output, int32_t, inputType, input); + } + break; + } case TSDB_DATA_TYPE_BIGINT: { if (inputType == TSDB_DATA_TYPE_BINARY) { *(int64_t *)output = taosStr2Int64(varDataVal(input), NULL, 10); @@ -748,6 +802,60 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp } break; } + case TSDB_DATA_TYPE_UTINYINT: { + if (inputType == TSDB_DATA_TYPE_BINARY) { + *(uint8_t *)output = taosStr2UInt8(varDataVal(input), NULL, 10); + } else if (inputType == TSDB_DATA_TYPE_NCHAR) { + char *newBuf = taosMemoryCalloc(1, inputLen); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + if (len < 0) { + taosMemoryFree(newBuf); + return TSDB_CODE_FAILED; + } + newBuf[len] = 0; + *(uint8_t *)output = taosStr2UInt8(newBuf, NULL, 10); + taosMemoryFree(newBuf); + } else { + GET_TYPED_DATA(*(uint8_t *)output, uint8_t, inputType, input); + } + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + if (inputType == TSDB_DATA_TYPE_BINARY) { + *(uint16_t *)output = taosStr2UInt16(varDataVal(input), NULL, 10); + } else if (inputType == TSDB_DATA_TYPE_NCHAR) { + char *newBuf = taosMemoryCalloc(1, inputLen); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + if (len < 0) { + taosMemoryFree(newBuf); + return TSDB_CODE_FAILED; + } + newBuf[len] = 0; + *(uint16_t *)output = taosStr2UInt16(newBuf, NULL, 10); + taosMemoryFree(newBuf); + } else { + GET_TYPED_DATA(*(uint16_t *)output, uint16_t, inputType, input); + } + break; + } + case TSDB_DATA_TYPE_UINT: { + if (inputType == TSDB_DATA_TYPE_BINARY) { + *(uint32_t *)output = taosStr2UInt32(varDataVal(input), NULL, 10); + } else if (inputType == TSDB_DATA_TYPE_NCHAR) { + char *newBuf = taosMemoryCalloc(1, inputLen); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + if (len < 0) { + taosMemoryFree(newBuf); + return TSDB_CODE_FAILED; + } + newBuf[len] = 0; + *(uint32_t *)output = taosStr2UInt32(newBuf, NULL, 10); + taosMemoryFree(newBuf); + } else { + GET_TYPED_DATA(*(uint32_t *)output, uint32_t, inputType, input); + } + break; + } case TSDB_DATA_TYPE_UBIGINT: { if (inputType == TSDB_DATA_TYPE_BINARY) { *(uint64_t *)output = taosStr2UInt64(varDataVal(input), NULL, 10); From a215844467f8b5da581984aa01d04cda4c37ffc9 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 24 Jun 2022 17:02:53 +0800 Subject: [PATCH 15/49] handle except --- source/libs/transport/inc/transComm.h | 17 +++++++++-------- source/libs/transport/src/transCli.c | 2 +- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 5367f6b49d..db8de1c8c3 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -120,14 +120,15 @@ typedef struct SCvtAddr { } SCvtAddr; typedef struct { - SEpSet epSet; // ip list provided by app - SEpSet origEpSet; - void* ahandle; // handle provided by app - tmsg_t msgType; // message type - int8_t connType; // connection type cli/srv - int64_t rid; // refId returned by taosAddRef + SEpSet epSet; // ip list provided by app + SEpSet origEpSet; + void* ahandle; // handle provided by app + tmsg_t msgType; // message type + int8_t connType; // connection type cli/srv - int8_t retryCount; + int8_t retryCnt; + int8_t retryLimit; + // bool setMaxRetry; STransCtx appCtx; // STransMsg* pRsp; // for synchronous API tsem_t* pSem; // for synchronous API @@ -381,7 +382,7 @@ void transDQDestroy(SDelayQueue* queue); int transDQSched(SDelayQueue* queue, void (*func)(void* arg), void* arg, uint64_t timeoutMs); -void transPrintEpSet(SEpSet* pEpSet); +// void transPrintEpSet(SEpSet* pEpSet); bool transEpSetIsEqual(SEpSet* a, SEpSet* b); /* * init global func diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 1cacc84d79..b9eaae6cba 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1039,7 +1039,6 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { } } else { addConnToPool(pThrd->pool, pConn); - cliUpdateRetryLimit(&pCtx->retryLimit, TRANS_RETRY_COUNT_LIMIT, TRANS_RETRY_COUNT_LIMIT); if (pCtx->retryCnt < pCtx->retryLimit) { if (pResp->contLen == 0) { @@ -1047,6 +1046,7 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { } else { tDeserializeSEpSet(pResp->pCont, pResp->contLen, &pCtx->epSet); } + transFreeMsg(pResp->pCont); cliSchedMsgToNextNode(pMsg, pThrd); return -1; } From e54044de732badcbb4eda63731bbe07e7d3bce45 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 24 Jun 2022 19:24:14 +0800 Subject: [PATCH 16/49] update retry --- source/libs/transport/src/transCli.c | 34 ++++++++++++++-------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index b9eaae6cba..653666ee70 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -272,17 +272,17 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { do { \ (epSet)->inUse = (++((epSet)->inUse)) % ((epSet)->numOfEps); \ } while (0) -#define EPSET_DEBUG_STR(epSet, buf) \ - do { \ - int len = snprintf(buf, sizeof(buf), "epset:{"); \ - for (int i = 0; i < (epSet)->numOfEps; i++) { \ - if (i == (epSet)->numOfEps - 1) { \ - len += snprintf(buf + len, sizeof(buf) - len, "%d. %s:%d", i, (epSet)->eps[i].fqdn, (epSet)->eps[i].port); \ - } else { \ - len += snprintf(buf + len, sizeof(buf) - len, "%d. %s:%d, ", i, (epSet)->eps[i].fqdn, (epSet)->eps[i].port); \ - } \ - } \ - len += snprintf(buf + len, sizeof(buf) - len, "}"); \ +#define EPSET_DEBUG_STR(epSet, tbuf) \ + do { \ + int len = snprintf(tbuf, sizeof(tbuf), "epset:{"); \ + for (int i = 0; i < (epSet)->numOfEps; i++) { \ + if (i == (epSet)->numOfEps - 1) { \ + len += snprintf(tbuf + len, sizeof(tbuf) - len, "%d. %s:%d", i, (epSet)->eps[i].fqdn, (epSet)->eps[i].port); \ + } else { \ + len += snprintf(tbuf + len, sizeof(tbuf) - len, "%d. %s:%d, ", i, (epSet)->eps[i].fqdn, (epSet)->eps[i].port); \ + } \ + } \ + len += snprintf(tbuf + len, sizeof(tbuf) - len, "}, inUse:%d", (epSet)->inUse); \ } while (0); static void* cliWorkThread(void* arg); @@ -989,9 +989,10 @@ static void cliSchedMsgToNextNode(SCliMsg* pMsg, SCliThrd* pThrd) { STraceId* trace = &pMsg->msg.info.traceId; STransConnCtx* pCtx = pMsg->ctx; - char buf[256] = {0}; - EPSET_DEBUG_STR(&pCtx->epSet, buf); - tGTrace("%s %s, retryCnt:%d, limit:%d", transLabel(pThrd), buf, pCtx->retryCnt + 1, pCtx->retryLimit); + char tbuf[256] = {0}; + EPSET_DEBUG_STR(&pCtx->epSet, tbuf); + tGTrace("%s retry to send msg to next node %dms later , use %s, retryCnt:%d, limit:%d", transLabel(pThrd->pTransInst), + TRANS_RETRY_INTERVAL, tbuf, pCtx->retryCnt + 1, pCtx->retryLimit); STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); arg->param1 = pMsg; @@ -1029,18 +1030,17 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { pMsg->sent = 0; pCtx->retryCnt += 1; if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { - transUnrefCliHandle(pConn); - cliUpdateRetryLimit(&pCtx->retryLimit, TRANS_RETRY_COUNT_LIMIT, EPSET_GET_SIZE(&pCtx->epSet) * 3); if (pCtx->retryCnt < pCtx->retryLimit) { + transUnrefCliHandle(pConn); EPSET_FORWARD_INUSE(&pCtx->epSet); cliSchedMsgToNextNode(pMsg, pThrd); return -1; } } else { - addConnToPool(pThrd->pool, pConn); cliUpdateRetryLimit(&pCtx->retryLimit, TRANS_RETRY_COUNT_LIMIT, TRANS_RETRY_COUNT_LIMIT); if (pCtx->retryCnt < pCtx->retryLimit) { + addConnToPool(pThrd->pool, pConn); if (pResp->contLen == 0) { EPSET_FORWARD_INUSE(&pCtx->epSet); } else { From 720645800c43ec1ef7385ae4f612aac87a9d0af1 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 24 Jun 2022 20:51:00 +0800 Subject: [PATCH 17/49] feat: refactor rpc quit --- source/libs/transport/src/transCli.c | 53 +++++++++++++--------------- 1 file changed, 24 insertions(+), 29 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 653666ee70..9ca5e8e73e 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1014,42 +1014,36 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { pTransInst->cfp(pTransInst->parent, pResp, NULL); return 0; } - - STransConnCtx* pCtx = pMsg->ctx; - if (pCtx->retryCnt == 0) { - pCtx->origEpSet = pCtx->epSet; - } /* * no retry * 1. query conn * 2. rpc thread already receive quit msg */ - int32_t code = pResp->code; - if (CONN_NO_PERSIST_BY_APP(pConn)) { - if (pTransInst->retry != NULL && pTransInst->retry(code)) { - pMsg->sent = 0; - pCtx->retryCnt += 1; - if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { - cliUpdateRetryLimit(&pCtx->retryLimit, TRANS_RETRY_COUNT_LIMIT, EPSET_GET_SIZE(&pCtx->epSet) * 3); - if (pCtx->retryCnt < pCtx->retryLimit) { - transUnrefCliHandle(pConn); + STransConnCtx* pCtx = pMsg->ctx; + int32_t code = pResp->code; + if (pTransInst->retry != NULL && pTransInst->retry(code)) { + pMsg->sent = 0; + pCtx->retryCnt += 1; + if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { + cliUpdateRetryLimit(&pCtx->retryLimit, TRANS_RETRY_COUNT_LIMIT, EPSET_GET_SIZE(&pCtx->epSet) * 3); + if (pCtx->retryCnt < pCtx->retryLimit) { + transUnrefCliHandle(pConn); + EPSET_FORWARD_INUSE(&pCtx->epSet); + cliSchedMsgToNextNode(pMsg, pThrd); + return -1; + } + } else { + cliUpdateRetryLimit(&pCtx->retryLimit, TRANS_RETRY_COUNT_LIMIT, TRANS_RETRY_COUNT_LIMIT); + if (pCtx->retryCnt < pCtx->retryLimit) { + addConnToPool(pThrd->pool, pConn); + if (pResp->contLen == 0) { EPSET_FORWARD_INUSE(&pCtx->epSet); - cliSchedMsgToNextNode(pMsg, pThrd); - return -1; - } - } else { - cliUpdateRetryLimit(&pCtx->retryLimit, TRANS_RETRY_COUNT_LIMIT, TRANS_RETRY_COUNT_LIMIT); - if (pCtx->retryCnt < pCtx->retryLimit) { - addConnToPool(pThrd->pool, pConn); - if (pResp->contLen == 0) { - EPSET_FORWARD_INUSE(&pCtx->epSet); - } else { - tDeserializeSEpSet(pResp->pCont, pResp->contLen, &pCtx->epSet); - } - transFreeMsg(pResp->pCont); - cliSchedMsgToNextNode(pMsg, pThrd); - return -1; + } else { + tDeserializeSEpSet(pResp->pCont, pResp->contLen, &pCtx->epSet); } + transFreeMsg(pResp->pCont); + cliSchedMsgToNextNode(pMsg, pThrd); + return -1; } } } @@ -1185,6 +1179,7 @@ void transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransM STransConnCtx* pCtx = taosMemoryCalloc(1, sizeof(STransConnCtx)); pCtx->epSet = *pEpSet; + pCtx->origEpSet = *pEpSet; pCtx->ahandle = pReq->info.ahandle; pCtx->msgType = pReq->msgType; pCtx->pSem = sem; From 853f1df3abcfd071a6d584f36aea036edbd976a3 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 24 Jun 2022 21:07:12 +0800 Subject: [PATCH 18/49] feat: support partition by expression and aggregate function output together --- include/libs/nodes/plannodes.h | 1 + source/libs/function/src/builtins.c | 18 ++++++-- source/libs/parser/src/parTranslater.c | 44 ++++++++++++++----- source/libs/parser/test/parSelectTest.cpp | 23 ++++++++++ source/libs/planner/src/planLogicCreater.c | 8 ++-- source/libs/planner/src/planOptimizer.c | 15 ++++++- source/libs/planner/src/planSpliter.c | 20 ++++----- source/libs/planner/test/planOptimizeTest.cpp | 10 ++++- source/libs/planner/test/planPartByTest.cpp | 6 +++ 9 files changed, 115 insertions(+), 30 deletions(-) diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index f8cf58d8cb..a7b8c97740 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -75,6 +75,7 @@ typedef struct SScanLogicNode { double filesFactor; SArray* pSmaIndexes; SNodeList* pPartTags; + bool partSort; } SScanLogicNode; typedef struct SJoinLogicNode { diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 6516a10795..af5bebc601 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1274,13 +1274,12 @@ static bool validateTimestampDigits(const SValueNode* pVal) { } int64_t tsVal = pVal->datum.i; - char fraction[20] = {0}; + char fraction[20] = {0}; NUM_TO_STRING(pVal->node.resType.type, &tsVal, sizeof(fraction), fraction); int32_t tsDigits = (int32_t)strlen(fraction); if (tsDigits > TSDB_TIME_PRECISION_SEC_DIGITS) { - if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS || - tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS || + if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS || tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS || tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) { return true; } else { @@ -1510,6 +1509,11 @@ static int32_t translateBlockDistInfoFunc(SFunctionNode* pFunc, char* pErrBuf, i return TSDB_CODE_SUCCESS; } +static int32_t translateGroupKeyFunc(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + pFunc->node.resType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType; + return TSDB_CODE_SUCCESS; +} + static bool getBlockDistFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { pEnv->calcMemSize = sizeof(STableBlockDistInfo); return true; @@ -2499,6 +2503,14 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .type = FUNCTION_TYPE_BLOCK_DIST_INFO, .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_SCAN_PC_FUNC, .translateFunc = translateBlockDistInfoFunc, + }, + { + .name = "_group_key", + .type = FUNCTION_TYPE_BLOCK_DIST_INFO, + .classification = FUNC_MGT_AGG_FUNC, + .translateFunc = translateGroupKeyFunc, + .pPartialFunc = "_group_key", + .pMergeFunc = "_group_key" } }; // clang-format on diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 5d464fdf27..d94ec3460b 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1355,6 +1355,24 @@ static EDealRes rewriteColToSelectValFunc(STranslateContext* pCxt, SNode** pNode return TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR; } +static EDealRes rewriteExprToGroupKeyFunc(STranslateContext* pCxt, SNode** pNode) { + SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); + if (NULL == pFunc) { + pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; + return DEAL_RES_ERROR; + } + + strcpy(pFunc->functionName, "_group_key"); + strcpy(pFunc->node.aliasName, ((SExprNode*)*pNode)->aliasName); + pCxt->errCode = nodesListMakeAppend(&pFunc->pParameterList, *pNode); + if (TSDB_CODE_SUCCESS == pCxt->errCode) { + *pNode = (SNode*)pFunc; + pCxt->errCode = fmGetFuncInfo(pFunc, pCxt->msgBuf.buf, pCxt->msgBuf.len); + } + + return (TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR); +} + static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) { SCheckExprForGroupByCxt* pCxt = (SCheckExprForGroupByCxt*)pContext; if (!nodesIsExprNode(*pNode) || isAliasColumn(*pNode)) { @@ -1371,10 +1389,10 @@ static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) { if (isAggFunc(*pNode) && !isDistinctOrderBy(pCxt->pTranslateCxt)) { return DEAL_RES_IGNORE_CHILD; } - SNode* pGroupNode; + SNode* pGroupNode = NULL; FOREACH(pGroupNode, getGroupByList(pCxt->pTranslateCxt)) { if (nodesEqualNode(getGroupByNode(pGroupNode), *pNode)) { - return DEAL_RES_IGNORE_CHILD; + return rewriteExprToGroupKeyFunc(pCxt->pTranslateCxt, pNode); } } if (isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) { @@ -1441,22 +1459,28 @@ typedef struct CheckAggColCoexistCxt { bool existOtherAggFunc; } CheckAggColCoexistCxt; -static EDealRes doCheckAggColCoexist(SNode* pNode, void* pContext) { +static EDealRes doCheckAggColCoexist(SNode** pNode, void* pContext) { CheckAggColCoexistCxt* pCxt = (CheckAggColCoexistCxt*)pContext; - if (isSelectFunc(pNode)) { + if (isSelectFunc(*pNode)) { ++(pCxt->selectFuncNum); - } else if (isAggFunc(pNode)) { + } else if (isAggFunc(*pNode)) { pCxt->existOtherAggFunc = true; } - if (isAggFunc(pNode)) { + if (isAggFunc(*pNode)) { pCxt->existAggFunc = true; return DEAL_RES_IGNORE_CHILD; } - if (isIndefiniteRowsFunc(pNode)) { + if (isIndefiniteRowsFunc(*pNode)) { pCxt->existIndefiniteRowsFunc = true; return DEAL_RES_IGNORE_CHILD; } - if (isScanPseudoColumnFunc(pNode) || QUERY_NODE_COLUMN == nodeType(pNode)) { + SNode* pPartKey = NULL; + FOREACH(pPartKey, pCxt->pTranslateCxt->pCurrSelectStmt->pPartitionByList) { + if (nodesEqualNode(pPartKey, *pNode)) { + return rewriteExprToGroupKeyFunc(pCxt->pTranslateCxt, pNode); + } + } + if (isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) { pCxt->existCol = true; } return DEAL_RES_CONTINUE; @@ -1472,9 +1496,9 @@ static int32_t checkAggColCoexist(STranslateContext* pCxt, SSelectStmt* pSelect) .existIndefiniteRowsFunc = false, .selectFuncNum = 0, .existOtherAggFunc = false}; - nodesWalkExprs(pSelect->pProjectionList, doCheckAggColCoexist, &cxt); + nodesRewriteExprs(pSelect->pProjectionList, doCheckAggColCoexist, &cxt); if (!pSelect->isDistinct) { - nodesWalkExprs(pSelect->pOrderByList, doCheckAggColCoexist, &cxt); + nodesRewriteExprs(pSelect->pOrderByList, doCheckAggColCoexist, &cxt); } if (1 == cxt.selectFuncNum && !cxt.existOtherAggFunc) { return rewriteColsToSelectValFunc(pCxt, pSelect); diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index 5c8439f846..f8b8dc58fc 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -199,6 +199,20 @@ TEST_F(ParserSelectTest, tailFuncSemanticCheck) { run("SELECT TAIL(c1, 10) FROM t1 GROUP BY c2", TSDB_CODE_PAR_GROUP_BY_NOT_ALLOWED_FUNC); } +TEST_F(ParserSelectTest, partitionBy) { + useDb("root", "test"); + + run("SELECT c1, c2 FROM t1 PARTITION BY c2"); + + run("SELECT SUM(c1), c2 FROM t1 PARTITION BY c2"); +} + +TEST_F(ParserSelectTest, partitionBySemanticCheck) { + useDb("root", "test"); + + run("SELECT SUM(c1), c2, c3 FROM t1 PARTITION BY c2", TSDB_CODE_PAR_NOT_SINGLE_GROUP); +} + TEST_F(ParserSelectTest, groupBy) { useDb("root", "test"); @@ -213,6 +227,15 @@ TEST_F(ParserSelectTest, groupBy) { run("SELECT COUNT(*), c1 + 10, c2 cnt FROM t1 WHERE c1 > 0 GROUP BY c1 + 10, c2"); } +TEST_F(ParserSelectTest, groupBySemanticCheck) { + useDb("root", "test"); + + run("SELECT COUNT(*) cnt, c1 FROM t1 WHERE c1 > 0", TSDB_CODE_PAR_NOT_SINGLE_GROUP); + run("SELECT COUNT(*) cnt, c2 FROM t1 WHERE c1 > 0 GROUP BY c1", TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION); + run("SELECT COUNT(*) cnt, c2 FROM t1 WHERE c1 > 0 PARTITION BY c2 GROUP BY c1", + TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION); +} + TEST_F(ParserSelectTest, orderBy) { useDb("root", "test"); diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index d4f18a663c..6d4aa5ae23 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -457,15 +457,15 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, } } + if (TSDB_CODE_SUCCESS == code && pSelect->hasAggFuncs) { + code = nodesCollectFuncs(pSelect, SQL_CLAUSE_GROUP_BY, fmIsAggFunc, &pAgg->pAggFuncs); + } + // rewrite the expression in subsequent clauses if (TSDB_CODE_SUCCESS == code) { code = rewriteExprsForSelect(pAgg->pGroupKeys, pSelect, SQL_CLAUSE_GROUP_BY); } - if (TSDB_CODE_SUCCESS == code && pSelect->hasAggFuncs) { - code = nodesCollectFuncs(pSelect, SQL_CLAUSE_GROUP_BY, fmIsAggFunc, &pAgg->pAggFuncs); - } - // rewrite the expression in subsequent clauses if (TSDB_CODE_SUCCESS == code) { code = rewriteExprsForSelect(pAgg->pAggFuncs, pSelect, SQL_CLAUSE_GROUP_BY); diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index d4964d3cd4..7abfdb1f50 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1075,6 +1075,16 @@ static int32_t partTagsOptRebuildTbanme(SNodeList* pPartKeys) { return code; } +static int32_t partTagOptRewriteGroupKey(SAggLogicNode* pAgg, SScanLogicNode* pScan) { + // SNode* pNode = NULL; + // FOREACH(pNode, pScan->pPartTags) { + // if (QUERY_NODE_COLUMN != nodeType(pNode)) { + // createColumnByRewriteExpr(pNode, ); + // } + // } + return TSDB_CODE_SUCCESS; +} + static int32_t partTagsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { SLogicNode* pNode = optFindPossibleNode(pLogicSubplan->pNode, partTagsOptMayBeOptimized); if (NULL == pNode) { @@ -1100,6 +1110,9 @@ static int32_t partTagsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSub } } NODES_DESTORY_LIST(((SAggLogicNode*)pNode)->pGroupKeys); + if (TSDB_CODE_SUCCESS == code) { + code = partTagOptRewriteGroupKey((SAggLogicNode*)pNode, pScan); + } } if (TSDB_CODE_SUCCESS == code) { code = partTagsOptRebuildTbanme(pScan->pPartTags); @@ -1189,7 +1202,7 @@ static const SOptimizeRule optimizeRuleSet[] = { {.pName = "ConditionPushDown", .optimizeFunc = cpdOptimize}, {.pName = "OrderByPrimaryKey", .optimizeFunc = opkOptimize}, {.pName = "SmaIndex", .optimizeFunc = smaOptimize}, - // {.pName = "PartitionTags", .optimizeFunc = partTagsOptimize}, + {.pName = "PartitionTags", .optimizeFunc = partTagsOptimize}, {.pName = "EliminateProject", .optimizeFunc = eliminateProjOptimize} }; // clang-format on diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 8e8559ba3f..b4a966f206 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -390,9 +390,6 @@ static int32_t stbSplCreateMergeNode(SSplitContext* pCxt, SLogicSubplan* pSubpla code = replaceLogicNode(pSubplan, pSplitNode, (SLogicNode*)pMerge); } } - if (TSDB_CODE_SUCCESS == code && NULL != pSubplan) { - nodesDestroyNode((SNode*)pSplitNode); - } if (TSDB_CODE_SUCCESS != code) { nodesDestroyNode((SNode*)pMerge); } @@ -564,6 +561,8 @@ static int32_t stbSplSplitState(SSplitContext* pCxt, SStableSplitInfo* pInfo) { static SNodeList* stbSplGetPartKeys(SLogicNode* pNode) { if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) { return ((SScanLogicNode*)pNode)->pPartTags; + } else if (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode)) { + return ((SPartitionLogicNode*)pNode)->pPartitionKeys; } else { return NULL; } @@ -574,14 +573,15 @@ static bool stbSplIsPartTbanme(SNodeList* pPartKeys) { return false; } SNode* pPartKey = nodesListGetNode(pPartKeys, 0); - return QUERY_NODE_FUNCTION == nodeType(pPartKey) && FUNCTION_TYPE_TBNAME == ((SFunctionNode*)pPartKey)->funcType; + return (QUERY_NODE_FUNCTION == nodeType(pPartKey) && FUNCTION_TYPE_TBNAME == ((SFunctionNode*)pPartKey)->funcType) || + (QUERY_NODE_COLUMN == nodeType(pPartKey) && COLUMN_TYPE_TBNAME == ((SColumnNode*)pPartKey)->colType); } -static bool stbSplIsMultiTableWinodw(SWindowLogicNode* pWindow) { +static bool stbSplIsPartTableWinodw(SWindowLogicNode* pWindow) { return stbSplIsPartTbanme(stbSplGetPartKeys((SLogicNode*)nodesListGetNode(pWindow->node.pChildren, 0))); } -static int32_t stbSplSplitWindowForMergeTable(SSplitContext* pCxt, SStableSplitInfo* pInfo) { +static int32_t stbSplSplitWindowForCrossTable(SSplitContext* pCxt, SStableSplitInfo* pInfo) { switch (((SWindowLogicNode*)pInfo->pSplitNode)->winType) { case WINDOW_TYPE_INTERVAL: return stbSplSplitInterval(pCxt, pInfo); @@ -595,7 +595,7 @@ static int32_t stbSplSplitWindowForMergeTable(SSplitContext* pCxt, SStableSplitI return TSDB_CODE_PLAN_INTERNAL_ERROR; } -static int32_t stbSplSplitWindowForMultiTable(SSplitContext* pCxt, SStableSplitInfo* pInfo) { +static int32_t stbSplSplitWindowForPartTable(SSplitContext* pCxt, SStableSplitInfo* pInfo) { if (pCxt->pPlanCxt->streamQuery) { SPLIT_FLAG_SET_MASK(pInfo->pSubplan->splitFlag, SPLIT_FLAG_STABLE_SPLIT); return TSDB_CODE_SUCCESS; @@ -616,10 +616,10 @@ static int32_t stbSplSplitWindowForMultiTable(SSplitContext* pCxt, SStableSplitI } static int32_t stbSplSplitWindowNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) { - if (stbSplIsMultiTableWinodw((SWindowLogicNode*)pInfo->pSplitNode)) { - return stbSplSplitWindowForMultiTable(pCxt, pInfo); + if (stbSplIsPartTableWinodw((SWindowLogicNode*)pInfo->pSplitNode)) { + return stbSplSplitWindowForPartTable(pCxt, pInfo); } else { - return stbSplSplitWindowForMergeTable(pCxt, pInfo); + return stbSplSplitWindowForCrossTable(pCxt, pInfo); } } diff --git a/source/libs/planner/test/planOptimizeTest.cpp b/source/libs/planner/test/planOptimizeTest.cpp index 8b3d263d66..9bd95a79f9 100644 --- a/source/libs/planner/test/planOptimizeTest.cpp +++ b/source/libs/planner/test/planOptimizeTest.cpp @@ -57,9 +57,15 @@ TEST_F(PlanOptimizeTest, orderByPrimaryKey) { TEST_F(PlanOptimizeTest, PartitionTags) { useDb("root", "test"); - run("SELECT c1 FROM st1 PARTITION BY tag1"); + run("SELECT c1, tag1 FROM st1 PARTITION BY tag1"); - run("SELECT SUM(c1) FROM st1 GROUP BY tag1"); + run("SELECT SUM(c1), tag1 FROM st1 PARTITION BY tag1"); + + run("SELECT SUM(c1), tag1 + 10 FROM st1 PARTITION BY tag1 + 10"); + + run("SELECT SUM(c1), tag1 FROM st1 GROUP BY tag1"); + + run("SELECT SUM(c1), tag1 + 10 FROM st1 GROUP BY tag1 + 10"); } TEST_F(PlanOptimizeTest, eliminateProjection) { diff --git a/source/libs/planner/test/planPartByTest.cpp b/source/libs/planner/test/planPartByTest.cpp index 9437f6ad3f..2abcb44dfd 100644 --- a/source/libs/planner/test/planPartByTest.cpp +++ b/source/libs/planner/test/planPartByTest.cpp @@ -34,6 +34,8 @@ TEST_F(PlanPartitionByTest, withAggFunc) { useDb("root", "test"); run("select count(*) from t1 partition by c1"); + + run("select count(*), c1 from t1 partition by c1"); } TEST_F(PlanPartitionByTest, withInterval) { @@ -41,8 +43,12 @@ TEST_F(PlanPartitionByTest, withInterval) { // normal/child table run("select count(*) from t1 partition by c1 interval(10s)"); + + run("select count(*), c1 from t1 partition by c1 interval(10s)"); // super table run("select count(*) from st1 partition by tag1, tag2 interval(10s)"); + + run("select count(*), tag1 from st1 partition by tag1, tag2 interval(10s)"); } TEST_F(PlanPartitionByTest, withGroupBy) { From 6c49aecb700c4a4130bcf04b25ff8aafa36aa022 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 24 Jun 2022 22:54:06 +0800 Subject: [PATCH 19/49] fix(query): fix memory leak. --- source/libs/executor/src/scanoperator.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index b521983d8e..5834d33a80 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1022,6 +1022,8 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { } } + taosArrayDestroy(block.pDataBlock); + if (pInfo->pRes->pDataBlock == NULL) { // TODO add log pOperator->status = OP_EXEC_DONE; From fb1e845256f278d0da3f9ac5a1363b4515fb6dea Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sat, 25 Jun 2022 08:44:45 +0800 Subject: [PATCH 20/49] feat: support partition by expression and aggregate function output together --- source/libs/parser/src/parTranslater.c | 44 ++++++++++++++++++------- source/libs/planner/src/planOptimizer.c | 15 +-------- 2 files changed, 34 insertions(+), 25 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 62cf2a125b..7a6c340474 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1450,6 +1450,25 @@ static int32_t rewriteColsToSelectValFunc(STranslateContext* pCxt, SSelectStmt* return pCxt->errCode; } +static EDealRes rewriteExprsToGroupKeyFuncImpl(SNode** pNode, void* pContext) { + STranslateContext* pCxt = pContext; + SNode* pPartKey = NULL; + FOREACH(pPartKey, pCxt->pCurrSelectStmt->pPartitionByList) { + if (nodesEqualNode(pPartKey, *pNode)) { + return rewriteExprToGroupKeyFunc(pCxt, pNode); + } + } + return DEAL_RES_CONTINUE; +} + +static int32_t rewriteExprsToGroupKeyFunc(STranslateContext* pCxt, SSelectStmt* pSelect) { + nodesRewriteExprs(pSelect->pProjectionList, rewriteExprsToGroupKeyFuncImpl, pCxt); + if (TSDB_CODE_SUCCESS == pCxt->errCode && !pSelect->isDistinct) { + nodesRewriteExprs(pSelect->pOrderByList, rewriteExprsToGroupKeyFuncImpl, pCxt); + } + return pCxt->errCode; +} + typedef struct CheckAggColCoexistCxt { STranslateContext* pTranslateCxt; bool existAggFunc; @@ -1459,28 +1478,28 @@ typedef struct CheckAggColCoexistCxt { bool existOtherAggFunc; } CheckAggColCoexistCxt; -static EDealRes doCheckAggColCoexist(SNode** pNode, void* pContext) { +static EDealRes doCheckAggColCoexist(SNode* pNode, void* pContext) { CheckAggColCoexistCxt* pCxt = (CheckAggColCoexistCxt*)pContext; - if (isSelectFunc(*pNode)) { + if (isSelectFunc(pNode)) { ++(pCxt->selectFuncNum); - } else if (isAggFunc(*pNode)) { + } else if (isAggFunc(pNode)) { pCxt->existOtherAggFunc = true; } - if (isAggFunc(*pNode)) { + if (isAggFunc(pNode)) { pCxt->existAggFunc = true; return DEAL_RES_IGNORE_CHILD; } - if (isIndefiniteRowsFunc(*pNode)) { + if (isIndefiniteRowsFunc(pNode)) { pCxt->existIndefiniteRowsFunc = true; return DEAL_RES_IGNORE_CHILD; } SNode* pPartKey = NULL; FOREACH(pPartKey, pCxt->pTranslateCxt->pCurrSelectStmt->pPartitionByList) { - if (nodesEqualNode(pPartKey, *pNode)) { - return rewriteExprToGroupKeyFunc(pCxt->pTranslateCxt, pNode); + if (nodesEqualNode(pPartKey, pNode)) { + return DEAL_RES_IGNORE_CHILD; } } - if (isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) { + if (isScanPseudoColumnFunc(pNode) || QUERY_NODE_COLUMN == nodeType(pNode)) { pCxt->existCol = true; } return DEAL_RES_CONTINUE; @@ -1496,9 +1515,9 @@ static int32_t checkAggColCoexist(STranslateContext* pCxt, SSelectStmt* pSelect) .existIndefiniteRowsFunc = false, .selectFuncNum = 0, .existOtherAggFunc = false}; - nodesRewriteExprs(pSelect->pProjectionList, doCheckAggColCoexist, &cxt); + nodesWalkExprs(pSelect->pProjectionList, doCheckAggColCoexist, &cxt); if (!pSelect->isDistinct) { - nodesRewriteExprs(pSelect->pOrderByList, doCheckAggColCoexist, &cxt); + nodesWalkExprs(pSelect->pOrderByList, doCheckAggColCoexist, &cxt); } if (1 == cxt.selectFuncNum && !cxt.existOtherAggFunc) { return rewriteColsToSelectValFunc(pCxt, pSelect); @@ -1509,6 +1528,9 @@ static int32_t checkAggColCoexist(STranslateContext* pCxt, SSelectStmt* pSelect) if (cxt.existIndefiniteRowsFunc && cxt.existCol) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC); } + if (cxt.existAggFunc && NULL != pSelect->pPartitionByList) { + return rewriteExprsToGroupKeyFunc(pCxt, pSelect); + } return TSDB_CODE_SUCCESS; } @@ -6039,7 +6061,7 @@ static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) { TSWAP(pQuery->pCmdMsg, pCxt->pCmdMsg); pQuery->msgType = pQuery->pCmdMsg->msgType; } - break; + break; default: pQuery->execMode = QUERY_EXEC_MODE_RPC; if (NULL != pCxt->pCmdMsg) { diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 7abfdb1f50..d4964d3cd4 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1075,16 +1075,6 @@ static int32_t partTagsOptRebuildTbanme(SNodeList* pPartKeys) { return code; } -static int32_t partTagOptRewriteGroupKey(SAggLogicNode* pAgg, SScanLogicNode* pScan) { - // SNode* pNode = NULL; - // FOREACH(pNode, pScan->pPartTags) { - // if (QUERY_NODE_COLUMN != nodeType(pNode)) { - // createColumnByRewriteExpr(pNode, ); - // } - // } - return TSDB_CODE_SUCCESS; -} - static int32_t partTagsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { SLogicNode* pNode = optFindPossibleNode(pLogicSubplan->pNode, partTagsOptMayBeOptimized); if (NULL == pNode) { @@ -1110,9 +1100,6 @@ static int32_t partTagsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSub } } NODES_DESTORY_LIST(((SAggLogicNode*)pNode)->pGroupKeys); - if (TSDB_CODE_SUCCESS == code) { - code = partTagOptRewriteGroupKey((SAggLogicNode*)pNode, pScan); - } } if (TSDB_CODE_SUCCESS == code) { code = partTagsOptRebuildTbanme(pScan->pPartTags); @@ -1202,7 +1189,7 @@ static const SOptimizeRule optimizeRuleSet[] = { {.pName = "ConditionPushDown", .optimizeFunc = cpdOptimize}, {.pName = "OrderByPrimaryKey", .optimizeFunc = opkOptimize}, {.pName = "SmaIndex", .optimizeFunc = smaOptimize}, - {.pName = "PartitionTags", .optimizeFunc = partTagsOptimize}, + // {.pName = "PartitionTags", .optimizeFunc = partTagsOptimize}, {.pName = "EliminateProject", .optimizeFunc = eliminateProjOptimize} }; // clang-format on From e187f4290236ecc4c9bc5ad93ba692cb235330c0 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sat, 25 Jun 2022 09:42:50 +0800 Subject: [PATCH 21/49] feat: support partition by expression and aggregate function output together --- source/libs/parser/src/parTranslater.c | 21 ++++++++++++++++++-- source/libs/planner/src/planOptimizer.c | 2 +- source/libs/planner/test/planGroupByTest.cpp | 10 ++-------- 3 files changed, 22 insertions(+), 11 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 7a6c340474..c81ecfc3e9 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1369,6 +1369,7 @@ static EDealRes rewriteExprToGroupKeyFunc(STranslateContext* pCxt, SNode** pNode *pNode = (SNode*)pFunc; pCxt->errCode = fmGetFuncInfo(pFunc, pCxt->msgBuf.buf, pCxt->msgBuf.len); } + pCxt->pCurrSelectStmt->hasAggFuncs = true; return (TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR); } @@ -2506,8 +2507,24 @@ static SNode* createOrderByExpr(STranslateContext* pCxt) { return (SNode*)pOrder; } -// from: select tail(expr, k, f) from t where_clause partition_by_clause order_by_clause ... -// to: select expr from t where_clause order by _rowts desc limit k offset f +/* case 1: + * in: select tail(expr, k, f) from t where_clause + * out: select expr from t where_clause order by _rowts desc limit k offset f + * + * case 2: + * in: select tail(expr, k, f) from t where_clause partition_by_clause + * out: select expr from t where_clause partition_by_clause sort by _rowts desc limit k offset f + * + * case 3: + * in: select tail(expr, k, f) from t where_clause order_by_clause limit_clause + * out: select expr from ( + * select expr from t where_clause order by _rowts desc limit k offset f + * ) order_by_clause limit_clause + * + * case 4: + * in: select tail(expr, k, f) from t where_clause partition_by_clause limit_clause + * out: + */ static int32_t rewriteTailStmt(STranslateContext* pCxt, SSelectStmt* pSelect) { if (!pSelect->hasTailFunc) { return TSDB_CODE_SUCCESS; diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index d4964d3cd4..c1b12d9d20 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1189,7 +1189,7 @@ static const SOptimizeRule optimizeRuleSet[] = { {.pName = "ConditionPushDown", .optimizeFunc = cpdOptimize}, {.pName = "OrderByPrimaryKey", .optimizeFunc = opkOptimize}, {.pName = "SmaIndex", .optimizeFunc = smaOptimize}, - // {.pName = "PartitionTags", .optimizeFunc = partTagsOptimize}, + {.pName = "PartitionTags", .optimizeFunc = partTagsOptimize}, {.pName = "EliminateProject", .optimizeFunc = eliminateProjOptimize} }; // clang-format on diff --git a/source/libs/planner/test/planGroupByTest.cpp b/source/libs/planner/test/planGroupByTest.cpp index a73c99bf68..657d8a6244 100644 --- a/source/libs/planner/test/planGroupByTest.cpp +++ b/source/libs/planner/test/planGroupByTest.cpp @@ -53,14 +53,6 @@ TEST_F(PlanGroupByTest, aggFunc) { run("SELECT SUM(10), COUNT(c1) FROM t1 GROUP BY c2"); } -TEST_F(PlanGroupByTest, rewriteFunc) { - useDb("root", "test"); - - run("SELECT AVG(c1) FROM t1"); - - run("SELECT AVG(c1) FROM t1 GROUP BY c2"); -} - TEST_F(PlanGroupByTest, selectFunc) { useDb("root", "test"); @@ -81,6 +73,8 @@ TEST_F(PlanGroupByTest, stable) { run("SELECT COUNT(*) FROM st1"); + run("SELECT c1 FROM st1 GROUP BY c1"); + run("SELECT COUNT(*) FROM st1 GROUP BY c1"); run("SELECT COUNT(*) FROM st1 PARTITION BY c2 GROUP BY c1"); From 96b3603bfda19844cd9f3e06438ddb402b1e731b Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 25 Jun 2022 09:09:33 +0800 Subject: [PATCH 22/49] refactor: rename auth to privilege --- source/dnode/mnode/impl/inc/mndAuth.h | 13 ++++-- source/dnode/mnode/impl/src/mndAcct.c | 13 ++++++ source/dnode/mnode/impl/src/mndAuth.c | 10 ++-- source/dnode/mnode/impl/src/mndBnode.c | 4 +- source/dnode/mnode/impl/src/mndDb.c | 15 +++--- source/dnode/mnode/impl/src/mndDnode.c | 6 +-- source/dnode/mnode/impl/src/mndFunc.c | 4 +- source/dnode/mnode/impl/src/mndMnode.c | 4 +- source/dnode/mnode/impl/src/mndOffset.c | 16 ++++--- source/dnode/mnode/impl/src/mndProfile.c | 48 ++++++++------------ source/dnode/mnode/impl/src/mndQnode.c | 4 +- source/dnode/mnode/impl/src/mndShow.c | 2 +- source/dnode/mnode/impl/src/mndSma.c | 4 +- source/dnode/mnode/impl/src/mndSnode.c | 4 +- source/dnode/mnode/impl/src/mndStb.c | 6 +-- source/dnode/mnode/impl/src/mndStream.c | 4 +- source/dnode/mnode/impl/src/mndTopic.c | 2 +- source/dnode/mnode/impl/src/mndTrans.c | 3 +- source/dnode/mnode/impl/src/mndUser.c | 18 ++++---- source/dnode/mnode/impl/src/mndVgroup.c | 6 +-- tests/script/tsim/user/privilege_sysinfo.sim | 21 +++++++++ 21 files changed, 117 insertions(+), 90 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndAuth.h b/source/dnode/mnode/impl/inc/mndAuth.h index 81a776b652..c6f337b44e 100644 --- a/source/dnode/mnode/impl/inc/mndAuth.h +++ b/source/dnode/mnode/impl/inc/mndAuth.h @@ -24,6 +24,9 @@ extern "C" { typedef enum { MND_OPER_CONNECT = 1, + MND_OPER_CREATE_ACCT, + MND_OPER_DROP_ACCT, + MND_OPER_ALTER_ACCT, MND_OPER_CREATE_USER, MND_OPER_DROP_USER, MND_OPER_ALTER_USER, @@ -45,6 +48,8 @@ typedef enum { MND_OPER_CREATE_FUNC, MND_OPER_DROP_FUNC, MND_OPER_KILL_TRANS, + MND_OPER_KILL_CONN, + MND_OPER_KILL_QUERY, MND_OPER_CREATE_DB, MND_OPER_ALTER_DB, MND_OPER_DROP_DB, @@ -57,10 +62,10 @@ typedef enum { int32_t mndInitAuth(SMnode *pMnode); void mndCleanupAuth(SMnode *pMnode); -int32_t mndCheckOperAuth(SMnode *pMnode, const char *user, EOperType operType); -int32_t mndCheckDbAuth(SMnode *pMnode, const char *user, EOperType operType, SDbObj *pDb); -int32_t mndCheckShowAuth(SMnode *pMnode, const char *user, int32_t showType); -int32_t mndCheckAlterUserAuth(SUserObj *pOperUser, SUserObj *pUser, SAlterUserReq *pAlter); +int32_t mndCheckOperPrivilege(SMnode *pMnode, const char *user, EOperType operType); +int32_t mndCheckDbPrivilege(SMnode *pMnode, const char *user, EOperType operType, SDbObj *pDb); +int32_t mndCheckShowPrivilege(SMnode *pMnode, const char *user, int32_t showType); +int32_t mndCheckAlterUserPrivilege(SUserObj *pOperUser, SUserObj *pUser, SAlterUserReq *pAlter); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndAcct.c b/source/dnode/mnode/impl/src/mndAcct.c index 0ce4a8c76e..da78abf5c0 100644 --- a/source/dnode/mnode/impl/src/mndAcct.c +++ b/source/dnode/mnode/impl/src/mndAcct.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "mndAcct.h" +#include "mndAuth.h" #include "mndShow.h" #include "mndTrans.h" @@ -212,18 +213,30 @@ static int32_t mndAcctActionUpdate(SSdb *pSdb, SAcctObj *pOld, SAcctObj *pNew) { } static int32_t mndProcessCreateAcctReq(SRpcMsg *pReq) { + if (mndCheckOperPrivilege(pReq->info.node, pReq->info.conn.user, MND_OPER_CREATE_ACCT) != 0) { + return -1; + } + terrno = TSDB_CODE_MSG_NOT_PROCESSED; mError("failed to process create acct request since %s", terrstr()); return -1; } static int32_t mndProcessAlterAcctReq(SRpcMsg *pReq) { + if (mndCheckOperPrivilege(pReq->info.node, pReq->info.conn.user, MND_OPER_ALTER_ACCT) != 0) { + return -1; + } + terrno = TSDB_CODE_MSG_NOT_PROCESSED; mError("failed to process create acct request since %s", terrstr()); return -1; } static int32_t mndProcessDropAcctReq(SRpcMsg *pReq) { + if (mndCheckOperPrivilege(pReq->info.node, pReq->info.conn.user, MND_OPER_DROP_ACCT) != 0) { + return -1; + } + terrno = TSDB_CODE_MSG_NOT_PROCESSED; mError("failed to process create acct request since %s", terrstr()); return -1; diff --git a/source/dnode/mnode/impl/src/mndAuth.c b/source/dnode/mnode/impl/src/mndAuth.c index 4445e3b9f7..f1542f5d42 100644 --- a/source/dnode/mnode/impl/src/mndAuth.c +++ b/source/dnode/mnode/impl/src/mndAuth.c @@ -73,7 +73,7 @@ static int32_t mndProcessAuthReq(SRpcMsg *pReq) { return code; } -int32_t mndCheckOperAuth(SMnode *pMnode, const char *user, EOperType operType) { +int32_t mndCheckOperPrivilege(SMnode *pMnode, const char *user, EOperType operType) { int32_t code = 0; SUserObj *pUser = mndAcquireUser(pMnode, user); @@ -95,6 +95,8 @@ int32_t mndCheckOperAuth(SMnode *pMnode, const char *user, EOperType operType) { switch (operType) { case MND_OPER_CONNECT: + case MND_OPER_CREATE_FUNC: + case MND_OPER_DROP_FUNC: break; default: terrno = TSDB_CODE_MND_NO_RIGHTS; @@ -106,7 +108,7 @@ _OVER: return code; } -int32_t mndCheckAlterUserAuth(SUserObj *pOperUser, SUserObj *pUser, SAlterUserReq *pAlter) { +int32_t mndCheckAlterUserPrivilege(SUserObj *pOperUser, SUserObj *pUser, SAlterUserReq *pAlter) { if (pUser->superUser && pAlter->alterType != TSDB_ALTER_USER_PASSWD) { terrno = TSDB_CODE_MND_NO_RIGHTS; return -1; @@ -129,7 +131,7 @@ int32_t mndCheckAlterUserAuth(SUserObj *pOperUser, SUserObj *pUser, SAlterUserRe return -1; } -int32_t mndCheckShowAuth(SMnode *pMnode, const char *user, int32_t showType) { +int32_t mndCheckShowPrivilege(SMnode *pMnode, const char *user, int32_t showType) { int32_t code = 0; SUserObj *pUser = mndAcquireUser(pMnode, user); @@ -162,7 +164,7 @@ _OVER: return code; } -int32_t mndCheckDbAuth(SMnode *pMnode, const char *user, EOperType operType, SDbObj *pDb) { +int32_t mndCheckDbPrivilege(SMnode *pMnode, const char *user, EOperType operType, SDbObj *pDb) { int32_t code = 0; SUserObj *pUser = mndAcquireUser(pMnode, user); diff --git a/source/dnode/mnode/impl/src/mndBnode.c b/source/dnode/mnode/impl/src/mndBnode.c index e2b1aad008..debd7cbcc0 100644 --- a/source/dnode/mnode/impl/src/mndBnode.c +++ b/source/dnode/mnode/impl/src/mndBnode.c @@ -277,7 +277,7 @@ static int32_t mndProcessCreateBnodeReq(SRpcMsg *pReq) { } mDebug("bnode:%d, start to create", createReq.dnodeId); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_BNODE) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_BNODE) != 0) { goto _OVER; } @@ -382,7 +382,7 @@ static int32_t mndProcessDropBnodeReq(SRpcMsg *pReq) { } mDebug("bnode:%d, start to drop", dropReq.dnodeId); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_BNODE) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_BNODE) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 345464399e..00a5e609d7 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -506,6 +506,9 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) { } mDebug("db:%s, start to create, vgroups:%d", createReq.db, createReq.numOfVgroups); + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_DB, NULL) != 0) { + goto _OVER; + } pDb = mndAcquireDb(pMnode, createReq.db); if (pDb != NULL) { @@ -526,10 +529,6 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_DB, NULL) != 0) { - goto _OVER; - } - code = mndCreateDb(pMnode, pReq, &createReq, pUser); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; @@ -700,7 +699,7 @@ static int32_t mndProcessAlterDbReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_ALTER_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_ALTER_DB, pDb) != 0) { goto _OVER; } @@ -980,7 +979,7 @@ static int32_t mndProcessDropDbReq(SRpcMsg *pReq) { } } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_DB, pDb) != 0) { goto _OVER; } @@ -1127,7 +1126,7 @@ static int32_t mndProcessUseDbReq(SRpcMsg *pReq) { mError("db:%s, failed to process use db req since %s", usedbReq.db, terrstr()); } else { - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_USE_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_USE_DB, pDb) != 0) { goto _OVER; } @@ -1252,7 +1251,7 @@ static int32_t mndProcessCompactDbReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_COMPACT_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_COMPACT_DB, pDb) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 113777bc1f..6cf80ffe47 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -621,7 +621,7 @@ static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) { } mInfo("dnode:%s:%d, start to create", createReq.fqdn, createReq.port); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_DNODE) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_DNODE) != 0) { goto _OVER; } @@ -715,7 +715,7 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) { } mInfo("dnode:%d, start to drop, ep:%s:%d", dropReq.dnodeId, dropReq.fqdn, dropReq.port); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_MNODE) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_MNODE) != 0) { goto _OVER; } @@ -779,7 +779,7 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { } mInfo("dnode:%d, start to config, option:%s, value:%s", cfgReq.dnodeId, cfgReq.config, cfgReq.value); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CONFIG_DNODE) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONFIG_DNODE) != 0) { return -1; } diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index 37e0a719dd..7baa641aa7 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -283,7 +283,7 @@ static int32_t mndProcessCreateFuncReq(SRpcMsg *pReq) { } mDebug("func:%s, start to create", createReq.name); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_FUNC) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_FUNC) != 0) { goto _OVER; } @@ -346,7 +346,7 @@ static int32_t mndProcessDropFuncReq(SRpcMsg *pReq) { } mDebug("func:%s, start to drop", dropReq.name); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_FUNC) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_FUNC) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index bc3d23282c..1919340250 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -389,7 +389,7 @@ static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) { } mDebug("mnode:%d, start to create", createReq.dnodeId); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_MNODE) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_MNODE) != 0) { goto _OVER; } @@ -594,7 +594,7 @@ static int32_t mndProcessDropMnodeReq(SRpcMsg *pReq) { } mDebug("mnode:%d, start to drop", dropReq.dnodeId); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_MNODE) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_MNODE) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndOffset.c b/source/dnode/mnode/impl/src/mndOffset.c index 18f2e993b2..8b5a1401f0 100644 --- a/source/dnode/mnode/impl/src/mndOffset.c +++ b/source/dnode/mnode/impl/src/mndOffset.c @@ -36,13 +36,15 @@ static int32_t mndOffsetActionUpdate(SSdb *pSdb, SMqOffsetObj *pOffset, SMqOffse static int32_t mndProcessCommitOffsetReq(SRpcMsg *pReq); int32_t mndInitOffset(SMnode *pMnode) { - SSdbTable table = {.sdbType = SDB_OFFSET, - .keyType = SDB_KEY_BINARY, - .encodeFp = (SdbEncodeFp)mndOffsetActionEncode, - .decodeFp = (SdbDecodeFp)mndOffsetActionDecode, - .insertFp = (SdbInsertFp)mndOffsetActionInsert, - .updateFp = (SdbUpdateFp)mndOffsetActionUpdate, - .deleteFp = (SdbDeleteFp)mndOffsetActionDelete}; + SSdbTable table = { + .sdbType = SDB_OFFSET, + .keyType = SDB_KEY_BINARY, + .encodeFp = (SdbEncodeFp)mndOffsetActionEncode, + .decodeFp = (SdbDecodeFp)mndOffsetActionDecode, + .insertFp = (SdbInsertFp)mndOffsetActionInsert, + .updateFp = (SdbUpdateFp)mndOffsetActionUpdate, + .deleteFp = (SdbDeleteFp)mndOffsetActionDelete, + }; mndSetMsgHandle(pMnode, TDMT_MND_MQ_COMMIT_OFFSET, mndProcessCommitOffsetReq); diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index e9df4ae1d0..5e3fa3c1e3 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -227,6 +227,10 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { } taosIp2String(pReq->info.conn.clientIp, ip); + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONNECT) != 0) { + mGError("user:%s, failed to login from %s since %s", pReq->info.conn.user, ip, terrstr()); + goto _OVER; + } pUser = mndAcquireUser(pMnode, pReq->info.conn.user); if (pUser == NULL) { @@ -240,11 +244,6 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CONNECT) != 0) { - mGError("user:%s, failed to login from %s since %s", pReq->info.conn.user, ip, terrstr()); - goto _OVER; - } - if (connReq.db[0]) { char db[TSDB_DB_FNAME_LEN] = {0}; snprintf(db, TSDB_DB_FNAME_LEN, "%d%s%s", pUser->acctId, TS_PATH_DELIMITER, connReq.db); @@ -271,7 +270,7 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { connectRsp.connId = pConn->id; connectRsp.connType = connReq.connType; connectRsp.dnodeNum = mndGetDnodeSize(pMnode); - + strcpy(connectRsp.sVer, version); snprintf(connectRsp.sDetailVer, sizeof(connectRsp.sDetailVer), "ver:%s\nbuild:%s\ngitinfo:%s", version, buildinfo, gitinfo); @@ -475,16 +474,16 @@ static int32_t mndGetOnlineDnodeNum(SMnode *pMnode, int32_t *num) { SDnodeObj *pDnode = NULL; int64_t curMs = taosGetTimestampMs(); void *pIter = NULL; - + while (true) { pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pDnode); if (pIter == NULL) break; - + bool online = mndIsDnodeOnline(pDnode, curMs); if (online) { (*num)++; } - + sdbRelease(pSdb, pDnode); } @@ -652,15 +651,6 @@ static int32_t mndProcessKillQueryReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SProfileMgmt *pMgmt = &pMnode->profileMgmt; - SUserObj *pUser = mndAcquireUser(pMnode, pReq->info.conn.user); - if (pUser == NULL) return 0; - if (!pUser->superUser) { - mndReleaseUser(pMnode, pUser); - terrno = TSDB_CODE_MND_NO_RIGHTS; - return -1; - } - mndReleaseUser(pMnode, pUser); - SKillQueryReq killReq = {0}; if (tDeserializeSKillQueryReq(pReq->pCont, pReq->contLen, &killReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; @@ -668,6 +658,10 @@ static int32_t mndProcessKillQueryReq(SRpcMsg *pReq) { } mInfo("kill query msg is received, queryId:%s", killReq.queryStrId); + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_KILL_QUERY) != 0) { + return -1; + } + int32_t connId = 0; uint64_t queryId = 0; char *p = strchr(killReq.queryStrId, ':'); @@ -697,21 +691,16 @@ static int32_t mndProcessKillConnReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SProfileMgmt *pMgmt = &pMnode->profileMgmt; - SUserObj *pUser = mndAcquireUser(pMnode, pReq->info.conn.user); - if (pUser == NULL) return 0; - if (!pUser->superUser) { - mndReleaseUser(pMnode, pUser); - terrno = TSDB_CODE_MND_NO_RIGHTS; - return -1; - } - mndReleaseUser(pMnode, pUser); - SKillConnReq killReq = {0}; if (tDeserializeSKillConnReq(pReq->pCont, pReq->contLen, &killReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; return -1; } + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_KILL_CONN) != 0) { + return -1; + } + SConnObj *pConn = taosCacheAcquireByKey(pMgmt->connCache, &killReq.connId, sizeof(uint32_t)); if (pConn == NULL) { mError("connId:%u, failed to kill connection, conn not exist", killReq.connId); @@ -726,10 +715,10 @@ static int32_t mndProcessKillConnReq(SRpcMsg *pReq) { } static int32_t mndProcessSvrVerReq(SRpcMsg *pReq) { - int32_t code = -1; + int32_t code = -1; SServerVerRsp rsp = {0}; strcpy(rsp.ver, version); - + int32_t contLen = tSerializeSServerVerRsp(NULL, 0, &rsp); if (contLen < 0) goto _over; void *pRsp = rpcMallocCont(contLen); @@ -746,7 +735,6 @@ _over: return code; } - static int32_t mndRetrieveConns(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { SMnode *pMnode = pReq->info.node; SSdb *pSdb = pMnode->pSdb; diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index 9f1eb4ee24..4992a99c5c 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -279,7 +279,7 @@ static int32_t mndProcessCreateQnodeReq(SRpcMsg *pReq) { } mDebug("qnode:%d, start to create", createReq.dnodeId); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_QNODE) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_QNODE) != 0) { goto _OVER; } @@ -390,7 +390,7 @@ static int32_t mndProcessDropQnodeReq(SRpcMsg *pReq) { } mDebug("qnode:%d, start to drop", dropReq.dnodeId); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_QNODE) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_QNODE) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 27de3883e9..bd129a8045 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -231,7 +231,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { mDebug("show:0x%" PRIx64 ", start retrieve data, type:%d", pShow->id, pShow->type); - // if (mndCheckShowAuth(pMnode, pReq->info.conn.user, pShow->type) != 0) return -1; + // if (mndCheckShowPrivilege(pMnode, pReq->info.conn.user, pShow->type) != 0) return -1; int32_t numOfCols = pShow->pMeta->numOfColumns; SSDataBlock *pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 05603f8554..5445c1e09f 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -713,7 +713,7 @@ static int32_t mndProcessCreateSmaReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { goto _OVER; } @@ -974,7 +974,7 @@ static int32_t mndProcessDropSmaReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index a638bdf61f..ea5011a659 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -285,7 +285,7 @@ static int32_t mndProcessCreateSnodeReq(SRpcMsg *pReq) { } mDebug("snode:%d, start to create", createReq.dnodeId); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_SNODE) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_SNODE) != 0) { goto _OVER; } @@ -397,7 +397,7 @@ static int32_t mndProcessDropSnodeReq(SRpcMsg *pReq) { } mDebug("snode:%d, start to drop", dropReq.dnodeId); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_SNODE) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_SNODE) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index f1bae14c07..026ce97776 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -876,7 +876,7 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { goto _OVER; } @@ -1607,7 +1607,7 @@ static int32_t mndProcessAlterStbReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { goto _OVER; } @@ -1737,7 +1737,7 @@ static int32_t mndProcessDropStbReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 5e2f5bc2dd..8847344493 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -437,7 +437,7 @@ static int32_t mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStre goto _OVER; } - if (mndCheckDbAuth(pMnode, user, MND_OPER_WRITE_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, user, MND_OPER_WRITE_DB, pDb) != 0) { goto _OVER; } @@ -550,7 +550,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { goto _OVER; } #endif diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index a650ed29f1..d3c51d18fc 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -480,7 +480,7 @@ static int32_t mndProcessCreateTopicReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index d1d88fdc90..963efc9233 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -1384,8 +1384,7 @@ static int32_t mndProcessKillTransReq(SRpcMsg *pReq) { } mInfo("trans:%d, start to kill", killReq.transId); - - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_KILL_TRANS) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_KILL_TRANS) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 03c9647bfe..95e011db28 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -295,7 +295,7 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate tstrncpy(userObj.acct, acct, TSDB_USER_LEN); userObj.createdTime = taosGetTimestampMs(); userObj.updateTime = userObj.createdTime; - userObj.superUser = 0;//pCreate->superUser; + userObj.superUser = 0; // pCreate->superUser; userObj.sysInfo = pCreate->sysInfo; userObj.enable = pCreate->enable; @@ -337,6 +337,9 @@ static int32_t mndProcessCreateUserReq(SRpcMsg *pReq) { } mDebug("user:%s, start to create", createReq.user); + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_USER) != 0) { + goto _OVER; + } if (createReq.user[0] == 0) { terrno = TSDB_CODE_MND_INVALID_USER_FORMAT; @@ -360,10 +363,6 @@ static int32_t mndProcessCreateUserReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_USER) != 0) { - goto _OVER; - } - code = mndCreateUser(pMnode, pOperUser->acct, &createReq, pReq); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; @@ -466,7 +465,7 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckAlterUserAuth(pOperUser, pUser, &alterReq) != 0) { + if (mndCheckAlterUserPrivilege(pOperUser, pUser, &alterReq) != 0) { goto _OVER; } @@ -631,6 +630,9 @@ static int32_t mndProcessDropUserReq(SRpcMsg *pReq) { } mDebug("user:%s, start to drop", dropReq.user); + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_USER) != 0) { + goto _OVER; + } if (dropReq.user[0] == 0) { terrno = TSDB_CODE_MND_INVALID_USER_FORMAT; @@ -643,10 +645,6 @@ static int32_t mndProcessDropUserReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_USER) != 0) { - goto _OVER; - } - code = mndDropUser(pMnode, pReq, pUser); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 1c5d73031f..ccdeb2626e 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -1212,7 +1212,7 @@ static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq) { } mInfo("vgId:%d, start to redistribute vgroup to dnode %d:%d:%d", req.vgId, req.dnodeId1, req.dnodeId2, req.dnodeId3); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_REDISTRIBUTE_VGROUP) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_REDISTRIBUTE_VGROUP) != 0) { goto _OVER; } @@ -1507,7 +1507,7 @@ static int32_t mndProcessSplitVgroupMsg(SRpcMsg *pReq) { SDbObj *pDb = NULL; mDebug("vgId:%d, start to split", vgId); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_SPLIT_VGROUP) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_SPLIT_VGROUP) != 0) { goto _OVER; } @@ -1657,7 +1657,7 @@ static int32_t mndProcessBalanceVgroupMsg(SRpcMsg *pReq) { } mInfo("start to balance vgroup"); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_BALANCE_VGROUP) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_BALANCE_VGROUP) != 0) { goto _OVER; } diff --git a/tests/script/tsim/user/privilege_sysinfo.sim b/tests/script/tsim/user/privilege_sysinfo.sim index 9ddfce8a97..ea3294765c 100644 --- a/tests/script/tsim/user/privilege_sysinfo.sim +++ b/tests/script/tsim/user/privilege_sysinfo.sim @@ -22,5 +22,26 @@ sql_error drop user sysinfo1 sql_error alter user sysinfo1 pass '1' sql_error alter user sysinfo0 pass '1' +sql_error create dnode $hostname port 7200 +sql_error drop dnode 1 + +sql_error create qnode on dnode 1 +sql_error drop qnode on dnode 1 + +sql_error create mnode on dnode 1 +sql_error drop mnode on dnode 1 + +sql_error create snode on dnode 1 +sql_error drop snode on dnode 1 + +sql_error redistribute vgroup 2 dnode 1 dnode 2 +sql_error balance vgroup + +sql_error kill transaction 1 +sql_error kill connection 1 +sql_error kill query 1 + +print =============== check db +sql_error create database db system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file From 09ce47ae9ff32292c9deb484cd01d1999e6ef44a Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 25 Jun 2022 09:09:33 +0800 Subject: [PATCH 23/49] refactor: rename auth to privilege --- .../impl/inc/{mndAuth.h => mndPrivilege.h} | 23 ++++--- source/dnode/mnode/impl/src/mndAcct.c | 13 ++++ source/dnode/mnode/impl/src/mndBnode.c | 6 +- source/dnode/mnode/impl/src/mndConsumer.c | 2 +- source/dnode/mnode/impl/src/mndDb.c | 17 +++-- source/dnode/mnode/impl/src/mndDnode.c | 8 +-- source/dnode/mnode/impl/src/mndFunc.c | 6 +- source/dnode/mnode/impl/src/mndMain.c | 4 +- source/dnode/mnode/impl/src/mndMnode.c | 6 +- source/dnode/mnode/impl/src/mndOffset.c | 18 ++--- .../impl/src/{mndAuth.c => mndPrivilege.c} | 68 +++---------------- source/dnode/mnode/impl/src/mndProfile.c | 50 ++++++-------- source/dnode/mnode/impl/src/mndQnode.c | 6 +- source/dnode/mnode/impl/src/mndShow.c | 4 +- source/dnode/mnode/impl/src/mndSma.c | 6 +- source/dnode/mnode/impl/src/mndSnode.c | 6 +- source/dnode/mnode/impl/src/mndStb.c | 8 +-- source/dnode/mnode/impl/src/mndStream.c | 6 +- source/dnode/mnode/impl/src/mndTopic.c | 4 +- source/dnode/mnode/impl/src/mndTrans.c | 5 +- source/dnode/mnode/impl/src/mndUser.c | 20 +++--- source/dnode/mnode/impl/src/mndVgroup.c | 8 +-- tests/script/tsim/user/privilege_sysinfo.sim | 21 ++++++ 23 files changed, 145 insertions(+), 170 deletions(-) rename source/dnode/mnode/impl/inc/{mndAuth.h => mndPrivilege.h} (68%) rename source/dnode/mnode/impl/src/{mndAuth.c => mndPrivilege.c} (66%) diff --git a/source/dnode/mnode/impl/inc/mndAuth.h b/source/dnode/mnode/impl/inc/mndPrivilege.h similarity index 68% rename from source/dnode/mnode/impl/inc/mndAuth.h rename to source/dnode/mnode/impl/inc/mndPrivilege.h index 81a776b652..15f9e4e6b5 100644 --- a/source/dnode/mnode/impl/inc/mndAuth.h +++ b/source/dnode/mnode/impl/inc/mndPrivilege.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef _TD_MND_AUTH_H_ -#define _TD_MND_AUTH_H_ +#ifndef _TD_MND_PRIVILEGE_H +#define _TD_MND_PRIVILEGE_H #include "mndInt.h" @@ -24,6 +24,9 @@ extern "C" { typedef enum { MND_OPER_CONNECT = 1, + MND_OPER_CREATE_ACCT, + MND_OPER_DROP_ACCT, + MND_OPER_ALTER_ACCT, MND_OPER_CREATE_USER, MND_OPER_DROP_USER, MND_OPER_ALTER_USER, @@ -45,6 +48,8 @@ typedef enum { MND_OPER_CREATE_FUNC, MND_OPER_DROP_FUNC, MND_OPER_KILL_TRANS, + MND_OPER_KILL_CONN, + MND_OPER_KILL_QUERY, MND_OPER_CREATE_DB, MND_OPER_ALTER_DB, MND_OPER_DROP_DB, @@ -54,16 +59,16 @@ typedef enum { MND_OPER_READ_DB, } EOperType; -int32_t mndInitAuth(SMnode *pMnode); -void mndCleanupAuth(SMnode *pMnode); +int32_t mndInitPrivilege(SMnode *pMnode); +void mndCleanupPrivilege(SMnode *pMnode); -int32_t mndCheckOperAuth(SMnode *pMnode, const char *user, EOperType operType); -int32_t mndCheckDbAuth(SMnode *pMnode, const char *user, EOperType operType, SDbObj *pDb); -int32_t mndCheckShowAuth(SMnode *pMnode, const char *user, int32_t showType); -int32_t mndCheckAlterUserAuth(SUserObj *pOperUser, SUserObj *pUser, SAlterUserReq *pAlter); +int32_t mndCheckOperPrivilege(SMnode *pMnode, const char *user, EOperType operType); +int32_t mndCheckDbPrivilege(SMnode *pMnode, const char *user, EOperType operType, SDbObj *pDb); +int32_t mndCheckShowPrivilege(SMnode *pMnode, const char *user, int32_t showType); +int32_t mndCheckAlterUserPrivilege(SUserObj *pOperUser, SUserObj *pUser, SAlterUserReq *pAlter); #ifdef __cplusplus } #endif -#endif /*_TD_MND_AUTH_H_*/ +#endif /*_TD_MND_PRIVILEGE_H*/ diff --git a/source/dnode/mnode/impl/src/mndAcct.c b/source/dnode/mnode/impl/src/mndAcct.c index 0ce4a8c76e..33f0bb7a34 100644 --- a/source/dnode/mnode/impl/src/mndAcct.c +++ b/source/dnode/mnode/impl/src/mndAcct.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "mndAcct.h" +#include "mndPrivilege.h" #include "mndShow.h" #include "mndTrans.h" @@ -212,18 +213,30 @@ static int32_t mndAcctActionUpdate(SSdb *pSdb, SAcctObj *pOld, SAcctObj *pNew) { } static int32_t mndProcessCreateAcctReq(SRpcMsg *pReq) { + if (mndCheckOperPrivilege(pReq->info.node, pReq->info.conn.user, MND_OPER_CREATE_ACCT) != 0) { + return -1; + } + terrno = TSDB_CODE_MSG_NOT_PROCESSED; mError("failed to process create acct request since %s", terrstr()); return -1; } static int32_t mndProcessAlterAcctReq(SRpcMsg *pReq) { + if (mndCheckOperPrivilege(pReq->info.node, pReq->info.conn.user, MND_OPER_ALTER_ACCT) != 0) { + return -1; + } + terrno = TSDB_CODE_MSG_NOT_PROCESSED; mError("failed to process create acct request since %s", terrstr()); return -1; } static int32_t mndProcessDropAcctReq(SRpcMsg *pReq) { + if (mndCheckOperPrivilege(pReq->info.node, pReq->info.conn.user, MND_OPER_DROP_ACCT) != 0) { + return -1; + } + terrno = TSDB_CODE_MSG_NOT_PROCESSED; mError("failed to process create acct request since %s", terrstr()); return -1; diff --git a/source/dnode/mnode/impl/src/mndBnode.c b/source/dnode/mnode/impl/src/mndBnode.c index e2b1aad008..aafcd19992 100644 --- a/source/dnode/mnode/impl/src/mndBnode.c +++ b/source/dnode/mnode/impl/src/mndBnode.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "mndBnode.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndDnode.h" #include "mndShow.h" #include "mndTrans.h" @@ -277,7 +277,7 @@ static int32_t mndProcessCreateBnodeReq(SRpcMsg *pReq) { } mDebug("bnode:%d, start to create", createReq.dnodeId); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_BNODE) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_BNODE) != 0) { goto _OVER; } @@ -382,7 +382,7 @@ static int32_t mndProcessDropBnodeReq(SRpcMsg *pReq) { } mDebug("bnode:%d, start to drop", dropReq.dnodeId); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_BNODE) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_BNODE) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 4da3c906d7..7dc5ee1ea1 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "mndConsumer.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 345464399e..0345f1b345 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "mndDb.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndDnode.h" #include "mndOffset.h" #include "mndShow.h" @@ -506,6 +506,9 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) { } mDebug("db:%s, start to create, vgroups:%d", createReq.db, createReq.numOfVgroups); + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_DB, NULL) != 0) { + goto _OVER; + } pDb = mndAcquireDb(pMnode, createReq.db); if (pDb != NULL) { @@ -526,10 +529,6 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_DB, NULL) != 0) { - goto _OVER; - } - code = mndCreateDb(pMnode, pReq, &createReq, pUser); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; @@ -700,7 +699,7 @@ static int32_t mndProcessAlterDbReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_ALTER_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_ALTER_DB, pDb) != 0) { goto _OVER; } @@ -980,7 +979,7 @@ static int32_t mndProcessDropDbReq(SRpcMsg *pReq) { } } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_DB, pDb) != 0) { goto _OVER; } @@ -1127,7 +1126,7 @@ static int32_t mndProcessUseDbReq(SRpcMsg *pReq) { mError("db:%s, failed to process use db req since %s", usedbReq.db, terrstr()); } else { - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_USE_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_USE_DB, pDb) != 0) { goto _OVER; } @@ -1252,7 +1251,7 @@ static int32_t mndProcessCompactDbReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_COMPACT_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_COMPACT_DB, pDb) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 113777bc1f..af1d641ebf 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "mndDnode.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndMnode.h" #include "mndQnode.h" #include "mndShow.h" @@ -621,7 +621,7 @@ static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) { } mInfo("dnode:%s:%d, start to create", createReq.fqdn, createReq.port); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_DNODE) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_DNODE) != 0) { goto _OVER; } @@ -715,7 +715,7 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) { } mInfo("dnode:%d, start to drop, ep:%s:%d", dropReq.dnodeId, dropReq.fqdn, dropReq.port); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_MNODE) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_MNODE) != 0) { goto _OVER; } @@ -779,7 +779,7 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { } mInfo("dnode:%d, start to config, option:%s, value:%s", cfgReq.dnodeId, cfgReq.config, cfgReq.value); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CONFIG_DNODE) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONFIG_DNODE) != 0) { return -1; } diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index 37e0a719dd..b626c1fb04 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "mndFunc.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndShow.h" #include "mndSync.h" #include "mndTrans.h" @@ -283,7 +283,7 @@ static int32_t mndProcessCreateFuncReq(SRpcMsg *pReq) { } mDebug("func:%s, start to create", createReq.name); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_FUNC) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_FUNC) != 0) { goto _OVER; } @@ -346,7 +346,7 @@ static int32_t mndProcessDropFuncReq(SRpcMsg *pReq) { } mDebug("func:%s, start to drop", dropReq.name); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_FUNC) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_FUNC) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index f76dd31614..b454cf91fc 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "mndAcct.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndBnode.h" #include "mndCluster.h" #include "mndConsumer.h" @@ -239,7 +239,7 @@ static int32_t mndInitSteps(SMnode *pMnode) { if (mndAllocStep(pMnode, "mnode-dnode", mndInitDnode, mndCleanupDnode) != 0) return -1; if (mndAllocStep(pMnode, "mnode-user", mndInitUser, mndCleanupUser) != 0) return -1; if (mndAllocStep(pMnode, "mnode-grant", mndInitGrant, mndCleanupGrant) != 0) return -1; - if (mndAllocStep(pMnode, "mnode-auth", mndInitAuth, mndCleanupAuth) != 0) return -1; + if (mndAllocStep(pMnode, "mnode-privilege", mndInitPrivilege, mndCleanupPrivilege) != 0) return -1; if (mndAllocStep(pMnode, "mnode-acct", mndInitAcct, mndCleanupAcct) != 0) return -1; if (mndAllocStep(pMnode, "mnode-stream", mndInitStream, mndCleanupStream) != 0) return -1; if (mndAllocStep(pMnode, "mnode-topic", mndInitTopic, mndCleanupTopic) != 0) return -1; diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index bc3d23282c..c03951b1d8 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "mndMnode.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndDnode.h" #include "mndShow.h" #include "mndSync.h" @@ -389,7 +389,7 @@ static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) { } mDebug("mnode:%d, start to create", createReq.dnodeId); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_MNODE) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_MNODE) != 0) { goto _OVER; } @@ -594,7 +594,7 @@ static int32_t mndProcessDropMnodeReq(SRpcMsg *pReq) { } mDebug("mnode:%d, start to drop", dropReq.dnodeId); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_MNODE) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_MNODE) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndOffset.c b/source/dnode/mnode/impl/src/mndOffset.c index 18f2e993b2..e2b20b2163 100644 --- a/source/dnode/mnode/impl/src/mndOffset.c +++ b/source/dnode/mnode/impl/src/mndOffset.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "mndOffset.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" @@ -36,13 +36,15 @@ static int32_t mndOffsetActionUpdate(SSdb *pSdb, SMqOffsetObj *pOffset, SMqOffse static int32_t mndProcessCommitOffsetReq(SRpcMsg *pReq); int32_t mndInitOffset(SMnode *pMnode) { - SSdbTable table = {.sdbType = SDB_OFFSET, - .keyType = SDB_KEY_BINARY, - .encodeFp = (SdbEncodeFp)mndOffsetActionEncode, - .decodeFp = (SdbDecodeFp)mndOffsetActionDecode, - .insertFp = (SdbInsertFp)mndOffsetActionInsert, - .updateFp = (SdbUpdateFp)mndOffsetActionUpdate, - .deleteFp = (SdbDeleteFp)mndOffsetActionDelete}; + SSdbTable table = { + .sdbType = SDB_OFFSET, + .keyType = SDB_KEY_BINARY, + .encodeFp = (SdbEncodeFp)mndOffsetActionEncode, + .decodeFp = (SdbDecodeFp)mndOffsetActionDecode, + .insertFp = (SdbInsertFp)mndOffsetActionInsert, + .updateFp = (SdbUpdateFp)mndOffsetActionUpdate, + .deleteFp = (SdbDeleteFp)mndOffsetActionDelete, + }; mndSetMsgHandle(pMnode, TDMT_MND_MQ_COMMIT_OFFSET, mndProcessCommitOffsetReq); diff --git a/source/dnode/mnode/impl/src/mndAuth.c b/source/dnode/mnode/impl/src/mndPrivilege.c similarity index 66% rename from source/dnode/mnode/impl/src/mndAuth.c rename to source/dnode/mnode/impl/src/mndPrivilege.c index 4445e3b9f7..478ba2bee4 100644 --- a/source/dnode/mnode/impl/src/mndAuth.c +++ b/source/dnode/mnode/impl/src/mndPrivilege.c @@ -14,66 +14,14 @@ */ #define _DEFAULT_SOURCE -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndUser.h" -static int32_t mndProcessAuthReq(SRpcMsg *pReq); +int32_t mndInitPrivilege(SMnode *pMnode) { return 0; } -int32_t mndInitAuth(SMnode *pMnode) { - mndSetMsgHandle(pMnode, TDMT_MND_AUTH, mndProcessAuthReq); - return 0; -} +void mndCleanupPrivilege(SMnode *pMnode) {} -void mndCleanupAuth(SMnode *pMnode) {} - -static int32_t mndRetriveAuth(SMnode *pMnode, SAuthRsp *pRsp) { - SUserObj *pUser = mndAcquireUser(pMnode, pRsp->user); - if (pUser == NULL) { - *pRsp->secret = 0; - mError("user:%s, failed to auth user since %s", pRsp->user, terrstr()); - return -1; - } - - pRsp->spi = 1; - pRsp->encrypt = 0; - *pRsp->ckey = 0; - - memcpy(pRsp->secret, pUser->pass, TSDB_PASSWORD_LEN); - mndReleaseUser(pMnode, pUser); - - mDebug("user:%s, auth info is returned", pRsp->user); - return 0; -} - -static int32_t mndProcessAuthReq(SRpcMsg *pReq) { - SAuthReq authReq = {0}; - if (tDeserializeSAuthReq(pReq->pCont, pReq->contLen, &authReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - SAuthReq authRsp = {0}; - memcpy(authRsp.user, authReq.user, TSDB_USER_LEN); - - int32_t code = mndRetriveAuth(pReq->info.node, &authRsp); - mTrace("user:%s, auth req received, spi:%d encrypt:%d ruser:%s", pReq->info.conn.user, authRsp.spi, authRsp.encrypt, - authRsp.user); - - int32_t contLen = tSerializeSAuthReq(NULL, 0, &authRsp); - void *pRsp = rpcMallocCont(contLen); - if (pRsp == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - tSerializeSAuthReq(pRsp, contLen, &authRsp); - - pReq->info.rsp = pRsp; - pReq->info.rspLen = contLen; - return code; -} - -int32_t mndCheckOperAuth(SMnode *pMnode, const char *user, EOperType operType) { +int32_t mndCheckOperPrivilege(SMnode *pMnode, const char *user, EOperType operType) { int32_t code = 0; SUserObj *pUser = mndAcquireUser(pMnode, user); @@ -95,6 +43,8 @@ int32_t mndCheckOperAuth(SMnode *pMnode, const char *user, EOperType operType) { switch (operType) { case MND_OPER_CONNECT: + case MND_OPER_CREATE_FUNC: + case MND_OPER_DROP_FUNC: break; default: terrno = TSDB_CODE_MND_NO_RIGHTS; @@ -106,7 +56,7 @@ _OVER: return code; } -int32_t mndCheckAlterUserAuth(SUserObj *pOperUser, SUserObj *pUser, SAlterUserReq *pAlter) { +int32_t mndCheckAlterUserPrivilege(SUserObj *pOperUser, SUserObj *pUser, SAlterUserReq *pAlter) { if (pUser->superUser && pAlter->alterType != TSDB_ALTER_USER_PASSWD) { terrno = TSDB_CODE_MND_NO_RIGHTS; return -1; @@ -129,7 +79,7 @@ int32_t mndCheckAlterUserAuth(SUserObj *pOperUser, SUserObj *pUser, SAlterUserRe return -1; } -int32_t mndCheckShowAuth(SMnode *pMnode, const char *user, int32_t showType) { +int32_t mndCheckShowPrivilege(SMnode *pMnode, const char *user, int32_t showType) { int32_t code = 0; SUserObj *pUser = mndAcquireUser(pMnode, user); @@ -162,7 +112,7 @@ _OVER: return code; } -int32_t mndCheckDbAuth(SMnode *pMnode, const char *user, EOperType operType, SDbObj *pDb) { +int32_t mndCheckDbPrivilege(SMnode *pMnode, const char *user, EOperType operType, SDbObj *pDb) { int32_t code = 0; SUserObj *pUser = mndAcquireUser(pMnode, user); diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index e9df4ae1d0..f2e599b073 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "mndProfile.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" @@ -227,6 +227,10 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { } taosIp2String(pReq->info.conn.clientIp, ip); + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONNECT) != 0) { + mGError("user:%s, failed to login from %s since %s", pReq->info.conn.user, ip, terrstr()); + goto _OVER; + } pUser = mndAcquireUser(pMnode, pReq->info.conn.user); if (pUser == NULL) { @@ -240,11 +244,6 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CONNECT) != 0) { - mGError("user:%s, failed to login from %s since %s", pReq->info.conn.user, ip, terrstr()); - goto _OVER; - } - if (connReq.db[0]) { char db[TSDB_DB_FNAME_LEN] = {0}; snprintf(db, TSDB_DB_FNAME_LEN, "%d%s%s", pUser->acctId, TS_PATH_DELIMITER, connReq.db); @@ -271,7 +270,7 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { connectRsp.connId = pConn->id; connectRsp.connType = connReq.connType; connectRsp.dnodeNum = mndGetDnodeSize(pMnode); - + strcpy(connectRsp.sVer, version); snprintf(connectRsp.sDetailVer, sizeof(connectRsp.sDetailVer), "ver:%s\nbuild:%s\ngitinfo:%s", version, buildinfo, gitinfo); @@ -475,16 +474,16 @@ static int32_t mndGetOnlineDnodeNum(SMnode *pMnode, int32_t *num) { SDnodeObj *pDnode = NULL; int64_t curMs = taosGetTimestampMs(); void *pIter = NULL; - + while (true) { pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pDnode); if (pIter == NULL) break; - + bool online = mndIsDnodeOnline(pDnode, curMs); if (online) { (*num)++; } - + sdbRelease(pSdb, pDnode); } @@ -652,15 +651,6 @@ static int32_t mndProcessKillQueryReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SProfileMgmt *pMgmt = &pMnode->profileMgmt; - SUserObj *pUser = mndAcquireUser(pMnode, pReq->info.conn.user); - if (pUser == NULL) return 0; - if (!pUser->superUser) { - mndReleaseUser(pMnode, pUser); - terrno = TSDB_CODE_MND_NO_RIGHTS; - return -1; - } - mndReleaseUser(pMnode, pUser); - SKillQueryReq killReq = {0}; if (tDeserializeSKillQueryReq(pReq->pCont, pReq->contLen, &killReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; @@ -668,6 +658,10 @@ static int32_t mndProcessKillQueryReq(SRpcMsg *pReq) { } mInfo("kill query msg is received, queryId:%s", killReq.queryStrId); + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_KILL_QUERY) != 0) { + return -1; + } + int32_t connId = 0; uint64_t queryId = 0; char *p = strchr(killReq.queryStrId, ':'); @@ -697,21 +691,16 @@ static int32_t mndProcessKillConnReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SProfileMgmt *pMgmt = &pMnode->profileMgmt; - SUserObj *pUser = mndAcquireUser(pMnode, pReq->info.conn.user); - if (pUser == NULL) return 0; - if (!pUser->superUser) { - mndReleaseUser(pMnode, pUser); - terrno = TSDB_CODE_MND_NO_RIGHTS; - return -1; - } - mndReleaseUser(pMnode, pUser); - SKillConnReq killReq = {0}; if (tDeserializeSKillConnReq(pReq->pCont, pReq->contLen, &killReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; return -1; } + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_KILL_CONN) != 0) { + return -1; + } + SConnObj *pConn = taosCacheAcquireByKey(pMgmt->connCache, &killReq.connId, sizeof(uint32_t)); if (pConn == NULL) { mError("connId:%u, failed to kill connection, conn not exist", killReq.connId); @@ -726,10 +715,10 @@ static int32_t mndProcessKillConnReq(SRpcMsg *pReq) { } static int32_t mndProcessSvrVerReq(SRpcMsg *pReq) { - int32_t code = -1; + int32_t code = -1; SServerVerRsp rsp = {0}; strcpy(rsp.ver, version); - + int32_t contLen = tSerializeSServerVerRsp(NULL, 0, &rsp); if (contLen < 0) goto _over; void *pRsp = rpcMallocCont(contLen); @@ -746,7 +735,6 @@ _over: return code; } - static int32_t mndRetrieveConns(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { SMnode *pMnode = pReq->info.node; SSdb *pSdb = pMnode->pSdb; diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index 9f1eb4ee24..f057f6190d 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "mndQnode.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndDnode.h" #include "mndShow.h" #include "mndTrans.h" @@ -279,7 +279,7 @@ static int32_t mndProcessCreateQnodeReq(SRpcMsg *pReq) { } mDebug("qnode:%d, start to create", createReq.dnodeId); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_QNODE) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_QNODE) != 0) { goto _OVER; } @@ -390,7 +390,7 @@ static int32_t mndProcessDropQnodeReq(SRpcMsg *pReq) { } mDebug("qnode:%d, start to drop", dropReq.dnodeId); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_QNODE) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_QNODE) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 27de3883e9..5c2531c25f 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "mndShow.h" #include "systable.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #define SHOW_STEP_SIZE 100 @@ -231,7 +231,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { mDebug("show:0x%" PRIx64 ", start retrieve data, type:%d", pShow->id, pShow->type); - // if (mndCheckShowAuth(pMnode, pReq->info.conn.user, pShow->type) != 0) return -1; + // if (mndCheckShowPrivilege(pMnode, pReq->info.conn.user, pShow->type) != 0) return -1; int32_t numOfCols = pShow->pMeta->numOfColumns; SSDataBlock *pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 05603f8554..ef24cd0ba4 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "mndSma.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndDb.h" #include "mndDnode.h" #include "mndInfoSchema.h" @@ -713,7 +713,7 @@ static int32_t mndProcessCreateSmaReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { goto _OVER; } @@ -974,7 +974,7 @@ static int32_t mndProcessDropSmaReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index a638bdf61f..2dd8592bf8 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "mndSnode.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndDnode.h" #include "mndShow.h" #include "mndTrans.h" @@ -285,7 +285,7 @@ static int32_t mndProcessCreateSnodeReq(SRpcMsg *pReq) { } mDebug("snode:%d, start to create", createReq.dnodeId); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_SNODE) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_SNODE) != 0) { goto _OVER; } @@ -397,7 +397,7 @@ static int32_t mndProcessDropSnodeReq(SRpcMsg *pReq) { } mDebug("snode:%d, start to drop", dropReq.dnodeId); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_SNODE) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_SNODE) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index f1bae14c07..77b13cd82d 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "mndStb.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndDb.h" #include "mndDnode.h" #include "mndInfoSchema.h" @@ -876,7 +876,7 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { goto _OVER; } @@ -1607,7 +1607,7 @@ static int32_t mndProcessAlterStbReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { goto _OVER; } @@ -1737,7 +1737,7 @@ static int32_t mndProcessDropStbReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 5e2f5bc2dd..e49756c837 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -14,7 +14,7 @@ */ #include "mndStream.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" @@ -437,7 +437,7 @@ static int32_t mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStre goto _OVER; } - if (mndCheckDbAuth(pMnode, user, MND_OPER_WRITE_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, user, MND_OPER_WRITE_DB, pDb) != 0) { goto _OVER; } @@ -550,7 +550,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { goto _OVER; } #endif diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index a650ed29f1..b8c17378c4 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -14,7 +14,7 @@ */ #include "mndTopic.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndConsumer.h" #include "mndDb.h" #include "mndDnode.h" @@ -480,7 +480,7 @@ static int32_t mndProcessCreateTopicReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index d1d88fdc90..a9de1a05a7 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "mndTrans.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndConsumer.h" #include "mndDb.h" #include "mndShow.h" @@ -1384,8 +1384,7 @@ static int32_t mndProcessKillTransReq(SRpcMsg *pReq) { } mInfo("trans:%d, start to kill", killReq.transId); - - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_KILL_TRANS) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_KILL_TRANS) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 03c9647bfe..921dba422d 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "mndUser.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndDb.h" #include "mndShow.h" #include "mndTrans.h" @@ -295,7 +295,7 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate tstrncpy(userObj.acct, acct, TSDB_USER_LEN); userObj.createdTime = taosGetTimestampMs(); userObj.updateTime = userObj.createdTime; - userObj.superUser = 0;//pCreate->superUser; + userObj.superUser = 0; // pCreate->superUser; userObj.sysInfo = pCreate->sysInfo; userObj.enable = pCreate->enable; @@ -337,6 +337,9 @@ static int32_t mndProcessCreateUserReq(SRpcMsg *pReq) { } mDebug("user:%s, start to create", createReq.user); + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_USER) != 0) { + goto _OVER; + } if (createReq.user[0] == 0) { terrno = TSDB_CODE_MND_INVALID_USER_FORMAT; @@ -360,10 +363,6 @@ static int32_t mndProcessCreateUserReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_USER) != 0) { - goto _OVER; - } - code = mndCreateUser(pMnode, pOperUser->acct, &createReq, pReq); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; @@ -466,7 +465,7 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckAlterUserAuth(pOperUser, pUser, &alterReq) != 0) { + if (mndCheckAlterUserPrivilege(pOperUser, pUser, &alterReq) != 0) { goto _OVER; } @@ -631,6 +630,9 @@ static int32_t mndProcessDropUserReq(SRpcMsg *pReq) { } mDebug("user:%s, start to drop", dropReq.user); + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_USER) != 0) { + goto _OVER; + } if (dropReq.user[0] == 0) { terrno = TSDB_CODE_MND_INVALID_USER_FORMAT; @@ -643,10 +645,6 @@ static int32_t mndProcessDropUserReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_USER) != 0) { - goto _OVER; - } - code = mndDropUser(pMnode, pReq, pUser); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 1c5d73031f..0e931e0a9c 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "mndVgroup.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" @@ -1212,7 +1212,7 @@ static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq) { } mInfo("vgId:%d, start to redistribute vgroup to dnode %d:%d:%d", req.vgId, req.dnodeId1, req.dnodeId2, req.dnodeId3); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_REDISTRIBUTE_VGROUP) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_REDISTRIBUTE_VGROUP) != 0) { goto _OVER; } @@ -1507,7 +1507,7 @@ static int32_t mndProcessSplitVgroupMsg(SRpcMsg *pReq) { SDbObj *pDb = NULL; mDebug("vgId:%d, start to split", vgId); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_SPLIT_VGROUP) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_SPLIT_VGROUP) != 0) { goto _OVER; } @@ -1657,7 +1657,7 @@ static int32_t mndProcessBalanceVgroupMsg(SRpcMsg *pReq) { } mInfo("start to balance vgroup"); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_BALANCE_VGROUP) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_BALANCE_VGROUP) != 0) { goto _OVER; } diff --git a/tests/script/tsim/user/privilege_sysinfo.sim b/tests/script/tsim/user/privilege_sysinfo.sim index 9ddfce8a97..ea3294765c 100644 --- a/tests/script/tsim/user/privilege_sysinfo.sim +++ b/tests/script/tsim/user/privilege_sysinfo.sim @@ -22,5 +22,26 @@ sql_error drop user sysinfo1 sql_error alter user sysinfo1 pass '1' sql_error alter user sysinfo0 pass '1' +sql_error create dnode $hostname port 7200 +sql_error drop dnode 1 + +sql_error create qnode on dnode 1 +sql_error drop qnode on dnode 1 + +sql_error create mnode on dnode 1 +sql_error drop mnode on dnode 1 + +sql_error create snode on dnode 1 +sql_error drop snode on dnode 1 + +sql_error redistribute vgroup 2 dnode 1 dnode 2 +sql_error balance vgroup + +sql_error kill transaction 1 +sql_error kill connection 1 +sql_error kill query 1 + +print =============== check db +sql_error create database db system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file From 16ee8eeb2812d2eb134dcf9bf1dbffcfd70f07e2 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sat, 25 Jun 2022 10:29:44 +0800 Subject: [PATCH 24/49] feat: support partition by expression and aggregate function output together --- source/libs/planner/src/planPhysiCreater.c | 4 ++-- source/libs/planner/test/planDistinctTest.cpp | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index ec6181a7d4..4f46abd8da 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -348,8 +348,8 @@ static SPhysiNode* makePhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode return NULL; } - pPhysiNode->pLimit = pLogicNode->pLimit; - pPhysiNode->pSlimit = pLogicNode->pSlimit; + TSWAP(pPhysiNode->pLimit, pLogicNode->pLimit); + TSWAP(pPhysiNode->pSlimit, pLogicNode->pSlimit); int32_t code = createDataBlockDesc(pCxt, pLogicNode->pTargets, &pPhysiNode->pOutputDataBlockDesc); if (TSDB_CODE_SUCCESS != code) { diff --git a/source/libs/planner/test/planDistinctTest.cpp b/source/libs/planner/test/planDistinctTest.cpp index 58473dcff2..8caee01d35 100644 --- a/source/libs/planner/test/planDistinctTest.cpp +++ b/source/libs/planner/test/planDistinctTest.cpp @@ -40,3 +40,9 @@ TEST_F(PlanDistinctTest, withOrderBy) { run("select distinct c1 + 10 a from t1 order by a"); } + +TEST_F(PlanDistinctTest, withLimit) { + useDb("root", "test"); + + run("SELECT DISTINCT c1 FROM t1 LIMIT 3"); +} From 37d1cc28a6cb268907777b64c419a329779147fb Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sat, 25 Jun 2022 10:43:06 +0800 Subject: [PATCH 25/49] refactor(query): refactor _group_key function --- source/libs/function/src/builtinsimpl.c | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 61f3154811..c4d3a26ab4 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -264,6 +264,7 @@ typedef struct SRateInfo { typedef struct SGroupKeyInfo{ bool hasResult; + bool isNull; char data[]; } SGroupKeyInfo; @@ -5371,14 +5372,21 @@ int32_t groupKeyFunction(SqlFunctionCtx* pCtx) { int32_t bytes = pInputCol->info.bytes; int32_t startIndex = pInput->startRowIndex; - if (colDataIsNull_s(pInputCol, startIndex)) { - pInfo->hasResult = false; + + //escape rest of data blocks to avoid first entry be overwritten. + if (pInfo->hasResult) { + goto _group_key_over; + } + + if (colDataIsNull_s(pInputCol, startIndex)) { + pInfo->isNull = true; + pInfo->hasResult = true; goto _group_key_over; } - pInfo->hasResult = true; char* data = colDataGetData(pInputCol, startIndex); memcpy(pInfo->data, data, bytes); + pInfo->hasResult = true; _group_key_over: @@ -5393,7 +5401,12 @@ int32_t groupKeyFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); SGroupKeyInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo); - colDataAppend(pCol, pBlock->info.rows, pInfo->data, pInfo->hasResult ? false : true); + + if (pInfo->hasResult) { + colDataAppend(pCol, pBlock->info.rows, pInfo->data, pInfo->isNull ? true : false); + } else { + pResInfo->numOfRes = 0; + } return pResInfo->numOfRes; } From aa8957bcc54b3a2d775cea36ddf3f19cfdc15ae3 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 25 Jun 2022 11:02:49 +0800 Subject: [PATCH 26/49] handle redirect --- source/libs/transport/src/transCli.c | 16 ++++++++-------- source/libs/transport/src/transSvr.c | 1 + 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 9ca5e8e73e..0671aa39d1 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -513,7 +513,7 @@ static void allocConnRef(SCliConn* conn, bool update) { } static void addConnToPool(void* pool, SCliConn* conn) { if (conn->status == ConnInPool) { - assert(0); + // assert(0); return; } SCliThrd* thrd = conn->hostThrd; @@ -986,13 +986,13 @@ static void doDelayTask(void* param) { } static void cliSchedMsgToNextNode(SCliMsg* pMsg, SCliThrd* pThrd) { - STraceId* trace = &pMsg->msg.info.traceId; STransConnCtx* pCtx = pMsg->ctx; - char tbuf[256] = {0}; + STraceId* trace = &pMsg->msg.info.traceId; + char tbuf[256] = {0}; EPSET_DEBUG_STR(&pCtx->epSet, tbuf); - tGTrace("%s retry to send msg to next node %dms later , use %s, retryCnt:%d, limit:%d", transLabel(pThrd->pTransInst), - TRANS_RETRY_INTERVAL, tbuf, pCtx->retryCnt + 1, pCtx->retryLimit); + tGTrace("%s retry on next node, use %s, retryCnt:%d, limit:%d", transLabel(pThrd->pTransInst), tbuf, + pCtx->retryCnt + 1, pCtx->retryLimit); STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); arg->param1 = pMsg; @@ -1000,7 +1000,7 @@ static void cliSchedMsgToNextNode(SCliMsg* pMsg, SCliThrd* pThrd) { transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL); } -void cliUpdateRetryLimit(int8_t* val, int8_t exp, int8_t newVal) { +void cliCompareAndSwap(int8_t* val, int8_t exp, int8_t newVal) { if (*val != exp) { *val = newVal; } @@ -1025,7 +1025,7 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { pMsg->sent = 0; pCtx->retryCnt += 1; if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { - cliUpdateRetryLimit(&pCtx->retryLimit, TRANS_RETRY_COUNT_LIMIT, EPSET_GET_SIZE(&pCtx->epSet) * 3); + cliCompareAndSwap(&pCtx->retryLimit, TRANS_RETRY_COUNT_LIMIT, EPSET_GET_SIZE(&pCtx->epSet) * 3); if (pCtx->retryCnt < pCtx->retryLimit) { transUnrefCliHandle(pConn); EPSET_FORWARD_INUSE(&pCtx->epSet); @@ -1033,7 +1033,7 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { return -1; } } else { - cliUpdateRetryLimit(&pCtx->retryLimit, TRANS_RETRY_COUNT_LIMIT, TRANS_RETRY_COUNT_LIMIT); + cliCompareAndSwap(&pCtx->retryLimit, TRANS_RETRY_COUNT_LIMIT, TRANS_RETRY_COUNT_LIMIT); if (pCtx->retryCnt < pCtx->retryLimit) { addConnToPool(pThrd->pool, pConn); if (pResp->contLen == 0) { diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 215323f69d..7651860224 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -422,6 +422,7 @@ static void uvPrepareSendData(SSvrMsg* smsg, uv_buf_t* wb) { transUnrefSrvHandle(pConn); } else { pHead->msgType = pMsg->msgType; + if (pHead->msgType == 0) pHead->msgType = pConn->inType + 1; } } From 925276e44921407cec3f7376d2d641f25cb18346 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Sat, 25 Jun 2022 11:14:10 +0800 Subject: [PATCH 27/49] os: add alloca func --- include/os/os.h | 1 + include/os/osMath.h | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/os/os.h b/include/os/os.h index 254c16efbe..3e72a618a0 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -52,6 +52,7 @@ extern "C" { #endif #else +#include #include #ifndef TD_USE_WINSOCK #include diff --git a/include/os/osMath.h b/include/os/osMath.h index f17ca56c9e..e13c32422e 100644 --- a/include/os/osMath.h +++ b/include/os/osMath.h @@ -25,11 +25,10 @@ extern "C" { #define TSWAP(a, b) \ do { \ - char *__tmp = taosMemoryMalloc(sizeof(a)); \ + char *__tmp = alloca(sizeof(a)); \ memcpy(__tmp, &(a), sizeof(a)); \ memcpy(&(a), &(b), sizeof(a)); \ memcpy(&(b), __tmp, sizeof(a)); \ - taosMemoryFree(__tmp); \ } while (0) #ifdef WINDOWS From 23653fd8430f8221e642d8bf7a15178603376ff0 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 25 Jun 2022 11:22:34 +0800 Subject: [PATCH 28/49] enh: privilege for topic and stream --- source/dnode/mnode/impl/inc/mndPrivilege.h | 1 + source/dnode/mnode/impl/src/mndConsumer.c | 4 +++ source/dnode/mnode/impl/src/mndPrivilege.c | 20 ++++++----- source/dnode/mnode/impl/src/mndStream.c | 35 +++++++------------- source/dnode/mnode/impl/src/mndTopic.c | 8 +++-- tests/script/tsim/user/privilege_sysinfo.sim | 3 -- 6 files changed, 34 insertions(+), 37 deletions(-) diff --git a/source/dnode/mnode/impl/inc/mndPrivilege.h b/source/dnode/mnode/impl/inc/mndPrivilege.h index 15f9e4e6b5..a1bec69790 100644 --- a/source/dnode/mnode/impl/inc/mndPrivilege.h +++ b/source/dnode/mnode/impl/inc/mndPrivilege.h @@ -64,6 +64,7 @@ void mndCleanupPrivilege(SMnode *pMnode); int32_t mndCheckOperPrivilege(SMnode *pMnode, const char *user, EOperType operType); int32_t mndCheckDbPrivilege(SMnode *pMnode, const char *user, EOperType operType, SDbObj *pDb); +int32_t mndCheckDbPrivilegeByName(SMnode *pMnode, const char *user, EOperType operType, const char *name); int32_t mndCheckShowPrivilege(SMnode *pMnode, const char *user, int32_t showType); int32_t mndCheckAlterUserPrivilege(SUserObj *pOperUser, SUserObj *pUser, SAlterUserReq *pAlter); diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 7dc5ee1ea1..69f58f33cf 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -431,6 +431,10 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { goto SUBSCRIBE_OVER; } + if (mndCheckDbPrivilege(pMnode, pMsg->info.conn.user, MND_OPER_READ_DB, pTopic->db) != 0) { + goto SUBSCRIBE_OVER; + } + #if 0 // ref topic to prevent drop // TODO make topic complete diff --git a/source/dnode/mnode/impl/src/mndPrivilege.c b/source/dnode/mnode/impl/src/mndPrivilege.c index 478ba2bee4..752b11540d 100644 --- a/source/dnode/mnode/impl/src/mndPrivilege.c +++ b/source/dnode/mnode/impl/src/mndPrivilege.c @@ -16,6 +16,7 @@ #define _DEFAULT_SOURCE #include "mndPrivilege.h" #include "mndUser.h" +#include "mndDb.h" int32_t mndInitPrivilege(SMnode *pMnode) { return 0; } @@ -133,15 +134,7 @@ int32_t mndCheckDbPrivilege(SMnode *pMnode, const char *user, EOperType operType if (pUser->sysInfo) goto _OVER; } - if (operType == MND_OPER_ALTER_DB) { - if (strcmp(pUser->user, pDb->createUser) == 0 && pUser->sysInfo) goto _OVER; - } - - if (operType == MND_OPER_DROP_DB) { - if (strcmp(pUser->user, pDb->createUser) == 0 && pUser->sysInfo) goto _OVER; - } - - if (operType == MND_OPER_COMPACT_DB) { + if (operType == MND_OPER_ALTER_DB || operType == MND_OPER_DROP_DB || operType == MND_OPER_COMPACT_DB) { if (strcmp(pUser->user, pDb->createUser) == 0 && pUser->sysInfo) goto _OVER; } @@ -168,3 +161,12 @@ _OVER: mndReleaseUser(pMnode, pUser); return code; } + +int32_t mndCheckDbPrivilegeByName(SMnode *pMnode, const char *user, EOperType operType, const char *name) { + SDbObj *pDb = mndAcquireDb(pMnode, name); + if (pDb == NULL) return -1; + + int32_t code = mndCheckDbPrivilege(pMnode, user, operType, pDb); + mndReleaseDb(pMnode, pDb); + return code; +} \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index e49756c837..c45421baa9 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -437,10 +437,6 @@ static int32_t mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStre goto _OVER; } - if (mndCheckDbPrivilege(pMnode, user, MND_OPER_WRITE_DB, pDb) != 0) { - goto _OVER; - } - int32_t numOfStbs = -1; if (mndGetNumOfStbs(pMnode, pDb->name, &numOfStbs) != 0) { goto _OVER; @@ -542,19 +538,6 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { goto _OVER; } - // TODO check read auth for source and write auth for target -#if 0 - pDb = mndAcquireDb(pMnode, createStreamReq.sourceDB); - if (pDb == NULL) { - terrno = TSDB_CODE_MND_DB_NOT_SELECTED; - goto _OVER; - } - - if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { - goto _OVER; - } -#endif - // build stream obj from request SStreamObj streamObj = {0}; if (mndBuildStreamObjFromCreateReq(pMnode, &streamObj, &createStreamReq) < 0) { @@ -592,6 +575,16 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { goto _OVER; } + if (mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_READ_DB, streamObj.sourceDb) != 0) { + mndTransDrop(pTrans); + goto _OVER; + } + + if (mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, streamObj.targetDb) != 0) { + mndTransDrop(pTrans); + goto _OVER; + } + // execute creation if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); @@ -641,13 +634,9 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { } } -#if 0 - // todo check auth - pUser = mndAcquireUser(pMnode, pReq->info.conn.user); - if (pUser == NULL) { - goto DROP_STREAM_OVER; + if (mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pStream->targetDb) != 0) { + return -1; } -#endif STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq); if (pTrans == NULL) { diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index b8c17378c4..90c4e96517 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -480,7 +480,7 @@ static int32_t mndProcessCreateTopicReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_READ_DB, pDb) != 0) { goto _OVER; } @@ -571,6 +571,10 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { } #endif + if (mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_READ_DB, pTopic->db) != 0) { + return -1; + } + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pReq); mndTransSetDbName(pTrans, pTopic->db, NULL); if (pTrans == NULL) { @@ -579,7 +583,7 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { } mDebug("trans:%d, used to drop topic:%s", pTrans->id, pTopic->name); - + if (mndDropOffsetByTopic(pMnode, pTrans, dropReq.name) < 0) { ASSERT(0); return -1; diff --git a/tests/script/tsim/user/privilege_sysinfo.sim b/tests/script/tsim/user/privilege_sysinfo.sim index ea3294765c..35760d45fd 100644 --- a/tests/script/tsim/user/privilege_sysinfo.sim +++ b/tests/script/tsim/user/privilege_sysinfo.sim @@ -13,9 +13,6 @@ print user sysinfo0 login sql close sql connect sysinfo0 -system sh/exec.sh -n dnode1 -s stop -return - print =============== check oper sql_error create user u1 pass 'u1' sql_error drop user sysinfo1 From 43f1d411220ad559ab1c795893e24c3b3c10b504 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 25 Jun 2022 11:23:44 +0800 Subject: [PATCH 29/49] fix: compile error --- source/dnode/mnode/impl/src/mndConsumer.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 69f58f33cf..5b5de10fba 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -431,7 +431,7 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { goto SUBSCRIBE_OVER; } - if (mndCheckDbPrivilege(pMnode, pMsg->info.conn.user, MND_OPER_READ_DB, pTopic->db) != 0) { + if (mndCheckDbPrivilegeByName(pMnode, pMsg->info.conn.user, MND_OPER_READ_DB, pTopic->db) != 0) { goto SUBSCRIBE_OVER; } From 0df1415caf617f63e55d255a06f777125cf2ad63 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sat, 25 Jun 2022 11:30:34 +0800 Subject: [PATCH 30/49] refactor(sync): add trace log --- source/libs/sync/src/syncAppendEntries.c | 276 ++++++++++++++++++ source/libs/sync/src/syncAppendEntriesReply.c | 73 +++++ source/libs/sync/src/syncRequestVote.c | 70 +++++ source/libs/sync/src/syncRequestVoteReply.c | 67 +++++ 4 files changed, 486 insertions(+) diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 47d0976aca..ba3dd66550 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -89,6 +89,280 @@ // /\ UNCHANGED <> // +int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { + int32_t ret = 0; + + // print log + syncAppendEntriesLog2("==syncNodeOnAppendEntriesCb==", pMsg); + + // if already drop replica, do not process + if (!syncNodeInRaftGroup(ths, &(pMsg->srcId)) && !ths->pRaftCfg->isStandBy) { + syncNodeEventLog(ths, "recv sync-append-entries, maybe replica already dropped"); + return ret; + } + + // maybe update term + if (pMsg->term > ths->pRaftStore->currentTerm) { + syncNodeUpdateTerm(ths, pMsg->term); + } + ASSERT(pMsg->term <= ths->pRaftStore->currentTerm); + + // reset elect timer + if (pMsg->term == ths->pRaftStore->currentTerm) { + ths->leaderCache = pMsg->srcId; + syncNodeResetElectTimer(ths); + } + ASSERT(pMsg->dataLen >= 0); + + do { + // return to follower state + if (pMsg->term == ths->pRaftStore->currentTerm && ths->state == TAOS_SYNC_STATE_CANDIDATE) { + syncNodeEventLog(ths, "recv sync-append-entries, candidate to follower"); + + syncNodeBecomeFollower(ths, "from candidate by append entries"); + + // ret or reply? + return ret; + } + } while (0); + + SyncTerm localPreLogTerm = 0; + if (pMsg->prevLogIndex >= SYNC_INDEX_BEGIN && pMsg->prevLogIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)) { + SSyncRaftEntry* pEntry = ths->pLogStore->getEntry(ths->pLogStore, pMsg->prevLogIndex); + if (pEntry == NULL) { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "getEntry error, index:%ld, since %s", pMsg->prevLogIndex, terrstr()); + syncNodeErrorLog(ths, logBuf); + return -1; + } + + localPreLogTerm = pEntry->term; + syncEntryDestory(pEntry); + } + + bool logOK = + (pMsg->prevLogIndex == SYNC_INDEX_INVALID) || + ((pMsg->prevLogIndex >= SYNC_INDEX_BEGIN) && + (pMsg->prevLogIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)) && (pMsg->prevLogTerm == localPreLogTerm)); + + // reject request + if ((pMsg->term < ths->pRaftStore->currentTerm) || + ((pMsg->term == ths->pRaftStore->currentTerm) && (ths->state == TAOS_SYNC_STATE_FOLLOWER) && !logOK)) { + do { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries, reject, pre-index:%ld, pre-term:%lu, datalen:%d", + pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen); + syncNodeEventLog(ths, logBuf); + } while (0); + + SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId); + pReply->srcId = ths->myRaftId; + pReply->destId = pMsg->srcId; + pReply->term = ths->pRaftStore->currentTerm; + pReply->success = false; + pReply->matchIndex = SYNC_INDEX_INVALID; + + SRpcMsg rpcMsg; + syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); + syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); + syncAppendEntriesReplyDestroy(pReply); + + return ret; + } + + // accept request + if (pMsg->term == ths->pRaftStore->currentTerm && ths->state == TAOS_SYNC_STATE_FOLLOWER && logOK) { + // preIndex = -1, or has preIndex entry in local log + ASSERT(pMsg->prevLogIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)); + + // has extra entries (> preIndex) in local log + bool hasExtraEntries = pMsg->prevLogIndex < ths->pLogStore->getLastIndex(ths->pLogStore); + + // has entries in SyncAppendEntries msg + bool hasAppendEntries = pMsg->dataLen > 0; + + do { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries, accept, pre-index:%ld, pre-term:%lu, datalen:%d", + pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen); + syncNodeEventLog(ths, logBuf); + } while (0); + + if (hasExtraEntries && hasAppendEntries) { + // not conflict by default + bool conflict = false; + + SyncIndex extraIndex = pMsg->prevLogIndex + 1; + SSyncRaftEntry* pExtraEntry = ths->pLogStore->getEntry(ths->pLogStore, extraIndex); + if (pExtraEntry == NULL) { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "getEntry error2, index:%ld, since %s", extraIndex, terrstr()); + syncNodeErrorLog(ths, logBuf); + return -1; + } + + SSyncRaftEntry* pAppendEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen); + if (pAppendEntry == NULL) { + syncNodeErrorLog(ths, "syncEntryDeserialize pAppendEntry error"); + return -1; + } + + // log not match, conflict + ASSERT(extraIndex == pAppendEntry->index); + if (pExtraEntry->term != pAppendEntry->term) { + conflict = true; + } + + if (conflict) { + // roll back + SyncIndex delBegin = ths->pLogStore->getLastIndex(ths->pLogStore); + SyncIndex delEnd = extraIndex; + + sTrace("syncNodeOnAppendEntriesCb --> conflict:%d, delBegin:%ld, delEnd:%ld", conflict, delBegin, delEnd); + + // notice! reverse roll back! + for (SyncIndex index = delEnd; index >= delBegin; --index) { + if (ths->pFsm->FpRollBackCb != NULL) { + SSyncRaftEntry* pRollBackEntry = ths->pLogStore->getEntry(ths->pLogStore, index); + if (pRollBackEntry == NULL) { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "getEntry error3, index:%ld, since %s", index, terrstr()); + syncNodeErrorLog(ths, logBuf); + return -1; + } + + // if (pRollBackEntry->msgType != TDMT_SYNC_NOOP) { + if (syncUtilUserRollback(pRollBackEntry->msgType)) { + SRpcMsg rpcMsg; + syncEntry2OriginalRpc(pRollBackEntry, &rpcMsg); + + SFsmCbMeta cbMeta = {0}; + cbMeta.index = pRollBackEntry->index; + cbMeta.lastConfigIndex = syncNodeGetSnapshotConfigIndex(ths, cbMeta.index); + cbMeta.isWeak = pRollBackEntry->isWeak; + cbMeta.code = 0; + cbMeta.state = ths->state; + cbMeta.seqNum = pRollBackEntry->seqNum; + ths->pFsm->FpRollBackCb(ths->pFsm, &rpcMsg, cbMeta); + rpcFreeCont(rpcMsg.pCont); + } + + syncEntryDestory(pRollBackEntry); + } + } + + // delete confict entries + ths->pLogStore->truncate(ths->pLogStore, extraIndex); + + // append new entries + ths->pLogStore->appendEntry(ths->pLogStore, pAppendEntry); + + // pre commit + SRpcMsg rpcMsg; + syncEntry2OriginalRpc(pAppendEntry, &rpcMsg); + if (ths->pFsm != NULL) { + // if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->originalRpcType != TDMT_SYNC_NOOP) { + if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pAppendEntry->originalRpcType)) { + SFsmCbMeta cbMeta = {0}; + cbMeta.index = pAppendEntry->index; + cbMeta.lastConfigIndex = syncNodeGetSnapshotConfigIndex(ths, cbMeta.index); + cbMeta.isWeak = pAppendEntry->isWeak; + cbMeta.code = 2; + cbMeta.state = ths->state; + cbMeta.seqNum = pAppendEntry->seqNum; + ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, cbMeta); + } + } + rpcFreeCont(rpcMsg.pCont); + } + + // free memory + syncEntryDestory(pExtraEntry); + syncEntryDestory(pAppendEntry); + + } else if (hasExtraEntries && !hasAppendEntries) { + // do nothing + + } else if (!hasExtraEntries && hasAppendEntries) { + SSyncRaftEntry* pAppendEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen); + if (pAppendEntry == NULL) { + syncNodeErrorLog(ths, "syncEntryDeserialize pAppendEntry2 error"); + return -1; + } + + // append new entries + ths->pLogStore->appendEntry(ths->pLogStore, pAppendEntry); + + // pre commit + SRpcMsg rpcMsg; + syncEntry2OriginalRpc(pAppendEntry, &rpcMsg); + if (ths->pFsm != NULL) { + // if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->originalRpcType != TDMT_SYNC_NOOP) { + if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pAppendEntry->originalRpcType)) { + SFsmCbMeta cbMeta = {0}; + cbMeta.index = pAppendEntry->index; + cbMeta.lastConfigIndex = syncNodeGetSnapshotConfigIndex(ths, cbMeta.index); + cbMeta.isWeak = pAppendEntry->isWeak; + cbMeta.code = 3; + cbMeta.state = ths->state; + cbMeta.seqNum = pAppendEntry->seqNum; + ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, cbMeta); + } + } + rpcFreeCont(rpcMsg.pCont); + + // free memory + syncEntryDestory(pAppendEntry); + + } else if (!hasExtraEntries && !hasAppendEntries) { + // do nothing + + } else { + syncNodeLog3("", ths); + ASSERT(0); + } + + SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId); + pReply->srcId = ths->myRaftId; + pReply->destId = pMsg->srcId; + pReply->term = ths->pRaftStore->currentTerm; + pReply->success = true; + + if (hasAppendEntries) { + pReply->matchIndex = pMsg->prevLogIndex + 1; + } else { + pReply->matchIndex = pMsg->prevLogIndex; + } + + SRpcMsg rpcMsg; + syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); + syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); + syncAppendEntriesReplyDestroy(pReply); + + // maybe update commit index from leader + if (pMsg->commitIndex > ths->commitIndex) { + // has commit entry in local + if (pMsg->commitIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)) { + SyncIndex beginIndex = ths->commitIndex + 1; + SyncIndex endIndex = pMsg->commitIndex; + + // update commit index + ths->commitIndex = pMsg->commitIndex; + + // call back Wal + ths->pLogStore->updateCommitIndex(ths->pLogStore, ths->commitIndex); + + int32_t code = syncNodeCommit(ths, beginIndex, endIndex, ths->state); + ASSERT(code == 0); + } + } + } + + return ret; +} + +#if 0 + int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { int32_t ret = 0; @@ -352,6 +626,8 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { return ret; } +#endif + static int32_t syncNodeMakeLogSame(SSyncNode* ths, SyncAppendEntries* pMsg) { int32_t code; diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index 6ca356b4f6..b7122b9c52 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -40,6 +40,78 @@ int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg) { int32_t ret = 0; + // print log + syncAppendEntriesReplyLog2("==syncNodeOnAppendEntriesReplyCb==", pMsg); + + // if already drop replica, do not process + if (!syncNodeInRaftGroup(ths, &(pMsg->srcId)) && !ths->pRaftCfg->isStandBy) { + syncNodeEventLog(ths, "recv sync-append-entries-reply, maybe replica already dropped"); + return 0; + } + + // drop stale response + if (pMsg->term < ths->pRaftStore->currentTerm) { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries-reply, recv-term:%lu, drop stale response", pMsg->term); + syncNodeEventLog(ths, logBuf); + return 0; + } + + if (gRaftDetailLog) { + syncNodeEventLog(ths, "recv sync-append-entries-reply, before"); + } + syncIndexMgrLog2("==syncNodeOnAppendEntriesReplyCb== before pNextIndex", ths->pNextIndex); + syncIndexMgrLog2("==syncNodeOnAppendEntriesReplyCb== before pMatchIndex", ths->pMatchIndex); + + // no need this code, because if I receive reply.term, then I must have sent for that term. + // if (pMsg->term > ths->pRaftStore->currentTerm) { + // syncNodeUpdateTerm(ths, pMsg->term); + // } + + if (pMsg->term > ths->pRaftStore->currentTerm) { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries-reply, error term, recv-term:%lu", pMsg->term); + syncNodeErrorLog(ths, logBuf); + return -1; + } + + ASSERT(pMsg->term == ths->pRaftStore->currentTerm); + + if (pMsg->success) { + // nextIndex' = [nextIndex EXCEPT ![i][j] = m.mmatchIndex + 1] + syncIndexMgrSetIndex(ths->pNextIndex, &(pMsg->srcId), pMsg->matchIndex + 1); + + // matchIndex' = [matchIndex EXCEPT ![i][j] = m.mmatchIndex] + syncIndexMgrSetIndex(ths->pMatchIndex, &(pMsg->srcId), pMsg->matchIndex); + + // maybe commit + syncMaybeAdvanceCommitIndex(ths); + + } else { + SyncIndex nextIndex = syncIndexMgrGetIndex(ths->pNextIndex, &(pMsg->srcId)); + + // notice! int64, uint64 + if (nextIndex > SYNC_INDEX_BEGIN) { + --nextIndex; + } else { + nextIndex = SYNC_INDEX_BEGIN; + } + syncIndexMgrSetIndex(ths->pNextIndex, &(pMsg->srcId), nextIndex); + } + + if (gRaftDetailLog) { + syncNodeEventLog(ths, "recv sync-append-entries-reply, after"); + } + syncIndexMgrLog2("==syncNodeOnAppendEntriesReplyCb== after pNextIndex", ths->pNextIndex); + syncIndexMgrLog2("==syncNodeOnAppendEntriesReplyCb== after pMatchIndex", ths->pMatchIndex); + + return ret; +} + +#if 0 +int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg) { + int32_t ret = 0; + char logBuf[128] = {0}; snprintf(logBuf, sizeof(logBuf), "==syncNodeOnAppendEntriesReplyCb== term:%lu", ths->pRaftStore->currentTerm); syncAppendEntriesReplyLog2(logBuf, pMsg); @@ -96,6 +168,7 @@ int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* p return ret; } +#endif int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg) { int32_t ret = 0; diff --git a/source/libs/sync/src/syncRequestVote.c b/source/libs/sync/src/syncRequestVote.c index 3270b5c0e4..95dec6cb83 100644 --- a/source/libs/sync/src/syncRequestVote.c +++ b/source/libs/sync/src/syncRequestVote.c @@ -45,6 +45,75 @@ int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg) { int32_t ret = 0; + syncRequestVoteLog2("==syncNodeOnRequestVoteCb==", pMsg); + + // if already drop replica, do not process + if (!syncNodeInRaftGroup(ths, &(pMsg->srcId)) && !ths->pRaftCfg->isStandBy) { + do { + char logBuf[256]; + char host[64]; + uint16_t port; + syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); + snprintf(logBuf, sizeof(logBuf), + "recv sync-request-vote from %s:%d, term:%lu, lindex:%ld, lterm:%lu, maybe replica already dropped", + host, port, pMsg->term, pMsg->lastLogIndex, pMsg->lastLogTerm); + syncNodeEventLog(ths, logBuf); + } while (0); + + return -1; + } + + // maybe update term + if (pMsg->term > ths->pRaftStore->currentTerm) { + syncNodeUpdateTerm(ths, pMsg->term); + } + ASSERT(pMsg->term <= ths->pRaftStore->currentTerm); + + bool logOK = (pMsg->lastLogTerm > ths->pLogStore->getLastTerm(ths->pLogStore)) || + ((pMsg->lastLogTerm == ths->pLogStore->getLastTerm(ths->pLogStore)) && + (pMsg->lastLogIndex >= ths->pLogStore->getLastIndex(ths->pLogStore))); + bool grant = (pMsg->term == ths->pRaftStore->currentTerm) && logOK && + ((!raftStoreHasVoted(ths->pRaftStore)) || (syncUtilSameId(&(ths->pRaftStore->voteFor), &(pMsg->srcId)))); + if (grant) { + // maybe has already voted for pMsg->srcId + // vote again, no harm + raftStoreVote(ths->pRaftStore, &(pMsg->srcId)); + + // forbid elect for this round + syncNodeResetElectTimer(ths); + } + + // send msg + SyncRequestVoteReply* pReply = syncRequestVoteReplyBuild(ths->vgId); + pReply->srcId = ths->myRaftId; + pReply->destId = pMsg->srcId; + pReply->term = ths->pRaftStore->currentTerm; + pReply->voteGranted = grant; + + // trace log + do { + char logBuf[256]; + char host[64]; + uint16_t port; + syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); + snprintf(logBuf, sizeof(logBuf), + "recv sync-request-vote from %s:%d, term:%lu, lindex:%ld, lterm:%lu, reply-grant:%d", host, port, + pMsg->term, pMsg->lastLogIndex, pMsg->lastLogTerm, pReply->voteGranted); + syncNodeEventLog(ths, logBuf); + } while (0); + + SRpcMsg rpcMsg; + syncRequestVoteReply2RpcMsg(pReply, &rpcMsg); + syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); + syncRequestVoteReplyDestroy(pReply); + + return ret; +} + +#if 0 +int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg) { + int32_t ret = 0; + char logBuf[128] = {0}; snprintf(logBuf, sizeof(logBuf), "==syncNodeOnRequestVoteCb== term:%lu", ths->pRaftStore->currentTerm); syncRequestVoteLog2(logBuf, pMsg); @@ -81,6 +150,7 @@ int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg) { return ret; } +#endif static bool syncNodeOnRequestVoteLogOK(SSyncNode* pSyncNode, SyncRequestVote* pMsg) { SyncTerm myLastTerm = syncNodeGetLastTerm(pSyncNode); diff --git a/source/libs/sync/src/syncRequestVoteReply.c b/source/libs/sync/src/syncRequestVoteReply.c index 60963d0dd3..4c205d16ec 100644 --- a/source/libs/sync/src/syncRequestVoteReply.c +++ b/source/libs/sync/src/syncRequestVoteReply.c @@ -40,6 +40,72 @@ int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg) { int32_t ret = 0; + // print log + char logBuf[128] = {0}; + snprintf(logBuf, sizeof(logBuf), "==syncNodeOnRequestVoteReplyCb== term:%lu", ths->pRaftStore->currentTerm); + syncRequestVoteReplyLog2(logBuf, pMsg); + + // if already drop replica, do not process + if (!syncNodeInRaftGroup(ths, &(pMsg->srcId)) && !ths->pRaftCfg->isStandBy) { + sInfo("recv SyncRequestVoteReply, maybe replica already dropped"); + return ret; + } + + // drop stale response + if (pMsg->term < ths->pRaftStore->currentTerm) { + sTrace("recv SyncRequestVoteReply, drop stale response, receive_term:%lu current_term:%lu", pMsg->term, + ths->pRaftStore->currentTerm); + return ret; + } + + // ASSERT(!(pMsg->term > ths->pRaftStore->currentTerm)); + // no need this code, because if I receive reply.term, then I must have sent for that term. + // if (pMsg->term > ths->pRaftStore->currentTerm) { + // syncNodeUpdateTerm(ths, pMsg->term); + // } + + if (pMsg->term > ths->pRaftStore->currentTerm) { + char logBuf[128] = {0}; + snprintf(logBuf, sizeof(logBuf), "syncNodeOnRequestVoteReplyCb error term, receive:%lu current:%lu", pMsg->term, + ths->pRaftStore->currentTerm); + syncNodePrint2(logBuf, ths); + sError("%s", logBuf); + return ret; + } + + ASSERT(pMsg->term == ths->pRaftStore->currentTerm); + + // This tallies votes even when the current state is not Candidate, + // but they won't be looked at, so it doesn't matter. + if (ths->state == TAOS_SYNC_STATE_CANDIDATE) { + votesRespondAdd(ths->pVotesRespond, pMsg); + if (pMsg->voteGranted) { + // add vote + voteGrantedVote(ths->pVotesGranted, pMsg); + + // maybe to leader + if (voteGrantedMajority(ths->pVotesGranted)) { + if (!ths->pVotesGranted->toLeader) { + syncNodeCandidate2Leader(ths); + + // prevent to leader again! + ths->pVotesGranted->toLeader = true; + } + } + } else { + ; + // do nothing + // UNCHANGED <> + } + } + + return ret; +} + +#if 0 +int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg) { + int32_t ret = 0; + char logBuf[128] = {0}; snprintf(logBuf, sizeof(logBuf), "==syncNodeOnRequestVoteReplyCb== term:%lu", ths->pRaftStore->currentTerm); syncRequestVoteReplyLog2(logBuf, pMsg); @@ -93,6 +159,7 @@ int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg) return ret; } +#endif int32_t syncNodeOnRequestVoteReplySnapshotCb(SSyncNode* ths, SyncRequestVoteReply* pMsg) { int32_t ret = 0; From d08835d51821a31eee196b8c16562a5201714580 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Sat, 25 Jun 2022 12:03:15 +0800 Subject: [PATCH 31/49] fix: privilege for create topic --- source/dnode/mnode/impl/src/mndTopic.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 078bbc1db7..f881f237c2 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -14,12 +14,12 @@ */ #include "mndTopic.h" -#include "mndPrivilege.h" #include "mndConsumer.h" #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" #include "mndOffset.h" +#include "mndPrivilege.h" #include "mndShow.h" #include "mndStb.h" #include "mndSubscribe.h" @@ -480,6 +480,11 @@ static int32_t mndProcessCreateTopicReq(SRpcMsg *pReq) { goto _OVER; } + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_READ_DB, pDb) != 0) { + goto _OVER; + } + + code = mndCreateTopic(pMnode, pReq, &createTopicReq, pDb); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: @@ -578,7 +583,7 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { } mDebug("trans:%d, used to drop topic:%s", pTrans->id, pTopic->name); - + if (mndDropOffsetByTopic(pMnode, pTrans, dropReq.name) < 0) { ASSERT(0); return -1; From 4a7938e9fffe1c8466a7c84dd059ef0c9c2ee043 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sat, 25 Jun 2022 12:05:17 +0800 Subject: [PATCH 32/49] cast support nchar->binary, binary/nchar->timestamp --- source/libs/scalar/src/sclfunc.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index a3cd1aba6d..e8f48cd0fa 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -881,8 +881,8 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp } case TSDB_DATA_TYPE_TIMESTAMP: { if (inputType == TSDB_DATA_TYPE_BINARY || inputType == TSDB_DATA_TYPE_NCHAR) { - //not support - return TSDB_CODE_FAILED; + //convert to 0 + *(int64_t *)output = 0; } else { GET_TYPED_DATA(*(int64_t *)output, int64_t, inputType, input); } @@ -897,8 +897,16 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp len = sprintf(varDataVal(output), "%.*s", len, varDataVal(input)); varDataSetLen(output, len); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { - //not support - return TSDB_CODE_FAILED; + char *newBuf = taosMemoryCalloc(1, inputLen); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + if (len < 0) { + taosMemoryFree(newBuf); + return TSDB_CODE_FAILED; + } + len = TMIN(len, outputLen - VARSTR_HEADER_SIZE); + memcpy(varDataVal(output), newBuf, len); + varDataSetLen(output, len); + taosMemoryFree(newBuf); } else { char tmp[400] = {0}; NUM_TO_STRING(inputType, input, sizeof(tmp), tmp); From cf4fcb739887ffdf0935277d53e9ffb876d48590 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sat, 25 Jun 2022 12:06:50 +0800 Subject: [PATCH 33/49] feat: support partition by expression and aggregate function output together --- source/libs/planner/src/planLogicCreater.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 6d4aa5ae23..a4cdcd35d3 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -450,6 +450,15 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, int32_t code = TSDB_CODE_SUCCESS; // set grouyp keys, agg funcs and having conditions + if (TSDB_CODE_SUCCESS == code && pSelect->hasAggFuncs) { + code = nodesCollectFuncs(pSelect, SQL_CLAUSE_GROUP_BY, fmIsAggFunc, &pAgg->pAggFuncs); + } + + // rewrite the expression in subsequent clauses + if (TSDB_CODE_SUCCESS == code) { + code = rewriteExprsForSelect(pAgg->pAggFuncs, pSelect, SQL_CLAUSE_GROUP_BY); + } + if (NULL != pSelect->pGroupByList) { pAgg->pGroupKeys = nodesCloneList(pSelect->pGroupByList); if (NULL == pAgg->pGroupKeys) { @@ -457,20 +466,11 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, } } - if (TSDB_CODE_SUCCESS == code && pSelect->hasAggFuncs) { - code = nodesCollectFuncs(pSelect, SQL_CLAUSE_GROUP_BY, fmIsAggFunc, &pAgg->pAggFuncs); - } - // rewrite the expression in subsequent clauses if (TSDB_CODE_SUCCESS == code) { code = rewriteExprsForSelect(pAgg->pGroupKeys, pSelect, SQL_CLAUSE_GROUP_BY); } - // rewrite the expression in subsequent clauses - if (TSDB_CODE_SUCCESS == code) { - code = rewriteExprsForSelect(pAgg->pAggFuncs, pSelect, SQL_CLAUSE_GROUP_BY); - } - if (TSDB_CODE_SUCCESS == code && NULL != pSelect->pHaving) { pAgg->node.pConditions = nodesCloneNode(pSelect->pHaving); if (NULL == pAgg->node.pConditions) { From 916adb82c624ab81a3624c4c0fd2a53df27ae9ca Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 25 Jun 2022 12:10:34 +0800 Subject: [PATCH 34/49] enh(query): add user name when retrieving meta-data from mnode. --- include/common/tmsg.h | 1 + source/common/src/tmsg.c | 3 +++ source/dnode/mnode/impl/src/mndShow.c | 1 + source/libs/executor/inc/executorimpl.h | 3 ++- source/libs/executor/src/executorimpl.c | 9 ++++----- source/libs/executor/src/scanoperator.c | 24 +++++++++++++----------- 6 files changed, 24 insertions(+), 17 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 29eebc332b..a5688af18a 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1284,6 +1284,7 @@ void tFreeSShowRsp(SShowRsp* pRsp); typedef struct { char db[TSDB_DB_FNAME_LEN]; char tb[TSDB_TABLE_NAME_LEN]; + char user[TSDB_USER_LEN]; int64_t showId; } SRetrieveTableReq; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index d96a5ec574..9db5019512 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -3017,6 +3017,7 @@ int32_t tSerializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableReq if (tEncodeI64(&encoder, pReq->showId) < 0) return -1; if (tEncodeCStr(&encoder, pReq->db) < 0) return -1; if (tEncodeCStr(&encoder, pReq->tb) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->user) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -3032,6 +3033,8 @@ int32_t tDeserializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableR if (tDecodeI64(&decoder, &pReq->showId) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->tb) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1; + tEndDecode(&decoder); tDecoderClear(&decoder); return 0; diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 5c2531c25f..3351cff3a3 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -122,6 +122,7 @@ static SShowObj *mndCreateShowObj(SMnode *pMnode, SRetrieveTableReq *pReq) { int32_t size = sizeof(SShowObj); SShowObj showObj = {0}; + showObj.id = showId; showObj.pMnode = pMnode; showObj.type = convertToRetrieveType(pReq->tb, tListLen(pReq->tb)); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index eb7beb19db..d8d231e952 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -358,6 +358,7 @@ typedef struct SSysTableScanInfo { tsem_t ready; SReadHandle readHandle; int32_t accountId; + const char* pUser; bool showRewrite; SNode* pCondition; // db_name filter condition, to discard data that are not in current database SMTbCursor* pCur; // cursor for iterate the local table meta store. @@ -713,7 +714,7 @@ SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, SExecTaskInfo* pTaskInfo, uint64_t queryId, uint64_t taskId); SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode *pScanPhyNode, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode *pScanPhyNode, const char* pUser, SExecTaskInfo* pTaskInfo); SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SExprInfo* pScalarExprInfo, int32_t numOfScalarExpr, SExecTaskInfo* pTaskInfo); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index c23d9a5040..f352049810 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4043,7 +4043,7 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, } SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, - uint64_t queryId, uint64_t taskId, STableListInfo* pTableListInfo) { + uint64_t queryId, uint64_t taskId, STableListInfo* pTableListInfo, const char* pUser) { int32_t type = nodeType(pPhyNode); if (pPhyNode->pChildren == NULL || LIST_LENGTH(pPhyNode->pChildren) == 0) { @@ -4103,8 +4103,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) { SSystemTableScanPhysiNode* pSysScanPhyNode = (SSystemTableScanPhysiNode*)pPhyNode; - return createSysTableScanOperatorInfo(pHandle, pSysScanPhyNode, pTaskInfo); - + return createSysTableScanOperatorInfo(pHandle, pSysScanPhyNode, pUser, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN == type) { STagScanPhysiNode* pScanPhyNode = (STagScanPhysiNode*)pPhyNode; @@ -4167,7 +4166,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SOperatorInfo** ops = taosMemoryCalloc(size, POINTER_BYTES); for (int32_t i = 0; i < size; ++i) { SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, i); - ops[i] = createOperatorTree(pChildNode, pTaskInfo, pHandle, queryId, taskId, pTableListInfo); + ops[i] = createOperatorTree(pChildNode, pTaskInfo, pHandle, queryId, taskId, pTableListInfo, pUser); if (ops[i] == NULL) { return NULL; } @@ -4600,7 +4599,7 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead (*pTaskInfo)->tableqinfoList.pTagCond = pPlan->pTagCond; (*pTaskInfo)->tableqinfoList.pTagIndexCond = pPlan->pTagIndexCond; (*pTaskInfo)->pRoot = - createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId, &(*pTaskInfo)->tableqinfoList); + createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId, &(*pTaskInfo)->tableqinfoList, pPlan->user); if (NULL == (*pTaskInfo)->pRoot) { code = (*pTaskInfo)->code; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index b4595b695a..7affac76d2 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1572,6 +1572,7 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { while (1) { int64_t startTs = taosGetTimestampUs(); strncpy(pInfo->req.tb, tNameGetTableName(&pInfo->name), tListLen(pInfo->req.tb)); + strcpy(pInfo->req.user, pInfo->pUser); if (pInfo->showRewrite) { char dbName[TSDB_DB_NAME_LEN] = {0}; @@ -1704,7 +1705,7 @@ int32_t buildDbTableInfoBlock(const SSDataBlock* p, const SSysTableMeta* pSysDbT } SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode* pScanPhyNode, - SExecTaskInfo* pTaskInfo) { + const char* pUser, SExecTaskInfo* pTaskInfo) { SSysTableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SSysTableScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -1717,13 +1718,14 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan SSDataBlock* pResBlock = createResDataBlock(pDescNode); int32_t num = 0; - SArray* colList = extractColMatchInfo(pScanNode->pScanCols, pDescNode, &num, COL_MATCH_FROM_COL_ID); + SArray* colList = extractColMatchInfo(pScanNode->pScanCols, pDescNode, &num, COL_MATCH_FROM_COL_ID); - pInfo->accountId = pScanPhyNode->accountId; + pInfo->accountId = pScanPhyNode->accountId; + pInfo->pUser = taosMemoryStrDup((void*) pUser); pInfo->showRewrite = pScanPhyNode->showRewrite; - pInfo->pRes = pResBlock; - pInfo->pCondition = pScanNode->node.pConditions; - pInfo->scanCols = colList; + pInfo->pRes = pResBlock; + pInfo->pCondition = pScanNode->node.pConditions; + pInfo->scanCols = colList; initResultSizeInfo(pOperator, 4096); @@ -1739,13 +1741,13 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan pInfo->readHandle = *(SReadHandle*)readHandle; } - pOperator->name = "SysTableScanOperator"; + pOperator->name = "SysTableScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; pOperator->exprSupp.numOfExprs = taosArrayGetSize(pResBlock->pDataBlock); - pOperator->pTaskInfo = pTaskInfo; + pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSysTableScan, NULL, NULL, destroySysScanOperator, NULL, NULL, NULL); From 3b9953f483a8ec3794144b8649c3ac63e7986765 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sat, 25 Jun 2022 12:15:17 +0800 Subject: [PATCH 35/49] cast function support ->float, ->double --- source/libs/scalar/src/sclfunc.c | 36 ++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index e8f48cd0fa..4caccfc7f3 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -879,6 +879,42 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp } break; } + case TSDB_DATA_TYPE_FLOAT: { + if (inputType == TSDB_DATA_TYPE_BINARY) { + *(float *)output = taosStr2Float(varDataVal(input), NULL); + } else if (inputType == TSDB_DATA_TYPE_NCHAR) { + char *newBuf = taosMemoryCalloc(1, inputLen); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + if (len < 0) { + taosMemoryFree(newBuf); + return TSDB_CODE_FAILED; + } + newBuf[len] = 0; + *(float *)output = taosStr2Float(newBuf, NULL); + taosMemoryFree(newBuf); + } else { + GET_TYPED_DATA(*(float *)output, float, inputType, input); + } + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + if (inputType == TSDB_DATA_TYPE_BINARY) { + *(double *)output = taosStr2Double(varDataVal(input), NULL); + } else if (inputType == TSDB_DATA_TYPE_NCHAR) { + char *newBuf = taosMemoryCalloc(1, inputLen); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + if (len < 0) { + taosMemoryFree(newBuf); + return TSDB_CODE_FAILED; + } + newBuf[len] = 0; + *(double *)output = taosStr2Double(newBuf, NULL); + taosMemoryFree(newBuf); + } else { + GET_TYPED_DATA(*(double *)output, double, inputType, input); + } + break; + } case TSDB_DATA_TYPE_TIMESTAMP: { if (inputType == TSDB_DATA_TYPE_BINARY || inputType == TSDB_DATA_TYPE_NCHAR) { //convert to 0 From f7b1960725e5900c6551371878fb48a45bfea0da Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Sat, 25 Jun 2022 12:18:47 +0800 Subject: [PATCH 36/49] ci: drop sync windows test with linux --- Jenkinsfile2 | 36 +----------------------------------- 1 file changed, 1 insertion(+), 35 deletions(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index b65576deaf..6b48aabab8 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -4,11 +4,6 @@ import jenkins.model.CauseOfInterruption node { } -win_test_stage = 0 -linux_ready = 0 -linux_node_ip = "" -linux_node_pass = "" - def abortPreviousBuilds() { def currentJobName = env.JOB_NAME def currentBuildNumber = env.BUILD_NUMBER.toInteger() @@ -289,7 +284,6 @@ def run_win_ctest() { ''' } def run_win_test() { - echo "LINUX NODE: ${linux_node_ip} - ${linux_node_pass}" bat ''' echo "windows test ..." cd %WIN_CONNECTOR_ROOT% @@ -298,9 +292,8 @@ def run_win_test() { ls -l C:\\Windows\\System32\\taos.dll time /t cd %WIN_SYSTEM_TEST_ROOT% - echo "node: ''' + linux_node_ip + ''':''' + linux_node_pass + '''" echo "testing ..." - test-all.bat "{\\\"host\\\":\\\"''' + linux_node_ip + '''\\\",\\\"port\\\":22,\\\"user\\\":\\\"root\\\",\\\"password\\\":\\\"''' + linux_node_pass + '''\\\",\\\"path\\\":\\\"/var/lib/jenkins/workspace/TDinternal\\\"}" + test-all.bat time /t ''' } @@ -331,17 +324,9 @@ pipeline { pre_test_win() pre_test_build_win() run_win_ctest() - script { - while(linux_ready == 0) { - sleep(8) - } - } run_win_test() } } - script { - win_test_stage = 1 - } } } stage('linux test') { @@ -351,17 +336,6 @@ pipeline { changeRequest() } steps { - script { - linux_node_ip = sh ( - script: 'jq .ip /home/node_info.json | sed "s/\\\"//g"', - returnStdout: true - ).trim() - linux_node_pass = sh ( - script: 'jq .password /home/node_info.json | sed "s/\\\"//g" |sed "s/\\!/^^^^^^^^\\!/g"', - returnStdout: true - ).trim() - echo "${linux_node_ip}:${linux_node_pass}" - } catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { timeout(time: 120, unit: 'MINUTES'){ pre_test() @@ -442,14 +416,6 @@ pipeline { } } } - script { - linux_ready = 1 - } - script { - while(win_test_stage == 0){ - sleep(12) - } - } } } } From 7c57b03de71b99bdfa975d4b12e8c39998c29641 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 25 Jun 2022 12:19:52 +0800 Subject: [PATCH 37/49] handle rpc retry --- source/client/src/clientEnv.c | 2 +- source/dnode/mgmt/node_mgmt/src/dmTransport.c | 17 +++++--- source/libs/function/src/udfd.c | 40 ++++++++++--------- source/libs/transport/src/transCli.c | 13 ++++-- source/libs/transport/src/transSvr.c | 3 +- 5 files changed, 46 insertions(+), 29 deletions(-) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index ff9003b8fc..657c21c4f9 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -86,7 +86,7 @@ void closeTransporter(STscObj *pTscObj) { static bool clientRpcRfp(int32_t code) { if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED || - code == TSDB_CODE_SYN_NOT_LEADER) { + code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY) { return true; } else { return false; diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index 63d2a65df1..a4745abd5b 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -70,9 +70,9 @@ int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg) { } static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) { - SDnodeTrans *pTrans = &pDnode->trans; + SDnodeTrans * pTrans = &pDnode->trans; int32_t code = -1; - SRpcMsg *pMsg = NULL; + SRpcMsg * pMsg = NULL; SMgmtWrapper *pWrapper = NULL; SDnodeHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(pRpc->msgType)]; @@ -194,11 +194,11 @@ int32_t dmInitMsgHandle(SDnode *pDnode) { for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) { SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype]; - SArray *pArray = (*pWrapper->func.getHandlesFp)(); + SArray * pArray = (*pWrapper->func.getHandlesFp)(); if (pArray == NULL) return -1; for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { - SMgmtHandle *pMgmt = taosArrayGet(pArray, i); + SMgmtHandle * pMgmt = taosArrayGet(pArray, i); SDnodeHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(pMgmt->msgType)]; if (pMgmt->needCheckVgId) { pHandle->needCheckVgId = pMgmt->needCheckVgId; @@ -248,7 +248,14 @@ static inline void dmReleaseHandle(SRpcHandleInfo *pHandle, int8_t type) { } } -static bool rpcRfp(int32_t code) { return code == TSDB_CODE_RPC_REDIRECT; } +static bool rpcRfp(int32_t code) { + if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED || + code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY) { + return true; + } else { + return false; + } +} int32_t dmInitClient(SDnode *pDnode) { SDnodeTrans *pTrans = &pDnode->trans; diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 838071dbf1..983cffe9dc 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -12,6 +12,8 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ + +// clang-format off #include "uv.h" #include "os.h" #include "fnLog.h" @@ -25,6 +27,7 @@ #include "tglobal.h" #include "tmsg.h" #include "trpc.h" +// clang-foramt on typedef struct SUdfdContext { uv_loop_t * loop; @@ -103,12 +106,12 @@ typedef struct SUdfdRpcSendRecvInfo { uv_sem_t resultSem; } SUdfdRpcSendRecvInfo; -static void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); +static void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); static int32_t udfdFillUdfInfoFromMNode(void *clientRpc, char *udfName, SUdf *udf); static int32_t udfdConnectToMnode(); static int32_t udfdLoadUdf(char *udfName, SUdf *udf); -static bool udfdRpcRfp(int32_t code); -static int initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet); +static bool udfdRpcRfp(int32_t code); +static int initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet); static int32_t udfdOpenClientRpc(); static int32_t udfdCloseClientRpc(); @@ -126,19 +129,19 @@ static void udfdUvHandleError(SUdfdUvConn *conn) { uv_close((uv_handle_t *)conn- static void udfdPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf); static void udfdOnNewConnection(uv_stream_t *server, int status); -static void udfdIntrSignalHandler(uv_signal_t *handle, int signum); +static void udfdIntrSignalHandler(uv_signal_t *handle, int signum); static int32_t removeListeningPipe(); -static void udfdPrintVersion(); +static void udfdPrintVersion(); static int32_t udfdParseArgs(int32_t argc, char *argv[]); static int32_t udfdInitLog(); -static void udfdCtrlAllocBufCb(uv_handle_t *handle, size_t suggested_size, uv_buf_t *buf); -static void udfdCtrlReadCb(uv_stream_t *q, ssize_t nread, const uv_buf_t *buf); +static void udfdCtrlAllocBufCb(uv_handle_t *handle, size_t suggested_size, uv_buf_t *buf); +static void udfdCtrlReadCb(uv_stream_t *q, ssize_t nread, const uv_buf_t *buf); static int32_t udfdUvInit(); -static void udfdCloseWalkCb(uv_handle_t *handle, void *arg); +static void udfdCloseWalkCb(uv_handle_t *handle, void *arg); static int32_t udfdRun(); -static void udfdConnectMnodeThreadFunc(void* args); +static void udfdConnectMnodeThreadFunc(void *args); void udfdProcessRequest(uv_work_t *req) { SUvUdfWork *uvUdf = (SUvUdfWork *)(req->data); @@ -401,11 +404,11 @@ void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { udf->bufSize = pFuncInfo->bufSize; char path[PATH_MAX] = {0}; - #ifdef WINDOWS +#ifdef WINDOWS snprintf(path, sizeof(path), "%s%s.dll", TD_TMP_DIR_PATH, pFuncInfo->name); - #else +#else snprintf(path, sizeof(path), "%s/lib%s.so", TD_TMP_DIR_PATH, pFuncInfo->name); - #endif +#endif TdFilePtr file = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC | TD_FILE_AUTO_DEL); if (file == NULL) { @@ -544,7 +547,8 @@ int32_t udfdLoadUdf(char *udfName, SUdf *udf) { return 0; } static bool udfdRpcRfp(int32_t code) { - if (code == TSDB_CODE_RPC_REDIRECT) { + if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED || + code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY) { return true; } else { return false; @@ -652,8 +656,7 @@ void udfdAllocBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) { buf->base = ctx->inputBuf; buf->len = ctx->inputCap; } else { - fnError("udfd can not allocate enough memory") - buf->base = NULL; + fnError("udfd can not allocate enough memory") buf->base = NULL; buf->len = 0; } } else { @@ -664,8 +667,7 @@ void udfdAllocBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) { buf->base = ctx->inputBuf + ctx->inputLen; buf->len = ctx->inputCap - ctx->inputLen; } else { - fnError("udfd can not allocate enough memory") - buf->base = NULL; + fnError("udfd can not allocate enough memory") buf->base = NULL; buf->len = 0; } } @@ -881,7 +883,7 @@ static int32_t udfdRun() { return 0; } -void udfdConnectMnodeThreadFunc(void* args) { +void udfdConnectMnodeThreadFunc(void *args) { int32_t retryMnodeTimes = 0; int32_t code = 0; while (retryMnodeTimes++ <= TSDB_MAX_REPLICA) { @@ -939,7 +941,7 @@ int main(int argc, char *argv[]) { uv_thread_create(&mnodeConnectThread, udfdConnectMnodeThreadFunc, NULL); udfdRun(); - + removeListeningPipe(); udfdCloseClientRpc(); diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 0671aa39d1..a21b753d6e 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -758,13 +758,16 @@ static void cliHandleUpdate(SCliMsg* pMsg, SCliThrd* pThrd) { destroyCmsg(pMsg); } -SCliConn* cliGetConn(SCliMsg* pMsg, SCliThrd* pThrd) { +SCliConn* cliGetConn(SCliMsg* pMsg, SCliThrd* pThrd, bool* ignore) { SCliConn* conn = NULL; int64_t refId = (int64_t)(pMsg->msg.info.handle); if (refId != 0) { SExHandle* exh = transAcquireExHandle(refMgt, refId); if (exh == NULL) { - assert(0); + *ignore = true; + destroyCmsg(pMsg); + return NULL; + // assert(0); } else { conn = exh->handle; transReleaseExHandle(refMgt, refId); @@ -799,7 +802,11 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) { cliMayCvtFqdnToIp(&pCtx->epSet, &pThrd->cvtAddr); // transPrintEpSet(&pCtx->epSet); - SCliConn* conn = cliGetConn(pMsg, pThrd); + bool ignore = false; + SCliConn* conn = cliGetConn(pMsg, pThrd, &ignore); + if (ignore == true) { + return; + } if (conn != NULL) { transCtxMerge(&conn->ctx, &pCtx->appCtx); transQueuePush(&conn->cliMsgs, pMsg); diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 7651860224..892d32696e 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -422,7 +422,8 @@ static void uvPrepareSendData(SSvrMsg* smsg, uv_buf_t* wb) { transUnrefSrvHandle(pConn); } else { pHead->msgType = pMsg->msgType; - if (pHead->msgType == 0) pHead->msgType = pConn->inType + 1; + if (pHead->msgType == 0 && transMsgLenFromCont(pMsg->contLen) == sizeof(STransMsgHead)) + pHead->msgType = pConn->inType + 1; } } From b139daf08bbcdd958e9e930413f1ea87fb691a41 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sat, 25 Jun 2022 13:32:14 +0800 Subject: [PATCH 38/49] add cast function support ->bool --- source/libs/scalar/src/sclfunc.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 4caccfc7f3..80bebafef2 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -915,6 +915,24 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp } break; } + case TSDB_DATA_TYPE_BOOL: { + if (inputType == TSDB_DATA_TYPE_BINARY) { + *(bool *)output = taosStr2Int8(varDataVal(input), NULL, 10); + } else if (inputType == TSDB_DATA_TYPE_NCHAR) { + char *newBuf = taosMemoryCalloc(1, inputLen); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + if (len < 0) { + taosMemoryFree(newBuf); + return TSDB_CODE_FAILED; + } + newBuf[len] = 0; + *(bool *)output = taosStr2Int8(newBuf, NULL, 10); + taosMemoryFree(newBuf); + } else { + GET_TYPED_DATA(*(bool *)output, bool, inputType, input); + } + break; + } case TSDB_DATA_TYPE_TIMESTAMP: { if (inputType == TSDB_DATA_TYPE_BINARY || inputType == TSDB_DATA_TYPE_NCHAR) { //convert to 0 From 27a67652824dc3ea3903d16819e40b01caf5500a Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Sat, 25 Jun 2022 13:46:53 +0800 Subject: [PATCH 39/49] fix: add a default parameter to windows python test script --- Jenkinsfile2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 6b48aabab8..515667d8cc 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -293,7 +293,7 @@ def run_win_test() { time /t cd %WIN_SYSTEM_TEST_ROOT% echo "testing ..." - test-all.bat + test-all.bat ci time /t ''' } From 7acf01225391e02a7eedc0665aff907a1c5654ab Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Sat, 25 Jun 2022 13:51:43 +0800 Subject: [PATCH 40/49] fix:memory error --- source/dnode/vnode/src/meta/metaTable.c | 2 +- source/libs/parser/src/parTranslater.c | 3 +++ source/libs/scalar/src/sclvector.c | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 4a3feca8d0..7109bf1dfc 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -388,7 +388,7 @@ int metaTtlDropTable(SMeta *pMeta, int64_t ttl, SArray *tbUids) { } static void metaBuildTtlIdxKey(STtlIdxKey *ttlKey, const SMetaEntry *pME){ - int32_t ttlDays; + int64_t ttlDays; int64_t ctime; if (pME->type == TSDB_CHILD_TABLE) { ctime = pME->ctbEntry.ctime; diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index d4bde5d624..fe562fc936 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -4963,6 +4963,7 @@ typedef struct SVgroupCreateTableBatch { static void destroyCreateTbReq(SVCreateTbReq* pReq) { taosMemoryFreeClear(pReq->name); + taosMemoryFreeClear(pReq->comment); taosMemoryFreeClear(pReq->ntb.schemaRow.pSchema); } @@ -4980,6 +4981,7 @@ static int32_t buildNormalTableBatchReq(int32_t acctId, const SCreateTableStmt* if (pStmt->pOptions->commentNull == false) { req.comment = strdup(pStmt->pOptions->comment); if (NULL == req.comment) { + destroyCreateTbReq(&req); return TSDB_CODE_OUT_OF_MEMORY; } req.commentLen = strlen(pStmt->pOptions->comment); @@ -5051,6 +5053,7 @@ static void destroyCreateTbReqBatch(SVgroupCreateTableBatch* pTbBatch) { for (int32_t i = 0; i < size; ++i) { SVCreateTbReq* pTableReq = taosArrayGet(pTbBatch->req.pArray, i); taosMemoryFreeClear(pTableReq->name); + taosMemoryFreeClear(pTableReq->comment); if (pTableReq->type == TSDB_NORMAL_TABLE) { taosMemoryFreeClear(pTableReq->ntb.schemaRow.pSchema); diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index f9a9ec0f72..292db3db1f 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -468,7 +468,7 @@ double getVectorDoubleValue_JSON(void *src, int32_t index){ } void* ncharTobinary(void *buf){ // todo need to remove , if tobinary is nchar - int32_t inputLen = varDataLen(buf); + int32_t inputLen = varDataTLen(buf); void* t = taosMemoryCalloc(1, inputLen); int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(buf), varDataLen(buf), varDataVal(t)); From a9f094613a47047acab797feec6d8eb58798e718 Mon Sep 17 00:00:00 2001 From: slzhou Date: Sat, 25 Jun 2022 13:52:52 +0800 Subject: [PATCH 41/49] fix: restore table merge scan operator --- source/dnode/vnode/inc/vnode.h | 1 + source/dnode/vnode/src/tsdb/tsdbRead.c | 2 +- source/libs/executor/src/executil.c | 4 +- source/libs/executor/src/executorimpl.c | 3 +- source/libs/executor/src/scanoperator.c | 122 ++++++++++++++---------- 5 files changed, 77 insertions(+), 55 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index a32bf0ecdb..b97d4605e7 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -196,6 +196,7 @@ struct SVnodeCfg { typedef struct { TSKEY lastKey; uint64_t uid; + uint64_t groupId; } STableKeyInfo; struct SMetaEntry { diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 540810f876..ab31d65d68 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2852,7 +2852,7 @@ int32_t tsdbGetAllTableList(SMeta* pMeta, uint64_t uid, SArray* list) { break; } - STableKeyInfo info = {.lastKey = TSKEY_INITIAL_VAL, uid = id}; + STableKeyInfo info = {.lastKey = TSKEY_INITIAL_VAL, uid = id, .groupId = 0}; taosArrayPush(list, &info); } diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 5ac5957f2b..374a3a736d 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -315,7 +315,7 @@ int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo } for (int i = 0; i < taosArrayGetSize(res); i++) { - STableKeyInfo info = {.lastKey = TSKEY_INITIAL_VAL, .uid = *(uint64_t*)taosArrayGet(res, i)}; + STableKeyInfo info = {.lastKey = TSKEY_INITIAL_VAL, .uid = *(uint64_t*)taosArrayGet(res, i), .groupId = 0}; taosArrayPush(pListInfo->pTableList, &info); } taosArrayDestroy(res); @@ -336,7 +336,7 @@ int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo } } }else { // Create one table group. - STableKeyInfo info = {.lastKey = 0, .uid = tableUid}; + STableKeyInfo info = {.lastKey = 0, .uid = tableUid, .groupId = 0}; taosArrayPush(pListInfo->pTableList, &info); } pListInfo->pGroupList = taosArrayInit(4, POINTER_BYTES); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index c23d9a5040..b4d87a53b1 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4028,6 +4028,7 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, int32_t len = (int32_t)(pStart - (char*)keyBuf); uint64_t groupId = calcGroupId(keyBuf, len); taosHashPut(pTableListInfo->map, &(info->uid), sizeof(uint64_t), &groupId, sizeof(uint64_t)); + info->groupId = groupId; groupNum++; nodesClearList(groupNew); @@ -4127,7 +4128,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo return NULL; } } else { // Create one table group. - STableKeyInfo info = {.lastKey = 0, .uid = pBlockNode->uid}; + STableKeyInfo info = {.lastKey = 0, .uid = pBlockNode->uid, .groupId = 0}; taosArrayPush(pTableListInfo->pTableList, &info); } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 9c0ed40c30..dedf6c0707 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -1965,7 +1965,10 @@ _error: typedef struct STableMergeScanInfo { STableListInfo* tableListInfo; - int32_t currentGroupId; + int32_t tableStartIndex; + int32_t tableEndIndex; + bool hasGroupId; + uint64_t groupId; SArray* dataReaders; // array of tsdbReaderT* SReadHandle readHandle; @@ -2006,7 +2009,7 @@ typedef struct STableMergeScanInfo { int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan int32_t dataBlockLoadFlag; SInterval interval; // if the upstream is an interval operator, the interval info is also kept here to get the time - // window to check if current data block needs to be loaded. + // window to check if current data block needs to be loaded. SSampleExecInfo sample; // sample execution info } STableMergeScanInfo; @@ -2030,6 +2033,22 @@ int32_t createScanTableListInfo(STableScanPhysiNode* pTableScanNode, SReadHandle return TSDB_CODE_SUCCESS; } +int32_t createMultipleDataReaders(SQueryTableDataCond* pQueryCond, SReadHandle* pHandle, STableListInfo* pTableListInfo, + int32_t tableStartIdx, int32_t tableEndIdx, SArray* arrayReader, uint64_t queryId, + uint64_t taskId) { + for (int32_t i = tableStartIdx; i <= tableEndIdx; ++i) { + SArray* subTableList = taosArrayInit(1, sizeof(STableKeyInfo)); + taosArrayPush(subTableList, taosArrayGet(pTableListInfo->pTableList, i)); + + tsdbReaderT* pReader = tsdbReaderOpen(pHandle->vnode, pQueryCond, subTableList, queryId, taskId); + taosArrayPush(arrayReader, &pReader); + + taosArrayDestroy(subTableList); + } + + return TSDB_CODE_SUCCESS; +} + // todo refactor static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeScanInfo* pTableScanInfo, int32_t readerIdx, SSDataBlock* pBlock, uint32_t* status) { @@ -2216,34 +2235,32 @@ SArray* generateSortByTsInfo(int32_t order) { return pList; } -static int32_t createMultipleDataReaders(SQueryTableDataCond* pQueryCond, SReadHandle* pHandle, SArray* tableList, SArray* arrayReader, uint64_t queryId, - uint64_t taskId) { - for (int32_t i = 0; i < taosArrayGetSize(tableList); ++i) { - SArray* tmp = taosArrayInit(1, sizeof(STableKeyInfo)); - taosArrayPush(tmp, taosArrayGet(tableList, i)); - - tsdbReaderT* pReader = tsdbReaderOpen(pHandle->vnode, pQueryCond, tmp, queryId, taskId); - taosArrayPush(arrayReader, &pReader); - - taosArrayDestroy(tmp); - } - - return TSDB_CODE_SUCCESS; -} - int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) { STableMergeScanInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SArray* tableList = taosArrayGetP(pInfo->tableListInfo->pGroupList, pInfo->currentGroupId); + { + size_t tableListSize = taosArrayGetSize(pInfo->tableListInfo->pTableList); + int32_t i = pInfo->tableStartIndex + 1; + for (; i < tableListSize; ++i) { + STableKeyInfo* tableKeyInfo = taosArrayGet(pInfo->tableListInfo->pTableList, i); + if (tableKeyInfo->groupId != pInfo->groupId) { + break; + } + } + pInfo->tableEndIndex = i - 1; + } - createMultipleDataReaders(&pInfo->cond, &pInfo->readHandle, tableList, + int32_t tableStartIdx = pInfo->tableStartIndex; + int32_t tableEndIdx = pInfo->tableEndIndex; + + STableListInfo* tableListInfo = pInfo->tableListInfo; + createMultipleDataReaders(&pInfo->cond, &pInfo->readHandle, tableListInfo, tableStartIdx, tableEndIdx, pInfo->dataReaders, pInfo->queryId, pInfo->taskId); // todo the total available buffer should be determined by total capacity of buffer of this task. // the additional one is reserved for merge result - int32_t tableLen = taosArrayGetSize(tableList); - pInfo->sortBufSize = pInfo->bufPageSize * ((tableLen==0?1:tableLen) + 1); + pInfo->sortBufSize = pInfo->bufPageSize * (tableEndIdx - tableStartIdx + 1 + 1); int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_MULTISOURCE_MERGE, pInfo->bufPageSize, numOfBufPage, pInfo->pSortInputBlock, pTaskInfo->id.str); @@ -2330,43 +2347,38 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) { if (code != TSDB_CODE_SUCCESS) { longjmp(pTaskInfo->env, code); } + size_t tableListSize = taosArrayGetSize(pInfo->tableListInfo->pTableList); + if (!pInfo->hasGroupId) { + pInfo->hasGroupId = true; - if (pInfo->currentGroupId == -1) { - pInfo->currentGroupId++; - if (pInfo->currentGroupId >= taosArrayGetSize(pInfo->tableListInfo->pGroupList)) { + if (tableListSize == 0) { doSetOperatorCompleted(pOperator); return NULL; } + pInfo->tableStartIndex = 0; + pInfo->groupId = ((STableKeyInfo*)taosArrayGet(pInfo->tableListInfo->pTableList, pInfo->tableStartIndex))->groupId; startGroupTableMergeScan(pOperator); } - SSDataBlock* pBlock = getSortedTableMergeScanBlockData(pInfo->pSortHandle, pOperator->resultInfo.capacity, pOperator); - if (pBlock != NULL) { - uint64_t* groupId = taosHashGet(pInfo->tableListInfo->map, &(pBlock->info.uid), sizeof(uint64_t)); - if(groupId) pBlock->info.groupId = *groupId; - - pOperator->resultInfo.totalRows += pBlock->info.rows; - return pBlock; + SSDataBlock* pBlock = NULL; + while (pInfo->tableStartIndex < tableListSize) { + pBlock = getSortedTableMergeScanBlockData(pInfo->pSortHandle, pOperator->resultInfo.capacity, pOperator); + if (pBlock != NULL) { + pBlock->info.groupId = pInfo->groupId; + pOperator->resultInfo.totalRows += pBlock->info.rows; + return pBlock; + } else { + stopGroupTableMergeScan(pOperator); + if (pInfo->tableEndIndex >= tableListSize - 1) { + doSetOperatorCompleted(pOperator); + break; + } + pInfo->tableStartIndex = pInfo->tableEndIndex + 1; + pInfo->groupId = + ((STableKeyInfo*)taosArrayGet(pInfo->tableListInfo->pTableList, pInfo->tableStartIndex))->groupId; + startGroupTableMergeScan(pOperator); + } } - stopGroupTableMergeScan(pOperator); - pInfo->currentGroupId++; - if (pInfo->currentGroupId >= taosArrayGetSize(pInfo->tableListInfo->pGroupList)) { - doSetOperatorCompleted(pOperator); - return NULL; - } - startGroupTableMergeScan(pOperator); - - pBlock = getSortedTableMergeScanBlockData(pInfo->pSortHandle, pOperator->resultInfo.capacity, pOperator); - if (pBlock != NULL) { - uint64_t* groupId = taosHashGet(pInfo->tableListInfo->map, &(pBlock->info.uid), sizeof(uint64_t)); - if(groupId) pBlock->info.groupId = *groupId; - - pOperator->resultInfo.totalRows += pBlock->info.rows; - return pBlock; - } - - doSetOperatorCompleted(pOperator); - return pBlock; } @@ -2403,6 +2415,12 @@ int32_t getTableMergeScanExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExpla return TSDB_CODE_SUCCESS; } +int32_t compareTableKeyInfoByGid(const void* p1, const void* p2) { + const STableKeyInfo* info1 = p1; + const STableKeyInfo* info2 = p2; + return info1->groupId - info2->groupId; +} + SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, STableListInfo* pTableListInfo, SReadHandle* readHandle, SExecTaskInfo* pTaskInfo, uint64_t queryId, uint64_t taskId) { @@ -2411,6 +2429,9 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN if (pInfo == NULL || pOperator == NULL) { goto _error; } + if (pTableScanNode->pPartitionTags) { + taosArraySort(pTableListInfo->pTableList, compareTableKeyInfoByGid); + } SDataBlockDescNode* pDescNode = pTableScanNode->scan.node.pOutputDataBlockDesc; @@ -2443,7 +2464,6 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN pInfo->dataReaders = taosArrayInit(64, POINTER_BYTES); pInfo->queryId = queryId; pInfo->taskId = taskId; - pInfo->currentGroupId = -1; pInfo->sortSourceParams = taosArrayInit(64, sizeof(STableMergeScanSortSourceParam)); From 25a12d96b05f6234c069de516b3c632d2a700d68 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Sat, 25 Jun 2022 14:06:33 +0800 Subject: [PATCH 42/49] handle rpc retry --- source/client/src/clientEnv.c | 2 +- source/libs/transport/src/trans.c | 1 + source/libs/transport/src/transCli.c | 9 ++++++--- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 657c21c4f9..d7bf4b60f1 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -136,7 +136,7 @@ void destroyTscObj(void *pObj) { schedulerStopQueryHb(pTscObj->pAppInfo->pTransporter); if (0 == connNum) { // TODO - closeTransporter(pTscObj); + // closeTransporter(pTscObj); } tscDebug("connObj 0x%" PRIx64 " destroyed, totalConn:%" PRId64, *(int64_t *)pTscObj->id, pTscObj->pAppInfo->numOfConns); diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 1ec96f4a7a..4f7b19b539 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -79,6 +79,7 @@ void* rpcOpen(const SRpcInit* pInit) { return pRpc; } void rpcClose(void* arg) { + tInfo("start to close rpc"); SRpcInfo* pRpc = (SRpcInfo*)arg; (*taosCloseHandle[pRpc->connType])(pRpc->tcphandle); transCloseExHandleMgt(pRpc->refMgt); diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index a21b753d6e..7374d1fffc 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -724,14 +724,13 @@ void cliConnCb(uv_connect_t* req, int status) { } static void cliHandleQuit(SCliMsg* pMsg, SCliThrd* pThrd) { + pThrd->quit = true; tDebug("cli work thread %p start to quit", pThrd); destroyCmsg(pMsg); destroyConnPool(pThrd->pool); uv_timer_stop(&pThrd->timer); uv_walk(pThrd->loop, cliWalkCb, NULL); - pThrd->quit = true; - // uv_stop(pThrd->loop); } static void cliHandleRelease(SCliMsg* pMsg, SCliThrd* pThrd) { @@ -977,7 +976,10 @@ void cliWalkCb(uv_handle_t* handle, void* arg) { } int cliRBChoseIdx(STrans* pTransInst) { - int64_t index = pTransInst->index; + int8_t index = pTransInst->index; + if (pTransInst->numOfThreads == 0) { + return -1; + } if (pTransInst->index++ >= pTransInst->numOfThreads) { pTransInst->index = 0; } @@ -1120,6 +1122,7 @@ SCliThrd* transGetWorkThrdFromHandle(int64_t handle) { SCliThrd* transGetWorkThrd(STrans* trans, int64_t handle) { if (handle == 0) { int idx = cliRBChoseIdx(trans); + if (idx < 0) return NULL; return ((SCliObj*)trans->tcphandle)->pThreadObj[idx]; } return transGetWorkThrdFromHandle(handle); From d05ef645077a3401154f037806550bfc7f8eeaf7 Mon Sep 17 00:00:00 2001 From: tangfangzhi Date: Sat, 25 Jun 2022 14:10:34 +0800 Subject: [PATCH 43/49] enh: skip install package --- Jenkinsfile2 | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 515667d8cc..861478160f 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -392,27 +392,6 @@ pipeline { cd ${WKC}/packaging ./release.sh -v cluster -n 3.0.0.100 -s static ''' - sh ''' - echo "install ..." - cd ${WKC}/release - tar xzf TDengine-enterprise-server-3.0.0.100-Linux-x64.tar.gz - cd TDengine-enterprise-server-3.0.0.100 - service taosd stop || : - rm -rf /var/lib/taos - ./install.sh -e no - ''' - sh ''' - echo "checking ..." - which taos - which taosd - rm -rf ${WK}/debug - mv ${WKC}/debug ${WK}/ - ''' - sh ''' - echo "install taospy ..." - cd ${WKPY} - pip3 install . - ''' } } } From 52bfe2e74cac305bebb3adaf8a2a49dcedce1a6a Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Sat, 25 Jun 2022 14:19:17 +0800 Subject: [PATCH 44/49] modify test cases for cast.py --- tests/system-test/2-query/cast.py | 38 +++++++++++++++---------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/tests/system-test/2-query/cast.py b/tests/system-test/2-query/cast.py index 387170991f..934bbbd7b4 100644 --- a/tests/system-test/2-query/cast.py +++ b/tests/system-test/2-query/cast.py @@ -630,27 +630,27 @@ class TDTestCase: ( tdSql.checkData(i, 0, '12') for i in range(tdSql.queryRows) ) tdLog.printNoPrefix("==========step40: error cast condition, should return error ") - tdSql.error("select cast(c1 as int) as b from ct4") - tdSql.error("select cast(c1 as bool) as b from ct4") - tdSql.error("select cast(c1 as tinyint) as b from ct4") - tdSql.error("select cast(c1 as smallint) as b from ct4") - tdSql.error("select cast(c1 as float) as b from ct4") - tdSql.error("select cast(c1 as double) as b from ct4") - tdSql.error("select cast(c1 as tinyint unsigned) as b from ct4") - tdSql.error("select cast(c1 as smallint unsigned) as b from ct4") - tdSql.error("select cast(c1 as int unsigned) as b from ct4") + #tdSql.error("select cast(c1 as int) as b from ct4") + #tdSql.error("select cast(c1 as bool) as b from ct4") + #tdSql.error("select cast(c1 as tinyint) as b from ct4") + #tdSql.error("select cast(c1 as smallint) as b from ct4") + #tdSql.error("select cast(c1 as float) as b from ct4") + #tdSql.error("select cast(c1 as double) as b from ct4") + #tdSql.error("select cast(c1 as tinyint unsigned) as b from ct4") + #tdSql.error("select cast(c1 as smallint unsigned) as b from ct4") + #tdSql.error("select cast(c1 as int unsigned) as b from ct4") - tdSql.error("select cast(c2 as int) as b from ct4") - tdSql.error("select cast(c3 as bool) as b from ct4") - tdSql.error("select cast(c4 as tinyint) as b from ct4") - tdSql.error("select cast(c5 as smallint) as b from ct4") - tdSql.error("select cast(c6 as float) as b from ct4") - tdSql.error("select cast(c7 as double) as b from ct4") - tdSql.error("select cast(c8 as tinyint unsigned) as b from ct4") + #tdSql.error("select cast(c2 as int) as b from ct4") + #tdSql.error("select cast(c3 as bool) as b from ct4") + #tdSql.error("select cast(c4 as tinyint) as b from ct4") + #tdSql.error("select cast(c5 as smallint) as b from ct4") + #tdSql.error("select cast(c6 as float) as b from ct4") + #tdSql.error("select cast(c7 as double) as b from ct4") + #tdSql.error("select cast(c8 as tinyint unsigned) as b from ct4") - tdSql.error("select cast(c8 as timestamp ) as b from ct4") - tdSql.error("select cast(c9 as timestamp ) as b from ct4") - tdSql.error("select cast(c9 as binary(64) ) as b from ct4") + #tdSql.error("select cast(c8 as timestamp ) as b from ct4") + #tdSql.error("select cast(c9 as timestamp ) as b from ct4") + #tdSql.error("select cast(c9 as binary(64) ) as b from ct4") pass def run(self): From b0f16e105566fbb149dfba4ddaaf5e7a835124f0 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 25 Jun 2022 14:27:19 +0800 Subject: [PATCH 45/49] feat: taos-tools update for3.0 (#14163) * feat: update taos-tools for 3.0 prepare for 3.0 [TD-13052] * feat: update taos-tools fix -I -s [TD-13052] --- tools/taos-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/taos-tools b/tools/taos-tools index 28a49b447f..a875a057d1 160000 --- a/tools/taos-tools +++ b/tools/taos-tools @@ -1 +1 @@ -Subproject commit 28a49b447f71c4f014ebbac858b7215b897d57fd +Subproject commit a875a057d1225d85c6323b9edaccc2b1a9641987 From 6adc19d4437705552669c7459e80b763c7344de5 Mon Sep 17 00:00:00 2001 From: slzhou Date: Sat, 25 Jun 2022 15:20:11 +0800 Subject: [PATCH 46/49] fix: overlapping intervals problem --- source/libs/executor/src/timewindowoperator.c | 51 +++++++++++-------- 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 48b0b1c071..0ba898e4f7 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -3900,18 +3900,22 @@ _error: // merge interval operator typedef struct SMergeIntervalAggOperatorInfo { SIntervalAggOperatorInfo intervalAggOperatorInfo; - - SHashObj* groupIntervalHash; - void* groupIntervalIter; + SList* groupIntervals; + SListIter groupIntervalsIter; bool hasGroupId; uint64_t groupId; SSDataBlock* prefetchedBlock; bool inputBlocksFinished; } SMergeIntervalAggOperatorInfo; +typedef struct SGroupTimeWindow { + uint64_t groupId; + STimeWindow window; +} SGroupTimeWindow; + void destroyMergeIntervalOperatorInfo(void* param, int32_t numOfOutput) { SMergeIntervalAggOperatorInfo* miaInfo = (SMergeIntervalAggOperatorInfo*)param; - taosHashCleanup(miaInfo->groupIntervalHash); + tdListFree(miaInfo->groupIntervals); destroyIntervalOperatorInfo(&miaInfo->intervalAggOperatorInfo, numOfOutput); } @@ -3940,15 +3944,22 @@ static int32_t outputPrevIntervalResult(SOperatorInfo* pOperatorInfo, uint64_t t bool ascScan = (iaInfo->order == TSDB_ORDER_ASC); SExprSupp* pExprSup = &pOperatorInfo->exprSupp; - STimeWindow* prevWin = taosHashGet(miaInfo->groupIntervalHash, &tableGroupId, sizeof(tableGroupId)); - if (prevWin == NULL) { - taosHashPut(miaInfo->groupIntervalHash, &tableGroupId, sizeof(tableGroupId), newWin, sizeof(STimeWindow)); - return 0; - } + SGroupTimeWindow groupTimeWindow = {.groupId = tableGroupId, .window = *newWin}; + tdListAppend(miaInfo->groupIntervals, &groupTimeWindow); - if ((ascScan && newWin->skey > prevWin->skey || (!ascScan) && newWin->skey < prevWin->skey)) { - finalizeWindowResult(pOperatorInfo, tableGroupId, prevWin, pResultBlock); - taosHashPut(miaInfo->groupIntervalHash, &tableGroupId, sizeof(tableGroupId), newWin, sizeof(STimeWindow)); + SListIter iter = {0}; + tdListInitIter(miaInfo->groupIntervals, &iter, TD_LIST_FORWARD); + SListNode* listNode = NULL; + while ((listNode = tdListNext(&iter)) != NULL) { + SGroupTimeWindow* prevGrpWin = (SGroupTimeWindow*)listNode->data; + if (prevGrpWin->groupId != tableGroupId ) { + continue; + } + STimeWindow* prevWin = &prevGrpWin->window; + if ((ascScan && newWin->skey > prevWin->ekey || (!ascScan) && newWin->skey < prevWin->ekey)) { + finalizeWindowResult(pOperatorInfo, tableGroupId, prevWin, pResultBlock); + tdListPopNode(miaInfo->groupIntervals, listNode); + } } return 0; @@ -4075,6 +4086,7 @@ static SSDataBlock* doMergeIntervalAgg(SOperatorInfo* pOperator) { } if (pBlock == NULL) { + tdListInitIter(miaInfo->groupIntervals, &miaInfo->groupIntervalsIter, TD_LIST_FORWARD); miaInfo->inputBlocksFinished = true; break; } @@ -4100,14 +4112,12 @@ static SSDataBlock* doMergeIntervalAgg(SOperatorInfo* pOperator) { } if (miaInfo->inputBlocksFinished) { - void* win = taosHashIterate(miaInfo->groupIntervalHash, miaInfo->groupIntervalIter); - if (win != NULL) { - miaInfo->groupIntervalIter = win; + SListNode* listNode = tdListNext(&miaInfo->groupIntervalsIter); - size_t len = 0; - uint64_t* pTableGroupId = taosHashGetKey(win, &len); - finalizeWindowResult(pOperator, *pTableGroupId, win, pRes); - pRes->info.groupId = *pTableGroupId; + if (listNode != NULL) { + SGroupTimeWindow* grpWin = (SGroupTimeWindow*)(listNode->data); + finalizeWindowResult(pOperator, grpWin->groupId, &grpWin->window, pRes); + pRes->info.groupId = grpWin->groupId; } } @@ -4129,8 +4139,7 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SExprI goto _error; } - miaInfo->groupIntervalHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_NO_LOCK); - miaInfo->groupIntervalIter = NULL; + miaInfo->groupIntervals = tdListNew(sizeof(SGroupTimeWindow)); SIntervalAggOperatorInfo* iaInfo = &miaInfo->intervalAggOperatorInfo; From 592d8e488f5c877f41b7ea3bed67956f3d94a4c4 Mon Sep 17 00:00:00 2001 From: afwerar <1296468573@qq.com> Date: Sat, 25 Jun 2022 16:14:54 +0800 Subject: [PATCH 47/49] test: add sim full test --- tests/pytest/util/dnodes.py | 24 +++++---- tests/script/test-all.bat | 58 +++++++++++++++++++++ tests/system-test/6-cluster/5dnode1mnode.py | 6 +-- tests/system-test/6-cluster/5dnode2mnode.py | 5 +- tests/system-test/7-tmq/tmqCommon.py | 8 ++- tests/system-test/7-tmq/tmqError.py | 5 +- tests/system-test/test-all.bat | 2 +- tests/system-test/test.py | 46 +++++++++++++--- tools/shell/src/shellEngine.c | 1 + 9 files changed, 128 insertions(+), 27 deletions(-) create mode 100644 tests/script/test-all.bat diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index be3454f78f..a38b14a52d 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -508,7 +508,6 @@ class TDDnode: def stoptaosd(self): if (not self.remoteIP == ""): - print("123") self.remoteExec(self.cfgDict, "tdDnodes.dnodes[%d].running=1\ntdDnodes.dnodes[%d].stop()"%(self.index-1,self.index-1)) tdLog.info("stop dnode%d"%self.index) return @@ -518,18 +517,21 @@ class TDDnode: toBeKilled = "valgrind.bin" if self.running != 0: - psCmd = "ps -ef|grep -w %s| grep dnode%d|grep -v grep | awk '{print $2}'" % (toBeKilled,self.index) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - while(processID): - killCmd = "kill -INT %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) + if platform.system().lower() == 'windows': + os.system("wmic process where \"name='taosd.exe' and CommandLine like '%%dnode%d%%'\" get processId | xargs echo | awk '{print $2}' | xargs taskkill -f -pid"%self.index) + else: + psCmd = "ps -ef|grep -w %s| grep dnode%d|grep -v grep | awk '{print $2}'" % (toBeKilled,self.index) processID = subprocess.check_output( psCmd, shell=True).decode("utf-8") - if self.valgrind: - time.sleep(2) + + while(processID): + killCmd = "kill -INT %s > /dev/null 2>&1" % processID + os.system(killCmd) + time.sleep(1) + processID = subprocess.check_output( + psCmd, shell=True).decode("utf-8") + if self.valgrind: + time.sleep(2) self.running = 0 tdLog.debug("dnode:%d is stopped by kill -INT" % (self.index)) diff --git a/tests/script/test-all.bat b/tests/script/test-all.bat new file mode 100644 index 0000000000..7a1a4bc7fa --- /dev/null +++ b/tests/script/test-all.bat @@ -0,0 +1,58 @@ +@echo off +SETLOCAL EnableDelayedExpansion +for /F "tokens=1,2 delims=#" %%a in ('"prompt #$H#$E# & echo on & for %%b in (1) do rem"') do ( set "DEL=%%a") +set /a a=0 +echo Windows Taosd Full Test +set /a exitNum=0 +rm -rf failed.txt +set caseFile="jenkins\\basic.txt" +if not "%2" == "" ( + set caseFile="%2" +) +for /F "usebackq tokens=*" %%i in (!caseFile!) do ( + set line=%%i + if "!line:~,9!" == "./test.sh" ( + set /a a+=1 + echo !a! Processing %%i + call :GetTimeSeconds !time! + set time1=!_timeTemp! + echo Start at !time! + call !line:./test.sh=wtest.bat! > result_!a!.txt 2>error_!a!.txt + if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && set /a exitNum=8 && echo %%i >>failed.txt ) else ( call :colorEcho 0a "Success" &echo. ) + ) +) +exit !exitNum! + +:colorEcho +set timeNow=%time% +call :GetTimeSeconds %timeNow% +set time2=%_timeTemp% +set /a interTime=%time2% - %time1% +echo End at %timeNow% , cast %interTime%s +echo off + "%~2" +findstr /v /a:%1 /R "^$" "%~2" nul +del "%~2" > nul 2>&1i +goto :eof + +:GetTimeSeconds +set tt=%1 +set tt=%tt:.= % +set tt=%tt::= % +set tt=%tt: 0= % +set /a index=1 +for %%a in (%tt%) do ( + if !index! EQU 1 ( + set /a hh=%%a + )^ + else if !index! EQU 2 ( + set /a mm=%%a + + )^ + else if !index! EQU 3 ( + set /a ss=%%a + ) + set /a index=index+1 +) +set /a _timeTemp=(%hh%*60+%mm%)*60+%ss% +goto :eof diff --git a/tests/system-test/6-cluster/5dnode1mnode.py b/tests/system-test/6-cluster/5dnode1mnode.py index 75134224db..5f4ab7357b 100644 --- a/tests/system-test/6-cluster/5dnode1mnode.py +++ b/tests/system-test/6-cluster/5dnode1mnode.py @@ -20,7 +20,7 @@ class MyDnodes(TDDnodes): self.simDeployed = False class TDTestCase: - + noConn = True def init(self,conn ,logSql): tdLog.debug(f"start to excute {__file__}") self.TDDnodes = None @@ -40,7 +40,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] @@ -81,7 +81,7 @@ class TDTestCase: dnode_id = dnode.cfgDict["fqdn"] + ":" +dnode.cfgDict["serverPort"] dnode_first_host = dnode.cfgDict["firstEp"].split(":")[0] dnode_first_port = dnode.cfgDict["firstEp"].split(":")[-1] - cmd = f" taos -h {dnode_first_host} -P {dnode_first_port} -s ' create dnode \"{dnode_id} \" ' ;" + cmd = f"{self.getBuildPath()}/build/bin/taos -h {dnode_first_host} -P {dnode_first_port} -s \"create dnode \\\"{dnode_id}\\\"\"" print(cmd) os.system(cmd) diff --git a/tests/system-test/6-cluster/5dnode2mnode.py b/tests/system-test/6-cluster/5dnode2mnode.py index d3cde987c6..e08e738be6 100644 --- a/tests/system-test/6-cluster/5dnode2mnode.py +++ b/tests/system-test/6-cluster/5dnode2mnode.py @@ -20,6 +20,7 @@ class MyDnodes(TDDnodes): self.simDeployed = False class TDTestCase: + noConn = True def init(self,conn ,logSql): tdLog.debug(f"start to excute {__file__}") @@ -40,7 +41,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] @@ -85,7 +86,7 @@ class TDTestCase: dnode_id = dnode.cfgDict["fqdn"] + ":" +dnode.cfgDict["serverPort"] dnode_first_host = dnode.cfgDict["firstEp"].split(":")[0] dnode_first_port = dnode.cfgDict["firstEp"].split(":")[-1] - cmd = f" taos -h {dnode_first_host} -P {dnode_first_port} -s ' create dnode \"{dnode_id} \" ' ;" + cmd = f"{self.getBuildPath()}/build/bin/taos -h {dnode_first_host} -P {dnode_first_port} -s \"create dnode \\\"{dnode_id}\\\"\"" print(cmd) os.system(cmd) diff --git a/tests/system-test/7-tmq/tmqCommon.py b/tests/system-test/7-tmq/tmqCommon.py index b8aa78e3ac..788ae3474c 100644 --- a/tests/system-test/7-tmq/tmqCommon.py +++ b/tests/system-test/7-tmq/tmqCommon.py @@ -86,7 +86,13 @@ class TMQCom: shellCmd += '--tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all --num-callers=20 -v --workaround-gcc296-bugs=yes ' if (platform.system().lower() == 'windows'): - shellCmd = 'mintty -h never -w hide ' + buildPath + '\\build\\bin\\tmq_sim.exe -c ' + cfgPath + processorName = buildPath + '\\build\\bin\\tmq_sim.exe' + if alias != 0: + processorNameNew = buildPath + '\\build\\bin\\tmq_sim_new.exe' + shellCmd = 'cp %s %s'%(processorName, processorNameNew) + os.system(shellCmd) + processorName = processorNameNew + shellCmd = 'mintty -h never ' + processorName + ' -c ' + cfgPath shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) shellCmd += "> nul 2>&1 &" else: diff --git a/tests/system-test/7-tmq/tmqError.py b/tests/system-test/7-tmq/tmqError.py index 5b5658d528..bd8ec565d8 100644 --- a/tests/system-test/7-tmq/tmqError.py +++ b/tests/system-test/7-tmq/tmqError.py @@ -288,7 +288,10 @@ class TDTestCase: tdLog.exit("tmq consume rows error!") tdSql.query("drop topic %s"%topicFromStb1) - os.system('pkill tmq_sim') + if (platform.system().lower() == 'windows'): + os.system("TASKKILL /F /IM tmq_sim.exe") + else: + os.system('pkill tmq_sim') tdLog.printNoPrefix("======== test case 1 end ...... ") diff --git a/tests/system-test/test-all.bat b/tests/system-test/test-all.bat index 275cbeebbb..adc9e0ce28 100644 --- a/tests/system-test/test-all.bat +++ b/tests/system-test/test-all.bat @@ -2,7 +2,7 @@ SETLOCAL EnableDelayedExpansion for /F "tokens=1,2 delims=#" %%a in ('"prompt #$H#$E# & echo on & for %%b in (1) do rem"') do ( set "DEL=%%a") set /a a=0 -if %1 == full ( +if "%1" == "full" ( echo Windows Taosd Full Test set /a exitNum=0 del /Q /F failed.txt diff --git a/tests/system-test/test.py b/tests/system-test/test.py index 8a8356449c..35f8ea953c 100644 --- a/tests/system-test/test.py +++ b/tests/system-test/test.py @@ -23,6 +23,7 @@ import platform import socket import threading from distutils.log import warn as printf +from tkinter import N from fabric2 import Connection sys.path.append("../pytest") from util.log import * @@ -187,9 +188,9 @@ if __name__ == "__main__": tdLog.info("Procedures for tdengine deployed in %s" % (host)) if platform.system().lower() == 'windows': + fileName = fileName.replace("/", os.sep) if (masterIp == "" and not fileName[0:12] == "0-others\\udf"): threading.Thread(target=checkRunTimeError,daemon=True).start() - tdCases.logSql(logSql) tdLog.info("Procedures for testing self-deployment") tdDnodes.init(deployPath, masterIp) tdDnodes.setTestCluster(testCluster) @@ -208,18 +209,46 @@ if __name__ == "__main__": uModule = importlib.import_module(moduleName) try: ucase = uModule.TDTestCase() - if ((json.dumps(updateCfgDict) == '{}') and (ucase.updatecfgDict is not None)): + if ((json.dumps(updateCfgDict) == '{}') and hasattr(ucase, 'updatecfgDict')): updateCfgDict = ucase.updatecfgDict updateCfgDictStr = "-d %s"%base64.b64encode(json.dumps(updateCfgDict).encode()).decode() except Exception as r: print(r) else: pass - tdDnodes.deploy(1,updateCfgDict) - tdDnodes.start(1) - conn = taos.connect( - host="%s"%(host), - config=tdDnodes.sim.getCfgDir()) + if dnodeNums == 1 : + tdDnodes.deploy(1,updateCfgDict) + tdDnodes.start(1) + tdCases.logSql(logSql) + else : + tdLog.debug("create an cluster with %s nodes and make %s dnode as independent mnode"%(dnodeNums,mnodeNums)) + dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums,mnodeNums=mnodeNums) + tdDnodes = ClusterDnodes(dnodeslist) + tdDnodes.init(deployPath, masterIp) + tdDnodes.setTestCluster(testCluster) + tdDnodes.setValgrind(valgrind) + tdDnodes.stopAll() + for dnode in tdDnodes.dnodes: + tdDnodes.deploy(dnode.index,{}) + for dnode in tdDnodes.dnodes: + tdDnodes.starttaosd(dnode.index) + tdCases.logSql(logSql) + conn = taos.connect( + host, + config=tdDnodes.getSimCfgPath()) + print(tdDnodes.getSimCfgPath(),host) + cluster.create_dnode(conn) + try: + if cluster.check_dnode(conn) : + print("check dnode ready") + except Exception as r: + print(r) + if ucase is not None and hasattr(ucase, 'noConn') and ucase.noConn == True: + conn = None + else: + conn = taos.connect( + host="%s"%(host), + config=tdDnodes.sim.getCfgDir()) if is_test_framework: tdCases.runOneWindows(conn, fileName) else: @@ -307,4 +336,5 @@ if __name__ == "__main__": tdCases.runOneLinux(conn, sp[0] + "_" + "restart.py") else: tdLog.info("not need to query") - conn.close() + if conn is not None: + conn.close() diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 8a017d378d..8bc99a2665 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -156,6 +156,7 @@ void shellRunSingleCommandImp(char *command) { } fname = sptr + 2; + while (*fname == ' ') fname++; *sptr = '\0'; } From 77b365f0ccd6cf68c58e24d2e8afecc4c6a63678 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Sat, 25 Jun 2022 16:27:05 +0800 Subject: [PATCH 48/49] refactor(sync): do not replicate when one replica --- source/libs/sync/src/syncMain.c | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index c5af72c971..3100d0525c 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -914,6 +914,9 @@ void syncNodeStart(SSyncNode* pSyncNode) { syncNodeBecomeLeader(pSyncNode, "one replica start"); // Raft 3.6.2 Committing entries from previous terms + syncNodeAppendNoop(pSyncNode); + syncMaybeAdvanceCommitIndex(pSyncNode); + return; } @@ -1662,6 +1665,12 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde // change isStandBy to normal (election timeout) if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { syncNodeBecomeLeader(pSyncNode, tmpbuf); + + // Raft 3.6.2 Committing entries from previous terms + syncNodeReplicate(pSyncNode); + syncNodeAppendNoop(pSyncNode); + syncMaybeAdvanceCommitIndex(pSyncNode); + } else { syncNodeBecomeFollower(pSyncNode, tmpbuf); } @@ -1807,16 +1816,9 @@ void syncNodeBecomeLeader(SSyncNode* pSyncNode, const char* debugStr) { // stop elect timer syncNodeStopElectTimer(pSyncNode); - // start replicate right now! - syncNodeReplicate(pSyncNode); - // start heartbeat timer syncNodeStartHeartbeatTimer(pSyncNode); - // append noop - syncNodeAppendNoop(pSyncNode); - syncMaybeAdvanceCommitIndex(pSyncNode); // maybe only one replica - // trace log do { int32_t debugStrLen = strlen(debugStr); @@ -1841,9 +1843,9 @@ void syncNodeCandidate2Leader(SSyncNode* pSyncNode) { syncNodeLog2("==state change syncNodeCandidate2Leader==", pSyncNode); // Raft 3.6.2 Committing entries from previous terms - - // do not use this - // syncNodeEqNoop(pSyncNode); + syncNodeReplicate(pSyncNode); + syncNodeAppendNoop(pSyncNode); + syncMaybeAdvanceCommitIndex(pSyncNode); } void syncNodeFollower2Candidate(SSyncNode* pSyncNode) { From 10e90ce973830815b616e0ba28b626766a1d4e96 Mon Sep 17 00:00:00 2001 From: 54liuyao <54liuyao@163.com> Date: Sat, 25 Jun 2022 16:12:00 +0800 Subject: [PATCH 49/49] feat(stream): auto pull data --- include/common/tcommon.h | 3 +- include/common/tdatablock.h | 3 + include/libs/stream/tstream.h | 2 +- include/libs/stream/tstreamUpdate.h | 3 + source/common/src/tdatablock.c | 78 +++- source/dnode/snode/src/snode.c | 2 +- source/dnode/vnode/src/tq/tqRead.c | 6 +- source/libs/executor/inc/executorimpl.h | 11 +- source/libs/executor/src/scanoperator.c | 59 ++- source/libs/executor/src/timewindowoperator.c | 382 ++++++++++++++---- source/libs/stream/src/streamDispatch.c | 3 +- source/libs/stream/src/streamExec.c | 15 +- source/libs/stream/src/streamUpdate.c | 20 + .../tsim/stream/distributeInterval0.sim | 8 +- tests/script/tsim/stream/partitionby.sim | 40 +- tests/script/tsim/stream/schedSnode.sim | 6 +- 16 files changed, 519 insertions(+), 122 deletions(-) diff --git a/include/common/tcommon.h b/include/common/tcommon.h index a14f7eff8a..928fe0aa0e 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -42,12 +42,13 @@ enum { typedef enum EStreamType { STREAM_NORMAL = 1, STREAM_INVERT, - STREAM_REPROCESS, + STREAM_CLEAR, STREAM_INVALID, STREAM_GET_ALL, STREAM_DELETE, STREAM_RETRIEVE, STREAM_PUSH_DATA, + STREAM_PUSH_EMPTY, } EStreamType; typedef struct { diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 2a0d4e7ff6..2e2c7d1700 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -224,6 +224,7 @@ size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize); int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n); int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src); +int32_t copyDataBlock(SSDataBlock* dst, const SSDataBlock* src); SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData); SSDataBlock* createDataBlock(); int32_t blockDataAppendColInfo(SSDataBlock* pBlock, SColumnInfoData* pColInfoData); @@ -236,6 +237,8 @@ void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData); void blockDebugShowData(const SArray* dataBlocks, const char* flag); +// for debug +char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** dumpBuf); int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, STSchema* pTSchema, int32_t vgId, tb_uid_t suid); diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 594344ba8a..db928f194c 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -319,7 +319,7 @@ static FORCE_INLINE int32_t streamTaskInput(SStreamTask* pTask, SStreamQueueItem return -1; } taosWriteQitem(pTask->inputQueue->queue, pSubmitClone); - } else if (pItem->type == STREAM_INPUT__DATA_BLOCK) { + } else if (pItem->type == STREAM_INPUT__DATA_BLOCK || pItem->type == STREAM_INPUT__DATA_RETRIEVE) { taosWriteQitem(pTask->inputQueue->queue, pItem); } else if (pItem->type == STREAM_INPUT__CHECKPOINT) { taosWriteQitem(pTask->inputQueue->queue, pItem); diff --git a/include/libs/stream/tstreamUpdate.h b/include/libs/stream/tstreamUpdate.h index 398851a09f..21a1515d8f 100644 --- a/include/libs/stream/tstreamUpdate.h +++ b/include/libs/stream/tstreamUpdate.h @@ -32,12 +32,15 @@ typedef struct SUpdateInfo { int64_t interval; int64_t watermark; TSKEY minTS; + SScalableBf* pCloseWinSBF; } SUpdateInfo; SUpdateInfo *updateInfoInitP(SInterval* pInterval, int64_t watermark); SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t watermark); bool updateInfoIsUpdated(SUpdateInfo *pInfo, tb_uid_t tableId, TSKEY ts); void updateInfoDestroy(SUpdateInfo *pInfo); +void updateInfoAddCloseWindowSBF(SUpdateInfo *pInfo); +void updateInfoDestoryColseWinSBF(SUpdateInfo *pInfo); #ifdef __cplusplus } diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 593f8c5c0b..cc995c4d64 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1164,7 +1164,7 @@ int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows) int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) { int32_t code = 0; - ASSERT(numOfRows > 0); + //ASSERT(numOfRows > 0); if (numOfRows == 0) { return TSDB_CODE_SUCCESS; @@ -1230,6 +1230,32 @@ int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src) { return 0; } +int32_t copyDataBlock(SSDataBlock* dst, const SSDataBlock* src) { + ASSERT(src != NULL && dst != NULL); + + blockDataCleanup(dst); + int32_t code = blockDataEnsureCapacity(dst, src->info.rows); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + return code; + } + + size_t numOfCols = taosArrayGetSize(src->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pDst = taosArrayGet(dst->pDataBlock, i); + SColumnInfoData* pSrc = taosArrayGet(src->pDataBlock, i); + if (pSrc->pData == NULL) { + continue; + } + + colDataAssign(pDst, pSrc, src->info.rows, &src->info); + } + + dst->info.rows = src->info.rows; + dst->info.window = src->info.window; + return TSDB_CODE_SUCCESS; +} + SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) { if (pDataBlock == NULL) { return NULL; @@ -1627,6 +1653,56 @@ void blockDebugShowData(const SArray* dataBlocks, const char* flag) { } } +// for debug +char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) { + int32_t size = 2048; + *pDataBuf = taosMemoryCalloc(size, 1); + char* dumpBuf = *pDataBuf; + char pBuf[128] = {0}; + int32_t colNum = taosArrayGetSize(pDataBlock->pDataBlock); + int32_t rows = pDataBlock->info.rows; + int32_t len = 0; + len += snprintf(dumpBuf + len, size - len, "\n%s |block type %d |child id %d|\n", flag, (int32_t)pDataBlock->info.type, pDataBlock->info.childId); + for (int32_t j = 0; j < rows; j++) { + len += snprintf(dumpBuf + len, size - len, "%s |", flag); + for (int32_t k = 0; k < colNum; k++) { + SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); + void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); + if (colDataIsNull(pColInfoData, rows, j, NULL)) { + len += snprintf(dumpBuf + len, size - len, " %15s |", "NULL"); + continue; + } + switch (pColInfoData->info.type) { + case TSDB_DATA_TYPE_TIMESTAMP: + formatTimestamp(pBuf, *(uint64_t*)var, TSDB_TIME_PRECISION_MILLI); + len += snprintf(dumpBuf + len, size - len, " %25s |", pBuf); + break; + case TSDB_DATA_TYPE_INT: + len += snprintf(dumpBuf + len, size - len, " %15d |", *(int32_t*)var); + break; + case TSDB_DATA_TYPE_UINT: + len += snprintf(dumpBuf + len, size - len, " %15u |", *(uint32_t*)var); + break; + case TSDB_DATA_TYPE_BIGINT: + len += snprintf(dumpBuf + len, size - len, " %15ld |", *(int64_t*)var); + break; + case TSDB_DATA_TYPE_UBIGINT: + len += snprintf(dumpBuf + len, size - len, " %15lu |", *(uint64_t*)var); + break; + case TSDB_DATA_TYPE_FLOAT: + len += snprintf(dumpBuf + len, size - len, " %15f |", *(float*)var); + break; + case TSDB_DATA_TYPE_DOUBLE: + len += snprintf(dumpBuf + len, size - len, " %15lf |", *(double*)var); + break; + } + } + len += snprintf(dumpBuf + len, size - len, "\n"); + } + len += snprintf(dumpBuf + len, size - len, "%s |end\n", flag); + return dumpBuf; +} + /** * @brief TODO: Assume that the final generated result it less than 3M * diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 3a92cba773..b13e654caf 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -257,7 +257,7 @@ int32_t sndProcessSMsg(SSnode *pSnode, SRpcMsg *pMsg) { case TDMT_STREAM_TASK_RECOVER_RSP: return sndProcessTaskRecoverRsp(pSnode, pMsg); case TDMT_STREAM_RETRIEVE_RSP: - return sndProcessTaskRecoverRsp(pSnode, pMsg); + return sndProcessTaskRetrieveRsp(pSnode, pMsg); default: ASSERT(0); } diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 6184d8f810..96f4eb3fd9 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -109,11 +109,15 @@ int32_t tqReadHandleSetMsg(STqReadHandle* pReadHandle, SSubmitReq* pMsg, int64_t } bool tqNextDataBlock(STqReadHandle* pHandle) { + if (pHandle->pMsg == NULL) return false; while (1) { if (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) < 0) { return false; } - if (pHandle->pBlock == NULL) return false; + if (pHandle->pBlock == NULL) { + pHandle->pMsg = NULL; + return false; + } if (pHandle->tbIdHash == NULL) { return true; diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index d8d231e952..36f81e86ff 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -293,6 +293,7 @@ typedef enum EStreamScanMode { STREAM_SCAN_FROM_RES, STREAM_SCAN_FROM_UPDATERES, STREAM_SCAN_FROM_DATAREADER, + STREAM_SCAN_FROM_DATAREADER_RETRIEVE, } EStreamScanMode; typedef struct SCatchSupporter { @@ -348,7 +349,9 @@ typedef struct SStreamBlockScanInfo { SArray* childIds; SessionWindowSupporter sessionSup; bool assignBlockUid; // assign block uid to groupId, temporarily used for generating rollup SMA. - int32_t scanWinIndex; + int32_t scanWinIndex; // for state operator + int32_t pullDataResIndex; + SSDataBlock* pPullDataRes; // pull data SSDataBlock } SStreamBlockScanInfo; typedef struct SSysTableScanInfo { @@ -427,8 +430,13 @@ typedef struct SStreamFinalIntervalOperatorInfo { STimeWindowAggSupp twAggSup; SArray* pChildren; SSDataBlock* pUpdateRes; + bool returnUpdate; SPhysiNode* pPhyNode; // create new child bool isFinal; + SHashObj* pPullDataMap; + SArray* pPullWins; // SPullWindowInfo + int32_t pullIndex; + SSDataBlock* pPullDataRes; } SStreamFinalIntervalOperatorInfo; typedef struct SAggOperatorInfo { @@ -851,6 +859,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex); int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* groupKey); +SSDataBlock* createPullDataBlock(); #ifdef __cplusplus } diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 7affac76d2..edf98deed4 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -776,15 +776,14 @@ static bool isStateWindow(SStreamBlockScanInfo* pInfo) { return pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE; } -static bool prepareDataScan(SStreamBlockScanInfo* pInfo) { - SSDataBlock* pSDB = pInfo->pUpdateRes; +static bool prepareDataScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pSDB, int32_t tsColIndex, int32_t* pRowIndex) { STimeWindow win = { .skey = INT64_MIN, .ekey = INT64_MAX, }; bool needRead = false; - if (!isStateWindow(pInfo) && pInfo->updateResIndex < pSDB->info.rows) { - SColumnInfoData* pColDataInfo = taosArrayGet(pSDB->pDataBlock, pInfo->primaryTsIndex); + if (!isStateWindow(pInfo) && (*pRowIndex) < pSDB->info.rows) { + SColumnInfoData* pColDataInfo = taosArrayGet(pSDB->pDataBlock, tsColIndex); TSKEY* tsCols = (TSKEY*)pColDataInfo->pData; SResultRowInfo dumyInfo; dumyInfo.cur.pageId = -1; @@ -793,14 +792,14 @@ static bool prepareDataScan(SStreamBlockScanInfo* pInfo) { int64_t gap = pInfo->sessionSup.gap; int32_t winIndex = 0; SResultWindowInfo* pCurWin = - getSessionTimeWindow(pAggSup, tsCols[pInfo->updateResIndex], INT64_MIN, pSDB->info.groupId, gap, &winIndex); + getSessionTimeWindow(pAggSup, tsCols[(*pRowIndex)], INT64_MIN, pSDB->info.groupId, gap, &winIndex); win = pCurWin->win; - pInfo->updateResIndex += - updateSessionWindowInfo(pCurWin, tsCols, NULL, pSDB->info.rows, pInfo->updateResIndex, gap, NULL); + (*pRowIndex) += + updateSessionWindowInfo(pCurWin, tsCols, NULL, pSDB->info.rows, (*pRowIndex), gap, NULL); } else { - win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[pInfo->updateResIndex], &pInfo->interval, + win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[(*pRowIndex)], &pInfo->interval, pInfo->interval.precision, NULL); - pInfo->updateResIndex += getNumOfRowsInTimeWindow(&pSDB->info, tsCols, pInfo->updateResIndex, win.ekey, + (*pRowIndex) += getNumOfRowsInTimeWindow(&pSDB->info, tsCols, (*pRowIndex), win.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); } needRead = true; @@ -823,6 +822,9 @@ static bool prepareDataScan(SStreamBlockScanInfo* pInfo) { pTableScanInfo->cond.twindows[0] = win; pTableScanInfo->curTWinIdx = 0; // tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0); + // if (!pTableScanInfo->dataReader) { + // return false; + // } pTableScanInfo->scanTimes = 0; pTableScanInfo->currentGroupId = -1; return true; @@ -862,12 +864,12 @@ static uint64_t getGroupId(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_ */ } -static SSDataBlock* doDataScan(SStreamBlockScanInfo* pInfo) { +static SSDataBlock* doDataScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pSDB, int32_t tsColIndex, int32_t* pRowIndex) { while (1) { SSDataBlock* pResult = NULL; pResult = doTableScan(pInfo->pSnapshotReadOp); if (pResult == NULL) { - if (prepareDataScan(pInfo)) { + if (prepareDataScan(pInfo, pSDB, tsColIndex, pRowIndex)) { // scan next window data pResult = doTableScan(pInfo->pSnapshotReadOp); } @@ -916,7 +918,7 @@ static void setUpdateData(SStreamBlockScanInfo* pInfo, SSDataBlock* pBlock, SSDa pUpdateBlock->info.rows = i; pInfo->tsArrayIndex += i; pUpdateBlock->info.groupId = pInfo->groupId; - pUpdateBlock->info.type = STREAM_REPROCESS; + pUpdateBlock->info.type = STREAM_CLEAR; blockDataUpdateTsWindow(pUpdateBlock, 0); } // all rows have same group id @@ -970,6 +972,14 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { int32_t current = pInfo->validBlockIndex++; SSDataBlock* pBlock = taosArrayGetP(pInfo->pBlockLists, current); blockDataUpdateTsWindow(pBlock, 0); + if (pBlock->info.type == STREAM_RETRIEVE) { + pInfo->blockType = STREAM_DATA_TYPE_SUBMIT_BLOCK; + pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RETRIEVE; + copyDataBlock(pInfo->pPullDataRes, pBlock); + pInfo->pullDataResIndex = 0; + prepareDataScan(pInfo, pInfo->pPullDataRes, 0, &pInfo->pullDataResIndex); + updateInfoAddCloseWindowSBF(pInfo->pUpdateInfo); + } return pBlock; } else if (pInfo->blockType == STREAM_DATA_TYPE_SUBMIT_BLOCK) { if (pInfo->scanMode == STREAM_SCAN_FROM_RES) { @@ -979,28 +989,39 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { } else if (pInfo->scanMode == STREAM_SCAN_FROM_UPDATERES) { pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER; if (!isStateWindow(pInfo)) { - prepareDataScan(pInfo); + prepareDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex); } return pInfo->pUpdateRes; + } else if (pInfo->scanMode == STREAM_SCAN_FROM_DATAREADER_RETRIEVE) { + SSDataBlock* pSDB = doDataScan(pInfo, pInfo->pPullDataRes, 0, &pInfo->pullDataResIndex); + if (pSDB != NULL) { + getUpdateDataBlock(pInfo, true, pSDB, NULL); + pSDB->info.type = STREAM_PUSH_DATA; + return pSDB; + } + pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER; } else { if (isStateWindow(pInfo) && taosArrayGetSize(pInfo->sessionSup.pStreamAggSup->pScanWindow) > 0) { pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER; pInfo->updateResIndex = pInfo->pUpdateRes->info.rows; - prepareDataScan(pInfo); + prepareDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex); } if (pInfo->scanMode == STREAM_SCAN_FROM_DATAREADER) { - SSDataBlock* pSDB = doDataScan(pInfo); + SSDataBlock* pSDB = doDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex); if (pSDB == NULL) { setUpdateData(pInfo, pInfo->pRes, pInfo->pUpdateRes); if (pInfo->pUpdateRes->info.rows > 0) { if (!isStateWindow(pInfo)) { - prepareDataScan(pInfo); + // Todo(liuyao) mybe can delete this. + bool test = prepareDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex); + ASSERT(test == false); } return pInfo->pUpdateRes; } else { pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; } } else { + pSDB->info.type = STREAM_NORMAL; getUpdateDataBlock(pInfo, true, pSDB, NULL); return pSDB; } @@ -1070,10 +1091,12 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { taosArrayDestroy(block.pDataBlock); if (pInfo->pRes->pDataBlock == NULL) { // TODO add log + updateInfoDestoryColseWinSBF(pInfo->pUpdateInfo); pOperator->status = OP_EXEC_DONE; pTaskInfo->code = terrno; return NULL; } + // currently only the tbname pseudo column if (pInfo->numOfPseudoExpr > 0) { addTagPseudoColumnData(&pInfo->readHandle, pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, pInfo->pRes); @@ -1091,12 +1114,13 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { pOperator->resultInfo.totalRows += pBlockInfo->rows; if (pBlockInfo->rows == 0) { + updateInfoDestoryColseWinSBF(pInfo->pUpdateInfo); pOperator->status = OP_EXEC_DONE; } else if (pInfo->pUpdateInfo) { pInfo->tsArrayIndex = 0; getUpdateDataBlock(pInfo, true, pInfo->pRes, pInfo->pUpdateRes); if (pInfo->pUpdateRes->info.rows > 0) { - if (pInfo->pUpdateRes->info.type == STREAM_REPROCESS) { + if (pInfo->pUpdateRes->info.type == STREAM_CLEAR) { pInfo->updateResIndex = 0; pInfo->scanMode = STREAM_SCAN_FROM_UPDATERES; } else if (pInfo->pUpdateRes->info.type == STREAM_INVERT) { @@ -1209,6 +1233,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; pInfo->sessionSup = (SessionWindowSupporter){.pStreamAggSup = NULL, .gap = -1}; pInfo->groupId = 0; + pInfo->pPullDataRes = createPullDataBlock(); pOperator->name = "StreamBlockScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN; diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 0ba898e4f7..b7d39207d8 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -15,6 +15,7 @@ #include "executorimpl.h" #include "function.h" #include "functionMgt.h" +#include "tcompare.h" #include "tdatablock.h" #include "tfill.h" #include "ttime.h" @@ -26,6 +27,16 @@ typedef enum SResultTsInterpType { #define IS_FINAL_OP(op) ((op)->isFinal) +typedef struct SWinRes { + TSKEY ts; + uint64_t groupId; +} SWinRes; + +typedef struct SPullWindowInfo { + STimeWindow window; + uint64_t groupId; +} SPullWindowInfo; + static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator); static int64_t* extractTsCol(SSDataBlock* pBlock, const SIntervalAggOperatorInfo* pInfo); @@ -684,11 +695,13 @@ static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t num } void printDataBlock(SSDataBlock* pBlock, const char* flag) { - if (pBlock == NULL) return; - SArray* blocks = taosArrayInit(1, sizeof(SSDataBlock)); - taosArrayPush(blocks, pBlock); - blockDebugShowData(blocks, flag); - taosArrayDestroy(blocks); + if (pBlock == NULL){ + qDebug("======printDataBlock Block is Null"); + return; + } + char *pBuf = NULL; + qDebug("%s", dumpBlockData(pBlock, flag, &pBuf)); + taosMemoryFree(pBuf); } typedef int64_t (*__get_value_fn_t)(void* data, int32_t index); @@ -1217,30 +1230,40 @@ void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf, SExprS } } -void doClearWindow(SAggSupporter* pAggSup, SExprSupp* pSup, char* pData, int16_t bytes, uint64_t groupId, +bool doClearWindow(SAggSupporter* pAggSup, SExprSupp* pSup, char* pData, int16_t bytes, uint64_t groupId, int32_t numOfOutput) { SET_RES_WINDOW_KEY(pAggSup->keyBuf, pData, bytes, groupId); SResultRowPosition* p1 = (SResultRowPosition*)taosHashGet(pAggSup->pResultRowHashTable, pAggSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); if (!p1) { // window has been closed - return; + return false; } doClearWindowImpl(p1, pAggSup->pResultBuf, pSup, numOfOutput); + return true; } static void doClearWindows(SAggSupporter* pAggSup, SExprSupp* pSup1, SInterval* pInterval, int32_t tsIndex, int32_t numOfOutput, SSDataBlock* pBlock, SArray* pUpWins) { - SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, tsIndex); - TSKEY* tsCols = (TSKEY*)pColDataInfo->pData; + SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, tsIndex); + TSKEY* tsCols = (TSKEY*)pTsCol->pData; + uint64_t* pGpDatas = NULL; + if (pBlock->info.type == STREAM_RETRIEVE) { + SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, 2); + pGpDatas = (uint64_t*)pGpCol->pData; + } int32_t step = 0; for (int32_t i = 0; i < pBlock->info.rows; i += step) { SResultRowInfo dumyInfo; dumyInfo.cur.pageId = -1; STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[i], pInterval, pInterval->precision, NULL); step = getNumOfRowsInTimeWindow(&pBlock->info, tsCols, i, win.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); - doClearWindow(pAggSup, pSup1, (char*)&win.skey, sizeof(TKEY), pBlock->info.groupId, numOfOutput); - if (pUpWins) { + uint64_t groupId = pBlock->info.groupId; + if (pGpDatas) { + groupId = pGpDatas[i]; + } + bool res = doClearWindow(pAggSup, pSup1, (char*)&win.skey, sizeof(TKEY), groupId, numOfOutput); + if (pUpWins && res) { taosArrayPush(pUpWins, &win); } } @@ -1268,8 +1291,8 @@ bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup) { return pSup->maxTs != INT64_MIN && pWin->ekey < pSup->maxTs - pSup->waterMark; } -static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, SInterval* pInterval, - SArray* closeWins) { +static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, + SInterval* pInterval, SHashObj* pPullDataMap, SArray* closeWins) { void* pIte = NULL; size_t keyLen = 0; while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) { @@ -1280,10 +1303,20 @@ static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, SResultRowInfo dumyInfo; dumyInfo.cur.pageId = -1; STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, ts, pInterval, pInterval->precision, NULL); + SWinRes winRe = {.ts = win.skey, .groupId = groupId,}; + void* chIds = taosHashGet(pPullDataMap, &winRe, sizeof(SWinRes)); if (isCloseWindow(&win, pSup)) { - char keyBuf[GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY))]; - SET_RES_WINDOW_KEY(keyBuf, &ts, sizeof(TSKEY), groupId); - taosHashRemove(pHashMap, keyBuf, keyLen); + if (chIds && pPullDataMap) { + SArray* chAy = *(SArray**) chIds; + int32_t size = taosArrayGetSize(chAy); + qInfo("======window %ld wait child size:%d", win.skey ,size); + for (int32_t i = 0; i < size; i++) { + qInfo("======window %ld wait chid id:%d", win.skey ,*(int32_t*)taosArrayGet(chAy, i)); + } + continue; + } else if (pPullDataMap) { + qInfo("======close window %ld", win.skey); + } SResultRowPosition* pPos = (SResultRowPosition*)pIte; if (pSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { int32_t code = saveResult(ts, pPos->pageId, pPos->offset, groupId, closeWins); @@ -1291,11 +1324,25 @@ static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, return code; } } + char keyBuf[GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY))]; + SET_RES_WINDOW_KEY(keyBuf, &ts, sizeof(TSKEY), groupId); + taosHashRemove(pHashMap, keyBuf, keyLen); } } return TSDB_CODE_SUCCESS; } +static void closeChildIntervalWindow(SArray* pChildren, TSKEY maxTs) { + int32_t size = taosArrayGetSize(pChildren); + for (int32_t i = 0; i < size; i++) { + SOperatorInfo* pChildOp = taosArrayGetP(pChildren, i); + SStreamFinalIntervalOperatorInfo* pChInfo = pChildOp->info; + ASSERT(pChInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE); + pChInfo->twAggSup.maxTs = TMAX(pChInfo->twAggSup.maxTs, maxTs); + closeIntervalWindow(pChInfo->aggSup.pResultRowHashTable, &pChInfo->twAggSup, &pChInfo->interval, NULL, NULL); + } +} + static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { SIntervalAggOperatorInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -1324,7 +1371,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { break; } - if (pBlock->info.type == STREAM_REPROCESS) { + if (pBlock->info.type == STREAM_CLEAR) { doClearWindows(&pInfo->aggSup, &pOperator->exprSupp, &pInfo->interval, 0, pOperator->exprSupp.numOfExprs, pBlock, NULL); qDebug("%s clear existed time window results for updates checked", GET_TASKID(pTaskInfo)); @@ -1345,7 +1392,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey); hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, MAIN_SCAN, pUpdated); } - closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, pUpdated); + closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, NULL, pUpdated); finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset); initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); @@ -1373,6 +1420,11 @@ void destroyStreamFinalIntervalOperatorInfo(void* param, int32_t numOfOutput) { SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)param; cleanupBasicInfo(&pInfo->binfo); cleanupAggSup(&pInfo->aggSup); + //it should be empty. + taosHashCleanup(pInfo->pPullDataMap); + taosArrayDestroy(pInfo->pPullWins); + blockDataDestroy(pInfo->pPullDataRes); + if (pInfo->pChildren) { int32_t size = taosArrayGetSize(pInfo->pChildren); for (int32_t i = 0; i < size; i++) { @@ -2164,6 +2216,24 @@ bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup) { return p1 == NULL; } +int32_t getNexWindowPos(SInterval* pInterval, SDataBlockInfo* pBlockInfo, TSKEY* tsCols, + int32_t startPos, TSKEY eKey, STimeWindow* pNextWin) { + int32_t forwardRows = getNumOfRowsInTimeWindow(pBlockInfo, tsCols, startPos, + eKey, binarySearchForKey, NULL, TSDB_ORDER_ASC); + int32_t prevEndPos = forwardRows - 1 + startPos; + return getNextQualifiedWindow(pInterval, pNextWin, pBlockInfo, tsCols, prevEndPos, TSDB_ORDER_ASC); +} + +void addPullWindow(SHashObj* pMap, SWinRes* pWinRes, int32_t size) { + SArray* childIds = taosArrayInit(8, sizeof(int32_t)); + for (int32_t i = 0; i < size; i++) { + taosArrayPush(childIds, &i); + } + taosHashPut(pMap, pWinRes, sizeof(SWinRes), &childIds, sizeof(void*)); +} + +static int32_t getChildIndex(SSDataBlock* pBlock) { return pBlock->info.childId; } + static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, uint64_t tableGroupId, SArray* pUpdated) { SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)pOperatorInfo->info; @@ -2177,35 +2247,59 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc SResultRow* pResult = NULL; int32_t forwardRows = 0; - if (pSDataBlock->pDataBlock != NULL) { - SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); - tsCols = (int64_t*)pColDataInfo->pData; - } else { - return; - } + ASSERT(pSDataBlock->pDataBlock != NULL); + SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); + tsCols = (int64_t*)pColDataInfo->pData; int32_t startPos = ascScan ? 0 : (pSDataBlock->info.rows - 1); TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols); STimeWindow nextWin = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, pInfo->interval.precision, NULL); while (1) { - if (IS_FINAL_OP(pInfo) && isCloseWindow(&nextWin, &pInfo->twAggSup) && - isDeletedWindow(&nextWin, tableGroupId, &pInfo->aggSup)) { - SArray* pUpWins = taosArrayInit(8, sizeof(STimeWindow)); - taosArrayPush(pUpWins, &nextWin); - rebuildIntervalWindow(pInfo, pSup, pUpWins, pInfo->binfo.pRes->info.groupId, pSup->numOfExprs, - pOperatorInfo->pTaskInfo); - taosArrayDestroy(pUpWins); + if (IS_FINAL_OP(pInfo) && isCloseWindow(&nextWin, &pInfo->twAggSup) && pInfo->pChildren) { + bool ignore = true; + SWinRes winRes = {.ts = nextWin.skey, .groupId = tableGroupId,}; + void* chIds = taosHashGet(pInfo->pPullDataMap, &winRes, sizeof(SWinRes)); + if (isDeletedWindow(&nextWin, tableGroupId, &pInfo->aggSup) && !chIds) { + SPullWindowInfo pull = {.window = nextWin, .groupId = tableGroupId}; + // add pull data request + taosArrayPush(pInfo->pPullWins, &pull); + addPullWindow(pInfo->pPullDataMap, &winRes, taosArrayGetSize(pInfo->pChildren)); + } else { + int32_t index = -1; + SArray* chArray = NULL; + if (chIds) { + chArray = *(void**) chIds; + int32_t chId = getChildIndex(pSDataBlock); + index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ); + } + if (index != -1 && pSDataBlock->info.type == STREAM_PUSH_DATA) { + taosArrayRemove(chArray, index); + if (taosArrayGetSize(chArray) == 0) { + // pull data is over + taosHashRemove(pInfo->pPullDataMap, &winRes, sizeof(SWinRes)); + } + } + if ( index == -1 || pSDataBlock->info.type == STREAM_PUSH_DATA) { + ignore = false; + } + } + + if (ignore) { + startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin); + if (startPos < 0) { + break; + } + continue; + } } + int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, true, &pResult, tableGroupId, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); if (code != TSDB_CODE_SUCCESS || pResult == NULL) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } - SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t)); - pos->groupId = tableGroupId; - pos->pos = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset}; - *(int64_t*)pos->key = pResult->win.skey; + forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdated) { @@ -2230,14 +2324,17 @@ static void clearStreamIntervalOperator(SStreamFinalIntervalOperatorInfo* pInfo) initResultRowInfo(&pInfo->binfo.resultRowInfo); } -static void clearUpdateDataBlock(SSDataBlock* pBlock) { +static void clearSpecialDataBlock(SSDataBlock* pBlock) { + if (pBlock->info.rows <= 0) { + return; + } blockDataCleanup(pBlock); } void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex) { // ASSERT(pDest->info.capacity >= pSource->info.rows); blockDataEnsureCapacity(pDest, pSource->info.rows); - clearUpdateDataBlock(pDest); + clearSpecialDataBlock(pDest); SColumnInfoData* pDestCol = taosArrayGet(pDest->pDataBlock, 0); SColumnInfoData* pSourceCol = taosArrayGet(pSource->pDataBlock, tsColIndex); @@ -2254,7 +2351,63 @@ void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsCol blockDataUpdateTsWindow(pDest, 0); } -static int32_t getChildIndex(SSDataBlock* pBlock) { return pBlock->info.childId; } +static bool needBreak(SStreamFinalIntervalOperatorInfo* pInfo) { + int32_t size = taosArrayGetSize(pInfo->pPullWins); + if (pInfo->pullIndex < size) { + return true; + } + return false; +} + +static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pBlock) { + clearSpecialDataBlock(pBlock); + int32_t size = taosArrayGetSize(array); + if (size - (*pIndex) == 0) { + return; + } + blockDataEnsureCapacity(pBlock, size - (*pIndex) ); + ASSERT(3 <= taosArrayGetSize(pBlock->pDataBlock)); + for (; (*pIndex) < size; (*pIndex)++) { + SPullWindowInfo* pWin = taosArrayGet(array, (*pIndex) ); + SColumnInfoData* pStartTs = (SColumnInfoData*) taosArrayGet(pBlock->pDataBlock, 0); + colDataAppend(pStartTs, pBlock->info.rows, (const char*)&pWin->window.skey, false); + + SColumnInfoData* pEndTs = (SColumnInfoData*) taosArrayGet(pBlock->pDataBlock, 1); + colDataAppend(pEndTs, pBlock->info.rows, (const char*)&pWin->window.ekey, false); + + SColumnInfoData* pGroupId = (SColumnInfoData*) taosArrayGet(pBlock->pDataBlock, 2); + colDataAppend(pGroupId, pBlock->info.rows, (const char*)&pWin->groupId, false); + pBlock->info.rows++; + } + if ((*pIndex) == size) { + *pIndex = 0; + taosArrayClear(array); + } + blockDataUpdateTsWindow(pBlock, 0); +} + +void processPushEmpty(SSDataBlock* pBlock, SHashObj* pMap) { + SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, 0); + TSKEY* tsData = (TSKEY*)pStartCol->pData; + SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, 2); + uint64_t* groupIdData = (uint64_t*)pGroupCol->pData; + int32_t chId = getChildIndex(pBlock); + for (int32_t i = 0; i < pBlock->info.rows; i++) { + SWinRes winRes = {.ts = tsData[i], .groupId = groupIdData[i]}; + void* chIds = taosHashGet(pMap, &winRes, sizeof(SWinRes)); + if (chIds) { + SArray* chArray = *(SArray**) chIds; + int32_t index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ); + if (index != -1) { + taosArrayRemove(chArray, index); + if (taosArrayGetSize(chArray) == 0) { + // pull data is over + taosHashRemove(pMap, &winRes, sizeof(SWinRes)); + } + } + } + } +} static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { SStreamFinalIntervalOperatorInfo* pInfo = pOperator->info; @@ -2270,28 +2423,50 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); if (pInfo->binfo.pRes->info.rows == 0) { pOperator->status = OP_EXEC_DONE; - if (IS_FINAL_OP(pInfo) || pInfo->pUpdateRes->info.rows == 0) { - if (!IS_FINAL_OP(pInfo)) { - // semi interval operator clear disk buffer - clearStreamIntervalOperator(pInfo); - } - return NULL; + if (!IS_FINAL_OP(pInfo)) { + // semi interval operator clear disk buffer + clearStreamIntervalOperator(pInfo); } + return NULL; + } + printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); + return pInfo->binfo.pRes; + } else { + doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); + if (pInfo->binfo.pRes->info.rows != 0) { + printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); + return pInfo->binfo.pRes; + } + if (pInfo->pUpdateRes->info.rows != 0 && pInfo->returnUpdate) { + pInfo->returnUpdate = false; + ASSERT(!IS_FINAL_OP(pInfo)); + printDataBlock(pInfo->pUpdateRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); // process the rest of the data - pOperator->status = OP_OPENED; return pInfo->pUpdateRes; } - return pInfo->binfo.pRes; + doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes); + if (pInfo->pPullDataRes->info.rows != 0) { + // process the rest of the data + ASSERT(IS_FINAL_OP(pInfo)); + printDataBlock(pInfo->pPullDataRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); + return pInfo->pPullDataRes; + } } while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { - clearUpdateDataBlock(pInfo->pUpdateRes); + clearSpecialDataBlock(pInfo->pUpdateRes); + pOperator->status = OP_RES_TO_RETURN; + qInfo("Stream Final Interval return data"); break; } + printDataBlock(pBlock, IS_FINAL_OP(pInfo) ? "interval Final recv" : "interval Semi recv"); + maxTs = TMAX(maxTs, pBlock->info.window.ekey); - if (pBlock->info.type == STREAM_REPROCESS) { + if (pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_PUSH_DATA || pBlock->info.type == STREAM_INVALID) { + pInfo->binfo.pRes->info.type = pBlock->info.type; + } else if (pBlock->info.type == STREAM_CLEAR) { SArray* pUpWins = taosArrayInit(8, sizeof(STimeWindow)); doClearWindows(&pInfo->aggSup, pSup, &pInfo->interval, pInfo->primaryTsIndex, pOperator->exprSupp.numOfExprs, pBlock, pUpWins); @@ -2310,11 +2485,25 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { } removeResults(pUpWins, pUpdated); copyUpdateDataBlock(pInfo->pUpdateRes, pBlock, pInfo->primaryTsIndex); + pInfo->returnUpdate = true; taosArrayDestroy(pUpWins); break; } else if (pBlock->info.type == STREAM_GET_ALL && IS_FINAL_OP(pInfo)) { getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdated); continue; + } else if (pBlock->info.type == STREAM_RETRIEVE && !IS_FINAL_OP(pInfo)) { + SArray* pUpWins = taosArrayInit(8, sizeof(STimeWindow)); + doClearWindows(&pInfo->aggSup, pSup, &pInfo->interval, 0, pOperator->exprSupp.numOfExprs, + pBlock, pUpWins); + removeResults(pUpWins, pUpdated); + taosArrayDestroy(pUpWins); + if (taosArrayGetSize(pUpdated) > 0) { + break; + } + continue; + } else if (pBlock->info.type == STREAM_PUSH_EMPTY && IS_FINAL_OP(pInfo)) { + processPushEmpty(pBlock, pInfo->pPullDataMap); + continue; } setInputDataBlock(pOperator, pSup->pCtx, pBlock, pInfo->order, MAIN_SCAN, true); @@ -2334,31 +2523,70 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { SStreamFinalIntervalOperatorInfo* pChInfo = pChildOp->info; setInputDataBlock(pChildOp, pChildOp->exprSupp.pCtx, pBlock, pChInfo->order, MAIN_SCAN, true); doHashInterval(pChildOp, pBlock, pBlock->info.groupId, NULL); - pChInfo->twAggSup.maxTs = TMAX(pChInfo->twAggSup.maxTs, pBlock->info.window.ekey); + + if (needBreak(pInfo)) { + break; + } } - maxTs = TMAX(maxTs, pBlock->info.window.ekey); } pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); if (IS_FINAL_OP(pInfo)) { - closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, pUpdated); + closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, + &pInfo->interval, pInfo->pPullDataMap, pUpdated); + closeChildIntervalWindow(pInfo->pChildren, pInfo->twAggSup.maxTs); } finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset); initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - pOperator->status = OP_RES_TO_RETURN; - if (pInfo->binfo.pRes->info.rows == 0) { - pOperator->status = OP_EXEC_DONE; - if (pInfo->pUpdateRes->info.rows == 0) { - return NULL; - } + if (pInfo->binfo.pRes->info.rows != 0) { + printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); + return pInfo->binfo.pRes; + } + + if (pInfo->pUpdateRes->info.rows != 0 && pInfo->returnUpdate) { + pInfo->returnUpdate = false; + ASSERT(!IS_FINAL_OP(pInfo)); + printDataBlock(pInfo->pUpdateRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); // process the rest of the data - pOperator->status = OP_OPENED; return pInfo->pUpdateRes; } - return pInfo->binfo.pRes; + + doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes); + if (pInfo->pPullDataRes->info.rows != 0) { + // process the rest of the data + ASSERT(IS_FINAL_OP(pInfo)); + printDataBlock(pInfo->pPullDataRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); + return pInfo->pPullDataRes; + } + // ASSERT(false); + return NULL; +} + +SSDataBlock* createPullDataBlock() { + SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); + pBlock->info.hasVarCol = false; + pBlock->info.groupId = 0; + pBlock->info.rows = 0; + pBlock->info.type = STREAM_RETRIEVE; + pBlock->info.rowSize = sizeof(TSKEY) + sizeof(TSKEY) + sizeof(uint64_t); + + pBlock->pDataBlock = taosArrayInit(3, sizeof(SColumnInfoData)); + SColumnInfoData infoData = {0}; + infoData.info.type = TSDB_DATA_TYPE_TIMESTAMP; + infoData.info.bytes = sizeof(TSKEY); + // window start ts + taosArrayPush(pBlock->pDataBlock, &infoData); + // window end ts + taosArrayPush(pBlock->pDataBlock, &infoData); + + infoData.info.type = TSDB_DATA_TYPE_UBIGINT; + infoData.info.bytes = sizeof(uint64_t); + taosArrayPush(pBlock->pDataBlock, &infoData); + + return pBlock; } SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, @@ -2412,23 +2640,30 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, goto _error; } } - // semi interval operator does not catch result pInfo->pUpdateRes = createResDataBlock(pPhyNode->pOutputDataBlockDesc); - pInfo->pUpdateRes->info.type = STREAM_REPROCESS; + pInfo->pUpdateRes->info.type = STREAM_CLEAR; blockDataEnsureCapacity(pInfo->pUpdateRes, 128); + pInfo->returnUpdate = false; + pInfo->pPhyNode = (SPhysiNode*)nodesCloneNode((SNode*)pPhyNode); if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) { pInfo->isFinal = true; pOperator->name = "StreamFinalIntervalOperator"; } else { + // semi interval operator does not catch result pInfo->isFinal = false; pOperator->name = "StreamSemiIntervalOperator"; } - if (!IS_FINAL_OP(pInfo)) { + if (!IS_FINAL_OP(pInfo) || numOfChild == 0) { pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE; } + pInfo->pPullWins = taosArrayInit(8, sizeof(SPullWindowInfo)); + pInfo->pullIndex = 0; + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + pInfo->pPullDataMap = taosHashInit(64, hashFn, false, HASH_NO_LOCK); + pInfo->pPullDataRes = createPullDataBlock(); pOperator->operatorType = pPhyNode->type; pOperator->blocking = true; @@ -2811,11 +3046,6 @@ void compactTimeWindow(SStreamSessionAggOperatorInfo* pInfo, int32_t startIndex, } } -typedef struct SWinRes { - TSKEY ts; - uint64_t groupId; -} SWinRes; - static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SHashObj* pStUpdated, SHashObj* pStDeleted, bool hasEndTs) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -3027,14 +3257,14 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { } else if (pOperator->status == OP_RES_TO_RETURN) { doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator); if (pInfo->pDelRes->info.rows > 0) { - /*printDataBlock(pInfo->pDelRes, "session del");*/ + printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo)? "Final Session" : "Single Session"); return pInfo->pDelRes; } doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); if (pBInfo->pRes->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); } - /*printDataBlock(pBInfo->pRes, "session insert");*/ + printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo)? "Final Session" : "Single Session"); return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; } @@ -3048,7 +3278,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { break; } - if (pBlock->info.type == STREAM_REPROCESS) { + if (pBlock->info.type == STREAM_CLEAR) { SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo)); doClearSessionWindows(&pInfo->streamAggSup, &pOperator->exprSupp, pBlock, 0, pOperator->exprSupp.numOfExprs, pInfo->gap, pWins); @@ -3102,11 +3332,11 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator); if (pInfo->pDelRes->info.rows > 0) { - /*printDataBlock(pInfo->pDelRes, "session del");*/ + printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo)? "Final Session" : "Single Session"); return pInfo->pDelRes; } doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); - /*printDataBlock(pBInfo->pRes, "session insert");*/ + printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo)? "Final Session" : "Single Session"); return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; } @@ -3169,11 +3399,11 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { - clearUpdateDataBlock(pInfo->pUpdateRes); + clearSpecialDataBlock(pInfo->pUpdateRes); break; } - if (pBlock->info.type == STREAM_REPROCESS) { + if (pBlock->info.type == STREAM_CLEAR) { SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo)); doClearSessionWindows(&pInfo->streamAggSup, pSup, pBlock, 0, pSup->numOfExprs, pInfo->gap, pWins); removeSessionResults(pStUpdated, pWins); @@ -3236,7 +3466,7 @@ SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream } else { pInfo->isFinal = false; pInfo->pUpdateRes = createResDataBlock(pPhyNode->pOutputDataBlockDesc); - pInfo->pUpdateRes->info.type = STREAM_REPROCESS; + pInfo->pUpdateRes->info.type = STREAM_CLEAR; blockDataEnsureCapacity(pInfo->pUpdateRes, 128); pOperator->name = "StreamSessionSemiAggOperator"; pOperator->fpSet = @@ -3554,7 +3784,7 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { break; } - if (pBlock->info.type == STREAM_REPROCESS) { + if (pBlock->info.type == STREAM_CLEAR) { doClearStateWindows(&pInfo->streamAggSup, pBlock, pInfo->primaryTsIndex, &pInfo->stateCol, pInfo->stateCol.slotId, pSeUpdated, pInfo->pSeDeleted); continue; diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 0c9d46f055..a5e9b8edd9 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -115,6 +115,7 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) .srcNodeId = pTask->nodeId, .srcTaskId = pTask->taskId, .pRetrieve = pRetrieve, + .retrieveLen = dataStrLen, }; int32_t sz = taosArrayGetSize(pTask->childEpInfo); @@ -146,7 +147,7 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) .code = 0, .msgType = TDMT_STREAM_RETRIEVE, .pCont = buf, - .contLen = len, + .contLen = sizeof(SMsgHead) + len, }; if (tmsgSendReq(&pEpInfo->epSet, &rpcMsg) < 0) { diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index fe0f406f8d..c75e6c004a 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -45,11 +45,16 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) ASSERT(false); } if (output == NULL) { - if (pItem->type == STREAM_INPUT__DATA_RETRIEVE && !hasData) { - SSDataBlock block = {0}; - block.info.type = STREAM_PUSH_DATA; - block.info.childId = pTask->selfChildId; - taosArrayPush(pRes, &block); + if (pItem->type == STREAM_INPUT__DATA_RETRIEVE) { + //SSDataBlock block = {0}; + //block.info.type = STREAM_PUSH_EMPTY; + //block.info.childId = pTask->selfChildId; + SStreamDataBlock* pRetrieveBlock = (SStreamDataBlock*)data; + ASSERT(taosArrayGetSize(pRetrieveBlock->blocks) == 1); + SSDataBlock* pBlock = createOneDataBlock(taosArrayGet(pRetrieveBlock->blocks, 0), true); + pBlock->info.type = STREAM_PUSH_EMPTY; + pBlock->info.childId = pTask->selfChildId; + taosArrayPush(pRes, pBlock); } break; } diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index ada391b40a..3d64cec8d8 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -119,6 +119,7 @@ SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t waterma taosArrayPush(pInfo->pTsBuckets, &dumy); } pInfo->numBuckets = DEFAULT_BUCKET_SIZE; + pInfo->pCloseWinSBF = NULL; return pInfo; } @@ -154,6 +155,9 @@ bool updateInfoIsUpdated(SUpdateInfo *pInfo, tb_uid_t tableId, TSKEY ts) { TSKEY maxTs = *(TSKEY *)taosArrayGet(pInfo->pTsBuckets, index); if (ts < maxTs - pInfo->watermark) { // this window has been closed. + if (pInfo->pCloseWinSBF) { + return tScalableBfPut(pInfo->pCloseWinSBF, &ts, sizeof(TSKEY)); + } return true; } @@ -193,3 +197,19 @@ void updateInfoDestroy(SUpdateInfo *pInfo) { taosArrayDestroy(pInfo->pTsSBFs); taosMemoryFree(pInfo); } + +void updateInfoAddCloseWindowSBF(SUpdateInfo *pInfo) { + if (pInfo->pCloseWinSBF) { + return; + } + int64_t rows = adjustExpEntries(pInfo->interval * ROWS_PER_MILLISECOND); + pInfo->pCloseWinSBF = tScalableBfInit(rows, DEFAULT_FALSE_POSITIVE); +} + +void updateInfoDestoryColseWinSBF(SUpdateInfo *pInfo) { + if (!pInfo || !pInfo->pCloseWinSBF) { + return; + } + tScalableBfDestroy(pInfo->pCloseWinSBF); + pInfo->pCloseWinSBF = NULL; +} diff --git a/tests/script/tsim/stream/distributeInterval0.sim b/tests/script/tsim/stream/distributeInterval0.sim index 91ce49bc8c..3e38df2c89 100644 --- a/tests/script/tsim/stream/distributeInterval0.sim +++ b/tests/script/tsim/stream/distributeInterval0.sim @@ -80,17 +80,17 @@ endi if $data03 != 4 then print ======$data03 - return -1 + goto loop1 endi if $data04 != 52 then print ======$data04 - return -1 + goto loop1 endi if $data05 != 13 then print ======$data05 - return -1 + goto loop1 endi # row 1 @@ -179,7 +179,7 @@ sql use test1; sql create stable st(ts timestamp, a int, b int , c int) tags(ta int,tb int,tc int); sql create table ts1 using st tags(1,1,1); sql create table ts2 using st tags(2,2,2); -sql create stream stream_t2 trigger at_once into streamtST1 as select _wstartts, count(*) c1, count(a) c2 , sum(a) c3 , max(b) c5, min(c) c6 from st interval(10s) ; +sql create stream stream_t2 trigger at_once watermark 20s into streamtST1 as select _wstartts, count(*) c1, count(a) c2 , sum(a) c3 , max(b) c5, min(c) c6 from st interval(10s) ; sql insert into ts1 values(1648791211000,1,2,3); sql insert into ts1 values(1648791222001,2,2,3); diff --git a/tests/script/tsim/stream/partitionby.sim b/tests/script/tsim/stream/partitionby.sim index b84a01eb4a..c634ad85ee 100644 --- a/tests/script/tsim/stream/partitionby.sim +++ b/tests/script/tsim/stream/partitionby.sim @@ -74,7 +74,7 @@ sql create stable st(ts timestamp,a int,b int,c int,id int) tags(ta int,tb int,t sql create table ts1 using st tags(1,1,1); sql create table ts2 using st tags(2,2,2); -sql create stream stream_t2 trigger at_once into streamtST as select _wstartts, count(*) c1, count(a) c2 , sum(a) c3 , max(b) c5, min(c) c6, max(id) c7 from st partition by ta interval(10s) ; +sql create stream stream_t2 trigger at_once watermark 20s into streamtST as select _wstartts, count(*) c1, count(a) c2 , sum(a) c3 , max(b) c5, min(c) c6, max(id) c7 from st partition by ta interval(10s) ; sql insert into ts1 values(1648791211000,1,2,3,1); sql insert into ts1 values(1648791222001,2,2,3,2); sql insert into ts2 values(1648791211000,1,2,3,3); @@ -83,16 +83,16 @@ sql insert into ts2 values(1648791222001,2,2,3,4); sql insert into ts2 values(1648791222002,2,2,3,5); sql insert into ts2 values(1648791222002,2,2,3,6); -sql insert into ts1 values(1648791211000,1,2,3,1); -sql insert into ts1 values(1648791222001,2,2,3,2); -sql insert into ts2 values(1648791211000,1,2,3,3); -sql insert into ts2 values(1648791222001,2,2,3,4); +sql insert into ts1 values(1648791211000,1,2,3,7); +sql insert into ts1 values(1648791222001,2,2,3,8); +sql insert into ts2 values(1648791211000,1,2,3,9); +sql insert into ts2 values(1648791222001,2,2,3,10); $loop_count = 0 loop2: sleep 300 -sql select * from streamtST; +sql select * from streamtST order by c7 asc; $loop_count = $loop_count + 1 if $loop_count == 10 then @@ -104,8 +104,18 @@ print =====data01=$data01 goto loop2 endi -if $data02 != 1 then -print =====data02=$data02 +if $data11 != 1 then +print =====data11=$data11 +goto loop2 +endi + +if $data21 != 1 then +print =====data21=$data21 +goto loop2 +endi + +if $data31 != 2 then +print =====data31=$data31 goto loop2 endi @@ -114,8 +124,18 @@ print =====data03=$data03 goto loop2 endi -if $data04 != 2 then -print =====data04=$data04 +if $data13 != 2 then +print =====data13=$data13 +goto loop2 +endi + +if $data23 != 1 then +print =====data23=$data23 +goto loop2 +endi + +if $data33 != 4 then +print =====data33=$data33 goto loop2 endi diff --git a/tests/script/tsim/stream/schedSnode.sim b/tests/script/tsim/stream/schedSnode.sim index dbf714a96f..dbdaaf65d0 100644 --- a/tests/script/tsim/stream/schedSnode.sim +++ b/tests/script/tsim/stream/schedSnode.sim @@ -79,17 +79,17 @@ endi if $data03 != 4 then print ======$data03 - return -1 + goto loop1 endi if $data04 != 52 then print ======$data04 - return -1 + goto loop1 endi if $data05 != 13 then print ======$data05 - return -1 + goto loop1 endi # row 1