diff --git a/Jenkinsfile2 b/Jenkinsfile2 index b65576deaf..861478160f 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -4,11 +4,6 @@ import jenkins.model.CauseOfInterruption node { } -win_test_stage = 0 -linux_ready = 0 -linux_node_ip = "" -linux_node_pass = "" - def abortPreviousBuilds() { def currentJobName = env.JOB_NAME def currentBuildNumber = env.BUILD_NUMBER.toInteger() @@ -289,7 +284,6 @@ def run_win_ctest() { ''' } def run_win_test() { - echo "LINUX NODE: ${linux_node_ip} - ${linux_node_pass}" bat ''' echo "windows test ..." cd %WIN_CONNECTOR_ROOT% @@ -298,9 +292,8 @@ def run_win_test() { ls -l C:\\Windows\\System32\\taos.dll time /t cd %WIN_SYSTEM_TEST_ROOT% - echo "node: ''' + linux_node_ip + ''':''' + linux_node_pass + '''" echo "testing ..." - test-all.bat "{\\\"host\\\":\\\"''' + linux_node_ip + '''\\\",\\\"port\\\":22,\\\"user\\\":\\\"root\\\",\\\"password\\\":\\\"''' + linux_node_pass + '''\\\",\\\"path\\\":\\\"/var/lib/jenkins/workspace/TDinternal\\\"}" + test-all.bat ci time /t ''' } @@ -331,17 +324,9 @@ pipeline { pre_test_win() pre_test_build_win() run_win_ctest() - script { - while(linux_ready == 0) { - sleep(8) - } - } run_win_test() } } - script { - win_test_stage = 1 - } } } stage('linux test') { @@ -351,17 +336,6 @@ pipeline { changeRequest() } steps { - script { - linux_node_ip = sh ( - script: 'jq .ip /home/node_info.json | sed "s/\\\"//g"', - returnStdout: true - ).trim() - linux_node_pass = sh ( - script: 'jq .password /home/node_info.json | sed "s/\\\"//g" |sed "s/\\!/^^^^^^^^\\!/g"', - returnStdout: true - ).trim() - echo "${linux_node_ip}:${linux_node_pass}" - } catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') { timeout(time: 120, unit: 'MINUTES'){ pre_test() @@ -418,38 +392,9 @@ pipeline { cd ${WKC}/packaging ./release.sh -v cluster -n 3.0.0.100 -s static ''' - sh ''' - echo "install ..." - cd ${WKC}/release - tar xzf TDengine-enterprise-server-3.0.0.100-Linux-x64.tar.gz - cd TDengine-enterprise-server-3.0.0.100 - service taosd stop || : - rm -rf /var/lib/taos - ./install.sh -e no - ''' - sh ''' - echo "checking ..." - which taos - which taosd - rm -rf ${WK}/debug - mv ${WKC}/debug ${WK}/ - ''' - sh ''' - echo "install taospy ..." - cd ${WKPY} - pip3 install . - ''' } } } - script { - linux_ready = 1 - } - script { - while(win_test_stage == 0){ - sleep(12) - } - } } } } diff --git a/include/common/tcommon.h b/include/common/tcommon.h index a14f7eff8a..928fe0aa0e 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -42,12 +42,13 @@ enum { typedef enum EStreamType { STREAM_NORMAL = 1, STREAM_INVERT, - STREAM_REPROCESS, + STREAM_CLEAR, STREAM_INVALID, STREAM_GET_ALL, STREAM_DELETE, STREAM_RETRIEVE, STREAM_PUSH_DATA, + STREAM_PUSH_EMPTY, } EStreamType; typedef struct { diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 2a0d4e7ff6..2e2c7d1700 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -224,6 +224,7 @@ size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize); int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n); int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src); +int32_t copyDataBlock(SSDataBlock* dst, const SSDataBlock* src); SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData); SSDataBlock* createDataBlock(); int32_t blockDataAppendColInfo(SSDataBlock* pBlock, SColumnInfoData* pColInfoData); @@ -236,6 +237,8 @@ void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData); void blockDebugShowData(const SArray* dataBlocks, const char* flag); +// for debug +char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** dumpBuf); int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks, STSchema* pTSchema, int32_t vgId, tb_uid_t suid); diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 29eebc332b..a5688af18a 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1284,6 +1284,7 @@ void tFreeSShowRsp(SShowRsp* pRsp); typedef struct { char db[TSDB_DB_FNAME_LEN]; char tb[TSDB_TABLE_NAME_LEN]; + char user[TSDB_USER_LEN]; int64_t showId; } SRetrieveTableReq; diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index b7583b41b9..e23b84c1f8 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -75,6 +75,7 @@ typedef struct SScanLogicNode { double filesFactor; SArray* pSmaIndexes; SNodeList* pPartTags; + bool partSort; } SScanLogicNode; typedef struct SJoinLogicNode { diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index d86a8dae6e..0b767e96f6 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -16,6 +16,7 @@ #ifndef _TD_QUERY_H_ #define _TD_QUERY_H_ +// clang-foramt off #ifdef __cplusplus extern "C" { #endif @@ -71,7 +72,7 @@ typedef struct SIndexMeta { } SIndexMeta; typedef struct STbVerInfo { - char tbFName[TSDB_TABLE_FNAME_LEN]; + char tbFName[TSDB_TABLE_FNAME_LEN]; int32_t sversion; int32_t tversion; } STbVerInfo; @@ -141,7 +142,7 @@ typedef struct SDataBuf { typedef struct STargetInfo { ETargetType type; - char* dbFName; // used to update db's vgroup epset + char* dbFName; // used to update db's vgroup epset int32_t vgId; } STargetInfo; @@ -149,15 +150,15 @@ typedef int32_t (*__async_send_cb_fn_t)(void* param, const SDataBuf* pMsg, int32 typedef int32_t (*__async_exec_fn_t)(void* param); typedef struct SRequestConnInfo { - void* pTrans; - uint64_t requestId; - int64_t requestObjRefId; - SEpSet mgmtEps; + void* pTrans; + uint64_t requestId; + int64_t requestObjRefId; + SEpSet mgmtEps; } SRequestConnInfo; typedef struct SMsgSendInfo { - __async_send_cb_fn_t fp; // async callback function - STargetInfo target; // for update epset + __async_send_cb_fn_t fp; // async callback function + STargetInfo target; // for update epset void* param; uint64_t requestId; uint64_t requestObjRefId; @@ -206,13 +207,15 @@ int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STabl char* jobTaskStatusStr(int32_t status); SSchema createSchema(int8_t type, int32_t bytes, col_id_t colId, const char* name); -void destroyQueryExecRes(SQueryExecRes* pRes); -int32_t dataConverToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *len); -char* parseTagDatatoJson(void* p); + +void destroyQueryExecRes(SQueryExecRes* pRes); +int32_t dataConverToStr(char* str, int type, void* buf, int32_t bufSize, int32_t* len); +char* parseTagDatatoJson(void* p); int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst); int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst); -extern int32_t (*queryBuildMsg[TDMT_MAX])(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void*(*mallocFp)(int32_t)); +extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char** msg, int32_t msgSize, int32_t* msgLen, + void* (*mallocFp)(int32_t)); extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t msgSize); #define SET_META_TYPE_NULL(t) (t) = META_TYPE_NULL_TABLE @@ -223,7 +226,7 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t #define NEED_CLIENT_RM_TBLMETA_ERROR(_code) \ ((_code) == TSDB_CODE_PAR_TABLE_NOT_EXIST || (_code) == TSDB_CODE_VND_TB_NOT_EXIST || \ (_code) == TSDB_CODE_PAR_INVALID_COLUMNS_NUM || (_code) == TSDB_CODE_PAR_INVALID_COLUMN || \ - (_code) == TSDB_CODE_PAR_TAGS_NOT_MATCHED || (_code) == TSDB_CODE_PAR_VALUE_TOO_LONG || \ + (_code) == TSDB_CODE_PAR_TAGS_NOT_MATCHED || (_code) == TSDB_CODE_PAR_VALUE_TOO_LONG || \ (_code) == TSDB_CODE_PAR_INVALID_DROP_COL || ((_code) == TSDB_CODE_TDB_INVALID_TABLE_ID)) #define NEED_CLIENT_REFRESH_VG_ERROR(_code) \ ((_code) == TSDB_CODE_VND_HASH_MISMATCH || (_code) == TSDB_CODE_VND_INVALID_VGROUP_ID) @@ -231,11 +234,13 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t #define NEED_CLIENT_HANDLE_ERROR(_code) \ (NEED_CLIENT_RM_TBLMETA_ERROR(_code) || NEED_CLIENT_REFRESH_VG_ERROR(_code) || \ NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code)) -#define NEED_CLIENT_RM_TBLMETA_REQ(_type) ((_type) == TDMT_VND_CREATE_TABLE || (_type) == TDMT_VND_CREATE_STB \ - || (_type) == TDMT_VND_DROP_TABLE || (_type) == TDMT_VND_DROP_STB) +#define NEED_CLIENT_RM_TBLMETA_REQ(_type) \ + ((_type) == TDMT_VND_CREATE_TABLE || (_type) == TDMT_VND_CREATE_STB || (_type) == TDMT_VND_DROP_TABLE || \ + (_type) == TDMT_VND_DROP_STB) -#define NEED_SCHEDULER_RETRY_ERROR(_code) \ - ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL || (_code) == TSDB_CODE_SCH_TIMEOUT_ERROR) +#define NEED_SCHEDULER_RETRY_ERROR(_code) \ + ((_code) == TSDB_CODE_RPC_REDIRECT || (_code) == TSDB_CODE_RPC_NETWORK_UNAVAIL || \ + (_code) == TSDB_CODE_SCH_TIMEOUT_ERROR) #define REQUEST_TOTAL_EXEC_TIMES 2 @@ -312,3 +317,4 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t #endif #endif /*_TD_QUERY_H_*/ + // clang-foramt on diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 594344ba8a..db928f194c 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -319,7 +319,7 @@ static FORCE_INLINE int32_t streamTaskInput(SStreamTask* pTask, SStreamQueueItem return -1; } taosWriteQitem(pTask->inputQueue->queue, pSubmitClone); - } else if (pItem->type == STREAM_INPUT__DATA_BLOCK) { + } else if (pItem->type == STREAM_INPUT__DATA_BLOCK || pItem->type == STREAM_INPUT__DATA_RETRIEVE) { taosWriteQitem(pTask->inputQueue->queue, pItem); } else if (pItem->type == STREAM_INPUT__CHECKPOINT) { taosWriteQitem(pTask->inputQueue->queue, pItem); diff --git a/include/libs/stream/tstreamUpdate.h b/include/libs/stream/tstreamUpdate.h index 398851a09f..21a1515d8f 100644 --- a/include/libs/stream/tstreamUpdate.h +++ b/include/libs/stream/tstreamUpdate.h @@ -32,12 +32,15 @@ typedef struct SUpdateInfo { int64_t interval; int64_t watermark; TSKEY minTS; + SScalableBf* pCloseWinSBF; } SUpdateInfo; SUpdateInfo *updateInfoInitP(SInterval* pInterval, int64_t watermark); SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t watermark); bool updateInfoIsUpdated(SUpdateInfo *pInfo, tb_uid_t tableId, TSKEY ts); void updateInfoDestroy(SUpdateInfo *pInfo); +void updateInfoAddCloseWindowSBF(SUpdateInfo *pInfo); +void updateInfoDestoryColseWinSBF(SUpdateInfo *pInfo); #ifdef __cplusplus } diff --git a/include/os/os.h b/include/os/os.h index 254c16efbe..3e72a618a0 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -52,6 +52,7 @@ extern "C" { #endif #else +#include #include #ifndef TD_USE_WINSOCK #include diff --git a/include/os/osMath.h b/include/os/osMath.h index f17ca56c9e..e13c32422e 100644 --- a/include/os/osMath.h +++ b/include/os/osMath.h @@ -25,11 +25,10 @@ extern "C" { #define TSWAP(a, b) \ do { \ - char *__tmp = taosMemoryMalloc(sizeof(a)); \ + char *__tmp = alloca(sizeof(a)); \ memcpy(__tmp, &(a), sizeof(a)); \ memcpy(&(a), &(b), sizeof(a)); \ memcpy(&(b), __tmp, sizeof(a)); \ - taosMemoryFree(__tmp); \ } while (0) #ifdef WINDOWS diff --git a/include/os/osSocket.h b/include/os/osSocket.h index 213a6930ee..9dd5b972fa 100644 --- a/include/os/osSocket.h +++ b/include/os/osSocket.h @@ -157,7 +157,10 @@ int32_t taosNonblockwrite(TdSocketPtr pSocket, char *ptr, int32_t nbytes); int64_t taosCopyFds(TdSocketPtr pSrcSocket, TdSocketPtr pDestSocket, int64_t len); void taosWinSocketInit(); -int taosCreateSocketWithTimeOutOpt(uint32_t conn_timeout_sec); +/* + * set timeout(ms) + */ +int32_t taosCreateSocketWithTimeout(uint32_t timeout); TdSocketPtr taosOpenUdpSocket(uint32_t localIp, uint16_t localPort); TdSocketPtr taosOpenTcpClientSocket(uint32_t ip, uint16_t port, uint32_t localIp); diff --git a/include/util/ttrace.h b/include/util/ttrace.h index 206cbbf28d..579768228a 100644 --- a/include/util/ttrace.h +++ b/include/util/ttrace.h @@ -45,9 +45,11 @@ typedef struct STraceId { #define TRACE_GET_MSGID(traceId) (traceId)->msgId -#define TRACE_TO_STR(traceId, buf) \ - do { \ - sprintf(buf, "0x%" PRIx64 ":0x%" PRIx64 "", traceId->rootId, traceId->msgId); \ +#define TRACE_TO_STR(traceId, buf) \ + do { \ + int64_t rootId = (traceId) != NULL ? (traceId)->rootId : 0; \ + int64_t msgId = (traceId) != NULL ? (traceId)->msgId : 0; \ + sprintf(buf, "0x%" PRIx64 ":0x%" PRIx64 "", rootId, msgId); \ } while (0) #ifdef __cplusplus diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 9f04e89694..d7bf4b60f1 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -13,11 +13,11 @@ * along with this program. If not, see . */ -#include "os.h" #include "catalog.h" -#include "functionMgt.h" #include "clientInt.h" #include "clientLog.h" +#include "functionMgt.h" +#include "os.h" #include "query.h" #include "scheduler.h" #include "tcache.h" @@ -38,7 +38,7 @@ static TdThreadOnce tscinit = PTHREAD_ONCE_INIT; volatile int32_t tscInitRes = 0; static void registerRequest(SRequestObj *pRequest) { - STscObj *pTscObj = acquireTscObj(*(int64_t*)pRequest->pTscObj->id); + STscObj *pTscObj = acquireTscObj(*(int64_t *)pRequest->pTscObj->id); assert(pTscObj != NULL); @@ -54,14 +54,14 @@ static void registerRequest(SRequestObj *pRequest) { int32_t currentInst = atomic_add_fetch_64((int64_t *)&pSummary->currentRequests, 1); tscDebug("0x%" PRIx64 " new Request from connObj:0x%" PRIx64 ", current:%d, app current:%d, total:%d, reqId:0x%" PRIx64, - pRequest->self, *(int64_t*)pRequest->pTscObj->id, num, currentInst, total, pRequest->requestId); + pRequest->self, *(int64_t *)pRequest->pTscObj->id, num, currentInst, total, pRequest->requestId); } } static void deregisterRequest(SRequestObj *pRequest) { assert(pRequest != NULL); - STscObj *pTscObj = pRequest->pTscObj; + STscObj * pTscObj = pRequest->pTscObj; SAppClusterSummary *pActivity = &pTscObj->pAppInfo->summary; int32_t currentInst = atomic_sub_fetch_64((int64_t *)&pActivity->currentRequests, 1); @@ -70,8 +70,8 @@ static void deregisterRequest(SRequestObj *pRequest) { int64_t duration = taosGetTimestampUs() - pRequest->metric.start; tscDebug("0x%" PRIx64 " free Request from connObj: 0x%" PRIx64 ", reqId:0x%" PRIx64 " elapsed:%" PRIu64 " ms, current:%d, app current:%d", - pRequest->self, *(int64_t*)pTscObj->id, pRequest->requestId, duration / 1000, num, currentInst); - releaseTscObj(*(int64_t*)pTscObj->id); + pRequest->self, *(int64_t *)pTscObj->id, pRequest->requestId, duration / 1000, num, currentInst); + releaseTscObj(*(int64_t *)pTscObj->id); } // todo close the transporter properly @@ -80,12 +80,13 @@ void closeTransporter(STscObj *pTscObj) { return; } - tscDebug("free transporter:%p in connObj: 0x%" PRIx64, pTscObj->pAppInfo->pTransporter, *(int64_t*)pTscObj->id); + tscDebug("free transporter:%p in connObj: 0x%" PRIx64, pTscObj->pAppInfo->pTransporter, *(int64_t *)pTscObj->id); rpcClose(pTscObj->pAppInfo->pTransporter); } static bool clientRpcRfp(int32_t code) { - if (code == TSDB_CODE_RPC_REDIRECT) { + if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED || + code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY) { return true; } else { return false; @@ -128,16 +129,17 @@ void closeAllRequests(SHashObj *pRequests) { void destroyTscObj(void *pObj) { STscObj *pTscObj = pObj; - SClientHbKey connKey = {.tscRid = *(int64_t*)pTscObj->id, .connType = pTscObj->connType}; + SClientHbKey connKey = {.tscRid = *(int64_t *)pTscObj->id, .connType = pTscObj->connType}; hbDeregisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey); int64_t connNum = atomic_sub_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); closeAllRequests(pTscObj->pRequests); schedulerStopQueryHb(pTscObj->pAppInfo->pTransporter); if (0 == connNum) { - // TODO - //closeTransporter(pTscObj); + // TODO + // closeTransporter(pTscObj); } - tscDebug("connObj 0x%" PRIx64 " destroyed, totalConn:%" PRId64, *(int64_t*)pTscObj->id, pTscObj->pAppInfo->numOfConns); + tscDebug("connObj 0x%" PRIx64 " destroyed, totalConn:%" PRId64, *(int64_t *)pTscObj->id, + pTscObj->pAppInfo->numOfConns); taosThreadMutexDestroy(&pTscObj->mutex); taosMemoryFreeClear(pTscObj); } @@ -167,10 +169,10 @@ void *createTscObj(const char *user, const char *auth, const char *db, int32_t c taosThreadMutexInit(&pObj->mutex, NULL); pObj->id = taosMemoryMalloc(sizeof(int64_t)); - *(int64_t*)pObj->id = taosAddRef(clientConnRefPool, pObj); + *(int64_t *)pObj->id = taosAddRef(clientConnRefPool, pObj); pObj->schemalessType = 1; - tscDebug("connObj created, 0x%" PRIx64, *(int64_t*)pObj->id); + tscDebug("connObj created, 0x%" PRIx64, *(int64_t *)pObj->id); return pObj; } @@ -325,7 +327,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { return 0; } - SConfig *pCfg = taosGetCfg(); + SConfig * pCfg = taosGetCfg(); SConfigItem *pItem = NULL; switch (option) { diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 489966b636..ac9daa5119 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -617,12 +617,12 @@ int32_t scheduleAsyncQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNod .requestId = pRequest->requestId, .requestObjRefId = pRequest->self}; SSchedulerReq req = {.pConn = &conn, - .pNodeList = pNodeList, - .pDag = pDag, - .sql = pRequest->sqlstr, - .startTs = pRequest->metric.start, - .fp = schdExecCallback, - .cbParam = &res}; + .pNodeList = pNodeList, + .pDag = pDag, + .sql = pRequest->sqlstr, + .startTs = pRequest->metric.start, + .fp = schdExecCallback, + .cbParam = &res}; int32_t code = schedulerAsyncExecJob(&req, &pRequest->body.queryJob); @@ -669,13 +669,13 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList .requestId = pRequest->requestId, .requestObjRefId = pRequest->self}; SSchedulerReq req = {.pConn = &conn, - .pNodeList = pNodeList, - .pDag = pDag, - .sql = pRequest->sqlstr, - .startTs = pRequest->metric.start, - .fp = NULL, - .cbParam = NULL, - .reqKilled = &pRequest->killed}; + .pNodeList = pNodeList, + .pDag = pDag, + .sql = pRequest->sqlstr, + .startTs = pRequest->metric.start, + .fp = NULL, + .cbParam = NULL, + .reqKilled = &pRequest->killed}; int32_t code = schedulerExecJob(&req, &pRequest->body.queryJob, &res); pRequest->body.resInfo.execRes = res.res; diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 593f8c5c0b..cc995c4d64 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1164,7 +1164,7 @@ int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows) int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) { int32_t code = 0; - ASSERT(numOfRows > 0); + //ASSERT(numOfRows > 0); if (numOfRows == 0) { return TSDB_CODE_SUCCESS; @@ -1230,6 +1230,32 @@ int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src) { return 0; } +int32_t copyDataBlock(SSDataBlock* dst, const SSDataBlock* src) { + ASSERT(src != NULL && dst != NULL); + + blockDataCleanup(dst); + int32_t code = blockDataEnsureCapacity(dst, src->info.rows); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + return code; + } + + size_t numOfCols = taosArrayGetSize(src->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pDst = taosArrayGet(dst->pDataBlock, i); + SColumnInfoData* pSrc = taosArrayGet(src->pDataBlock, i); + if (pSrc->pData == NULL) { + continue; + } + + colDataAssign(pDst, pSrc, src->info.rows, &src->info); + } + + dst->info.rows = src->info.rows; + dst->info.window = src->info.window; + return TSDB_CODE_SUCCESS; +} + SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) { if (pDataBlock == NULL) { return NULL; @@ -1627,6 +1653,56 @@ void blockDebugShowData(const SArray* dataBlocks, const char* flag) { } } +// for debug +char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) { + int32_t size = 2048; + *pDataBuf = taosMemoryCalloc(size, 1); + char* dumpBuf = *pDataBuf; + char pBuf[128] = {0}; + int32_t colNum = taosArrayGetSize(pDataBlock->pDataBlock); + int32_t rows = pDataBlock->info.rows; + int32_t len = 0; + len += snprintf(dumpBuf + len, size - len, "\n%s |block type %d |child id %d|\n", flag, (int32_t)pDataBlock->info.type, pDataBlock->info.childId); + for (int32_t j = 0; j < rows; j++) { + len += snprintf(dumpBuf + len, size - len, "%s |", flag); + for (int32_t k = 0; k < colNum; k++) { + SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); + void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); + if (colDataIsNull(pColInfoData, rows, j, NULL)) { + len += snprintf(dumpBuf + len, size - len, " %15s |", "NULL"); + continue; + } + switch (pColInfoData->info.type) { + case TSDB_DATA_TYPE_TIMESTAMP: + formatTimestamp(pBuf, *(uint64_t*)var, TSDB_TIME_PRECISION_MILLI); + len += snprintf(dumpBuf + len, size - len, " %25s |", pBuf); + break; + case TSDB_DATA_TYPE_INT: + len += snprintf(dumpBuf + len, size - len, " %15d |", *(int32_t*)var); + break; + case TSDB_DATA_TYPE_UINT: + len += snprintf(dumpBuf + len, size - len, " %15u |", *(uint32_t*)var); + break; + case TSDB_DATA_TYPE_BIGINT: + len += snprintf(dumpBuf + len, size - len, " %15ld |", *(int64_t*)var); + break; + case TSDB_DATA_TYPE_UBIGINT: + len += snprintf(dumpBuf + len, size - len, " %15lu |", *(uint64_t*)var); + break; + case TSDB_DATA_TYPE_FLOAT: + len += snprintf(dumpBuf + len, size - len, " %15f |", *(float*)var); + break; + case TSDB_DATA_TYPE_DOUBLE: + len += snprintf(dumpBuf + len, size - len, " %15lf |", *(double*)var); + break; + } + } + len += snprintf(dumpBuf + len, size - len, "\n"); + } + len += snprintf(dumpBuf + len, size - len, "%s |end\n", flag); + return dumpBuf; +} + /** * @brief TODO: Assume that the final generated result it less than 3M * diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index d96a5ec574..9db5019512 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -3017,6 +3017,7 @@ int32_t tSerializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableReq if (tEncodeI64(&encoder, pReq->showId) < 0) return -1; if (tEncodeCStr(&encoder, pReq->db) < 0) return -1; if (tEncodeCStr(&encoder, pReq->tb) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->user) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -3032,6 +3033,8 @@ int32_t tDeserializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableR if (tDecodeI64(&decoder, &pReq->showId) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->tb) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1; + tEndDecode(&decoder); tDecoderClear(&decoder); return 0; diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index 63d2a65df1..a4745abd5b 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -70,9 +70,9 @@ int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg) { } static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) { - SDnodeTrans *pTrans = &pDnode->trans; + SDnodeTrans * pTrans = &pDnode->trans; int32_t code = -1; - SRpcMsg *pMsg = NULL; + SRpcMsg * pMsg = NULL; SMgmtWrapper *pWrapper = NULL; SDnodeHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(pRpc->msgType)]; @@ -194,11 +194,11 @@ int32_t dmInitMsgHandle(SDnode *pDnode) { for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) { SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype]; - SArray *pArray = (*pWrapper->func.getHandlesFp)(); + SArray * pArray = (*pWrapper->func.getHandlesFp)(); if (pArray == NULL) return -1; for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { - SMgmtHandle *pMgmt = taosArrayGet(pArray, i); + SMgmtHandle * pMgmt = taosArrayGet(pArray, i); SDnodeHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(pMgmt->msgType)]; if (pMgmt->needCheckVgId) { pHandle->needCheckVgId = pMgmt->needCheckVgId; @@ -248,7 +248,14 @@ static inline void dmReleaseHandle(SRpcHandleInfo *pHandle, int8_t type) { } } -static bool rpcRfp(int32_t code) { return code == TSDB_CODE_RPC_REDIRECT; } +static bool rpcRfp(int32_t code) { + if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED || + code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY) { + return true; + } else { + return false; + } +} int32_t dmInitClient(SDnode *pDnode) { SDnodeTrans *pTrans = &pDnode->trans; diff --git a/source/dnode/mnode/impl/inc/mndAuth.h b/source/dnode/mnode/impl/inc/mndPrivilege.h similarity index 65% rename from source/dnode/mnode/impl/inc/mndAuth.h rename to source/dnode/mnode/impl/inc/mndPrivilege.h index 81a776b652..a1bec69790 100644 --- a/source/dnode/mnode/impl/inc/mndAuth.h +++ b/source/dnode/mnode/impl/inc/mndPrivilege.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef _TD_MND_AUTH_H_ -#define _TD_MND_AUTH_H_ +#ifndef _TD_MND_PRIVILEGE_H +#define _TD_MND_PRIVILEGE_H #include "mndInt.h" @@ -24,6 +24,9 @@ extern "C" { typedef enum { MND_OPER_CONNECT = 1, + MND_OPER_CREATE_ACCT, + MND_OPER_DROP_ACCT, + MND_OPER_ALTER_ACCT, MND_OPER_CREATE_USER, MND_OPER_DROP_USER, MND_OPER_ALTER_USER, @@ -45,6 +48,8 @@ typedef enum { MND_OPER_CREATE_FUNC, MND_OPER_DROP_FUNC, MND_OPER_KILL_TRANS, + MND_OPER_KILL_CONN, + MND_OPER_KILL_QUERY, MND_OPER_CREATE_DB, MND_OPER_ALTER_DB, MND_OPER_DROP_DB, @@ -54,16 +59,17 @@ typedef enum { MND_OPER_READ_DB, } EOperType; -int32_t mndInitAuth(SMnode *pMnode); -void mndCleanupAuth(SMnode *pMnode); +int32_t mndInitPrivilege(SMnode *pMnode); +void mndCleanupPrivilege(SMnode *pMnode); -int32_t mndCheckOperAuth(SMnode *pMnode, const char *user, EOperType operType); -int32_t mndCheckDbAuth(SMnode *pMnode, const char *user, EOperType operType, SDbObj *pDb); -int32_t mndCheckShowAuth(SMnode *pMnode, const char *user, int32_t showType); -int32_t mndCheckAlterUserAuth(SUserObj *pOperUser, SUserObj *pUser, SAlterUserReq *pAlter); +int32_t mndCheckOperPrivilege(SMnode *pMnode, const char *user, EOperType operType); +int32_t mndCheckDbPrivilege(SMnode *pMnode, const char *user, EOperType operType, SDbObj *pDb); +int32_t mndCheckDbPrivilegeByName(SMnode *pMnode, const char *user, EOperType operType, const char *name); +int32_t mndCheckShowPrivilege(SMnode *pMnode, const char *user, int32_t showType); +int32_t mndCheckAlterUserPrivilege(SUserObj *pOperUser, SUserObj *pUser, SAlterUserReq *pAlter); #ifdef __cplusplus } #endif -#endif /*_TD_MND_AUTH_H_*/ +#endif /*_TD_MND_PRIVILEGE_H*/ diff --git a/source/dnode/mnode/impl/src/mndAcct.c b/source/dnode/mnode/impl/src/mndAcct.c index 0ce4a8c76e..33f0bb7a34 100644 --- a/source/dnode/mnode/impl/src/mndAcct.c +++ b/source/dnode/mnode/impl/src/mndAcct.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "mndAcct.h" +#include "mndPrivilege.h" #include "mndShow.h" #include "mndTrans.h" @@ -212,18 +213,30 @@ static int32_t mndAcctActionUpdate(SSdb *pSdb, SAcctObj *pOld, SAcctObj *pNew) { } static int32_t mndProcessCreateAcctReq(SRpcMsg *pReq) { + if (mndCheckOperPrivilege(pReq->info.node, pReq->info.conn.user, MND_OPER_CREATE_ACCT) != 0) { + return -1; + } + terrno = TSDB_CODE_MSG_NOT_PROCESSED; mError("failed to process create acct request since %s", terrstr()); return -1; } static int32_t mndProcessAlterAcctReq(SRpcMsg *pReq) { + if (mndCheckOperPrivilege(pReq->info.node, pReq->info.conn.user, MND_OPER_ALTER_ACCT) != 0) { + return -1; + } + terrno = TSDB_CODE_MSG_NOT_PROCESSED; mError("failed to process create acct request since %s", terrstr()); return -1; } static int32_t mndProcessDropAcctReq(SRpcMsg *pReq) { + if (mndCheckOperPrivilege(pReq->info.node, pReq->info.conn.user, MND_OPER_DROP_ACCT) != 0) { + return -1; + } + terrno = TSDB_CODE_MSG_NOT_PROCESSED; mError("failed to process create acct request since %s", terrstr()); return -1; diff --git a/source/dnode/mnode/impl/src/mndBnode.c b/source/dnode/mnode/impl/src/mndBnode.c index e2b1aad008..aafcd19992 100644 --- a/source/dnode/mnode/impl/src/mndBnode.c +++ b/source/dnode/mnode/impl/src/mndBnode.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "mndBnode.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndDnode.h" #include "mndShow.h" #include "mndTrans.h" @@ -277,7 +277,7 @@ static int32_t mndProcessCreateBnodeReq(SRpcMsg *pReq) { } mDebug("bnode:%d, start to create", createReq.dnodeId); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_BNODE) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_BNODE) != 0) { goto _OVER; } @@ -382,7 +382,7 @@ static int32_t mndProcessDropBnodeReq(SRpcMsg *pReq) { } mDebug("bnode:%d, start to drop", dropReq.dnodeId); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_BNODE) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_BNODE) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 4da3c906d7..5b5de10fba 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "mndConsumer.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" @@ -431,6 +431,10 @@ static int32_t mndProcessSubscribeReq(SRpcMsg *pMsg) { goto SUBSCRIBE_OVER; } + if (mndCheckDbPrivilegeByName(pMnode, pMsg->info.conn.user, MND_OPER_READ_DB, pTopic->db) != 0) { + goto SUBSCRIBE_OVER; + } + #if 0 // ref topic to prevent drop // TODO make topic complete diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 345464399e..0345f1b345 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "mndDb.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndDnode.h" #include "mndOffset.h" #include "mndShow.h" @@ -506,6 +506,9 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) { } mDebug("db:%s, start to create, vgroups:%d", createReq.db, createReq.numOfVgroups); + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_DB, NULL) != 0) { + goto _OVER; + } pDb = mndAcquireDb(pMnode, createReq.db); if (pDb != NULL) { @@ -526,10 +529,6 @@ static int32_t mndProcessCreateDbReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_DB, NULL) != 0) { - goto _OVER; - } - code = mndCreateDb(pMnode, pReq, &createReq, pUser); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; @@ -700,7 +699,7 @@ static int32_t mndProcessAlterDbReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_ALTER_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_ALTER_DB, pDb) != 0) { goto _OVER; } @@ -980,7 +979,7 @@ static int32_t mndProcessDropDbReq(SRpcMsg *pReq) { } } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_DB, pDb) != 0) { goto _OVER; } @@ -1127,7 +1126,7 @@ static int32_t mndProcessUseDbReq(SRpcMsg *pReq) { mError("db:%s, failed to process use db req since %s", usedbReq.db, terrstr()); } else { - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_USE_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_USE_DB, pDb) != 0) { goto _OVER; } @@ -1252,7 +1251,7 @@ static int32_t mndProcessCompactDbReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_COMPACT_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_COMPACT_DB, pDb) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 113777bc1f..af1d641ebf 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "mndDnode.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndMnode.h" #include "mndQnode.h" #include "mndShow.h" @@ -621,7 +621,7 @@ static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) { } mInfo("dnode:%s:%d, start to create", createReq.fqdn, createReq.port); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_DNODE) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_DNODE) != 0) { goto _OVER; } @@ -715,7 +715,7 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) { } mInfo("dnode:%d, start to drop, ep:%s:%d", dropReq.dnodeId, dropReq.fqdn, dropReq.port); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_MNODE) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_MNODE) != 0) { goto _OVER; } @@ -779,7 +779,7 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { } mInfo("dnode:%d, start to config, option:%s, value:%s", cfgReq.dnodeId, cfgReq.config, cfgReq.value); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CONFIG_DNODE) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONFIG_DNODE) != 0) { return -1; } diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index 37e0a719dd..b626c1fb04 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "mndFunc.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndShow.h" #include "mndSync.h" #include "mndTrans.h" @@ -283,7 +283,7 @@ static int32_t mndProcessCreateFuncReq(SRpcMsg *pReq) { } mDebug("func:%s, start to create", createReq.name); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_FUNC) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_FUNC) != 0) { goto _OVER; } @@ -346,7 +346,7 @@ static int32_t mndProcessDropFuncReq(SRpcMsg *pReq) { } mDebug("func:%s, start to drop", dropReq.name); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_FUNC) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_FUNC) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 9f78e21dd1..4c9974ba1a 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "mndAcct.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndBnode.h" #include "mndCluster.h" #include "mndConsumer.h" @@ -239,7 +239,7 @@ static int32_t mndInitSteps(SMnode *pMnode) { if (mndAllocStep(pMnode, "mnode-dnode", mndInitDnode, mndCleanupDnode) != 0) return -1; if (mndAllocStep(pMnode, "mnode-user", mndInitUser, mndCleanupUser) != 0) return -1; if (mndAllocStep(pMnode, "mnode-grant", mndInitGrant, mndCleanupGrant) != 0) return -1; - if (mndAllocStep(pMnode, "mnode-auth", mndInitAuth, mndCleanupAuth) != 0) return -1; + if (mndAllocStep(pMnode, "mnode-privilege", mndInitPrivilege, mndCleanupPrivilege) != 0) return -1; if (mndAllocStep(pMnode, "mnode-acct", mndInitAcct, mndCleanupAcct) != 0) return -1; if (mndAllocStep(pMnode, "mnode-stream", mndInitStream, mndCleanupStream) != 0) return -1; if (mndAllocStep(pMnode, "mnode-topic", mndInitTopic, mndCleanupTopic) != 0) return -1; diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index bc3d23282c..c03951b1d8 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "mndMnode.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndDnode.h" #include "mndShow.h" #include "mndSync.h" @@ -389,7 +389,7 @@ static int32_t mndProcessCreateMnodeReq(SRpcMsg *pReq) { } mDebug("mnode:%d, start to create", createReq.dnodeId); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_MNODE) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_MNODE) != 0) { goto _OVER; } @@ -594,7 +594,7 @@ static int32_t mndProcessDropMnodeReq(SRpcMsg *pReq) { } mDebug("mnode:%d, start to drop", dropReq.dnodeId); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_MNODE) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_MNODE) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndOffset.c b/source/dnode/mnode/impl/src/mndOffset.c index 18f2e993b2..e2b20b2163 100644 --- a/source/dnode/mnode/impl/src/mndOffset.c +++ b/source/dnode/mnode/impl/src/mndOffset.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "mndOffset.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" @@ -36,13 +36,15 @@ static int32_t mndOffsetActionUpdate(SSdb *pSdb, SMqOffsetObj *pOffset, SMqOffse static int32_t mndProcessCommitOffsetReq(SRpcMsg *pReq); int32_t mndInitOffset(SMnode *pMnode) { - SSdbTable table = {.sdbType = SDB_OFFSET, - .keyType = SDB_KEY_BINARY, - .encodeFp = (SdbEncodeFp)mndOffsetActionEncode, - .decodeFp = (SdbDecodeFp)mndOffsetActionDecode, - .insertFp = (SdbInsertFp)mndOffsetActionInsert, - .updateFp = (SdbUpdateFp)mndOffsetActionUpdate, - .deleteFp = (SdbDeleteFp)mndOffsetActionDelete}; + SSdbTable table = { + .sdbType = SDB_OFFSET, + .keyType = SDB_KEY_BINARY, + .encodeFp = (SdbEncodeFp)mndOffsetActionEncode, + .decodeFp = (SdbDecodeFp)mndOffsetActionDecode, + .insertFp = (SdbInsertFp)mndOffsetActionInsert, + .updateFp = (SdbUpdateFp)mndOffsetActionUpdate, + .deleteFp = (SdbDeleteFp)mndOffsetActionDelete, + }; mndSetMsgHandle(pMnode, TDMT_MND_MQ_COMMIT_OFFSET, mndProcessCommitOffsetReq); diff --git a/source/dnode/mnode/impl/src/mndAuth.c b/source/dnode/mnode/impl/src/mndPrivilege.c similarity index 61% rename from source/dnode/mnode/impl/src/mndAuth.c rename to source/dnode/mnode/impl/src/mndPrivilege.c index 4445e3b9f7..752b11540d 100644 --- a/source/dnode/mnode/impl/src/mndAuth.c +++ b/source/dnode/mnode/impl/src/mndPrivilege.c @@ -14,66 +14,15 @@ */ #define _DEFAULT_SOURCE -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndUser.h" +#include "mndDb.h" -static int32_t mndProcessAuthReq(SRpcMsg *pReq); +int32_t mndInitPrivilege(SMnode *pMnode) { return 0; } -int32_t mndInitAuth(SMnode *pMnode) { - mndSetMsgHandle(pMnode, TDMT_MND_AUTH, mndProcessAuthReq); - return 0; -} +void mndCleanupPrivilege(SMnode *pMnode) {} -void mndCleanupAuth(SMnode *pMnode) {} - -static int32_t mndRetriveAuth(SMnode *pMnode, SAuthRsp *pRsp) { - SUserObj *pUser = mndAcquireUser(pMnode, pRsp->user); - if (pUser == NULL) { - *pRsp->secret = 0; - mError("user:%s, failed to auth user since %s", pRsp->user, terrstr()); - return -1; - } - - pRsp->spi = 1; - pRsp->encrypt = 0; - *pRsp->ckey = 0; - - memcpy(pRsp->secret, pUser->pass, TSDB_PASSWORD_LEN); - mndReleaseUser(pMnode, pUser); - - mDebug("user:%s, auth info is returned", pRsp->user); - return 0; -} - -static int32_t mndProcessAuthReq(SRpcMsg *pReq) { - SAuthReq authReq = {0}; - if (tDeserializeSAuthReq(pReq->pCont, pReq->contLen, &authReq) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return -1; - } - - SAuthReq authRsp = {0}; - memcpy(authRsp.user, authReq.user, TSDB_USER_LEN); - - int32_t code = mndRetriveAuth(pReq->info.node, &authRsp); - mTrace("user:%s, auth req received, spi:%d encrypt:%d ruser:%s", pReq->info.conn.user, authRsp.spi, authRsp.encrypt, - authRsp.user); - - int32_t contLen = tSerializeSAuthReq(NULL, 0, &authRsp); - void *pRsp = rpcMallocCont(contLen); - if (pRsp == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - tSerializeSAuthReq(pRsp, contLen, &authRsp); - - pReq->info.rsp = pRsp; - pReq->info.rspLen = contLen; - return code; -} - -int32_t mndCheckOperAuth(SMnode *pMnode, const char *user, EOperType operType) { +int32_t mndCheckOperPrivilege(SMnode *pMnode, const char *user, EOperType operType) { int32_t code = 0; SUserObj *pUser = mndAcquireUser(pMnode, user); @@ -95,6 +44,8 @@ int32_t mndCheckOperAuth(SMnode *pMnode, const char *user, EOperType operType) { switch (operType) { case MND_OPER_CONNECT: + case MND_OPER_CREATE_FUNC: + case MND_OPER_DROP_FUNC: break; default: terrno = TSDB_CODE_MND_NO_RIGHTS; @@ -106,7 +57,7 @@ _OVER: return code; } -int32_t mndCheckAlterUserAuth(SUserObj *pOperUser, SUserObj *pUser, SAlterUserReq *pAlter) { +int32_t mndCheckAlterUserPrivilege(SUserObj *pOperUser, SUserObj *pUser, SAlterUserReq *pAlter) { if (pUser->superUser && pAlter->alterType != TSDB_ALTER_USER_PASSWD) { terrno = TSDB_CODE_MND_NO_RIGHTS; return -1; @@ -129,7 +80,7 @@ int32_t mndCheckAlterUserAuth(SUserObj *pOperUser, SUserObj *pUser, SAlterUserRe return -1; } -int32_t mndCheckShowAuth(SMnode *pMnode, const char *user, int32_t showType) { +int32_t mndCheckShowPrivilege(SMnode *pMnode, const char *user, int32_t showType) { int32_t code = 0; SUserObj *pUser = mndAcquireUser(pMnode, user); @@ -162,7 +113,7 @@ _OVER: return code; } -int32_t mndCheckDbAuth(SMnode *pMnode, const char *user, EOperType operType, SDbObj *pDb) { +int32_t mndCheckDbPrivilege(SMnode *pMnode, const char *user, EOperType operType, SDbObj *pDb) { int32_t code = 0; SUserObj *pUser = mndAcquireUser(pMnode, user); @@ -183,15 +134,7 @@ int32_t mndCheckDbAuth(SMnode *pMnode, const char *user, EOperType operType, SDb if (pUser->sysInfo) goto _OVER; } - if (operType == MND_OPER_ALTER_DB) { - if (strcmp(pUser->user, pDb->createUser) == 0 && pUser->sysInfo) goto _OVER; - } - - if (operType == MND_OPER_DROP_DB) { - if (strcmp(pUser->user, pDb->createUser) == 0 && pUser->sysInfo) goto _OVER; - } - - if (operType == MND_OPER_COMPACT_DB) { + if (operType == MND_OPER_ALTER_DB || operType == MND_OPER_DROP_DB || operType == MND_OPER_COMPACT_DB) { if (strcmp(pUser->user, pDb->createUser) == 0 && pUser->sysInfo) goto _OVER; } @@ -218,3 +161,12 @@ _OVER: mndReleaseUser(pMnode, pUser); return code; } + +int32_t mndCheckDbPrivilegeByName(SMnode *pMnode, const char *user, EOperType operType, const char *name) { + SDbObj *pDb = mndAcquireDb(pMnode, name); + if (pDb == NULL) return -1; + + int32_t code = mndCheckDbPrivilege(pMnode, user, operType, pDb); + mndReleaseDb(pMnode, pDb); + return code; +} \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index e9df4ae1d0..f2e599b073 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "mndProfile.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" @@ -227,6 +227,10 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { } taosIp2String(pReq->info.conn.clientIp, ip); + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CONNECT) != 0) { + mGError("user:%s, failed to login from %s since %s", pReq->info.conn.user, ip, terrstr()); + goto _OVER; + } pUser = mndAcquireUser(pMnode, pReq->info.conn.user); if (pUser == NULL) { @@ -240,11 +244,6 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CONNECT) != 0) { - mGError("user:%s, failed to login from %s since %s", pReq->info.conn.user, ip, terrstr()); - goto _OVER; - } - if (connReq.db[0]) { char db[TSDB_DB_FNAME_LEN] = {0}; snprintf(db, TSDB_DB_FNAME_LEN, "%d%s%s", pUser->acctId, TS_PATH_DELIMITER, connReq.db); @@ -271,7 +270,7 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { connectRsp.connId = pConn->id; connectRsp.connType = connReq.connType; connectRsp.dnodeNum = mndGetDnodeSize(pMnode); - + strcpy(connectRsp.sVer, version); snprintf(connectRsp.sDetailVer, sizeof(connectRsp.sDetailVer), "ver:%s\nbuild:%s\ngitinfo:%s", version, buildinfo, gitinfo); @@ -475,16 +474,16 @@ static int32_t mndGetOnlineDnodeNum(SMnode *pMnode, int32_t *num) { SDnodeObj *pDnode = NULL; int64_t curMs = taosGetTimestampMs(); void *pIter = NULL; - + while (true) { pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pDnode); if (pIter == NULL) break; - + bool online = mndIsDnodeOnline(pDnode, curMs); if (online) { (*num)++; } - + sdbRelease(pSdb, pDnode); } @@ -652,15 +651,6 @@ static int32_t mndProcessKillQueryReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SProfileMgmt *pMgmt = &pMnode->profileMgmt; - SUserObj *pUser = mndAcquireUser(pMnode, pReq->info.conn.user); - if (pUser == NULL) return 0; - if (!pUser->superUser) { - mndReleaseUser(pMnode, pUser); - terrno = TSDB_CODE_MND_NO_RIGHTS; - return -1; - } - mndReleaseUser(pMnode, pUser); - SKillQueryReq killReq = {0}; if (tDeserializeSKillQueryReq(pReq->pCont, pReq->contLen, &killReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; @@ -668,6 +658,10 @@ static int32_t mndProcessKillQueryReq(SRpcMsg *pReq) { } mInfo("kill query msg is received, queryId:%s", killReq.queryStrId); + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_KILL_QUERY) != 0) { + return -1; + } + int32_t connId = 0; uint64_t queryId = 0; char *p = strchr(killReq.queryStrId, ':'); @@ -697,21 +691,16 @@ static int32_t mndProcessKillConnReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SProfileMgmt *pMgmt = &pMnode->profileMgmt; - SUserObj *pUser = mndAcquireUser(pMnode, pReq->info.conn.user); - if (pUser == NULL) return 0; - if (!pUser->superUser) { - mndReleaseUser(pMnode, pUser); - terrno = TSDB_CODE_MND_NO_RIGHTS; - return -1; - } - mndReleaseUser(pMnode, pUser); - SKillConnReq killReq = {0}; if (tDeserializeSKillConnReq(pReq->pCont, pReq->contLen, &killReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; return -1; } + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_KILL_CONN) != 0) { + return -1; + } + SConnObj *pConn = taosCacheAcquireByKey(pMgmt->connCache, &killReq.connId, sizeof(uint32_t)); if (pConn == NULL) { mError("connId:%u, failed to kill connection, conn not exist", killReq.connId); @@ -726,10 +715,10 @@ static int32_t mndProcessKillConnReq(SRpcMsg *pReq) { } static int32_t mndProcessSvrVerReq(SRpcMsg *pReq) { - int32_t code = -1; + int32_t code = -1; SServerVerRsp rsp = {0}; strcpy(rsp.ver, version); - + int32_t contLen = tSerializeSServerVerRsp(NULL, 0, &rsp); if (contLen < 0) goto _over; void *pRsp = rpcMallocCont(contLen); @@ -746,7 +735,6 @@ _over: return code; } - static int32_t mndRetrieveConns(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { SMnode *pMnode = pReq->info.node; SSdb *pSdb = pMnode->pSdb; diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index 9f1eb4ee24..f057f6190d 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "mndQnode.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndDnode.h" #include "mndShow.h" #include "mndTrans.h" @@ -279,7 +279,7 @@ static int32_t mndProcessCreateQnodeReq(SRpcMsg *pReq) { } mDebug("qnode:%d, start to create", createReq.dnodeId); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_QNODE) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_QNODE) != 0) { goto _OVER; } @@ -390,7 +390,7 @@ static int32_t mndProcessDropQnodeReq(SRpcMsg *pReq) { } mDebug("qnode:%d, start to drop", dropReq.dnodeId); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_QNODE) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_QNODE) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 27de3883e9..3351cff3a3 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "mndShow.h" #include "systable.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #define SHOW_STEP_SIZE 100 @@ -122,6 +122,7 @@ static SShowObj *mndCreateShowObj(SMnode *pMnode, SRetrieveTableReq *pReq) { int32_t size = sizeof(SShowObj); SShowObj showObj = {0}; + showObj.id = showId; showObj.pMnode = pMnode; showObj.type = convertToRetrieveType(pReq->tb, tListLen(pReq->tb)); @@ -231,7 +232,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { mDebug("show:0x%" PRIx64 ", start retrieve data, type:%d", pShow->id, pShow->type); - // if (mndCheckShowAuth(pMnode, pReq->info.conn.user, pShow->type) != 0) return -1; + // if (mndCheckShowPrivilege(pMnode, pReq->info.conn.user, pShow->type) != 0) return -1; int32_t numOfCols = pShow->pMeta->numOfColumns; SSDataBlock *pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); diff --git a/source/dnode/mnode/impl/src/mndSma.c b/source/dnode/mnode/impl/src/mndSma.c index 3dadcf77f0..d880fb28d4 100644 --- a/source/dnode/mnode/impl/src/mndSma.c +++ b/source/dnode/mnode/impl/src/mndSma.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "mndSma.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndDb.h" #include "mndDnode.h" #include "mndInfoSchema.h" @@ -713,7 +713,7 @@ static int32_t mndProcessCreateSmaReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { goto _OVER; } @@ -992,7 +992,7 @@ static int32_t mndProcessDropSmaReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index a638bdf61f..2dd8592bf8 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "mndSnode.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndDnode.h" #include "mndShow.h" #include "mndTrans.h" @@ -285,7 +285,7 @@ static int32_t mndProcessCreateSnodeReq(SRpcMsg *pReq) { } mDebug("snode:%d, start to create", createReq.dnodeId); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_SNODE) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_SNODE) != 0) { goto _OVER; } @@ -397,7 +397,7 @@ static int32_t mndProcessDropSnodeReq(SRpcMsg *pReq) { } mDebug("snode:%d, start to drop", dropReq.dnodeId); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_SNODE) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_SNODE) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index f1bae14c07..77b13cd82d 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "mndStb.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndDb.h" #include "mndDnode.h" #include "mndInfoSchema.h" @@ -876,7 +876,7 @@ static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { goto _OVER; } @@ -1607,7 +1607,7 @@ static int32_t mndProcessAlterStbReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { goto _OVER; } @@ -1737,7 +1737,7 @@ static int32_t mndProcessDropStbReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index 29c6819163..444a8864ef 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -14,7 +14,7 @@ */ #include "mndStream.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" @@ -437,10 +437,6 @@ static int32_t mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStre goto _OVER; } - if (mndCheckDbAuth(pMnode, user, MND_OPER_WRITE_DB, pDb) != 0) { - goto _OVER; - } - int32_t numOfStbs = -1; if (mndGetNumOfStbs(pMnode, pDb->name, &numOfStbs) != 0) { goto _OVER; @@ -542,19 +538,6 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { goto _OVER; } - // TODO check read auth for source and write auth for target -#if 0 - pDb = mndAcquireDb(pMnode, createStreamReq.sourceDB); - if (pDb == NULL) { - terrno = TSDB_CODE_MND_DB_NOT_SELECTED; - goto _OVER; - } - - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { - goto _OVER; - } -#endif - // build stream obj from request SStreamObj streamObj = {0}; if (mndBuildStreamObjFromCreateReq(pMnode, &streamObj, &createStreamReq) < 0) { @@ -592,6 +575,16 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) { goto _OVER; } + if (mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_READ_DB, streamObj.sourceDb) != 0) { + mndTransDrop(pTrans); + goto _OVER; + } + + if (mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, streamObj.targetDb) != 0) { + mndTransDrop(pTrans); + goto _OVER; + } + // execute creation if (mndTransPrepare(pMnode, pTrans) != 0) { mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); @@ -641,13 +634,9 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { } } -#if 0 - // todo check auth - pUser = mndAcquireUser(pMnode, pReq->info.conn.user); - if (pUser == NULL) { - goto DROP_STREAM_OVER; + if (mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pStream->targetDb) != 0) { + return -1; } -#endif STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq); if (pTrans == NULL) { diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 8fcd345544..8d1220d596 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -14,12 +14,12 @@ */ #include "mndTopic.h" -#include "mndAuth.h" #include "mndConsumer.h" #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" #include "mndOffset.h" +#include "mndPrivilege.h" #include "mndShow.h" #include "mndStb.h" #include "mndSubscribe.h" @@ -484,7 +484,7 @@ static int32_t mndProcessCreateTopicReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckDbAuth(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pDb) != 0) { + if (mndCheckDbPrivilege(pMnode, pReq->info.conn.user, MND_OPER_READ_DB, pDb) != 0) { goto _OVER; } @@ -575,6 +575,10 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { } #endif + if (mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_READ_DB, pTopic->db) != 0) { + return -1; + } + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_DB_INSIDE, pReq); mndTransSetDbName(pTrans, pTopic->db, NULL); if (pTrans == NULL) { diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index d1d88fdc90..a9de1a05a7 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "mndTrans.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndConsumer.h" #include "mndDb.h" #include "mndShow.h" @@ -1384,8 +1384,7 @@ static int32_t mndProcessKillTransReq(SRpcMsg *pReq) { } mInfo("trans:%d, start to kill", killReq.transId); - - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_KILL_TRANS) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_KILL_TRANS) != 0) { goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 03c9647bfe..921dba422d 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "mndUser.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndDb.h" #include "mndShow.h" #include "mndTrans.h" @@ -295,7 +295,7 @@ static int32_t mndCreateUser(SMnode *pMnode, char *acct, SCreateUserReq *pCreate tstrncpy(userObj.acct, acct, TSDB_USER_LEN); userObj.createdTime = taosGetTimestampMs(); userObj.updateTime = userObj.createdTime; - userObj.superUser = 0;//pCreate->superUser; + userObj.superUser = 0; // pCreate->superUser; userObj.sysInfo = pCreate->sysInfo; userObj.enable = pCreate->enable; @@ -337,6 +337,9 @@ static int32_t mndProcessCreateUserReq(SRpcMsg *pReq) { } mDebug("user:%s, start to create", createReq.user); + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_CREATE_USER) != 0) { + goto _OVER; + } if (createReq.user[0] == 0) { terrno = TSDB_CODE_MND_INVALID_USER_FORMAT; @@ -360,10 +363,6 @@ static int32_t mndProcessCreateUserReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_CREATE_USER) != 0) { - goto _OVER; - } - code = mndCreateUser(pMnode, pOperUser->acct, &createReq, pReq); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; @@ -466,7 +465,7 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckAlterUserAuth(pOperUser, pUser, &alterReq) != 0) { + if (mndCheckAlterUserPrivilege(pOperUser, pUser, &alterReq) != 0) { goto _OVER; } @@ -631,6 +630,9 @@ static int32_t mndProcessDropUserReq(SRpcMsg *pReq) { } mDebug("user:%s, start to drop", dropReq.user); + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_DROP_USER) != 0) { + goto _OVER; + } if (dropReq.user[0] == 0) { terrno = TSDB_CODE_MND_INVALID_USER_FORMAT; @@ -643,10 +645,6 @@ static int32_t mndProcessDropUserReq(SRpcMsg *pReq) { goto _OVER; } - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_DROP_USER) != 0) { - goto _OVER; - } - code = mndDropUser(pMnode, pReq, pUser); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 1c5d73031f..0e931e0a9c 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -15,7 +15,7 @@ #define _DEFAULT_SOURCE #include "mndVgroup.h" -#include "mndAuth.h" +#include "mndPrivilege.h" #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" @@ -1212,7 +1212,7 @@ static int32_t mndProcessRedistributeVgroupMsg(SRpcMsg *pReq) { } mInfo("vgId:%d, start to redistribute vgroup to dnode %d:%d:%d", req.vgId, req.dnodeId1, req.dnodeId2, req.dnodeId3); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_REDISTRIBUTE_VGROUP) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_REDISTRIBUTE_VGROUP) != 0) { goto _OVER; } @@ -1507,7 +1507,7 @@ static int32_t mndProcessSplitVgroupMsg(SRpcMsg *pReq) { SDbObj *pDb = NULL; mDebug("vgId:%d, start to split", vgId); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_SPLIT_VGROUP) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_SPLIT_VGROUP) != 0) { goto _OVER; } @@ -1657,7 +1657,7 @@ static int32_t mndProcessBalanceVgroupMsg(SRpcMsg *pReq) { } mInfo("start to balance vgroup"); - if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_BALANCE_VGROUP) != 0) { + if (mndCheckOperPrivilege(pMnode, pReq->info.conn.user, MND_OPER_BALANCE_VGROUP) != 0) { goto _OVER; } diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 3a92cba773..b13e654caf 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -257,7 +257,7 @@ int32_t sndProcessSMsg(SSnode *pSnode, SRpcMsg *pMsg) { case TDMT_STREAM_TASK_RECOVER_RSP: return sndProcessTaskRecoverRsp(pSnode, pMsg); case TDMT_STREAM_RETRIEVE_RSP: - return sndProcessTaskRecoverRsp(pSnode, pMsg); + return sndProcessTaskRetrieveRsp(pSnode, pMsg); default: ASSERT(0); } diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index a32bf0ecdb..b97d4605e7 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -196,6 +196,7 @@ struct SVnodeCfg { typedef struct { TSKEY lastKey; uint64_t uid; + uint64_t groupId; } STableKeyInfo; struct SMetaEntry { diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 4a3feca8d0..7109bf1dfc 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -388,7 +388,7 @@ int metaTtlDropTable(SMeta *pMeta, int64_t ttl, SArray *tbUids) { } static void metaBuildTtlIdxKey(STtlIdxKey *ttlKey, const SMetaEntry *pME){ - int32_t ttlDays; + int64_t ttlDays; int64_t ctime; if (pME->type == TSDB_CHILD_TABLE) { ctime = pME->ctbEntry.ctime; diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 6184d8f810..96f4eb3fd9 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -109,11 +109,15 @@ int32_t tqReadHandleSetMsg(STqReadHandle* pReadHandle, SSubmitReq* pMsg, int64_t } bool tqNextDataBlock(STqReadHandle* pHandle) { + if (pHandle->pMsg == NULL) return false; while (1) { if (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) < 0) { return false; } - if (pHandle->pBlock == NULL) return false; + if (pHandle->pBlock == NULL) { + pHandle->pMsg = NULL; + return false; + } if (pHandle->tbIdHash == NULL) { return true; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 540810f876..ab31d65d68 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2852,7 +2852,7 @@ int32_t tsdbGetAllTableList(SMeta* pMeta, uint64_t uid, SArray* list) { break; } - STableKeyInfo info = {.lastKey = TSKEY_INITIAL_VAL, uid = id}; + STableKeyInfo info = {.lastKey = TSKEY_INITIAL_VAL, uid = id, .groupId = 0}; taosArrayPush(list, &info); } diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 0f3251515e..eb2c6f5102 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -293,6 +293,7 @@ typedef enum EStreamScanMode { STREAM_SCAN_FROM_RES, STREAM_SCAN_FROM_UPDATERES, STREAM_SCAN_FROM_DATAREADER, + STREAM_SCAN_FROM_DATAREADER_RETRIEVE, } EStreamScanMode; typedef struct SCatchSupporter { @@ -347,7 +348,9 @@ typedef struct SStreamBlockScanInfo { SInterval interval; // if the upstream is an interval operator, the interval info is also kept here. SArray* childIds; SessionWindowSupporter sessionSup; - int32_t scanWinIndex; + int32_t scanWinIndex; // for state operator + int32_t pullDataResIndex; + SSDataBlock* pPullDataRes; // pull data SSDataBlock } SStreamBlockScanInfo; typedef struct SSysTableScanInfo { @@ -357,6 +360,7 @@ typedef struct SSysTableScanInfo { tsem_t ready; SReadHandle readHandle; int32_t accountId; + const char* pUser; bool showRewrite; SNode* pCondition; // db_name filter condition, to discard data that are not in current database SMTbCursor* pCur; // cursor for iterate the local table meta store. @@ -425,8 +429,13 @@ typedef struct SStreamFinalIntervalOperatorInfo { STimeWindowAggSupp twAggSup; SArray* pChildren; SSDataBlock* pUpdateRes; + bool returnUpdate; SPhysiNode* pPhyNode; // create new child bool isFinal; + SHashObj* pPullDataMap; + SArray* pPullWins; // SPullWindowInfo + int32_t pullIndex; + SSDataBlock* pPullDataRes; } SStreamFinalIntervalOperatorInfo; typedef struct SAggOperatorInfo { @@ -712,7 +721,7 @@ SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, SExecTaskInfo* pTaskInfo, uint64_t queryId, uint64_t taskId); SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode *pScanPhyNode, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode *pScanPhyNode, const char* pUser, SExecTaskInfo* pTaskInfo); SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SExprInfo* pScalarExprInfo, int32_t numOfScalarExpr, SExecTaskInfo* pTaskInfo); @@ -849,6 +858,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex); int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* groupKey); +SSDataBlock* createPullDataBlock(); #ifdef __cplusplus } diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 5ac5957f2b..374a3a736d 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -315,7 +315,7 @@ int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo } for (int i = 0; i < taosArrayGetSize(res); i++) { - STableKeyInfo info = {.lastKey = TSKEY_INITIAL_VAL, .uid = *(uint64_t*)taosArrayGet(res, i)}; + STableKeyInfo info = {.lastKey = TSKEY_INITIAL_VAL, .uid = *(uint64_t*)taosArrayGet(res, i), .groupId = 0}; taosArrayPush(pListInfo->pTableList, &info); } taosArrayDestroy(res); @@ -336,7 +336,7 @@ int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo } } }else { // Create one table group. - STableKeyInfo info = {.lastKey = 0, .uid = tableUid}; + STableKeyInfo info = {.lastKey = 0, .uid = tableUid, .groupId = 0}; taosArrayPush(pListInfo->pTableList, &info); } pListInfo->pGroupList = taosArrayInit(4, POINTER_BYTES); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index c23d9a5040..f18da72f58 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -4028,6 +4028,7 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, int32_t len = (int32_t)(pStart - (char*)keyBuf); uint64_t groupId = calcGroupId(keyBuf, len); taosHashPut(pTableListInfo->map, &(info->uid), sizeof(uint64_t), &groupId, sizeof(uint64_t)); + info->groupId = groupId; groupNum++; nodesClearList(groupNew); @@ -4043,7 +4044,7 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, } SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, - uint64_t queryId, uint64_t taskId, STableListInfo* pTableListInfo) { + uint64_t queryId, uint64_t taskId, STableListInfo* pTableListInfo, const char* pUser) { int32_t type = nodeType(pPhyNode); if (pPhyNode->pChildren == NULL || LIST_LENGTH(pPhyNode->pChildren) == 0) { @@ -4103,8 +4104,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) { SSystemTableScanPhysiNode* pSysScanPhyNode = (SSystemTableScanPhysiNode*)pPhyNode; - return createSysTableScanOperatorInfo(pHandle, pSysScanPhyNode, pTaskInfo); - + return createSysTableScanOperatorInfo(pHandle, pSysScanPhyNode, pUser, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN == type) { STagScanPhysiNode* pScanPhyNode = (STagScanPhysiNode*)pPhyNode; @@ -4127,7 +4127,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo return NULL; } } else { // Create one table group. - STableKeyInfo info = {.lastKey = 0, .uid = pBlockNode->uid}; + STableKeyInfo info = {.lastKey = 0, .uid = pBlockNode->uid, .groupId = 0}; taosArrayPush(pTableListInfo->pTableList, &info); } @@ -4167,7 +4167,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SOperatorInfo** ops = taosMemoryCalloc(size, POINTER_BYTES); for (int32_t i = 0; i < size; ++i) { SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, i); - ops[i] = createOperatorTree(pChildNode, pTaskInfo, pHandle, queryId, taskId, pTableListInfo); + ops[i] = createOperatorTree(pChildNode, pTaskInfo, pHandle, queryId, taskId, pTableListInfo, pUser); if (ops[i] == NULL) { return NULL; } @@ -4600,7 +4600,7 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead (*pTaskInfo)->tableqinfoList.pTagCond = pPlan->pTagCond; (*pTaskInfo)->tableqinfoList.pTagIndexCond = pPlan->pTagIndexCond; (*pTaskInfo)->pRoot = - createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId, &(*pTaskInfo)->tableqinfoList); + createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId, &(*pTaskInfo)->tableqinfoList, pPlan->user); if (NULL == (*pTaskInfo)->pRoot) { code = (*pTaskInfo)->code; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 0c73ec72a5..aadf7b3b7e 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -507,21 +507,20 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { STableScanInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - if (pInfo->currentGroupId == -1) { + if(pInfo->currentGroupId == -1){ pInfo->currentGroupId++; if (pInfo->currentGroupId >= taosArrayGetSize(pTaskInfo->tableqinfoList.pGroupList)) { setTaskStatus(pTaskInfo, TASK_COMPLETED); return NULL; } - SArray* tableList = taosArrayGetP(pTaskInfo->tableqinfoList.pGroupList, pInfo->currentGroupId); + SArray *tableList = taosArrayGetP(pTaskInfo->tableqinfoList.pGroupList, pInfo->currentGroupId); tsdbCleanupReadHandle(pInfo->dataReader); - tsdbReaderT* pReader = - tsdbReaderOpen(pInfo->readHandle.vnode, &pInfo->cond, tableList, pInfo->queryId, pInfo->taskId); + tsdbReaderT* pReader = tsdbReaderOpen(pInfo->readHandle.vnode, &pInfo->cond, tableList, pInfo->queryId, pInfo->taskId); pInfo->dataReader = pReader; } SSDataBlock* result = doTableScanGroup(pOperator); - if (result) { + if(result){ return result; } @@ -531,7 +530,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { return NULL; } - SArray* tableList = taosArrayGetP(pTaskInfo->tableqinfoList.pGroupList, pInfo->currentGroupId); + SArray *tableList = taosArrayGetP(pTaskInfo->tableqinfoList.pGroupList, pInfo->currentGroupId); tsdbSetTableList(pInfo->dataReader, tableList); tsdbResetReadHandle(pInfo->dataReader, &pInfo->cond, 0); @@ -539,7 +538,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { pInfo->scanTimes = 0; result = doTableScanGroup(pOperator); - if (result) { + if(result){ return result; } @@ -777,15 +776,14 @@ static bool isStateWindow(SStreamBlockScanInfo* pInfo) { return pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE; } -static bool prepareDataScan(SStreamBlockScanInfo* pInfo) { - SSDataBlock* pSDB = pInfo->pUpdateRes; +static bool prepareDataScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pSDB, int32_t tsColIndex, int32_t* pRowIndex) { STimeWindow win = { .skey = INT64_MIN, .ekey = INT64_MAX, }; bool needRead = false; - if (!isStateWindow(pInfo) && pInfo->updateResIndex < pSDB->info.rows) { - SColumnInfoData* pColDataInfo = taosArrayGet(pSDB->pDataBlock, pInfo->primaryTsIndex); + if (!isStateWindow(pInfo) && (*pRowIndex) < pSDB->info.rows) { + SColumnInfoData* pColDataInfo = taosArrayGet(pSDB->pDataBlock, tsColIndex); TSKEY* tsCols = (TSKEY*)pColDataInfo->pData; SResultRowInfo dumyInfo; dumyInfo.cur.pageId = -1; @@ -794,14 +792,14 @@ static bool prepareDataScan(SStreamBlockScanInfo* pInfo) { int64_t gap = pInfo->sessionSup.gap; int32_t winIndex = 0; SResultWindowInfo* pCurWin = - getSessionTimeWindow(pAggSup, tsCols[pInfo->updateResIndex], INT64_MIN, pSDB->info.groupId, gap, &winIndex); + getSessionTimeWindow(pAggSup, tsCols[(*pRowIndex)], INT64_MIN, pSDB->info.groupId, gap, &winIndex); win = pCurWin->win; - pInfo->updateResIndex += - updateSessionWindowInfo(pCurWin, tsCols, NULL, pSDB->info.rows, pInfo->updateResIndex, gap, NULL); + (*pRowIndex) += + updateSessionWindowInfo(pCurWin, tsCols, NULL, pSDB->info.rows, (*pRowIndex), gap, NULL); } else { - win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[pInfo->updateResIndex], &pInfo->interval, + win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[(*pRowIndex)], &pInfo->interval, pInfo->interval.precision, NULL); - pInfo->updateResIndex += getNumOfRowsInTimeWindow(&pSDB->info, tsCols, pInfo->updateResIndex, win.ekey, + (*pRowIndex) += getNumOfRowsInTimeWindow(&pSDB->info, tsCols, (*pRowIndex), win.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); } needRead = true; @@ -823,7 +821,10 @@ static bool prepareDataScan(SStreamBlockScanInfo* pInfo) { STableScanInfo* pTableScanInfo = pInfo->pSnapshotReadOp->info; pTableScanInfo->cond.twindows[0] = win; pTableScanInfo->curTWinIdx = 0; - // tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0); +// tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0); + // if (!pTableScanInfo->dataReader) { + // return false; + // } pTableScanInfo->scanTimes = 0; pTableScanInfo->currentGroupId = -1; return true; @@ -863,12 +864,12 @@ static uint64_t getGroupId(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_ */ } -static SSDataBlock* doDataScan(SStreamBlockScanInfo* pInfo) { +static SSDataBlock* doDataScan(SStreamBlockScanInfo* pInfo, SSDataBlock* pSDB, int32_t tsColIndex, int32_t* pRowIndex) { while (1) { SSDataBlock* pResult = NULL; pResult = doTableScan(pInfo->pSnapshotReadOp); if (pResult == NULL) { - if (prepareDataScan(pInfo)) { + if (prepareDataScan(pInfo, pSDB, tsColIndex, pRowIndex)) { // scan next window data pResult = doTableScan(pInfo->pSnapshotReadOp); } @@ -917,7 +918,7 @@ static void setUpdateData(SStreamBlockScanInfo* pInfo, SSDataBlock* pBlock, SSDa pUpdateBlock->info.rows = i; pInfo->tsArrayIndex += i; pUpdateBlock->info.groupId = pInfo->groupId; - pUpdateBlock->info.type = STREAM_REPROCESS; + pUpdateBlock->info.type = STREAM_CLEAR; blockDataUpdateTsWindow(pUpdateBlock, 0); } // all rows have same group id @@ -971,6 +972,14 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { int32_t current = pInfo->validBlockIndex++; SSDataBlock* pBlock = taosArrayGetP(pInfo->pBlockLists, current); blockDataUpdateTsWindow(pBlock, 0); + if (pBlock->info.type == STREAM_RETRIEVE) { + pInfo->blockType = STREAM_DATA_TYPE_SUBMIT_BLOCK; + pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RETRIEVE; + copyDataBlock(pInfo->pPullDataRes, pBlock); + pInfo->pullDataResIndex = 0; + prepareDataScan(pInfo, pInfo->pPullDataRes, 0, &pInfo->pullDataResIndex); + updateInfoAddCloseWindowSBF(pInfo->pUpdateInfo); + } return pBlock; } else if (pInfo->blockType == STREAM_DATA_TYPE_SUBMIT_BLOCK) { if (pInfo->scanMode == STREAM_SCAN_FROM_RES) { @@ -980,28 +989,39 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { } else if (pInfo->scanMode == STREAM_SCAN_FROM_UPDATERES) { pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER; if (!isStateWindow(pInfo)) { - prepareDataScan(pInfo); + prepareDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex); } return pInfo->pUpdateRes; + } else if (pInfo->scanMode == STREAM_SCAN_FROM_DATAREADER_RETRIEVE) { + SSDataBlock* pSDB = doDataScan(pInfo, pInfo->pPullDataRes, 0, &pInfo->pullDataResIndex); + if (pSDB != NULL) { + getUpdateDataBlock(pInfo, true, pSDB, NULL); + pSDB->info.type = STREAM_PUSH_DATA; + return pSDB; + } + pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER; } else { if (isStateWindow(pInfo) && taosArrayGetSize(pInfo->sessionSup.pStreamAggSup->pScanWindow) > 0) { pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER; pInfo->updateResIndex = pInfo->pUpdateRes->info.rows; - prepareDataScan(pInfo); + prepareDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex); } if (pInfo->scanMode == STREAM_SCAN_FROM_DATAREADER) { - SSDataBlock* pSDB = doDataScan(pInfo); + SSDataBlock* pSDB = doDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex); if (pSDB == NULL) { setUpdateData(pInfo, pInfo->pRes, pInfo->pUpdateRes); if (pInfo->pUpdateRes->info.rows > 0) { if (!isStateWindow(pInfo)) { - prepareDataScan(pInfo); + // Todo(liuyao) mybe can delete this. + bool test = prepareDataScan(pInfo, pInfo->pUpdateRes, pInfo->primaryTsIndex, &pInfo->updateResIndex); + ASSERT(test == false); } return pInfo->pUpdateRes; } else { pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; } } else { + pSDB->info.type = STREAM_NORMAL; getUpdateDataBlock(pInfo, true, pSDB, NULL); return pSDB; } @@ -1060,15 +1080,15 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { } } - // TODO refactor @liao taosArrayDestroy(block.pDataBlock); - if (pInfo->pRes->pDataBlock == NULL) { // TODO add log + updateInfoDestoryColseWinSBF(pInfo->pUpdateInfo); pOperator->status = OP_EXEC_DONE; pTaskInfo->code = terrno; return NULL; } + // currently only the tbname pseudo column if (pInfo->numOfPseudoExpr > 0) { addTagPseudoColumnData(&pInfo->readHandle, pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, pInfo->pRes); @@ -1086,12 +1106,13 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { pOperator->resultInfo.totalRows += pBlockInfo->rows; if (pBlockInfo->rows == 0) { + updateInfoDestoryColseWinSBF(pInfo->pUpdateInfo); pOperator->status = OP_EXEC_DONE; } else if (pInfo->pUpdateInfo) { pInfo->tsArrayIndex = 0; getUpdateDataBlock(pInfo, true, pInfo->pRes, pInfo->pUpdateRes); if (pInfo->pUpdateRes->info.rows > 0) { - if (pInfo->pUpdateRes->info.type == STREAM_REPROCESS) { + if (pInfo->pUpdateRes->info.type == STREAM_CLEAR) { pInfo->updateResIndex = 0; pInfo->scanMode = STREAM_SCAN_FROM_UPDATERES; } else if (pInfo->pUpdateRes->info.type == STREAM_INVERT) { @@ -1125,9 +1146,9 @@ static SArray* extractTableIdList(const STableListInfo* pTableGroupInfo) { return tableIdList; } -SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, - SExecTaskInfo* pTaskInfo, STimeWindowAggSupp* pTwSup, uint64_t queryId, - uint64_t taskId) { +SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, + STableScanPhysiNode* pTableScanNode, SExecTaskInfo* pTaskInfo, + STimeWindowAggSupp* pTwSup, uint64_t queryId, uint64_t taskId) { SStreamBlockScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamBlockScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); @@ -1204,6 +1225,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; pInfo->sessionSup = (SessionWindowSupporter){.pStreamAggSup = NULL, .gap = -1}; pInfo->groupId = 0; + pInfo->pPullDataRes = createPullDataBlock(); pOperator->name = "StreamBlockScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN; @@ -1567,6 +1589,7 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { while (1) { int64_t startTs = taosGetTimestampUs(); strncpy(pInfo->req.tb, tNameGetTableName(&pInfo->name), tListLen(pInfo->req.tb)); + strcpy(pInfo->req.user, pInfo->pUser); if (pInfo->showRewrite) { char dbName[TSDB_DB_NAME_LEN] = {0}; @@ -1699,7 +1722,7 @@ int32_t buildDbTableInfoBlock(const SSDataBlock* p, const SSysTableMeta* pSysDbT } SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode* pScanPhyNode, - SExecTaskInfo* pTaskInfo) { + const char* pUser, SExecTaskInfo* pTaskInfo) { SSysTableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SSysTableScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -1712,13 +1735,14 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan SSDataBlock* pResBlock = createResDataBlock(pDescNode); int32_t num = 0; - SArray* colList = extractColMatchInfo(pScanNode->pScanCols, pDescNode, &num, COL_MATCH_FROM_COL_ID); + SArray* colList = extractColMatchInfo(pScanNode->pScanCols, pDescNode, &num, COL_MATCH_FROM_COL_ID); - pInfo->accountId = pScanPhyNode->accountId; + pInfo->accountId = pScanPhyNode->accountId; + pInfo->pUser = taosMemoryStrDup((void*) pUser); pInfo->showRewrite = pScanPhyNode->showRewrite; - pInfo->pRes = pResBlock; - pInfo->pCondition = pScanNode->node.pConditions; - pInfo->scanCols = colList; + pInfo->pRes = pResBlock; + pInfo->pCondition = pScanNode->node.pConditions; + pInfo->scanCols = colList; initResultSizeInfo(pOperator, 4096); @@ -1734,13 +1758,13 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan pInfo->readHandle = *(SReadHandle*)readHandle; } - pOperator->name = "SysTableScanOperator"; + pOperator->name = "SysTableScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; pOperator->exprSupp.numOfExprs = taosArrayGetSize(pResBlock->pDataBlock); - pOperator->pTaskInfo = pTaskInfo; + pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSysTableScan, NULL, NULL, destroySysScanOperator, NULL, NULL, NULL); @@ -1927,11 +1951,11 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi goto _error; } - pInfo->pTableList = pTableListInfo; - pInfo->pColMatchInfo = colList; - pInfo->pRes = createResDataBlock(pDescNode); - pInfo->readHandle = *pReadHandle; - pInfo->curPos = 0; + pInfo->pTableList = pTableListInfo; + pInfo->pColMatchInfo = colList; + pInfo->pRes = createResDataBlock(pDescNode); + pInfo->readHandle = *pReadHandle; + pInfo->curPos = 0; pOperator->name = "TagScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN; @@ -1958,7 +1982,10 @@ _error: typedef struct STableMergeScanInfo { STableListInfo* tableListInfo; - int32_t currentGroupId; + int32_t tableStartIndex; + int32_t tableEndIndex; + bool hasGroupId; + uint64_t groupId; SArray* dataReaders; // array of tsdbReaderT* SReadHandle readHandle; @@ -1999,7 +2026,7 @@ typedef struct STableMergeScanInfo { int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan int32_t dataBlockLoadFlag; SInterval interval; // if the upstream is an interval operator, the interval info is also kept here to get the time - // window to check if current data block needs to be loaded. + // window to check if current data block needs to be loaded. SSampleExecInfo sample; // sample execution info } STableMergeScanInfo; @@ -2023,6 +2050,22 @@ int32_t createScanTableListInfo(STableScanPhysiNode* pTableScanNode, SReadHandle return TSDB_CODE_SUCCESS; } +int32_t createMultipleDataReaders(SQueryTableDataCond* pQueryCond, SReadHandle* pHandle, STableListInfo* pTableListInfo, + int32_t tableStartIdx, int32_t tableEndIdx, SArray* arrayReader, uint64_t queryId, + uint64_t taskId) { + for (int32_t i = tableStartIdx; i <= tableEndIdx; ++i) { + SArray* subTableList = taosArrayInit(1, sizeof(STableKeyInfo)); + taosArrayPush(subTableList, taosArrayGet(pTableListInfo->pTableList, i)); + + tsdbReaderT* pReader = tsdbReaderOpen(pHandle->vnode, pQueryCond, subTableList, queryId, taskId); + taosArrayPush(arrayReader, &pReader); + + taosArrayDestroy(subTableList); + } + + return TSDB_CODE_SUCCESS; +} + // todo refactor static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeScanInfo* pTableScanInfo, int32_t readerIdx, SSDataBlock* pBlock, uint32_t* status) { @@ -2209,34 +2252,32 @@ SArray* generateSortByTsInfo(int32_t order) { return pList; } -static int32_t createMultipleDataReaders(SQueryTableDataCond* pQueryCond, SReadHandle* pHandle, SArray* tableList, - SArray* arrayReader, uint64_t queryId, uint64_t taskId) { - for (int32_t i = 0; i < taosArrayGetSize(tableList); ++i) { - SArray* tmp = taosArrayInit(1, sizeof(STableKeyInfo)); - taosArrayPush(tmp, taosArrayGet(tableList, i)); - - tsdbReaderT* pReader = tsdbReaderOpen(pHandle->vnode, pQueryCond, tmp, queryId, taskId); - taosArrayPush(arrayReader, &pReader); - - taosArrayDestroy(tmp); - } - - return TSDB_CODE_SUCCESS; -} - int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) { STableMergeScanInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SArray* tableList = taosArrayGetP(pInfo->tableListInfo->pGroupList, pInfo->currentGroupId); + { + size_t tableListSize = taosArrayGetSize(pInfo->tableListInfo->pTableList); + int32_t i = pInfo->tableStartIndex + 1; + for (; i < tableListSize; ++i) { + STableKeyInfo* tableKeyInfo = taosArrayGet(pInfo->tableListInfo->pTableList, i); + if (tableKeyInfo->groupId != pInfo->groupId) { + break; + } + } + pInfo->tableEndIndex = i - 1; + } - createMultipleDataReaders(&pInfo->cond, &pInfo->readHandle, tableList, pInfo->dataReaders, pInfo->queryId, - pInfo->taskId); + int32_t tableStartIdx = pInfo->tableStartIndex; + int32_t tableEndIdx = pInfo->tableEndIndex; + + STableListInfo* tableListInfo = pInfo->tableListInfo; + createMultipleDataReaders(&pInfo->cond, &pInfo->readHandle, tableListInfo, tableStartIdx, tableEndIdx, + pInfo->dataReaders, pInfo->queryId, pInfo->taskId); // todo the total available buffer should be determined by total capacity of buffer of this task. // the additional one is reserved for merge result - int32_t tableLen = taosArrayGetSize(tableList); - pInfo->sortBufSize = pInfo->bufPageSize * ((tableLen == 0 ? 1 : tableLen) + 1); + pInfo->sortBufSize = pInfo->bufPageSize * (tableEndIdx - tableStartIdx + 1 + 1); int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_MULTISOURCE_MERGE, pInfo->bufPageSize, numOfBufPage, pInfo->pSortInputBlock, pTaskInfo->id.str); @@ -2323,43 +2364,38 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) { if (code != TSDB_CODE_SUCCESS) { longjmp(pTaskInfo->env, code); } + size_t tableListSize = taosArrayGetSize(pInfo->tableListInfo->pTableList); + if (!pInfo->hasGroupId) { + pInfo->hasGroupId = true; - if (pInfo->currentGroupId == -1) { - pInfo->currentGroupId++; - if (pInfo->currentGroupId >= taosArrayGetSize(pInfo->tableListInfo->pGroupList)) { + if (tableListSize == 0) { doSetOperatorCompleted(pOperator); return NULL; } + pInfo->tableStartIndex = 0; + pInfo->groupId = ((STableKeyInfo*)taosArrayGet(pInfo->tableListInfo->pTableList, pInfo->tableStartIndex))->groupId; startGroupTableMergeScan(pOperator); } - SSDataBlock* pBlock = getSortedTableMergeScanBlockData(pInfo->pSortHandle, pOperator->resultInfo.capacity, pOperator); - if (pBlock != NULL) { - uint64_t* groupId = taosHashGet(pInfo->tableListInfo->map, &(pBlock->info.uid), sizeof(uint64_t)); - if (groupId) pBlock->info.groupId = *groupId; - - pOperator->resultInfo.totalRows += pBlock->info.rows; - return pBlock; + SSDataBlock* pBlock = NULL; + while (pInfo->tableStartIndex < tableListSize) { + pBlock = getSortedTableMergeScanBlockData(pInfo->pSortHandle, pOperator->resultInfo.capacity, pOperator); + if (pBlock != NULL) { + pBlock->info.groupId = pInfo->groupId; + pOperator->resultInfo.totalRows += pBlock->info.rows; + return pBlock; + } else { + stopGroupTableMergeScan(pOperator); + if (pInfo->tableEndIndex >= tableListSize - 1) { + doSetOperatorCompleted(pOperator); + break; + } + pInfo->tableStartIndex = pInfo->tableEndIndex + 1; + pInfo->groupId = + ((STableKeyInfo*)taosArrayGet(pInfo->tableListInfo->pTableList, pInfo->tableStartIndex))->groupId; + startGroupTableMergeScan(pOperator); + } } - stopGroupTableMergeScan(pOperator); - pInfo->currentGroupId++; - if (pInfo->currentGroupId >= taosArrayGetSize(pInfo->tableListInfo->pGroupList)) { - doSetOperatorCompleted(pOperator); - return NULL; - } - startGroupTableMergeScan(pOperator); - - pBlock = getSortedTableMergeScanBlockData(pInfo->pSortHandle, pOperator->resultInfo.capacity, pOperator); - if (pBlock != NULL) { - uint64_t* groupId = taosHashGet(pInfo->tableListInfo->map, &(pBlock->info.uid), sizeof(uint64_t)); - if (groupId) pBlock->info.groupId = *groupId; - - pOperator->resultInfo.totalRows += pBlock->info.rows; - return pBlock; - } - - doSetOperatorCompleted(pOperator); - return pBlock; } @@ -2396,6 +2432,12 @@ int32_t getTableMergeScanExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExpla return TSDB_CODE_SUCCESS; } +int32_t compareTableKeyInfoByGid(const void* p1, const void* p2) { + const STableKeyInfo* info1 = p1; + const STableKeyInfo* info2 = p2; + return info1->groupId - info2->groupId; +} + SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, STableListInfo* pTableListInfo, SReadHandle* readHandle, SExecTaskInfo* pTaskInfo, uint64_t queryId, uint64_t taskId) { @@ -2404,6 +2446,9 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN if (pInfo == NULL || pOperator == NULL) { goto _error; } + if (pTableScanNode->pPartitionTags) { + taosArraySort(pTableListInfo->pTableList, compareTableKeyInfoByGid); + } SDataBlockDescNode* pDescNode = pTableScanNode->scan.node.pOutputDataBlockDesc; @@ -2436,7 +2481,6 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN pInfo->dataReaders = taosArrayInit(64, POINTER_BYTES); pInfo->queryId = queryId; pInfo->taskId = taskId; - pInfo->currentGroupId = -1; pInfo->sortSourceParams = taosArrayInit(64, sizeof(STableMergeScanSortSourceParam)); diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 48b0b1c071..b7d39207d8 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -15,6 +15,7 @@ #include "executorimpl.h" #include "function.h" #include "functionMgt.h" +#include "tcompare.h" #include "tdatablock.h" #include "tfill.h" #include "ttime.h" @@ -26,6 +27,16 @@ typedef enum SResultTsInterpType { #define IS_FINAL_OP(op) ((op)->isFinal) +typedef struct SWinRes { + TSKEY ts; + uint64_t groupId; +} SWinRes; + +typedef struct SPullWindowInfo { + STimeWindow window; + uint64_t groupId; +} SPullWindowInfo; + static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator); static int64_t* extractTsCol(SSDataBlock* pBlock, const SIntervalAggOperatorInfo* pInfo); @@ -684,11 +695,13 @@ static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t num } void printDataBlock(SSDataBlock* pBlock, const char* flag) { - if (pBlock == NULL) return; - SArray* blocks = taosArrayInit(1, sizeof(SSDataBlock)); - taosArrayPush(blocks, pBlock); - blockDebugShowData(blocks, flag); - taosArrayDestroy(blocks); + if (pBlock == NULL){ + qDebug("======printDataBlock Block is Null"); + return; + } + char *pBuf = NULL; + qDebug("%s", dumpBlockData(pBlock, flag, &pBuf)); + taosMemoryFree(pBuf); } typedef int64_t (*__get_value_fn_t)(void* data, int32_t index); @@ -1217,30 +1230,40 @@ void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf, SExprS } } -void doClearWindow(SAggSupporter* pAggSup, SExprSupp* pSup, char* pData, int16_t bytes, uint64_t groupId, +bool doClearWindow(SAggSupporter* pAggSup, SExprSupp* pSup, char* pData, int16_t bytes, uint64_t groupId, int32_t numOfOutput) { SET_RES_WINDOW_KEY(pAggSup->keyBuf, pData, bytes, groupId); SResultRowPosition* p1 = (SResultRowPosition*)taosHashGet(pAggSup->pResultRowHashTable, pAggSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); if (!p1) { // window has been closed - return; + return false; } doClearWindowImpl(p1, pAggSup->pResultBuf, pSup, numOfOutput); + return true; } static void doClearWindows(SAggSupporter* pAggSup, SExprSupp* pSup1, SInterval* pInterval, int32_t tsIndex, int32_t numOfOutput, SSDataBlock* pBlock, SArray* pUpWins) { - SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, tsIndex); - TSKEY* tsCols = (TSKEY*)pColDataInfo->pData; + SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, tsIndex); + TSKEY* tsCols = (TSKEY*)pTsCol->pData; + uint64_t* pGpDatas = NULL; + if (pBlock->info.type == STREAM_RETRIEVE) { + SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, 2); + pGpDatas = (uint64_t*)pGpCol->pData; + } int32_t step = 0; for (int32_t i = 0; i < pBlock->info.rows; i += step) { SResultRowInfo dumyInfo; dumyInfo.cur.pageId = -1; STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsCols[i], pInterval, pInterval->precision, NULL); step = getNumOfRowsInTimeWindow(&pBlock->info, tsCols, i, win.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); - doClearWindow(pAggSup, pSup1, (char*)&win.skey, sizeof(TKEY), pBlock->info.groupId, numOfOutput); - if (pUpWins) { + uint64_t groupId = pBlock->info.groupId; + if (pGpDatas) { + groupId = pGpDatas[i]; + } + bool res = doClearWindow(pAggSup, pSup1, (char*)&win.skey, sizeof(TKEY), groupId, numOfOutput); + if (pUpWins && res) { taosArrayPush(pUpWins, &win); } } @@ -1268,8 +1291,8 @@ bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup) { return pSup->maxTs != INT64_MIN && pWin->ekey < pSup->maxTs - pSup->waterMark; } -static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, SInterval* pInterval, - SArray* closeWins) { +static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, + SInterval* pInterval, SHashObj* pPullDataMap, SArray* closeWins) { void* pIte = NULL; size_t keyLen = 0; while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) { @@ -1280,10 +1303,20 @@ static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, SResultRowInfo dumyInfo; dumyInfo.cur.pageId = -1; STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, ts, pInterval, pInterval->precision, NULL); + SWinRes winRe = {.ts = win.skey, .groupId = groupId,}; + void* chIds = taosHashGet(pPullDataMap, &winRe, sizeof(SWinRes)); if (isCloseWindow(&win, pSup)) { - char keyBuf[GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY))]; - SET_RES_WINDOW_KEY(keyBuf, &ts, sizeof(TSKEY), groupId); - taosHashRemove(pHashMap, keyBuf, keyLen); + if (chIds && pPullDataMap) { + SArray* chAy = *(SArray**) chIds; + int32_t size = taosArrayGetSize(chAy); + qInfo("======window %ld wait child size:%d", win.skey ,size); + for (int32_t i = 0; i < size; i++) { + qInfo("======window %ld wait chid id:%d", win.skey ,*(int32_t*)taosArrayGet(chAy, i)); + } + continue; + } else if (pPullDataMap) { + qInfo("======close window %ld", win.skey); + } SResultRowPosition* pPos = (SResultRowPosition*)pIte; if (pSup->calTrigger == STREAM_TRIGGER_WINDOW_CLOSE) { int32_t code = saveResult(ts, pPos->pageId, pPos->offset, groupId, closeWins); @@ -1291,11 +1324,25 @@ static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, return code; } } + char keyBuf[GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY))]; + SET_RES_WINDOW_KEY(keyBuf, &ts, sizeof(TSKEY), groupId); + taosHashRemove(pHashMap, keyBuf, keyLen); } } return TSDB_CODE_SUCCESS; } +static void closeChildIntervalWindow(SArray* pChildren, TSKEY maxTs) { + int32_t size = taosArrayGetSize(pChildren); + for (int32_t i = 0; i < size; i++) { + SOperatorInfo* pChildOp = taosArrayGetP(pChildren, i); + SStreamFinalIntervalOperatorInfo* pChInfo = pChildOp->info; + ASSERT(pChInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE); + pChInfo->twAggSup.maxTs = TMAX(pChInfo->twAggSup.maxTs, maxTs); + closeIntervalWindow(pChInfo->aggSup.pResultRowHashTable, &pChInfo->twAggSup, &pChInfo->interval, NULL, NULL); + } +} + static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { SIntervalAggOperatorInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -1324,7 +1371,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { break; } - if (pBlock->info.type == STREAM_REPROCESS) { + if (pBlock->info.type == STREAM_CLEAR) { doClearWindows(&pInfo->aggSup, &pOperator->exprSupp, &pInfo->interval, 0, pOperator->exprSupp.numOfExprs, pBlock, NULL); qDebug("%s clear existed time window results for updates checked", GET_TASKID(pTaskInfo)); @@ -1345,7 +1392,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey); hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, MAIN_SCAN, pUpdated); } - closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, pUpdated); + closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, NULL, pUpdated); finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset); initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); @@ -1373,6 +1420,11 @@ void destroyStreamFinalIntervalOperatorInfo(void* param, int32_t numOfOutput) { SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)param; cleanupBasicInfo(&pInfo->binfo); cleanupAggSup(&pInfo->aggSup); + //it should be empty. + taosHashCleanup(pInfo->pPullDataMap); + taosArrayDestroy(pInfo->pPullWins); + blockDataDestroy(pInfo->pPullDataRes); + if (pInfo->pChildren) { int32_t size = taosArrayGetSize(pInfo->pChildren); for (int32_t i = 0; i < size; i++) { @@ -2164,6 +2216,24 @@ bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup) { return p1 == NULL; } +int32_t getNexWindowPos(SInterval* pInterval, SDataBlockInfo* pBlockInfo, TSKEY* tsCols, + int32_t startPos, TSKEY eKey, STimeWindow* pNextWin) { + int32_t forwardRows = getNumOfRowsInTimeWindow(pBlockInfo, tsCols, startPos, + eKey, binarySearchForKey, NULL, TSDB_ORDER_ASC); + int32_t prevEndPos = forwardRows - 1 + startPos; + return getNextQualifiedWindow(pInterval, pNextWin, pBlockInfo, tsCols, prevEndPos, TSDB_ORDER_ASC); +} + +void addPullWindow(SHashObj* pMap, SWinRes* pWinRes, int32_t size) { + SArray* childIds = taosArrayInit(8, sizeof(int32_t)); + for (int32_t i = 0; i < size; i++) { + taosArrayPush(childIds, &i); + } + taosHashPut(pMap, pWinRes, sizeof(SWinRes), &childIds, sizeof(void*)); +} + +static int32_t getChildIndex(SSDataBlock* pBlock) { return pBlock->info.childId; } + static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, uint64_t tableGroupId, SArray* pUpdated) { SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)pOperatorInfo->info; @@ -2177,35 +2247,59 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc SResultRow* pResult = NULL; int32_t forwardRows = 0; - if (pSDataBlock->pDataBlock != NULL) { - SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); - tsCols = (int64_t*)pColDataInfo->pData; - } else { - return; - } + ASSERT(pSDataBlock->pDataBlock != NULL); + SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); + tsCols = (int64_t*)pColDataInfo->pData; int32_t startPos = ascScan ? 0 : (pSDataBlock->info.rows - 1); TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols); STimeWindow nextWin = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, pInfo->interval.precision, NULL); while (1) { - if (IS_FINAL_OP(pInfo) && isCloseWindow(&nextWin, &pInfo->twAggSup) && - isDeletedWindow(&nextWin, tableGroupId, &pInfo->aggSup)) { - SArray* pUpWins = taosArrayInit(8, sizeof(STimeWindow)); - taosArrayPush(pUpWins, &nextWin); - rebuildIntervalWindow(pInfo, pSup, pUpWins, pInfo->binfo.pRes->info.groupId, pSup->numOfExprs, - pOperatorInfo->pTaskInfo); - taosArrayDestroy(pUpWins); + if (IS_FINAL_OP(pInfo) && isCloseWindow(&nextWin, &pInfo->twAggSup) && pInfo->pChildren) { + bool ignore = true; + SWinRes winRes = {.ts = nextWin.skey, .groupId = tableGroupId,}; + void* chIds = taosHashGet(pInfo->pPullDataMap, &winRes, sizeof(SWinRes)); + if (isDeletedWindow(&nextWin, tableGroupId, &pInfo->aggSup) && !chIds) { + SPullWindowInfo pull = {.window = nextWin, .groupId = tableGroupId}; + // add pull data request + taosArrayPush(pInfo->pPullWins, &pull); + addPullWindow(pInfo->pPullDataMap, &winRes, taosArrayGetSize(pInfo->pChildren)); + } else { + int32_t index = -1; + SArray* chArray = NULL; + if (chIds) { + chArray = *(void**) chIds; + int32_t chId = getChildIndex(pSDataBlock); + index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ); + } + if (index != -1 && pSDataBlock->info.type == STREAM_PUSH_DATA) { + taosArrayRemove(chArray, index); + if (taosArrayGetSize(chArray) == 0) { + // pull data is over + taosHashRemove(pInfo->pPullDataMap, &winRes, sizeof(SWinRes)); + } + } + if ( index == -1 || pSDataBlock->info.type == STREAM_PUSH_DATA) { + ignore = false; + } + } + + if (ignore) { + startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin); + if (startPos < 0) { + break; + } + continue; + } } + int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, true, &pResult, tableGroupId, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); if (code != TSDB_CODE_SUCCESS || pResult == NULL) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } - SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t)); - pos->groupId = tableGroupId; - pos->pos = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset}; - *(int64_t*)pos->key = pResult->win.skey; + forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdated) { @@ -2230,14 +2324,17 @@ static void clearStreamIntervalOperator(SStreamFinalIntervalOperatorInfo* pInfo) initResultRowInfo(&pInfo->binfo.resultRowInfo); } -static void clearUpdateDataBlock(SSDataBlock* pBlock) { +static void clearSpecialDataBlock(SSDataBlock* pBlock) { + if (pBlock->info.rows <= 0) { + return; + } blockDataCleanup(pBlock); } void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex) { // ASSERT(pDest->info.capacity >= pSource->info.rows); blockDataEnsureCapacity(pDest, pSource->info.rows); - clearUpdateDataBlock(pDest); + clearSpecialDataBlock(pDest); SColumnInfoData* pDestCol = taosArrayGet(pDest->pDataBlock, 0); SColumnInfoData* pSourceCol = taosArrayGet(pSource->pDataBlock, tsColIndex); @@ -2254,7 +2351,63 @@ void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsCol blockDataUpdateTsWindow(pDest, 0); } -static int32_t getChildIndex(SSDataBlock* pBlock) { return pBlock->info.childId; } +static bool needBreak(SStreamFinalIntervalOperatorInfo* pInfo) { + int32_t size = taosArrayGetSize(pInfo->pPullWins); + if (pInfo->pullIndex < size) { + return true; + } + return false; +} + +static void doBuildPullDataBlock(SArray* array, int32_t* pIndex, SSDataBlock* pBlock) { + clearSpecialDataBlock(pBlock); + int32_t size = taosArrayGetSize(array); + if (size - (*pIndex) == 0) { + return; + } + blockDataEnsureCapacity(pBlock, size - (*pIndex) ); + ASSERT(3 <= taosArrayGetSize(pBlock->pDataBlock)); + for (; (*pIndex) < size; (*pIndex)++) { + SPullWindowInfo* pWin = taosArrayGet(array, (*pIndex) ); + SColumnInfoData* pStartTs = (SColumnInfoData*) taosArrayGet(pBlock->pDataBlock, 0); + colDataAppend(pStartTs, pBlock->info.rows, (const char*)&pWin->window.skey, false); + + SColumnInfoData* pEndTs = (SColumnInfoData*) taosArrayGet(pBlock->pDataBlock, 1); + colDataAppend(pEndTs, pBlock->info.rows, (const char*)&pWin->window.ekey, false); + + SColumnInfoData* pGroupId = (SColumnInfoData*) taosArrayGet(pBlock->pDataBlock, 2); + colDataAppend(pGroupId, pBlock->info.rows, (const char*)&pWin->groupId, false); + pBlock->info.rows++; + } + if ((*pIndex) == size) { + *pIndex = 0; + taosArrayClear(array); + } + blockDataUpdateTsWindow(pBlock, 0); +} + +void processPushEmpty(SSDataBlock* pBlock, SHashObj* pMap) { + SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, 0); + TSKEY* tsData = (TSKEY*)pStartCol->pData; + SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, 2); + uint64_t* groupIdData = (uint64_t*)pGroupCol->pData; + int32_t chId = getChildIndex(pBlock); + for (int32_t i = 0; i < pBlock->info.rows; i++) { + SWinRes winRes = {.ts = tsData[i], .groupId = groupIdData[i]}; + void* chIds = taosHashGet(pMap, &winRes, sizeof(SWinRes)); + if (chIds) { + SArray* chArray = *(SArray**) chIds; + int32_t index = taosArraySearchIdx(chArray, &chId, compareInt32Val, TD_EQ); + if (index != -1) { + taosArrayRemove(chArray, index); + if (taosArrayGetSize(chArray) == 0) { + // pull data is over + taosHashRemove(pMap, &winRes, sizeof(SWinRes)); + } + } + } + } +} static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { SStreamFinalIntervalOperatorInfo* pInfo = pOperator->info; @@ -2270,28 +2423,50 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); if (pInfo->binfo.pRes->info.rows == 0) { pOperator->status = OP_EXEC_DONE; - if (IS_FINAL_OP(pInfo) || pInfo->pUpdateRes->info.rows == 0) { - if (!IS_FINAL_OP(pInfo)) { - // semi interval operator clear disk buffer - clearStreamIntervalOperator(pInfo); - } - return NULL; + if (!IS_FINAL_OP(pInfo)) { + // semi interval operator clear disk buffer + clearStreamIntervalOperator(pInfo); } + return NULL; + } + printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); + return pInfo->binfo.pRes; + } else { + doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); + if (pInfo->binfo.pRes->info.rows != 0) { + printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); + return pInfo->binfo.pRes; + } + if (pInfo->pUpdateRes->info.rows != 0 && pInfo->returnUpdate) { + pInfo->returnUpdate = false; + ASSERT(!IS_FINAL_OP(pInfo)); + printDataBlock(pInfo->pUpdateRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); // process the rest of the data - pOperator->status = OP_OPENED; return pInfo->pUpdateRes; } - return pInfo->binfo.pRes; + doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes); + if (pInfo->pPullDataRes->info.rows != 0) { + // process the rest of the data + ASSERT(IS_FINAL_OP(pInfo)); + printDataBlock(pInfo->pPullDataRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); + return pInfo->pPullDataRes; + } } while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { - clearUpdateDataBlock(pInfo->pUpdateRes); + clearSpecialDataBlock(pInfo->pUpdateRes); + pOperator->status = OP_RES_TO_RETURN; + qInfo("Stream Final Interval return data"); break; } + printDataBlock(pBlock, IS_FINAL_OP(pInfo) ? "interval Final recv" : "interval Semi recv"); + maxTs = TMAX(maxTs, pBlock->info.window.ekey); - if (pBlock->info.type == STREAM_REPROCESS) { + if (pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_PUSH_DATA || pBlock->info.type == STREAM_INVALID) { + pInfo->binfo.pRes->info.type = pBlock->info.type; + } else if (pBlock->info.type == STREAM_CLEAR) { SArray* pUpWins = taosArrayInit(8, sizeof(STimeWindow)); doClearWindows(&pInfo->aggSup, pSup, &pInfo->interval, pInfo->primaryTsIndex, pOperator->exprSupp.numOfExprs, pBlock, pUpWins); @@ -2310,11 +2485,25 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { } removeResults(pUpWins, pUpdated); copyUpdateDataBlock(pInfo->pUpdateRes, pBlock, pInfo->primaryTsIndex); + pInfo->returnUpdate = true; taosArrayDestroy(pUpWins); break; } else if (pBlock->info.type == STREAM_GET_ALL && IS_FINAL_OP(pInfo)) { getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdated); continue; + } else if (pBlock->info.type == STREAM_RETRIEVE && !IS_FINAL_OP(pInfo)) { + SArray* pUpWins = taosArrayInit(8, sizeof(STimeWindow)); + doClearWindows(&pInfo->aggSup, pSup, &pInfo->interval, 0, pOperator->exprSupp.numOfExprs, + pBlock, pUpWins); + removeResults(pUpWins, pUpdated); + taosArrayDestroy(pUpWins); + if (taosArrayGetSize(pUpdated) > 0) { + break; + } + continue; + } else if (pBlock->info.type == STREAM_PUSH_EMPTY && IS_FINAL_OP(pInfo)) { + processPushEmpty(pBlock, pInfo->pPullDataMap); + continue; } setInputDataBlock(pOperator, pSup->pCtx, pBlock, pInfo->order, MAIN_SCAN, true); @@ -2334,31 +2523,70 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { SStreamFinalIntervalOperatorInfo* pChInfo = pChildOp->info; setInputDataBlock(pChildOp, pChildOp->exprSupp.pCtx, pBlock, pChInfo->order, MAIN_SCAN, true); doHashInterval(pChildOp, pBlock, pBlock->info.groupId, NULL); - pChInfo->twAggSup.maxTs = TMAX(pChInfo->twAggSup.maxTs, pBlock->info.window.ekey); + + if (needBreak(pInfo)) { + break; + } } - maxTs = TMAX(maxTs, pBlock->info.window.ekey); } pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); if (IS_FINAL_OP(pInfo)) { - closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, pUpdated); + closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, + &pInfo->interval, pInfo->pPullDataMap, pUpdated); + closeChildIntervalWindow(pInfo->pChildren, pInfo->twAggSup.maxTs); } finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset); initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - pOperator->status = OP_RES_TO_RETURN; - if (pInfo->binfo.pRes->info.rows == 0) { - pOperator->status = OP_EXEC_DONE; - if (pInfo->pUpdateRes->info.rows == 0) { - return NULL; - } + if (pInfo->binfo.pRes->info.rows != 0) { + printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); + return pInfo->binfo.pRes; + } + + if (pInfo->pUpdateRes->info.rows != 0 && pInfo->returnUpdate) { + pInfo->returnUpdate = false; + ASSERT(!IS_FINAL_OP(pInfo)); + printDataBlock(pInfo->pUpdateRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); // process the rest of the data - pOperator->status = OP_OPENED; return pInfo->pUpdateRes; } - return pInfo->binfo.pRes; + + doBuildPullDataBlock(pInfo->pPullWins, &pInfo->pullIndex, pInfo->pPullDataRes); + if (pInfo->pPullDataRes->info.rows != 0) { + // process the rest of the data + ASSERT(IS_FINAL_OP(pInfo)); + printDataBlock(pInfo->pPullDataRes, IS_FINAL_OP(pInfo) ? "interval Final" : "interval Semi"); + return pInfo->pPullDataRes; + } + // ASSERT(false); + return NULL; +} + +SSDataBlock* createPullDataBlock() { + SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); + pBlock->info.hasVarCol = false; + pBlock->info.groupId = 0; + pBlock->info.rows = 0; + pBlock->info.type = STREAM_RETRIEVE; + pBlock->info.rowSize = sizeof(TSKEY) + sizeof(TSKEY) + sizeof(uint64_t); + + pBlock->pDataBlock = taosArrayInit(3, sizeof(SColumnInfoData)); + SColumnInfoData infoData = {0}; + infoData.info.type = TSDB_DATA_TYPE_TIMESTAMP; + infoData.info.bytes = sizeof(TSKEY); + // window start ts + taosArrayPush(pBlock->pDataBlock, &infoData); + // window end ts + taosArrayPush(pBlock->pDataBlock, &infoData); + + infoData.info.type = TSDB_DATA_TYPE_UBIGINT; + infoData.info.bytes = sizeof(uint64_t); + taosArrayPush(pBlock->pDataBlock, &infoData); + + return pBlock; } SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, @@ -2412,23 +2640,30 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, goto _error; } } - // semi interval operator does not catch result pInfo->pUpdateRes = createResDataBlock(pPhyNode->pOutputDataBlockDesc); - pInfo->pUpdateRes->info.type = STREAM_REPROCESS; + pInfo->pUpdateRes->info.type = STREAM_CLEAR; blockDataEnsureCapacity(pInfo->pUpdateRes, 128); + pInfo->returnUpdate = false; + pInfo->pPhyNode = (SPhysiNode*)nodesCloneNode((SNode*)pPhyNode); if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL) { pInfo->isFinal = true; pOperator->name = "StreamFinalIntervalOperator"; } else { + // semi interval operator does not catch result pInfo->isFinal = false; pOperator->name = "StreamSemiIntervalOperator"; } - if (!IS_FINAL_OP(pInfo)) { + if (!IS_FINAL_OP(pInfo) || numOfChild == 0) { pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE; } + pInfo->pPullWins = taosArrayInit(8, sizeof(SPullWindowInfo)); + pInfo->pullIndex = 0; + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + pInfo->pPullDataMap = taosHashInit(64, hashFn, false, HASH_NO_LOCK); + pInfo->pPullDataRes = createPullDataBlock(); pOperator->operatorType = pPhyNode->type; pOperator->blocking = true; @@ -2811,11 +3046,6 @@ void compactTimeWindow(SStreamSessionAggOperatorInfo* pInfo, int32_t startIndex, } } -typedef struct SWinRes { - TSKEY ts; - uint64_t groupId; -} SWinRes; - static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SHashObj* pStUpdated, SHashObj* pStDeleted, bool hasEndTs) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -3027,14 +3257,14 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { } else if (pOperator->status == OP_RES_TO_RETURN) { doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator); if (pInfo->pDelRes->info.rows > 0) { - /*printDataBlock(pInfo->pDelRes, "session del");*/ + printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo)? "Final Session" : "Single Session"); return pInfo->pDelRes; } doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); if (pBInfo->pRes->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); } - /*printDataBlock(pBInfo->pRes, "session insert");*/ + printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo)? "Final Session" : "Single Session"); return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; } @@ -3048,7 +3278,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { break; } - if (pBlock->info.type == STREAM_REPROCESS) { + if (pBlock->info.type == STREAM_CLEAR) { SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo)); doClearSessionWindows(&pInfo->streamAggSup, &pOperator->exprSupp, pBlock, 0, pOperator->exprSupp.numOfExprs, pInfo->gap, pWins); @@ -3102,11 +3332,11 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator); if (pInfo->pDelRes->info.rows > 0) { - /*printDataBlock(pInfo->pDelRes, "session del");*/ + printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo)? "Final Session" : "Single Session"); return pInfo->pDelRes; } doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); - /*printDataBlock(pBInfo->pRes, "session insert");*/ + printDataBlock(pBInfo->pRes, IS_FINAL_OP(pInfo)? "Final Session" : "Single Session"); return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; } @@ -3169,11 +3399,11 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { - clearUpdateDataBlock(pInfo->pUpdateRes); + clearSpecialDataBlock(pInfo->pUpdateRes); break; } - if (pBlock->info.type == STREAM_REPROCESS) { + if (pBlock->info.type == STREAM_CLEAR) { SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo)); doClearSessionWindows(&pInfo->streamAggSup, pSup, pBlock, 0, pSup->numOfExprs, pInfo->gap, pWins); removeSessionResults(pStUpdated, pWins); @@ -3236,7 +3466,7 @@ SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream } else { pInfo->isFinal = false; pInfo->pUpdateRes = createResDataBlock(pPhyNode->pOutputDataBlockDesc); - pInfo->pUpdateRes->info.type = STREAM_REPROCESS; + pInfo->pUpdateRes->info.type = STREAM_CLEAR; blockDataEnsureCapacity(pInfo->pUpdateRes, 128); pOperator->name = "StreamSessionSemiAggOperator"; pOperator->fpSet = @@ -3554,7 +3784,7 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { break; } - if (pBlock->info.type == STREAM_REPROCESS) { + if (pBlock->info.type == STREAM_CLEAR) { doClearStateWindows(&pInfo->streamAggSup, pBlock, pInfo->primaryTsIndex, &pInfo->stateCol, pInfo->stateCol.slotId, pSeUpdated, pInfo->pSeDeleted); continue; @@ -3900,18 +4130,22 @@ _error: // merge interval operator typedef struct SMergeIntervalAggOperatorInfo { SIntervalAggOperatorInfo intervalAggOperatorInfo; - - SHashObj* groupIntervalHash; - void* groupIntervalIter; + SList* groupIntervals; + SListIter groupIntervalsIter; bool hasGroupId; uint64_t groupId; SSDataBlock* prefetchedBlock; bool inputBlocksFinished; } SMergeIntervalAggOperatorInfo; +typedef struct SGroupTimeWindow { + uint64_t groupId; + STimeWindow window; +} SGroupTimeWindow; + void destroyMergeIntervalOperatorInfo(void* param, int32_t numOfOutput) { SMergeIntervalAggOperatorInfo* miaInfo = (SMergeIntervalAggOperatorInfo*)param; - taosHashCleanup(miaInfo->groupIntervalHash); + tdListFree(miaInfo->groupIntervals); destroyIntervalOperatorInfo(&miaInfo->intervalAggOperatorInfo, numOfOutput); } @@ -3940,15 +4174,22 @@ static int32_t outputPrevIntervalResult(SOperatorInfo* pOperatorInfo, uint64_t t bool ascScan = (iaInfo->order == TSDB_ORDER_ASC); SExprSupp* pExprSup = &pOperatorInfo->exprSupp; - STimeWindow* prevWin = taosHashGet(miaInfo->groupIntervalHash, &tableGroupId, sizeof(tableGroupId)); - if (prevWin == NULL) { - taosHashPut(miaInfo->groupIntervalHash, &tableGroupId, sizeof(tableGroupId), newWin, sizeof(STimeWindow)); - return 0; - } + SGroupTimeWindow groupTimeWindow = {.groupId = tableGroupId, .window = *newWin}; + tdListAppend(miaInfo->groupIntervals, &groupTimeWindow); - if ((ascScan && newWin->skey > prevWin->skey || (!ascScan) && newWin->skey < prevWin->skey)) { - finalizeWindowResult(pOperatorInfo, tableGroupId, prevWin, pResultBlock); - taosHashPut(miaInfo->groupIntervalHash, &tableGroupId, sizeof(tableGroupId), newWin, sizeof(STimeWindow)); + SListIter iter = {0}; + tdListInitIter(miaInfo->groupIntervals, &iter, TD_LIST_FORWARD); + SListNode* listNode = NULL; + while ((listNode = tdListNext(&iter)) != NULL) { + SGroupTimeWindow* prevGrpWin = (SGroupTimeWindow*)listNode->data; + if (prevGrpWin->groupId != tableGroupId ) { + continue; + } + STimeWindow* prevWin = &prevGrpWin->window; + if ((ascScan && newWin->skey > prevWin->ekey || (!ascScan) && newWin->skey < prevWin->ekey)) { + finalizeWindowResult(pOperatorInfo, tableGroupId, prevWin, pResultBlock); + tdListPopNode(miaInfo->groupIntervals, listNode); + } } return 0; @@ -4075,6 +4316,7 @@ static SSDataBlock* doMergeIntervalAgg(SOperatorInfo* pOperator) { } if (pBlock == NULL) { + tdListInitIter(miaInfo->groupIntervals, &miaInfo->groupIntervalsIter, TD_LIST_FORWARD); miaInfo->inputBlocksFinished = true; break; } @@ -4100,14 +4342,12 @@ static SSDataBlock* doMergeIntervalAgg(SOperatorInfo* pOperator) { } if (miaInfo->inputBlocksFinished) { - void* win = taosHashIterate(miaInfo->groupIntervalHash, miaInfo->groupIntervalIter); - if (win != NULL) { - miaInfo->groupIntervalIter = win; + SListNode* listNode = tdListNext(&miaInfo->groupIntervalsIter); - size_t len = 0; - uint64_t* pTableGroupId = taosHashGetKey(win, &len); - finalizeWindowResult(pOperator, *pTableGroupId, win, pRes); - pRes->info.groupId = *pTableGroupId; + if (listNode != NULL) { + SGroupTimeWindow* grpWin = (SGroupTimeWindow*)(listNode->data); + finalizeWindowResult(pOperator, grpWin->groupId, &grpWin->window, pRes); + pRes->info.groupId = grpWin->groupId; } } @@ -4129,8 +4369,7 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SExprI goto _error; } - miaInfo->groupIntervalHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_NO_LOCK); - miaInfo->groupIntervalIter = NULL; + miaInfo->groupIntervals = tdListNew(sizeof(SGroupTimeWindow)); SIntervalAggOperatorInfo* iaInfo = &miaInfo->intervalAggOperatorInfo; diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 6744ebe4b9..d41bc89a5f 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1220,18 +1220,19 @@ static int32_t translateSubstr(SFunctionNode* pFunc, char* pErrBuf, int32_t len) static int32_t translateCast(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { // The number of parameters has been limited by the syntax definition - uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; + //uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; + // The function return type has been set during syntax parsing uint8_t para2Type = pFunc->node.resType.type; - if (para2Type != TSDB_DATA_TYPE_BIGINT && para2Type != TSDB_DATA_TYPE_UBIGINT && - para2Type != TSDB_DATA_TYPE_VARCHAR && para2Type != TSDB_DATA_TYPE_NCHAR && - para2Type != TSDB_DATA_TYPE_TIMESTAMP) { - return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); - } - if ((para2Type == TSDB_DATA_TYPE_TIMESTAMP && IS_VAR_DATA_TYPE(para1Type)) || - (para2Type == TSDB_DATA_TYPE_BINARY && para1Type == TSDB_DATA_TYPE_NCHAR)) { - return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); - } + //if (para2Type != TSDB_DATA_TYPE_BIGINT && para2Type != TSDB_DATA_TYPE_UBIGINT && + // para2Type != TSDB_DATA_TYPE_VARCHAR && para2Type != TSDB_DATA_TYPE_NCHAR && + // para2Type != TSDB_DATA_TYPE_TIMESTAMP) { + // return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + //} + //if ((para2Type == TSDB_DATA_TYPE_TIMESTAMP && IS_VAR_DATA_TYPE(para1Type)) || + // (para2Type == TSDB_DATA_TYPE_BINARY && para1Type == TSDB_DATA_TYPE_NCHAR)) { + // return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + //} int32_t para2Bytes = pFunc->node.resType.bytes; if (IS_VAR_DATA_TYPE(para2Type)) { @@ -1274,13 +1275,12 @@ static bool validateTimestampDigits(const SValueNode* pVal) { } int64_t tsVal = pVal->datum.i; - char fraction[20] = {0}; + char fraction[20] = {0}; NUM_TO_STRING(pVal->node.resType.type, &tsVal, sizeof(fraction), fraction); int32_t tsDigits = (int32_t)strlen(fraction); if (tsDigits > TSDB_TIME_PRECISION_SEC_DIGITS) { - if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS || - tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS || + if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS || tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS || tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) { return true; } else { @@ -1510,6 +1510,11 @@ static int32_t translateBlockDistInfoFunc(SFunctionNode* pFunc, char* pErrBuf, i return TSDB_CODE_SUCCESS; } +static int32_t translateGroupKeyFunc(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + pFunc->node.resType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType; + return TSDB_CODE_SUCCESS; +} + static bool getBlockDistFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { pEnv->calcMemSize = sizeof(STableBlockDistInfo); return true; @@ -2519,6 +2524,8 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .initFunc = functionSetup, .processFunc = groupKeyFunction, .finalizeFunc = groupKeyFinalize, + .pPartialFunc = "_group_key", + .pMergeFunc = "_group_key" }, }; // clang-format on diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 61f3154811..c4d3a26ab4 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -264,6 +264,7 @@ typedef struct SRateInfo { typedef struct SGroupKeyInfo{ bool hasResult; + bool isNull; char data[]; } SGroupKeyInfo; @@ -5371,14 +5372,21 @@ int32_t groupKeyFunction(SqlFunctionCtx* pCtx) { int32_t bytes = pInputCol->info.bytes; int32_t startIndex = pInput->startRowIndex; - if (colDataIsNull_s(pInputCol, startIndex)) { - pInfo->hasResult = false; + + //escape rest of data blocks to avoid first entry be overwritten. + if (pInfo->hasResult) { + goto _group_key_over; + } + + if (colDataIsNull_s(pInputCol, startIndex)) { + pInfo->isNull = true; + pInfo->hasResult = true; goto _group_key_over; } - pInfo->hasResult = true; char* data = colDataGetData(pInputCol, startIndex); memcpy(pInfo->data, data, bytes); + pInfo->hasResult = true; _group_key_over: @@ -5393,7 +5401,12 @@ int32_t groupKeyFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); SGroupKeyInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo); - colDataAppend(pCol, pBlock->info.rows, pInfo->data, pInfo->hasResult ? false : true); + + if (pInfo->hasResult) { + colDataAppend(pCol, pBlock->info.rows, pInfo->data, pInfo->isNull ? true : false); + } else { + pResInfo->numOfRes = 0; + } return pResInfo->numOfRes; } diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 838071dbf1..983cffe9dc 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -12,6 +12,8 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ + +// clang-format off #include "uv.h" #include "os.h" #include "fnLog.h" @@ -25,6 +27,7 @@ #include "tglobal.h" #include "tmsg.h" #include "trpc.h" +// clang-foramt on typedef struct SUdfdContext { uv_loop_t * loop; @@ -103,12 +106,12 @@ typedef struct SUdfdRpcSendRecvInfo { uv_sem_t resultSem; } SUdfdRpcSendRecvInfo; -static void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); +static void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); static int32_t udfdFillUdfInfoFromMNode(void *clientRpc, char *udfName, SUdf *udf); static int32_t udfdConnectToMnode(); static int32_t udfdLoadUdf(char *udfName, SUdf *udf); -static bool udfdRpcRfp(int32_t code); -static int initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet); +static bool udfdRpcRfp(int32_t code); +static int initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet); static int32_t udfdOpenClientRpc(); static int32_t udfdCloseClientRpc(); @@ -126,19 +129,19 @@ static void udfdUvHandleError(SUdfdUvConn *conn) { uv_close((uv_handle_t *)conn- static void udfdPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf); static void udfdOnNewConnection(uv_stream_t *server, int status); -static void udfdIntrSignalHandler(uv_signal_t *handle, int signum); +static void udfdIntrSignalHandler(uv_signal_t *handle, int signum); static int32_t removeListeningPipe(); -static void udfdPrintVersion(); +static void udfdPrintVersion(); static int32_t udfdParseArgs(int32_t argc, char *argv[]); static int32_t udfdInitLog(); -static void udfdCtrlAllocBufCb(uv_handle_t *handle, size_t suggested_size, uv_buf_t *buf); -static void udfdCtrlReadCb(uv_stream_t *q, ssize_t nread, const uv_buf_t *buf); +static void udfdCtrlAllocBufCb(uv_handle_t *handle, size_t suggested_size, uv_buf_t *buf); +static void udfdCtrlReadCb(uv_stream_t *q, ssize_t nread, const uv_buf_t *buf); static int32_t udfdUvInit(); -static void udfdCloseWalkCb(uv_handle_t *handle, void *arg); +static void udfdCloseWalkCb(uv_handle_t *handle, void *arg); static int32_t udfdRun(); -static void udfdConnectMnodeThreadFunc(void* args); +static void udfdConnectMnodeThreadFunc(void *args); void udfdProcessRequest(uv_work_t *req) { SUvUdfWork *uvUdf = (SUvUdfWork *)(req->data); @@ -401,11 +404,11 @@ void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { udf->bufSize = pFuncInfo->bufSize; char path[PATH_MAX] = {0}; - #ifdef WINDOWS +#ifdef WINDOWS snprintf(path, sizeof(path), "%s%s.dll", TD_TMP_DIR_PATH, pFuncInfo->name); - #else +#else snprintf(path, sizeof(path), "%s/lib%s.so", TD_TMP_DIR_PATH, pFuncInfo->name); - #endif +#endif TdFilePtr file = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC | TD_FILE_AUTO_DEL); if (file == NULL) { @@ -544,7 +547,8 @@ int32_t udfdLoadUdf(char *udfName, SUdf *udf) { return 0; } static bool udfdRpcRfp(int32_t code) { - if (code == TSDB_CODE_RPC_REDIRECT) { + if (code == TSDB_CODE_RPC_REDIRECT || code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_NODE_NOT_DEPLOYED || + code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_APP_NOT_READY) { return true; } else { return false; @@ -652,8 +656,7 @@ void udfdAllocBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) { buf->base = ctx->inputBuf; buf->len = ctx->inputCap; } else { - fnError("udfd can not allocate enough memory") - buf->base = NULL; + fnError("udfd can not allocate enough memory") buf->base = NULL; buf->len = 0; } } else { @@ -664,8 +667,7 @@ void udfdAllocBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) { buf->base = ctx->inputBuf + ctx->inputLen; buf->len = ctx->inputCap - ctx->inputLen; } else { - fnError("udfd can not allocate enough memory") - buf->base = NULL; + fnError("udfd can not allocate enough memory") buf->base = NULL; buf->len = 0; } } @@ -881,7 +883,7 @@ static int32_t udfdRun() { return 0; } -void udfdConnectMnodeThreadFunc(void* args) { +void udfdConnectMnodeThreadFunc(void *args) { int32_t retryMnodeTimes = 0; int32_t code = 0; while (retryMnodeTimes++ <= TSDB_MAX_REPLICA) { @@ -939,7 +941,7 @@ int main(int argc, char *argv[]) { uv_thread_create(&mnodeConnectThread, udfdConnectMnodeThreadFunc, NULL); udfdRun(); - + removeListeningPipe(); udfdCloseClientRpc(); diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 227914449a..734f287cfb 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -346,9 +346,11 @@ static void destroyVgDataBlockArray(SArray* pArray) { } static void destroyLogicNode(SLogicNode* pNode) { - nodesDestroyList(pNode->pChildren); - nodesDestroyNode(pNode->pConditions); nodesDestroyList(pNode->pTargets); + nodesDestroyNode(pNode->pConditions); + nodesDestroyList(pNode->pChildren); + nodesDestroyNode(pNode->pLimit); + nodesDestroyNode(pNode->pSlimit); } static void destroyPhysiNode(SPhysiNode* pNode) { @@ -368,6 +370,7 @@ static void destroyWinodwPhysiNode(SWinodwPhysiNode* pNode) { static void destroyScanPhysiNode(SScanPhysiNode* pNode) { destroyPhysiNode((SPhysiNode*)pNode); nodesDestroyList(pNode->pScanCols); + nodesDestroyList(pNode->pScanPseudoCols); } static void destroyDataSinkNode(SDataSinkNode* pNode) { nodesDestroyNode((SNode*)pNode->pInputDataBlockDesc); } @@ -516,6 +519,9 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyNode(pStmt->pWindow); nodesDestroyList(pStmt->pGroupByList); nodesDestroyNode(pStmt->pHaving); + nodesDestroyNode(pStmt->pRange); + nodesDestroyNode(pStmt->pEvery); + nodesDestroyNode(pStmt->pFill); nodesDestroyList(pStmt->pOrderByList); nodesDestroyNode((SNode*)pStmt->pLimit); nodesDestroyNode((SNode*)pStmt->pSlimit); @@ -779,6 +785,8 @@ void nodesDestroyNode(SNode* pNode) { SInterpFuncLogicNode* pLogicNode = (SInterpFuncLogicNode*)pNode; destroyLogicNode((SLogicNode*)pLogicNode); nodesDestroyList(pLogicNode->pFuncs); + nodesDestroyNode(pLogicNode->pFillValues); + nodesDestroyNode(pLogicNode->pTimeSeries); break; } case QUERY_NODE_LOGIC_SUBPLAN: { @@ -793,14 +801,21 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyList(((SQueryLogicPlan*)pNode)->pTopSubplans); break; case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: - case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: - case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN: - case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN: case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN: case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN: case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN: destroyScanPhysiNode((SScanPhysiNode*)pNode); break; + case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN: { + STableScanPhysiNode* pPhyNode = (STableScanPhysiNode*)pNode; + destroyScanPhysiNode((SScanPhysiNode*)pNode); + nodesDestroyList(pPhyNode->pDynamicScanFuncs); + nodesDestroyList(pPhyNode->pPartitionTags); + break; + } case QUERY_NODE_PHYSICAL_PLAN_PROJECT: { SProjectPhysiNode* pPhyNode = (SProjectPhysiNode*)pNode; destroyPhysiNode((SPhysiNode*)pPhyNode); @@ -891,6 +906,8 @@ void nodesDestroyNode(SNode* pNode) { destroyPhysiNode((SPhysiNode*)pPhyNode); nodesDestroyList(pPhyNode->pExprs); nodesDestroyList(pPhyNode->pFuncs); + nodesDestroyNode(pPhyNode->pFillValues); + nodesDestroyNode(pPhyNode->pTimeSeries); break; } case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index d4bde5d624..b68a4cc568 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -1355,6 +1355,25 @@ static EDealRes rewriteColToSelectValFunc(STranslateContext* pCxt, SNode** pNode return TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR; } +static EDealRes rewriteExprToGroupKeyFunc(STranslateContext* pCxt, SNode** pNode) { + SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); + if (NULL == pFunc) { + pCxt->errCode = TSDB_CODE_OUT_OF_MEMORY; + return DEAL_RES_ERROR; + } + + strcpy(pFunc->functionName, "_group_key"); + strcpy(pFunc->node.aliasName, ((SExprNode*)*pNode)->aliasName); + pCxt->errCode = nodesListMakeAppend(&pFunc->pParameterList, *pNode); + if (TSDB_CODE_SUCCESS == pCxt->errCode) { + *pNode = (SNode*)pFunc; + pCxt->errCode = fmGetFuncInfo(pFunc, pCxt->msgBuf.buf, pCxt->msgBuf.len); + } + pCxt->pCurrSelectStmt->hasAggFuncs = true; + + return (TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR); +} + static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) { SCheckExprForGroupByCxt* pCxt = (SCheckExprForGroupByCxt*)pContext; if (!nodesIsExprNode(*pNode) || isAliasColumn(*pNode)) { @@ -1371,10 +1390,10 @@ static EDealRes doCheckExprForGroupBy(SNode** pNode, void* pContext) { if (isAggFunc(*pNode) && !isDistinctOrderBy(pCxt->pTranslateCxt)) { return DEAL_RES_IGNORE_CHILD; } - SNode* pGroupNode; + SNode* pGroupNode = NULL; FOREACH(pGroupNode, getGroupByList(pCxt->pTranslateCxt)) { if (nodesEqualNode(getGroupByNode(pGroupNode), *pNode)) { - return DEAL_RES_IGNORE_CHILD; + return rewriteExprToGroupKeyFunc(pCxt->pTranslateCxt, pNode); } } if (isScanPseudoColumnFunc(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode)) { @@ -1432,6 +1451,25 @@ static int32_t rewriteColsToSelectValFunc(STranslateContext* pCxt, SSelectStmt* return pCxt->errCode; } +static EDealRes rewriteExprsToGroupKeyFuncImpl(SNode** pNode, void* pContext) { + STranslateContext* pCxt = pContext; + SNode* pPartKey = NULL; + FOREACH(pPartKey, pCxt->pCurrSelectStmt->pPartitionByList) { + if (nodesEqualNode(pPartKey, *pNode)) { + return rewriteExprToGroupKeyFunc(pCxt, pNode); + } + } + return DEAL_RES_CONTINUE; +} + +static int32_t rewriteExprsToGroupKeyFunc(STranslateContext* pCxt, SSelectStmt* pSelect) { + nodesRewriteExprs(pSelect->pProjectionList, rewriteExprsToGroupKeyFuncImpl, pCxt); + if (TSDB_CODE_SUCCESS == pCxt->errCode && !pSelect->isDistinct) { + nodesRewriteExprs(pSelect->pOrderByList, rewriteExprsToGroupKeyFuncImpl, pCxt); + } + return pCxt->errCode; +} + typedef struct CheckAggColCoexistCxt { STranslateContext* pTranslateCxt; bool existAggFunc; @@ -1456,6 +1494,12 @@ static EDealRes doCheckAggColCoexist(SNode* pNode, void* pContext) { pCxt->existIndefiniteRowsFunc = true; return DEAL_RES_IGNORE_CHILD; } + SNode* pPartKey = NULL; + FOREACH(pPartKey, pCxt->pTranslateCxt->pCurrSelectStmt->pPartitionByList) { + if (nodesEqualNode(pPartKey, pNode)) { + return DEAL_RES_IGNORE_CHILD; + } + } if (isScanPseudoColumnFunc(pNode) || QUERY_NODE_COLUMN == nodeType(pNode)) { pCxt->existCol = true; } @@ -1485,6 +1529,9 @@ static int32_t checkAggColCoexist(STranslateContext* pCxt, SSelectStmt* pSelect) if (cxt.existIndefiniteRowsFunc && cxt.existCol) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC); } + if (cxt.existAggFunc && NULL != pSelect->pPartitionByList) { + return rewriteExprsToGroupKeyFunc(pCxt, pSelect); + } return TSDB_CODE_SUCCESS; } @@ -1840,10 +1887,7 @@ static int32_t createMultiResFuncsFromStar(STranslateContext* pCxt, SFunctionNod code = createMultiResFuncs(pSrcFunc, pExprs, pOutput); } - if (TSDB_CODE_SUCCESS != code) { - nodesDestroyList(pExprs); - } - + nodesDestroyList(pExprs); return code; } @@ -2463,8 +2507,24 @@ static SNode* createOrderByExpr(STranslateContext* pCxt) { return (SNode*)pOrder; } -// from: select tail(expr, k, f) from t where_clause partition_by_clause order_by_clause ... -// to: select expr from t where_clause order by _rowts desc limit k offset f +/* case 1: + * in: select tail(expr, k, f) from t where_clause + * out: select expr from t where_clause order by _rowts desc limit k offset f + * + * case 2: + * in: select tail(expr, k, f) from t where_clause partition_by_clause + * out: select expr from t where_clause partition_by_clause sort by _rowts desc limit k offset f + * + * case 3: + * in: select tail(expr, k, f) from t where_clause order_by_clause limit_clause + * out: select expr from ( + * select expr from t where_clause order by _rowts desc limit k offset f + * ) order_by_clause limit_clause + * + * case 4: + * in: select tail(expr, k, f) from t where_clause partition_by_clause limit_clause + * out: + */ static int32_t rewriteTailStmt(STranslateContext* pCxt, SSelectStmt* pSelect) { if (!pSelect->hasTailFunc) { return TSDB_CODE_SUCCESS; @@ -4963,6 +5023,7 @@ typedef struct SVgroupCreateTableBatch { static void destroyCreateTbReq(SVCreateTbReq* pReq) { taosMemoryFreeClear(pReq->name); + taosMemoryFreeClear(pReq->comment); taosMemoryFreeClear(pReq->ntb.schemaRow.pSchema); } @@ -4980,6 +5041,7 @@ static int32_t buildNormalTableBatchReq(int32_t acctId, const SCreateTableStmt* if (pStmt->pOptions->commentNull == false) { req.comment = strdup(pStmt->pOptions->comment); if (NULL == req.comment) { + destroyCreateTbReq(&req); return TSDB_CODE_OUT_OF_MEMORY; } req.commentLen = strlen(pStmt->pOptions->comment); @@ -5051,6 +5113,7 @@ static void destroyCreateTbReqBatch(SVgroupCreateTableBatch* pTbBatch) { for (int32_t i = 0; i < size; ++i) { SVCreateTbReq* pTableReq = taosArrayGet(pTbBatch->req.pArray, i); taosMemoryFreeClear(pTableReq->name); + taosMemoryFreeClear(pTableReq->comment); if (pTableReq->type == TSDB_NORMAL_TABLE) { taosMemoryFreeClear(pTableReq->ntb.schemaRow.pSchema); @@ -6018,7 +6081,7 @@ static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) { TSWAP(pQuery->pCmdMsg, pCxt->pCmdMsg); pQuery->msgType = pQuery->pCmdMsg->msgType; } - break; + break; default: pQuery->execMode = QUERY_EXEC_MODE_RPC; if (NULL != pCxt->pCmdMsg) { diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index 5c8439f846..f8b8dc58fc 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -199,6 +199,20 @@ TEST_F(ParserSelectTest, tailFuncSemanticCheck) { run("SELECT TAIL(c1, 10) FROM t1 GROUP BY c2", TSDB_CODE_PAR_GROUP_BY_NOT_ALLOWED_FUNC); } +TEST_F(ParserSelectTest, partitionBy) { + useDb("root", "test"); + + run("SELECT c1, c2 FROM t1 PARTITION BY c2"); + + run("SELECT SUM(c1), c2 FROM t1 PARTITION BY c2"); +} + +TEST_F(ParserSelectTest, partitionBySemanticCheck) { + useDb("root", "test"); + + run("SELECT SUM(c1), c2, c3 FROM t1 PARTITION BY c2", TSDB_CODE_PAR_NOT_SINGLE_GROUP); +} + TEST_F(ParserSelectTest, groupBy) { useDb("root", "test"); @@ -213,6 +227,15 @@ TEST_F(ParserSelectTest, groupBy) { run("SELECT COUNT(*), c1 + 10, c2 cnt FROM t1 WHERE c1 > 0 GROUP BY c1 + 10, c2"); } +TEST_F(ParserSelectTest, groupBySemanticCheck) { + useDb("root", "test"); + + run("SELECT COUNT(*) cnt, c1 FROM t1 WHERE c1 > 0", TSDB_CODE_PAR_NOT_SINGLE_GROUP); + run("SELECT COUNT(*) cnt, c2 FROM t1 WHERE c1 > 0 GROUP BY c1", TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION); + run("SELECT COUNT(*) cnt, c2 FROM t1 WHERE c1 > 0 PARTITION BY c2 GROUP BY c1", + TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION); +} + TEST_F(ParserSelectTest, orderBy) { useDb("root", "test"); diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index ffb97bcc04..a4cdcd35d3 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -450,6 +450,15 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, int32_t code = TSDB_CODE_SUCCESS; // set grouyp keys, agg funcs and having conditions + if (TSDB_CODE_SUCCESS == code && pSelect->hasAggFuncs) { + code = nodesCollectFuncs(pSelect, SQL_CLAUSE_GROUP_BY, fmIsAggFunc, &pAgg->pAggFuncs); + } + + // rewrite the expression in subsequent clauses + if (TSDB_CODE_SUCCESS == code) { + code = rewriteExprsForSelect(pAgg->pAggFuncs, pSelect, SQL_CLAUSE_GROUP_BY); + } + if (NULL != pSelect->pGroupByList) { pAgg->pGroupKeys = nodesCloneList(pSelect->pGroupByList); if (NULL == pAgg->pGroupKeys) { @@ -462,15 +471,6 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, code = rewriteExprsForSelect(pAgg->pGroupKeys, pSelect, SQL_CLAUSE_GROUP_BY); } - if (TSDB_CODE_SUCCESS == code && pSelect->hasAggFuncs) { - code = nodesCollectFuncs(pSelect, SQL_CLAUSE_GROUP_BY, fmIsAggFunc, &pAgg->pAggFuncs); - } - - // rewrite the expression in subsequent clauses - if (TSDB_CODE_SUCCESS == code) { - code = rewriteExprsForSelect(pAgg->pAggFuncs, pSelect, SQL_CLAUSE_GROUP_BY); - } - if (TSDB_CODE_SUCCESS == code && NULL != pSelect->pHaving) { pAgg->node.pConditions = nodesCloneNode(pSelect->pHaving); if (NULL == pAgg->node.pConditions) { @@ -780,8 +780,8 @@ static int32_t createProjectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSel return TSDB_CODE_OUT_OF_MEMORY; } - pProject->node.pLimit = (SNode*)pSelect->pLimit; - pProject->node.pSlimit = (SNode*)pSelect->pSlimit; + TSWAP(pProject->node.pLimit, pSelect->pLimit); + TSWAP(pProject->node.pSlimit, pSelect->pSlimit); int32_t code = TSDB_CODE_SUCCESS; @@ -940,7 +940,7 @@ static int32_t createSetOpSortLogicNode(SLogicPlanContext* pCxt, SSetOperator* p return TSDB_CODE_OUT_OF_MEMORY; } - pSort->node.pLimit = pSetOperator->pLimit; + TSWAP(pSort->node.pLimit, pSetOperator->pLimit); int32_t code = TSDB_CODE_SUCCESS; @@ -973,7 +973,7 @@ static int32_t createSetOpProjectLogicNode(SLogicPlanContext* pCxt, SSetOperator } if (NULL == pSetOperator->pOrderByList) { - pProject->node.pLimit = pSetOperator->pLimit; + TSWAP(pProject->node.pLimit, pSetOperator->pLimit); } int32_t code = TSDB_CODE_SUCCESS; @@ -1004,7 +1004,7 @@ static int32_t createSetOpAggLogicNode(SLogicPlanContext* pCxt, SSetOperator* pS } if (NULL == pSetOperator->pOrderByList) { - pAgg->node.pLimit = pSetOperator->pLimit; + TSWAP(pAgg->node.pSlimit, pSetOperator->pLimit); } int32_t code = TSDB_CODE_SUCCESS; diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 10275ffd9d..c1b12d9d20 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -682,7 +682,7 @@ static EOrder opkGetPrimaryKeyOrder(SSortLogicNode* pSort) { static SNode* opkRewriteDownNode(SSortLogicNode* pSort) { SNode* pDownNode = nodesListGetNode(pSort->node.pChildren, 0); // todo - pSort->node.pChildren = NULL; + NODES_CLEAR_LIST(pSort->node.pChildren); return pDownNode; } diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 37b69ffdfe..4f46abd8da 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -348,8 +348,8 @@ static SPhysiNode* makePhysiNode(SPhysiPlanContext* pCxt, SLogicNode* pLogicNode return NULL; } - pPhysiNode->pLimit = pLogicNode->pLimit; - pPhysiNode->pSlimit = pLogicNode->pSlimit; + TSWAP(pPhysiNode->pLimit, pLogicNode->pLimit); + TSWAP(pPhysiNode->pSlimit, pLogicNode->pSlimit); int32_t code = createDataBlockDesc(pCxt, pLogicNode->pTargets, &pPhysiNode->pOutputDataBlockDesc); if (TSDB_CODE_SUCCESS != code) { @@ -862,6 +862,9 @@ static int32_t createIndefRowsFuncPhysiNode(SPhysiPlanContext* pCxt, SNodeList* nodesDestroyNode((SNode*)pIdfRowsFunc); } + nodesDestroyList(pPrecalcExprs); + nodesDestroyList(pFuncs); + return code; } @@ -913,6 +916,9 @@ static int32_t createInterpFuncPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pCh nodesDestroyNode((SNode*)pInterpFunc); } + nodesDestroyList(pPrecalcExprs); + nodesDestroyList(pFuncs); + return code; } @@ -1048,6 +1054,9 @@ static int32_t createWindowPhysiNodeFinalize(SPhysiPlanContext* pCxt, SNodeList* nodesDestroyNode((SNode*)pWindow); } + nodesDestroyList(pPrecalcExprs); + nodesDestroyList(pFuncs); + return code; } @@ -1241,6 +1250,9 @@ static int32_t createPartitionPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChi nodesDestroyNode((SNode*)pPart); } + nodesDestroyList(pPrecalcExprs); + nodesDestroyList(pPartitionKeys); + return code; } diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 0e35239621..b4a966f206 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -179,7 +179,7 @@ static bool stbSplNeedSplitWindow(bool streamQuery, SLogicNode* pNode) { return !stbSplHasGatherExecFunc(pWindow->pFuncs) && stbSplHasMultiTbScan(streamQuery, pNode); } } - + if (WINDOW_TYPE_STATE == pWindow->winType) { if (!streamQuery) { return stbSplHasMultiTbScan(streamQuery, pNode); @@ -374,7 +374,7 @@ static int32_t stbSplCreateMergeNode(SSplitContext* pCxt, SLogicSubplan* pSubpla int32_t code = TSDB_CODE_SUCCESS; pMerge->pInputs = nodesCloneList(pPartChild->pTargets); - // NULL == pSubplan means 'merge node' replaces 'split node'. + // NULL != pSubplan means 'merge node' replaces 'split node'. if (NULL == pSubplan) { pMerge->node.pTargets = nodesCloneList(pPartChild->pTargets); } else { @@ -512,7 +512,7 @@ static int32_t stbSplSplitSessionOrStateForBatch(SSplitContext* pCxt, SStableSpl SLogicNode* pChild = (SLogicNode*)nodesListGetNode(pWindow->pChildren, 0); SNodeList* pMergeKeys = NULL; - int32_t code = stbSplCreateMergeKeysByPrimaryKey(((SWindowLogicNode*)pWindow)->pTspk, &pMergeKeys); + int32_t code = stbSplCreateMergeKeysByPrimaryKey(((SWindowLogicNode*)pWindow)->pTspk, &pMergeKeys); if (TSDB_CODE_SUCCESS == code) { code = stbSplCreateMergeNode(pCxt, pInfo->pSubplan, pChild, pMergeKeys, (SLogicNode*)pChild); @@ -561,6 +561,8 @@ static int32_t stbSplSplitState(SSplitContext* pCxt, SStableSplitInfo* pInfo) { static SNodeList* stbSplGetPartKeys(SLogicNode* pNode) { if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) { return ((SScanLogicNode*)pNode)->pPartTags; + } else if (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode)) { + return ((SPartitionLogicNode*)pNode)->pPartitionKeys; } else { return NULL; } @@ -571,14 +573,15 @@ static bool stbSplIsPartTbanme(SNodeList* pPartKeys) { return false; } SNode* pPartKey = nodesListGetNode(pPartKeys, 0); - return QUERY_NODE_FUNCTION == nodeType(pPartKey) && FUNCTION_TYPE_TBNAME == ((SFunctionNode*)pPartKey)->funcType; + return (QUERY_NODE_FUNCTION == nodeType(pPartKey) && FUNCTION_TYPE_TBNAME == ((SFunctionNode*)pPartKey)->funcType) || + (QUERY_NODE_COLUMN == nodeType(pPartKey) && COLUMN_TYPE_TBNAME == ((SColumnNode*)pPartKey)->colType); } -static bool stbSplIsMultiTableWinodw(SWindowLogicNode* pWindow) { +static bool stbSplIsPartTableWinodw(SWindowLogicNode* pWindow) { return stbSplIsPartTbanme(stbSplGetPartKeys((SLogicNode*)nodesListGetNode(pWindow->node.pChildren, 0))); } -static int32_t stbSplSplitWindowForMergeTable(SSplitContext* pCxt, SStableSplitInfo* pInfo) { +static int32_t stbSplSplitWindowForCrossTable(SSplitContext* pCxt, SStableSplitInfo* pInfo) { switch (((SWindowLogicNode*)pInfo->pSplitNode)->winType) { case WINDOW_TYPE_INTERVAL: return stbSplSplitInterval(pCxt, pInfo); @@ -592,7 +595,7 @@ static int32_t stbSplSplitWindowForMergeTable(SSplitContext* pCxt, SStableSplitI return TSDB_CODE_PLAN_INTERNAL_ERROR; } -static int32_t stbSplSplitWindowForMultiTable(SSplitContext* pCxt, SStableSplitInfo* pInfo) { +static int32_t stbSplSplitWindowForPartTable(SSplitContext* pCxt, SStableSplitInfo* pInfo) { if (pCxt->pPlanCxt->streamQuery) { SPLIT_FLAG_SET_MASK(pInfo->pSubplan->splitFlag, SPLIT_FLAG_STABLE_SPLIT); return TSDB_CODE_SUCCESS; @@ -613,10 +616,10 @@ static int32_t stbSplSplitWindowForMultiTable(SSplitContext* pCxt, SStableSplitI } static int32_t stbSplSplitWindowNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) { - if (stbSplIsMultiTableWinodw((SWindowLogicNode*)pInfo->pSplitNode)) { - return stbSplSplitWindowForMultiTable(pCxt, pInfo); + if (stbSplIsPartTableWinodw((SWindowLogicNode*)pInfo->pSplitNode)) { + return stbSplSplitWindowForPartTable(pCxt, pInfo); } else { - return stbSplSplitWindowForMergeTable(pCxt, pInfo); + return stbSplSplitWindowForCrossTable(pCxt, pInfo); } } diff --git a/source/libs/planner/test/planDistinctTest.cpp b/source/libs/planner/test/planDistinctTest.cpp index 58473dcff2..8caee01d35 100644 --- a/source/libs/planner/test/planDistinctTest.cpp +++ b/source/libs/planner/test/planDistinctTest.cpp @@ -40,3 +40,9 @@ TEST_F(PlanDistinctTest, withOrderBy) { run("select distinct c1 + 10 a from t1 order by a"); } + +TEST_F(PlanDistinctTest, withLimit) { + useDb("root", "test"); + + run("SELECT DISTINCT c1 FROM t1 LIMIT 3"); +} diff --git a/source/libs/planner/test/planGroupByTest.cpp b/source/libs/planner/test/planGroupByTest.cpp index a73c99bf68..657d8a6244 100644 --- a/source/libs/planner/test/planGroupByTest.cpp +++ b/source/libs/planner/test/planGroupByTest.cpp @@ -53,14 +53,6 @@ TEST_F(PlanGroupByTest, aggFunc) { run("SELECT SUM(10), COUNT(c1) FROM t1 GROUP BY c2"); } -TEST_F(PlanGroupByTest, rewriteFunc) { - useDb("root", "test"); - - run("SELECT AVG(c1) FROM t1"); - - run("SELECT AVG(c1) FROM t1 GROUP BY c2"); -} - TEST_F(PlanGroupByTest, selectFunc) { useDb("root", "test"); @@ -81,6 +73,8 @@ TEST_F(PlanGroupByTest, stable) { run("SELECT COUNT(*) FROM st1"); + run("SELECT c1 FROM st1 GROUP BY c1"); + run("SELECT COUNT(*) FROM st1 GROUP BY c1"); run("SELECT COUNT(*) FROM st1 PARTITION BY c2 GROUP BY c1"); diff --git a/source/libs/planner/test/planOptimizeTest.cpp b/source/libs/planner/test/planOptimizeTest.cpp index 8b3d263d66..9bd95a79f9 100644 --- a/source/libs/planner/test/planOptimizeTest.cpp +++ b/source/libs/planner/test/planOptimizeTest.cpp @@ -57,9 +57,15 @@ TEST_F(PlanOptimizeTest, orderByPrimaryKey) { TEST_F(PlanOptimizeTest, PartitionTags) { useDb("root", "test"); - run("SELECT c1 FROM st1 PARTITION BY tag1"); + run("SELECT c1, tag1 FROM st1 PARTITION BY tag1"); - run("SELECT SUM(c1) FROM st1 GROUP BY tag1"); + run("SELECT SUM(c1), tag1 FROM st1 PARTITION BY tag1"); + + run("SELECT SUM(c1), tag1 + 10 FROM st1 PARTITION BY tag1 + 10"); + + run("SELECT SUM(c1), tag1 FROM st1 GROUP BY tag1"); + + run("SELECT SUM(c1), tag1 + 10 FROM st1 GROUP BY tag1 + 10"); } TEST_F(PlanOptimizeTest, eliminateProjection) { diff --git a/source/libs/planner/test/planPartByTest.cpp b/source/libs/planner/test/planPartByTest.cpp index 9437f6ad3f..2abcb44dfd 100644 --- a/source/libs/planner/test/planPartByTest.cpp +++ b/source/libs/planner/test/planPartByTest.cpp @@ -34,6 +34,8 @@ TEST_F(PlanPartitionByTest, withAggFunc) { useDb("root", "test"); run("select count(*) from t1 partition by c1"); + + run("select count(*), c1 from t1 partition by c1"); } TEST_F(PlanPartitionByTest, withInterval) { @@ -41,8 +43,12 @@ TEST_F(PlanPartitionByTest, withInterval) { // normal/child table run("select count(*) from t1 partition by c1 interval(10s)"); + + run("select count(*), c1 from t1 partition by c1 interval(10s)"); // super table run("select count(*) from st1 partition by tag1, tag2 interval(10s)"); + + run("select count(*), tag1 from st1 partition by tag1, tag2 interval(10s)"); } TEST_F(PlanPartitionByTest, withGroupBy) { diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index c807c1c4cd..bee0461e58 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -19,6 +19,7 @@ #include "tmsg.h" #include "trpc.h" #include "tsched.h" +// clang-format off #include "cJSON.h" #define VALIDNUMOFCOLS(x) ((x) >= TSDB_MIN_COLUMNS && (x) <= TSDB_MAX_COLUMNS) @@ -147,13 +148,15 @@ int32_t asyncSendMsgToServerExt(void* pTransporter, SEpSet* epSet, int64_t* pTra } memcpy(pMsg, pInfo->msgInfo.pData, pInfo->msgInfo.len); - SRpcMsg rpcMsg = {.msgType = pInfo->msgType, - .pCont = pMsg, - .contLen = pInfo->msgInfo.len, - .info.ahandle = (void*)pInfo, - .info.handle = pInfo->msgInfo.handle, - .info.persistHandle = persistHandle, - .code = 0}; + SRpcMsg rpcMsg = { + .msgType = pInfo->msgType, + .pCont = pMsg, + .contLen = pInfo->msgInfo.len, + .info.ahandle = (void*)pInfo, + .info.handle = pInfo->msgInfo.handle, + .info.persistHandle = persistHandle, + .code = 0 + }; assert(pInfo->fp != NULL); TRACE_SET_ROOTID(&rpcMsg.info.traceId, pInfo->requestId); rpcSendRequestWithCtx(pTransporter, epSet, &rpcMsg, pTransporterId, rpcCtx); @@ -221,8 +224,9 @@ void destroyQueryExecRes(SQueryExecRes* pRes) { qError("invalid exec result for request type %d", pRes->msgType); } } +// clang-format on -int32_t dataConverToStr(char *str, int type, void *buf, int32_t bufSize, int32_t *len) { +int32_t dataConverToStr(char* str, int type, void* buf, int32_t bufSize, int32_t* len) { int32_t n = 0; switch (type) { @@ -262,7 +266,7 @@ int32_t dataConverToStr(char *str, int type, void *buf, int32_t bufSize, int32_t case TSDB_DATA_TYPE_BINARY: case TSDB_DATA_TYPE_NCHAR: if (bufSize < 0) { -// tscError("invalid buf size"); + // tscError("invalid buf size"); return TSDB_CODE_TSC_INVALID_VALUE; } @@ -289,7 +293,7 @@ int32_t dataConverToStr(char *str, int type, void *buf, int32_t bufSize, int32_t break; default: -// tscError("unsupported type:%d", type); + // tscError("unsupported type:%d", type); return TSDB_CODE_TSC_INVALID_VALUE; } @@ -332,7 +336,7 @@ char* parseTagDatatoJson(void* p) { int32_t length = taosUcs4ToMbs((TdUcs4*)pTagVal->pData, pTagVal->nData, tagJsonValue); if (length < 0) { qError("charset:%s to %s. val:%s convert json value failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, - pTagVal->pData); + pTagVal->pData); taosMemoryFree(tagJsonValue); goto end; } @@ -372,7 +376,6 @@ end: return string; } - int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst) { if (NULL == pSrc) { *pDst = NULL; @@ -393,37 +396,36 @@ int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst) { *pDst = NULL; return TSDB_CODE_SUCCESS; } - + *pDst = taosMemoryMalloc(sizeof(*pSrc)); if (NULL == *pDst) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } memcpy(*pDst, pSrc, sizeof(*pSrc)); if (pSrc->vgHash) { - (*pDst)->vgHash = taosHashInit(taosHashGetSize(pSrc->vgHash), taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); + (*pDst)->vgHash = taosHashInit(taosHashGetSize(pSrc->vgHash), taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, + HASH_ENTRY_LOCK); if (NULL == (*pDst)->vgHash) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } SVgroupInfo* vgInfo = NULL; - void *pIter = taosHashIterate(pSrc->vgHash, NULL); + void* pIter = taosHashIterate(pSrc->vgHash, NULL); while (pIter) { vgInfo = pIter; int32_t* vgId = taosHashGetKey(pIter, NULL); - + if (0 != taosHashPut((*pDst)->vgHash, vgId, sizeof(*vgId), vgInfo, sizeof(*vgInfo))) { qError("taosHashPut failed, vgId:%d", vgInfo->vgId); - taosHashCancelIterate(pSrc->vgHash, pIter); + taosHashCancelIterate(pSrc->vgHash, pIter); taosHashCleanup((*pDst)->vgHash); taosMemoryFreeClear(*pDst); return TSDB_CODE_CTG_MEM_ERROR; } - + pIter = taosHashIterate(pSrc->vgHash, pIter); } } - + return TSDB_CODE_SUCCESS; } - - diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index d7f0733129..80bebafef2 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -735,6 +735,60 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp char *input = colDataGetData(pInput[0].columnData, i); switch(outputType) { + case TSDB_DATA_TYPE_TINYINT: { + if (inputType == TSDB_DATA_TYPE_BINARY) { + *(int8_t *)output = taosStr2Int8(varDataVal(input), NULL, 10); + } else if (inputType == TSDB_DATA_TYPE_NCHAR) { + char *newBuf = taosMemoryCalloc(1, inputLen); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + if (len < 0) { + taosMemoryFree(newBuf); + return TSDB_CODE_FAILED; + } + newBuf[len] = 0; + *(int8_t *)output = taosStr2Int8(newBuf, NULL, 10); + taosMemoryFree(newBuf); + } else { + GET_TYPED_DATA(*(int8_t *)output, int8_t, inputType, input); + } + break; + } + case TSDB_DATA_TYPE_SMALLINT: { + if (inputType == TSDB_DATA_TYPE_BINARY) { + *(int16_t *)output = taosStr2Int16(varDataVal(input), NULL, 10); + } else if (inputType == TSDB_DATA_TYPE_NCHAR) { + char *newBuf = taosMemoryCalloc(1, inputLen); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + if (len < 0) { + taosMemoryFree(newBuf); + return TSDB_CODE_FAILED; + } + newBuf[len] = 0; + *(int16_t *)output = taosStr2Int16(newBuf, NULL, 10); + taosMemoryFree(newBuf); + } else { + GET_TYPED_DATA(*(int16_t *)output, int16_t, inputType, input); + } + break; + } + case TSDB_DATA_TYPE_INT: { + if (inputType == TSDB_DATA_TYPE_BINARY) { + *(int32_t *)output = taosStr2Int32(varDataVal(input), NULL, 10); + } else if (inputType == TSDB_DATA_TYPE_NCHAR) { + char *newBuf = taosMemoryCalloc(1, inputLen); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + if (len < 0) { + taosMemoryFree(newBuf); + return TSDB_CODE_FAILED; + } + newBuf[len] = 0; + *(int32_t *)output = taosStr2Int32(newBuf, NULL, 10); + taosMemoryFree(newBuf); + } else { + GET_TYPED_DATA(*(int32_t *)output, int32_t, inputType, input); + } + break; + } case TSDB_DATA_TYPE_BIGINT: { if (inputType == TSDB_DATA_TYPE_BINARY) { *(int64_t *)output = taosStr2Int64(varDataVal(input), NULL, 10); @@ -753,6 +807,60 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp } break; } + case TSDB_DATA_TYPE_UTINYINT: { + if (inputType == TSDB_DATA_TYPE_BINARY) { + *(uint8_t *)output = taosStr2UInt8(varDataVal(input), NULL, 10); + } else if (inputType == TSDB_DATA_TYPE_NCHAR) { + char *newBuf = taosMemoryCalloc(1, inputLen); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + if (len < 0) { + taosMemoryFree(newBuf); + return TSDB_CODE_FAILED; + } + newBuf[len] = 0; + *(uint8_t *)output = taosStr2UInt8(newBuf, NULL, 10); + taosMemoryFree(newBuf); + } else { + GET_TYPED_DATA(*(uint8_t *)output, uint8_t, inputType, input); + } + break; + } + case TSDB_DATA_TYPE_USMALLINT: { + if (inputType == TSDB_DATA_TYPE_BINARY) { + *(uint16_t *)output = taosStr2UInt16(varDataVal(input), NULL, 10); + } else if (inputType == TSDB_DATA_TYPE_NCHAR) { + char *newBuf = taosMemoryCalloc(1, inputLen); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + if (len < 0) { + taosMemoryFree(newBuf); + return TSDB_CODE_FAILED; + } + newBuf[len] = 0; + *(uint16_t *)output = taosStr2UInt16(newBuf, NULL, 10); + taosMemoryFree(newBuf); + } else { + GET_TYPED_DATA(*(uint16_t *)output, uint16_t, inputType, input); + } + break; + } + case TSDB_DATA_TYPE_UINT: { + if (inputType == TSDB_DATA_TYPE_BINARY) { + *(uint32_t *)output = taosStr2UInt32(varDataVal(input), NULL, 10); + } else if (inputType == TSDB_DATA_TYPE_NCHAR) { + char *newBuf = taosMemoryCalloc(1, inputLen); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + if (len < 0) { + taosMemoryFree(newBuf); + return TSDB_CODE_FAILED; + } + newBuf[len] = 0; + *(uint32_t *)output = taosStr2UInt32(newBuf, NULL, 10); + taosMemoryFree(newBuf); + } else { + GET_TYPED_DATA(*(uint32_t *)output, uint32_t, inputType, input); + } + break; + } case TSDB_DATA_TYPE_UBIGINT: { if (inputType == TSDB_DATA_TYPE_BINARY) { *(uint64_t *)output = taosStr2UInt64(varDataVal(input), NULL, 10); @@ -771,10 +879,64 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp } break; } + case TSDB_DATA_TYPE_FLOAT: { + if (inputType == TSDB_DATA_TYPE_BINARY) { + *(float *)output = taosStr2Float(varDataVal(input), NULL); + } else if (inputType == TSDB_DATA_TYPE_NCHAR) { + char *newBuf = taosMemoryCalloc(1, inputLen); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + if (len < 0) { + taosMemoryFree(newBuf); + return TSDB_CODE_FAILED; + } + newBuf[len] = 0; + *(float *)output = taosStr2Float(newBuf, NULL); + taosMemoryFree(newBuf); + } else { + GET_TYPED_DATA(*(float *)output, float, inputType, input); + } + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + if (inputType == TSDB_DATA_TYPE_BINARY) { + *(double *)output = taosStr2Double(varDataVal(input), NULL); + } else if (inputType == TSDB_DATA_TYPE_NCHAR) { + char *newBuf = taosMemoryCalloc(1, inputLen); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + if (len < 0) { + taosMemoryFree(newBuf); + return TSDB_CODE_FAILED; + } + newBuf[len] = 0; + *(double *)output = taosStr2Double(newBuf, NULL); + taosMemoryFree(newBuf); + } else { + GET_TYPED_DATA(*(double *)output, double, inputType, input); + } + break; + } + case TSDB_DATA_TYPE_BOOL: { + if (inputType == TSDB_DATA_TYPE_BINARY) { + *(bool *)output = taosStr2Int8(varDataVal(input), NULL, 10); + } else if (inputType == TSDB_DATA_TYPE_NCHAR) { + char *newBuf = taosMemoryCalloc(1, inputLen); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + if (len < 0) { + taosMemoryFree(newBuf); + return TSDB_CODE_FAILED; + } + newBuf[len] = 0; + *(bool *)output = taosStr2Int8(newBuf, NULL, 10); + taosMemoryFree(newBuf); + } else { + GET_TYPED_DATA(*(bool *)output, bool, inputType, input); + } + break; + } case TSDB_DATA_TYPE_TIMESTAMP: { if (inputType == TSDB_DATA_TYPE_BINARY || inputType == TSDB_DATA_TYPE_NCHAR) { - //not support - return TSDB_CODE_FAILED; + //convert to 0 + *(int64_t *)output = 0; } else { GET_TYPED_DATA(*(int64_t *)output, int64_t, inputType, input); } @@ -789,8 +951,16 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp len = sprintf(varDataVal(output), "%.*s", len, varDataVal(input)); varDataSetLen(output, len); } else if (inputType == TSDB_DATA_TYPE_NCHAR) { - //not support - return TSDB_CODE_FAILED; + char *newBuf = taosMemoryCalloc(1, inputLen); + int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(input), varDataLen(input), newBuf); + if (len < 0) { + taosMemoryFree(newBuf); + return TSDB_CODE_FAILED; + } + len = TMIN(len, outputLen - VARSTR_HEADER_SIZE); + memcpy(varDataVal(output), newBuf, len); + varDataSetLen(output, len); + taosMemoryFree(newBuf); } else { char tmp[400] = {0}; NUM_TO_STRING(inputType, input, sizeof(tmp), tmp); diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index f9a9ec0f72..292db3db1f 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -468,7 +468,7 @@ double getVectorDoubleValue_JSON(void *src, int32_t index){ } void* ncharTobinary(void *buf){ // todo need to remove , if tobinary is nchar - int32_t inputLen = varDataLen(buf); + int32_t inputLen = varDataTLen(buf); void* t = taosMemoryCalloc(1, inputLen); int32_t len = taosUcs4ToMbs((TdUcs4 *)varDataVal(buf), varDataLen(buf), varDataVal(t)); diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 0c9d46f055..a5e9b8edd9 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -115,6 +115,7 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) .srcNodeId = pTask->nodeId, .srcTaskId = pTask->taskId, .pRetrieve = pRetrieve, + .retrieveLen = dataStrLen, }; int32_t sz = taosArrayGetSize(pTask->childEpInfo); @@ -146,7 +147,7 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) .code = 0, .msgType = TDMT_STREAM_RETRIEVE, .pCont = buf, - .contLen = len, + .contLen = sizeof(SMsgHead) + len, }; if (tmsgSendReq(&pEpInfo->epSet, &rpcMsg) < 0) { diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index fe0f406f8d..c75e6c004a 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -45,11 +45,16 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) ASSERT(false); } if (output == NULL) { - if (pItem->type == STREAM_INPUT__DATA_RETRIEVE && !hasData) { - SSDataBlock block = {0}; - block.info.type = STREAM_PUSH_DATA; - block.info.childId = pTask->selfChildId; - taosArrayPush(pRes, &block); + if (pItem->type == STREAM_INPUT__DATA_RETRIEVE) { + //SSDataBlock block = {0}; + //block.info.type = STREAM_PUSH_EMPTY; + //block.info.childId = pTask->selfChildId; + SStreamDataBlock* pRetrieveBlock = (SStreamDataBlock*)data; + ASSERT(taosArrayGetSize(pRetrieveBlock->blocks) == 1); + SSDataBlock* pBlock = createOneDataBlock(taosArrayGet(pRetrieveBlock->blocks, 0), true); + pBlock->info.type = STREAM_PUSH_EMPTY; + pBlock->info.childId = pTask->selfChildId; + taosArrayPush(pRes, pBlock); } break; } diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index ada391b40a..3d64cec8d8 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -119,6 +119,7 @@ SUpdateInfo *updateInfoInit(int64_t interval, int32_t precision, int64_t waterma taosArrayPush(pInfo->pTsBuckets, &dumy); } pInfo->numBuckets = DEFAULT_BUCKET_SIZE; + pInfo->pCloseWinSBF = NULL; return pInfo; } @@ -154,6 +155,9 @@ bool updateInfoIsUpdated(SUpdateInfo *pInfo, tb_uid_t tableId, TSKEY ts) { TSKEY maxTs = *(TSKEY *)taosArrayGet(pInfo->pTsBuckets, index); if (ts < maxTs - pInfo->watermark) { // this window has been closed. + if (pInfo->pCloseWinSBF) { + return tScalableBfPut(pInfo->pCloseWinSBF, &ts, sizeof(TSKEY)); + } return true; } @@ -193,3 +197,19 @@ void updateInfoDestroy(SUpdateInfo *pInfo) { taosArrayDestroy(pInfo->pTsSBFs); taosMemoryFree(pInfo); } + +void updateInfoAddCloseWindowSBF(SUpdateInfo *pInfo) { + if (pInfo->pCloseWinSBF) { + return; + } + int64_t rows = adjustExpEntries(pInfo->interval * ROWS_PER_MILLISECOND); + pInfo->pCloseWinSBF = tScalableBfInit(rows, DEFAULT_FALSE_POSITIVE); +} + +void updateInfoDestoryColseWinSBF(SUpdateInfo *pInfo) { + if (!pInfo || !pInfo->pCloseWinSBF) { + return; + } + tScalableBfDestroy(pInfo->pCloseWinSBF); + pInfo->pCloseWinSBF = NULL; +} diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 47d0976aca..ba3dd66550 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -89,6 +89,280 @@ // /\ UNCHANGED <> // +int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { + int32_t ret = 0; + + // print log + syncAppendEntriesLog2("==syncNodeOnAppendEntriesCb==", pMsg); + + // if already drop replica, do not process + if (!syncNodeInRaftGroup(ths, &(pMsg->srcId)) && !ths->pRaftCfg->isStandBy) { + syncNodeEventLog(ths, "recv sync-append-entries, maybe replica already dropped"); + return ret; + } + + // maybe update term + if (pMsg->term > ths->pRaftStore->currentTerm) { + syncNodeUpdateTerm(ths, pMsg->term); + } + ASSERT(pMsg->term <= ths->pRaftStore->currentTerm); + + // reset elect timer + if (pMsg->term == ths->pRaftStore->currentTerm) { + ths->leaderCache = pMsg->srcId; + syncNodeResetElectTimer(ths); + } + ASSERT(pMsg->dataLen >= 0); + + do { + // return to follower state + if (pMsg->term == ths->pRaftStore->currentTerm && ths->state == TAOS_SYNC_STATE_CANDIDATE) { + syncNodeEventLog(ths, "recv sync-append-entries, candidate to follower"); + + syncNodeBecomeFollower(ths, "from candidate by append entries"); + + // ret or reply? + return ret; + } + } while (0); + + SyncTerm localPreLogTerm = 0; + if (pMsg->prevLogIndex >= SYNC_INDEX_BEGIN && pMsg->prevLogIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)) { + SSyncRaftEntry* pEntry = ths->pLogStore->getEntry(ths->pLogStore, pMsg->prevLogIndex); + if (pEntry == NULL) { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "getEntry error, index:%ld, since %s", pMsg->prevLogIndex, terrstr()); + syncNodeErrorLog(ths, logBuf); + return -1; + } + + localPreLogTerm = pEntry->term; + syncEntryDestory(pEntry); + } + + bool logOK = + (pMsg->prevLogIndex == SYNC_INDEX_INVALID) || + ((pMsg->prevLogIndex >= SYNC_INDEX_BEGIN) && + (pMsg->prevLogIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)) && (pMsg->prevLogTerm == localPreLogTerm)); + + // reject request + if ((pMsg->term < ths->pRaftStore->currentTerm) || + ((pMsg->term == ths->pRaftStore->currentTerm) && (ths->state == TAOS_SYNC_STATE_FOLLOWER) && !logOK)) { + do { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries, reject, pre-index:%ld, pre-term:%lu, datalen:%d", + pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen); + syncNodeEventLog(ths, logBuf); + } while (0); + + SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId); + pReply->srcId = ths->myRaftId; + pReply->destId = pMsg->srcId; + pReply->term = ths->pRaftStore->currentTerm; + pReply->success = false; + pReply->matchIndex = SYNC_INDEX_INVALID; + + SRpcMsg rpcMsg; + syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); + syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); + syncAppendEntriesReplyDestroy(pReply); + + return ret; + } + + // accept request + if (pMsg->term == ths->pRaftStore->currentTerm && ths->state == TAOS_SYNC_STATE_FOLLOWER && logOK) { + // preIndex = -1, or has preIndex entry in local log + ASSERT(pMsg->prevLogIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)); + + // has extra entries (> preIndex) in local log + bool hasExtraEntries = pMsg->prevLogIndex < ths->pLogStore->getLastIndex(ths->pLogStore); + + // has entries in SyncAppendEntries msg + bool hasAppendEntries = pMsg->dataLen > 0; + + do { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries, accept, pre-index:%ld, pre-term:%lu, datalen:%d", + pMsg->prevLogIndex, pMsg->prevLogTerm, pMsg->dataLen); + syncNodeEventLog(ths, logBuf); + } while (0); + + if (hasExtraEntries && hasAppendEntries) { + // not conflict by default + bool conflict = false; + + SyncIndex extraIndex = pMsg->prevLogIndex + 1; + SSyncRaftEntry* pExtraEntry = ths->pLogStore->getEntry(ths->pLogStore, extraIndex); + if (pExtraEntry == NULL) { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "getEntry error2, index:%ld, since %s", extraIndex, terrstr()); + syncNodeErrorLog(ths, logBuf); + return -1; + } + + SSyncRaftEntry* pAppendEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen); + if (pAppendEntry == NULL) { + syncNodeErrorLog(ths, "syncEntryDeserialize pAppendEntry error"); + return -1; + } + + // log not match, conflict + ASSERT(extraIndex == pAppendEntry->index); + if (pExtraEntry->term != pAppendEntry->term) { + conflict = true; + } + + if (conflict) { + // roll back + SyncIndex delBegin = ths->pLogStore->getLastIndex(ths->pLogStore); + SyncIndex delEnd = extraIndex; + + sTrace("syncNodeOnAppendEntriesCb --> conflict:%d, delBegin:%ld, delEnd:%ld", conflict, delBegin, delEnd); + + // notice! reverse roll back! + for (SyncIndex index = delEnd; index >= delBegin; --index) { + if (ths->pFsm->FpRollBackCb != NULL) { + SSyncRaftEntry* pRollBackEntry = ths->pLogStore->getEntry(ths->pLogStore, index); + if (pRollBackEntry == NULL) { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "getEntry error3, index:%ld, since %s", index, terrstr()); + syncNodeErrorLog(ths, logBuf); + return -1; + } + + // if (pRollBackEntry->msgType != TDMT_SYNC_NOOP) { + if (syncUtilUserRollback(pRollBackEntry->msgType)) { + SRpcMsg rpcMsg; + syncEntry2OriginalRpc(pRollBackEntry, &rpcMsg); + + SFsmCbMeta cbMeta = {0}; + cbMeta.index = pRollBackEntry->index; + cbMeta.lastConfigIndex = syncNodeGetSnapshotConfigIndex(ths, cbMeta.index); + cbMeta.isWeak = pRollBackEntry->isWeak; + cbMeta.code = 0; + cbMeta.state = ths->state; + cbMeta.seqNum = pRollBackEntry->seqNum; + ths->pFsm->FpRollBackCb(ths->pFsm, &rpcMsg, cbMeta); + rpcFreeCont(rpcMsg.pCont); + } + + syncEntryDestory(pRollBackEntry); + } + } + + // delete confict entries + ths->pLogStore->truncate(ths->pLogStore, extraIndex); + + // append new entries + ths->pLogStore->appendEntry(ths->pLogStore, pAppendEntry); + + // pre commit + SRpcMsg rpcMsg; + syncEntry2OriginalRpc(pAppendEntry, &rpcMsg); + if (ths->pFsm != NULL) { + // if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->originalRpcType != TDMT_SYNC_NOOP) { + if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pAppendEntry->originalRpcType)) { + SFsmCbMeta cbMeta = {0}; + cbMeta.index = pAppendEntry->index; + cbMeta.lastConfigIndex = syncNodeGetSnapshotConfigIndex(ths, cbMeta.index); + cbMeta.isWeak = pAppendEntry->isWeak; + cbMeta.code = 2; + cbMeta.state = ths->state; + cbMeta.seqNum = pAppendEntry->seqNum; + ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, cbMeta); + } + } + rpcFreeCont(rpcMsg.pCont); + } + + // free memory + syncEntryDestory(pExtraEntry); + syncEntryDestory(pAppendEntry); + + } else if (hasExtraEntries && !hasAppendEntries) { + // do nothing + + } else if (!hasExtraEntries && hasAppendEntries) { + SSyncRaftEntry* pAppendEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen); + if (pAppendEntry == NULL) { + syncNodeErrorLog(ths, "syncEntryDeserialize pAppendEntry2 error"); + return -1; + } + + // append new entries + ths->pLogStore->appendEntry(ths->pLogStore, pAppendEntry); + + // pre commit + SRpcMsg rpcMsg; + syncEntry2OriginalRpc(pAppendEntry, &rpcMsg); + if (ths->pFsm != NULL) { + // if (ths->pFsm->FpPreCommitCb != NULL && pAppendEntry->originalRpcType != TDMT_SYNC_NOOP) { + if (ths->pFsm->FpPreCommitCb != NULL && syncUtilUserPreCommit(pAppendEntry->originalRpcType)) { + SFsmCbMeta cbMeta = {0}; + cbMeta.index = pAppendEntry->index; + cbMeta.lastConfigIndex = syncNodeGetSnapshotConfigIndex(ths, cbMeta.index); + cbMeta.isWeak = pAppendEntry->isWeak; + cbMeta.code = 3; + cbMeta.state = ths->state; + cbMeta.seqNum = pAppendEntry->seqNum; + ths->pFsm->FpPreCommitCb(ths->pFsm, &rpcMsg, cbMeta); + } + } + rpcFreeCont(rpcMsg.pCont); + + // free memory + syncEntryDestory(pAppendEntry); + + } else if (!hasExtraEntries && !hasAppendEntries) { + // do nothing + + } else { + syncNodeLog3("", ths); + ASSERT(0); + } + + SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId); + pReply->srcId = ths->myRaftId; + pReply->destId = pMsg->srcId; + pReply->term = ths->pRaftStore->currentTerm; + pReply->success = true; + + if (hasAppendEntries) { + pReply->matchIndex = pMsg->prevLogIndex + 1; + } else { + pReply->matchIndex = pMsg->prevLogIndex; + } + + SRpcMsg rpcMsg; + syncAppendEntriesReply2RpcMsg(pReply, &rpcMsg); + syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); + syncAppendEntriesReplyDestroy(pReply); + + // maybe update commit index from leader + if (pMsg->commitIndex > ths->commitIndex) { + // has commit entry in local + if (pMsg->commitIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)) { + SyncIndex beginIndex = ths->commitIndex + 1; + SyncIndex endIndex = pMsg->commitIndex; + + // update commit index + ths->commitIndex = pMsg->commitIndex; + + // call back Wal + ths->pLogStore->updateCommitIndex(ths->pLogStore, ths->commitIndex); + + int32_t code = syncNodeCommit(ths, beginIndex, endIndex, ths->state); + ASSERT(code == 0); + } + } + } + + return ret; +} + +#if 0 + int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { int32_t ret = 0; @@ -352,6 +626,8 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { return ret; } +#endif + static int32_t syncNodeMakeLogSame(SSyncNode* ths, SyncAppendEntries* pMsg) { int32_t code; diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index 6ca356b4f6..b7122b9c52 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -40,6 +40,78 @@ int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg) { int32_t ret = 0; + // print log + syncAppendEntriesReplyLog2("==syncNodeOnAppendEntriesReplyCb==", pMsg); + + // if already drop replica, do not process + if (!syncNodeInRaftGroup(ths, &(pMsg->srcId)) && !ths->pRaftCfg->isStandBy) { + syncNodeEventLog(ths, "recv sync-append-entries-reply, maybe replica already dropped"); + return 0; + } + + // drop stale response + if (pMsg->term < ths->pRaftStore->currentTerm) { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries-reply, recv-term:%lu, drop stale response", pMsg->term); + syncNodeEventLog(ths, logBuf); + return 0; + } + + if (gRaftDetailLog) { + syncNodeEventLog(ths, "recv sync-append-entries-reply, before"); + } + syncIndexMgrLog2("==syncNodeOnAppendEntriesReplyCb== before pNextIndex", ths->pNextIndex); + syncIndexMgrLog2("==syncNodeOnAppendEntriesReplyCb== before pMatchIndex", ths->pMatchIndex); + + // no need this code, because if I receive reply.term, then I must have sent for that term. + // if (pMsg->term > ths->pRaftStore->currentTerm) { + // syncNodeUpdateTerm(ths, pMsg->term); + // } + + if (pMsg->term > ths->pRaftStore->currentTerm) { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "recv sync-append-entries-reply, error term, recv-term:%lu", pMsg->term); + syncNodeErrorLog(ths, logBuf); + return -1; + } + + ASSERT(pMsg->term == ths->pRaftStore->currentTerm); + + if (pMsg->success) { + // nextIndex' = [nextIndex EXCEPT ![i][j] = m.mmatchIndex + 1] + syncIndexMgrSetIndex(ths->pNextIndex, &(pMsg->srcId), pMsg->matchIndex + 1); + + // matchIndex' = [matchIndex EXCEPT ![i][j] = m.mmatchIndex] + syncIndexMgrSetIndex(ths->pMatchIndex, &(pMsg->srcId), pMsg->matchIndex); + + // maybe commit + syncMaybeAdvanceCommitIndex(ths); + + } else { + SyncIndex nextIndex = syncIndexMgrGetIndex(ths->pNextIndex, &(pMsg->srcId)); + + // notice! int64, uint64 + if (nextIndex > SYNC_INDEX_BEGIN) { + --nextIndex; + } else { + nextIndex = SYNC_INDEX_BEGIN; + } + syncIndexMgrSetIndex(ths->pNextIndex, &(pMsg->srcId), nextIndex); + } + + if (gRaftDetailLog) { + syncNodeEventLog(ths, "recv sync-append-entries-reply, after"); + } + syncIndexMgrLog2("==syncNodeOnAppendEntriesReplyCb== after pNextIndex", ths->pNextIndex); + syncIndexMgrLog2("==syncNodeOnAppendEntriesReplyCb== after pMatchIndex", ths->pMatchIndex); + + return ret; +} + +#if 0 +int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg) { + int32_t ret = 0; + char logBuf[128] = {0}; snprintf(logBuf, sizeof(logBuf), "==syncNodeOnAppendEntriesReplyCb== term:%lu", ths->pRaftStore->currentTerm); syncAppendEntriesReplyLog2(logBuf, pMsg); @@ -96,6 +168,7 @@ int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* p return ret; } +#endif int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntriesReply* pMsg) { int32_t ret = 0; diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index c5af72c971..3100d0525c 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -914,6 +914,9 @@ void syncNodeStart(SSyncNode* pSyncNode) { syncNodeBecomeLeader(pSyncNode, "one replica start"); // Raft 3.6.2 Committing entries from previous terms + syncNodeAppendNoop(pSyncNode); + syncMaybeAdvanceCommitIndex(pSyncNode); + return; } @@ -1662,6 +1665,12 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde // change isStandBy to normal (election timeout) if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { syncNodeBecomeLeader(pSyncNode, tmpbuf); + + // Raft 3.6.2 Committing entries from previous terms + syncNodeReplicate(pSyncNode); + syncNodeAppendNoop(pSyncNode); + syncMaybeAdvanceCommitIndex(pSyncNode); + } else { syncNodeBecomeFollower(pSyncNode, tmpbuf); } @@ -1807,16 +1816,9 @@ void syncNodeBecomeLeader(SSyncNode* pSyncNode, const char* debugStr) { // stop elect timer syncNodeStopElectTimer(pSyncNode); - // start replicate right now! - syncNodeReplicate(pSyncNode); - // start heartbeat timer syncNodeStartHeartbeatTimer(pSyncNode); - // append noop - syncNodeAppendNoop(pSyncNode); - syncMaybeAdvanceCommitIndex(pSyncNode); // maybe only one replica - // trace log do { int32_t debugStrLen = strlen(debugStr); @@ -1841,9 +1843,9 @@ void syncNodeCandidate2Leader(SSyncNode* pSyncNode) { syncNodeLog2("==state change syncNodeCandidate2Leader==", pSyncNode); // Raft 3.6.2 Committing entries from previous terms - - // do not use this - // syncNodeEqNoop(pSyncNode); + syncNodeReplicate(pSyncNode); + syncNodeAppendNoop(pSyncNode); + syncMaybeAdvanceCommitIndex(pSyncNode); } void syncNodeFollower2Candidate(SSyncNode* pSyncNode) { diff --git a/source/libs/sync/src/syncRequestVote.c b/source/libs/sync/src/syncRequestVote.c index 3270b5c0e4..95dec6cb83 100644 --- a/source/libs/sync/src/syncRequestVote.c +++ b/source/libs/sync/src/syncRequestVote.c @@ -45,6 +45,75 @@ int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg) { int32_t ret = 0; + syncRequestVoteLog2("==syncNodeOnRequestVoteCb==", pMsg); + + // if already drop replica, do not process + if (!syncNodeInRaftGroup(ths, &(pMsg->srcId)) && !ths->pRaftCfg->isStandBy) { + do { + char logBuf[256]; + char host[64]; + uint16_t port; + syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); + snprintf(logBuf, sizeof(logBuf), + "recv sync-request-vote from %s:%d, term:%lu, lindex:%ld, lterm:%lu, maybe replica already dropped", + host, port, pMsg->term, pMsg->lastLogIndex, pMsg->lastLogTerm); + syncNodeEventLog(ths, logBuf); + } while (0); + + return -1; + } + + // maybe update term + if (pMsg->term > ths->pRaftStore->currentTerm) { + syncNodeUpdateTerm(ths, pMsg->term); + } + ASSERT(pMsg->term <= ths->pRaftStore->currentTerm); + + bool logOK = (pMsg->lastLogTerm > ths->pLogStore->getLastTerm(ths->pLogStore)) || + ((pMsg->lastLogTerm == ths->pLogStore->getLastTerm(ths->pLogStore)) && + (pMsg->lastLogIndex >= ths->pLogStore->getLastIndex(ths->pLogStore))); + bool grant = (pMsg->term == ths->pRaftStore->currentTerm) && logOK && + ((!raftStoreHasVoted(ths->pRaftStore)) || (syncUtilSameId(&(ths->pRaftStore->voteFor), &(pMsg->srcId)))); + if (grant) { + // maybe has already voted for pMsg->srcId + // vote again, no harm + raftStoreVote(ths->pRaftStore, &(pMsg->srcId)); + + // forbid elect for this round + syncNodeResetElectTimer(ths); + } + + // send msg + SyncRequestVoteReply* pReply = syncRequestVoteReplyBuild(ths->vgId); + pReply->srcId = ths->myRaftId; + pReply->destId = pMsg->srcId; + pReply->term = ths->pRaftStore->currentTerm; + pReply->voteGranted = grant; + + // trace log + do { + char logBuf[256]; + char host[64]; + uint16_t port; + syncUtilU642Addr(pMsg->srcId.addr, host, sizeof(host), &port); + snprintf(logBuf, sizeof(logBuf), + "recv sync-request-vote from %s:%d, term:%lu, lindex:%ld, lterm:%lu, reply-grant:%d", host, port, + pMsg->term, pMsg->lastLogIndex, pMsg->lastLogTerm, pReply->voteGranted); + syncNodeEventLog(ths, logBuf); + } while (0); + + SRpcMsg rpcMsg; + syncRequestVoteReply2RpcMsg(pReply, &rpcMsg); + syncNodeSendMsgById(&pReply->destId, ths, &rpcMsg); + syncRequestVoteReplyDestroy(pReply); + + return ret; +} + +#if 0 +int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg) { + int32_t ret = 0; + char logBuf[128] = {0}; snprintf(logBuf, sizeof(logBuf), "==syncNodeOnRequestVoteCb== term:%lu", ths->pRaftStore->currentTerm); syncRequestVoteLog2(logBuf, pMsg); @@ -81,6 +150,7 @@ int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg) { return ret; } +#endif static bool syncNodeOnRequestVoteLogOK(SSyncNode* pSyncNode, SyncRequestVote* pMsg) { SyncTerm myLastTerm = syncNodeGetLastTerm(pSyncNode); diff --git a/source/libs/sync/src/syncRequestVoteReply.c b/source/libs/sync/src/syncRequestVoteReply.c index 60963d0dd3..4c205d16ec 100644 --- a/source/libs/sync/src/syncRequestVoteReply.c +++ b/source/libs/sync/src/syncRequestVoteReply.c @@ -40,6 +40,72 @@ int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg) { int32_t ret = 0; + // print log + char logBuf[128] = {0}; + snprintf(logBuf, sizeof(logBuf), "==syncNodeOnRequestVoteReplyCb== term:%lu", ths->pRaftStore->currentTerm); + syncRequestVoteReplyLog2(logBuf, pMsg); + + // if already drop replica, do not process + if (!syncNodeInRaftGroup(ths, &(pMsg->srcId)) && !ths->pRaftCfg->isStandBy) { + sInfo("recv SyncRequestVoteReply, maybe replica already dropped"); + return ret; + } + + // drop stale response + if (pMsg->term < ths->pRaftStore->currentTerm) { + sTrace("recv SyncRequestVoteReply, drop stale response, receive_term:%lu current_term:%lu", pMsg->term, + ths->pRaftStore->currentTerm); + return ret; + } + + // ASSERT(!(pMsg->term > ths->pRaftStore->currentTerm)); + // no need this code, because if I receive reply.term, then I must have sent for that term. + // if (pMsg->term > ths->pRaftStore->currentTerm) { + // syncNodeUpdateTerm(ths, pMsg->term); + // } + + if (pMsg->term > ths->pRaftStore->currentTerm) { + char logBuf[128] = {0}; + snprintf(logBuf, sizeof(logBuf), "syncNodeOnRequestVoteReplyCb error term, receive:%lu current:%lu", pMsg->term, + ths->pRaftStore->currentTerm); + syncNodePrint2(logBuf, ths); + sError("%s", logBuf); + return ret; + } + + ASSERT(pMsg->term == ths->pRaftStore->currentTerm); + + // This tallies votes even when the current state is not Candidate, + // but they won't be looked at, so it doesn't matter. + if (ths->state == TAOS_SYNC_STATE_CANDIDATE) { + votesRespondAdd(ths->pVotesRespond, pMsg); + if (pMsg->voteGranted) { + // add vote + voteGrantedVote(ths->pVotesGranted, pMsg); + + // maybe to leader + if (voteGrantedMajority(ths->pVotesGranted)) { + if (!ths->pVotesGranted->toLeader) { + syncNodeCandidate2Leader(ths); + + // prevent to leader again! + ths->pVotesGranted->toLeader = true; + } + } + } else { + ; + // do nothing + // UNCHANGED <> + } + } + + return ret; +} + +#if 0 +int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg) { + int32_t ret = 0; + char logBuf[128] = {0}; snprintf(logBuf, sizeof(logBuf), "==syncNodeOnRequestVoteReplyCb== term:%lu", ths->pRaftStore->currentTerm); syncRequestVoteReplyLog2(logBuf, pMsg); @@ -93,6 +159,7 @@ int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg) return ret; } +#endif int32_t syncNodeOnRequestVoteReplySnapshotCb(SSyncNode* ths, SyncRequestVoteReply* pMsg) { int32_t ret = 0; diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index bbd9ce3a78..963a85922f 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -105,6 +105,13 @@ typedef SRpcCtxVal STransCtxVal; typedef SRpcInfo STrans; typedef SRpcConnInfo STransHandleInfo; +// ref mgt +// handle +typedef struct SExHandle { + void* handle; + int64_t refId; + void* pThrd; +} SExHandle; /*convet from fqdn to ip */ typedef struct SCvtAddr { char ip[TSDB_FQDN_LEN]; @@ -113,14 +120,15 @@ typedef struct SCvtAddr { } SCvtAddr; typedef struct { - SEpSet epSet; // ip list provided by app - SEpSet origEpSet; - void* ahandle; // handle provided by app - tmsg_t msgType; // message type - int8_t connType; // connection type cli/srv - int64_t rid; // refId returned by taosAddRef + SEpSet epSet; // ip list provided by app + SEpSet origEpSet; + void* ahandle; // handle provided by app + tmsg_t msgType; // message type + int8_t connType; // connection type cli/srv - int8_t retryCount; + int8_t retryCnt; + int8_t retryLimit; + // bool setMaxRetry; STransCtx appCtx; // STransMsg* pRsp; // for synchronous API tsem_t* pSem; // for synchronous API @@ -239,6 +247,32 @@ int transSendAsync(SAsyncPool* pool, queue* mq); } \ } \ } while (0) + +#define ASYNC_CHECK_HANDLE(exh1, id) \ + do { \ + if (id > 0) { \ + tTrace("handle step1"); \ + SExHandle* exh2 = transAcquireExHandle(refMgt, id); \ + if (exh2 == NULL || id != exh2->refId) { \ + tTrace("handle %p except, may already freed, ignore msg, ref1: %" PRIu64 ", ref2 : %" PRIu64 "", exh1, \ + exh2 ? exh2->refId : 0, id); \ + goto _return1; \ + } \ + } else if (id == 0) { \ + tTrace("handle step2"); \ + SExHandle* exh2 = transAcquireExHandle(refMgt, id); \ + if (exh2 == NULL || id == exh2->refId) { \ + tTrace("handle %p except, may already freed, ignore msg, ref1: %" PRIu64 ", ref2 : %" PRIu64 "", exh1, id, \ + exh2 ? exh2->refId : 0); \ + goto _return1; \ + } else { \ + id = exh1->refId; \ + } \ + } else if (id < 0) { \ + tTrace("handle step3"); \ + goto _return2; \ + } \ + } while (0) int transInitBuffer(SConnBuffer* buf); int transClearBuffer(SConnBuffer* buf); int transDestroyBuffer(SConnBuffer* buf); @@ -349,21 +383,13 @@ void transDQDestroy(SDelayQueue* queue); int transDQSched(SDelayQueue* queue, void (*func)(void* arg), void* arg, uint64_t timeoutMs); -void transPrintEpSet(SEpSet* pEpSet); +// void transPrintEpSet(SEpSet* pEpSet); bool transEpSetIsEqual(SEpSet* a, SEpSet* b); /* * init global func */ void transThreadOnce(); -// ref mgt -// handle -typedef struct SExHandle { - void* handle; - int64_t refId; - void* pThrd; -} SExHandle; - void transInitEnv(); int32_t transOpenExHandleMgt(int size); void transCloseExHandleMgt(int32_t mgt); diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index 1ec96f4a7a..4f7b19b539 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -79,6 +79,7 @@ void* rpcOpen(const SRpcInit* pInit) { return pRpc; } void rpcClose(void* arg) { + tInfo("start to close rpc"); SRpcInfo* pRpc = (SRpcInfo*)arg; (*taosCloseHandle[pRpc->connType])(pRpc->tcphandle); transCloseExHandleMgt(pRpc->refMgt); diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index e563bf0ddd..7374d1fffc 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -1,4 +1,4 @@ -/* * Copyright (c) 2019 TAOS Data, Inc. +/** Copyright (c) 2019 TAOS Data, Inc. * * This program is free software: you can use, redistribute, and/or modify * it under the terms of the GNU Affero General Public License, version 3 @@ -25,7 +25,6 @@ typedef struct SCliConn { uv_write_t writeReq; void* hostThrd; - int hThrdIdx; SConnBuffer readBuf; STransQueue cliMsgs; @@ -36,6 +35,7 @@ typedef struct SCliConn { bool broken; // link broken or not ConnStatus status; // + int64_t refId; char* ip; uint32_t port; @@ -54,7 +54,7 @@ typedef struct SCliMsg { int sent; //(0: no send, 1: alread sent) } SCliMsg; -typedef struct SCliThrdObj { +typedef struct SCliThrd { TdThread thread; // tid int64_t pid; // pid uv_loop_t* loop; @@ -72,13 +72,13 @@ typedef struct SCliThrdObj { SCvtAddr cvtAddr; bool quit; -} SCliThrdObj; +} SCliThrd; typedef struct SCliObj { - char label[TSDB_LABEL_LEN]; - int32_t index; - int numOfThreads; - SCliThrdObj** pThreadObj; + char label[TSDB_LABEL_LEN]; + int32_t index; + int numOfThreads; + SCliThrd** pThreadObj; } SCliObj; typedef struct SConnList { @@ -106,11 +106,18 @@ static void cliAsyncCb(uv_async_t* handle); static int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg); -static SCliConn* cliCreateConn(SCliThrdObj* thrd); +static SCliConn* cliCreateConn(SCliThrd* thrd); static void cliDestroyConn(SCliConn* pConn, bool clear /*clear tcp handle or not*/); static void cliDestroy(uv_handle_t* handle); static void cliSend(SCliConn* pConn); +static bool cliIsEpsetUpdated(int32_t code, STransConnCtx* pCtx) { + if (code != 0) return false; + if (pCtx->retryCnt == 0) return false; + if (transEpSetIsEqual(&pCtx->epSet, &pCtx->origEpSet)) return false; + return true; +} + void cliMayCvtFqdnToIp(SEpSet* pEpSet, SCvtAddr* pCvtAddr); /* * set TCP connection timeout per-socket level @@ -122,14 +129,14 @@ static void cliHandleResp(SCliConn* conn); static void cliHandleExcept(SCliConn* conn); // handle req from app -static void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd); -static void cliHandleQuit(SCliMsg* pMsg, SCliThrdObj* pThrd); -static void cliHandleRelease(SCliMsg* pMsg, SCliThrdObj* pThrd); -static void cliHandleUpdate(SCliMsg* pMsg, SCliThrdObj* pThrd); -static void (*cliAsyncHandle[])(SCliMsg* pMsg, SCliThrdObj* pThrd) = {cliHandleReq, cliHandleQuit, cliHandleRelease, - NULL, cliHandleUpdate}; +static void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd); +static void cliHandleQuit(SCliMsg* pMsg, SCliThrd* pThrd); +static void cliHandleRelease(SCliMsg* pMsg, SCliThrd* pThrd); +static void cliHandleUpdate(SCliMsg* pMsg, SCliThrd* pThrd); +static void (*cliAsyncHandle[])(SCliMsg* pMsg, SCliThrd* pThrd) = {cliHandleReq, cliHandleQuit, cliHandleRelease, NULL, + cliHandleUpdate}; -static void cliSendQuit(SCliThrdObj* thrd); +static void cliSendQuit(SCliThrd* thrd); static void destroyUserdata(STransMsg* userdata); static int cliRBChoseIdx(STrans* pTransInst); @@ -137,8 +144,8 @@ static int cliRBChoseIdx(STrans* pTransInst); static void destroyCmsg(SCliMsg* cmsg); static void transDestroyConnCtx(STransConnCtx* ctx); // thread obj -static SCliThrdObj* createThrdObj(); -static void destroyThrdObj(SCliThrdObj* pThrd); +static SCliThrd* createThrdObj(); +static void destroyThrdObj(SCliThrd* pThrd); static void cliWalkCb(uv_handle_t* handle, void* arg); @@ -154,7 +161,6 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { destroyCmsg(pMsg); } } - #define CLI_RELEASE_UV(loop) \ do { \ uv_walk(loop, cliWalkCb, NULL); \ @@ -168,17 +174,23 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { snprintf(key, sizeof(key), "%s:%d", ip, (int)port); \ } while (0) -#define CONN_HOST_THREAD_IDX(conn) (conn ? ((SCliConn*)conn)->hThrdIdx : -1) +#define CONN_HOST_THREAD_IDX1(idx, exh, refId, pThrd) \ + do { \ + if (exh == NULL) { \ + idx = -1; \ + } else { \ + ASYNC_CHECK_HANDLE((exh), refId); \ + pThrd = (SCliThrd*)(exh)->pThrd; \ + } \ + } while (0) #define CONN_PERSIST_TIME(para) (para * 1000 * 10) #define CONN_GET_HOST_THREAD(conn) (conn ? ((SCliConn*)conn)->hostThrd : NULL) -#define CONN_GET_INST_LABEL(conn) (((STrans*)(((SCliThrdObj*)(conn)->hostThrd)->pTransInst))->label) +#define CONN_GET_INST_LABEL(conn) (((STrans*)(((SCliThrd*)(conn)->hostThrd)->pTransInst))->label) #define CONN_SHOULD_RELEASE(conn, head) \ do { \ if ((head)->release == 1 && (head->msgLen) == sizeof(*head)) { \ - int connStatus = conn->status; \ uint64_t ahandle = head->ahandle; \ CONN_GET_MSGCTX_BY_AHANDLE(conn, ahandle); \ - conn->status = ConnRelease; \ transClearBuffer(&conn->readBuf); \ transFreeMsg(transContFromHead((char*)head)); \ tDebug("%s conn %p receive release request, ref: %d", CONN_GET_INST_LABEL(conn), conn, T_REF_VAL_GET(conn)); \ @@ -187,9 +199,7 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { } \ destroyCmsg(pMsg); \ cliReleaseUnfinishedMsg(conn); \ - if (connStatus != ConnInPool) { \ - addConnToPool(((SCliThrdObj*)conn->hostThrd)->pool, conn); \ - } \ + addConnToPool(((SCliThrd*)conn->hostThrd)->pool, conn); \ return; \ } \ } while (0) @@ -255,8 +265,25 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { #define REQUEST_PERSIS_HANDLE(msg) ((msg)->info.persistHandle == 1) #define REQUEST_RELEASE_HANDLE(cmsg) ((cmsg)->type == Release) +#define EPSET_GET_SIZE(epSet) (epSet)->numOfEps #define EPSET_GET_INUSE_IP(epSet) ((epSet)->eps[(epSet)->inUse].fqdn) #define EPSET_GET_INUSE_PORT(epSet) ((epSet)->eps[(epSet)->inUse].port) +#define EPSET_FORWARD_INUSE(epSet) \ + do { \ + (epSet)->inUse = (++((epSet)->inUse)) % ((epSet)->numOfEps); \ + } while (0) +#define EPSET_DEBUG_STR(epSet, tbuf) \ + do { \ + int len = snprintf(tbuf, sizeof(tbuf), "epset:{"); \ + for (int i = 0; i < (epSet)->numOfEps; i++) { \ + if (i == (epSet)->numOfEps - 1) { \ + len += snprintf(tbuf + len, sizeof(tbuf) - len, "%d. %s:%d", i, (epSet)->eps[i].fqdn, (epSet)->eps[i].port); \ + } else { \ + len += snprintf(tbuf + len, sizeof(tbuf) - len, "%d. %s:%d, ", i, (epSet)->eps[i].fqdn, (epSet)->eps[i].port); \ + } \ + } \ + len += snprintf(tbuf + len, sizeof(tbuf) - len, "}, inUse:%d", (epSet)->inUse); \ + } while (0); static void* cliWorkThread(void* arg); @@ -271,8 +298,8 @@ _RETURN: return false; } void cliHandleResp(SCliConn* conn) { - SCliThrdObj* pThrd = conn->hostThrd; - STrans* pTransInst = pThrd->pTransInst; + SCliThrd* pThrd = conn->hostThrd; + STrans* pTransInst = pThrd->pTransInst; STransMsgHead* pHead = (STransMsgHead*)(conn->readBuf.buf); pHead->code = htonl(pHead->code); @@ -292,17 +319,9 @@ void cliHandleResp(SCliConn* conn) { if (CONN_NO_PERSIST_BY_APP(conn)) { pMsg = transQueuePop(&conn->cliMsgs); - pCtx = pMsg ? pMsg->ctx : NULL; - if (pMsg == NULL && !CONN_NO_PERSIST_BY_APP(conn)) { - transMsg.info.ahandle = transCtxDumpVal(&conn->ctx, transMsg.msgType); - if (transMsg.info.ahandle == NULL) { - transMsg.info.ahandle = transCtxDumpBrokenlinkVal(&conn->ctx, (int32_t*)&(transMsg.msgType)); - } - tDebug("%s conn %p construct ahandle %p, persist: 0", CONN_GET_INST_LABEL(conn), conn, transMsg.info.ahandle); - } else { - transMsg.info.ahandle = pCtx ? pCtx->ahandle : NULL; - tDebug("%s conn %p get ahandle %p, persist: 0", CONN_GET_INST_LABEL(conn), conn, transMsg.info.ahandle); - } + pCtx = pMsg->ctx; + transMsg.info.ahandle = pCtx->ahandle; + tDebug("%s conn %p get ahandle %p, persist: 0", CONN_GET_INST_LABEL(conn), conn, transMsg.info.ahandle); } else { uint64_t ahandle = (uint64_t)pHead->ahandle; CONN_GET_MSGCTX_BY_AHANDLE(conn, ahandle); @@ -324,26 +343,22 @@ void cliHandleResp(SCliConn* conn) { } // buf's mem alread translated to transMsg.pCont transClearBuffer(&conn->readBuf); - if (!CONN_NO_PERSIST_BY_APP(conn)) { - transMsg.info.handle = conn; + transMsg.info.handle = (void*)conn->refId; tDebug("%s conn %p ref by app", CONN_GET_INST_LABEL(conn), conn); } - // char buf[64] = {0}; - // TRACE_TO_STR(&transMsg.info.traceId, buf); + STraceId* trace = &transMsg.info.traceId; - tGTrace("conn %p %s received from %s:%d, local info: %s:%d, msg size: %d, code: %d", conn, TMSG_INFO(pHead->msgType), - taosInetNtoa(conn->addr.sin_addr), ntohs(conn->addr.sin_port), taosInetNtoa(conn->localAddr.sin_addr), - ntohs(conn->localAddr.sin_port), transMsg.contLen, transMsg.code); + tGTrace("%s conn %p %s received from %s:%d, local info: %s:%d, msg size: %d, code: %d", CONN_GET_INST_LABEL(conn), + conn, TMSG_INFO(pHead->msgType), taosInetNtoa(conn->addr.sin_addr), ntohs(conn->addr.sin_port), + taosInetNtoa(conn->localAddr.sin_addr), ntohs(conn->localAddr.sin_port), transMsg.contLen, transMsg.code); if (pCtx == NULL && CONN_NO_PERSIST_BY_APP(conn)) { - tDebug("%s except, server continue send while cli ignore it", CONN_GET_INST_LABEL(conn)); - // transUnrefCliHandle(conn); + tDebug("%s except, conn %p read while cli ignore it", CONN_GET_INST_LABEL(conn), conn); return; } if (CONN_RELEASE_BY_SERVER(conn) && transMsg.info.ahandle == NULL) { - tDebug("%s except, server continue send while cli ignore it", CONN_GET_INST_LABEL(conn)); - // transUnrefCliHandle(conn); + tDebug("%s except, conn %p read while cli ignore it", CONN_GET_INST_LABEL(conn), conn); return; } @@ -375,9 +390,9 @@ void cliHandleExcept(SCliConn* pConn) { return; } } - SCliThrdObj* pThrd = pConn->hostThrd; - STrans* pTransInst = pThrd->pTransInst; - bool once = false; + SCliThrd* pThrd = pConn->hostThrd; + STrans* pTransInst = pThrd->pTransInst; + bool once = false; do { SCliMsg* pMsg = transQueuePop(&pConn->cliMsgs); if (pMsg == NULL && once) { @@ -389,7 +404,6 @@ void cliHandleExcept(SCliConn* pConn) { transMsg.code = TSDB_CODE_RPC_NETWORK_UNAVAIL; transMsg.msgType = pMsg ? pMsg->msg.msgType + 1 : 0; transMsg.info.ahandle = NULL; - transMsg.info.handle = pConn; if (pMsg == NULL && !CONN_NO_PERSIST_BY_APP(pConn)) { transMsg.info.ahandle = transCtxDumpVal(&pConn->ctx, transMsg.msgType); @@ -414,15 +428,15 @@ void cliHandleExcept(SCliConn* pConn) { return; } destroyCmsg(pMsg); - tTrace("%s conn %p start to destroy", CONN_GET_INST_LABEL(pConn), pConn); + tTrace("%s conn %p start to destroy, ref:%d", CONN_GET_INST_LABEL(pConn), pConn, T_REF_VAL_GET(pConn)); } while (!transQueueEmpty(&pConn->cliMsgs)); transUnrefCliHandle(pConn); } void cliTimeoutCb(uv_timer_t* handle) { - SCliThrdObj* pThrd = handle->data; - STrans* pTransInst = pThrd->pTransInst; - int64_t currentTime = pThrd->nextTimeout; + SCliThrd* pThrd = handle->data; + STrans* pTransInst = pThrd->pTransInst; + int64_t currentTime = pThrd->nextTimeout; tTrace("%s conn timeout, try to remove expire conn from conn pool", pTransInst->label); SConnList* p = taosHashIterate((SHashObj*)pThrd->pool, NULL); @@ -487,10 +501,26 @@ static SCliConn* getConnFromPool(void* pool, char* ip, uint32_t port) { assert(h == &conn->conn); return conn; } +static void allocConnRef(SCliConn* conn, bool update) { + if (update) { + transRemoveExHandle(refMgt, conn->refId); + } + SExHandle* exh = taosMemoryCalloc(1, sizeof(SExHandle)); + exh->handle = conn; + exh->pThrd = conn->hostThrd; + exh->refId = transAddExHandle(refMgt, exh); + conn->refId = exh->refId; +} static void addConnToPool(void* pool, SCliConn* conn) { - SCliThrdObj* thrd = conn->hostThrd; + if (conn->status == ConnInPool) { + // assert(0); + return; + } + SCliThrd* thrd = conn->hostThrd; CONN_HANDLE_THREAD_QUIT(thrd); + allocConnRef(conn, true); + STrans* pTransInst = thrd->pTransInst; conn->expireTime = taosGetTimestampMs() + CONN_PERSIST_TIME(pTransInst->idleTime); transQueueClear(&conn->cliMsgs); @@ -499,7 +529,7 @@ static void addConnToPool(void* pool, SCliConn* conn) { char key[128] = {0}; CONN_CONSTRUCT_HASH_KEY(key, conn->ip, conn->port); - tTrace("%s conn %p added to conn pool, read buf cap: %d", CONN_GET_INST_LABEL(conn), conn, conn->readBuf.cap); + tTrace("%s conn %p added to conn pool, read buf cap:%d", CONN_GET_INST_LABEL(conn), conn, conn->readBuf.cap); SConnList* plist = taosHashGet((SHashObj*)pool, key, strlen(key)); // list already create before @@ -540,13 +570,14 @@ static void cliRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) { return; } if (nread < 0) { - tError("%s conn %p read error: %s", CONN_GET_INST_LABEL(conn), conn, uv_err_name(nread)); + tError("%s conn %p read error: %s, ref: %d", CONN_GET_INST_LABEL(conn), conn, uv_err_name(nread), + T_REF_VAL_GET(conn)); conn->broken = true; cliHandleExcept(conn); } } -static SCliConn* cliCreateConn(SCliThrdObj* pThrd) { +static SCliConn* cliCreateConn(SCliThrd* pThrd) { SCliConn* conn = taosMemoryCalloc(1, sizeof(SCliConn)); // read/write stream handle conn->stream = (uv_stream_t*)taosMemoryMalloc(sizeof(uv_tcp_t)); @@ -562,11 +593,16 @@ static SCliConn* cliCreateConn(SCliThrdObj* pThrd) { conn->status = ConnNormal; conn->broken = 0; transRefCliHandle(conn); + + allocConnRef(conn, false); + return conn; } static void cliDestroyConn(SCliConn* conn, bool clear) { tTrace("%s conn %p remove from conn pool", CONN_GET_INST_LABEL(conn), conn); QUEUE_REMOVE(&conn->conn); + QUEUE_INIT(&conn->conn); + transRemoveExHandle(refMgt, conn->refId); if (clear) { uv_close((uv_handle_t*)conn->stream, cliDestroy); } @@ -593,7 +629,7 @@ static bool cliHandleNoResp(SCliConn* conn) { } if (res == true) { if (cliMaySendCachedMsg(conn) == false) { - SCliThrdObj* thrd = conn->hostThrd; + SCliThrd* thrd = conn->hostThrd; addConnToPool(thrd->pool, conn); } } @@ -629,8 +665,8 @@ void cliSend(SCliConn* pConn) { STransConnCtx* pCtx = pCliMsg->ctx; - SCliThrdObj* pThrd = pConn->hostThrd; - STrans* pTransInst = pThrd->pTransInst; + SCliThrd* pThrd = pConn->hostThrd; + STrans* pTransInst = pThrd->pTransInst; STransMsg* pMsg = (STransMsg*)(&pCliMsg->msg); if (pMsg->pCont == 0) { @@ -651,12 +687,10 @@ void cliSend(SCliConn* pConn) { uv_buf_t wb = uv_buf_init((char*)pHead, msgLen); - // char buf[64] = {0}; - // TRACE_TO_STR(&pMsg->info.traceId, buf); STraceId* trace = &pMsg->info.traceId; - tGTrace("conn %p %s is sent to %s:%d, local info %s:%d", pConn, TMSG_INFO(pHead->msgType), - taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), taosInetNtoa(pConn->localAddr.sin_addr), - ntohs(pConn->localAddr.sin_port)); + tGTrace("%s conn %p %s is sent to %s:%d, local info %s:%d", CONN_GET_INST_LABEL(pConn), pConn, + TMSG_INFO(pHead->msgType), taosInetNtoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), + taosInetNtoa(pConn->localAddr.sin_addr), ntohs(pConn->localAddr.sin_port)); if (pHead->persist == 1) { CONN_SET_PERSIST_BY_APP(pConn); @@ -664,7 +698,6 @@ void cliSend(SCliConn* pConn) { pConn->writeReq.data = pConn; uv_write(&pConn->writeReq, (uv_stream_t*)pConn->stream, &wb, 1, cliSendCb); - return; _RETURN: return; @@ -690,19 +723,24 @@ void cliConnCb(uv_connect_t* req, int status) { cliSend(pConn); } -static void cliHandleQuit(SCliMsg* pMsg, SCliThrdObj* pThrd) { +static void cliHandleQuit(SCliMsg* pMsg, SCliThrd* pThrd) { + pThrd->quit = true; tDebug("cli work thread %p start to quit", pThrd); destroyCmsg(pMsg); destroyConnPool(pThrd->pool); uv_timer_stop(&pThrd->timer); uv_walk(pThrd->loop, cliWalkCb, NULL); - pThrd->quit = true; - // uv_stop(pThrd->loop); } -static void cliHandleRelease(SCliMsg* pMsg, SCliThrdObj* pThrd) { - SCliConn* conn = pMsg->msg.info.handle; +static void cliHandleRelease(SCliMsg* pMsg, SCliThrd* pThrd) { + int64_t refId = (int64_t)(pMsg->msg.info.handle); + SExHandle* exh = transAcquireExHandle(refMgt, refId); + if (exh == NULL) { + tDebug("%" PRId64 " already release", refId); + } + + SCliConn* conn = exh->handle; tDebug("%s conn %p start to release to inst", CONN_GET_INST_LABEL(conn), conn); if (T_REF_VAL_GET(conn) == 2) { @@ -711,33 +749,37 @@ static void cliHandleRelease(SCliMsg* pMsg, SCliThrdObj* pThrd) { return; } cliSend(conn); - } else { - // conn already broken down - transUnrefCliHandle(conn); } } -static void cliHandleUpdate(SCliMsg* pMsg, SCliThrdObj* pThrd) { +static void cliHandleUpdate(SCliMsg* pMsg, SCliThrd* pThrd) { STransConnCtx* pCtx = pMsg->ctx; - pThrd->cvtAddr = pCtx->cvtAddr; destroyCmsg(pMsg); } -SCliConn* cliGetConn(SCliMsg* pMsg, SCliThrdObj* pThrd) { +SCliConn* cliGetConn(SCliMsg* pMsg, SCliThrd* pThrd, bool* ignore) { SCliConn* conn = NULL; - if (pMsg->msg.info.handle != NULL) { - conn = (SCliConn*)(pMsg->msg.info.handle); - if (conn != NULL) { - tTrace("%s conn %p reused", CONN_GET_INST_LABEL(conn), conn); - } - } else { - STransConnCtx* pCtx = pMsg->ctx; - conn = getConnFromPool(pThrd->pool, EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet)); - if (conn != NULL) { - tTrace("%s conn %p get from conn pool", CONN_GET_INST_LABEL(conn), conn); + int64_t refId = (int64_t)(pMsg->msg.info.handle); + if (refId != 0) { + SExHandle* exh = transAcquireExHandle(refMgt, refId); + if (exh == NULL) { + *ignore = true; + destroyCmsg(pMsg); + return NULL; + // assert(0); } else { - tTrace("%s not found conn in conn pool %p", ((STrans*)pThrd->pTransInst)->label, pThrd->pool); + conn = exh->handle; + transReleaseExHandle(refMgt, refId); } + return conn; + }; + + STransConnCtx* pCtx = pMsg->ctx; + conn = getConnFromPool(pThrd->pool, EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet)); + if (conn != NULL) { + tTrace("%s conn %p get from conn pool:%p", CONN_GET_INST_LABEL(conn), conn, pThrd->pool); + } else { + tTrace("%s not found conn in conn pool:%p", ((STrans*)pThrd->pTransInst)->label, pThrd->pool); } return conn; } @@ -752,22 +794,19 @@ void cliMayCvtFqdnToIp(SEpSet* pEpSet, SCvtAddr* pCvtAddr) { } } } -void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { - uint64_t et = taosGetTimestampUs(); - uint64_t el = et - pMsg->st; - // tTrace("%s cli msg tran time cost: %" PRIu64 "us", ((STrans*)pThrd->pTransInst)->label, el); - +void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) { STransConnCtx* pCtx = pMsg->ctx; STrans* pTransInst = pThrd->pTransInst; cliMayCvtFqdnToIp(&pCtx->epSet, &pThrd->cvtAddr); - transPrintEpSet(&pCtx->epSet); - - SCliConn* conn = cliGetConn(pMsg, pThrd); + // transPrintEpSet(&pCtx->epSet); + bool ignore = false; + SCliConn* conn = cliGetConn(pMsg, pThrd, &ignore); + if (ignore == true) { + return; + } if (conn != NULL) { - conn->hThrdIdx = pCtx->hThrdIdx; - transCtxMerge(&conn->ctx, &pCtx->appCtx); transQueuePush(&conn->cliMsgs, pMsg); cliSend(conn); @@ -776,7 +815,6 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { transCtxMerge(&conn->ctx, &pCtx->appCtx); transQueuePush(&conn->cliMsgs, pMsg); - conn->hThrdIdx = pCtx->hThrdIdx; conn->ip = strdup(EPSET_GET_INUSE_IP(&pCtx->epSet)); conn->port = EPSET_GET_INUSE_PORT(&pCtx->epSet); @@ -784,7 +822,7 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { if (ret) { tError("%s conn %p failed to set conn option, errmsg %s", transLabel(pTransInst), conn, uv_err_name(ret)); } - int fd = taosCreateSocketWithTimeOutOpt(TRANS_CONN_TIMEOUT); + int32_t fd = taosCreateSocketWithTimeout(TRANS_CONN_TIMEOUT); if (fd == -1) { tTrace("%s conn %p failed to create socket", transLabel(pTransInst), conn); cliHandleExcept(conn); @@ -807,9 +845,9 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { } } static void cliAsyncCb(uv_async_t* handle) { - SAsyncItem* item = handle->data; - SCliThrdObj* pThrd = item->pThrd; - SCliMsg* pMsg = NULL; + SAsyncItem* item = handle->data; + SCliThrd* pThrd = item->pThrd; + SCliMsg* pMsg = NULL; // batch process to avoid to lock/unlock frequently queue wq; @@ -835,7 +873,7 @@ static void cliAsyncCb(uv_async_t* handle) { } static void* cliWorkThread(void* arg) { - SCliThrdObj* pThrd = (SCliThrdObj*)arg; + SCliThrd* pThrd = (SCliThrd*)arg; pThrd->pid = taosGetSelfPthreadId(); setThreadName("trans-cli-work"); uv_run(pThrd->loop, UV_RUN_DEFAULT); @@ -848,10 +886,10 @@ void* transInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, STrans* pTransInst = shandle; memcpy(cli->label, label, strlen(label)); cli->numOfThreads = numOfThreads; - cli->pThreadObj = (SCliThrdObj**)taosMemoryCalloc(cli->numOfThreads, sizeof(SCliThrdObj*)); + cli->pThreadObj = (SCliThrd**)taosMemoryCalloc(cli->numOfThreads, sizeof(SCliThrd*)); for (int i = 0; i < cli->numOfThreads; i++) { - SCliThrdObj* pThrd = createThrdObj(); + SCliThrd* pThrd = createThrdObj(); pThrd->nextTimeout = taosGetTimestampMs() + CONN_PERSIST_TIME(pTransInst->idleTime); pThrd->pTransInst = shandle; @@ -885,8 +923,8 @@ static void destroyCmsg(SCliMsg* pMsg) { taosMemoryFree(pMsg); } -static SCliThrdObj* createThrdObj() { - SCliThrdObj* pThrd = (SCliThrdObj*)taosMemoryCalloc(1, sizeof(SCliThrdObj)); +static SCliThrd* createThrdObj() { + SCliThrd* pThrd = (SCliThrd*)taosMemoryCalloc(1, sizeof(SCliThrd)); QUEUE_INIT(&pThrd->msg); taosThreadMutexInit(&pThrd->msgMtx, NULL); @@ -904,7 +942,7 @@ static SCliThrdObj* createThrdObj() { pThrd->quit = false; return pThrd; } -static void destroyThrdObj(SCliThrdObj* pThrd) { +static void destroyThrdObj(SCliThrd* pThrd) { if (pThrd == NULL) { return; } @@ -925,7 +963,7 @@ static void transDestroyConnCtx(STransConnCtx* ctx) { taosMemoryFree(ctx); } -void cliSendQuit(SCliThrdObj* thrd) { +void cliSendQuit(SCliThrd* thrd) { // cli can stop gracefully SCliMsg* msg = taosMemoryCalloc(1, sizeof(SCliMsg)); msg->type = Quit; @@ -938,7 +976,10 @@ void cliWalkCb(uv_handle_t* handle, void* arg) { } int cliRBChoseIdx(STrans* pTransInst) { - int64_t index = pTransInst->index; + int8_t index = pTransInst->index; + if (pTransInst->numOfThreads == 0) { + return -1; + } if (pTransInst->index++ >= pTransInst->numOfThreads) { pTransInst->index = 0; } @@ -946,82 +987,71 @@ int cliRBChoseIdx(STrans* pTransInst) { } static void doDelayTask(void* param) { STaskArg* arg = param; - - SCliMsg* pMsg = arg->param1; - SCliThrdObj* pThrd = arg->param2; - cliHandleReq(pMsg, pThrd); - + SCliMsg* pMsg = arg->param1; + SCliThrd* pThrd = arg->param2; taosMemoryFree(arg); + + cliHandleReq(pMsg, pThrd); +} + +static void cliSchedMsgToNextNode(SCliMsg* pMsg, SCliThrd* pThrd) { + STransConnCtx* pCtx = pMsg->ctx; + + STraceId* trace = &pMsg->msg.info.traceId; + char tbuf[256] = {0}; + EPSET_DEBUG_STR(&pCtx->epSet, tbuf); + tGTrace("%s retry on next node, use %s, retryCnt:%d, limit:%d", transLabel(pThrd->pTransInst), tbuf, + pCtx->retryCnt + 1, pCtx->retryLimit); + + STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); + arg->param1 = pMsg; + arg->param2 = pThrd; + transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL); +} + +void cliCompareAndSwap(int8_t* val, int8_t exp, int8_t newVal) { + if (*val != exp) { + *val = newVal; + } } int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { - SCliThrdObj* pThrd = pConn->hostThrd; - STrans* pTransInst = pThrd->pTransInst; + SCliThrd* pThrd = pConn->hostThrd; + STrans* pTransInst = pThrd->pTransInst; if (pMsg == NULL || pMsg->ctx == NULL) { tTrace("%s conn %p handle resp", pTransInst->label, pConn); pTransInst->cfp(pTransInst->parent, pResp, NULL); return 0; } - - STransConnCtx* pCtx = pMsg->ctx; - SEpSet* pEpSet = &pCtx->epSet; - - if (pCtx->retryCount == 0) { - pCtx->origEpSet = pCtx->epSet; - } /* - * upper layer handle retry if code equal TSDB_CODE_RPC_NETWORK_UNAVAIL + * no retry + * 1. query conn + * 2. rpc thread already receive quit msg */ - tmsg_t msgType = pCtx->msgType; - if ((pTransInst->retry != NULL && pEpSet->numOfEps > 1 && (pTransInst->retry(pResp->code))) || - (pResp->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || pResp->code == TSDB_CODE_APP_NOT_READY || - pResp->code == TSDB_CODE_NODE_NOT_DEPLOYED || pResp->code == TSDB_CODE_SYN_NOT_LEADER)) { + STransConnCtx* pCtx = pMsg->ctx; + int32_t code = pResp->code; + if (pTransInst->retry != NULL && pTransInst->retry(code)) { pMsg->sent = 0; - tTrace("try to send req to next node"); - pMsg->st = taosGetTimestampUs(); - pCtx->retryCount += 1; - if (pResp->code == TSDB_CODE_RPC_NETWORK_UNAVAIL && pCtx->setMaxRetry == false) { - if (pCtx->retryCount < pEpSet->numOfEps * 3) { - pEpSet->inUse = (++pEpSet->inUse) % pEpSet->numOfEps; - if (pThrd->quit == false) { - STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); - arg->param1 = pMsg; - arg->param2 = pThrd; - transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL); - transPrintEpSet(pEpSet); - tTrace("%s use local epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse, - pCtx->retryCount + 1, pEpSet->numOfEps * 3); - - transUnrefCliHandle(pConn); - return -1; - } + pCtx->retryCnt += 1; + if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { + cliCompareAndSwap(&pCtx->retryLimit, TRANS_RETRY_COUNT_LIMIT, EPSET_GET_SIZE(&pCtx->epSet) * 3); + if (pCtx->retryCnt < pCtx->retryLimit) { + transUnrefCliHandle(pConn); + EPSET_FORWARD_INUSE(&pCtx->epSet); + cliSchedMsgToNextNode(pMsg, pThrd); + return -1; } - } else if (pCtx->retryCount < TRANS_RETRY_COUNT_LIMIT) { - pCtx->setMaxRetry = true; - if (pResp->contLen == 0) { - pEpSet->inUse = (++pEpSet->inUse) % pEpSet->numOfEps; - transPrintEpSet(&pCtx->epSet); - tTrace("%s use local epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse, - pCtx->retryCount + 1, TRANS_RETRY_COUNT_LIMIT); - } else { - SEpSet epSet = {0}; - tDeserializeSEpSet(pResp->pCont, pResp->contLen, &epSet); - pCtx->epSet = epSet; - - transPrintEpSet(&pCtx->epSet); - tTrace("%s use remote epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse, - pCtx->retryCount + 1, TRANS_RETRY_COUNT_LIMIT); - } - if (pThrd->quit == false) { - if (pResp->code != TSDB_CODE_RPC_NETWORK_UNAVAIL) { - if (pConn->status != ConnInPool) addConnToPool(pThrd->pool, pConn); + } else { + cliCompareAndSwap(&pCtx->retryLimit, TRANS_RETRY_COUNT_LIMIT, TRANS_RETRY_COUNT_LIMIT); + if (pCtx->retryCnt < pCtx->retryLimit) { + addConnToPool(pThrd->pool, pConn); + if (pResp->contLen == 0) { + EPSET_FORWARD_INUSE(&pCtx->epSet); } else { - transUnrefCliHandle(pConn); + tDeserializeSEpSet(pResp->pCont, pResp->contLen, &pCtx->epSet); } - STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); - arg->param1 = pMsg; - arg->param2 = pThrd; - transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL); + transFreeMsg(pResp->pCont); + cliSchedMsgToNextNode(pMsg, pThrd); return -1; } } @@ -1029,20 +1059,20 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { STraceId* trace = &pResp->info.traceId; if (pCtx->pSem != NULL) { - tGTrace("conn %p(sync) handle resp", pConn); + tGTrace("%s conn %p(sync) handle resp", CONN_GET_INST_LABEL(pConn), pConn); if (pCtx->pRsp == NULL) { - tGTrace("conn %p(sync) failed to resp, ignore", pConn); + tGTrace("%s conn %p(sync) failed to resp, ignore", CONN_GET_INST_LABEL(pConn), pConn); } else { memcpy((char*)pCtx->pRsp, (char*)pResp, sizeof(*pResp)); } tsem_post(pCtx->pSem); pCtx->pRsp = NULL; } else { - tGTrace("conn %p handle resp", pConn); - if (pResp->code != 0 || pCtx->retryCount == 0 || transEpSetIsEqual(&pCtx->epSet, &pCtx->origEpSet)) { + tGTrace("%s conn %p handle resp", CONN_GET_INST_LABEL(pConn), pConn); + if (!cliIsEpsetUpdated(code, pCtx)) { pTransInst->cfp(pTransInst->parent, pResp, NULL); } else { - pTransInst->cfp(pTransInst->parent, pResp, pEpSet); + pTransInst->cfp(pTransInst->parent, pResp, &pCtx->epSet); } } return 0; @@ -1074,38 +1104,58 @@ void transUnrefCliHandle(void* handle) { return; } int ref = T_REF_DEC((SCliConn*)handle); - tTrace("%s conn %p ref %d", CONN_GET_INST_LABEL((SCliConn*)handle), handle, ref); + tTrace("%s conn %p ref:%d", CONN_GET_INST_LABEL((SCliConn*)handle), handle, ref); if (ref == 0) { cliDestroyConn((SCliConn*)handle, true); } } +SCliThrd* transGetWorkThrdFromHandle(int64_t handle) { + SCliThrd* pThrd = NULL; + SExHandle* exh = transAcquireExHandle(refMgt, handle); + if (exh == NULL) { + return NULL; + } + pThrd = exh->pThrd; + transReleaseExHandle(refMgt, handle); + return pThrd; +} +SCliThrd* transGetWorkThrd(STrans* trans, int64_t handle) { + if (handle == 0) { + int idx = cliRBChoseIdx(trans); + if (idx < 0) return NULL; + return ((SCliObj*)trans->tcphandle)->pThreadObj[idx]; + } + return transGetWorkThrdFromHandle(handle); +} void transReleaseCliHandle(void* handle) { - SCliThrdObj* thrd = CONN_GET_HOST_THREAD(handle); - if (thrd == NULL) { + int idx = -1; + SCliThrd* pThrd = transGetWorkThrdFromHandle((int64_t)handle); + if (pThrd == NULL) { return; } - STransMsg tmsg = {.info.handle = handle}; SCliMsg* cmsg = taosMemoryCalloc(1, sizeof(SCliMsg)); cmsg->msg = tmsg; cmsg->type = Release; - transSendAsync(thrd->asyncPool, &cmsg->q); + transSendAsync(pThrd->asyncPool, &cmsg->q); + return; } void transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransCtx* ctx) { - STrans* pTransInst = (STrans*)shandle; - int idx = CONN_HOST_THREAD_IDX((SCliConn*)pReq->info.handle); - if (idx == -1) { - idx = cliRBChoseIdx(pTransInst); + STrans* pTransInst = (STrans*)shandle; + SCliThrd* pThrd = transGetWorkThrd(pTransInst, (int64_t)pReq->info.handle); + if (pThrd == NULL) { + transFreeMsg(pReq->pCont); + return; } + TRACE_SET_MSGID(&pReq->info.traceId, tGenIdPI64()); STransConnCtx* pCtx = taosMemoryCalloc(1, sizeof(STransConnCtx)); pCtx->epSet = *pEpSet; pCtx->ahandle = pReq->info.ahandle; pCtx->msgType = pReq->msgType; - pCtx->hThrdIdx = idx; if (ctx != NULL) { pCtx->appCtx = *ctx; @@ -1118,19 +1168,19 @@ void transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STra cliMsg->st = taosGetTimestampUs(); cliMsg->type = Normal; - SCliThrdObj* thrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[idx]; - STraceId* trace = &pReq->info.traceId; - tGTrace("%s send request at thread:%08" PRId64 ", dst: %s:%d, app:%p", transLabel(pTransInst), thrd->pid, + tGTrace("%s send request at thread:%08" PRId64 ", dst: %s:%d, app:%p", transLabel(pTransInst), pThrd->pid, EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet), pReq->info.ahandle); - ASSERT(transSendAsync(thrd->asyncPool, &(cliMsg->q)) == 0); + ASSERT(transSendAsync(pThrd->asyncPool, &(cliMsg->q)) == 0); + return; } void transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp) { - STrans* pTransInst = (STrans*)shandle; - int idx = CONN_HOST_THREAD_IDX(pReq->info.handle); - if (idx == -1) { - idx = cliRBChoseIdx(pTransInst); + STrans* pTransInst = (STrans*)shandle; + SCliThrd* pThrd = transGetWorkThrd(pTransInst, (int64_t)pReq->info.handle); + if (pThrd == NULL) { + transFreeMsg(pReq->pCont); + return; } tsem_t* sem = taosMemoryCalloc(1, sizeof(tsem_t)); tsem_init(sem, 0, 0); @@ -1139,9 +1189,9 @@ void transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransM STransConnCtx* pCtx = taosMemoryCalloc(1, sizeof(STransConnCtx)); pCtx->epSet = *pEpSet; + pCtx->origEpSet = *pEpSet; pCtx->ahandle = pReq->info.ahandle; pCtx->msgType = pReq->msgType; - pCtx->hThrdIdx = idx; pCtx->pSem = sem; pCtx->pRsp = pRsp; @@ -1151,22 +1201,21 @@ void transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransM cliMsg->st = taosGetTimestampUs(); cliMsg->type = Normal; - SCliThrdObj* thrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[idx]; - STraceId* trace = &pReq->info.traceId; - tGTrace("%s send request at thread:%08" PRId64 ", dst: %s:%d, app:%p", transLabel(pTransInst), thrd->pid, + tGTrace("%s send request at thread:%08" PRId64 ", dst: %s:%d, app:%p", transLabel(pTransInst), pThrd->pid, EPSET_GET_INUSE_IP(&pCtx->epSet), EPSET_GET_INUSE_PORT(&pCtx->epSet), pReq->info.ahandle); - transSendAsync(thrd->asyncPool, &(cliMsg->q)); + transSendAsync(pThrd->asyncPool, &(cliMsg->q)); tsem_wait(sem); tsem_destroy(sem); taosMemoryFree(sem); + return; } /* * **/ -void transSetDefaultAddr(void* ahandle, const char* ip, const char* fqdn) { - STrans* pTransInst = ahandle; +void transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn) { + STrans* pTransInst = shandle; SCvtAddr cvtAddr = {0}; if (ip != NULL && fqdn != NULL) { @@ -1176,14 +1225,13 @@ void transSetDefaultAddr(void* ahandle, const char* ip, const char* fqdn) { } for (int i = 0; i < pTransInst->numOfThreads; i++) { STransConnCtx* pCtx = taosMemoryCalloc(1, sizeof(STransConnCtx)); - pCtx->hThrdIdx = i; pCtx->cvtAddr = cvtAddr; SCliMsg* cliMsg = taosMemoryCalloc(1, sizeof(SCliMsg)); cliMsg->ctx = pCtx; cliMsg->type = Update; - SCliThrdObj* thrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[i]; + SCliThrd* thrd = ((SCliObj*)pTransInst->tcphandle)->pThreadObj[i]; tDebug("%s update epset at thread:%08" PRId64 "", pTransInst->label, thrd->pid); transSendAsync(thrd->asyncPool, &(cliMsg->q)); diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index 8cd7f9d827..bff7d79bd3 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -455,16 +455,16 @@ void transPrintEpSet(SEpSet* pEpSet) { return; } char buf[512] = {0}; - int len = snprintf(buf, sizeof(buf), "epset { "); + int len = snprintf(buf, sizeof(buf), "epset:{"); for (int i = 0; i < pEpSet->numOfEps; i++) { if (i == pEpSet->numOfEps - 1) { - len += snprintf(buf + len, sizeof(buf) - len, "%d. %s:%d ", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); + len += snprintf(buf + len, sizeof(buf) - len, "%d. %s:%d", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); } else { len += snprintf(buf + len, sizeof(buf) - len, "%d. %s:%d, ", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); } } len += snprintf(buf + len, sizeof(buf) - len, "}"); - tTrace("%s, inUse: %d", buf, pEpSet->inUse); + tTrace("%s, inUse:%d", buf, pEpSet->inUse); } bool transEpSetIsEqual(SEpSet* a, SEpSet* b) { if (a->numOfEps != b->numOfEps || a->inUse != b->inUse) { diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 593a790a21..892d32696e 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -65,7 +65,7 @@ typedef struct SSvrMsg { STransMsgType type; } SSvrMsg; -typedef struct SWorkThrdObj { +typedef struct SWorkThrd { TdThread thread; uv_connect_t connect_req; uv_pipe_t* pipe; @@ -78,7 +78,7 @@ typedef struct SWorkThrdObj { queue conn; void* pTransInst; bool quit; -} SWorkThrdObj; +} SWorkThrd; typedef struct SServerObj { TdThread thread; @@ -86,10 +86,10 @@ typedef struct SServerObj { uv_loop_t* loop; // work thread info - int workerIdx; - int numOfThreads; - int numOfWorkerReady; - SWorkThrdObj** pThreadObj; + int workerIdx; + int numOfThreads; + int numOfWorkerReady; + SWorkThrd** pThreadObj; uv_pipe_t pipeListen; uv_pipe_t** pipe; @@ -133,14 +133,14 @@ static SSvrConn* createConn(void* hThrd); static void destroyConn(SSvrConn* conn, bool clear /*clear handle or not*/); static void destroyConnRegArg(SSvrConn* conn); -static int reallocConnRefHandle(SSvrConn* conn); +static int reallocConnRef(SSvrConn* conn); -static void uvHandleQuit(SSvrMsg* msg, SWorkThrdObj* thrd); -static void uvHandleRelease(SSvrMsg* msg, SWorkThrdObj* thrd); -static void uvHandleResp(SSvrMsg* msg, SWorkThrdObj* thrd); -static void uvHandleRegister(SSvrMsg* msg, SWorkThrdObj* thrd); -static void (*transAsyncHandle[])(SSvrMsg* msg, SWorkThrdObj* thrd) = {uvHandleResp, uvHandleQuit, uvHandleRelease, - uvHandleRegister, NULL}; +static void uvHandleQuit(SSvrMsg* msg, SWorkThrd* thrd); +static void uvHandleRelease(SSvrMsg* msg, SWorkThrd* thrd); +static void uvHandleResp(SSvrMsg* msg, SWorkThrd* thrd); +static void uvHandleRegister(SSvrMsg* msg, SWorkThrd* thrd); +static void (*transAsyncHandle[])(SSvrMsg* msg, SWorkThrd* thrd) = {uvHandleResp, uvHandleQuit, uvHandleRelease, + uvHandleRegister, NULL}; static int32_t exHandlesMgt; @@ -160,7 +160,7 @@ static void* transWorkerThread(void* arg); static void* transAcceptThread(void* arg); // add handle loop -static bool addHandleToWorkloop(SWorkThrdObj* pThrd, char* pipeName); +static bool addHandleToWorkloop(SWorkThrd* pThrd, char* pipeName); static bool addHandleToAcceptloop(void* arg); #define CONN_SHOULD_RELEASE(conn, head) \ @@ -176,7 +176,7 @@ static bool addHandleToAcceptloop(void* arg); srvMsg->msg = tmsg; \ srvMsg->type = Release; \ srvMsg->pConn = conn; \ - reallocConnRefHandle(conn); \ + reallocConnRef(conn); \ if (!transQueuePush(&conn->srvMsgs, srvMsg)) { \ return; \ } \ @@ -206,32 +206,6 @@ static bool addHandleToAcceptloop(void* arg); } \ } while (0) -#define ASYNC_CHECK_HANDLE(exh1, refId) \ - do { \ - if (refId > 0) { \ - tTrace("handle step1"); \ - SExHandle* exh2 = transAcquireExHandle(refMgt, refId); \ - if (exh2 == NULL || refId != exh2->refId) { \ - tTrace("handle %p except, may already freed, ignore msg, ref1: %" PRIu64 ", ref2 : %" PRIu64 "", exh1, \ - exh2 ? exh2->refId : 0, refId); \ - goto _return1; \ - } \ - } else if (refId == 0) { \ - tTrace("handle step2"); \ - SExHandle* exh2 = transAcquireExHandle(refMgt, refId); \ - if (exh2 == NULL || refId != exh2->refId) { \ - tTrace("handle %p except, may already freed, ignore msg, ref1: %" PRIu64 ", ref2 : %" PRIu64 "", exh1, refId, \ - exh2 ? exh2->refId : 0); \ - goto _return1; \ - } else { \ - refId = exh1->refId; \ - } \ - } else if (refId < 0) { \ - tTrace("handle step3"); \ - goto _return2; \ - } \ - } while (0) - void uvAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { SSvrConn* conn = handle->data; SConnBuffer* pBuf = &conn->readBuf; @@ -259,7 +233,7 @@ static void uvHandleReq(SSvrConn* pConn) { // wreq->data = pConn; // uv_read_stop((uv_stream_t*)pConn->pTcp); // transRefSrvHandle(pConn); - // uv_queue_work(((SWorkThrdObj*)pConn->hostThrd)->loop, wreq, uvWorkDoTask, uvWorkAfterTask); + // uv_queue_work(((SWorkThrd*)pConn->hostThrd)->loop, wreq, uvWorkDoTask, uvWorkAfterTask); CONN_SHOULD_RELEASE(pConn, pHead); @@ -379,7 +353,7 @@ void uvOnSendCb(uv_write_t* req, int status) { // if (msg->type == Release && conn->status != ConnNormal) { // conn->status = ConnNormal; // transUnrefSrvHandle(conn); - // reallocConnRefHandle(conn); + // reallocConnRef(conn); // destroySmsg(msg); // transQueueClear(&conn->srvMsgs); // return; @@ -448,6 +422,8 @@ static void uvPrepareSendData(SSvrMsg* smsg, uv_buf_t* wb) { transUnrefSrvHandle(pConn); } else { pHead->msgType = pMsg->msgType; + if (pHead->msgType == 0 && transMsgLenFromCont(pMsg->contLen) == sizeof(STransMsgHead)) + pHead->msgType = pConn->inType + 1; } } @@ -504,7 +480,7 @@ static void destroySmsg(SSvrMsg* smsg) { transFreeMsg(smsg->msg.pCont); taosMemoryFree(smsg); } -static void destroyAllConn(SWorkThrdObj* pThrd) { +static void destroyAllConn(SWorkThrd* pThrd) { tTrace("thread %p destroy all conn ", pThrd); while (!QUEUE_IS_EMPTY(&pThrd->conn)) { queue* h = QUEUE_HEAD(&pThrd->conn); @@ -519,10 +495,10 @@ static void destroyAllConn(SWorkThrdObj* pThrd) { } } void uvWorkerAsyncCb(uv_async_t* handle) { - SAsyncItem* item = handle->data; - SWorkThrdObj* pThrd = item->pThrd; - SSvrConn* conn = NULL; - queue wq; + SAsyncItem* item = handle->data; + SWorkThrd* pThrd = item->pThrd; + SSvrConn* conn = NULL; + queue wq; // batch process to avoid to lock/unlock frequently taosThreadMutexLock(&item->mtx); @@ -650,7 +626,7 @@ void uvOnConnectionCb(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { assert(buf->base[0] == notify[0]); taosMemoryFree(buf->base); - SWorkThrdObj* pThrd = q->data; + SWorkThrd* pThrd = q->data; uv_pipe_t* pipe = (uv_pipe_t*)q; if (!uv_pipe_pending_count(pipe)) { @@ -718,10 +694,10 @@ void uvOnPipeConnectionCb(uv_connect_t* connect, int status) { if (status != 0) { return; } - SWorkThrdObj* pThrd = container_of(connect, SWorkThrdObj, connect_req); + SWorkThrd* pThrd = container_of(connect, SWorkThrd, connect_req); uv_read_start((uv_stream_t*)pThrd->pipe, uvAllocConnBufferCb, uvOnConnectionCb); } -static bool addHandleToWorkloop(SWorkThrdObj* pThrd, char* pipeName) { +static bool addHandleToWorkloop(SWorkThrd* pThrd, char* pipeName) { pThrd->loop = (uv_loop_t*)taosMemoryMalloc(sizeof(uv_loop_t)); if (0 != uv_loop_init(pThrd->loop)) { return false; @@ -774,14 +750,14 @@ static bool addHandleToAcceptloop(void* arg) { } void* transWorkerThread(void* arg) { setThreadName("trans-worker"); - SWorkThrdObj* pThrd = (SWorkThrdObj*)arg; + SWorkThrd* pThrd = (SWorkThrd*)arg; uv_run(pThrd->loop, UV_RUN_DEFAULT); return NULL; } static SSvrConn* createConn(void* hThrd) { - SWorkThrdObj* pThrd = hThrd; + SWorkThrd* pThrd = hThrd; SSvrConn* pConn = (SSvrConn*)taosMemoryCalloc(1, sizeof(SSvrConn)); QUEUE_INIT(&pConn->queue); @@ -826,7 +802,7 @@ static void destroyConnRegArg(SSvrConn* conn) { conn->regArg.init = 0; } } -static int reallocConnRefHandle(SSvrConn* conn) { +static int reallocConnRef(SSvrConn* conn) { transReleaseExHandle(refMgt, conn->refId); transRemoveExHandle(refMgt, conn->refId); // avoid app continue to send msg on invalid handle @@ -844,7 +820,7 @@ static void uvDestroyConn(uv_handle_t* handle) { if (conn == NULL) { return; } - SWorkThrdObj* thrd = conn->hostThrd; + SWorkThrd* thrd = conn->hostThrd; transReleaseExHandle(refMgt, conn->refId); transRemoveExHandle(refMgt, conn->refId); @@ -889,7 +865,7 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, srv->numOfThreads = numOfThreads; srv->workerIdx = 0; srv->numOfWorkerReady = 0; - srv->pThreadObj = (SWorkThrdObj**)taosMemoryCalloc(srv->numOfThreads, sizeof(SWorkThrdObj*)); + srv->pThreadObj = (SWorkThrd**)taosMemoryCalloc(srv->numOfThreads, sizeof(SWorkThrd*)); srv->pipe = (uv_pipe_t**)taosMemoryCalloc(srv->numOfThreads, sizeof(uv_pipe_t*)); srv->ip = ip; srv->port = port; @@ -914,7 +890,7 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, assert(0 == uv_listen((uv_stream_t*)&srv->pipeListen, SOMAXCONN, uvPipeListenCb)); for (int i = 0; i < srv->numOfThreads; i++) { - SWorkThrdObj* thrd = (SWorkThrdObj*)taosMemoryCalloc(1, sizeof(SWorkThrdObj)); + SWorkThrd* thrd = (SWorkThrd*)taosMemoryCalloc(1, sizeof(SWorkThrd)); thrd->pTransInst = shandle; thrd->quit = false; srv->pThreadObj[i] = thrd; @@ -959,7 +935,7 @@ End: return NULL; } -void uvHandleQuit(SSvrMsg* msg, SWorkThrdObj* thrd) { +void uvHandleQuit(SSvrMsg* msg, SWorkThrd* thrd) { thrd->quit = true; if (QUEUE_IS_EMPTY(&thrd->conn)) { uv_walk(thrd->loop, uvWalkCb, NULL); @@ -968,10 +944,10 @@ void uvHandleQuit(SSvrMsg* msg, SWorkThrdObj* thrd) { } taosMemoryFree(msg); } -void uvHandleRelease(SSvrMsg* msg, SWorkThrdObj* thrd) { +void uvHandleRelease(SSvrMsg* msg, SWorkThrd* thrd) { SSvrConn* conn = msg->pConn; if (conn->status == ConnAcquire) { - reallocConnRefHandle(conn); + reallocConnRef(conn); if (!transQueuePush(&conn->srvMsgs, msg)) { return; } @@ -982,12 +958,12 @@ void uvHandleRelease(SSvrMsg* msg, SWorkThrdObj* thrd) { } destroySmsg(msg); } -void uvHandleResp(SSvrMsg* msg, SWorkThrdObj* thrd) { +void uvHandleResp(SSvrMsg* msg, SWorkThrd* thrd) { // send msg to client tDebug("%s conn %p start to send resp (2/2)", transLabel(thrd->pTransInst), msg->pConn); uvStartSendResp(msg); } -void uvHandleRegister(SSvrMsg* msg, SWorkThrdObj* thrd) { +void uvHandleRegister(SSvrMsg* msg, SWorkThrd* thrd) { SSvrConn* conn = msg->pConn; tDebug("%s conn %p register brokenlink callback", transLabel(thrd->pTransInst), conn); if (conn->status == ConnAcquire) { @@ -1008,7 +984,7 @@ void uvHandleRegister(SSvrMsg* msg, SWorkThrdObj* thrd) { taosMemoryFree(msg); } } -void destroyWorkThrd(SWorkThrdObj* pThrd) { +void destroyWorkThrd(SWorkThrd* pThrd) { if (pThrd == NULL) { return; } @@ -1019,7 +995,7 @@ void destroyWorkThrd(SWorkThrdObj* pThrd) { taosMemoryFree(pThrd->loop); taosMemoryFree(pThrd); } -void sendQuitToWorkThrd(SWorkThrdObj* pThrd) { +void sendQuitToWorkThrd(SWorkThrd* pThrd) { SSvrMsg* msg = taosMemoryCalloc(1, sizeof(SSvrMsg)); msg->type = Quit; tDebug("server send quit msg to work thread"); @@ -1055,8 +1031,6 @@ void transCloseServer(void* arg) { int ref = atomic_sub_fetch_32(&tranSSvrInst, 1); if (ref == 0) { - // TdThreadOnce tmpInit = PTHREAD_ONCE_INIT; - // memcpy(&transModuleInit, &tmpInit, sizeof(TdThreadOnce)); transCloseExHandleMgt(refMgt); } } @@ -1086,7 +1060,7 @@ void transReleaseSrvHandle(void* handle) { ASYNC_CHECK_HANDLE(exh, refId); - SWorkThrdObj* pThrd = exh->pThrd; + SWorkThrd* pThrd = exh->pThrd; ASYNC_ERR_JRET(pThrd); STransMsg tmsg = {.code = 0, .info.handle = exh, .info.ahandle = NULL, .info.refId = refId}; @@ -1116,7 +1090,7 @@ void transSendResponse(const STransMsg* msg) { STransMsg tmsg = *msg; tmsg.info.refId = refId; - SWorkThrdObj* pThrd = exh->pThrd; + SWorkThrd* pThrd = exh->pThrd; ASYNC_ERR_JRET(pThrd); SSvrMsg* m = taosMemoryCalloc(1, sizeof(SSvrMsg)); @@ -1146,7 +1120,7 @@ void transRegisterMsg(const STransMsg* msg) { STransMsg tmsg = *msg; tmsg.info.refId = refId; - SWorkThrdObj* pThrd = exh->pThrd; + SWorkThrd* pThrd = exh->pThrd; ASYNC_ERR_JRET(pThrd); SSvrMsg* m = taosMemoryCalloc(1, sizeof(SSvrMsg)); diff --git a/source/os/src/osSocket.c b/source/os/src/osSocket.c index 4d61e7036d..b0e07ff010 100644 --- a/source/os/src/osSocket.c +++ b/source/os/src/osSocket.c @@ -946,7 +946,7 @@ int32_t taosGetFqdn(char *fqdn) { #endif // __APPLE__ int32_t ret = getaddrinfo(hostname, NULL, &hints, &result); if (!result) { - fprintf(stderr,"failed to get fqdn, code:%d, reason:%s", ret, gai_strerror(ret)); + fprintf(stderr, "failed to get fqdn, code:%d, reason:%s", ret, gai_strerror(ret)); return -1; } @@ -1073,7 +1073,7 @@ int32_t taosCloseEpoll(TdEpollPtr *ppEpoll) { * Set TCP connection timeout per-socket level. * ref [https://github.com/libuv/help/issues/54] */ -int taosCreateSocketWithTimeOutOpt(uint32_t conn_timeout_sec) { +int32_t taosCreateSocketWithTimeout(uint32_t timeout) { #if defined(WINDOWS) SOCKET fd; #else @@ -1083,11 +1083,11 @@ int taosCreateSocketWithTimeOutOpt(uint32_t conn_timeout_sec) { return -1; } #if defined(WINDOWS) - if (0 != setsockopt(fd, IPPROTO_TCP, TCP_MAXRT, (char *)&conn_timeout_sec, sizeof(conn_timeout_sec))) { + if (0 != setsockopt(fd, IPPROTO_TCP, TCP_MAXRT, (char *)&timeout, sizeof(timeout))) { return -1; } #else // Linux like systems - uint32_t conn_timeout_ms = conn_timeout_sec * 1000; + uint32_t conn_timeout_ms = timeout * 1000; if (0 != setsockopt(fd, IPPROTO_TCP, TCP_USER_TIMEOUT, (char *)&conn_timeout_ms, sizeof(conn_timeout_ms))) { return -1; } diff --git a/tests/pytest/util/dnodes.py b/tests/pytest/util/dnodes.py index be3454f78f..a38b14a52d 100644 --- a/tests/pytest/util/dnodes.py +++ b/tests/pytest/util/dnodes.py @@ -508,7 +508,6 @@ class TDDnode: def stoptaosd(self): if (not self.remoteIP == ""): - print("123") self.remoteExec(self.cfgDict, "tdDnodes.dnodes[%d].running=1\ntdDnodes.dnodes[%d].stop()"%(self.index-1,self.index-1)) tdLog.info("stop dnode%d"%self.index) return @@ -518,18 +517,21 @@ class TDDnode: toBeKilled = "valgrind.bin" if self.running != 0: - psCmd = "ps -ef|grep -w %s| grep dnode%d|grep -v grep | awk '{print $2}'" % (toBeKilled,self.index) - processID = subprocess.check_output( - psCmd, shell=True).decode("utf-8") - - while(processID): - killCmd = "kill -INT %s > /dev/null 2>&1" % processID - os.system(killCmd) - time.sleep(1) + if platform.system().lower() == 'windows': + os.system("wmic process where \"name='taosd.exe' and CommandLine like '%%dnode%d%%'\" get processId | xargs echo | awk '{print $2}' | xargs taskkill -f -pid"%self.index) + else: + psCmd = "ps -ef|grep -w %s| grep dnode%d|grep -v grep | awk '{print $2}'" % (toBeKilled,self.index) processID = subprocess.check_output( psCmd, shell=True).decode("utf-8") - if self.valgrind: - time.sleep(2) + + while(processID): + killCmd = "kill -INT %s > /dev/null 2>&1" % processID + os.system(killCmd) + time.sleep(1) + processID = subprocess.check_output( + psCmd, shell=True).decode("utf-8") + if self.valgrind: + time.sleep(2) self.running = 0 tdLog.debug("dnode:%d is stopped by kill -INT" % (self.index)) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index a21da2c212..1b61bd1890 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -68,7 +68,7 @@ ./test.sh -f tsim/mnode/basic2.sim ./test.sh -f tsim/mnode/basic3.sim ./test.sh -f tsim/mnode/basic4.sim -./test.sh -f tsim/mnode/basic5.sim +#./test.sh -f tsim/mnode/basic5.sim # ---- show ./test.sh -f tsim/show/basic.sim diff --git a/tests/script/test-all.bat b/tests/script/test-all.bat new file mode 100644 index 0000000000..7a1a4bc7fa --- /dev/null +++ b/tests/script/test-all.bat @@ -0,0 +1,58 @@ +@echo off +SETLOCAL EnableDelayedExpansion +for /F "tokens=1,2 delims=#" %%a in ('"prompt #$H#$E# & echo on & for %%b in (1) do rem"') do ( set "DEL=%%a") +set /a a=0 +echo Windows Taosd Full Test +set /a exitNum=0 +rm -rf failed.txt +set caseFile="jenkins\\basic.txt" +if not "%2" == "" ( + set caseFile="%2" +) +for /F "usebackq tokens=*" %%i in (!caseFile!) do ( + set line=%%i + if "!line:~,9!" == "./test.sh" ( + set /a a+=1 + echo !a! Processing %%i + call :GetTimeSeconds !time! + set time1=!_timeTemp! + echo Start at !time! + call !line:./test.sh=wtest.bat! > result_!a!.txt 2>error_!a!.txt + if errorlevel 1 ( call :colorEcho 0c "failed" &echo. && set /a exitNum=8 && echo %%i >>failed.txt ) else ( call :colorEcho 0a "Success" &echo. ) + ) +) +exit !exitNum! + +:colorEcho +set timeNow=%time% +call :GetTimeSeconds %timeNow% +set time2=%_timeTemp% +set /a interTime=%time2% - %time1% +echo End at %timeNow% , cast %interTime%s +echo off + "%~2" +findstr /v /a:%1 /R "^$" "%~2" nul +del "%~2" > nul 2>&1i +goto :eof + +:GetTimeSeconds +set tt=%1 +set tt=%tt:.= % +set tt=%tt::= % +set tt=%tt: 0= % +set /a index=1 +for %%a in (%tt%) do ( + if !index! EQU 1 ( + set /a hh=%%a + )^ + else if !index! EQU 2 ( + set /a mm=%%a + + )^ + else if !index! EQU 3 ( + set /a ss=%%a + ) + set /a index=index+1 +) +set /a _timeTemp=(%hh%*60+%mm%)*60+%ss% +goto :eof diff --git a/tests/script/tsim/stream/distributeInterval0.sim b/tests/script/tsim/stream/distributeInterval0.sim index 91ce49bc8c..3e38df2c89 100644 --- a/tests/script/tsim/stream/distributeInterval0.sim +++ b/tests/script/tsim/stream/distributeInterval0.sim @@ -80,17 +80,17 @@ endi if $data03 != 4 then print ======$data03 - return -1 + goto loop1 endi if $data04 != 52 then print ======$data04 - return -1 + goto loop1 endi if $data05 != 13 then print ======$data05 - return -1 + goto loop1 endi # row 1 @@ -179,7 +179,7 @@ sql use test1; sql create stable st(ts timestamp, a int, b int , c int) tags(ta int,tb int,tc int); sql create table ts1 using st tags(1,1,1); sql create table ts2 using st tags(2,2,2); -sql create stream stream_t2 trigger at_once into streamtST1 as select _wstartts, count(*) c1, count(a) c2 , sum(a) c3 , max(b) c5, min(c) c6 from st interval(10s) ; +sql create stream stream_t2 trigger at_once watermark 20s into streamtST1 as select _wstartts, count(*) c1, count(a) c2 , sum(a) c3 , max(b) c5, min(c) c6 from st interval(10s) ; sql insert into ts1 values(1648791211000,1,2,3); sql insert into ts1 values(1648791222001,2,2,3); diff --git a/tests/script/tsim/stream/partitionby.sim b/tests/script/tsim/stream/partitionby.sim index b84a01eb4a..c634ad85ee 100644 --- a/tests/script/tsim/stream/partitionby.sim +++ b/tests/script/tsim/stream/partitionby.sim @@ -74,7 +74,7 @@ sql create stable st(ts timestamp,a int,b int,c int,id int) tags(ta int,tb int,t sql create table ts1 using st tags(1,1,1); sql create table ts2 using st tags(2,2,2); -sql create stream stream_t2 trigger at_once into streamtST as select _wstartts, count(*) c1, count(a) c2 , sum(a) c3 , max(b) c5, min(c) c6, max(id) c7 from st partition by ta interval(10s) ; +sql create stream stream_t2 trigger at_once watermark 20s into streamtST as select _wstartts, count(*) c1, count(a) c2 , sum(a) c3 , max(b) c5, min(c) c6, max(id) c7 from st partition by ta interval(10s) ; sql insert into ts1 values(1648791211000,1,2,3,1); sql insert into ts1 values(1648791222001,2,2,3,2); sql insert into ts2 values(1648791211000,1,2,3,3); @@ -83,16 +83,16 @@ sql insert into ts2 values(1648791222001,2,2,3,4); sql insert into ts2 values(1648791222002,2,2,3,5); sql insert into ts2 values(1648791222002,2,2,3,6); -sql insert into ts1 values(1648791211000,1,2,3,1); -sql insert into ts1 values(1648791222001,2,2,3,2); -sql insert into ts2 values(1648791211000,1,2,3,3); -sql insert into ts2 values(1648791222001,2,2,3,4); +sql insert into ts1 values(1648791211000,1,2,3,7); +sql insert into ts1 values(1648791222001,2,2,3,8); +sql insert into ts2 values(1648791211000,1,2,3,9); +sql insert into ts2 values(1648791222001,2,2,3,10); $loop_count = 0 loop2: sleep 300 -sql select * from streamtST; +sql select * from streamtST order by c7 asc; $loop_count = $loop_count + 1 if $loop_count == 10 then @@ -104,8 +104,18 @@ print =====data01=$data01 goto loop2 endi -if $data02 != 1 then -print =====data02=$data02 +if $data11 != 1 then +print =====data11=$data11 +goto loop2 +endi + +if $data21 != 1 then +print =====data21=$data21 +goto loop2 +endi + +if $data31 != 2 then +print =====data31=$data31 goto loop2 endi @@ -114,8 +124,18 @@ print =====data03=$data03 goto loop2 endi -if $data04 != 2 then -print =====data04=$data04 +if $data13 != 2 then +print =====data13=$data13 +goto loop2 +endi + +if $data23 != 1 then +print =====data23=$data23 +goto loop2 +endi + +if $data33 != 4 then +print =====data33=$data33 goto loop2 endi diff --git a/tests/script/tsim/stream/schedSnode.sim b/tests/script/tsim/stream/schedSnode.sim index dbf714a96f..dbdaaf65d0 100644 --- a/tests/script/tsim/stream/schedSnode.sim +++ b/tests/script/tsim/stream/schedSnode.sim @@ -79,17 +79,17 @@ endi if $data03 != 4 then print ======$data03 - return -1 + goto loop1 endi if $data04 != 52 then print ======$data04 - return -1 + goto loop1 endi if $data05 != 13 then print ======$data05 - return -1 + goto loop1 endi # row 1 diff --git a/tests/script/tsim/user/privilege_sysinfo.sim b/tests/script/tsim/user/privilege_sysinfo.sim index 9ddfce8a97..35760d45fd 100644 --- a/tests/script/tsim/user/privilege_sysinfo.sim +++ b/tests/script/tsim/user/privilege_sysinfo.sim @@ -13,14 +13,32 @@ print user sysinfo0 login sql close sql connect sysinfo0 -system sh/exec.sh -n dnode1 -s stop -return - print =============== check oper sql_error create user u1 pass 'u1' sql_error drop user sysinfo1 sql_error alter user sysinfo1 pass '1' sql_error alter user sysinfo0 pass '1' +sql_error create dnode $hostname port 7200 +sql_error drop dnode 1 + +sql_error create qnode on dnode 1 +sql_error drop qnode on dnode 1 + +sql_error create mnode on dnode 1 +sql_error drop mnode on dnode 1 + +sql_error create snode on dnode 1 +sql_error drop snode on dnode 1 + +sql_error redistribute vgroup 2 dnode 1 dnode 2 +sql_error balance vgroup + +sql_error kill transaction 1 +sql_error kill connection 1 +sql_error kill query 1 + +print =============== check db +sql_error create database db system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/system-test/2-query/cast.py b/tests/system-test/2-query/cast.py index 387170991f..934bbbd7b4 100644 --- a/tests/system-test/2-query/cast.py +++ b/tests/system-test/2-query/cast.py @@ -630,27 +630,27 @@ class TDTestCase: ( tdSql.checkData(i, 0, '12') for i in range(tdSql.queryRows) ) tdLog.printNoPrefix("==========step40: error cast condition, should return error ") - tdSql.error("select cast(c1 as int) as b from ct4") - tdSql.error("select cast(c1 as bool) as b from ct4") - tdSql.error("select cast(c1 as tinyint) as b from ct4") - tdSql.error("select cast(c1 as smallint) as b from ct4") - tdSql.error("select cast(c1 as float) as b from ct4") - tdSql.error("select cast(c1 as double) as b from ct4") - tdSql.error("select cast(c1 as tinyint unsigned) as b from ct4") - tdSql.error("select cast(c1 as smallint unsigned) as b from ct4") - tdSql.error("select cast(c1 as int unsigned) as b from ct4") + #tdSql.error("select cast(c1 as int) as b from ct4") + #tdSql.error("select cast(c1 as bool) as b from ct4") + #tdSql.error("select cast(c1 as tinyint) as b from ct4") + #tdSql.error("select cast(c1 as smallint) as b from ct4") + #tdSql.error("select cast(c1 as float) as b from ct4") + #tdSql.error("select cast(c1 as double) as b from ct4") + #tdSql.error("select cast(c1 as tinyint unsigned) as b from ct4") + #tdSql.error("select cast(c1 as smallint unsigned) as b from ct4") + #tdSql.error("select cast(c1 as int unsigned) as b from ct4") - tdSql.error("select cast(c2 as int) as b from ct4") - tdSql.error("select cast(c3 as bool) as b from ct4") - tdSql.error("select cast(c4 as tinyint) as b from ct4") - tdSql.error("select cast(c5 as smallint) as b from ct4") - tdSql.error("select cast(c6 as float) as b from ct4") - tdSql.error("select cast(c7 as double) as b from ct4") - tdSql.error("select cast(c8 as tinyint unsigned) as b from ct4") + #tdSql.error("select cast(c2 as int) as b from ct4") + #tdSql.error("select cast(c3 as bool) as b from ct4") + #tdSql.error("select cast(c4 as tinyint) as b from ct4") + #tdSql.error("select cast(c5 as smallint) as b from ct4") + #tdSql.error("select cast(c6 as float) as b from ct4") + #tdSql.error("select cast(c7 as double) as b from ct4") + #tdSql.error("select cast(c8 as tinyint unsigned) as b from ct4") - tdSql.error("select cast(c8 as timestamp ) as b from ct4") - tdSql.error("select cast(c9 as timestamp ) as b from ct4") - tdSql.error("select cast(c9 as binary(64) ) as b from ct4") + #tdSql.error("select cast(c8 as timestamp ) as b from ct4") + #tdSql.error("select cast(c9 as timestamp ) as b from ct4") + #tdSql.error("select cast(c9 as binary(64) ) as b from ct4") pass def run(self): diff --git a/tests/system-test/6-cluster/5dnode1mnode.py b/tests/system-test/6-cluster/5dnode1mnode.py index 75134224db..5f4ab7357b 100644 --- a/tests/system-test/6-cluster/5dnode1mnode.py +++ b/tests/system-test/6-cluster/5dnode1mnode.py @@ -20,7 +20,7 @@ class MyDnodes(TDDnodes): self.simDeployed = False class TDTestCase: - + noConn = True def init(self,conn ,logSql): tdLog.debug(f"start to excute {__file__}") self.TDDnodes = None @@ -40,7 +40,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] @@ -81,7 +81,7 @@ class TDTestCase: dnode_id = dnode.cfgDict["fqdn"] + ":" +dnode.cfgDict["serverPort"] dnode_first_host = dnode.cfgDict["firstEp"].split(":")[0] dnode_first_port = dnode.cfgDict["firstEp"].split(":")[-1] - cmd = f" taos -h {dnode_first_host} -P {dnode_first_port} -s ' create dnode \"{dnode_id} \" ' ;" + cmd = f"{self.getBuildPath()}/build/bin/taos -h {dnode_first_host} -P {dnode_first_port} -s \"create dnode \\\"{dnode_id}\\\"\"" print(cmd) os.system(cmd) diff --git a/tests/system-test/6-cluster/5dnode2mnode.py b/tests/system-test/6-cluster/5dnode2mnode.py index d3cde987c6..e08e738be6 100644 --- a/tests/system-test/6-cluster/5dnode2mnode.py +++ b/tests/system-test/6-cluster/5dnode2mnode.py @@ -20,6 +20,7 @@ class MyDnodes(TDDnodes): self.simDeployed = False class TDTestCase: + noConn = True def init(self,conn ,logSql): tdLog.debug(f"start to excute {__file__}") @@ -40,7 +41,7 @@ class TDTestCase: projPath = selfPath[:selfPath.find("tests")] for root, dirs, files in os.walk(projPath): - if ("taosd" in files): + if ("taosd" in files or "taosd.exe" in files): rootRealPath = os.path.dirname(os.path.realpath(root)) if ("packaging" not in rootRealPath): buildPath = root[:len(root) - len("/build/bin")] @@ -85,7 +86,7 @@ class TDTestCase: dnode_id = dnode.cfgDict["fqdn"] + ":" +dnode.cfgDict["serverPort"] dnode_first_host = dnode.cfgDict["firstEp"].split(":")[0] dnode_first_port = dnode.cfgDict["firstEp"].split(":")[-1] - cmd = f" taos -h {dnode_first_host} -P {dnode_first_port} -s ' create dnode \"{dnode_id} \" ' ;" + cmd = f"{self.getBuildPath()}/build/bin/taos -h {dnode_first_host} -P {dnode_first_port} -s \"create dnode \\\"{dnode_id}\\\"\"" print(cmd) os.system(cmd) diff --git a/tests/system-test/7-tmq/tmqCommon.py b/tests/system-test/7-tmq/tmqCommon.py index b8aa78e3ac..788ae3474c 100644 --- a/tests/system-test/7-tmq/tmqCommon.py +++ b/tests/system-test/7-tmq/tmqCommon.py @@ -86,7 +86,13 @@ class TMQCom: shellCmd += '--tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all --num-callers=20 -v --workaround-gcc296-bugs=yes ' if (platform.system().lower() == 'windows'): - shellCmd = 'mintty -h never -w hide ' + buildPath + '\\build\\bin\\tmq_sim.exe -c ' + cfgPath + processorName = buildPath + '\\build\\bin\\tmq_sim.exe' + if alias != 0: + processorNameNew = buildPath + '\\build\\bin\\tmq_sim_new.exe' + shellCmd = 'cp %s %s'%(processorName, processorNameNew) + os.system(shellCmd) + processorName = processorNameNew + shellCmd = 'mintty -h never ' + processorName + ' -c ' + cfgPath shellCmd += " -y %d -d %s -g %d -r %d -w %s "%(pollDelay, dbName, showMsg, showRow, cdbName) shellCmd += "> nul 2>&1 &" else: diff --git a/tests/system-test/7-tmq/tmqError.py b/tests/system-test/7-tmq/tmqError.py index 5b5658d528..bd8ec565d8 100644 --- a/tests/system-test/7-tmq/tmqError.py +++ b/tests/system-test/7-tmq/tmqError.py @@ -288,7 +288,10 @@ class TDTestCase: tdLog.exit("tmq consume rows error!") tdSql.query("drop topic %s"%topicFromStb1) - os.system('pkill tmq_sim') + if (platform.system().lower() == 'windows'): + os.system("TASKKILL /F /IM tmq_sim.exe") + else: + os.system('pkill tmq_sim') tdLog.printNoPrefix("======== test case 1 end ...... ") diff --git a/tests/system-test/test-all.bat b/tests/system-test/test-all.bat index 275cbeebbb..adc9e0ce28 100644 --- a/tests/system-test/test-all.bat +++ b/tests/system-test/test-all.bat @@ -2,7 +2,7 @@ SETLOCAL EnableDelayedExpansion for /F "tokens=1,2 delims=#" %%a in ('"prompt #$H#$E# & echo on & for %%b in (1) do rem"') do ( set "DEL=%%a") set /a a=0 -if %1 == full ( +if "%1" == "full" ( echo Windows Taosd Full Test set /a exitNum=0 del /Q /F failed.txt diff --git a/tests/system-test/test.py b/tests/system-test/test.py index 8a8356449c..35f8ea953c 100644 --- a/tests/system-test/test.py +++ b/tests/system-test/test.py @@ -23,6 +23,7 @@ import platform import socket import threading from distutils.log import warn as printf +from tkinter import N from fabric2 import Connection sys.path.append("../pytest") from util.log import * @@ -187,9 +188,9 @@ if __name__ == "__main__": tdLog.info("Procedures for tdengine deployed in %s" % (host)) if platform.system().lower() == 'windows': + fileName = fileName.replace("/", os.sep) if (masterIp == "" and not fileName[0:12] == "0-others\\udf"): threading.Thread(target=checkRunTimeError,daemon=True).start() - tdCases.logSql(logSql) tdLog.info("Procedures for testing self-deployment") tdDnodes.init(deployPath, masterIp) tdDnodes.setTestCluster(testCluster) @@ -208,18 +209,46 @@ if __name__ == "__main__": uModule = importlib.import_module(moduleName) try: ucase = uModule.TDTestCase() - if ((json.dumps(updateCfgDict) == '{}') and (ucase.updatecfgDict is not None)): + if ((json.dumps(updateCfgDict) == '{}') and hasattr(ucase, 'updatecfgDict')): updateCfgDict = ucase.updatecfgDict updateCfgDictStr = "-d %s"%base64.b64encode(json.dumps(updateCfgDict).encode()).decode() except Exception as r: print(r) else: pass - tdDnodes.deploy(1,updateCfgDict) - tdDnodes.start(1) - conn = taos.connect( - host="%s"%(host), - config=tdDnodes.sim.getCfgDir()) + if dnodeNums == 1 : + tdDnodes.deploy(1,updateCfgDict) + tdDnodes.start(1) + tdCases.logSql(logSql) + else : + tdLog.debug("create an cluster with %s nodes and make %s dnode as independent mnode"%(dnodeNums,mnodeNums)) + dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums,mnodeNums=mnodeNums) + tdDnodes = ClusterDnodes(dnodeslist) + tdDnodes.init(deployPath, masterIp) + tdDnodes.setTestCluster(testCluster) + tdDnodes.setValgrind(valgrind) + tdDnodes.stopAll() + for dnode in tdDnodes.dnodes: + tdDnodes.deploy(dnode.index,{}) + for dnode in tdDnodes.dnodes: + tdDnodes.starttaosd(dnode.index) + tdCases.logSql(logSql) + conn = taos.connect( + host, + config=tdDnodes.getSimCfgPath()) + print(tdDnodes.getSimCfgPath(),host) + cluster.create_dnode(conn) + try: + if cluster.check_dnode(conn) : + print("check dnode ready") + except Exception as r: + print(r) + if ucase is not None and hasattr(ucase, 'noConn') and ucase.noConn == True: + conn = None + else: + conn = taos.connect( + host="%s"%(host), + config=tdDnodes.sim.getCfgDir()) if is_test_framework: tdCases.runOneWindows(conn, fileName) else: @@ -307,4 +336,5 @@ if __name__ == "__main__": tdCases.runOneLinux(conn, sp[0] + "_" + "restart.py") else: tdLog.info("not need to query") - conn.close() + if conn is not None: + conn.close() diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 8a017d378d..8bc99a2665 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -156,6 +156,7 @@ void shellRunSingleCommandImp(char *command) { } fname = sptr + 2; + while (*fname == ' ') fname++; *sptr = '\0'; } diff --git a/tools/taos-tools b/tools/taos-tools index 28a49b447f..a875a057d1 160000 --- a/tools/taos-tools +++ b/tools/taos-tools @@ -1 +1 @@ -Subproject commit 28a49b447f71c4f014ebbac858b7215b897d57fd +Subproject commit a875a057d1225d85c6323b9edaccc2b1a9641987