diff --git a/.gitignore b/.gitignore index 65e03e1933..a69d2e44bd 100644 --- a/.gitignore +++ b/.gitignore @@ -101,6 +101,7 @@ tests/examples/JDBC/JDBCDemo/.settings/ source/libs/parser/inc/sql.* tests/script/tmqResult.txt tests/tmqResult.txt +tests/script/jenkins/basic.txt # Emacs # -*- mode: gitignore; -*- diff --git a/Jenkinsfile2 b/Jenkinsfile2 index 33c3ef55c9..de39f4fcec 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -303,7 +303,7 @@ def pre_test_build_win() { set CL=/MP8 echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> cmake" time /t - cmake .. -G "NMake Makefiles JOM" -DBUILD_TEST=true || exit 7 + cmake .. -G "NMake Makefiles JOM" -DBUILD_TEST=true -DBUILD_TOOLS=true || exit 7 echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> jom -j 6" time /t jom -j 6 || exit 8 diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index b996ffcd17..d1c448b094 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG e00ebd9 + GIT_TAG efa2a5f SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/docs/examples/python/tmq_example.py b/docs/examples/python/tmq_example.py index 836beb2417..a4625ca11a 100644 --- a/docs/examples/python/tmq_example.py +++ b/docs/examples/python/tmq_example.py @@ -4,6 +4,7 @@ from taos.tmq import * conn = taos.connect() print("init") +conn.execute("drop topic if exists topic_ctb_column") conn.execute("drop database if exists py_tmq") conn.execute("create database if not exists py_tmq vgroups 2") conn.select_db("py_tmq") @@ -15,7 +16,6 @@ conn.execute("create table if not exists tb2 using stb1 tags(2)") conn.execute("create table if not exists tb3 using stb1 tags(3)") print("create topic") -conn.execute("drop topic if exists topic_ctb_column") conn.execute( "create topic if not exists topic_ctb_column as select ts, c1, c2, c3 from stb1" ) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 02d4c2279c..29d68aba14 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1629,7 +1629,6 @@ int32_t tSerializeSSubQueryMsg(void *buf, int32_t bufLen, SSubQueryMsg *pReq); int32_t tDeserializeSSubQueryMsg(void *buf, int32_t bufLen, SSubQueryMsg *pReq); void tFreeSSubQueryMsg(SSubQueryMsg *pReq); - typedef struct { SMsgHead header; uint64_t sId; @@ -1667,6 +1666,10 @@ typedef struct { int32_t execId; } SResFetchReq; +int32_t tSerializeSResFetchReq(void *buf, int32_t bufLen, SResFetchReq *pReq); +int32_t tDeserializeSResFetchReq(void *buf, int32_t bufLen, SResFetchReq *pReq); + + typedef struct { SMsgHead header; uint64_t sId; @@ -2948,6 +2951,10 @@ typedef struct { STqOffsetVal reqOffset; } SMqPollReq; +int32_t tSerializeSMqPollReq(void *buf, int32_t bufLen, SMqPollReq *pReq); +int32_t tDeserializeSMqPollReq(void *buf, int32_t bufLen, SMqPollReq *pReq); + + typedef struct { int32_t vgId; int64_t offset; diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 660bfd4fbf..63009e5421 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -213,7 +213,7 @@ function install_bin() { [ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -s ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${demoName} || : [ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -s ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${benchmarkName} || : [ -x ${install_main_dir}/bin/${dumpName} ] && ${csudo}ln -s ${install_main_dir}/bin/${dumpName} ${bin_link_dir}/${dumpName} || : - [ -x ${install_main_dir}/bin/${xname} ] && ${csudo}ln -s ${install_main_dir}/bin/${dumpName} ${bin_link_dir}/${xname} || : + [ -x ${install_main_dir}/bin/${xname} ] && ${csudo}ln -s ${install_main_dir}/bin/${xname} ${bin_link_dir}/${xname} || : [ -x ${install_main_dir}/bin/TDinsight.sh ] && ${csudo}ln -s ${install_main_dir}/bin/TDinsight.sh ${bin_link_dir}/TDinsight.sh || : [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || : [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 0aa88382fe..423cfcdc5b 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -1106,6 +1106,8 @@ int taos_get_table_vgId(TAOS *taos, const char *db, const char *table, int *vgId return terrno; } + pRequest->syncQuery = true; + STscObj *pTscObj = pRequest->pTscObj; SCatalog *pCtg = NULL; code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &pCtg); diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 28b41b97c5..1dd3174c29 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -1461,12 +1461,7 @@ int32_t tmqAskEp(tmq_t* tmq, bool async) { return code; } -SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t timeout, SMqClientTopic* pTopic, SMqClientVg* pVg) { - SMqPollReq* pReq = taosMemoryCalloc(1, sizeof(SMqPollReq)); - if (pReq == NULL) { - return NULL; - } - +void tmqBuildConsumeReqImpl(SMqPollReq *pReq, tmq_t* tmq, int64_t timeout, SMqClientTopic* pTopic, SMqClientVg* pVg) { /*strcpy(pReq->topic, pTopic->topicName);*/ /*strcpy(pReq->cgroup, tmq->groupId);*/ @@ -1485,9 +1480,7 @@ SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t timeout, SMqClientTopic* pReq->useSnapshot = tmq->useSnapshot; - pReq->head.vgId = htonl(pVg->vgId); - pReq->head.contLen = htonl(sizeof(SMqPollReq)); - return pReq; + pReq->head.vgId = pVg->vgId; } SMqMetaRspObj* tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) { @@ -1559,15 +1552,32 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { #endif } atomic_store_32(&pVg->vgSkipCnt, 0); - SMqPollReq* pReq = tmqBuildConsumeReqImpl(tmq, timeout, pTopic, pVg); - if (pReq == NULL) { + + SMqPollReq req = {0}; + tmqBuildConsumeReqImpl(&req, tmq, timeout, pTopic, pVg); + int32_t msgSize = tSerializeSMqPollReq(NULL, 0, &req); + if (msgSize < 0) { atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); tsem_post(&tmq->rspSem); return -1; } + char *msg = taosMemoryCalloc(1, msgSize); + if (NULL == msg) { + atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); + tsem_post(&tmq->rspSem); + return -1; + } + + if (tSerializeSMqPollReq(msg, msgSize, &req) < 0) { + taosMemoryFree(msg); + atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); + tsem_post(&tmq->rspSem); + return -1; + } + SMqPollCbParam* pParam = taosMemoryMalloc(sizeof(SMqPollCbParam)); if (pParam == NULL) { - taosMemoryFree(pReq); + taosMemoryFree(msg); atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); tsem_post(&tmq->rspSem); return -1; @@ -1581,7 +1591,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (sendInfo == NULL) { - taosMemoryFree(pReq); + taosMemoryFree(msg); taosMemoryFree(pParam); atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); tsem_post(&tmq->rspSem); @@ -1589,11 +1599,11 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { } sendInfo->msgInfo = (SDataBuf){ - .pData = pReq, - .len = sizeof(SMqPollReq), + .pData = msg, + .len = msgSize, .handle = NULL, }; - sendInfo->requestId = pReq->reqId; + sendInfo->requestId = req.reqId; sendInfo->requestObjRefId = 0; sendInfo->param = pParam; sendInfo->fp = tmqPollCb; @@ -1605,7 +1615,7 @@ int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { char offsetFormatBuf[80]; tFormatOffset(offsetFormatBuf, 80, &pVg->currentOffset); tscDebug("consumer:%" PRId64 ", send poll to %s vgId:%d, epoch %d, req offset:%s, reqId:%" PRIu64, - tmq->consumerId, pTopic->topicName, pVg->vgId, tmq->epoch, offsetFormatBuf, pReq->reqId); + tmq->consumerId, pTopic->topicName, pVg->vgId, tmq->epoch, offsetFormatBuf, req.reqId); /*printf("send vgId:%d %" PRId64 "\n", pVg->vgId, pVg->currentOffset);*/ asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pVg->epSet, &transporterId, sendInfo); pVg->pollCnt++; diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index f3b90a8fd2..607db72ec6 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1084,8 +1084,6 @@ int32_t dataBlockCompar_rv(const void* p1, const void* p2, const void* param) { return 0; } -int32_t varColSort(SColumnInfoData* pColumnInfoData, SBlockOrderInfo* pOrder) { return 0; } - int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst) { // Allocate the additional buffer. int64_t p0 = taosGetTimestampUs(); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 35900638dd..7575554bcf 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -4723,6 +4723,139 @@ void tFreeSSubQueryMsg(SSubQueryMsg *pReq) { taosMemoryFreeClear(pReq->msg); } + +int32_t tSerializeSResFetchReq(void *buf, int32_t bufLen, SResFetchReq *pReq) { + int32_t headLen = sizeof(SMsgHead); + if (buf != NULL) { + buf = (char *)buf + headLen; + bufLen -= headLen; + } + + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + if (tStartEncode(&encoder) < 0) return -1; + + if (tEncodeU64(&encoder, pReq->sId) < 0) return -1; + if (tEncodeU64(&encoder, pReq->queryId) < 0) return -1; + if (tEncodeU64(&encoder, pReq->taskId) < 0) return -1; + if (tEncodeI32(&encoder, pReq->execId) < 0) return -1; + + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + + if (buf != NULL) { + SMsgHead *pHead = (SMsgHead *)((char *)buf - headLen); + pHead->vgId = htonl(pReq->header.vgId); + pHead->contLen = htonl(tlen + headLen); + } + + return tlen + headLen; +} + +int32_t tDeserializeSResFetchReq(void *buf, int32_t bufLen, SResFetchReq *pReq) { + int32_t headLen = sizeof(SMsgHead); + + SMsgHead *pHead = buf; + pHead->vgId = pReq->header.vgId; + pHead->contLen = pReq->header.contLen; + + SDecoder decoder = {0}; + tDecoderInit(&decoder, (char *)buf + headLen, bufLen - headLen); + + if (tStartDecode(&decoder) < 0) return -1; + + if (tDecodeU64(&decoder, &pReq->sId) < 0) return -1; + if (tDecodeU64(&decoder, &pReq->queryId) < 0) return -1; + if (tDecodeU64(&decoder, &pReq->taskId) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->execId) < 0) return -1; + + tEndDecode(&decoder); + + tDecoderClear(&decoder); + return 0; +} + + +int32_t tSerializeSTqOffsetVal(SEncoder *pEncoder, STqOffsetVal *pOffset) { + if (tEncodeI8(pEncoder, pOffset->type) < 0) return -1; + if (tEncodeI64(pEncoder, pOffset->uid) < 0) return -1; + if (tEncodeI64(pEncoder, pOffset->ts) < 0) return -1; + + return 0; +} + +int32_t tDerializeSTqOffsetVal(SDecoder *pDecoder, STqOffsetVal *pOffset) { + if (tDecodeI8(pDecoder, &pOffset->type) < 0) return -1; + if (tDecodeI64(pDecoder, &pOffset->uid) < 0) return -1; + if (tDecodeI64(pDecoder, &pOffset->ts) < 0) return -1; + + return 0; +} + +int32_t tSerializeSMqPollReq(void *buf, int32_t bufLen, SMqPollReq *pReq) { + int32_t headLen = sizeof(SMsgHead); + if (buf != NULL) { + buf = (char *)buf + headLen; + bufLen -= headLen; + } + + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + if (tStartEncode(&encoder) < 0) return -1; + + if (tEncodeCStr(&encoder, pReq->subKey) < 0) return -1; + if (tEncodeI8(&encoder, pReq->withTbName) < 0) return -1; + if (tEncodeI8(&encoder, pReq->useSnapshot) < 0) return -1; + if (tEncodeI32(&encoder, pReq->epoch) < 0) return -1; + if (tEncodeU64(&encoder, pReq->reqId) < 0) return -1; + if (tEncodeI64(&encoder, pReq->consumerId) < 0) return -1; + if (tEncodeI64(&encoder, pReq->timeout) < 0) return -1; + if (tSerializeSTqOffsetVal(&encoder, &pReq->reqOffset) < 0) return -1; + + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + + if (buf != NULL) { + SMsgHead *pHead = (SMsgHead *)((char *)buf - headLen); + pHead->vgId = htonl(pReq->head.vgId); + pHead->contLen = htonl(tlen + headLen); + } + + return tlen + headLen; +} + +int32_t tDeserializeSMqPollReq(void *buf, int32_t bufLen, SMqPollReq *pReq) { + int32_t headLen = sizeof(SMsgHead); + + SMsgHead *pHead = buf; + pHead->vgId = pReq->head.vgId; + pHead->contLen = pReq->head.contLen; + + SDecoder decoder = {0}; + tDecoderInit(&decoder, (char *)buf + headLen, bufLen - headLen); + + if (tStartDecode(&decoder) < 0) return -1; + + if (tDecodeCStrTo(&decoder, pReq->subKey) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->withTbName) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->useSnapshot) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->epoch) < 0) return -1; + if (tDecodeU64(&decoder, &pReq->reqId) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->consumerId) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->timeout) < 0) return -1; + if (tDerializeSTqOffsetVal(&decoder, &pReq->reqOffset) < 0) return -1; + + tEndDecode(&decoder); + + tDecoderClear(&decoder); + return 0; +} + + int32_t tSerializeSTaskDropReq(void *buf, int32_t bufLen, STaskDropReq *pReq) { int32_t headLen = sizeof(SMsgHead); if (buf != NULL) { diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 5e35e34b87..927c0db287 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -458,20 +458,26 @@ static int32_t tqInitTaosxRsp(STaosxRsp* pRsp, const SMqPollReq* pReq) { } int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { - SMqPollReq* pReq = pMsg->pCont; - int64_t consumerId = pReq->consumerId; - int32_t reqEpoch = pReq->epoch; + SMqPollReq req = {0}; int32_t code = 0; - STqOffsetVal reqOffset = pReq->reqOffset; STqOffsetVal fetchOffsetNew; SWalCkHead* pCkHead = NULL; + if (tDeserializeSMqPollReq(pMsg->pCont, pMsg->contLen, &req) < 0) { + tqError("tDeserializeSMqPollReq %d failed", pMsg->contLen); + return -1; + } + + int64_t consumerId = req.consumerId; + int32_t reqEpoch = req.epoch; + STqOffsetVal reqOffset = req.reqOffset; + // 1.find handle - STqHandle* pHandle = taosHashGet(pTq->pHandle, pReq->subKey, strlen(pReq->subKey)); + STqHandle* pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey)); /*ASSERT(pHandle);*/ if (pHandle == NULL) { tqError("tmq poll: no consumer handle for consumer:%" PRId64 ", in vgId:%d, subkey %s", consumerId, - TD_VID(pTq->pVnode), pReq->subKey); + TD_VID(pTq->pVnode), req.subKey); return -1; } @@ -479,7 +485,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { if (pHandle->consumerId != consumerId) { tqError("tmq poll: consumer handle mismatch for consumer:%" PRId64 ", in vgId:%d, subkey %s, handle consumer id %" PRId64, - consumerId, TD_VID(pTq->pVnode), pReq->subKey, pHandle->consumerId); + consumerId, TD_VID(pTq->pVnode), req.subKey, pHandle->consumerId); terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH; return -1; } @@ -493,13 +499,13 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { char buf[80]; tFormatOffset(buf, 80, &reqOffset); tqDebug("tmq poll: consumer %" PRId64 " (epoch %d), subkey %s, recv poll req in vg %d, req offset %s", consumerId, - pReq->epoch, pHandle->subKey, TD_VID(pTq->pVnode), buf); + req.epoch, pHandle->subKey, TD_VID(pTq->pVnode), buf); // 2.reset offset if needed if (reqOffset.type > 0) { fetchOffsetNew = reqOffset; } else { - STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, pReq->subKey); + STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, req.subKey); if (pOffset != NULL) { fetchOffsetNew = pOffset->val; char formatBuf[80]; @@ -508,7 +514,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { TD_VID(pTq->pVnode), formatBuf); } else { if (reqOffset.type == TMQ_OFFSET__RESET_EARLIEAST) { - if (pReq->useSnapshot) { + if (req.useSnapshot) { if (pHandle->fetchMeta) { tqOffsetResetToMeta(&fetchOffsetNew, 0); } else { @@ -520,21 +526,21 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) { if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { SMqDataRsp dataRsp = {0}; - tqInitDataRsp(&dataRsp, pReq, pHandle->execHandle.subType); + tqInitDataRsp(&dataRsp, &req, pHandle->execHandle.subType); tqOffsetResetToLog(&dataRsp.rspOffset, walGetLastVer(pTq->pVnode->pWal)); tqDebug("tmq poll: consumer %" PRId64 ", subkey %s, vg %d, offset reset to %" PRId64, consumerId, pHandle->subKey, TD_VID(pTq->pVnode), dataRsp.rspOffset.version); - if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) { + if (tqSendDataRsp(pTq, pMsg, &req, &dataRsp) < 0) { code = -1; } tDeleteSMqDataRsp(&dataRsp); return code; } else { STaosxRsp taosxRsp = {0}; - tqInitTaosxRsp(&taosxRsp, pReq); + tqInitTaosxRsp(&taosxRsp, &req); tqOffsetResetToLog(&taosxRsp.rspOffset, walGetLastVer(pTq->pVnode->pWal)); - if (tqSendTaosxRsp(pTq, pMsg, pReq, &taosxRsp) < 0) { + if (tqSendTaosxRsp(pTq, pMsg, &req, &taosxRsp) < 0) { code = -1; } tDeleteSTaosxRsp(&taosxRsp); @@ -543,7 +549,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { } else if (reqOffset.type == TMQ_OFFSET__RESET_NONE) { tqError("tmq poll: subkey %s, no offset committed for consumer %" PRId64 " in vg %d, subkey %s, reset none failed", - pHandle->subKey, consumerId, TD_VID(pTq->pVnode), pReq->subKey); + pHandle->subKey, consumerId, TD_VID(pTq->pVnode), req.subKey); terrno = TSDB_CODE_TQ_NO_COMMITTED_OFFSET; return -1; } @@ -552,7 +558,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { SMqDataRsp dataRsp = {0}; - tqInitDataRsp(&dataRsp, pReq, pHandle->execHandle.subType); + tqInitDataRsp(&dataRsp, &req, pHandle->execHandle.subType); // lock taosWLockLatch(&pTq->pushLock); tqScanData(pTq, pHandle, &dataRsp, &fetchOffsetNew); @@ -580,7 +586,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { #endif taosWUnLockLatch(&pTq->pushLock); - if (tqSendDataRsp(pTq, pMsg, pReq, &dataRsp) < 0) { + if (tqSendDataRsp(pTq, pMsg, &req, &dataRsp) < 0) { code = -1; } @@ -599,13 +605,13 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { SMqMetaRsp metaRsp = {0}; STaosxRsp taosxRsp = {0}; - tqInitTaosxRsp(&taosxRsp, pReq); + tqInitTaosxRsp(&taosxRsp, &req); if (fetchOffsetNew.type != TMQ_OFFSET__LOG) { tqScanTaosx(pTq, pHandle, &taosxRsp, &metaRsp, &fetchOffsetNew); if (metaRsp.metaRspLen > 0) { - if (tqSendMetaPollRsp(pTq, pMsg, pReq, &metaRsp) < 0) { + if (tqSendMetaPollRsp(pTq, pMsg, &req, &metaRsp) < 0) { code = -1; } tqDebug("tmq poll: consumer %" PRId64 ", subkey %s, vg %d, send meta offset type:%d,uid:%" PRId64 @@ -618,7 +624,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { } if (taosxRsp.blockNum > 0) { - if (tqSendTaosxRsp(pTq, pMsg, pReq, &taosxRsp) < 0) { + if (tqSendTaosxRsp(pTq, pMsg, &req, &taosxRsp) < 0) { code = -1; } tDeleteSTaosxRsp(&taosxRsp); @@ -648,13 +654,13 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { if (consumerEpoch > reqEpoch) { tqWarn("tmq poll: consumer %" PRId64 " (epoch %d), subkey %s, vg %d offset %" PRId64 ", found new consumer epoch %d, discard req epoch %d", - consumerId, pReq->epoch, pHandle->subKey, TD_VID(pTq->pVnode), fetchVer, consumerEpoch, reqEpoch); + consumerId, req.epoch, pHandle->subKey, TD_VID(pTq->pVnode), fetchVer, consumerEpoch, reqEpoch); break; } if (tqFetchLog(pTq, pHandle, &fetchVer, &pCkHead) < 0) { tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer); - if (tqSendTaosxRsp(pTq, pMsg, pReq, &taosxRsp) < 0) { + if (tqSendTaosxRsp(pTq, pMsg, &req, &taosxRsp) < 0) { code = -1; } tDeleteSTaosxRsp(&taosxRsp); @@ -665,7 +671,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { SWalCont* pHead = &pCkHead->head; tqDebug("tmq poll: consumer:%" PRId64 ", (epoch %d) iter log, vgId:%d offset %" PRId64 " msgType %d", consumerId, - pReq->epoch, TD_VID(pTq->pVnode), fetchVer, pHead->msgType); + req.epoch, TD_VID(pTq->pVnode), fetchVer, pHead->msgType); if (pHead->msgType == TDMT_VND_SUBMIT) { SSubmitReq* pCont = (SSubmitReq*)&pHead->body; @@ -674,7 +680,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { } if (taosxRsp.blockNum > 0 /* threshold */) { tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer); - if (tqSendTaosxRsp(pTq, pMsg, pReq, &taosxRsp) < 0) { + if (tqSendTaosxRsp(pTq, pMsg, &req, &taosxRsp) < 0) { code = -1; } tDeleteSTaosxRsp(&taosxRsp); @@ -692,7 +698,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { metaRsp.resMsgType = pHead->msgType; metaRsp.metaRspLen = pHead->bodyLen; metaRsp.metaRsp = pHead->body; - if (tqSendMetaPollRsp(pTq, pMsg, pReq, &metaRsp) < 0) { + if (tqSendMetaPollRsp(pTq, pMsg, &req, &metaRsp) < 0) { code = -1; taosMemoryFreeClear(pCkHead); tDeleteSTaosxRsp(&taosxRsp); diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index 12d5b4112b..f89bc20362 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -308,9 +308,8 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) } if (vnodeIsRoleLeader(pTq->pVnode)) { + if (taosHashGetSize(pTq->pStreamMeta->pTasks) == 0) return 0; if (msgType == TDMT_VND_SUBMIT) { - if (taosHashGetSize(pTq->pStreamMeta->pTasks) == 0) return 0; - void* data = taosMemoryMalloc(msgLen); if (data == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/libs/executor/CMakeLists.txt b/source/libs/executor/CMakeLists.txt index 89d08b3078..8b3d04e32c 100644 --- a/source/libs/executor/CMakeLists.txt +++ b/source/libs/executor/CMakeLists.txt @@ -2,10 +2,6 @@ aux_source_directory(src EXECUTOR_SRC) #add_library(executor ${EXECUTOR_SRC}) add_library(executor STATIC ${EXECUTOR_SRC}) -#set_target_properties(executor PROPERTIES -# IMPORTED_LOCATION "${CMAKE_CURRENT_SOURCE_DIR}/libexecutor.a" -# INTERFACE_INCLUDE_DIRECTORIES "${TD_SOURCE_DIR}/include/libs/executor" -# ) target_link_libraries(executor PRIVATE os util common function parser planner qcom vnode scalar nodes index stream diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 7dc9e9ab03..b442034860 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -235,16 +235,6 @@ typedef enum { #define COL_MATCH_FROM_COL_ID 0x1 #define COL_MATCH_FROM_SLOT_ID 0x2 -typedef struct SSourceDataInfo { - int32_t index; - SRetrieveTableRsp* pRsp; - uint64_t totalRows; - int64_t startTime; - int32_t code; - EX_SOURCE_STATUS status; - const char* taskId; -} SSourceDataInfo; - typedef struct SLoadRemoteDataInfo { uint64_t totalSize; // total load bytes from remote uint64_t totalRows; // total number of rows @@ -371,23 +361,8 @@ typedef struct STagScanInfo { SColMatchInfo matchInfo; int32_t curPos; SReadHandle readHandle; - STableListInfo* pTableList; } STagScanInfo; -typedef struct SLastrowScanInfo { - SSDataBlock* pRes; - SReadHandle readHandle; - void* pLastrowReader; - SColMatchInfo matchInfo; - int32_t* pSlotIds; - SExprSupp pseudoExprSup; - int32_t retrieveType; - int32_t currentGroupIndex; - SSDataBlock* pBufferredRes; - SArray* pUidList; - int32_t indexOfBufferedRes; -} SLastrowScanInfo; - typedef enum EStreamScanMode { STREAM_SCAN_FROM_READERHANDLE = 1, STREAM_SCAN_FROM_RES, @@ -504,40 +479,6 @@ typedef struct { SSnapContext* sContext; } SStreamRawScanInfo; -typedef struct SSysTableIndex { - int8_t init; - SArray* uids; - int32_t lastIdx; -} SSysTableIndex; - -typedef struct SSysTableScanInfo { - SRetrieveMetaTableRsp* pRsp; - SRetrieveTableReq req; - SEpSet epSet; - tsem_t ready; - SReadHandle readHandle; - int32_t accountId; - const char* pUser; - bool sysInfo; - bool showRewrite; - SNode* pCondition; // db_name filter condition, to discard data that are not in current database - SMTbCursor* pCur; // cursor for iterate the local table meta store. - SSysTableIndex* pIdx; // idx for local table meta - SColMatchInfo matchInfo; - SName name; - SSDataBlock* pRes; - int64_t numOfBlocks; // extract basic running information. - SLoadRemoteDataInfo loadInfo; -} SSysTableScanInfo; - -typedef struct SBlockDistInfo { - SSDataBlock* pResBlock; - STsdbReader* pHandle; - SReadHandle readHandle; - uint64_t uid; // table uid -} SBlockDistInfo; - -// todo remove this typedef struct SOptrBasicInfo { SResultRowInfo resultRowInfo; SSDataBlock* pRes; @@ -603,24 +544,6 @@ typedef struct SAggOperatorInfo { SExprSupp scalarExprSup; } SAggOperatorInfo; -typedef struct SProjectOperatorInfo { - SOptrBasicInfo binfo; - SAggSupporter aggSup; - SArray* pPseudoColInfo; - SLimitInfo limitInfo; - bool mergeDataBlocks; - SSDataBlock* pFinalRes; -} SProjectOperatorInfo; - -typedef struct SIndefOperatorInfo { - SOptrBasicInfo binfo; - SAggSupporter aggSup; - SArray* pPseudoColInfo; - SExprSupp scalarSup; - uint64_t groupId; - SSDataBlock* pNextGroupRes; -} SIndefOperatorInfo; - typedef struct SFillOperatorInfo { struct SFillInfo* pFillInfo; SSDataBlock* pRes; @@ -638,42 +561,12 @@ typedef struct SFillOperatorInfo { SExprSupp noFillExprSupp; } SFillOperatorInfo; -typedef struct SGroupbyOperatorInfo { - SOptrBasicInfo binfo; - SAggSupporter aggSup; - SArray* pGroupCols; // group by columns, SArray - SArray* pGroupColVals; // current group column values, SArray - bool isInit; // denote if current val is initialized or not - char* keyBuf; // group by keys for hash - int32_t groupKeyLen; // total group by column width - SGroupResInfo groupResInfo; - SExprSupp scalarSup; -} SGroupbyOperatorInfo; - typedef struct SDataGroupInfo { uint64_t groupId; int64_t numOfRows; SArray* pPageList; } SDataGroupInfo; -// The sort in partition may be needed later. -typedef struct SPartitionOperatorInfo { - SOptrBasicInfo binfo; - SArray* pGroupCols; - SArray* pGroupColVals; // current group column values, SArray - char* keyBuf; // group by keys for hash - int32_t groupKeyLen; // total group by column width - SHashObj* pGroupSet; // quick locate the window object for each result - - SDiskbasedBuf* pBuf; // query result buffer based on blocked-wised disk file - int32_t rowCapacity; // maximum number of rows for each buffer page - int32_t* columnOffset; // start position for each column data - SArray* sortedGroupArray; // SDataGroupInfo sorted by group id - int32_t groupIndex; // group index - int32_t pageIndex; // page index of current group - SExprSupp scalarSup; -} SPartitionOperatorInfo; - typedef struct SWindowRowsSup { STimeWindow win; TSKEY prevTs; @@ -817,33 +710,6 @@ typedef struct SStateWindowOperatorInfo { STimeWindowAggSupp twAggSup; } SStateWindowOperatorInfo; -typedef struct SSortOperatorInfo { - SOptrBasicInfo binfo; - uint32_t sortBufSize; // max buffer size for in-memory sort - SArray* pSortInfo; - SSortHandle* pSortHandle; - SColMatchInfo matchInfo; - int32_t bufPageSize; - int64_t startTs; // sort start time - uint64_t sortElapsed; // sort elapsed time, time to flush to disk not included. - SLimitInfo limitInfo; -} SSortOperatorInfo; - -typedef struct SJoinOperatorInfo { - SSDataBlock* pRes; - int32_t joinType; - int32_t inputOrder; - - SSDataBlock* pLeft; - int32_t leftPos; - SColumnInfo leftCol; - - SSDataBlock* pRight; - int32_t rightPos; - SColumnInfo rightCol; - SNode* pCondAfterMerge; -} SJoinOperatorInfo; - #define OPTR_IS_OPENED(_optr) (((_optr)->status & OP_OPENED) == OP_OPENED) #define OPTR_SET_OPENED(_optr) ((_optr)->status |= OP_OPENED) @@ -867,7 +733,6 @@ void doBuildStreamResBlock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGr void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, SDiskbasedBuf* pBuf); -int32_t handleLimitOffset(SOperatorInfo* pOperator, SLimitInfo* pLimitInfo, SSDataBlock* pBlock, bool holdDataInBuf); bool hasLimitOffsetInfo(SLimitInfo* pLimitInfo); void initLimitInfo(const SNode* pLimit, const SNode* pSLimit, SLimitInfo* pLimitInfo); void applyLimitOffset(SLimitInfo* pLimitInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo, SOperatorInfo* pOperator); @@ -897,9 +762,6 @@ void cleanupAggSup(SAggSupporter* pAggSup); void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle); void setTbNameColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId, const char* name); -int32_t doPrepareScan(SOperatorInfo* pOperator, uint64_t uid, int64_t ts); -int32_t doGetScanStatus(SOperatorInfo* pOperator, uint64_t* uid, int64_t* ts); - SSDataBlock* loadNextDataBlock(void* param); void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset); @@ -982,9 +844,8 @@ void setInputDataBlock(SExprSupp* pExprSupp, SSDataBlock* pBlock, int32_t order, bool isTaskKilled(SExecTaskInfo* pTaskInfo); int32_t checkForQueryBuf(size_t numOfTables); -void setTaskKilled(SExecTaskInfo* pTaskInfo); -void queryCostStatis(SExecTaskInfo* pTaskInfo); - +void setTaskKilled(SExecTaskInfo* pTaskInfo); +void queryCostStatis(SExecTaskInfo* pTaskInfo); void doDestroyTask(SExecTaskInfo* pTaskInfo); void destroyOperatorInfo(SOperatorInfo* pOperator); int32_t getMaximumIdleDurationSec(); @@ -1012,9 +873,6 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, qTaskInfo_t* pTaskInfo, SReadHandle* readHandle); int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SArray* pExecInfoList); -int32_t aggDecodeResultRow(SOperatorInfo* pOperator, char* result); -int32_t aggEncodeResultRow(SOperatorInfo* pOperator, char** result, int32_t* length); - STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, SInterval* pInterval, int32_t order); int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos, TSKEY ekey, diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index 5e5c01201f..6b5f773fe3 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -25,6 +25,20 @@ #include "thash.h" #include "ttypes.h" +typedef struct SCacheRowsScanInfo { + SSDataBlock* pRes; + SReadHandle readHandle; + void* pLastrowReader; + SColMatchInfo matchInfo; + int32_t* pSlotIds; + SExprSupp pseudoExprSup; + int32_t retrieveType; + int32_t currentGroupIndex; + SSDataBlock* pBufferredRes; + SArray* pUidList; + int32_t indexOfBufferedRes; +} SCacheRowsScanInfo; + static SSDataBlock* doScanCache(SOperatorInfo* pOperator); static void destroyCacheScanOperator(void* param); static int32_t extractCacheScanSlotId(const SArray* pColMatchInfo, SExecTaskInfo* pTaskInfo, int32_t** pSlotIds); @@ -33,7 +47,7 @@ static int32_t removeRedundantTsCol(SLastRowScanPhysiNode* pScanNode, SColM SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SReadHandle* readHandle, SExecTaskInfo* pTaskInfo) { int32_t code = TSDB_CODE_SUCCESS; - SLastrowScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SLastrowScanInfo)); + SCacheRowsScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SCacheRowsScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -114,7 +128,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { return NULL; } - SLastrowScanInfo* pInfo = pOperator->info; + SCacheRowsScanInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; STableListInfo* pTableList = pTaskInfo->pTableInfoList; @@ -240,7 +254,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { } void destroyCacheScanOperator(void* param) { - SLastrowScanInfo* pInfo = (SLastrowScanInfo*)param; + SCacheRowsScanInfo* pInfo = (SCacheRowsScanInfo*)param; blockDataDestroy(pInfo->pRes); blockDataDestroy(pInfo->pBufferredRes); taosMemoryFree(pInfo->pSlotIds); diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index 78afdd16b7..09ca1d27b9 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -250,6 +250,8 @@ static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, return code; } + taosArrayClear(pInserter->pDataBlocks); + code = sendSubmitRequest(pInserter, pMsg, pInserter->pParam->readHandle->pMsgCb->clientRpc, &pInserter->pNode->epSet); if (code) { return code; diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index a28066003a..3e3c73797d 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -41,6 +41,16 @@ typedef struct SFetchRspHandleWrapper { int32_t sourceIndex; } SFetchRspHandleWrapper; +typedef struct SSourceDataInfo { + int32_t index; + SRetrieveTableRsp* pRsp; + uint64_t totalRows; + int64_t startTime; + int32_t code; + EX_SOURCE_STATUS status; + const char* taskId; +} SSourceDataInfo; + static void destroyExchangeOperatorInfo(void* param); static void freeBlock(void* pParam); static void freeSourceDataInfo(void* param); @@ -52,6 +62,7 @@ static int32_t getCompletedSources(const SArray* pArray); static int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator); static int32_t seqLoadRemoteData(SOperatorInfo* pOperator); static int32_t prepareLoadRemoteData(SOperatorInfo* pOperator); +static int32_t handleLimitOffset(SOperatorInfo* pOperator, SLimitInfo* pLimitInfo, SSDataBlock* pBlock, bool holdDataInBuf); static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTaskInfo) { @@ -405,27 +416,42 @@ int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTas loadRemoteDataCallback(pWrapper, &pBuf, code); taosMemoryFree(pWrapper); } else { - SResFetchReq* pMsg = taosMemoryCalloc(1, sizeof(SResFetchReq)); - if (NULL == pMsg) { + SResFetchReq req = {0}; + req.header.vgId = pSource->addr.nodeId; + req.sId = pSource->schedId; + req.taskId = pSource->taskId; + req.queryId = pTaskInfo->id.queryId; + req.execId = pSource->execId; + + int32_t msgSize = tSerializeSResFetchReq(NULL, 0, &req); + if (msgSize < 0) { pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; taosMemoryFree(pWrapper); return pTaskInfo->code; } + + void* msg = taosMemoryCalloc(1, msgSize); + if (NULL == msg) { + pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + taosMemoryFree(pWrapper); + return pTaskInfo->code; + } + + if (tSerializeSResFetchReq(msg, msgSize, &req) < 0) { + pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + taosMemoryFree(pWrapper); + taosMemoryFree(msg); + return pTaskInfo->code; + } qDebug("%s build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", execId:%d, %d/%" PRIzu, GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->addr.epSet.eps[0].fqdn, pSource->taskId, pSource->execId, sourceIndex, totalSources); - pMsg->header.vgId = htonl(pSource->addr.nodeId); - pMsg->sId = htobe64(pSource->schedId); - pMsg->taskId = htobe64(pSource->taskId); - pMsg->queryId = htobe64(pTaskInfo->id.queryId); - pMsg->execId = htonl(pSource->execId); - // send the fetch remote task result reques SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); if (NULL == pMsgSendInfo) { - taosMemoryFreeClear(pMsg); + taosMemoryFreeClear(msg); taosMemoryFree(pWrapper); qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; @@ -434,8 +460,8 @@ int32_t doSendFetchDataRequest(SExchangeInfo* pExchangeInfo, SExecTaskInfo* pTas pMsgSendInfo->param = pWrapper; pMsgSendInfo->paramFreeFp = taosMemoryFree; - pMsgSendInfo->msgInfo.pData = pMsg; - pMsgSendInfo->msgInfo.len = sizeof(SResFetchReq); + pMsgSendInfo->msgInfo.pData = msg; + pMsgSendInfo->msgInfo.len = msgSize; pMsgSendInfo->msgType = pSource->fetchMsgType; pMsgSendInfo->fp = loadRemoteDataCallback; @@ -647,3 +673,80 @@ int32_t prepareLoadRemoteData(SOperatorInfo* pOperator) { pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0; return TSDB_CODE_SUCCESS; } + +int32_t handleLimitOffset(SOperatorInfo* pOperator, SLimitInfo* pLimitInfo, SSDataBlock* pBlock, bool holdDataInBuf) { + if (pLimitInfo->remainGroupOffset > 0) { + if (pLimitInfo->currentGroupId == 0) { // it is the first group + pLimitInfo->currentGroupId = pBlock->info.groupId; + blockDataCleanup(pBlock); + return PROJECT_RETRIEVE_CONTINUE; + } else if (pLimitInfo->currentGroupId != pBlock->info.groupId) { + // now it is the data from a new group + pLimitInfo->remainGroupOffset -= 1; + + // ignore data block in current group + if (pLimitInfo->remainGroupOffset > 0) { + blockDataCleanup(pBlock); + return PROJECT_RETRIEVE_CONTINUE; + } + } + + // set current group id of the project operator + pLimitInfo->currentGroupId = pBlock->info.groupId; + } + + // here check for a new group data, we need to handle the data of the previous group. + if (pLimitInfo->currentGroupId != 0 && pLimitInfo->currentGroupId != pBlock->info.groupId) { + pLimitInfo->numOfOutputGroups += 1; + if ((pLimitInfo->slimit.limit > 0) && (pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups)) { + pOperator->status = OP_EXEC_DONE; + blockDataCleanup(pBlock); + + return PROJECT_RETRIEVE_DONE; + } + + // reset the value for a new group data + pLimitInfo->numOfOutputRows = 0; + pLimitInfo->remainOffset = pLimitInfo->limit.offset; + + // existing rows that belongs to previous group. + if (pBlock->info.rows > 0) { + return PROJECT_RETRIEVE_DONE; + } + } + + // here we reach the start position, according to the limit/offset requirements. + + // set current group id + pLimitInfo->currentGroupId = pBlock->info.groupId; + + if (pLimitInfo->remainOffset >= pBlock->info.rows) { + pLimitInfo->remainOffset -= pBlock->info.rows; + blockDataCleanup(pBlock); + return PROJECT_RETRIEVE_CONTINUE; + } else if (pLimitInfo->remainOffset < pBlock->info.rows && pLimitInfo->remainOffset > 0) { + blockDataTrimFirstNRows(pBlock, pLimitInfo->remainOffset); + pLimitInfo->remainOffset = 0; + } + + // check for the limitation in each group + if (pLimitInfo->limit.limit >= 0 && pLimitInfo->numOfOutputRows + pBlock->info.rows >= pLimitInfo->limit.limit) { + int32_t keepRows = (int32_t)(pLimitInfo->limit.limit - pLimitInfo->numOfOutputRows); + blockDataKeepFirstNRows(pBlock, keepRows); + if (pLimitInfo->slimit.limit > 0 && pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups) { + pOperator->status = OP_EXEC_DONE; + } + + return PROJECT_RETRIEVE_DONE; + } + + // todo optimize performance + // If there are slimit/soffset value exists, multi-round result can not be packed into one group, since the + // they may not belong to the same group the limit/offset value is not valid in this case. + if ((!holdDataInBuf) || (pBlock->info.rows >= pOperator->resultInfo.threshold) || pLimitInfo->slimit.offset != -1 || + pLimitInfo->slimit.limit != -1) { + return PROJECT_RETRIEVE_DONE; + } else { // not full enough, continue to accumulate the output data in the buffer. + return PROJECT_RETRIEVE_CONTINUE; + } +} diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 7dad9245d5..c1974ff30e 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -480,51 +480,6 @@ _error: return code; } -#ifdef TEST_IMPL -// wait moment -int waitMoment(SQInfo* pQInfo) { - if (pQInfo->sql) { - int ms = 0; - char* pcnt = strstr(pQInfo->sql, " count(*)"); - if (pcnt) return 0; - - char* pos = strstr(pQInfo->sql, " t_"); - if (pos) { - pos += 3; - ms = atoi(pos); - while (*pos >= '0' && *pos <= '9') { - pos++; - } - char unit_char = *pos; - if (unit_char == 'h') { - ms *= 3600 * 1000; - } else if (unit_char == 'm') { - ms *= 60 * 1000; - } else if (unit_char == 's') { - ms *= 1000; - } - } - if (ms == 0) return 0; - printf("test wait sleep %dms. sql=%s ...\n", ms, pQInfo->sql); - - if (ms < 1000) { - taosMsleep(ms); - } else { - int used_ms = 0; - while (used_ms < ms) { - taosMsleep(1000); - used_ms += 1000; - if (isTaskKilled(pQInfo)) { - printf("test check query is canceled, sleep break.%s\n", pQInfo->sql); - break; - } - } - } - } - return 1; -} -#endif - static void freeBlock(void* param) { SSDataBlock* pBlock = *(SSDataBlock**)param; blockDataDestroy(pBlock); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 20c39ec921..4319dd379a 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -24,7 +24,6 @@ #include "tdatablock.h" #include "tglobal.h" #include "tmsg.h" -#include "tsort.h" #include "ttime.h" #include "executorimpl.h" @@ -297,8 +296,6 @@ void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow colDataAppendInt64(pColData, 4, &pQueryWindow->ekey); } -void cleanupExecTimeWindowInfo(SColumnInfoData* pColData) { colDataDestroy(pColData); } - typedef struct { bool hasAgg; int32_t numOfRows; @@ -1347,42 +1344,6 @@ void queryCostStatis(SExecTaskInfo* pTaskInfo) { } } -// static void updateOffsetVal(STaskRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBlockInfo) { -// STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; -// STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; -// -// int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); -// -// if (pQueryAttr->limit.offset == pBlockInfo->rows) { // current block will ignore completed -// pTableQueryInfo->lastKey = QUERY_IS_ASC_QUERY(pQueryAttr) ? pBlockInfo->window.ekey + step : -// pBlockInfo->window.skey + step; pQueryAttr->limit.offset = 0; return; -// } -// -// if (QUERY_IS_ASC_QUERY(pQueryAttr)) { -// pQueryAttr->pos = (int32_t)pQueryAttr->limit.offset; -// } else { -// pQueryAttr->pos = pBlockInfo->rows - (int32_t)pQueryAttr->limit.offset - 1; -// } -// -// assert(pQueryAttr->pos >= 0 && pQueryAttr->pos <= pBlockInfo->rows - 1); -// -// SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pTsdbReadHandle, NULL); -// SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); -// -// // update the pQueryAttr->limit.offset value, and pQueryAttr->pos value -// TSKEY *keys = (TSKEY *) pColInfoData->pData; -// -// // update the offset value -// pTableQueryInfo->lastKey = keys[pQueryAttr->pos]; -// pQueryAttr->limit.offset = 0; -// -// int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, pBlockInfo, NULL, binarySearchForKey, pDataBlock); -// -// //qDebug("QInfo:0x%"PRIx64" check data block, brange:%" PRId64 "-%" PRId64 ", numBlocksOfStep:%d, numOfRes:%d, -// lastKey:%"PRId64, GET_TASKID(pRuntimeEnv), -// pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes, pQuery->current->lastKey); -// } - // void skipBlocks(STaskRuntimeEnv *pRuntimeEnv) { // STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; // @@ -1723,159 +1684,6 @@ static SSDataBlock* getAggregateResult(SOperatorInfo* pOperator) { return (rows == 0) ? NULL : pInfo->pRes; } -int32_t aggEncodeResultRow(SOperatorInfo* pOperator, char** result, int32_t* length) { - if (result == NULL || length == NULL) { - return TSDB_CODE_TSC_INVALID_INPUT; - } - SOptrBasicInfo* pInfo = (SOptrBasicInfo*)(pOperator->info); - SAggSupporter* pSup = (SAggSupporter*)POINTER_SHIFT(pOperator->info, sizeof(SOptrBasicInfo)); - int32_t size = tSimpleHashGetSize(pSup->pResultRowHashTable); - size_t keyLen = sizeof(uint64_t) * 2; // estimate the key length - int32_t totalSize = - sizeof(int32_t) + sizeof(int32_t) + size * (sizeof(int32_t) + keyLen + sizeof(int32_t) + pSup->resultRowSize); - - // no result - if (getTotalBufSize(pSup->pResultBuf) == 0) { - *result = NULL; - *length = 0; - return TSDB_CODE_SUCCESS; - } - - *result = (char*)taosMemoryCalloc(1, totalSize); - if (*result == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - int32_t offset = sizeof(int32_t); - *(int32_t*)(*result + offset) = size; - offset += sizeof(int32_t); - - // prepare memory - SResultRowPosition* pos = &pInfo->resultRowInfo.cur; - void* pPage = getBufPage(pSup->pResultBuf, pos->pageId); - SResultRow* pRow = (SResultRow*)((char*)pPage + pos->offset); - setBufPageDirty(pPage, true); - releaseBufPage(pSup->pResultBuf, pPage); - - int32_t iter = 0; - void* pIter = NULL; - while ((pIter = tSimpleHashIterate(pSup->pResultRowHashTable, pIter, &iter))) { - void* key = tSimpleHashGetKey(pIter, &keyLen); - SResultRowPosition* p1 = (SResultRowPosition*)pIter; - - pPage = (SFilePage*)getBufPage(pSup->pResultBuf, p1->pageId); - pRow = (SResultRow*)((char*)pPage + p1->offset); - setBufPageDirty(pPage, true); - releaseBufPage(pSup->pResultBuf, pPage); - - // recalculate the result size - int32_t realTotalSize = offset + sizeof(int32_t) + keyLen + sizeof(int32_t) + pSup->resultRowSize; - if (realTotalSize > totalSize) { - char* tmp = (char*)taosMemoryRealloc(*result, realTotalSize); - if (tmp == NULL) { - taosMemoryFree(*result); - *result = NULL; - return TSDB_CODE_OUT_OF_MEMORY; - } else { - *result = tmp; - } - } - // save key - *(int32_t*)(*result + offset) = keyLen; - offset += sizeof(int32_t); - memcpy(*result + offset, key, keyLen); - offset += keyLen; - - // save value - *(int32_t*)(*result + offset) = pSup->resultRowSize; - offset += sizeof(int32_t); - memcpy(*result + offset, pRow, pSup->resultRowSize); - offset += pSup->resultRowSize; - } - - *(int32_t*)(*result) = offset; - *length = offset; - - return TDB_CODE_SUCCESS; -} - -int32_t handleLimitOffset(SOperatorInfo* pOperator, SLimitInfo* pLimitInfo, SSDataBlock* pBlock, bool holdDataInBuf) { - if (pLimitInfo->remainGroupOffset > 0) { - if (pLimitInfo->currentGroupId == 0) { // it is the first group - pLimitInfo->currentGroupId = pBlock->info.groupId; - blockDataCleanup(pBlock); - return PROJECT_RETRIEVE_CONTINUE; - } else if (pLimitInfo->currentGroupId != pBlock->info.groupId) { - // now it is the data from a new group - pLimitInfo->remainGroupOffset -= 1; - - // ignore data block in current group - if (pLimitInfo->remainGroupOffset > 0) { - blockDataCleanup(pBlock); - return PROJECT_RETRIEVE_CONTINUE; - } - } - - // set current group id of the project operator - pLimitInfo->currentGroupId = pBlock->info.groupId; - } - - // here check for a new group data, we need to handle the data of the previous group. - if (pLimitInfo->currentGroupId != 0 && pLimitInfo->currentGroupId != pBlock->info.groupId) { - pLimitInfo->numOfOutputGroups += 1; - if ((pLimitInfo->slimit.limit > 0) && (pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups)) { - pOperator->status = OP_EXEC_DONE; - blockDataCleanup(pBlock); - - return PROJECT_RETRIEVE_DONE; - } - - // reset the value for a new group data - pLimitInfo->numOfOutputRows = 0; - pLimitInfo->remainOffset = pLimitInfo->limit.offset; - - // existing rows that belongs to previous group. - if (pBlock->info.rows > 0) { - return PROJECT_RETRIEVE_DONE; - } - } - - // here we reach the start position, according to the limit/offset requirements. - - // set current group id - pLimitInfo->currentGroupId = pBlock->info.groupId; - - if (pLimitInfo->remainOffset >= pBlock->info.rows) { - pLimitInfo->remainOffset -= pBlock->info.rows; - blockDataCleanup(pBlock); - return PROJECT_RETRIEVE_CONTINUE; - } else if (pLimitInfo->remainOffset < pBlock->info.rows && pLimitInfo->remainOffset > 0) { - blockDataTrimFirstNRows(pBlock, pLimitInfo->remainOffset); - pLimitInfo->remainOffset = 0; - } - - // check for the limitation in each group - if (pLimitInfo->limit.limit >= 0 && pLimitInfo->numOfOutputRows + pBlock->info.rows >= pLimitInfo->limit.limit) { - int32_t keepRows = (int32_t)(pLimitInfo->limit.limit - pLimitInfo->numOfOutputRows); - blockDataKeepFirstNRows(pBlock, keepRows); - if (pLimitInfo->slimit.limit > 0 && pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups) { - pOperator->status = OP_EXEC_DONE; - } - - return PROJECT_RETRIEVE_DONE; - } - - // todo optimize performance - // If there are slimit/soffset value exists, multi-round result can not be packed into one group, since the - // they may not belong to the same group the limit/offset value is not valid in this case. - if ((!holdDataInBuf) || (pBlock->info.rows >= pOperator->resultInfo.threshold) || pLimitInfo->slimit.offset != -1 || - pLimitInfo->slimit.limit != -1) { - return PROJECT_RETRIEVE_DONE; - } else { // not full enough, continue to accumulate the output data in the buffer. - return PROJECT_RETRIEVE_CONTINUE; - } -} - static void doApplyScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_t order, int32_t scanFlag); static void doHandleRemainBlockForNewGroupImpl(SOperatorInfo* pOperator, SFillOperatorInfo* pInfo, SResultInfo* pResultInfo, SExecTaskInfo* pTaskInfo) { diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 26a5f6838d..fcfb79eb4b 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -27,6 +27,36 @@ #include "thash.h" #include "ttypes.h" +typedef struct SGroupbyOperatorInfo { + SOptrBasicInfo binfo; + SAggSupporter aggSup; + SArray* pGroupCols; // group by columns, SArray + SArray* pGroupColVals; // current group column values, SArray + bool isInit; // denote if current val is initialized or not + char* keyBuf; // group by keys for hash + int32_t groupKeyLen; // total group by column width + SGroupResInfo groupResInfo; + SExprSupp scalarSup; +} SGroupbyOperatorInfo; + +// The sort in partition may be needed later. +typedef struct SPartitionOperatorInfo { + SOptrBasicInfo binfo; + SArray* pGroupCols; + SArray* pGroupColVals; // current group column values, SArray + char* keyBuf; // group by keys for hash + int32_t groupKeyLen; // total group by column width + SHashObj* pGroupSet; // quick locate the window object for each result + + SDiskbasedBuf* pBuf; // query result buffer based on blocked-wised disk file + int32_t rowCapacity; // maximum number of rows for each buffer page + int32_t* columnOffset; // start position for each column data + SArray* sortedGroupArray; // SDataGroupInfo sorted by group id + int32_t groupIndex; // group index + int32_t pageIndex; // page index of current group + SExprSupp scalarSup; +} SPartitionOperatorInfo; + static void* getCurrentDataGroupInfo(const SPartitionOperatorInfo* pInfo, SDataGroupInfo** pGroupInfo, int32_t len); static int32_t* setupColumnOffset(const SSDataBlock* pBlock, int32_t rowCapacity); static int32_t setGroupResultOutputBuf(SOperatorInfo* pOperator, SOptrBasicInfo* binfo, int32_t numOfCols, char* pData, diff --git a/source/libs/executor/src/joinoperator.c b/source/libs/executor/src/joinoperator.c index 4e1daac643..61dc7a1b76 100644 --- a/source/libs/executor/src/joinoperator.c +++ b/source/libs/executor/src/joinoperator.c @@ -24,6 +24,21 @@ #include "tmsg.h" #include "ttypes.h" +typedef struct SJoinOperatorInfo { + SSDataBlock* pRes; + int32_t joinType; + int32_t inputOrder; + + SSDataBlock* pLeft; + int32_t leftPos; + SColumnInfo leftCol; + + SSDataBlock* pRight; + int32_t rightPos; + SColumnInfo rightCol; + SNode* pCondAfterMerge; +} SJoinOperatorInfo; + static void setJoinColumnInfo(SColumnInfo* pColumn, const SColumnNode* pColumnNode); static SSDataBlock* doMergeJoin(struct SOperatorInfo* pOperator); static void destroyMergeJoinOperator(void* param); diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index ce1d13775c..ada7964c67 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -17,6 +17,24 @@ #include "executorimpl.h" #include "functionMgt.h" +typedef struct SProjectOperatorInfo { + SOptrBasicInfo binfo; + SAggSupporter aggSup; + SArray* pPseudoColInfo; + SLimitInfo limitInfo; + bool mergeDataBlocks; + SSDataBlock* pFinalRes; +} SProjectOperatorInfo; + +typedef struct SIndefOperatorInfo { + SOptrBasicInfo binfo; + SAggSupporter aggSup; + SArray* pPseudoColInfo; + SExprSupp scalarSup; + uint64_t groupId; + SSDataBlock* pNextGroupRes; +} SIndefOperatorInfo; + static SSDataBlock* doGenerateSourceData(SOperatorInfo* pOperator); static SSDataBlock* doProjectOperation(SOperatorInfo* pOperator); static SSDataBlock* doApplyIndefinitFunction(SOperatorInfo* pOperator); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 7128e0a2c9..de2a7b9dac 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -13,7 +13,6 @@ * along with this program. If not, see . */ -#include #include "executorimpl.h" #include "filter.h" #include "function.h" @@ -36,29 +35,6 @@ #define SET_REVERSE_SCAN_FLAG(_info) ((_info)->scanFlag = REVERSE_SCAN) #define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC)) -static int32_t buildSysDbTableInfo(const SSysTableScanInfo* pInfo, int32_t capacity); -static int32_t buildDbTableInfoBlock(bool sysInfo, const SSDataBlock* p, const SSysTableMeta* pSysDbTableMeta, - size_t size, const char* dbName); - -static char* SYSTABLE_IDX_COLUMN[] = {"table_name", "db_name", "create_time", "columns", - "ttl", "stable_name", "vgroup_id', 'uid", "type"}; - -static char* SYSTABLE_SPECIAL_COL[] = {"db_name", "vgroup_id"}; - -typedef int32_t (*__sys_filte)(void* pMeta, SNode* cond, SArray* result); -typedef int32_t (*__sys_check)(SNode* cond); - -typedef struct { - const char* name; - __sys_check chkFunc; - __sys_filte fltFunc; -} SSTabFltFuncDef; - -typedef struct { - void* pMeta; - void* pVnode; -} SSTabFltArg; - typedef struct STableMergeScanExecInfo { SFileBlockLoadRecorder blockRecorder; SSortExecInfo sortExecInfo; @@ -71,54 +47,8 @@ typedef struct STableMergeScanSortSourceParam { SSDataBlock* inputBlock; } STableMergeScanSortSourceParam; -static int32_t sysChkFilter__Comm(SNode* pNode); -static int32_t sysChkFilter__DBName(SNode* pNode); -static int32_t sysChkFilter__VgroupId(SNode* pNode); -static int32_t sysChkFilter__TableName(SNode* pNode); -static int32_t sysChkFilter__CreateTime(SNode* pNode); -static int32_t sysChkFilter__Ncolumn(SNode* pNode); -static int32_t sysChkFilter__Ttl(SNode* pNode); -static int32_t sysChkFilter__STableName(SNode* pNode); -static int32_t sysChkFilter__Uid(SNode* pNode); -static int32_t sysChkFilter__Type(SNode* pNode); - -static int32_t sysFilte__DbName(void* arg, SNode* pNode, SArray* result); -static int32_t sysFilte__VgroupId(void* arg, SNode* pNode, SArray* result); -static int32_t sysFilte__TableName(void* arg, SNode* pNode, SArray* result); -static int32_t sysFilte__CreateTime(void* arg, SNode* pNode, SArray* result); -static int32_t sysFilte__Ncolumn(void* arg, SNode* pNode, SArray* result); -static int32_t sysFilte__Ttl(void* arg, SNode* pNode, SArray* result); -static int32_t sysFilte__STableName(void* arg, SNode* pNode, SArray* result); -static int32_t sysFilte__Uid(void* arg, SNode* pNode, SArray* result); -static int32_t sysFilte__Type(void* arg, SNode* pNode, SArray* result); - -const SSTabFltFuncDef filterDict[] = { - {.name = "table_name", .chkFunc = sysChkFilter__TableName, .fltFunc = sysFilte__TableName}, - {.name = "db_name", .chkFunc = sysChkFilter__DBName, .fltFunc = sysFilte__DbName}, - {.name = "create_time", .chkFunc = sysChkFilter__CreateTime, .fltFunc = sysFilte__CreateTime}, - {.name = "columns", .chkFunc = sysChkFilter__Ncolumn, .fltFunc = sysFilte__Ncolumn}, - {.name = "ttl", .chkFunc = sysChkFilter__Ttl, .fltFunc = sysFilte__Ttl}, - {.name = "stable_name", .chkFunc = sysChkFilter__STableName, .fltFunc = sysFilte__STableName}, - {.name = "vgroup_id", .chkFunc = sysChkFilter__VgroupId, .fltFunc = sysFilte__VgroupId}, - {.name = "uid", .chkFunc = sysChkFilter__Uid, .fltFunc = sysFilte__Uid}, - {.name = "type", .chkFunc = sysChkFilter__Type, .fltFunc = sysFilte__Type}}; - -#define SYSTAB_FILTER_DICT_SIZE (sizeof(filterDict) / sizeof(filterDict[0])) - -static int32_t optSysTabFilte(void* arg, SNode* cond, SArray* result); -static int32_t optSysTabFilteImpl(void* arg, SNode* cond, SArray* result); -static int32_t optSysCheckOper(SNode* pOpear); -static int32_t optSysMergeRslt(SArray* mRslt, SArray* rslt); - static bool processBlockWithProbability(const SSampleExecInfo* pInfo); -static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, SMetaReader* smrSuperTable, - SMetaReader* smrChildTable, const char* dbname, const char* tableName, - int32_t* pNumOfRows, const SSDataBlock* dataBlock); - -static void relocateAndFilterSysTagsScanResult(SSysTableScanInfo* pInfo, int32_t numOfRows, SSDataBlock* dataBlock, - SFilterInfo* pFilterInfo); - bool processBlockWithProbability(const SSampleExecInfo* pInfo) { #if 0 if (pInfo->sampleRatio == 1) { @@ -1007,169 +937,6 @@ SOperatorInfo* createTableSeqScanOperatorInfo(void* pReadHandle, SExecTaskInfo* return pOperator; } -static int32_t doGetTableRowSize(void* pMeta, uint64_t uid, int32_t* rowLen, const char* idstr) { - *rowLen = 0; - - SMetaReader mr = {0}; - metaReaderInit(&mr, pMeta, 0); - int32_t code = metaGetTableEntryByUid(&mr, uid); - if (code != TSDB_CODE_SUCCESS) { - qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", uid, tstrerror(terrno), idstr); - metaReaderClear(&mr); - return terrno; - } - - if (mr.me.type == TSDB_SUPER_TABLE) { - int32_t numOfCols = mr.me.stbEntry.schemaRow.nCols; - for (int32_t i = 0; i < numOfCols; ++i) { - (*rowLen) += mr.me.stbEntry.schemaRow.pSchema[i].bytes; - } - } else if (mr.me.type == TSDB_CHILD_TABLE) { - uint64_t suid = mr.me.ctbEntry.suid; - tDecoderClear(&mr.coder); - code = metaGetTableEntryByUid(&mr, suid); - if (code != TSDB_CODE_SUCCESS) { - qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", suid, tstrerror(terrno), idstr); - metaReaderClear(&mr); - return terrno; - } - - int32_t numOfCols = mr.me.stbEntry.schemaRow.nCols; - - for (int32_t i = 0; i < numOfCols; ++i) { - (*rowLen) += mr.me.stbEntry.schemaRow.pSchema[i].bytes; - } - } else if (mr.me.type == TSDB_NORMAL_TABLE) { - int32_t numOfCols = mr.me.ntbEntry.schemaRow.nCols; - for (int32_t i = 0; i < numOfCols; ++i) { - (*rowLen) += mr.me.ntbEntry.schemaRow.pSchema[i].bytes; - } - } - - metaReaderClear(&mr); - return TSDB_CODE_SUCCESS; -} - -static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) { - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - SBlockDistInfo* pBlockScanInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - STableBlockDistInfo blockDistInfo = {.minRows = INT_MAX, .maxRows = INT_MIN}; - int32_t code = doGetTableRowSize(pBlockScanInfo->readHandle.meta, pBlockScanInfo->uid, - (int32_t*)&blockDistInfo.rowSize, GET_TASKID(pTaskInfo)); - if (code != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pTaskInfo->env, code); - } - - tsdbGetFileBlocksDistInfo(pBlockScanInfo->pHandle, &blockDistInfo); - blockDistInfo.numOfInmemRows = (int32_t)tsdbGetNumOfRowsInMemTable(pBlockScanInfo->pHandle); - - SSDataBlock* pBlock = pBlockScanInfo->pResBlock; - - int32_t slotId = pOperator->exprSupp.pExprInfo->base.resSchema.slotId; - SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, slotId); - - int32_t len = tSerializeBlockDistInfo(NULL, 0, &blockDistInfo); - char* p = taosMemoryCalloc(1, len + VARSTR_HEADER_SIZE); - tSerializeBlockDistInfo(varDataVal(p), len, &blockDistInfo); - varDataSetLen(p, len); - - colDataAppend(pColInfo, 0, p, false); - taosMemoryFree(p); - - pBlock->info.rows = 1; - pOperator->status = OP_EXEC_DONE; - return pBlock; -} - -static void destroyBlockDistScanOperatorInfo(void* param) { - SBlockDistInfo* pDistInfo = (SBlockDistInfo*)param; - blockDataDestroy(pDistInfo->pResBlock); - tsdbReaderClose(pDistInfo->pHandle); - taosMemoryFreeClear(param); -} - -static int32_t initTableblockDistQueryCond(uint64_t uid, SQueryTableDataCond* pCond) { - memset(pCond, 0, sizeof(SQueryTableDataCond)); - - pCond->order = TSDB_ORDER_ASC; - pCond->numOfCols = 1; - pCond->colList = taosMemoryCalloc(1, sizeof(SColumnInfo)); - if (pCond->colList == NULL) { - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - return terrno; - } - - pCond->colList->colId = 1; - pCond->colList->type = TSDB_DATA_TYPE_TIMESTAMP; - pCond->colList->bytes = sizeof(TSKEY); - - pCond->twindows = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX}; - pCond->suid = uid; - pCond->type = TIMEWINDOW_RANGE_CONTAINED; - pCond->startVersion = -1; - pCond->endVersion = -1; - - return TSDB_CODE_SUCCESS; -} - -SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDistScanPhysiNode* pBlockScanNode, - SExecTaskInfo* pTaskInfo) { - SBlockDistInfo* pInfo = taosMemoryCalloc(1, sizeof(SBlockDistInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - if (pInfo == NULL || pOperator == NULL) { - pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; - goto _error; - } - - { - SQueryTableDataCond cond = {0}; - - int32_t code = initTableblockDistQueryCond(pBlockScanNode->suid, &cond); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - STableListInfo* pTableListInfo = pTaskInfo->pTableInfoList; - size_t num = tableListGetSize(pTableListInfo); - void* pList = tableListGetInfo(pTableListInfo, 0); - - code = tsdbReaderOpen(readHandle->vnode, &cond, pList, num, &pInfo->pHandle, pTaskInfo->id.str); - cleanupQueryTableDataCond(&cond); - if (code != 0) { - goto _error; - } - } - - pInfo->readHandle = *readHandle; - pInfo->uid = pBlockScanNode->suid; - - pInfo->pResBlock = createResDataBlock(pBlockScanNode->node.pOutputDataBlockDesc); - blockDataEnsureCapacity(pInfo->pResBlock, 1); - - int32_t numOfCols = 0; - SExprInfo* pExprInfo = createExprInfo(pBlockScanNode->pScanPseudoCols, NULL, &numOfCols); - int32_t code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfCols); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - setOperatorInfo(pOperator, "DataBlockDistScanOperator", QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN, false, - OP_NOT_OPENED, pInfo, pTaskInfo); - pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doBlockInfoScan, NULL, destroyBlockDistScanOperatorInfo, NULL); - return pOperator; - -_error: - taosMemoryFreeClear(pInfo); - taosMemoryFreeClear(pOperator); - return NULL; -} - static FORCE_INLINE void doClearBufferedBlocks(SStreamScanInfo* pInfo) { taosArrayClear(pInfo->pBlockLists); pInfo->validBlockIndex = 0; @@ -2615,1638 +2382,6 @@ _error: return NULL; } -static void destroySysScanOperator(void* param) { - SSysTableScanInfo* pInfo = (SSysTableScanInfo*)param; - tsem_destroy(&pInfo->ready); - blockDataDestroy(pInfo->pRes); - - const char* name = tNameGetTableName(&pInfo->name); - if (strncasecmp(name, TSDB_INS_TABLE_TABLES, TSDB_TABLE_FNAME_LEN) == 0 || - strncasecmp(name, TSDB_INS_TABLE_TAGS, TSDB_TABLE_FNAME_LEN) == 0 || pInfo->pCur != NULL) { - metaCloseTbCursor(pInfo->pCur); - pInfo->pCur = NULL; - } - if (pInfo->pIdx) { - taosArrayDestroy(pInfo->pIdx->uids); - taosMemoryFree(pInfo->pIdx); - pInfo->pIdx = NULL; - } - - taosArrayDestroy(pInfo->matchInfo.pList); - taosMemoryFreeClear(pInfo->pUser); - - taosMemoryFreeClear(param); -} - -static int32_t getSysTableDbNameColId(const char* pTable) { - // if (0 == strcmp(TSDB_INS_TABLE_INDEXES, pTable)) { - // return 1; - // } - return TSDB_INS_USER_STABLES_DBNAME_COLID; -} - -EDealRes getDBNameFromConditionWalker(SNode* pNode, void* pContext) { - int32_t code = TSDB_CODE_SUCCESS; - ENodeType nType = nodeType(pNode); - - switch (nType) { - case QUERY_NODE_OPERATOR: { - SOperatorNode* node = (SOperatorNode*)pNode; - if (OP_TYPE_EQUAL == node->opType) { - *(int32_t*)pContext = 1; - return DEAL_RES_CONTINUE; - } - - *(int32_t*)pContext = 0; - return DEAL_RES_IGNORE_CHILD; - } - case QUERY_NODE_COLUMN: { - if (1 != *(int32_t*)pContext) { - return DEAL_RES_CONTINUE; - } - - SColumnNode* node = (SColumnNode*)pNode; - if (getSysTableDbNameColId(node->tableName) == node->colId) { - *(int32_t*)pContext = 2; - return DEAL_RES_CONTINUE; - } - - *(int32_t*)pContext = 0; - return DEAL_RES_CONTINUE; - } - case QUERY_NODE_VALUE: { - if (2 != *(int32_t*)pContext) { - return DEAL_RES_CONTINUE; - } - - SValueNode* node = (SValueNode*)pNode; - char* dbName = nodesGetValueFromNode(node); - strncpy(pContext, varDataVal(dbName), varDataLen(dbName)); - *((char*)pContext + varDataLen(dbName)) = 0; - return DEAL_RES_END; // stop walk - } - default: - break; - } - return DEAL_RES_CONTINUE; -} - -static void getDBNameFromCondition(SNode* pCondition, const char* dbName) { - if (NULL == pCondition) { - return; - } - nodesWalkExpr(pCondition, getDBNameFromConditionWalker, (char*)dbName); -} - -static int32_t loadSysTableCallback(void* param, SDataBuf* pMsg, int32_t code) { - SOperatorInfo* operator=(SOperatorInfo*) param; - SSysTableScanInfo* pScanResInfo = (SSysTableScanInfo*)operator->info; - if (TSDB_CODE_SUCCESS == code) { - pScanResInfo->pRsp = pMsg->pData; - - SRetrieveMetaTableRsp* pRsp = pScanResInfo->pRsp; - pRsp->numOfRows = htonl(pRsp->numOfRows); - pRsp->useconds = htobe64(pRsp->useconds); - pRsp->handle = htobe64(pRsp->handle); - pRsp->compLen = htonl(pRsp->compLen); - } else { - operator->pTaskInfo->code = code; - } - - tsem_post(&pScanResInfo->ready); - return TSDB_CODE_SUCCESS; -} - -static SSDataBlock* doFilterResult(SSDataBlock* pDataBlock, SFilterInfo* pFilterInfo) { - if (pFilterInfo == NULL) { - return pDataBlock->info.rows == 0 ? NULL : pDataBlock; - } - - doFilter(pDataBlock, pFilterInfo, NULL); - return pDataBlock->info.rows == 0 ? NULL : pDataBlock; -} - -static SSDataBlock* buildInfoSchemaTableMetaBlock(char* tableName) { - size_t size = 0; - const SSysTableMeta* pMeta = NULL; - getInfosDbMeta(&pMeta, &size); - - int32_t index = 0; - for (int32_t i = 0; i < size; ++i) { - if (strcmp(pMeta[i].name, tableName) == 0) { - index = i; - break; - } - } - - SSDataBlock* pBlock = createDataBlock(); - for (int32_t i = 0; i < pMeta[index].colNum; ++i) { - SColumnInfoData colInfoData = - createColumnInfoData(pMeta[index].schema[i].type, pMeta[index].schema[i].bytes, i + 1); - blockDataAppendColInfo(pBlock, &colInfoData); - } - - return pBlock; -} - -int32_t convertTagDataToStr(char* str, int type, void* buf, int32_t bufSize, int32_t* len) { - int32_t n = 0; - - switch (type) { - case TSDB_DATA_TYPE_NULL: - n = sprintf(str, "null"); - break; - - case TSDB_DATA_TYPE_BOOL: - n = sprintf(str, (*(int8_t*)buf) ? "true" : "false"); - break; - - case TSDB_DATA_TYPE_TINYINT: - n = sprintf(str, "%d", *(int8_t*)buf); - break; - - case TSDB_DATA_TYPE_SMALLINT: - n = sprintf(str, "%d", *(int16_t*)buf); - break; - - case TSDB_DATA_TYPE_INT: - n = sprintf(str, "%d", *(int32_t*)buf); - break; - - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_TIMESTAMP: - n = sprintf(str, "%" PRId64, *(int64_t*)buf); - break; - - case TSDB_DATA_TYPE_FLOAT: - n = sprintf(str, "%.5f", GET_FLOAT_VAL(buf)); - break; - - case TSDB_DATA_TYPE_DOUBLE: - n = sprintf(str, "%.9f", GET_DOUBLE_VAL(buf)); - break; - - case TSDB_DATA_TYPE_BINARY: - if (bufSize < 0) { - return TSDB_CODE_TSC_INVALID_VALUE; - } - - memcpy(str, buf, bufSize); - n = bufSize; - break; - case TSDB_DATA_TYPE_NCHAR: - if (bufSize < 0) { - return TSDB_CODE_TSC_INVALID_VALUE; - } - - int32_t length = taosUcs4ToMbs((TdUcs4*)buf, bufSize, str); - if (length <= 0) { - return TSDB_CODE_TSC_INVALID_VALUE; - } - n = length; - break; - case TSDB_DATA_TYPE_UTINYINT: - n = sprintf(str, "%u", *(uint8_t*)buf); - break; - - case TSDB_DATA_TYPE_USMALLINT: - n = sprintf(str, "%u", *(uint16_t*)buf); - break; - - case TSDB_DATA_TYPE_UINT: - n = sprintf(str, "%u", *(uint32_t*)buf); - break; - - case TSDB_DATA_TYPE_UBIGINT: - n = sprintf(str, "%" PRIu64, *(uint64_t*)buf); - break; - - default: - return TSDB_CODE_TSC_INVALID_VALUE; - } - - if (len) *len = n; - - return TSDB_CODE_SUCCESS; -} - -static bool sysTableIsOperatorCondOnOneTable(SNode* pCond, char* condTable) { - SOperatorNode* node = (SOperatorNode*)pCond; - if (node->opType == OP_TYPE_EQUAL) { - if (nodeType(node->pLeft) == QUERY_NODE_COLUMN && - strcasecmp(nodesGetNameFromColumnNode(node->pLeft), "table_name") == 0 && - nodeType(node->pRight) == QUERY_NODE_VALUE) { - SValueNode* pValue = (SValueNode*)node->pRight; - if (pValue->node.resType.type == TSDB_DATA_TYPE_NCHAR || pValue->node.resType.type == TSDB_DATA_TYPE_VARCHAR || - pValue->node.resType.type == TSDB_DATA_TYPE_BINARY) { - char* value = nodesGetValueFromNode(pValue); - strncpy(condTable, varDataVal(value), TSDB_TABLE_NAME_LEN); - return true; - } - } - } - return false; -} - -static bool sysTableIsCondOnOneTable(SNode* pCond, char* condTable) { - if (pCond == NULL) { - return false; - } - if (nodeType(pCond) == QUERY_NODE_LOGIC_CONDITION) { - SLogicConditionNode* node = (SLogicConditionNode*)pCond; - if (LOGIC_COND_TYPE_AND == node->condType) { - SNode* pChild = NULL; - FOREACH(pChild, node->pParameterList) { - if (QUERY_NODE_OPERATOR == nodeType(pChild) && sysTableIsOperatorCondOnOneTable(pChild, condTable)) { - return true; - } - } - } - } - - if (QUERY_NODE_OPERATOR == nodeType(pCond)) { - return sysTableIsOperatorCondOnOneTable(pCond, condTable); - } - - return false; -} - -static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SSysTableScanInfo* pInfo = pOperator->info; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - blockDataCleanup(pInfo->pRes); - int32_t numOfRows = 0; - - SSDataBlock* dataBlock = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_TAGS); - blockDataEnsureCapacity(dataBlock, pOperator->resultInfo.capacity); - - const char* db = NULL; - int32_t vgId = 0; - vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); - - SName sn = {0}; - char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; - tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); - - tNameGetDbName(&sn, varDataVal(dbname)); - varDataSetLen(dbname, strlen(varDataVal(dbname))); - - char condTableName[TSDB_TABLE_NAME_LEN] = {0}; - // optimize when sql like where table_name='tablename' and xxx. - if (pInfo->pCondition && sysTableIsCondOnOneTable(pInfo->pCondition, condTableName)) { - char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(tableName, condTableName); - - SMetaReader smrChildTable = {0}; - metaReaderInit(&smrChildTable, pInfo->readHandle.meta, 0); - int32_t code = metaGetTableEntryByName(&smrChildTable, condTableName); - if (code != TSDB_CODE_SUCCESS) { - // terrno has been set by metaGetTableEntryByName, therefore, return directly - return NULL; - } - - if (smrChildTable.me.type != TSDB_CHILD_TABLE) { - metaReaderClear(&smrChildTable); - blockDataDestroy(dataBlock); - pInfo->loadInfo.totalRows = 0; - return NULL; - } - - SMetaReader smrSuperTable = {0}; - metaReaderInit(&smrSuperTable, pInfo->readHandle.meta, META_READER_NOLOCK); - code = metaGetTableEntryByUid(&smrSuperTable, smrChildTable.me.ctbEntry.suid); - if (code != TSDB_CODE_SUCCESS) { - // terrno has been set by metaGetTableEntryByUid - return NULL; - } - - sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &smrChildTable, dbname, tableName, &numOfRows, dataBlock); - metaReaderClear(&smrSuperTable); - metaReaderClear(&smrChildTable); - if (numOfRows > 0) { - relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); - numOfRows = 0; - } - blockDataDestroy(dataBlock); - pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; - setOperatorCompleted(pOperator); - return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; - } - - int32_t ret = 0; - if (pInfo->pCur == NULL) { - pInfo->pCur = metaOpenTbCursor(pInfo->readHandle.meta); - } - - while ((ret = metaTbCursorNext(pInfo->pCur)) == 0) { - if (pInfo->pCur->mr.me.type != TSDB_CHILD_TABLE) { - continue; - } - - char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(tableName, pInfo->pCur->mr.me.name); - - SMetaReader smrSuperTable = {0}; - metaReaderInit(&smrSuperTable, pInfo->readHandle.meta, 0); - uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; - int32_t code = metaGetTableEntryByUid(&smrSuperTable, suid); - if (code != TSDB_CODE_SUCCESS) { - qError("failed to get super table meta, uid:0x%" PRIx64 ", code:%s, %s", suid, tstrerror(terrno), - GET_TASKID(pTaskInfo)); - metaReaderClear(&smrSuperTable); - metaCloseTbCursor(pInfo->pCur); - pInfo->pCur = NULL; - T_LONG_JMP(pTaskInfo->env, terrno); - } - - sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &pInfo->pCur->mr, dbname, tableName, &numOfRows, dataBlock); - - metaReaderClear(&smrSuperTable); - - if (numOfRows >= pOperator->resultInfo.capacity) { - relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); - numOfRows = 0; - - if (pInfo->pRes->info.rows > 0) { - break; - } - } - } - - if (numOfRows > 0) { - relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); - numOfRows = 0; - } - - blockDataDestroy(dataBlock); - if (ret != 0) { - metaCloseTbCursor(pInfo->pCur); - pInfo->pCur = NULL; - setOperatorCompleted(pOperator); - } - - pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; - return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; -} - -void relocateAndFilterSysTagsScanResult(SSysTableScanInfo* pInfo, int32_t numOfRows, SSDataBlock* dataBlock, - SFilterInfo* pFilterInfo) { - dataBlock->info.rows = numOfRows; - pInfo->pRes->info.rows = numOfRows; - - relocateColumnData(pInfo->pRes, pInfo->matchInfo.pList, dataBlock->pDataBlock, false); - doFilterResult(pInfo->pRes, pFilterInfo); - blockDataCleanup(dataBlock); -} - -static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, SMetaReader* smrSuperTable, - SMetaReader* smrChildTable, const char* dbname, const char* tableName, - int32_t* pNumOfRows, const SSDataBlock* dataBlock) { - char stableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(stableName, (*smrSuperTable).me.name); - - int32_t numOfRows = *pNumOfRows; - - int32_t numOfTags = (*smrSuperTable).me.stbEntry.schemaTag.nCols; - for (int32_t i = 0; i < numOfTags; ++i) { - SColumnInfoData* pColInfoData = NULL; - - // table name - pColInfoData = taosArrayGet(dataBlock->pDataBlock, 0); - colDataAppend(pColInfoData, numOfRows, tableName, false); - - // database name - pColInfoData = taosArrayGet(dataBlock->pDataBlock, 1); - colDataAppend(pColInfoData, numOfRows, dbname, false); - - // super table name - pColInfoData = taosArrayGet(dataBlock->pDataBlock, 2); - colDataAppend(pColInfoData, numOfRows, stableName, false); - - // tag name - char tagName[TSDB_COL_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(tagName, (*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].name); - pColInfoData = taosArrayGet(dataBlock->pDataBlock, 3); - colDataAppend(pColInfoData, numOfRows, tagName, false); - - // tag type - int8_t tagType = (*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].type; - pColInfoData = taosArrayGet(dataBlock->pDataBlock, 4); - char tagTypeStr[VARSTR_HEADER_SIZE + 32]; - int tagTypeLen = sprintf(varDataVal(tagTypeStr), "%s", tDataTypes[tagType].name); - if (tagType == TSDB_DATA_TYPE_VARCHAR) { - tagTypeLen += sprintf(varDataVal(tagTypeStr) + tagTypeLen, "(%d)", - (int32_t)((*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].bytes - VARSTR_HEADER_SIZE)); - } else if (tagType == TSDB_DATA_TYPE_NCHAR) { - tagTypeLen += sprintf( - varDataVal(tagTypeStr) + tagTypeLen, "(%d)", - (int32_t)(((*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); - } - varDataSetLen(tagTypeStr, tagTypeLen); - colDataAppend(pColInfoData, numOfRows, (char*)tagTypeStr, false); - - STagVal tagVal = {0}; - tagVal.cid = (*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].colId; - char* tagData = NULL; - uint32_t tagLen = 0; - - if (tagType == TSDB_DATA_TYPE_JSON) { - tagData = (char*)smrChildTable->me.ctbEntry.pTags; - } else { - bool exist = tTagGet((STag*)smrChildTable->me.ctbEntry.pTags, &tagVal); - if (exist) { - if (IS_VAR_DATA_TYPE(tagType)) { - tagData = (char*)tagVal.pData; - tagLen = tagVal.nData; - } else { - tagData = (char*)&tagVal.i64; - tagLen = tDataTypes[tagType].bytes; - } - } - } - - char* tagVarChar = NULL; - if (tagData != NULL) { - if (tagType == TSDB_DATA_TYPE_JSON) { - char* tagJson = parseTagDatatoJson(tagData); - tagVarChar = taosMemoryMalloc(strlen(tagJson) + VARSTR_HEADER_SIZE); - memcpy(varDataVal(tagVarChar), tagJson, strlen(tagJson)); - varDataSetLen(tagVarChar, strlen(tagJson)); - taosMemoryFree(tagJson); - } else { - int32_t bufSize = IS_VAR_DATA_TYPE(tagType) ? (tagLen + VARSTR_HEADER_SIZE) - : (3 + DBL_MANT_DIG - DBL_MIN_EXP + VARSTR_HEADER_SIZE); - tagVarChar = taosMemoryMalloc(bufSize); - int32_t len = -1; - convertTagDataToStr(varDataVal(tagVarChar), tagType, tagData, tagLen, &len); - varDataSetLen(tagVarChar, len); - } - } - pColInfoData = taosArrayGet(dataBlock->pDataBlock, 5); - colDataAppend(pColInfoData, numOfRows, tagVarChar, - (tagData == NULL) || (tagType == TSDB_DATA_TYPE_JSON && tTagIsJsonNull(tagData))); - taosMemoryFree(tagVarChar); - ++numOfRows; - } - - *pNumOfRows = numOfRows; - - return TSDB_CODE_SUCCESS; -} - -typedef int (*__optSysFilter)(void* a, void* b, int16_t dtype); - -int optSysDoCompare(__compar_fn_t func, int8_t comparType, void* a, void* b) { - int32_t cmp = func(a, b); - switch (comparType) { - case OP_TYPE_LOWER_THAN: - if (cmp < 0) return 0; - break; - case OP_TYPE_LOWER_EQUAL: { - if (cmp <= 0) return 0; - break; - } - case OP_TYPE_GREATER_THAN: { - if (cmp > 0) return 0; - break; - } - case OP_TYPE_GREATER_EQUAL: { - if (cmp >= 0) return 0; - break; - } - case OP_TYPE_EQUAL: { - if (cmp == 0) return 0; - break; - } - default: - return -1; - } - return cmp; -} - -static int optSysFilterFuncImpl__LowerThan(void* a, void* b, int16_t dtype) { - __compar_fn_t func = getComparFunc(dtype, 0); - return optSysDoCompare(func, OP_TYPE_LOWER_THAN, a, b); -} -static int optSysFilterFuncImpl__LowerEqual(void* a, void* b, int16_t dtype) { - __compar_fn_t func = getComparFunc(dtype, 0); - return optSysDoCompare(func, OP_TYPE_LOWER_EQUAL, a, b); -} -static int optSysFilterFuncImpl__GreaterThan(void* a, void* b, int16_t dtype) { - __compar_fn_t func = getComparFunc(dtype, 0); - return optSysDoCompare(func, OP_TYPE_GREATER_THAN, a, b); -} -static int optSysFilterFuncImpl__GreaterEqual(void* a, void* b, int16_t dtype) { - __compar_fn_t func = getComparFunc(dtype, 0); - return optSysDoCompare(func, OP_TYPE_GREATER_EQUAL, a, b); -} -static int optSysFilterFuncImpl__Equal(void* a, void* b, int16_t dtype) { - __compar_fn_t func = getComparFunc(dtype, 0); - return optSysDoCompare(func, OP_TYPE_EQUAL, a, b); -} - -static int optSysFilterFuncImpl__NoEqual(void* a, void* b, int16_t dtype) { - __compar_fn_t func = getComparFunc(dtype, 0); - return optSysDoCompare(func, OP_TYPE_NOT_EQUAL, a, b); -} -static __optSysFilter optSysGetFilterFunc(int32_t ctype, bool* reverse) { - if (ctype == OP_TYPE_LOWER_EQUAL || ctype == OP_TYPE_LOWER_THAN) { - *reverse = true; - } - if (ctype == OP_TYPE_LOWER_THAN) - return optSysFilterFuncImpl__LowerThan; - else if (ctype == OP_TYPE_LOWER_EQUAL) - return optSysFilterFuncImpl__LowerEqual; - else if (ctype == OP_TYPE_GREATER_THAN) - return optSysFilterFuncImpl__GreaterThan; - else if (ctype == OP_TYPE_GREATER_EQUAL) - return optSysFilterFuncImpl__GreaterEqual; - else if (ctype == OP_TYPE_EQUAL) - return optSysFilterFuncImpl__Equal; - else if (ctype == OP_TYPE_NOT_EQUAL) - return optSysFilterFuncImpl__NoEqual; - return NULL; -} -static int32_t sysFilte__DbName(void* arg, SNode* pNode, SArray* result) { - void* pVnode = ((SSTabFltArg*)arg)->pVnode; - - const char* db = NULL; - vnodeGetInfo(pVnode, &db, NULL); - - SName sn = {0}; - char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; - tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); - - tNameGetDbName(&sn, varDataVal(dbname)); - varDataSetLen(dbname, strlen(varDataVal(dbname))); - - SOperatorNode* pOper = (SOperatorNode*)pNode; - SValueNode* pVal = (SValueNode*)pOper->pRight; - - bool reverse = false; - __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); - if (func == NULL) return -1; - - int ret = func(dbname, pVal->datum.p, TSDB_DATA_TYPE_VARCHAR); - if (ret == 0) return 0; - - return -2; -} -static int32_t sysFilte__VgroupId(void* arg, SNode* pNode, SArray* result) { - void* pVnode = ((SSTabFltArg*)arg)->pVnode; - - int64_t vgId = 0; - vnodeGetInfo(pVnode, NULL, (int32_t*)&vgId); - - SOperatorNode* pOper = (SOperatorNode*)pNode; - SValueNode* pVal = (SValueNode*)pOper->pRight; - - bool reverse = false; - - __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); - if (func == NULL) return -1; - - int ret = func(&vgId, &pVal->datum.i, TSDB_DATA_TYPE_BIGINT); - if (ret == 0) return 0; - - return -1; -} -static int32_t sysFilte__TableName(void* arg, SNode* pNode, SArray* result) { - void* pMeta = ((SSTabFltArg*)arg)->pMeta; - - SOperatorNode* pOper = (SOperatorNode*)pNode; - SValueNode* pVal = (SValueNode*)pOper->pRight; - bool reverse = false; - - __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); - if (func == NULL) return -1; - - SMetaFltParam param = {.suid = 0, - .cid = 0, - .type = TSDB_DATA_TYPE_VARCHAR, - .val = pVal->datum.p, - .reverse = reverse, - .filterFunc = func}; - return -1; -} - -static int32_t sysFilte__CreateTime(void* arg, SNode* pNode, SArray* result) { - void* pMeta = ((SSTabFltArg*)arg)->pMeta; - - SOperatorNode* pOper = (SOperatorNode*)pNode; - SValueNode* pVal = (SValueNode*)pOper->pRight; - bool reverse = false; - - __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); - if (func == NULL) return -1; - - SMetaFltParam param = {.suid = 0, - .cid = 0, - .type = TSDB_DATA_TYPE_BIGINT, - .val = &pVal->datum.i, - .reverse = reverse, - .filterFunc = func}; - - int32_t ret = metaFilterCreateTime(pMeta, ¶m, result); - return ret; -} -static int32_t sysFilte__Ncolumn(void* arg, SNode* pNode, SArray* result) { - void* pMeta = ((SSTabFltArg*)arg)->pMeta; - - SOperatorNode* pOper = (SOperatorNode*)pNode; - SValueNode* pVal = (SValueNode*)pOper->pRight; - bool reverse = false; - - __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); - if (func == NULL) return -1; - return -1; -} - -static int32_t sysFilte__Ttl(void* arg, SNode* pNode, SArray* result) { - void* pMeta = ((SSTabFltArg*)arg)->pMeta; - - SOperatorNode* pOper = (SOperatorNode*)pNode; - SValueNode* pVal = (SValueNode*)pOper->pRight; - bool reverse = false; - - __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); - if (func == NULL) return -1; - return -1; -} -static int32_t sysFilte__STableName(void* arg, SNode* pNode, SArray* result) { - void* pMeta = ((SSTabFltArg*)arg)->pMeta; - - SOperatorNode* pOper = (SOperatorNode*)pNode; - SValueNode* pVal = (SValueNode*)pOper->pRight; - bool reverse = false; - - __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); - if (func == NULL) return -1; - return -1; -} -static int32_t sysFilte__Uid(void* arg, SNode* pNode, SArray* result) { - void* pMeta = ((SSTabFltArg*)arg)->pMeta; - - SOperatorNode* pOper = (SOperatorNode*)pNode; - SValueNode* pVal = (SValueNode*)pOper->pRight; - bool reverse = false; - - __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); - if (func == NULL) return -1; - return -1; -} -static int32_t sysFilte__Type(void* arg, SNode* pNode, SArray* result) { - void* pMeta = ((SSTabFltArg*)arg)->pMeta; - - SOperatorNode* pOper = (SOperatorNode*)pNode; - SValueNode* pVal = (SValueNode*)pOper->pRight; - bool reverse = false; - - __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); - if (func == NULL) return -1; - return -1; -} -static int32_t sysChkFilter__Comm(SNode* pNode) { - // impl - SOperatorNode* pOper = (SOperatorNode*)pNode; - EOperatorType opType = pOper->opType; - if (opType != OP_TYPE_EQUAL && opType != OP_TYPE_LOWER_EQUAL && opType != OP_TYPE_LOWER_THAN && - opType != OP_TYPE_GREATER_EQUAL && opType != OP_TYPE_GREATER_THAN) { - return -1; - } - return 0; -} - -static int32_t sysChkFilter__DBName(SNode* pNode) { - SOperatorNode* pOper = (SOperatorNode*)pNode; - - if (pOper->opType != OP_TYPE_EQUAL && pOper->opType != OP_TYPE_NOT_EQUAL) { - return -1; - } - - SValueNode* pVal = (SValueNode*)pOper->pRight; - if (!IS_STR_DATA_TYPE(pVal->node.resType.type)) { - return -1; - } - - return 0; -} -static int32_t sysChkFilter__VgroupId(SNode* pNode) { - SOperatorNode* pOper = (SOperatorNode*)pNode; - SValueNode* pVal = (SValueNode*)pOper->pRight; - if (!IS_INTEGER_TYPE(pVal->node.resType.type)) { - return -1; - } - return sysChkFilter__Comm(pNode); -} -static int32_t sysChkFilter__TableName(SNode* pNode) { - SOperatorNode* pOper = (SOperatorNode*)pNode; - SValueNode* pVal = (SValueNode*)pOper->pRight; - if (!IS_STR_DATA_TYPE(pVal->node.resType.type)) { - return -1; - } - return sysChkFilter__Comm(pNode); -} -static int32_t sysChkFilter__CreateTime(SNode* pNode) { - SOperatorNode* pOper = (SOperatorNode*)pNode; - SValueNode* pVal = (SValueNode*)pOper->pRight; - - if (!IS_TIMESTAMP_TYPE(pVal->node.resType.type)) { - return -1; - } - return sysChkFilter__Comm(pNode); -} - -static int32_t sysChkFilter__Ncolumn(SNode* pNode) { - SOperatorNode* pOper = (SOperatorNode*)pNode; - SValueNode* pVal = (SValueNode*)pOper->pRight; - - if (!IS_INTEGER_TYPE(pVal->node.resType.type)) { - return -1; - } - return sysChkFilter__Comm(pNode); -} -static int32_t sysChkFilter__Ttl(SNode* pNode) { - SOperatorNode* pOper = (SOperatorNode*)pNode; - SValueNode* pVal = (SValueNode*)pOper->pRight; - - if (!IS_INTEGER_TYPE(pVal->node.resType.type)) { - return -1; - } - return sysChkFilter__Comm(pNode); -} -static int32_t sysChkFilter__STableName(SNode* pNode) { - SOperatorNode* pOper = (SOperatorNode*)pNode; - SValueNode* pVal = (SValueNode*)pOper->pRight; - if (!IS_STR_DATA_TYPE(pVal->node.resType.type)) { - return -1; - } - return sysChkFilter__Comm(pNode); -} -static int32_t sysChkFilter__Uid(SNode* pNode) { - SOperatorNode* pOper = (SOperatorNode*)pNode; - SValueNode* pVal = (SValueNode*)pOper->pRight; - if (!IS_INTEGER_TYPE(pVal->node.resType.type)) { - return -1; - } - return sysChkFilter__Comm(pNode); -} -static int32_t sysChkFilter__Type(SNode* pNode) { - SOperatorNode* pOper = (SOperatorNode*)pNode; - SValueNode* pVal = (SValueNode*)pOper->pRight; - if (!IS_INTEGER_TYPE(pVal->node.resType.type)) { - return -1; - } - return sysChkFilter__Comm(pNode); -} -static int32_t optSysTabFilteImpl(void* arg, SNode* cond, SArray* result) { - if (optSysCheckOper(cond) != 0) return -1; - - SOperatorNode* pNode = (SOperatorNode*)cond; - - int8_t i = 0; - for (; i < SYSTAB_FILTER_DICT_SIZE; i++) { - if (strcmp(filterDict[i].name, ((SColumnNode*)(pNode->pLeft))->colName) == 0) { - break; - } - } - if (i >= SYSTAB_FILTER_DICT_SIZE) return -1; - - if (filterDict[i].chkFunc(cond) != 0) return -1; - - return filterDict[i].fltFunc(arg, cond, result); -} - -static int32_t optSysCheckOper(SNode* pOpear) { - if (nodeType(pOpear) != QUERY_NODE_OPERATOR) return -1; - - SOperatorNode* pOper = (SOperatorNode*)pOpear; - if (pOper->opType < OP_TYPE_GREATER_THAN || pOper->opType > OP_TYPE_NOT_EQUAL) { - return -1; - } - - if (nodeType(pOper->pLeft) != QUERY_NODE_COLUMN || nodeType(pOper->pRight) != QUERY_NODE_VALUE) { - return -1; - } - return 0; -} - -static int tableUidCompare(const void* a, const void* b) { - int64_t u1 = *(int64_t*)a; - int64_t u2 = *(int64_t*)b; - if (u1 == u2) { - return 0; - } - return u1 < u2 ? -1 : 1; -} - -typedef struct MergeIndex { - int idx; - int len; -} MergeIndex; - -static FORCE_INLINE int optSysBinarySearch(SArray* arr, int s, int e, uint64_t k) { - uint64_t v; - int32_t m; - while (s <= e) { - m = s + (e - s) / 2; - v = *(uint64_t*)taosArrayGet(arr, m); - if (v >= k) { - e = m - 1; - } else { - s = m + 1; - } - } - return s; -} - -void optSysIntersection(SArray* in, SArray* out) { - int32_t sz = (int32_t)taosArrayGetSize(in); - if (sz <= 0) { - return; - } - MergeIndex* mi = taosMemoryCalloc(sz, sizeof(MergeIndex)); - for (int i = 0; i < sz; i++) { - SArray* t = taosArrayGetP(in, i); - mi[i].len = (int32_t)taosArrayGetSize(t); - mi[i].idx = 0; - } - - SArray* base = taosArrayGetP(in, 0); - for (int i = 0; i < taosArrayGetSize(base); i++) { - uint64_t tgt = *(uint64_t*)taosArrayGet(base, i); - bool has = true; - for (int j = 1; j < taosArrayGetSize(in); j++) { - SArray* oth = taosArrayGetP(in, j); - int mid = optSysBinarySearch(oth, mi[j].idx, mi[j].len - 1, tgt); - if (mid >= 0 && mid < mi[j].len) { - uint64_t val = *(uint64_t*)taosArrayGet(oth, mid); - has = (val == tgt ? true : false); - mi[j].idx = mid; - } else { - has = false; - } - } - if (has == true) { - taosArrayPush(out, &tgt); - } - } - taosMemoryFreeClear(mi); -} - -static int32_t optSysMergeRslt(SArray* mRslt, SArray* rslt) { - // TODO, find comm mem from mRslt - for (int i = 0; i < taosArrayGetSize(mRslt); i++) { - SArray* arslt = taosArrayGetP(mRslt, i); - taosArraySort(arslt, tableUidCompare); - } - optSysIntersection(mRslt, rslt); - return 0; -} - -static int32_t optSysSpecialColumn(SNode* cond) { - SOperatorNode* pOper = (SOperatorNode*)cond; - SColumnNode* pCol = (SColumnNode*)pOper->pLeft; - for (int i = 0; i < sizeof(SYSTABLE_SPECIAL_COL) / sizeof(SYSTABLE_SPECIAL_COL[0]); i++) { - if (0 == strcmp(pCol->colName, SYSTABLE_SPECIAL_COL[i])) { - return 1; - } - } - return 0; -} - -static int32_t optSysTabFilte(void* arg, SNode* cond, SArray* result) { - int ret = -1; - if (nodeType(cond) == QUERY_NODE_OPERATOR) { - ret = optSysTabFilteImpl(arg, cond, result); - if (ret == 0) { - SOperatorNode* pOper = (SOperatorNode*)cond; - SColumnNode* pCol = (SColumnNode*)pOper->pLeft; - if (0 == strcmp(pCol->colName, "create_time")) { - return 0; - } - return -1; - } - return ret; - } - - if (nodeType(cond) != QUERY_NODE_LOGIC_CONDITION || ((SLogicConditionNode*)cond)->condType != LOGIC_COND_TYPE_AND) { - return ret; - } - - SLogicConditionNode* pNode = (SLogicConditionNode*)cond; - SNodeList* pList = (SNodeList*)pNode->pParameterList; - - int32_t len = LIST_LENGTH(pList); - - bool hasIdx = false; - bool hasRslt = true; - SArray* mRslt = taosArrayInit(len, POINTER_BYTES); - - SListCell* cell = pList->pHead; - for (int i = 0; i < len; i++) { - if (cell == NULL) break; - - SArray* aRslt = taosArrayInit(16, sizeof(int64_t)); - - ret = optSysTabFilteImpl(arg, cell->pNode, aRslt); - if (ret == 0) { - // has index - hasIdx = true; - if (optSysSpecialColumn(cell->pNode) == 0) { - taosArrayPush(mRslt, &aRslt); - } else { - // db_name/vgroup not result - taosArrayDestroy(aRslt); - } - } else if (ret == -2) { - // current vg - hasIdx = true; - hasRslt = false; - taosArrayDestroy(aRslt); - break; - } else { - taosArrayDestroy(aRslt); - } - cell = cell->pNext; - } - if (hasRslt && hasIdx) { - optSysMergeRslt(mRslt, result); - } - - for (int i = 0; i < taosArrayGetSize(mRslt); i++) { - SArray* aRslt = taosArrayGetP(mRslt, i); - taosArrayDestroy(aRslt); - } - taosArrayDestroy(mRslt); - if (hasRslt == false) { - return -2; - } - if (hasRslt && hasIdx) { - cell = pList->pHead; - for (int i = 0; i < len; i++) { - if (cell == NULL) break; - SOperatorNode* pOper = (SOperatorNode*)cell->pNode; - SColumnNode* pCol = (SColumnNode*)pOper->pLeft; - if (0 == strcmp(pCol->colName, "create_time")) { - return 0; - } - cell = cell->pNext; - } - return -1; - } - return -1; -} - -static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SSysTableScanInfo* pInfo = pOperator->info; - - SSysTableIndex* pIdx = pInfo->pIdx; - blockDataCleanup(pInfo->pRes); - int32_t numOfRows = 0; - - int ret = 0; - - const char* db = NULL; - int32_t vgId = 0; - vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); - - SName sn = {0}; - char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; - tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); - - tNameGetDbName(&sn, varDataVal(dbname)); - varDataSetLen(dbname, strlen(varDataVal(dbname))); - - SSDataBlock* p = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_TABLES); - blockDataEnsureCapacity(p, pOperator->resultInfo.capacity); - - char n[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - int32_t i = pIdx->lastIdx; - for (; i < taosArrayGetSize(pIdx->uids); i++) { - tb_uid_t* uid = taosArrayGet(pIdx->uids, i); - - SMetaReader mr = {0}; - metaReaderInit(&mr, pInfo->readHandle.meta, 0); - ret = metaGetTableEntryByUid(&mr, *uid); - if (ret < 0) { - metaReaderClear(&mr); - continue; - } - STR_TO_VARSTR(n, mr.me.name); - - // table name - SColumnInfoData* pColInfoData = taosArrayGet(p->pDataBlock, 0); - colDataAppend(pColInfoData, numOfRows, n, false); - - // database name - pColInfoData = taosArrayGet(p->pDataBlock, 1); - colDataAppend(pColInfoData, numOfRows, dbname, false); - - // vgId - pColInfoData = taosArrayGet(p->pDataBlock, 6); - colDataAppend(pColInfoData, numOfRows, (char*)&vgId, false); - - int32_t tableType = mr.me.type; - if (tableType == TSDB_CHILD_TABLE) { - // create time - int64_t ts = mr.me.ctbEntry.ctime; - pColInfoData = taosArrayGet(p->pDataBlock, 2); - colDataAppend(pColInfoData, numOfRows, (char*)&ts, false); - - SMetaReader mr1 = {0}; - metaReaderInit(&mr1, pInfo->readHandle.meta, META_READER_NOLOCK); - - int64_t suid = mr.me.ctbEntry.suid; - int32_t code = metaGetTableEntryByUid(&mr1, suid); - if (code != TSDB_CODE_SUCCESS) { - qError("failed to get super table meta, cname:%s, suid:0x%" PRIx64 ", code:%s, %s", pInfo->pCur->mr.me.name, - suid, tstrerror(terrno), GET_TASKID(pTaskInfo)); - metaReaderClear(&mr1); - metaReaderClear(&mr); - T_LONG_JMP(pTaskInfo->env, terrno); - } - pColInfoData = taosArrayGet(p->pDataBlock, 3); - colDataAppend(pColInfoData, numOfRows, (char*)&mr1.me.stbEntry.schemaRow.nCols, false); - - // super table name - STR_TO_VARSTR(n, mr1.me.name); - pColInfoData = taosArrayGet(p->pDataBlock, 4); - colDataAppend(pColInfoData, numOfRows, n, false); - metaReaderClear(&mr1); - - // table comment - pColInfoData = taosArrayGet(p->pDataBlock, 8); - if (mr.me.ctbEntry.commentLen > 0) { - char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(comment, mr.me.ctbEntry.comment); - colDataAppend(pColInfoData, numOfRows, comment, false); - } else if (mr.me.ctbEntry.commentLen == 0) { - char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(comment, ""); - colDataAppend(pColInfoData, numOfRows, comment, false); - } else { - colDataAppendNULL(pColInfoData, numOfRows); - } - - // uid - pColInfoData = taosArrayGet(p->pDataBlock, 5); - colDataAppend(pColInfoData, numOfRows, (char*)&mr.me.uid, false); - - // ttl - pColInfoData = taosArrayGet(p->pDataBlock, 7); - colDataAppend(pColInfoData, numOfRows, (char*)&mr.me.ctbEntry.ttlDays, false); - - STR_TO_VARSTR(n, "CHILD_TABLE"); - - } else if (tableType == TSDB_NORMAL_TABLE) { - // create time - pColInfoData = taosArrayGet(p->pDataBlock, 2); - colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.ctime, false); - - // number of columns - pColInfoData = taosArrayGet(p->pDataBlock, 3); - colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.schemaRow.nCols, false); - - // super table name - pColInfoData = taosArrayGet(p->pDataBlock, 4); - colDataAppendNULL(pColInfoData, numOfRows); - - // table comment - pColInfoData = taosArrayGet(p->pDataBlock, 8); - if (mr.me.ntbEntry.commentLen > 0) { - char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(comment, mr.me.ntbEntry.comment); - colDataAppend(pColInfoData, numOfRows, comment, false); - } else if (mr.me.ntbEntry.commentLen == 0) { - char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(comment, ""); - colDataAppend(pColInfoData, numOfRows, comment, false); - } else { - colDataAppendNULL(pColInfoData, numOfRows); - } - - // uid - pColInfoData = taosArrayGet(p->pDataBlock, 5); - colDataAppend(pColInfoData, numOfRows, (char*)&mr.me.uid, false); - - // ttl - pColInfoData = taosArrayGet(p->pDataBlock, 7); - colDataAppend(pColInfoData, numOfRows, (char*)&mr.me.ntbEntry.ttlDays, false); - - STR_TO_VARSTR(n, "NORMAL_TABLE"); - // impl later - } - - metaReaderClear(&mr); - - pColInfoData = taosArrayGet(p->pDataBlock, 9); - colDataAppend(pColInfoData, numOfRows, n, false); - - if (++numOfRows >= pOperator->resultInfo.capacity) { - p->info.rows = numOfRows; - pInfo->pRes->info.rows = numOfRows; - - relocateColumnData(pInfo->pRes, pInfo->matchInfo.pList, p->pDataBlock, false); - doFilterResult(pInfo->pRes, pOperator->exprSupp.pFilterInfo); - - blockDataCleanup(p); - numOfRows = 0; - - if (pInfo->pRes->info.rows > 0) { - break; - } - } - } - - if (numOfRows > 0) { - p->info.rows = numOfRows; - pInfo->pRes->info.rows = numOfRows; - - relocateColumnData(pInfo->pRes, pInfo->matchInfo.pList, p->pDataBlock, false); - doFilterResult(pInfo->pRes, pOperator->exprSupp.pFilterInfo); - - blockDataCleanup(p); - numOfRows = 0; - } - - if (i >= taosArrayGetSize(pIdx->uids)) { - setOperatorCompleted(pOperator); - } else { - pIdx->lastIdx = i + 1; - } - - blockDataDestroy(p); - - pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; - return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; -} - -static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - SSysTableScanInfo* pInfo = pOperator->info; - if (pInfo->pCur == NULL) { - pInfo->pCur = metaOpenTbCursor(pInfo->readHandle.meta); - } - - blockDataCleanup(pInfo->pRes); - int32_t numOfRows = 0; - - const char* db = NULL; - int32_t vgId = 0; - vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); - - SName sn = {0}; - char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; - tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); - - tNameGetDbName(&sn, varDataVal(dbname)); - varDataSetLen(dbname, strlen(varDataVal(dbname))); - - SSDataBlock* p = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_TABLES); - blockDataEnsureCapacity(p, pOperator->resultInfo.capacity); - - char n[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - - int32_t ret = 0; - while ((ret = metaTbCursorNext(pInfo->pCur)) == 0) { - STR_TO_VARSTR(n, pInfo->pCur->mr.me.name); - - // table name - SColumnInfoData* pColInfoData = taosArrayGet(p->pDataBlock, 0); - colDataAppend(pColInfoData, numOfRows, n, false); - - // database name - pColInfoData = taosArrayGet(p->pDataBlock, 1); - colDataAppend(pColInfoData, numOfRows, dbname, false); - - // vgId - pColInfoData = taosArrayGet(p->pDataBlock, 6); - colDataAppend(pColInfoData, numOfRows, (char*)&vgId, false); - - int32_t tableType = pInfo->pCur->mr.me.type; - if (tableType == TSDB_CHILD_TABLE) { - // create time - int64_t ts = pInfo->pCur->mr.me.ctbEntry.ctime; - pColInfoData = taosArrayGet(p->pDataBlock, 2); - colDataAppend(pColInfoData, numOfRows, (char*)&ts, false); - - SMetaReader mr = {0}; - metaReaderInit(&mr, pInfo->readHandle.meta, META_READER_NOLOCK); - - uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; - int32_t code = metaGetTableEntryByUid(&mr, suid); - if (code != TSDB_CODE_SUCCESS) { - qError("failed to get super table meta, cname:%s, suid:0x%" PRIx64 ", code:%s, %s", pInfo->pCur->mr.me.name, - suid, tstrerror(terrno), GET_TASKID(pTaskInfo)); - metaReaderClear(&mr); - metaCloseTbCursor(pInfo->pCur); - pInfo->pCur = NULL; - T_LONG_JMP(pTaskInfo->env, terrno); - } - - // number of columns - pColInfoData = taosArrayGet(p->pDataBlock, 3); - colDataAppend(pColInfoData, numOfRows, (char*)&mr.me.stbEntry.schemaRow.nCols, false); - - // super table name - STR_TO_VARSTR(n, mr.me.name); - pColInfoData = taosArrayGet(p->pDataBlock, 4); - colDataAppend(pColInfoData, numOfRows, n, false); - metaReaderClear(&mr); - - // table comment - pColInfoData = taosArrayGet(p->pDataBlock, 8); - if (pInfo->pCur->mr.me.ctbEntry.commentLen > 0) { - char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(comment, pInfo->pCur->mr.me.ctbEntry.comment); - colDataAppend(pColInfoData, numOfRows, comment, false); - } else if (pInfo->pCur->mr.me.ctbEntry.commentLen == 0) { - char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(comment, ""); - colDataAppend(pColInfoData, numOfRows, comment, false); - } else { - colDataAppendNULL(pColInfoData, numOfRows); - } - - // uid - pColInfoData = taosArrayGet(p->pDataBlock, 5); - colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.uid, false); - - // ttl - pColInfoData = taosArrayGet(p->pDataBlock, 7); - colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ctbEntry.ttlDays, false); - - STR_TO_VARSTR(n, "CHILD_TABLE"); - } else if (tableType == TSDB_NORMAL_TABLE) { - // create time - pColInfoData = taosArrayGet(p->pDataBlock, 2); - colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.ctime, false); - - // number of columns - pColInfoData = taosArrayGet(p->pDataBlock, 3); - colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.schemaRow.nCols, false); - - // super table name - pColInfoData = taosArrayGet(p->pDataBlock, 4); - colDataAppendNULL(pColInfoData, numOfRows); - - // table comment - pColInfoData = taosArrayGet(p->pDataBlock, 8); - if (pInfo->pCur->mr.me.ntbEntry.commentLen > 0) { - char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(comment, pInfo->pCur->mr.me.ntbEntry.comment); - colDataAppend(pColInfoData, numOfRows, comment, false); - } else if (pInfo->pCur->mr.me.ntbEntry.commentLen == 0) { - char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0}; - STR_TO_VARSTR(comment, ""); - colDataAppend(pColInfoData, numOfRows, comment, false); - } else { - colDataAppendNULL(pColInfoData, numOfRows); - } - - // uid - pColInfoData = taosArrayGet(p->pDataBlock, 5); - colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.uid, false); - - // ttl - pColInfoData = taosArrayGet(p->pDataBlock, 7); - colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.ttlDays, false); - - STR_TO_VARSTR(n, "NORMAL_TABLE"); - } - - pColInfoData = taosArrayGet(p->pDataBlock, 9); - colDataAppend(pColInfoData, numOfRows, n, false); - - if (++numOfRows >= pOperator->resultInfo.capacity) { - p->info.rows = numOfRows; - pInfo->pRes->info.rows = numOfRows; - - relocateColumnData(pInfo->pRes, pInfo->matchInfo.pList, p->pDataBlock, false); - doFilterResult(pInfo->pRes, pOperator->exprSupp.pFilterInfo); - - blockDataCleanup(p); - numOfRows = 0; - - if (pInfo->pRes->info.rows > 0) { - break; - } - } - } - - if (numOfRows > 0) { - p->info.rows = numOfRows; - pInfo->pRes->info.rows = numOfRows; - - relocateColumnData(pInfo->pRes, pInfo->matchInfo.pList, p->pDataBlock, false); - doFilterResult(pInfo->pRes, pOperator->exprSupp.pFilterInfo); - - blockDataCleanup(p); - numOfRows = 0; - } - - blockDataDestroy(p); - - // todo temporarily free the cursor here, the true reason why the free is not valid needs to be found - if (ret != 0) { - metaCloseTbCursor(pInfo->pCur); - pInfo->pCur = NULL; - setOperatorCompleted(pOperator); - } - - pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; - return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; -} - -static SSDataBlock* sysTableScanUserTables(SOperatorInfo* pOperator) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SSysTableScanInfo* pInfo = pOperator->info; - - SNode* pCondition = pInfo->pCondition; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - // the retrieve is executed on the mnode, so return tables that belongs to the information schema database. - if (pInfo->readHandle.mnd != NULL) { - buildSysDbTableInfo(pInfo, pOperator->resultInfo.capacity); - doFilterResult(pInfo->pRes, pOperator->exprSupp.pFilterInfo); - pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; - - setOperatorCompleted(pOperator); - return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; - } else { - if (pInfo->showRewrite == false) { - if (pCondition != NULL && pInfo->pIdx == NULL) { - SSTabFltArg arg = {.pMeta = pInfo->readHandle.meta, .pVnode = pInfo->readHandle.vnode}; - - SSysTableIndex* idx = taosMemoryMalloc(sizeof(SSysTableIndex)); - idx->init = 0; - idx->uids = taosArrayInit(128, sizeof(int64_t)); - idx->lastIdx = 0; - - pInfo->pIdx = idx; // set idx arg - - int flt = optSysTabFilte(&arg, pCondition, idx->uids); - if (flt == 0) { - pInfo->pIdx->init = 1; - SSDataBlock* blk = sysTableBuildUserTablesByUids(pOperator); - return blk; - } else if (flt == -2) { - qDebug("%s failed to get sys table info by idx, empty result", GET_TASKID(pTaskInfo)); - return NULL; - } else if (flt == -1) { - // not idx - qDebug("%s failed to get sys table info by idx, scan sys table one by one", GET_TASKID(pTaskInfo)); - } - } else if (pCondition != NULL && (pInfo->pIdx != NULL && pInfo->pIdx->init == 1)) { - SSDataBlock* blk = sysTableBuildUserTablesByUids(pOperator); - return blk; - } - } - - return sysTableBuildUserTables(pOperator); - } - return NULL; -} - -static SSDataBlock* sysTableScanUserSTables(SOperatorInfo* pOperator) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SSysTableScanInfo* pInfo = pOperator->info; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - pInfo->pRes->info.rows = 0; - pOperator->status = OP_EXEC_DONE; - - pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; - return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; -} - -static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { - // build message and send to mnode to fetch the content of system tables. - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SSysTableScanInfo* pInfo = pOperator->info; - char dbName[TSDB_DB_NAME_LEN] = {0}; - - const char* name = tNameGetTableName(&pInfo->name); - if (pInfo->showRewrite) { - getDBNameFromCondition(pInfo->pCondition, dbName); - sprintf(pInfo->req.db, "%d.%s", pInfo->accountId, dbName); - } - - if (strncasecmp(name, TSDB_INS_TABLE_TABLES, TSDB_TABLE_FNAME_LEN) == 0) { - return sysTableScanUserTables(pOperator); - } else if (strncasecmp(name, TSDB_INS_TABLE_TAGS, TSDB_TABLE_FNAME_LEN) == 0) { - return sysTableScanUserTags(pOperator); - } else if (strncasecmp(name, TSDB_INS_TABLE_STABLES, TSDB_TABLE_FNAME_LEN) == 0 && pInfo->showRewrite && - IS_SYS_DBNAME(dbName)) { - return sysTableScanUserSTables(pOperator); - } else { // load the meta from mnode of the given epset - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - while (1) { - int64_t startTs = taosGetTimestampUs(); - tstrncpy(pInfo->req.tb, tNameGetTableName(&pInfo->name), tListLen(pInfo->req.tb)); - tstrncpy(pInfo->req.user, pInfo->pUser, tListLen(pInfo->req.user)); - - int32_t contLen = tSerializeSRetrieveTableReq(NULL, 0, &pInfo->req); - char* buf1 = taosMemoryCalloc(1, contLen); - tSerializeSRetrieveTableReq(buf1, contLen, &pInfo->req); - - // send the fetch remote task result reques - SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); - if (NULL == pMsgSendInfo) { - qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); - pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; - return NULL; - } - - int32_t msgType = (strcasecmp(name, TSDB_INS_TABLE_DNODE_VARIABLES) == 0) ? TDMT_DND_SYSTABLE_RETRIEVE - : TDMT_MND_SYSTABLE_RETRIEVE; - - pMsgSendInfo->param = pOperator; - pMsgSendInfo->msgInfo.pData = buf1; - pMsgSendInfo->msgInfo.len = contLen; - pMsgSendInfo->msgType = msgType; - pMsgSendInfo->fp = loadSysTableCallback; - pMsgSendInfo->requestId = pTaskInfo->id.queryId; - - int64_t transporterId = 0; - int32_t code = - asyncSendMsgToServer(pInfo->readHandle.pMsgCb->clientRpc, &pInfo->epSet, &transporterId, pMsgSendInfo); - tsem_wait(&pInfo->ready); - - if (pTaskInfo->code) { - qDebug("%s load meta data from mnode failed, totalRows:%" PRIu64 ", code:%s", GET_TASKID(pTaskInfo), - pInfo->loadInfo.totalRows, tstrerror(pTaskInfo->code)); - return NULL; - } - - SRetrieveMetaTableRsp* pRsp = pInfo->pRsp; - pInfo->req.showId = pRsp->handle; - - if (pRsp->numOfRows == 0 || pRsp->completed) { - pOperator->status = OP_EXEC_DONE; - qDebug("%s load meta data from mnode completed, rowsOfSource:%d, totalRows:%" PRIu64, GET_TASKID(pTaskInfo), - pRsp->numOfRows, pInfo->loadInfo.totalRows); - - if (pRsp->numOfRows == 0) { - taosMemoryFree(pRsp); - return NULL; - } - } - - char* pStart = pRsp->data; - extractDataBlockFromFetchRsp(pInfo->pRes, pRsp->data, pInfo->matchInfo.pList, &pStart); - updateLoadRemoteInfo(&pInfo->loadInfo, pRsp->numOfRows, pRsp->compLen, startTs, pOperator); - - // todo log the filter info - doFilterResult(pInfo->pRes, pOperator->exprSupp.pFilterInfo); - taosMemoryFree(pRsp); - if (pInfo->pRes->info.rows > 0) { - return pInfo->pRes; - } else if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - } - } -} - -int32_t buildSysDbTableInfo(const SSysTableScanInfo* pInfo, int32_t capacity) { - SSDataBlock* p = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_TABLES); - blockDataEnsureCapacity(p, capacity); - - size_t size = 0; - const SSysTableMeta* pSysDbTableMeta = NULL; - - getInfosDbMeta(&pSysDbTableMeta, &size); - p->info.rows = buildDbTableInfoBlock(pInfo->sysInfo, p, pSysDbTableMeta, size, TSDB_INFORMATION_SCHEMA_DB); - - getPerfDbMeta(&pSysDbTableMeta, &size); - p->info.rows = buildDbTableInfoBlock(pInfo->sysInfo, p, pSysDbTableMeta, size, TSDB_PERFORMANCE_SCHEMA_DB); - - pInfo->pRes->info.rows = p->info.rows; - relocateColumnData(pInfo->pRes, pInfo->matchInfo.pList, p->pDataBlock, false); - blockDataDestroy(p); - - return pInfo->pRes->info.rows; -} - -int32_t buildDbTableInfoBlock(bool sysInfo, const SSDataBlock* p, const SSysTableMeta* pSysDbTableMeta, size_t size, - const char* dbName) { - char n[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; - int32_t numOfRows = p->info.rows; - - for (int32_t i = 0; i < size; ++i) { - const SSysTableMeta* pm = &pSysDbTableMeta[i]; - if (!sysInfo && pm->sysInfo) { - continue; - } - - SColumnInfoData* pColInfoData = taosArrayGet(p->pDataBlock, 0); - - STR_TO_VARSTR(n, pm->name); - colDataAppend(pColInfoData, numOfRows, n, false); - - // database name - STR_TO_VARSTR(n, dbName); - pColInfoData = taosArrayGet(p->pDataBlock, 1); - colDataAppend(pColInfoData, numOfRows, n, false); - - // create time - pColInfoData = taosArrayGet(p->pDataBlock, 2); - colDataAppendNULL(pColInfoData, numOfRows); - - // number of columns - pColInfoData = taosArrayGet(p->pDataBlock, 3); - colDataAppend(pColInfoData, numOfRows, (char*)&pm->colNum, false); - - for (int32_t j = 4; j <= 8; ++j) { - pColInfoData = taosArrayGet(p->pDataBlock, j); - colDataAppendNULL(pColInfoData, numOfRows); - } - - STR_TO_VARSTR(n, "SYSTEM_TABLE"); - - pColInfoData = taosArrayGet(p->pDataBlock, 9); - colDataAppend(pColInfoData, numOfRows, n, false); - - numOfRows += 1; - } - - return numOfRows; -} - -SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode* pScanPhyNode, - const char* pUser, SExecTaskInfo* pTaskInfo) { - SSysTableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SSysTableScanInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - if (pInfo == NULL || pOperator == NULL) { - goto _error; - } - - SScanPhysiNode* pScanNode = &pScanPhyNode->scan; - SDataBlockDescNode* pDescNode = pScanNode->node.pOutputDataBlockDesc; - - int32_t num = 0; - int32_t code = extractColMatchInfo(pScanNode->pScanCols, pDescNode, &num, COL_MATCH_FROM_COL_ID, &pInfo->matchInfo); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - pInfo->accountId = pScanPhyNode->accountId; - pInfo->pUser = taosMemoryStrDup((void*)pUser); - pInfo->sysInfo = pScanPhyNode->sysInfo; - pInfo->showRewrite = pScanPhyNode->showRewrite; - pInfo->pRes = createResDataBlock(pDescNode); - - pInfo->pCondition = pScanNode->node.pConditions; - code = filterInitFromNode(pScanNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); - if (code != TSDB_CODE_SUCCESS) { - goto _error; - } - - initResultSizeInfo(&pOperator->resultInfo, 4096); - blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); - - tNameAssign(&pInfo->name, &pScanNode->tableName); - const char* name = tNameGetTableName(&pInfo->name); - - if (strncasecmp(name, TSDB_INS_TABLE_TABLES, TSDB_TABLE_FNAME_LEN) == 0 || - strncasecmp(name, TSDB_INS_TABLE_TAGS, TSDB_TABLE_FNAME_LEN) == 0) { - pInfo->readHandle = *(SReadHandle*)readHandle; - } else { - tsem_init(&pInfo->ready, 0, 0); - pInfo->epSet = pScanPhyNode->mgmtEpSet; - pInfo->readHandle = *(SReadHandle*)readHandle; - } - - setOperatorInfo(pOperator, "SysTableScanOperator", QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN, false, OP_NOT_OPENED, - pInfo, pTaskInfo); - pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock); - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSysTableScan, NULL, destroySysScanOperator, NULL); - return pOperator; - -_error: - if (pInfo != NULL) { - destroySysScanOperator(pInfo); - } - taosMemoryFreeClear(pOperator); - pTaskInfo->code = code; - return NULL; -} - static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { if (pOperator->status == OP_EXEC_DONE) { return NULL; @@ -4271,7 +2406,7 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { metaReaderInit(&mr, pInfo->readHandle.meta, 0); while (pInfo->curPos < size && count < pOperator->resultInfo.capacity) { - STableKeyInfo* item = tableListGetInfo(pInfo->pTableList, pInfo->curPos); + STableKeyInfo* item = tableListGetInfo(pTaskInfo->pTableInfoList, pInfo->curPos); int32_t code = metaGetTableEntryByUid(&mr, item->uid); tDecoderClear(&mr.coder); if (code != TSDB_CODE_SUCCESS) { @@ -4358,7 +2493,6 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi goto _error; } - pInfo->pTableList = pTableListInfo; pInfo->pRes = createResDataBlock(pDescNode); pInfo->readHandle = *pReadHandle; pInfo->curPos = 0; diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 0e7644151a..6201dfc9cb 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -17,6 +17,18 @@ #include "executorimpl.h" #include "tdatablock.h" +typedef struct SSortOperatorInfo { + SOptrBasicInfo binfo; + uint32_t sortBufSize; // max buffer size for in-memory sort + SArray* pSortInfo; + SSortHandle* pSortHandle; + SColMatchInfo matchInfo; + int32_t bufPageSize; + int64_t startTs; // sort start time + uint64_t sortElapsed; // sort elapsed time, time to flush to disk not included. + SLimitInfo limitInfo; +} SSortOperatorInfo; + static SSDataBlock* doSort(SOperatorInfo* pOperator); static int32_t doOpenSortOperator(SOperatorInfo* pOperator); static int32_t getExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len); diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c new file mode 100644 index 0000000000..c5e1f2c214 --- /dev/null +++ b/source/libs/executor/src/sysscanoperator.c @@ -0,0 +1,1944 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "executorimpl.h" +#include "filter.h" +#include "function.h" +#include "functionMgt.h" +#include "os.h" +#include "querynodes.h" +#include "systable.h" +#include "tname.h" +#include "ttime.h" + +#include "tdatablock.h" +#include "tmsg.h" + +#include "query.h" +#include "tcompare.h" +#include "thash.h" +#include "ttypes.h" +#include "vnode.h" + +typedef int (*__optSysFilter)(void* a, void* b, int16_t dtype); +typedef int32_t (*__sys_filte)(void* pMeta, SNode* cond, SArray* result); +typedef int32_t (*__sys_check)(SNode* cond); + +typedef struct SSTabFltArg { + void* pMeta; + void* pVnode; +} SSTabFltArg; + +typedef struct SSysTableIndex { + int8_t init; + SArray* uids; + int32_t lastIdx; +} SSysTableIndex; + +typedef struct SSysTableScanInfo { + SRetrieveMetaTableRsp* pRsp; + SRetrieveTableReq req; + SEpSet epSet; + tsem_t ready; + SReadHandle readHandle; + int32_t accountId; + const char* pUser; + bool sysInfo; + bool showRewrite; + SNode* pCondition; // db_name filter condition, to discard data that are not in current database + SMTbCursor* pCur; // cursor for iterate the local table meta store. + SSysTableIndex* pIdx; // idx for local table meta + SColMatchInfo matchInfo; + SName name; + SSDataBlock* pRes; + int64_t numOfBlocks; // extract basic running information. + SLoadRemoteDataInfo loadInfo; +} SSysTableScanInfo; + +typedef struct { + const char* name; + __sys_check chkFunc; + __sys_filte fltFunc; +} SSTabFltFuncDef; + +typedef struct MergeIndex { + int idx; + int len; +} MergeIndex; + +typedef struct SBlockDistInfo { + SSDataBlock* pResBlock; + STsdbReader* pHandle; + SReadHandle readHandle; + uint64_t uid; // table uid +} SBlockDistInfo; + +static int32_t sysChkFilter__Comm(SNode* pNode); +static int32_t sysChkFilter__DBName(SNode* pNode); +static int32_t sysChkFilter__VgroupId(SNode* pNode); +static int32_t sysChkFilter__TableName(SNode* pNode); +static int32_t sysChkFilter__CreateTime(SNode* pNode); +static int32_t sysChkFilter__Ncolumn(SNode* pNode); +static int32_t sysChkFilter__Ttl(SNode* pNode); +static int32_t sysChkFilter__STableName(SNode* pNode); +static int32_t sysChkFilter__Uid(SNode* pNode); +static int32_t sysChkFilter__Type(SNode* pNode); + +static int32_t sysFilte__DbName(void* arg, SNode* pNode, SArray* result); +static int32_t sysFilte__VgroupId(void* arg, SNode* pNode, SArray* result); +static int32_t sysFilte__TableName(void* arg, SNode* pNode, SArray* result); +static int32_t sysFilte__CreateTime(void* arg, SNode* pNode, SArray* result); +static int32_t sysFilte__Ncolumn(void* arg, SNode* pNode, SArray* result); +static int32_t sysFilte__Ttl(void* arg, SNode* pNode, SArray* result); +static int32_t sysFilte__STableName(void* arg, SNode* pNode, SArray* result); +static int32_t sysFilte__Uid(void* arg, SNode* pNode, SArray* result); +static int32_t sysFilte__Type(void* arg, SNode* pNode, SArray* result); + +const SSTabFltFuncDef filterDict[] = { + {.name = "table_name", .chkFunc = sysChkFilter__TableName, .fltFunc = sysFilte__TableName}, + {.name = "db_name", .chkFunc = sysChkFilter__DBName, .fltFunc = sysFilte__DbName}, + {.name = "create_time", .chkFunc = sysChkFilter__CreateTime, .fltFunc = sysFilte__CreateTime}, + {.name = "columns", .chkFunc = sysChkFilter__Ncolumn, .fltFunc = sysFilte__Ncolumn}, + {.name = "ttl", .chkFunc = sysChkFilter__Ttl, .fltFunc = sysFilte__Ttl}, + {.name = "stable_name", .chkFunc = sysChkFilter__STableName, .fltFunc = sysFilte__STableName}, + {.name = "vgroup_id", .chkFunc = sysChkFilter__VgroupId, .fltFunc = sysFilte__VgroupId}, + {.name = "uid", .chkFunc = sysChkFilter__Uid, .fltFunc = sysFilte__Uid}, + {.name = "type", .chkFunc = sysChkFilter__Type, .fltFunc = sysFilte__Type}}; + +#define SYSTAB_FILTER_DICT_SIZE (sizeof(filterDict) / sizeof(filterDict[0])) + +static int32_t buildDbTableInfoBlock(bool sysInfo, const SSDataBlock* p, const SSysTableMeta* pSysDbTableMeta, + size_t size, const char* dbName); + +static char* SYSTABLE_IDX_COLUMN[] = {"table_name", "db_name", "create_time", "columns", + "ttl", "stable_name", "vgroup_id', 'uid", "type"}; + +static char* SYSTABLE_SPECIAL_COL[] = {"db_name", "vgroup_id"}; + +static int32_t buildSysDbTableInfo(const SSysTableScanInfo* pInfo, int32_t capacity); +static SSDataBlock* buildInfoSchemaTableMetaBlock(char* tableName); +static void destroySysScanOperator(void* param); +static int32_t loadSysTableCallback(void* param, SDataBuf* pMsg, int32_t code); +static SSDataBlock* doFilterResult(SSDataBlock* pDataBlock, SFilterInfo* pFilterInfo); +static __optSysFilter optSysGetFilterFunc(int32_t ctype, bool* reverse); + +static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, SMetaReader* smrSuperTable, + SMetaReader* smrChildTable, const char* dbname, const char* tableName, + int32_t* pNumOfRows, const SSDataBlock* dataBlock); + +static void relocateAndFilterSysTagsScanResult(SSysTableScanInfo* pInfo, int32_t numOfRows, SSDataBlock* dataBlock, + SFilterInfo* pFilterInfo); + +int32_t sysFilte__DbName(void* arg, SNode* pNode, SArray* result) { + void* pVnode = ((SSTabFltArg*)arg)->pVnode; + + const char* db = NULL; + vnodeGetInfo(pVnode, &db, NULL); + + SName sn = {0}; + char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); + + tNameGetDbName(&sn, varDataVal(dbname)); + varDataSetLen(dbname, strlen(varDataVal(dbname))); + + SOperatorNode* pOper = (SOperatorNode*)pNode; + SValueNode* pVal = (SValueNode*)pOper->pRight; + + bool reverse = false; + __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); + if (func == NULL) return -1; + + int ret = func(dbname, pVal->datum.p, TSDB_DATA_TYPE_VARCHAR); + if (ret == 0) return 0; + + return -2; +} + +int32_t sysFilte__VgroupId(void* arg, SNode* pNode, SArray* result) { + void* pVnode = ((SSTabFltArg*)arg)->pVnode; + + int64_t vgId = 0; + vnodeGetInfo(pVnode, NULL, (int32_t*)&vgId); + + SOperatorNode* pOper = (SOperatorNode*)pNode; + SValueNode* pVal = (SValueNode*)pOper->pRight; + + bool reverse = false; + + __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); + if (func == NULL) return -1; + + int ret = func(&vgId, &pVal->datum.i, TSDB_DATA_TYPE_BIGINT); + if (ret == 0) return 0; + + return -1; +} + +int32_t sysFilte__TableName(void* arg, SNode* pNode, SArray* result) { + void* pMeta = ((SSTabFltArg*)arg)->pMeta; + + SOperatorNode* pOper = (SOperatorNode*)pNode; + SValueNode* pVal = (SValueNode*)pOper->pRight; + bool reverse = false; + + __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); + if (func == NULL) return -1; + + SMetaFltParam param = {.suid = 0, + .cid = 0, + .type = TSDB_DATA_TYPE_VARCHAR, + .val = pVal->datum.p, + .reverse = reverse, + .filterFunc = func}; + return -1; +} + +int32_t sysFilte__CreateTime(void* arg, SNode* pNode, SArray* result) { + void* pMeta = ((SSTabFltArg*)arg)->pMeta; + + SOperatorNode* pOper = (SOperatorNode*)pNode; + SValueNode* pVal = (SValueNode*)pOper->pRight; + bool reverse = false; + + __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); + if (func == NULL) return -1; + + SMetaFltParam param = {.suid = 0, + .cid = 0, + .type = TSDB_DATA_TYPE_BIGINT, + .val = &pVal->datum.i, + .reverse = reverse, + .filterFunc = func}; + + int32_t ret = metaFilterCreateTime(pMeta, ¶m, result); + return ret; +} + +int32_t sysFilte__Ncolumn(void* arg, SNode* pNode, SArray* result) { + void* pMeta = ((SSTabFltArg*)arg)->pMeta; + + SOperatorNode* pOper = (SOperatorNode*)pNode; + SValueNode* pVal = (SValueNode*)pOper->pRight; + bool reverse = false; + + __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); + if (func == NULL) return -1; + return -1; +} + +int32_t sysFilte__Ttl(void* arg, SNode* pNode, SArray* result) { + void* pMeta = ((SSTabFltArg*)arg)->pMeta; + + SOperatorNode* pOper = (SOperatorNode*)pNode; + SValueNode* pVal = (SValueNode*)pOper->pRight; + bool reverse = false; + + __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); + if (func == NULL) return -1; + return -1; +} + +int32_t sysFilte__STableName(void* arg, SNode* pNode, SArray* result) { + void* pMeta = ((SSTabFltArg*)arg)->pMeta; + + SOperatorNode* pOper = (SOperatorNode*)pNode; + SValueNode* pVal = (SValueNode*)pOper->pRight; + bool reverse = false; + + __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); + if (func == NULL) return -1; + return -1; +} + +int32_t sysFilte__Uid(void* arg, SNode* pNode, SArray* result) { + void* pMeta = ((SSTabFltArg*)arg)->pMeta; + + SOperatorNode* pOper = (SOperatorNode*)pNode; + SValueNode* pVal = (SValueNode*)pOper->pRight; + bool reverse = false; + + __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); + if (func == NULL) return -1; + return -1; +} + +int32_t sysFilte__Type(void* arg, SNode* pNode, SArray* result) { + void* pMeta = ((SSTabFltArg*)arg)->pMeta; + + SOperatorNode* pOper = (SOperatorNode*)pNode; + SValueNode* pVal = (SValueNode*)pOper->pRight; + bool reverse = false; + + __optSysFilter func = optSysGetFilterFunc(pOper->opType, &reverse); + if (func == NULL) return -1; + return -1; +} + +int optSysDoCompare(__compar_fn_t func, int8_t comparType, void* a, void* b) { + int32_t cmp = func(a, b); + switch (comparType) { + case OP_TYPE_LOWER_THAN: + if (cmp < 0) return 0; + break; + case OP_TYPE_LOWER_EQUAL: { + if (cmp <= 0) return 0; + break; + } + case OP_TYPE_GREATER_THAN: { + if (cmp > 0) return 0; + break; + } + case OP_TYPE_GREATER_EQUAL: { + if (cmp >= 0) return 0; + break; + } + case OP_TYPE_EQUAL: { + if (cmp == 0) return 0; + break; + } + default: + return -1; + } + return cmp; +} + +static int optSysFilterFuncImpl__LowerThan(void* a, void* b, int16_t dtype) { + __compar_fn_t func = getComparFunc(dtype, 0); + return optSysDoCompare(func, OP_TYPE_LOWER_THAN, a, b); +} +static int optSysFilterFuncImpl__LowerEqual(void* a, void* b, int16_t dtype) { + __compar_fn_t func = getComparFunc(dtype, 0); + return optSysDoCompare(func, OP_TYPE_LOWER_EQUAL, a, b); +} +static int optSysFilterFuncImpl__GreaterThan(void* a, void* b, int16_t dtype) { + __compar_fn_t func = getComparFunc(dtype, 0); + return optSysDoCompare(func, OP_TYPE_GREATER_THAN, a, b); +} +static int optSysFilterFuncImpl__GreaterEqual(void* a, void* b, int16_t dtype) { + __compar_fn_t func = getComparFunc(dtype, 0); + return optSysDoCompare(func, OP_TYPE_GREATER_EQUAL, a, b); +} +static int optSysFilterFuncImpl__Equal(void* a, void* b, int16_t dtype) { + __compar_fn_t func = getComparFunc(dtype, 0); + return optSysDoCompare(func, OP_TYPE_EQUAL, a, b); +} + +static int optSysFilterFuncImpl__NoEqual(void* a, void* b, int16_t dtype) { + __compar_fn_t func = getComparFunc(dtype, 0); + return optSysDoCompare(func, OP_TYPE_NOT_EQUAL, a, b); +} + +static int32_t optSysTabFilte(void* arg, SNode* cond, SArray* result); +static int32_t optSysTabFilteImpl(void* arg, SNode* cond, SArray* result); +static int32_t optSysCheckOper(SNode* pOpear); +static int32_t optSysMergeRslt(SArray* mRslt, SArray* rslt); + +__optSysFilter optSysGetFilterFunc(int32_t ctype, bool* reverse) { + if (ctype == OP_TYPE_LOWER_EQUAL || ctype == OP_TYPE_LOWER_THAN) { + *reverse = true; + } + if (ctype == OP_TYPE_LOWER_THAN) + return optSysFilterFuncImpl__LowerThan; + else if (ctype == OP_TYPE_LOWER_EQUAL) + return optSysFilterFuncImpl__LowerEqual; + else if (ctype == OP_TYPE_GREATER_THAN) + return optSysFilterFuncImpl__GreaterThan; + else if (ctype == OP_TYPE_GREATER_EQUAL) + return optSysFilterFuncImpl__GreaterEqual; + else if (ctype == OP_TYPE_EQUAL) + return optSysFilterFuncImpl__Equal; + else if (ctype == OP_TYPE_NOT_EQUAL) + return optSysFilterFuncImpl__NoEqual; + return NULL; +} + +static bool sysTableIsOperatorCondOnOneTable(SNode* pCond, char* condTable) { + SOperatorNode* node = (SOperatorNode*)pCond; + if (node->opType == OP_TYPE_EQUAL) { + if (nodeType(node->pLeft) == QUERY_NODE_COLUMN && + strcasecmp(nodesGetNameFromColumnNode(node->pLeft), "table_name") == 0 && + nodeType(node->pRight) == QUERY_NODE_VALUE) { + SValueNode* pValue = (SValueNode*)node->pRight; + if (pValue->node.resType.type == TSDB_DATA_TYPE_NCHAR || pValue->node.resType.type == TSDB_DATA_TYPE_VARCHAR || + pValue->node.resType.type == TSDB_DATA_TYPE_BINARY) { + char* value = nodesGetValueFromNode(pValue); + strncpy(condTable, varDataVal(value), TSDB_TABLE_NAME_LEN); + return true; + } + } + } + return false; +} + +static bool sysTableIsCondOnOneTable(SNode* pCond, char* condTable) { + if (pCond == NULL) { + return false; + } + if (nodeType(pCond) == QUERY_NODE_LOGIC_CONDITION) { + SLogicConditionNode* node = (SLogicConditionNode*)pCond; + if (LOGIC_COND_TYPE_AND == node->condType) { + SNode* pChild = NULL; + FOREACH(pChild, node->pParameterList) { + if (QUERY_NODE_OPERATOR == nodeType(pChild) && sysTableIsOperatorCondOnOneTable(pChild, condTable)) { + return true; + } + } + } + } + + if (QUERY_NODE_OPERATOR == nodeType(pCond)) { + return sysTableIsOperatorCondOnOneTable(pCond, condTable); + } + + return false; +} + +static SSDataBlock* sysTableScanUserTags(SOperatorInfo* pOperator) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SSysTableScanInfo* pInfo = pOperator->info; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + blockDataCleanup(pInfo->pRes); + int32_t numOfRows = 0; + + SSDataBlock* dataBlock = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_TAGS); + blockDataEnsureCapacity(dataBlock, pOperator->resultInfo.capacity); + + const char* db = NULL; + int32_t vgId = 0; + vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); + + SName sn = {0}; + char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); + + tNameGetDbName(&sn, varDataVal(dbname)); + varDataSetLen(dbname, strlen(varDataVal(dbname))); + + char condTableName[TSDB_TABLE_NAME_LEN] = {0}; + // optimize when sql like where table_name='tablename' and xxx. + if (pInfo->pCondition && sysTableIsCondOnOneTable(pInfo->pCondition, condTableName)) { + char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(tableName, condTableName); + + SMetaReader smrChildTable = {0}; + metaReaderInit(&smrChildTable, pInfo->readHandle.meta, 0); + int32_t code = metaGetTableEntryByName(&smrChildTable, condTableName); + if (code != TSDB_CODE_SUCCESS) { + // terrno has been set by metaGetTableEntryByName, therefore, return directly + return NULL; + } + + if (smrChildTable.me.type != TSDB_CHILD_TABLE) { + metaReaderClear(&smrChildTable); + blockDataDestroy(dataBlock); + pInfo->loadInfo.totalRows = 0; + return NULL; + } + + SMetaReader smrSuperTable = {0}; + metaReaderInit(&smrSuperTable, pInfo->readHandle.meta, META_READER_NOLOCK); + code = metaGetTableEntryByUid(&smrSuperTable, smrChildTable.me.ctbEntry.suid); + if (code != TSDB_CODE_SUCCESS) { + // terrno has been set by metaGetTableEntryByUid + return NULL; + } + + sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &smrChildTable, dbname, tableName, &numOfRows, dataBlock); + metaReaderClear(&smrSuperTable); + metaReaderClear(&smrChildTable); + if (numOfRows > 0) { + relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); + numOfRows = 0; + } + blockDataDestroy(dataBlock); + pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; + setOperatorCompleted(pOperator); + return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; + } + + int32_t ret = 0; + if (pInfo->pCur == NULL) { + pInfo->pCur = metaOpenTbCursor(pInfo->readHandle.meta); + } + + while ((ret = metaTbCursorNext(pInfo->pCur)) == 0) { + if (pInfo->pCur->mr.me.type != TSDB_CHILD_TABLE) { + continue; + } + + char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(tableName, pInfo->pCur->mr.me.name); + + SMetaReader smrSuperTable = {0}; + metaReaderInit(&smrSuperTable, pInfo->readHandle.meta, 0); + uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; + int32_t code = metaGetTableEntryByUid(&smrSuperTable, suid); + if (code != TSDB_CODE_SUCCESS) { + qError("failed to get super table meta, uid:0x%" PRIx64 ", code:%s, %s", suid, tstrerror(terrno), + GET_TASKID(pTaskInfo)); + metaReaderClear(&smrSuperTable); + metaCloseTbCursor(pInfo->pCur); + pInfo->pCur = NULL; + T_LONG_JMP(pTaskInfo->env, terrno); + } + + sysTableUserTagsFillOneTableTags(pInfo, &smrSuperTable, &pInfo->pCur->mr, dbname, tableName, &numOfRows, dataBlock); + + metaReaderClear(&smrSuperTable); + + if (numOfRows >= pOperator->resultInfo.capacity) { + relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); + numOfRows = 0; + + if (pInfo->pRes->info.rows > 0) { + break; + } + } + } + + if (numOfRows > 0) { + relocateAndFilterSysTagsScanResult(pInfo, numOfRows, dataBlock, pOperator->exprSupp.pFilterInfo); + numOfRows = 0; + } + + blockDataDestroy(dataBlock); + if (ret != 0) { + metaCloseTbCursor(pInfo->pCur); + pInfo->pCur = NULL; + setOperatorCompleted(pOperator); + } + + pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; + return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; +} + +void relocateAndFilterSysTagsScanResult(SSysTableScanInfo* pInfo, int32_t numOfRows, SSDataBlock* dataBlock, + SFilterInfo* pFilterInfo) { + dataBlock->info.rows = numOfRows; + pInfo->pRes->info.rows = numOfRows; + + relocateColumnData(pInfo->pRes, pInfo->matchInfo.pList, dataBlock->pDataBlock, false); + doFilterResult(pInfo->pRes, pFilterInfo); + blockDataCleanup(dataBlock); +} + +int32_t convertTagDataToStr(char* str, int type, void* buf, int32_t bufSize, int32_t* len) { + int32_t n = 0; + + switch (type) { + case TSDB_DATA_TYPE_NULL: + n = sprintf(str, "null"); + break; + + case TSDB_DATA_TYPE_BOOL: + n = sprintf(str, (*(int8_t*)buf) ? "true" : "false"); + break; + + case TSDB_DATA_TYPE_TINYINT: + n = sprintf(str, "%d", *(int8_t*)buf); + break; + + case TSDB_DATA_TYPE_SMALLINT: + n = sprintf(str, "%d", *(int16_t*)buf); + break; + + case TSDB_DATA_TYPE_INT: + n = sprintf(str, "%d", *(int32_t*)buf); + break; + + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: + n = sprintf(str, "%" PRId64, *(int64_t*)buf); + break; + + case TSDB_DATA_TYPE_FLOAT: + n = sprintf(str, "%.5f", GET_FLOAT_VAL(buf)); + break; + + case TSDB_DATA_TYPE_DOUBLE: + n = sprintf(str, "%.9f", GET_DOUBLE_VAL(buf)); + break; + + case TSDB_DATA_TYPE_BINARY: + if (bufSize < 0) { + return TSDB_CODE_TSC_INVALID_VALUE; + } + + memcpy(str, buf, bufSize); + n = bufSize; + break; + case TSDB_DATA_TYPE_NCHAR: + if (bufSize < 0) { + return TSDB_CODE_TSC_INVALID_VALUE; + } + + int32_t length = taosUcs4ToMbs((TdUcs4*)buf, bufSize, str); + if (length <= 0) { + return TSDB_CODE_TSC_INVALID_VALUE; + } + n = length; + break; + case TSDB_DATA_TYPE_UTINYINT: + n = sprintf(str, "%u", *(uint8_t*)buf); + break; + + case TSDB_DATA_TYPE_USMALLINT: + n = sprintf(str, "%u", *(uint16_t*)buf); + break; + + case TSDB_DATA_TYPE_UINT: + n = sprintf(str, "%u", *(uint32_t*)buf); + break; + + case TSDB_DATA_TYPE_UBIGINT: + n = sprintf(str, "%" PRIu64, *(uint64_t*)buf); + break; + + default: + return TSDB_CODE_TSC_INVALID_VALUE; + } + + if (len) *len = n; + + return TSDB_CODE_SUCCESS; +} + +static int32_t sysTableUserTagsFillOneTableTags(const SSysTableScanInfo* pInfo, SMetaReader* smrSuperTable, + SMetaReader* smrChildTable, const char* dbname, const char* tableName, + int32_t* pNumOfRows, const SSDataBlock* dataBlock) { + char stableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(stableName, (*smrSuperTable).me.name); + + int32_t numOfRows = *pNumOfRows; + + int32_t numOfTags = (*smrSuperTable).me.stbEntry.schemaTag.nCols; + for (int32_t i = 0; i < numOfTags; ++i) { + SColumnInfoData* pColInfoData = NULL; + + // table name + pColInfoData = taosArrayGet(dataBlock->pDataBlock, 0); + colDataAppend(pColInfoData, numOfRows, tableName, false); + + // database name + pColInfoData = taosArrayGet(dataBlock->pDataBlock, 1); + colDataAppend(pColInfoData, numOfRows, dbname, false); + + // super table name + pColInfoData = taosArrayGet(dataBlock->pDataBlock, 2); + colDataAppend(pColInfoData, numOfRows, stableName, false); + + // tag name + char tagName[TSDB_COL_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(tagName, (*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].name); + pColInfoData = taosArrayGet(dataBlock->pDataBlock, 3); + colDataAppend(pColInfoData, numOfRows, tagName, false); + + // tag type + int8_t tagType = (*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].type; + pColInfoData = taosArrayGet(dataBlock->pDataBlock, 4); + char tagTypeStr[VARSTR_HEADER_SIZE + 32]; + int tagTypeLen = sprintf(varDataVal(tagTypeStr), "%s", tDataTypes[tagType].name); + if (tagType == TSDB_DATA_TYPE_VARCHAR) { + tagTypeLen += sprintf(varDataVal(tagTypeStr) + tagTypeLen, "(%d)", + (int32_t)((*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].bytes - VARSTR_HEADER_SIZE)); + } else if (tagType == TSDB_DATA_TYPE_NCHAR) { + tagTypeLen += sprintf( + varDataVal(tagTypeStr) + tagTypeLen, "(%d)", + (int32_t)(((*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].bytes - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE)); + } + varDataSetLen(tagTypeStr, tagTypeLen); + colDataAppend(pColInfoData, numOfRows, (char*)tagTypeStr, false); + + STagVal tagVal = {0}; + tagVal.cid = (*smrSuperTable).me.stbEntry.schemaTag.pSchema[i].colId; + char* tagData = NULL; + uint32_t tagLen = 0; + + if (tagType == TSDB_DATA_TYPE_JSON) { + tagData = (char*)smrChildTable->me.ctbEntry.pTags; + } else { + bool exist = tTagGet((STag*)smrChildTable->me.ctbEntry.pTags, &tagVal); + if (exist) { + if (IS_VAR_DATA_TYPE(tagType)) { + tagData = (char*)tagVal.pData; + tagLen = tagVal.nData; + } else { + tagData = (char*)&tagVal.i64; + tagLen = tDataTypes[tagType].bytes; + } + } + } + + char* tagVarChar = NULL; + if (tagData != NULL) { + if (tagType == TSDB_DATA_TYPE_JSON) { + char* tagJson = parseTagDatatoJson(tagData); + tagVarChar = taosMemoryMalloc(strlen(tagJson) + VARSTR_HEADER_SIZE); + memcpy(varDataVal(tagVarChar), tagJson, strlen(tagJson)); + varDataSetLen(tagVarChar, strlen(tagJson)); + taosMemoryFree(tagJson); + } else { + int32_t bufSize = IS_VAR_DATA_TYPE(tagType) ? (tagLen + VARSTR_HEADER_SIZE) + : (3 + DBL_MANT_DIG - DBL_MIN_EXP + VARSTR_HEADER_SIZE); + tagVarChar = taosMemoryMalloc(bufSize); + int32_t len = -1; + convertTagDataToStr(varDataVal(tagVarChar), tagType, tagData, tagLen, &len); + varDataSetLen(tagVarChar, len); + } + } + pColInfoData = taosArrayGet(dataBlock->pDataBlock, 5); + colDataAppend(pColInfoData, numOfRows, tagVarChar, + (tagData == NULL) || (tagType == TSDB_DATA_TYPE_JSON && tTagIsJsonNull(tagData))); + taosMemoryFree(tagVarChar); + ++numOfRows; + } + + *pNumOfRows = numOfRows; + + return TSDB_CODE_SUCCESS; +} + +static SSDataBlock* buildInfoSchemaTableMetaBlock(char* tableName) { + size_t size = 0; + const SSysTableMeta* pMeta = NULL; + getInfosDbMeta(&pMeta, &size); + + int32_t index = 0; + for (int32_t i = 0; i < size; ++i) { + if (strcmp(pMeta[i].name, tableName) == 0) { + index = i; + break; + } + } + + SSDataBlock* pBlock = createDataBlock(); + for (int32_t i = 0; i < pMeta[index].colNum; ++i) { + SColumnInfoData colInfoData = + createColumnInfoData(pMeta[index].schema[i].type, pMeta[index].schema[i].bytes, i + 1); + blockDataAppendColInfo(pBlock, &colInfoData); + } + + return pBlock; +} + +int32_t buildDbTableInfoBlock(bool sysInfo, const SSDataBlock* p, const SSysTableMeta* pSysDbTableMeta, size_t size, + const char* dbName) { + char n[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + int32_t numOfRows = p->info.rows; + + for (int32_t i = 0; i < size; ++i) { + const SSysTableMeta* pm = &pSysDbTableMeta[i]; + if (!sysInfo && pm->sysInfo) { + continue; + } + + SColumnInfoData* pColInfoData = taosArrayGet(p->pDataBlock, 0); + + STR_TO_VARSTR(n, pm->name); + colDataAppend(pColInfoData, numOfRows, n, false); + + // database name + STR_TO_VARSTR(n, dbName); + pColInfoData = taosArrayGet(p->pDataBlock, 1); + colDataAppend(pColInfoData, numOfRows, n, false); + + // create time + pColInfoData = taosArrayGet(p->pDataBlock, 2); + colDataAppendNULL(pColInfoData, numOfRows); + + // number of columns + pColInfoData = taosArrayGet(p->pDataBlock, 3); + colDataAppend(pColInfoData, numOfRows, (char*)&pm->colNum, false); + + for (int32_t j = 4; j <= 8; ++j) { + pColInfoData = taosArrayGet(p->pDataBlock, j); + colDataAppendNULL(pColInfoData, numOfRows); + } + + STR_TO_VARSTR(n, "SYSTEM_TABLE"); + + pColInfoData = taosArrayGet(p->pDataBlock, 9); + colDataAppend(pColInfoData, numOfRows, n, false); + + numOfRows += 1; + } + + return numOfRows; +} + +int32_t buildSysDbTableInfo(const SSysTableScanInfo* pInfo, int32_t capacity) { + SSDataBlock* p = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_TABLES); + blockDataEnsureCapacity(p, capacity); + + size_t size = 0; + const SSysTableMeta* pSysDbTableMeta = NULL; + + getInfosDbMeta(&pSysDbTableMeta, &size); + p->info.rows = buildDbTableInfoBlock(pInfo->sysInfo, p, pSysDbTableMeta, size, TSDB_INFORMATION_SCHEMA_DB); + + getPerfDbMeta(&pSysDbTableMeta, &size); + p->info.rows = buildDbTableInfoBlock(pInfo->sysInfo, p, pSysDbTableMeta, size, TSDB_PERFORMANCE_SCHEMA_DB); + + pInfo->pRes->info.rows = p->info.rows; + relocateColumnData(pInfo->pRes, pInfo->matchInfo.pList, p->pDataBlock, false); + blockDataDestroy(p); + + return pInfo->pRes->info.rows; +} + +static SSDataBlock* sysTableBuildUserTablesByUids(SOperatorInfo* pOperator) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SSysTableScanInfo* pInfo = pOperator->info; + + SSysTableIndex* pIdx = pInfo->pIdx; + blockDataCleanup(pInfo->pRes); + int32_t numOfRows = 0; + + int ret = 0; + + const char* db = NULL; + int32_t vgId = 0; + vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); + + SName sn = {0}; + char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); + + tNameGetDbName(&sn, varDataVal(dbname)); + varDataSetLen(dbname, strlen(varDataVal(dbname))); + + SSDataBlock* p = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_TABLES); + blockDataEnsureCapacity(p, pOperator->resultInfo.capacity); + + char n[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + int32_t i = pIdx->lastIdx; + for (; i < taosArrayGetSize(pIdx->uids); i++) { + tb_uid_t* uid = taosArrayGet(pIdx->uids, i); + + SMetaReader mr = {0}; + metaReaderInit(&mr, pInfo->readHandle.meta, 0); + ret = metaGetTableEntryByUid(&mr, *uid); + if (ret < 0) { + metaReaderClear(&mr); + continue; + } + STR_TO_VARSTR(n, mr.me.name); + + // table name + SColumnInfoData* pColInfoData = taosArrayGet(p->pDataBlock, 0); + colDataAppend(pColInfoData, numOfRows, n, false); + + // database name + pColInfoData = taosArrayGet(p->pDataBlock, 1); + colDataAppend(pColInfoData, numOfRows, dbname, false); + + // vgId + pColInfoData = taosArrayGet(p->pDataBlock, 6); + colDataAppend(pColInfoData, numOfRows, (char*)&vgId, false); + + int32_t tableType = mr.me.type; + if (tableType == TSDB_CHILD_TABLE) { + // create time + int64_t ts = mr.me.ctbEntry.ctime; + pColInfoData = taosArrayGet(p->pDataBlock, 2); + colDataAppend(pColInfoData, numOfRows, (char*)&ts, false); + + SMetaReader mr1 = {0}; + metaReaderInit(&mr1, pInfo->readHandle.meta, META_READER_NOLOCK); + + int64_t suid = mr.me.ctbEntry.suid; + int32_t code = metaGetTableEntryByUid(&mr1, suid); + if (code != TSDB_CODE_SUCCESS) { + qError("failed to get super table meta, cname:%s, suid:0x%" PRIx64 ", code:%s, %s", pInfo->pCur->mr.me.name, + suid, tstrerror(terrno), GET_TASKID(pTaskInfo)); + metaReaderClear(&mr1); + metaReaderClear(&mr); + T_LONG_JMP(pTaskInfo->env, terrno); + } + pColInfoData = taosArrayGet(p->pDataBlock, 3); + colDataAppend(pColInfoData, numOfRows, (char*)&mr1.me.stbEntry.schemaRow.nCols, false); + + // super table name + STR_TO_VARSTR(n, mr1.me.name); + pColInfoData = taosArrayGet(p->pDataBlock, 4); + colDataAppend(pColInfoData, numOfRows, n, false); + metaReaderClear(&mr1); + + // table comment + pColInfoData = taosArrayGet(p->pDataBlock, 8); + if (mr.me.ctbEntry.commentLen > 0) { + char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(comment, mr.me.ctbEntry.comment); + colDataAppend(pColInfoData, numOfRows, comment, false); + } else if (mr.me.ctbEntry.commentLen == 0) { + char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(comment, ""); + colDataAppend(pColInfoData, numOfRows, comment, false); + } else { + colDataAppendNULL(pColInfoData, numOfRows); + } + + // uid + pColInfoData = taosArrayGet(p->pDataBlock, 5); + colDataAppend(pColInfoData, numOfRows, (char*)&mr.me.uid, false); + + // ttl + pColInfoData = taosArrayGet(p->pDataBlock, 7); + colDataAppend(pColInfoData, numOfRows, (char*)&mr.me.ctbEntry.ttlDays, false); + + STR_TO_VARSTR(n, "CHILD_TABLE"); + + } else if (tableType == TSDB_NORMAL_TABLE) { + // create time + pColInfoData = taosArrayGet(p->pDataBlock, 2); + colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.ctime, false); + + // number of columns + pColInfoData = taosArrayGet(p->pDataBlock, 3); + colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.schemaRow.nCols, false); + + // super table name + pColInfoData = taosArrayGet(p->pDataBlock, 4); + colDataAppendNULL(pColInfoData, numOfRows); + + // table comment + pColInfoData = taosArrayGet(p->pDataBlock, 8); + if (mr.me.ntbEntry.commentLen > 0) { + char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(comment, mr.me.ntbEntry.comment); + colDataAppend(pColInfoData, numOfRows, comment, false); + } else if (mr.me.ntbEntry.commentLen == 0) { + char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(comment, ""); + colDataAppend(pColInfoData, numOfRows, comment, false); + } else { + colDataAppendNULL(pColInfoData, numOfRows); + } + + // uid + pColInfoData = taosArrayGet(p->pDataBlock, 5); + colDataAppend(pColInfoData, numOfRows, (char*)&mr.me.uid, false); + + // ttl + pColInfoData = taosArrayGet(p->pDataBlock, 7); + colDataAppend(pColInfoData, numOfRows, (char*)&mr.me.ntbEntry.ttlDays, false); + + STR_TO_VARSTR(n, "NORMAL_TABLE"); + // impl later + } + + metaReaderClear(&mr); + + pColInfoData = taosArrayGet(p->pDataBlock, 9); + colDataAppend(pColInfoData, numOfRows, n, false); + + if (++numOfRows >= pOperator->resultInfo.capacity) { + p->info.rows = numOfRows; + pInfo->pRes->info.rows = numOfRows; + + relocateColumnData(pInfo->pRes, pInfo->matchInfo.pList, p->pDataBlock, false); + doFilterResult(pInfo->pRes, pOperator->exprSupp.pFilterInfo); + + blockDataCleanup(p); + numOfRows = 0; + + if (pInfo->pRes->info.rows > 0) { + break; + } + } + } + + if (numOfRows > 0) { + p->info.rows = numOfRows; + pInfo->pRes->info.rows = numOfRows; + + relocateColumnData(pInfo->pRes, pInfo->matchInfo.pList, p->pDataBlock, false); + doFilterResult(pInfo->pRes, pOperator->exprSupp.pFilterInfo); + + blockDataCleanup(p); + numOfRows = 0; + } + + if (i >= taosArrayGetSize(pIdx->uids)) { + setOperatorCompleted(pOperator); + } else { + pIdx->lastIdx = i + 1; + } + + blockDataDestroy(p); + + pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; + return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; +} + +static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + SSysTableScanInfo* pInfo = pOperator->info; + if (pInfo->pCur == NULL) { + pInfo->pCur = metaOpenTbCursor(pInfo->readHandle.meta); + } + + blockDataCleanup(pInfo->pRes); + int32_t numOfRows = 0; + + const char* db = NULL; + int32_t vgId = 0; + vnodeGetInfo(pInfo->readHandle.vnode, &db, &vgId); + + SName sn = {0}; + char dbname[TSDB_DB_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + tNameFromString(&sn, db, T_NAME_ACCT | T_NAME_DB); + + tNameGetDbName(&sn, varDataVal(dbname)); + varDataSetLen(dbname, strlen(varDataVal(dbname))); + + SSDataBlock* p = buildInfoSchemaTableMetaBlock(TSDB_INS_TABLE_TABLES); + blockDataEnsureCapacity(p, pOperator->resultInfo.capacity); + + char n[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + + int32_t ret = 0; + while ((ret = metaTbCursorNext(pInfo->pCur)) == 0) { + STR_TO_VARSTR(n, pInfo->pCur->mr.me.name); + + // table name + SColumnInfoData* pColInfoData = taosArrayGet(p->pDataBlock, 0); + colDataAppend(pColInfoData, numOfRows, n, false); + + // database name + pColInfoData = taosArrayGet(p->pDataBlock, 1); + colDataAppend(pColInfoData, numOfRows, dbname, false); + + // vgId + pColInfoData = taosArrayGet(p->pDataBlock, 6); + colDataAppend(pColInfoData, numOfRows, (char*)&vgId, false); + + int32_t tableType = pInfo->pCur->mr.me.type; + if (tableType == TSDB_CHILD_TABLE) { + // create time + int64_t ts = pInfo->pCur->mr.me.ctbEntry.ctime; + pColInfoData = taosArrayGet(p->pDataBlock, 2); + colDataAppend(pColInfoData, numOfRows, (char*)&ts, false); + + SMetaReader mr = {0}; + metaReaderInit(&mr, pInfo->readHandle.meta, META_READER_NOLOCK); + + uint64_t suid = pInfo->pCur->mr.me.ctbEntry.suid; + int32_t code = metaGetTableEntryByUid(&mr, suid); + if (code != TSDB_CODE_SUCCESS) { + qError("failed to get super table meta, cname:%s, suid:0x%" PRIx64 ", code:%s, %s", pInfo->pCur->mr.me.name, + suid, tstrerror(terrno), GET_TASKID(pTaskInfo)); + metaReaderClear(&mr); + metaCloseTbCursor(pInfo->pCur); + pInfo->pCur = NULL; + T_LONG_JMP(pTaskInfo->env, terrno); + } + + // number of columns + pColInfoData = taosArrayGet(p->pDataBlock, 3); + colDataAppend(pColInfoData, numOfRows, (char*)&mr.me.stbEntry.schemaRow.nCols, false); + + // super table name + STR_TO_VARSTR(n, mr.me.name); + pColInfoData = taosArrayGet(p->pDataBlock, 4); + colDataAppend(pColInfoData, numOfRows, n, false); + metaReaderClear(&mr); + + // table comment + pColInfoData = taosArrayGet(p->pDataBlock, 8); + if (pInfo->pCur->mr.me.ctbEntry.commentLen > 0) { + char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(comment, pInfo->pCur->mr.me.ctbEntry.comment); + colDataAppend(pColInfoData, numOfRows, comment, false); + } else if (pInfo->pCur->mr.me.ctbEntry.commentLen == 0) { + char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(comment, ""); + colDataAppend(pColInfoData, numOfRows, comment, false); + } else { + colDataAppendNULL(pColInfoData, numOfRows); + } + + // uid + pColInfoData = taosArrayGet(p->pDataBlock, 5); + colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.uid, false); + + // ttl + pColInfoData = taosArrayGet(p->pDataBlock, 7); + colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ctbEntry.ttlDays, false); + + STR_TO_VARSTR(n, "CHILD_TABLE"); + } else if (tableType == TSDB_NORMAL_TABLE) { + // create time + pColInfoData = taosArrayGet(p->pDataBlock, 2); + colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.ctime, false); + + // number of columns + pColInfoData = taosArrayGet(p->pDataBlock, 3); + colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.schemaRow.nCols, false); + + // super table name + pColInfoData = taosArrayGet(p->pDataBlock, 4); + colDataAppendNULL(pColInfoData, numOfRows); + + // table comment + pColInfoData = taosArrayGet(p->pDataBlock, 8); + if (pInfo->pCur->mr.me.ntbEntry.commentLen > 0) { + char comment[TSDB_TB_COMMENT_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(comment, pInfo->pCur->mr.me.ntbEntry.comment); + colDataAppend(pColInfoData, numOfRows, comment, false); + } else if (pInfo->pCur->mr.me.ntbEntry.commentLen == 0) { + char comment[VARSTR_HEADER_SIZE + VARSTR_HEADER_SIZE] = {0}; + STR_TO_VARSTR(comment, ""); + colDataAppend(pColInfoData, numOfRows, comment, false); + } else { + colDataAppendNULL(pColInfoData, numOfRows); + } + + // uid + pColInfoData = taosArrayGet(p->pDataBlock, 5); + colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.uid, false); + + // ttl + pColInfoData = taosArrayGet(p->pDataBlock, 7); + colDataAppend(pColInfoData, numOfRows, (char*)&pInfo->pCur->mr.me.ntbEntry.ttlDays, false); + + STR_TO_VARSTR(n, "NORMAL_TABLE"); + } + + pColInfoData = taosArrayGet(p->pDataBlock, 9); + colDataAppend(pColInfoData, numOfRows, n, false); + + if (++numOfRows >= pOperator->resultInfo.capacity) { + p->info.rows = numOfRows; + pInfo->pRes->info.rows = numOfRows; + + relocateColumnData(pInfo->pRes, pInfo->matchInfo.pList, p->pDataBlock, false); + doFilterResult(pInfo->pRes, pOperator->exprSupp.pFilterInfo); + + blockDataCleanup(p); + numOfRows = 0; + + if (pInfo->pRes->info.rows > 0) { + break; + } + } + } + + if (numOfRows > 0) { + p->info.rows = numOfRows; + pInfo->pRes->info.rows = numOfRows; + + relocateColumnData(pInfo->pRes, pInfo->matchInfo.pList, p->pDataBlock, false); + doFilterResult(pInfo->pRes, pOperator->exprSupp.pFilterInfo); + + blockDataCleanup(p); + numOfRows = 0; + } + + blockDataDestroy(p); + + // todo temporarily free the cursor here, the true reason why the free is not valid needs to be found + if (ret != 0) { + metaCloseTbCursor(pInfo->pCur); + pInfo->pCur = NULL; + setOperatorCompleted(pOperator); + } + + pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; + return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; +} + +static SSDataBlock* sysTableScanUserTables(SOperatorInfo* pOperator) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SSysTableScanInfo* pInfo = pOperator->info; + + SNode* pCondition = pInfo->pCondition; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + // the retrieve is executed on the mnode, so return tables that belongs to the information schema database. + if (pInfo->readHandle.mnd != NULL) { + buildSysDbTableInfo(pInfo, pOperator->resultInfo.capacity); + doFilterResult(pInfo->pRes, pOperator->exprSupp.pFilterInfo); + pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; + + setOperatorCompleted(pOperator); + return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; + } else { + if (pInfo->showRewrite == false) { + if (pCondition != NULL && pInfo->pIdx == NULL) { + SSTabFltArg arg = {.pMeta = pInfo->readHandle.meta, .pVnode = pInfo->readHandle.vnode}; + + SSysTableIndex* idx = taosMemoryMalloc(sizeof(SSysTableIndex)); + idx->init = 0; + idx->uids = taosArrayInit(128, sizeof(int64_t)); + idx->lastIdx = 0; + + pInfo->pIdx = idx; // set idx arg + + int flt = optSysTabFilte(&arg, pCondition, idx->uids); + if (flt == 0) { + pInfo->pIdx->init = 1; + SSDataBlock* blk = sysTableBuildUserTablesByUids(pOperator); + return blk; + } else if (flt == -2) { + qDebug("%s failed to get sys table info by idx, empty result", GET_TASKID(pTaskInfo)); + return NULL; + } else if (flt == -1) { + // not idx + qDebug("%s failed to get sys table info by idx, scan sys table one by one", GET_TASKID(pTaskInfo)); + } + } else if (pCondition != NULL && (pInfo->pIdx != NULL && pInfo->pIdx->init == 1)) { + SSDataBlock* blk = sysTableBuildUserTablesByUids(pOperator); + return blk; + } + } + + return sysTableBuildUserTables(pOperator); + } + return NULL; +} + +static SSDataBlock* sysTableScanUserSTables(SOperatorInfo* pOperator) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SSysTableScanInfo* pInfo = pOperator->info; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + pInfo->pRes->info.rows = 0; + pOperator->status = OP_EXEC_DONE; + + pInfo->loadInfo.totalRows += pInfo->pRes->info.rows; + return (pInfo->pRes->info.rows == 0) ? NULL : pInfo->pRes; +} + +static int32_t getSysTableDbNameColId(const char* pTable) { + // if (0 == strcmp(TSDB_INS_TABLE_INDEXES, pTable)) { + // return 1; + // } + return TSDB_INS_USER_STABLES_DBNAME_COLID; +} + +static EDealRes getDBNameFromConditionWalker(SNode* pNode, void* pContext) { + int32_t code = TSDB_CODE_SUCCESS; + ENodeType nType = nodeType(pNode); + + switch (nType) { + case QUERY_NODE_OPERATOR: { + SOperatorNode* node = (SOperatorNode*)pNode; + if (OP_TYPE_EQUAL == node->opType) { + *(int32_t*)pContext = 1; + return DEAL_RES_CONTINUE; + } + + *(int32_t*)pContext = 0; + return DEAL_RES_IGNORE_CHILD; + } + case QUERY_NODE_COLUMN: { + if (1 != *(int32_t*)pContext) { + return DEAL_RES_CONTINUE; + } + + SColumnNode* node = (SColumnNode*)pNode; + if (getSysTableDbNameColId(node->tableName) == node->colId) { + *(int32_t*)pContext = 2; + return DEAL_RES_CONTINUE; + } + + *(int32_t*)pContext = 0; + return DEAL_RES_CONTINUE; + } + case QUERY_NODE_VALUE: { + if (2 != *(int32_t*)pContext) { + return DEAL_RES_CONTINUE; + } + + SValueNode* node = (SValueNode*)pNode; + char* dbName = nodesGetValueFromNode(node); + strncpy(pContext, varDataVal(dbName), varDataLen(dbName)); + *((char*)pContext + varDataLen(dbName)) = 0; + return DEAL_RES_END; // stop walk + } + default: + break; + } + return DEAL_RES_CONTINUE; +} + +static void getDBNameFromCondition(SNode* pCondition, const char* dbName) { + if (NULL == pCondition) { + return; + } + nodesWalkExpr(pCondition, getDBNameFromConditionWalker, (char*)dbName); +} + +static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { + // build message and send to mnode to fetch the content of system tables. + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SSysTableScanInfo* pInfo = pOperator->info; + char dbName[TSDB_DB_NAME_LEN] = {0}; + + const char* name = tNameGetTableName(&pInfo->name); + if (pInfo->showRewrite) { + getDBNameFromCondition(pInfo->pCondition, dbName); + sprintf(pInfo->req.db, "%d.%s", pInfo->accountId, dbName); + } + + if (strncasecmp(name, TSDB_INS_TABLE_TABLES, TSDB_TABLE_FNAME_LEN) == 0) { + return sysTableScanUserTables(pOperator); + } else if (strncasecmp(name, TSDB_INS_TABLE_TAGS, TSDB_TABLE_FNAME_LEN) == 0) { + return sysTableScanUserTags(pOperator); + } else if (strncasecmp(name, TSDB_INS_TABLE_STABLES, TSDB_TABLE_FNAME_LEN) == 0 && pInfo->showRewrite && + IS_SYS_DBNAME(dbName)) { + return sysTableScanUserSTables(pOperator); + } else { // load the meta from mnode of the given epset + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + while (1) { + int64_t startTs = taosGetTimestampUs(); + tstrncpy(pInfo->req.tb, tNameGetTableName(&pInfo->name), tListLen(pInfo->req.tb)); + tstrncpy(pInfo->req.user, pInfo->pUser, tListLen(pInfo->req.user)); + + int32_t contLen = tSerializeSRetrieveTableReq(NULL, 0, &pInfo->req); + char* buf1 = taosMemoryCalloc(1, contLen); + tSerializeSRetrieveTableReq(buf1, contLen, &pInfo->req); + + // send the fetch remote task result reques + SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); + if (NULL == pMsgSendInfo) { + qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); + pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + return NULL; + } + + int32_t msgType = (strcasecmp(name, TSDB_INS_TABLE_DNODE_VARIABLES) == 0) ? TDMT_DND_SYSTABLE_RETRIEVE + : TDMT_MND_SYSTABLE_RETRIEVE; + + pMsgSendInfo->param = pOperator; + pMsgSendInfo->msgInfo.pData = buf1; + pMsgSendInfo->msgInfo.len = contLen; + pMsgSendInfo->msgType = msgType; + pMsgSendInfo->fp = loadSysTableCallback; + pMsgSendInfo->requestId = pTaskInfo->id.queryId; + + int64_t transporterId = 0; + int32_t code = + asyncSendMsgToServer(pInfo->readHandle.pMsgCb->clientRpc, &pInfo->epSet, &transporterId, pMsgSendInfo); + tsem_wait(&pInfo->ready); + + if (pTaskInfo->code) { + qDebug("%s load meta data from mnode failed, totalRows:%" PRIu64 ", code:%s", GET_TASKID(pTaskInfo), + pInfo->loadInfo.totalRows, tstrerror(pTaskInfo->code)); + return NULL; + } + + SRetrieveMetaTableRsp* pRsp = pInfo->pRsp; + pInfo->req.showId = pRsp->handle; + + if (pRsp->numOfRows == 0 || pRsp->completed) { + pOperator->status = OP_EXEC_DONE; + qDebug("%s load meta data from mnode completed, rowsOfSource:%d, totalRows:%" PRIu64, GET_TASKID(pTaskInfo), + pRsp->numOfRows, pInfo->loadInfo.totalRows); + + if (pRsp->numOfRows == 0) { + taosMemoryFree(pRsp); + return NULL; + } + } + + char* pStart = pRsp->data; + extractDataBlockFromFetchRsp(pInfo->pRes, pRsp->data, pInfo->matchInfo.pList, &pStart); + updateLoadRemoteInfo(&pInfo->loadInfo, pRsp->numOfRows, pRsp->compLen, startTs, pOperator); + + // todo log the filter info + doFilterResult(pInfo->pRes, pOperator->exprSupp.pFilterInfo); + taosMemoryFree(pRsp); + if (pInfo->pRes->info.rows > 0) { + return pInfo->pRes; + } else if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + } + } +} + +SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode* pScanPhyNode, + const char* pUser, SExecTaskInfo* pTaskInfo) { + SSysTableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SSysTableScanInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } + + SScanPhysiNode* pScanNode = &pScanPhyNode->scan; + SDataBlockDescNode* pDescNode = pScanNode->node.pOutputDataBlockDesc; + + int32_t num = 0; + int32_t code = extractColMatchInfo(pScanNode->pScanCols, pDescNode, &num, COL_MATCH_FROM_COL_ID, &pInfo->matchInfo); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + pInfo->accountId = pScanPhyNode->accountId; + pInfo->pUser = taosMemoryStrDup((void*)pUser); + pInfo->sysInfo = pScanPhyNode->sysInfo; + pInfo->showRewrite = pScanPhyNode->showRewrite; + pInfo->pRes = createResDataBlock(pDescNode); + + pInfo->pCondition = pScanNode->node.pConditions; + code = filterInitFromNode(pScanNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + initResultSizeInfo(&pOperator->resultInfo, 4096); + blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); + + tNameAssign(&pInfo->name, &pScanNode->tableName); + const char* name = tNameGetTableName(&pInfo->name); + + if (strncasecmp(name, TSDB_INS_TABLE_TABLES, TSDB_TABLE_FNAME_LEN) == 0 || + strncasecmp(name, TSDB_INS_TABLE_TAGS, TSDB_TABLE_FNAME_LEN) == 0) { + pInfo->readHandle = *(SReadHandle*)readHandle; + } else { + tsem_init(&pInfo->ready, 0, 0); + pInfo->epSet = pScanPhyNode->mgmtEpSet; + pInfo->readHandle = *(SReadHandle*)readHandle; + } + + setOperatorInfo(pOperator, "SysTableScanOperator", QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN, false, OP_NOT_OPENED, + pInfo, pTaskInfo); + pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSysTableScan, NULL, destroySysScanOperator, NULL); + return pOperator; + + _error: + if (pInfo != NULL) { + destroySysScanOperator(pInfo); + } + taosMemoryFreeClear(pOperator); + pTaskInfo->code = code; + return NULL; +} + +void destroySysScanOperator(void* param) { + SSysTableScanInfo* pInfo = (SSysTableScanInfo*)param; + tsem_destroy(&pInfo->ready); + blockDataDestroy(pInfo->pRes); + + const char* name = tNameGetTableName(&pInfo->name); + if (strncasecmp(name, TSDB_INS_TABLE_TABLES, TSDB_TABLE_FNAME_LEN) == 0 || + strncasecmp(name, TSDB_INS_TABLE_TAGS, TSDB_TABLE_FNAME_LEN) == 0 || pInfo->pCur != NULL) { + metaCloseTbCursor(pInfo->pCur); + pInfo->pCur = NULL; + } + if (pInfo->pIdx) { + taosArrayDestroy(pInfo->pIdx->uids); + taosMemoryFree(pInfo->pIdx); + pInfo->pIdx = NULL; + } + + taosArrayDestroy(pInfo->matchInfo.pList); + taosMemoryFreeClear(pInfo->pUser); + + taosMemoryFreeClear(param); +} + +int32_t loadSysTableCallback(void* param, SDataBuf* pMsg, int32_t code) { + SOperatorInfo* operator=(SOperatorInfo*) param; + SSysTableScanInfo* pScanResInfo = (SSysTableScanInfo*)operator->info; + if (TSDB_CODE_SUCCESS == code) { + pScanResInfo->pRsp = pMsg->pData; + + SRetrieveMetaTableRsp* pRsp = pScanResInfo->pRsp; + pRsp->numOfRows = htonl(pRsp->numOfRows); + pRsp->useconds = htobe64(pRsp->useconds); + pRsp->handle = htobe64(pRsp->handle); + pRsp->compLen = htonl(pRsp->compLen); + } else { + operator->pTaskInfo->code = code; + } + + tsem_post(&pScanResInfo->ready); + return TSDB_CODE_SUCCESS; +} + +SSDataBlock* doFilterResult(SSDataBlock* pDataBlock, SFilterInfo* pFilterInfo) { + if (pFilterInfo == NULL) { + return pDataBlock->info.rows == 0 ? NULL : pDataBlock; + } + + doFilter(pDataBlock, pFilterInfo, NULL); + return pDataBlock->info.rows == 0 ? NULL : pDataBlock; +} + +static int32_t sysChkFilter__Comm(SNode* pNode) { + // impl + SOperatorNode* pOper = (SOperatorNode*)pNode; + EOperatorType opType = pOper->opType; + if (opType != OP_TYPE_EQUAL && opType != OP_TYPE_LOWER_EQUAL && opType != OP_TYPE_LOWER_THAN && + opType != OP_TYPE_GREATER_EQUAL && opType != OP_TYPE_GREATER_THAN) { + return -1; + } + return 0; +} + +static int32_t sysChkFilter__DBName(SNode* pNode) { + SOperatorNode* pOper = (SOperatorNode*)pNode; + + if (pOper->opType != OP_TYPE_EQUAL && pOper->opType != OP_TYPE_NOT_EQUAL) { + return -1; + } + + SValueNode* pVal = (SValueNode*)pOper->pRight; + if (!IS_STR_DATA_TYPE(pVal->node.resType.type)) { + return -1; + } + + return 0; +} +static int32_t sysChkFilter__VgroupId(SNode* pNode) { + SOperatorNode* pOper = (SOperatorNode*)pNode; + SValueNode* pVal = (SValueNode*)pOper->pRight; + if (!IS_INTEGER_TYPE(pVal->node.resType.type)) { + return -1; + } + return sysChkFilter__Comm(pNode); +} +static int32_t sysChkFilter__TableName(SNode* pNode) { + SOperatorNode* pOper = (SOperatorNode*)pNode; + SValueNode* pVal = (SValueNode*)pOper->pRight; + if (!IS_STR_DATA_TYPE(pVal->node.resType.type)) { + return -1; + } + return sysChkFilter__Comm(pNode); +} +static int32_t sysChkFilter__CreateTime(SNode* pNode) { + SOperatorNode* pOper = (SOperatorNode*)pNode; + SValueNode* pVal = (SValueNode*)pOper->pRight; + + if (!IS_TIMESTAMP_TYPE(pVal->node.resType.type)) { + return -1; + } + return sysChkFilter__Comm(pNode); +} + +static int32_t sysChkFilter__Ncolumn(SNode* pNode) { + SOperatorNode* pOper = (SOperatorNode*)pNode; + SValueNode* pVal = (SValueNode*)pOper->pRight; + + if (!IS_INTEGER_TYPE(pVal->node.resType.type)) { + return -1; + } + return sysChkFilter__Comm(pNode); +} +static int32_t sysChkFilter__Ttl(SNode* pNode) { + SOperatorNode* pOper = (SOperatorNode*)pNode; + SValueNode* pVal = (SValueNode*)pOper->pRight; + + if (!IS_INTEGER_TYPE(pVal->node.resType.type)) { + return -1; + } + return sysChkFilter__Comm(pNode); +} +static int32_t sysChkFilter__STableName(SNode* pNode) { + SOperatorNode* pOper = (SOperatorNode*)pNode; + SValueNode* pVal = (SValueNode*)pOper->pRight; + if (!IS_STR_DATA_TYPE(pVal->node.resType.type)) { + return -1; + } + return sysChkFilter__Comm(pNode); +} +static int32_t sysChkFilter__Uid(SNode* pNode) { + SOperatorNode* pOper = (SOperatorNode*)pNode; + SValueNode* pVal = (SValueNode*)pOper->pRight; + if (!IS_INTEGER_TYPE(pVal->node.resType.type)) { + return -1; + } + return sysChkFilter__Comm(pNode); +} +static int32_t sysChkFilter__Type(SNode* pNode) { + SOperatorNode* pOper = (SOperatorNode*)pNode; + SValueNode* pVal = (SValueNode*)pOper->pRight; + if (!IS_INTEGER_TYPE(pVal->node.resType.type)) { + return -1; + } + return sysChkFilter__Comm(pNode); +} +static int32_t optSysTabFilteImpl(void* arg, SNode* cond, SArray* result) { + if (optSysCheckOper(cond) != 0) return -1; + + SOperatorNode* pNode = (SOperatorNode*)cond; + + int8_t i = 0; + for (; i < SYSTAB_FILTER_DICT_SIZE; i++) { + if (strcmp(filterDict[i].name, ((SColumnNode*)(pNode->pLeft))->colName) == 0) { + break; + } + } + if (i >= SYSTAB_FILTER_DICT_SIZE) return -1; + + if (filterDict[i].chkFunc(cond) != 0) return -1; + + return filterDict[i].fltFunc(arg, cond, result); +} + +static int32_t optSysCheckOper(SNode* pOpear) { + if (nodeType(pOpear) != QUERY_NODE_OPERATOR) return -1; + + SOperatorNode* pOper = (SOperatorNode*)pOpear; + if (pOper->opType < OP_TYPE_GREATER_THAN || pOper->opType > OP_TYPE_NOT_EQUAL) { + return -1; + } + + if (nodeType(pOper->pLeft) != QUERY_NODE_COLUMN || nodeType(pOper->pRight) != QUERY_NODE_VALUE) { + return -1; + } + return 0; +} + +static FORCE_INLINE int optSysBinarySearch(SArray* arr, int s, int e, uint64_t k) { + uint64_t v; + int32_t m; + while (s <= e) { + m = s + (e - s) / 2; + v = *(uint64_t*)taosArrayGet(arr, m); + if (v >= k) { + e = m - 1; + } else { + s = m + 1; + } + } + return s; +} + +void optSysIntersection(SArray* in, SArray* out) { + int32_t sz = (int32_t)taosArrayGetSize(in); + if (sz <= 0) { + return; + } + MergeIndex* mi = taosMemoryCalloc(sz, sizeof(MergeIndex)); + for (int i = 0; i < sz; i++) { + SArray* t = taosArrayGetP(in, i); + mi[i].len = (int32_t)taosArrayGetSize(t); + mi[i].idx = 0; + } + + SArray* base = taosArrayGetP(in, 0); + for (int i = 0; i < taosArrayGetSize(base); i++) { + uint64_t tgt = *(uint64_t*)taosArrayGet(base, i); + bool has = true; + for (int j = 1; j < taosArrayGetSize(in); j++) { + SArray* oth = taosArrayGetP(in, j); + int mid = optSysBinarySearch(oth, mi[j].idx, mi[j].len - 1, tgt); + if (mid >= 0 && mid < mi[j].len) { + uint64_t val = *(uint64_t*)taosArrayGet(oth, mid); + has = (val == tgt ? true : false); + mi[j].idx = mid; + } else { + has = false; + } + } + if (has == true) { + taosArrayPush(out, &tgt); + } + } + taosMemoryFreeClear(mi); +} + +static int tableUidCompare(const void* a, const void* b) { + int64_t u1 = *(int64_t*)a; + int64_t u2 = *(int64_t*)b; + if (u1 == u2) { + return 0; + } + return u1 < u2 ? -1 : 1; +} + +static int32_t optSysMergeRslt(SArray* mRslt, SArray* rslt) { + // TODO, find comm mem from mRslt + for (int i = 0; i < taosArrayGetSize(mRslt); i++) { + SArray* arslt = taosArrayGetP(mRslt, i); + taosArraySort(arslt, tableUidCompare); + } + optSysIntersection(mRslt, rslt); + return 0; +} + +static int32_t optSysSpecialColumn(SNode* cond) { + SOperatorNode* pOper = (SOperatorNode*)cond; + SColumnNode* pCol = (SColumnNode*)pOper->pLeft; + for (int i = 0; i < sizeof(SYSTABLE_SPECIAL_COL) / sizeof(SYSTABLE_SPECIAL_COL[0]); i++) { + if (0 == strcmp(pCol->colName, SYSTABLE_SPECIAL_COL[i])) { + return 1; + } + } + return 0; +} + +static int32_t optSysTabFilte(void* arg, SNode* cond, SArray* result) { + int ret = -1; + if (nodeType(cond) == QUERY_NODE_OPERATOR) { + ret = optSysTabFilteImpl(arg, cond, result); + if (ret == 0) { + SOperatorNode* pOper = (SOperatorNode*)cond; + SColumnNode* pCol = (SColumnNode*)pOper->pLeft; + if (0 == strcmp(pCol->colName, "create_time")) { + return 0; + } + return -1; + } + return ret; + } + + if (nodeType(cond) != QUERY_NODE_LOGIC_CONDITION || ((SLogicConditionNode*)cond)->condType != LOGIC_COND_TYPE_AND) { + return ret; + } + + SLogicConditionNode* pNode = (SLogicConditionNode*)cond; + SNodeList* pList = (SNodeList*)pNode->pParameterList; + + int32_t len = LIST_LENGTH(pList); + + bool hasIdx = false; + bool hasRslt = true; + SArray* mRslt = taosArrayInit(len, POINTER_BYTES); + + SListCell* cell = pList->pHead; + for (int i = 0; i < len; i++) { + if (cell == NULL) break; + + SArray* aRslt = taosArrayInit(16, sizeof(int64_t)); + + ret = optSysTabFilteImpl(arg, cell->pNode, aRslt); + if (ret == 0) { + // has index + hasIdx = true; + if (optSysSpecialColumn(cell->pNode) == 0) { + taosArrayPush(mRslt, &aRslt); + } else { + // db_name/vgroup not result + taosArrayDestroy(aRslt); + } + } else if (ret == -2) { + // current vg + hasIdx = true; + hasRslt = false; + taosArrayDestroy(aRslt); + break; + } else { + taosArrayDestroy(aRslt); + } + cell = cell->pNext; + } + if (hasRslt && hasIdx) { + optSysMergeRslt(mRslt, result); + } + + for (int i = 0; i < taosArrayGetSize(mRslt); i++) { + SArray* aRslt = taosArrayGetP(mRslt, i); + taosArrayDestroy(aRslt); + } + taosArrayDestroy(mRslt); + if (hasRslt == false) { + return -2; + } + if (hasRslt && hasIdx) { + cell = pList->pHead; + for (int i = 0; i < len; i++) { + if (cell == NULL) break; + SOperatorNode* pOper = (SOperatorNode*)cell->pNode; + SColumnNode* pCol = (SColumnNode*)pOper->pLeft; + if (0 == strcmp(pCol->colName, "create_time")) { + return 0; + } + cell = cell->pNext; + } + return -1; + } + return -1; +} + +static int32_t doGetTableRowSize(void* pMeta, uint64_t uid, int32_t* rowLen, const char* idstr) { + *rowLen = 0; + + SMetaReader mr = {0}; + metaReaderInit(&mr, pMeta, 0); + int32_t code = metaGetTableEntryByUid(&mr, uid); + if (code != TSDB_CODE_SUCCESS) { + qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", uid, tstrerror(terrno), idstr); + metaReaderClear(&mr); + return terrno; + } + + if (mr.me.type == TSDB_SUPER_TABLE) { + int32_t numOfCols = mr.me.stbEntry.schemaRow.nCols; + for (int32_t i = 0; i < numOfCols; ++i) { + (*rowLen) += mr.me.stbEntry.schemaRow.pSchema[i].bytes; + } + } else if (mr.me.type == TSDB_CHILD_TABLE) { + uint64_t suid = mr.me.ctbEntry.suid; + tDecoderClear(&mr.coder); + code = metaGetTableEntryByUid(&mr, suid); + if (code != TSDB_CODE_SUCCESS) { + qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", suid, tstrerror(terrno), idstr); + metaReaderClear(&mr); + return terrno; + } + + int32_t numOfCols = mr.me.stbEntry.schemaRow.nCols; + + for (int32_t i = 0; i < numOfCols; ++i) { + (*rowLen) += mr.me.stbEntry.schemaRow.pSchema[i].bytes; + } + } else if (mr.me.type == TSDB_NORMAL_TABLE) { + int32_t numOfCols = mr.me.ntbEntry.schemaRow.nCols; + for (int32_t i = 0; i < numOfCols; ++i) { + (*rowLen) += mr.me.ntbEntry.schemaRow.pSchema[i].bytes; + } + } + + metaReaderClear(&mr); + return TSDB_CODE_SUCCESS; +} + +static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) { + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SBlockDistInfo* pBlockScanInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + STableBlockDistInfo blockDistInfo = {.minRows = INT_MAX, .maxRows = INT_MIN}; + int32_t code = doGetTableRowSize(pBlockScanInfo->readHandle.meta, pBlockScanInfo->uid, + (int32_t*)&blockDistInfo.rowSize, GET_TASKID(pTaskInfo)); + if (code != TSDB_CODE_SUCCESS) { + T_LONG_JMP(pTaskInfo->env, code); + } + + tsdbGetFileBlocksDistInfo(pBlockScanInfo->pHandle, &blockDistInfo); + blockDistInfo.numOfInmemRows = (int32_t)tsdbGetNumOfRowsInMemTable(pBlockScanInfo->pHandle); + + SSDataBlock* pBlock = pBlockScanInfo->pResBlock; + + int32_t slotId = pOperator->exprSupp.pExprInfo->base.resSchema.slotId; + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, slotId); + + int32_t len = tSerializeBlockDistInfo(NULL, 0, &blockDistInfo); + char* p = taosMemoryCalloc(1, len + VARSTR_HEADER_SIZE); + tSerializeBlockDistInfo(varDataVal(p), len, &blockDistInfo); + varDataSetLen(p, len); + + colDataAppend(pColInfo, 0, p, false); + taosMemoryFree(p); + + pBlock->info.rows = 1; + pOperator->status = OP_EXEC_DONE; + return pBlock; +} + +static void destroyBlockDistScanOperatorInfo(void* param) { + SBlockDistInfo* pDistInfo = (SBlockDistInfo*)param; + blockDataDestroy(pDistInfo->pResBlock); + tsdbReaderClose(pDistInfo->pHandle); + taosMemoryFreeClear(param); +} + +static int32_t initTableblockDistQueryCond(uint64_t uid, SQueryTableDataCond* pCond) { + memset(pCond, 0, sizeof(SQueryTableDataCond)); + + pCond->order = TSDB_ORDER_ASC; + pCond->numOfCols = 1; + pCond->colList = taosMemoryCalloc(1, sizeof(SColumnInfo)); + if (pCond->colList == NULL) { + terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + return terrno; + } + + pCond->colList->colId = 1; + pCond->colList->type = TSDB_DATA_TYPE_TIMESTAMP; + pCond->colList->bytes = sizeof(TSKEY); + + pCond->twindows = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX}; + pCond->suid = uid; + pCond->type = TIMEWINDOW_RANGE_CONTAINED; + pCond->startVersion = -1; + pCond->endVersion = -1; + + return TSDB_CODE_SUCCESS; +} + +SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDistScanPhysiNode* pBlockScanNode, + SExecTaskInfo* pTaskInfo) { + SBlockDistInfo* pInfo = taosMemoryCalloc(1, sizeof(SBlockDistInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; + goto _error; + } + + { + SQueryTableDataCond cond = {0}; + + int32_t code = initTableblockDistQueryCond(pBlockScanNode->suid, &cond); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + STableListInfo* pTableListInfo = pTaskInfo->pTableInfoList; + size_t num = tableListGetSize(pTableListInfo); + void* pList = tableListGetInfo(pTableListInfo, 0); + + code = tsdbReaderOpen(readHandle->vnode, &cond, pList, num, &pInfo->pHandle, pTaskInfo->id.str); + cleanupQueryTableDataCond(&cond); + if (code != 0) { + goto _error; + } + } + + pInfo->readHandle = *readHandle; + pInfo->uid = pBlockScanNode->suid; + + pInfo->pResBlock = createResDataBlock(pBlockScanNode->node.pOutputDataBlockDesc); + blockDataEnsureCapacity(pInfo->pResBlock, 1); + + int32_t numOfCols = 0; + SExprInfo* pExprInfo = createExprInfo(pBlockScanNode->pScanPseudoCols, NULL, &numOfCols); + int32_t code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfCols); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + setOperatorInfo(pOperator, "DataBlockDistScanOperator", QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN, false, + OP_NOT_OPENED, pInfo, pTaskInfo); + pOperator->fpSet = + createOperatorFpSet(operatorDummyOpenFn, doBlockInfoScan, NULL, destroyBlockDistScanOperatorInfo, NULL); + return pOperator; + + _error: + taosMemoryFreeClear(pInfo); + taosMemoryFreeClear(pOperator); + return NULL; +} \ No newline at end of file diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 4d69b4d45c..35f50cebca 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -5672,12 +5672,12 @@ int32_t modeFunction(SqlFunctionCtx* pCtx) { int32_t numOfElems = 0; int32_t startOffset = pCtx->offset; for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; ++i) { - char* data = colDataGetData(pInputCol, i); if (colDataIsNull_s(pInputCol, i)) { continue; } - numOfElems++; + + char* data = colDataGetData(pInputCol, i); doModeAdd(pInfo, i, pCtx, data); if (sizeof(SModeInfo) + pInfo->numOfPoints * (sizeof(SModeItem) + pInfo->colBytes) >= MODE_MAX_RESULT_SIZE) { diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 411adc680c..155fc7f831 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -172,8 +172,8 @@ static int32_t parseDuplicateUsingClause(SInsertParseContext* pCxt, SVnodeModifO } // pStmt->pSql -> field1_name, ...) -static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, SParsedDataColInfo* pColList, - SSchema* pSchema) { +static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, bool isTags, + SParsedDataColInfo* pColList, SSchema* pSchema) { col_id_t nCols = pColList->numOfCols; pColList->numOfBound = 0; @@ -227,6 +227,10 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, const char** pSql, S } } + if (!isTags && pColList->cols[0].valStat == VAL_STAT_NONE) { + return buildInvalidOperationMsg(&pCxt->msg, "primary timestamp column can not be null"); + } + pColList->orderStatus = isOrdered ? ORDER_STATUS_ORDERED : ORDER_STATUS_DISORDERED; if (!isOrdered) { @@ -525,7 +529,7 @@ static int32_t parseBoundTagsClause(SInsertParseContext* pCxt, SVnodeModifOpStmt } pStmt->pSql += index; - return parseBoundColumns(pCxt, &pStmt->pSql, &pCxt->tags, pTagsSchema); + return parseBoundColumns(pCxt, &pStmt->pSql, true, &pCxt->tags, pTagsSchema); } static int32_t parseTagValue(SInsertParseContext* pCxt, SVnodeModifOpStmt* pStmt, SSchema* pTagSchema, SToken* pToken, @@ -792,6 +796,8 @@ static int32_t getTableMeta(SInsertParseContext* pCxt, SName* pTbName, bool isSt *pMissCache = true; } else if (isStb && TSDB_SUPER_TABLE != (*pTableMeta)->tableType) { code = buildInvalidOperationMsg(&pCxt->msg, "create table only from super table is allowed"); + } else if (!isStb && TSDB_SUPER_TABLE == (*pTableMeta)->tableType) { + code = buildInvalidOperationMsg(&pCxt->msg, "insert data into super table is not supported"); } } return code; @@ -935,11 +941,12 @@ static int32_t parseBoundColumnsClause(SInsertParseContext* pCxt, SVnodeModifOpS return buildSyntaxErrMsg(&pCxt->msg, "keyword VALUES or FILE is expected", token.z); } // pStmt->pSql -> field1_name, ...) - return parseBoundColumns(pCxt, &pStmt->pSql, &pDataBuf->boundColumnInfo, getTableColumnSchema(pStmt->pTableMeta)); + return parseBoundColumns(pCxt, &pStmt->pSql, false, &pDataBuf->boundColumnInfo, + getTableColumnSchema(pStmt->pTableMeta)); } if (NULL != pStmt->pBoundCols) { - return parseBoundColumns(pCxt, &pStmt->pBoundCols, &pDataBuf->boundColumnInfo, + return parseBoundColumns(pCxt, &pStmt->pBoundCols, false, &pDataBuf->boundColumnInfo, getTableColumnSchema(pStmt->pTableMeta)); } @@ -1571,16 +1578,16 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt, SVnodeModifOpStmt* pSt static void destroySubTableHashElem(void* p) { taosMemoryFree(*(STableMeta**)p); } -static int32_t createVnodeModifOpStmt(SParseContext* pCxt, bool reentry, SNode** pOutput) { +static int32_t createVnodeModifOpStmt(SInsertParseContext* pCxt, bool reentry, SNode** pOutput) { SVnodeModifOpStmt* pStmt = (SVnodeModifOpStmt*)nodesMakeNode(QUERY_NODE_VNODE_MODIF_STMT); if (NULL == pStmt) { return TSDB_CODE_OUT_OF_MEMORY; } - if (pCxt->pStmtCb) { + if (pCxt->pComCxt->pStmtCb) { TSDB_QUERY_SET_TYPE(pStmt->insertType, TSDB_QUERY_TYPE_STMT_INSERT); } - pStmt->pSql = pCxt->pSql; + pStmt->pSql = pCxt->pComCxt->pSql; pStmt->freeHashFunc = insDestroyBlockHashmap; pStmt->freeArrayFunc = insDestroyBlockArrayList; @@ -1604,7 +1611,7 @@ static int32_t createVnodeModifOpStmt(SParseContext* pCxt, bool reentry, SNode** return TSDB_CODE_SUCCESS; } -static int32_t createInsertQuery(SParseContext* pCxt, SQuery** pOutput) { +static int32_t createInsertQuery(SInsertParseContext* pCxt, SQuery** pOutput) { SQuery* pQuery = (SQuery*)nodesMakeNode(QUERY_NODE_QUERY); if (NULL == pQuery) { return TSDB_CODE_OUT_OF_MEMORY; @@ -1667,11 +1674,15 @@ static int32_t getTableVgroupFromMetaData(const SArray* pTables, SVnodeModifOpSt sizeof(SVgroupInfo)); } -static int32_t getTableSchemaFromMetaData(const SMetaData* pMetaData, SVnodeModifOpStmt* pStmt, bool isStb) { +static int32_t getTableSchemaFromMetaData(SInsertParseContext* pCxt, const SMetaData* pMetaData, + SVnodeModifOpStmt* pStmt, bool isStb) { int32_t code = checkAuthFromMetaData(pMetaData->pUser); if (TSDB_CODE_SUCCESS == code) { code = getTableMetaFromMetaData(pMetaData->pTableMeta, &pStmt->pTableMeta); } + if (TSDB_CODE_SUCCESS == code && !isStb && TSDB_SUPER_TABLE == pStmt->pTableMeta->tableType) { + code = buildInvalidOperationMsg(&pCxt->msg, "insert data into super table is not supported"); + } if (TSDB_CODE_SUCCESS == code) { code = getTableVgroupFromMetaData(pMetaData->pTableHash, pStmt, isStb); } @@ -1696,24 +1707,25 @@ static void clearCatalogReq(SCatalogReq* pCatalogReq) { pCatalogReq->pUser = NULL; } -static int32_t setVnodeModifOpStmt(SParseContext* pCxt, SCatalogReq* pCatalogReq, const SMetaData* pMetaData, +static int32_t setVnodeModifOpStmt(SInsertParseContext* pCxt, SCatalogReq* pCatalogReq, const SMetaData* pMetaData, SVnodeModifOpStmt* pStmt) { clearCatalogReq(pCatalogReq); if (pStmt->usingTableProcessing) { - return getTableSchemaFromMetaData(pMetaData, pStmt, true); + return getTableSchemaFromMetaData(pCxt, pMetaData, pStmt, true); } - return getTableSchemaFromMetaData(pMetaData, pStmt, false); + return getTableSchemaFromMetaData(pCxt, pMetaData, pStmt, false); } -static int32_t resetVnodeModifOpStmt(SParseContext* pCxt, SQuery* pQuery) { +static int32_t resetVnodeModifOpStmt(SInsertParseContext* pCxt, SQuery* pQuery) { nodesDestroyNode(pQuery->pRoot); int32_t code = createVnodeModifOpStmt(pCxt, true, &pQuery->pRoot); if (TSDB_CODE_SUCCESS == code) { SVnodeModifOpStmt* pStmt = (SVnodeModifOpStmt*)pQuery->pRoot; - (*pCxt->pStmtCb->getExecInfoFn)(pCxt->pStmtCb->pStmt, &pStmt->pVgroupsHashObj, &pStmt->pTableBlockHashObj); + (*pCxt->pComCxt->pStmtCb->getExecInfoFn)(pCxt->pComCxt->pStmtCb->pStmt, &pStmt->pVgroupsHashObj, + &pStmt->pTableBlockHashObj); if (NULL == pStmt->pVgroupsHashObj) { pStmt->pVgroupsHashObj = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); } @@ -1729,13 +1741,13 @@ static int32_t resetVnodeModifOpStmt(SParseContext* pCxt, SQuery* pQuery) { return code; } -static int32_t initInsertQuery(SParseContext* pCxt, SCatalogReq* pCatalogReq, const SMetaData* pMetaData, +static int32_t initInsertQuery(SInsertParseContext* pCxt, SCatalogReq* pCatalogReq, const SMetaData* pMetaData, SQuery** pQuery) { if (NULL == *pQuery) { return createInsertQuery(pCxt, pQuery); } - if (NULL != pCxt->pStmtCb) { + if (NULL != pCxt->pComCxt->pStmtCb) { return resetVnodeModifOpStmt(pCxt, *pQuery); } @@ -1896,7 +1908,7 @@ int32_t parseInsertSql(SParseContext* pCxt, SQuery** pQuery, SCatalogReq* pCatal .usingDuplicateTable = false, }; - int32_t code = initInsertQuery(pCxt, pCatalogReq, pMetaData, pQuery); + int32_t code = initInsertQuery(&context, pCatalogReq, pMetaData, pQuery); if (TSDB_CODE_SUCCESS == code) { code = parseInsertSqlImpl(&context, (SVnodeModifOpStmt*)(*pQuery)->pRoot); } diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index bb8a7cd140..d9a7cea411 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -499,27 +499,22 @@ int32_t qWorkerProcessFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int return TSDB_CODE_QRY_INVALID_INPUT; } - SResFetchReq *msg = pMsg->pCont; + SResFetchReq req = {0}; SQWorker *mgmt = (SQWorker *)qWorkerMgmt; qwUpdateTimeInQueue(mgmt, ts, FETCH_QUEUE); QW_STAT_INC(mgmt->stat.msgStat.fetchProcessed, 1); - if (NULL == msg || pMsg->contLen < sizeof(*msg)) { - QW_ELOG("invalid fetch msg, msg:%p, msgLen:%d", msg, pMsg->contLen); + if (tDeserializeSResFetchReq(pMsg->pCont, pMsg->contLen, &req) < 0) { + QW_ELOG("tDeserializeSResFetchReq %d failed", pMsg->contLen); QW_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - msg->sId = be64toh(msg->sId); - msg->queryId = be64toh(msg->queryId); - msg->taskId = be64toh(msg->taskId); - msg->execId = ntohl(msg->execId); - - uint64_t sId = msg->sId; - uint64_t qId = msg->queryId; - uint64_t tId = msg->taskId; + uint64_t sId = req.sId; + uint64_t qId = req.queryId; + uint64_t tId = req.taskId; int64_t rId = 0; - int32_t eId = msg->execId; + int32_t eId = req.execId; SQWMsg qwMsg = {.node = node, .msg = NULL, .msgLen = 0, .connInfo = pMsg->info, .msgType = pMsg->msgType}; diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index b0bc0df850..c154060d21 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -1083,22 +1083,29 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, } case TDMT_SCH_FETCH: case TDMT_SCH_MERGE_FETCH: { - msgSize = sizeof(SResFetchReq); + SResFetchReq req = {0}; + req.header.vgId = addr->nodeId; + req.sId = schMgmt.sId; + req.queryId = pJob->queryId; + req.taskId = pTask->taskId; + req.execId = pTask->execId; + + msgSize = tSerializeSResFetchReq(NULL, 0, &req); + if (msgSize < 0) { + SCH_TASK_ELOG("tSerializeSResFetchReq get size, msgSize:%d", msgSize); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + msg = taosMemoryCalloc(1, msgSize); if (NULL == msg) { SCH_TASK_ELOG("calloc %d failed", msgSize); SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - SResFetchReq *pMsg = msg; - - pMsg->header.vgId = htonl(addr->nodeId); - - pMsg->sId = htobe64(schMgmt.sId); - pMsg->queryId = htobe64(pJob->queryId); - pMsg->taskId = htobe64(pTask->taskId); - pMsg->execId = htonl(pTask->execId); - + if (tSerializeSResFetchReq(msg, msgSize, &req) < 0) { + SCH_TASK_ELOG("tSerializeSResFetchReq %d failed", msgSize); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } break; } case TDMT_SCH_DROP_TASK: { diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c index 60bec6ab65..dbe72bea7a 100644 --- a/source/libs/sync/src/syncCommit.c +++ b/source/libs/sync/src/syncCommit.c @@ -171,7 +171,7 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { pSyncNode->pLogStore->syncLogUpdateCommitIndex(pSyncNode->pLogStore, pSyncNode->commitIndex); // execute fsm - if (pSyncNode->pFsm != NULL) { + if (pSyncNode != NULL && pSyncNode->pFsm != NULL) { int32_t code = syncNodeDoCommit(pSyncNode, beginIndex, endIndex, pSyncNode->state); if (code != 0) { sNError(pSyncNode, "advance commit index error, do commit begin:%" PRId64 ", end:%" PRId64, beginIndex, diff --git a/source/libs/sync/src/syncRaftEntry.c b/source/libs/sync/src/syncRaftEntry.c index a759409163..4329722958 100644 --- a/source/libs/sync/src/syncRaftEntry.c +++ b/source/libs/sync/src/syncRaftEntry.c @@ -64,7 +64,7 @@ SSyncRaftEntry* syncEntryBuildFromRpcMsg(const SRpcMsg* pMsg, SyncTerm term, Syn } SSyncRaftEntry* syncEntryBuildFromAppendEntries(const SyncAppendEntries* pMsg) { - SSyncRaftEntry* pEntry = syncEntryBuild(pMsg->dataLen); + SSyncRaftEntry* pEntry = syncEntryBuild((int32_t)(pMsg->dataLen)); if (pEntry == NULL) return NULL; memcpy(pEntry, pMsg->data, pMsg->dataLen); @@ -91,15 +91,14 @@ SSyncRaftEntry* syncEntryBuildNoop(SyncTerm term, SyncIndex index, int32_t vgId) void syncEntryDestory(SSyncRaftEntry* pEntry) { if (pEntry != NULL) { - taosMemoryFree(pEntry); - sTrace("free entry: %p", pEntry); + taosMemoryFree(pEntry); } } void syncEntry2OriginalRpc(const SSyncRaftEntry* pEntry, SRpcMsg* pRpcMsg) { pRpcMsg->msgType = pEntry->originalRpcType; - pRpcMsg->contLen = pEntry->dataLen; + pRpcMsg->contLen = (int32_t)(pEntry->dataLen); pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen); memcpy(pRpcMsg->pCont, pEntry->data, pRpcMsg->contLen); } @@ -339,7 +338,8 @@ int32_t raftEntryCacheGetEntry(struct SRaftEntryCache* pCache, SyncIndex index, SSyncRaftEntry* pEntry = NULL; int32_t code = raftEntryCacheGetEntryP(pCache, index, &pEntry); if (code == 1) { - *ppEntry = taosMemoryMalloc((int64_t)(pEntry->bytes)); + int32_t bytes = (int32_t)pEntry->bytes; + *ppEntry = taosMemoryMalloc((int64_t)bytes); memcpy(*ppEntry, pEntry, pEntry->bytes); (*ppEntry)->rid = -1; } else { diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index 802595c55a..de5f71e5a9 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -91,7 +91,7 @@ int32_t syncNodeReplicateOne(SSyncNode* pSyncNode, SRaftId* pDestId, bool snapsh if (code == 0) { ASSERT(pEntry != NULL); - code = syncBuildAppendEntries(&rpcMsg, pEntry->bytes, pSyncNode->vgId); + code = syncBuildAppendEntries(&rpcMsg, (int32_t)(pEntry->bytes), pSyncNode->vgId); ASSERT(code == 0); pMsg = rpcMsg.pCont; diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index 4fc7dd245d..da6f8a52d9 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -229,7 +229,10 @@ void syncPrintNodeLog(const char* flags, ELogLevel level, int32_t dflag, SSyncNo int32_t writeLen = vsnprintf(eventLog, sizeof(eventLog), format, argpointer); va_end(argpointer); - int32_t aqItems = pNode->pFsm->FpApplyQueueItems(pNode->pFsm); + int32_t aqItems = 0; + if (pNode != NULL && pNode->pFsm != NULL && pNode->pFsm->FpApplyQueueItems != NULL) { + aqItems = pNode->pFsm->FpApplyQueueItems(pNode->pFsm); + } // restore error code terrno = errCode; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 5b8f059e04..275048e94a 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -445,9 +445,11 @@ void cliHandleExceptImpl(SCliConn* pConn, int32_t code) { if (pCtx == NULL || pCtx->pSem == NULL) { if (transMsg.info.ahandle == NULL) { - if (pMsg == NULL || REQUEST_NO_RESP(&pMsg->msg) || pMsg->type == Release) destroyCmsg(pMsg); - once = true; - continue; + if (pMsg == NULL || REQUEST_NO_RESP(&pMsg->msg) || pMsg->type == Release) { + destroyCmsg(pMsg); + once = true; + continue; + } } } @@ -1217,6 +1219,7 @@ static FORCE_INLINE void destroyCmsg(void* arg) { if (pMsg == NULL) { return; } + transDestroyConnCtx(pMsg->ctx); destroyUserdata(&pMsg->msg); taosMemoryFree(pMsg); diff --git a/source/os/src/osFile.c b/source/os/src/osFile.c index 94a10322ed..3003cf84eb 100644 --- a/source/os/src/osFile.c +++ b/source/os/src/osFile.c @@ -739,7 +739,6 @@ void taosFprintfFile(TdFilePtr pFile, const char *format, ...) { va_start(ap, format); vfprintf(pFile->fp, format, ap); va_end(ap); - fflush(pFile->fp); } bool taosValidFile(TdFilePtr pFile) { return pFile != NULL && pFile->fd > 0; } diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 937ddb7a5e..86f4294d30 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -231,7 +231,7 @@ ,,y,script,./test.sh -f tsim/stream/windowClose.sim ,,y,script,./test.sh -f tsim/stream/ignoreExpiredData.sim ,,y,script,./test.sh -f tsim/stream/sliding.sim -,,n,script,./test.sh -f tsim/stream/partitionbyColumnInterval.sim +,,,script,./test.sh -f tsim/stream/partitionbyColumnInterval.sim ,,y,script,./test.sh -f tsim/stream/partitionbyColumnSession.sim ,,y,script,./test.sh -f tsim/stream/partitionbyColumnState.sim ,,y,script,./test.sh -f tsim/stream/deleteInterval.sim @@ -278,8 +278,8 @@ ,,y,script,./test.sh -f tsim/stable/values.sim ,,y,script,./test.sh -f tsim/stable/vnode3.sim ,,y,script,./test.sh -f tsim/stable/metrics_idx.sim -,,n,script,./test.sh -f tsim/sma/drop_sma.sim -,,n,script,./test.sh -f tsim/sma/sma_leak.sim +,,,script,./test.sh -f tsim/sma/drop_sma.sim +,,,script,./test.sh -f tsim/sma/sma_leak.sim ,,y,script,./test.sh -f tsim/sma/tsmaCreateInsertQuery.sim ,,y,script,./test.sh -f tsim/sma/rsmaCreateInsertQuery.sim ,,y,script,./test.sh -f tsim/sma/rsmaPersistenceRecovery.sim @@ -401,18 +401,18 @@ #system test -,,,system-test,python3 ./test.py -f 0-others/taosShell.py -,,,system-test,python3 ./test.py -f 0-others/taosShellError.py -,,,system-test,python3 ./test.py -f 0-others/taosShellNetChk.py -,,,system-test,python3 ./test.py -f 0-others/telemetry.py -,,,system-test,python3 ./test.py -f 0-others/taosdMonitor.py -,,,system-test,python3 ./test.py -f 0-others/udfTest.py -,,,system-test,python3 ./test.py -f 0-others/udf_create.py -,,,system-test,python3 ./test.py -f 0-others/udf_restart_taosd.py -,,,system-test,python3 ./test.py -f 0-others/cachemodel.py -,,,system-test,python3 ./test.py -f 0-others/udf_cfg1.py -,,,system-test,python3 ./test.py -f 0-others/udf_cfg2.py -,,,system-test,python3 ./test.py -f 0-others/taosdShell.py -N 5 -M 3 -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosShell.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosShellError.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosShellNetChk.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/telemetry.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/taosdMonitor.py +,,n,system-test,python3 ./test.py -f 0-others/udfTest.py +,,n,system-test,python3 ./test.py -f 0-others/udf_create.py +,,n,system-test,python3 ./test.py -f 0-others/udf_restart_taosd.py +,,n,system-test,python3 ./test.py -f 0-others/cachemodel.py +,,n,system-test,python3 ./test.py -f 0-others/udf_cfg1.py +,,n,system-test,python3 ./test.py -f 0-others/udf_cfg2.py +,,n,system-test,python3 ./test.py -f 0-others/taosdShell.py -N 5 -M 3 -Q 3 ,,,system-test,python3 ./test.py -f 0-others/sysinfo.py ,,,system-test,python3 ./test.py -f 0-others/user_control.py ,,,system-test,python3 ./test.py -f 0-others/fsync.py @@ -451,8 +451,8 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arcsin.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arctan.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arctan.py -R -,,,system-test,python3 ./test.py -f 2-query/avg.py -,,,system-test,python3 ./test.py -f 2-query/avg.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/avg.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/avg.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/between.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/between.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/bottom.py @@ -465,12 +465,12 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/char_length.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/check_tsdb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/check_tsdb.py -R -,,,system-test,python3 ./test.py -f 2-query/concat.py -,,,system-test,python3 ./test.py -f 2-query/concat.py -R -,,,system-test,python3 ./test.py -f 2-query/concat_ws.py -,,,system-test,python3 ./test.py -f 2-query/concat_ws.py -R -,,,system-test,python3 ./test.py -f 2-query/concat_ws2.py -,,,system-test,python3 ./test.py -f 2-query/concat_ws2.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat_ws.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat_ws.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat_ws2.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat_ws2.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cos.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cos.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py @@ -515,14 +515,14 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/histogram.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/hyperloglog.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/hyperloglog.py -R -,,,system-test,python3 ./test.py -f 2-query/interp.py -,,,system-test,python3 ./test.py -f 2-query/interp.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interp.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interp.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/irate.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/irate.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join.py -R -,,,system-test,python3 ./test.py -f 2-query/last_row.py -,,,system-test,python3 ./test.py -f 2-query/last_row.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/leastsquares.py @@ -531,10 +531,10 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/length.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/log.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/log.py -R -,,,system-test,python3 ./test.py -f 2-query/lower.py -,,,system-test,python3 ./test.py -f 2-query/lower.py -R -,,,system-test,python3 ./test.py -f 2-query/ltrim.py -,,,system-test,python3 ./test.py -f 2-query/ltrim.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/lower.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/lower.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ltrim.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ltrim.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mavg.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mavg.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_partition.py @@ -555,8 +555,8 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/query_cols_tags_and_or.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/round.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/round.py -R -,,,system-test,python3 ./test.py -f 2-query/rtrim.py -,,,system-test,python3 ./test.py -f 2-query/rtrim.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/rtrim.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/rtrim.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sample.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sample.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sin.py @@ -573,12 +573,12 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/statecount.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stateduration.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stateduration.py -R -,,,system-test,python3 ./test.py -f 2-query/substr.py -,,,system-test,python3 ./test.py -f 2-query/substr.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/substr.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/substr.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sum.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sum.py -R -,,,system-test,python3 ./test.py -f 2-query/tail.py -,,,system-test,python3 ./test.py -f 2-query/tail.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tail.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tail.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tan.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tan.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Timediff.py @@ -603,8 +603,8 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/twa.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/union.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/union.py -R -,,,system-test,python3 ./test.py -f 2-query/unique.py -,,,system-test,python3 ./test.py -f 2-query/unique.py -R +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/unique.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/unique.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/upper.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/upper.py -R ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/varchar.py @@ -618,7 +618,7 @@ ,,,system-test,python3 ./test.py -f 1-insert/delete_normaltable.py ,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/keep_expired.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join2.py -,,,system-test,python3 ./test.py -f 2-query/union1.py +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/union1.py ,,,system-test,python3 ./test.py -f 2-query/concat2.py ,,,system-test,python3 ./test.py -f 2-query/json_tag.py ,,,system-test,python3 ./test.py -f 2-query/nestedQuery.py @@ -725,11 +725,11 @@ ,,,system-test,python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py ,,,system-test,python3 ./test.py -f 99-TDcase/TD-19201.py ,,,system-test,python3 ./test.py -f 7-tmq/tmqSubscribeStb-r3.py -N 5 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/between.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/between.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distinct.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/varchar.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/ltrim.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/rtrim.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ltrim.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/rtrim.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/length.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/char_length.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/upper.py -Q 2 @@ -737,13 +737,13 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join2.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cast.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/substr.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/union.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/union1.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/concat.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/concat2.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/concat_ws.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/concat_ws2.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/substr.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/union.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/union1.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat2.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat_ws.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat_ws2.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/check_tsdb.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/spread.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/hyperloglog.py -Q 2 @@ -782,230 +782,230 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arcsin.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arccos.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arctan.py -Q 2 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/interp.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interp.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/nestedQuery.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/nestedQuery_str.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/nestedQuery_math.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/nestedQuery_time.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/stablity.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/stablity_1.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/avg.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/elapsed.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/csum.py -Q 2 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mavg.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/sample.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/function_diff.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/unique.py -Q 2 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stateduration.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/function_stateduration.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/statecount.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/tail.py -Q 2 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ttl_comment.py -Q 2 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_count.py -Q 2 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_max.py -Q 2 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_min.py -Q 2 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 2 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 2 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 2 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 2 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 2 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/twa.py -Q 2 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/irate.py -Q 2 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_null.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/avg.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/elapsed.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/csum.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mavg.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/sample.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/function_diff.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/unique.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stateduration.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/function_stateduration.py -Q 2 +,,,system-test,python3 ./test.py -f 2-query/statecount.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tail.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ttl_comment.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_count.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_max.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_min.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/twa.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/irate.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_null.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_partition.py -Q 2 -,,,system-test,python3 ./test.py -f 2-query/last_row.py -Q 2 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row.py -Q 2 ,,,system-test,python3 ./test.py -f 2-query/tsbsQuery.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -Q 2 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/case_when.py -Q 2 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/between.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distinct.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/varchar.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/ltrim.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/rtrim.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/length.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/char_length.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/upper.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/lower.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join2.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cast.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/substr.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/union.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/union1.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/concat.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/concat2.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/concat_ws.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/concat_ws2.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/check_tsdb.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/spread.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/hyperloglog.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/explain.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/leastsquares.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/timezone.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Now.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Today.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/min.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/mode.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/between.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distinct.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/varchar.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ltrim.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/rtrim.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/length.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/char_length.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/upper.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/lower.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join2.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cast.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/substr.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/union.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/union1.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat2.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat_ws.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat_ws2.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/check_tsdb.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/spread.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/hyperloglog.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/explain.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/leastsquares.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/timezone.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Now.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Today.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/min.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/mode.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/first.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/To_iso8601.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/To_unixtimestamp.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/timetruncate.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/diff.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Timediff.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/json_tag.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/top.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/bottom.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/percentile.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/apercentile.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ceil.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/floor.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/round.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/log.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/pow.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sqrt.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sin.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cos.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tan.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arcsin.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arccos.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arctan.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/first.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/To_iso8601.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/To_unixtimestamp.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/timetruncate.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/diff.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Timediff.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/json_tag.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/top.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/bottom.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/percentile.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/apercentile.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ceil.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/floor.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/round.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/log.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/pow.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sqrt.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sin.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cos.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tan.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arcsin.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arccos.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arctan.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/nestedQuery.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/nestedQuery_str.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/nestedQuery_math.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/nestedQuery_time.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/stablity.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/stablity_1.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/avg.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/elapsed.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/csum.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mavg.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/sample.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/function_diff.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/unique.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stateduration.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/function_stateduration.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/statecount.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/tail.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ttl_comment.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_count.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_max.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_min.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/twa.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/irate.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_null.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/avg.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/elapsed.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/csum.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mavg.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/sample.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/function_diff.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/unique.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stateduration.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/function_stateduration.py -Q 3 +,,,system-test,python3 ./test.py -f 2-query/statecount.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tail.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ttl_comment.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_count.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_max.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_min.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/twa.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/irate.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_null.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_partition.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/last_row.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row.py -Q 3 ,,,system-test,python3 ./test.py -f 2-query/tsbsQuery.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -Q 3 -,,,system-test,python3 ./test.py -f 2-query/interp.py -Q 3 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interp.py -Q 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/case_when.py -Q 3 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/between.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distinct.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/varchar.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/ltrim.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/rtrim.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/length.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/char_length.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/upper.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/lower.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/join2.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/substr.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/union.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/union1.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/concat.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/concat2.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/concat_ws.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/concat_ws2.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/check_tsdb.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/spread.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/hyperloglog.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/explain.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/leastsquares.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/timezone.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Now.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Today.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/min.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/mode.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/between.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distinct.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/varchar.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ltrim.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/rtrim.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/length.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/char_length.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/upper.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/lower.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/join2.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/substr.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/union.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/union1.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat2.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat_ws.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/concat_ws2.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/check_tsdb.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/spread.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/hyperloglog.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/explain.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/leastsquares.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/timezone.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Now.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Today.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/min.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/mode.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/countAlwaysReturnValue.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/first.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/To_iso8601.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/To_unixtimestamp.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/timetruncate.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/diff.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Timediff.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/json_tag.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/top.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/bottom.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/percentile.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/apercentile.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ceil.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/floor.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/round.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/log.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/pow.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sqrt.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sin.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cos.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tan.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arcsin.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arccos.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arctan.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/first.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/To_iso8601.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/To_unixtimestamp.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/timetruncate.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/diff.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/Timediff.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/json_tag.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/top.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/bottom.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/percentile.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/apercentile.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ceil.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/floor.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/round.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/log.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/pow.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sqrt.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sin.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cos.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tan.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arcsin.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arccos.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/arctan.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/query_cols_tags_and_or.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/nestedQuery.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/nestedQuery_str.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/nestedQuery_math.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/nestedQuery_time.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/stablity.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/stablity_1.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/avg.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/elapsed.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/csum.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mavg.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/sample.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cast.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/function_diff.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/unique.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stateduration.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/function_stateduration.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/statecount.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/tail.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ttl_comment.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_count.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_max.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_min.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/twa.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/irate.py -Q 4 -,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_null.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/avg.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/elapsed.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/csum.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/mavg.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/sample.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/cast.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/function_diff.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/unique.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/stateduration.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/function_stateduration.py -Q 4 +,,,system-test,python3 ./test.py -f 2-query/statecount.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/tail.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ttl_comment.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_count.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_max.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_min.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_sum.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_spread.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_apercentile.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_avg.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distribute_agg_stddev.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/twa.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/irate.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_null.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/count_partition.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/max_partition.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/last_row.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/last_row.py -Q 4 ,,,system-test,python3 ./test.py -f 2-query/tsbsQuery.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/sml.py -Q 4 -,,,system-test,python3 ./test.py -f 2-query/interp.py -Q 4 +,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/interp.py -Q 4 ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/case_when.py -Q 4 #develop test @@ -1023,8 +1023,8 @@ ,,,develop-test,python3 ./test.py -f 5-taos-tools/taosbenchmark/telnet_tcp.py -R #docs-examples test -,,,docs-examples-test,bash python.sh -,,,docs-examples-test,bash node.sh -,,,docs-examples-test,bash csharp.sh -,,,docs-examples-test,bash jdbc.sh -,,,docs-examples-test,bash go.sh +,,n,docs-examples-test,bash python.sh +,,n,docs-examples-test,bash node.sh +,,n,docs-examples-test,bash csharp.sh +,,n,docs-examples-test,bash jdbc.sh +,,n,docs-examples-test,bash go.sh diff --git a/tests/script/runAllSimCases.sh b/tests/script/runAllSimCases.sh index 97bebd9eb8..e9214caad1 100755 --- a/tests/script/runAllSimCases.sh +++ b/tests/script/runAllSimCases.sh @@ -11,6 +11,9 @@ set -e VALGRIND=0 LOG_BK_DIR=/data/valgrind_log_backup # 192.168.0.203 SIM_FILES=./jenkins/basic.txt +cases_task_file=../parallel_test/cases.task + +cat $cases_task_file | grep "./test.sh " | awk -F, '{print $5}' > $SIM_FILES while getopts "v:r:f:" arg do @@ -21,9 +24,9 @@ do r) LOG_BK_DIR=$(echo $OPTARG) ;; - f) - SIM_FILES=$(echo $OPTARG) - ;; + #f) + # SIM_FILES=$(echo $OPTARG) + # ;; ?) #unknow option echo "unkonw argument" exit 1 diff --git a/tests/script/sh/checkAsan.sh b/tests/script/sh/checkAsan.sh index 074956534f..8b478384cf 100755 --- a/tests/script/sh/checkAsan.sh +++ b/tests/script/sh/checkAsan.sh @@ -21,15 +21,24 @@ LOG_DIR=$TAOS_DIR/sim/asan error_num=`cat ${LOG_DIR}/*.asan | grep "ERROR" | wc -l` memory_leak=`cat ${LOG_DIR}/*.asan | grep "Direct leak" | wc -l` indirect_leak=`cat ${LOG_DIR}/*.asan | grep "Indirect leak" | wc -l` +python_error=`cat ${LOG_DIR}/*.info | grep -w "stack" | wc -l` # ignore + +# TD-20368 +# /root/TDengine/contrib/zlib/trees.c:873:5: runtime error: null pointer passed as argument 2, which is declared to never be null + +# TD-20494 TD-20452 # /root/TDengine/source/libs/scalar/src/sclfunc.c:735:11: runtime error: 4.75783e+11 is outside the range of representable values of type 'signed char' # /root/TDengine/source/libs/scalar/src/sclfunc.c:790:11: runtime error: 3.4e+38 is outside the range of representable values of type 'long int' # /root/TDengine/source/libs/scalar/src/sclfunc.c:772:11: runtime error: 3.52344e+09 is outside the range of representable values of type 'int' # /root/TDengine/source/libs/scalar/src/sclfunc.c:753:11: runtime error: 4.75783e+11 is outside the range of representable values of type 'short int' -runtime_error=`cat ${LOG_DIR}/*.asan | grep "runtime error" | grep -v "trees.c:873" | grep -v "sclfunc.c.*outside the range of representable values of type" | wc -l` -python_error=`cat ${LOG_DIR}/*.info | grep -w "stack" | wc -l` +# TD-20569 +# /root/TDengine/source/libs/function/src/builtinsimpl.c:856:29: runtime error: signed integer overflow: 9223372036854775806 + 9223372036854775805 cannot be represented in type 'long int' +# /root/TDengine/source/libs/scalar/src/sclvector.c:1075:66: runtime error: signed integer overflow: 9223372034707292160 + 1668838476672 cannot be represented in type 'long int' + +runtime_error=`cat ${LOG_DIR}/*.asan | grep "runtime error" | grep -v "trees.c:873" | grep -v "sclfunc.c.*outside the range of representable values of type"| grep -v "builtinsimpl.c.*signed integer overflow"| grep -v "sclvector.c.*signed integer overflow" | wc -l` echo -e "\033[44;32;1m"asan error_num: $error_num"\033[0m" echo -e "\033[44;32;1m"asan memory_leak: $memory_leak"\033[0m" diff --git a/tests/script/sh/stop_dnodes.sh b/tests/script/sh/stop_dnodes.sh index ce2d7144f9..ab9cfc1017 100755 --- a/tests/script/sh/stop_dnodes.sh +++ b/tests/script/sh/stop_dnodes.sh @@ -26,3 +26,17 @@ while [ -n "$PID" ]; do fi PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` done + +PID=`ps -ef|grep -w taos | grep -v grep | awk '{print $2}'` +while [ -n "$PID" ]; do + echo kill -9 $PID + #pkill -9 taosd + kill -9 $PID + echo "Killing taosd processes" + if [ "$OS_TYPE" != "Darwin" ]; then + fuser -k -n tcp 6030 + else + lsof -nti:6030 | xargs kill -9 + fi + PID=`ps -ef|grep -w taosd | grep -v grep | awk '{print $2}'` +done diff --git a/tests/system-test/0-others/taosShellNetChk.py b/tests/system-test/0-others/taosShellNetChk.py index 781fcae638..aac7808d24 100644 --- a/tests/system-test/0-others/taosShellNetChk.py +++ b/tests/system-test/0-others/taosShellNetChk.py @@ -230,9 +230,11 @@ class TDTestCase: tdLog.exit('taos -n client fail!') finally: if platform.system().lower() == 'windows': - os.system('ps -a | grep taos | awk \'{print $2}\' | xargs kill -9') + tdLog.info("ps -a | grep taos | awk \'{print $2}\' | xargs kill -9") + # os.system('ps -a | grep taos | awk \'{print $2}\' | xargs kill -9') else: - os.system('pkill -9 taos') + tdLog.info("pkill -9 taos") + # os.system('pkill -9 taos') def stop(self): tdSql.close()