diff --git a/Jenkinsfile b/Jenkinsfile index fc2b3562c1..4b84e1f88e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -113,7 +113,7 @@ pipeline { ''' sh''' cd ${WKC}/debug - ctest + ctest -VV ''' } } diff --git a/Jenkinsfile2 b/Jenkinsfile2 index cca886d4af..c9c9c3a7ca 100644 --- a/Jenkinsfile2 +++ b/Jenkinsfile2 @@ -121,7 +121,7 @@ pipeline { pre_test() sh''' cd ${WKC}/debug - ctest + ctest -VV ''' sh''' export LD_LIBRARY_PATH=${WKC}/debug/build/lib diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 19923a5ad6..1ddc765c5c 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -14,24 +14,9 @@ if(${BUILD_PTHREAD}) cat("${TD_SUPPORT_DIR}/pthread_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) endif() -# iconv -if(${BUILD_WITH_ICONV}) - cat("${TD_SUPPORT_DIR}/iconv_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) -endif() - -# msvc regex -if(${BUILD_MSVCREGEX}) - cat("${TD_SUPPORT_DIR}/msvcregex_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) -endif() - -# wcwidth -if(${BUILD_WCWIDTH}) - cat("${TD_SUPPORT_DIR}/wcwidth_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) -endif() - -# wingetopt -if(${BUILD_WINGETOPT}) - cat("${TD_SUPPORT_DIR}/wingetopt_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) +# gnu regex +if(${BUILD_GNUREGEX}) + cat("${TD_SUPPORT_DIR}/gnuregex_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) endif() # googletest @@ -114,27 +99,8 @@ if(${BUILD_TEST}) target_include_directories( gtest PUBLIC $ + PUBLIC $ ) - if(${TD_WINDOWS}) - target_include_directories( - gtest - PUBLIC $ - ) - endif(${TD_WINDOWS}) - if(${TD_LINUX}) - target_include_directories( - gtest - PUBLIC $ - ) - endif(${TD_LINUX}) - if(${TD_DARWIN}) - target_include_directories( - gtest - PUBLIC $ - ) - endif(${TD_DARWIN}) - - endif(${BUILD_TEST}) # cJson @@ -216,53 +182,6 @@ if(${BUILD_WITH_NURAFT}) add_subdirectory(nuraft) endif(${BUILD_WITH_NURAFT}) -# pthread -if(${BUILD_PTHREAD}) - set(CMAKE_BUILD_TYPE release) - add_definitions(-DPTW32_STATIC_LIB) - add_subdirectory(pthread) - set_target_properties(libpthreadVC3 PROPERTIES OUTPUT_NAME pthread) - add_library(pthread STATIC IMPORTED GLOBAL) - SET_PROPERTY(TARGET pthread PROPERTY IMPORTED_LOCATION ${LIBRARY_OUTPUT_PATH}/pthread.lib) -endif() - -# iconv -if(${BUILD_WITH_ICONV}) - add_subdirectory(iconv) -endif(${BUILD_WITH_ICONV}) - -# wingetopt -if(${BUILD_WINGETOPT}) - add_subdirectory(wingetopt) -endif(${BUILD_WINGETOPT}) - -# msvcregex -if(${BUILD_MSVCREGEX}) - add_library(msvcregex STATIC "") - target_sources(msvcregex - PRIVATE "msvcregex/regex.c" - ) - target_include_directories(msvcregex - PRIVATE "msvcregex" - ) - target_link_libraries(msvcregex - INTERFACE Shell32 - ) - SET_TARGET_PROPERTIES(msvcregex PROPERTIES OUTPUT_NAME msvcregex) -endif(${BUILD_MSVCREGEX}) - -# msvcregex -if(${BUILD_WCWIDTH}) - add_library(wcwidth STATIC "") - target_sources(wcwidth - PRIVATE "wcwidth/wcwidth.c" - ) - target_include_directories(wcwidth - PRIVATE "wcwidth" - ) - SET_TARGET_PROPERTIES(wcwidth PROPERTIES OUTPUT_NAME wcwidth) -endif(${BUILD_WCWIDTH}) - # CRAFT if(${BUILD_WITH_CRAFT}) add_library(craft STATIC IMPORTED GLOBAL) @@ -319,12 +238,8 @@ if(${BUILD_WITH_SQLITE}) target_link_libraries(sqlite INTERFACE m INTERFACE pthread + INTERFACE dl ) - if(NOT TD_WINDOWS) - target_link_libraries(sqlite - INTERFACE dl - ) - endif(NOT TD_WINDOWS) endif(${BUILD_WITH_SQLITE}) # pthread diff --git a/contrib/test/craft/CMakeLists.txt b/contrib/test/craft/CMakeLists.txt index ec8b44b673..e0f6ae64bd 100644 --- a/contrib/test/craft/CMakeLists.txt +++ b/contrib/test/craft/CMakeLists.txt @@ -1,9 +1,2 @@ add_executable(simulate_vnode "simulate_vnode.c") -target_link_libraries(simulate_vnode PUBLIC craft lz4 uv_a) -if(${BUILD_WINGETOPT}) - target_link_libraries(simulate_vnode PUBLIC wingetopt) - target_include_directories( - simulate_vnode - PUBLIC "${TD_SOURCE_DIR}/contrib/wingetopt/src" - ) -endif() \ No newline at end of file +target_link_libraries(simulate_vnode PUBLIC craft lz4 uv_a) \ No newline at end of file diff --git a/contrib/test/tdev/src/main.c b/contrib/test/tdev/src/main.c index e40040ce97..5e1de83e88 100644 --- a/contrib/test/tdev/src/main.c +++ b/contrib/test/tdev/src/main.c @@ -6,39 +6,43 @@ #define POINTER_SHIFT(ptr, s) ((void *)(((char *)ptr) + (s))) #define POINTER_DISTANCE(pa, pb) ((char *)(pb) - (char *)(pa)) -static inline void tPutA(void **buf, uint64_t val) { - memcpy(buf, &val, sizeof(val)); - *buf = POINTER_SHIFT(buf, sizeof(val)); -} +#define tPutA(buf, val) \ + ({ \ + memcpy(buf, &val, sizeof(val)); \ + POINTER_SHIFT(buf, sizeof(val)); \ + }) -static inline void tPutB(void **buf, uint64_t val) { - ((uint8_t *)buf)[7] = ((val) >> 56) & 0xff; - ((uint8_t *)buf)[6] = ((val) >> 48) & 0xff; - ((uint8_t *)buf)[5] = ((val) >> 40) & 0xff; - ((uint8_t *)buf)[4] = ((val) >> 32) & 0xff; - ((uint8_t *)buf)[3] = ((val) >> 24) & 0xff; - ((uint8_t *)buf)[2] = ((val) >> 16) & 0xff; - ((uint8_t *)buf)[1] = ((val) >> 8) & 0xff; - ((uint8_t *)buf)[0] = (val)&0xff; - *buf = POINTER_SHIFT(buf, sizeof(val)); -} +#define tPutB(buf, val) \ + ({ \ + ((uint8_t *)buf)[7] = ((val) >> 56) & 0xff; \ + ((uint8_t *)buf)[6] = ((val) >> 48) & 0xff; \ + ((uint8_t *)buf)[5] = ((val) >> 40) & 0xff; \ + ((uint8_t *)buf)[4] = ((val) >> 32) & 0xff; \ + ((uint8_t *)buf)[3] = ((val) >> 24) & 0xff; \ + ((uint8_t *)buf)[2] = ((val) >> 16) & 0xff; \ + ((uint8_t *)buf)[1] = ((val) >> 8) & 0xff; \ + ((uint8_t *)buf)[0] = (val)&0xff; \ + POINTER_SHIFT(buf, sizeof(val)); \ + }) -static inline void tPutC(void **buf, uint64_t val) { - if (buf) { - ((uint64_t *)buf)[0] = (val); - POINTER_SHIFT(buf, sizeof(val)); - } - *buf = NULL; -} +#define tPutC(buf, val) \ + ({ \ + if (buf) { \ + ((uint64_t *)buf)[0] = (val); \ + POINTER_SHIFT(buf, sizeof(val)); \ + } \ + NULL; \ + }) -static inline void tPutD(void **buf, uint64_t val) { - uint64_t tmp = val; - for (size_t i = 0; i < sizeof(val); i++) { - ((uint8_t *)buf)[i] = tmp & 0xff; - tmp >>= 8; - } - *buf = POINTER_SHIFT(buf, sizeof(val)); -} +#define tPutD(buf, val) \ + ({ \ + uint64_t tmp = val; \ + for (size_t i = 0; i < sizeof(val); i++) { \ + ((uint8_t *)buf)[i] = tmp & 0xff; \ + tmp >>= 8; \ + } \ + POINTER_SHIFT(buf, sizeof(val)); \ + }) static inline void tPutE(void **buf, uint64_t val) { if (buf) { @@ -57,7 +61,7 @@ static void func(T t) { switch (t) { case A: for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) { - tPutA(pBuf, val); + pBuf = tPutA(pBuf, val); if (POINTER_DISTANCE(buf, pBuf) == 1024) { pBuf = buf; } @@ -65,7 +69,7 @@ static void func(T t) { break; case B: for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) { - tPutB(pBuf, val); + pBuf = tPutB(pBuf, val); if (POINTER_DISTANCE(buf, pBuf) == 1024) { pBuf = buf; } @@ -73,7 +77,7 @@ static void func(T t) { break; case C: for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) { - tPutC(pBuf, val); + pBuf = tPutC(pBuf, val); if (POINTER_DISTANCE(buf, pBuf) == 1024) { pBuf = buf; } @@ -81,7 +85,7 @@ static void func(T t) { break; case D: for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) { - tPutD(pBuf, val); + pBuf = tPutD(pBuf, val); if (POINTER_DISTANCE(buf, pBuf) == 1024) { pBuf = buf; } diff --git a/example/src/tmq.c b/example/src/tmq.c index 2abf915fd8..b2cbf856c0 100644 --- a/example/src/tmq.c +++ b/example/src/tmq.c @@ -17,6 +17,7 @@ #include #include #include +#include #include "taos.h" static int running = 1; @@ -47,6 +48,7 @@ int32_t init_env() { return -1; } taos_free_result(pRes); + sleep(1); pRes = taos_query(pConn, "use abc1"); if (taos_errno(pRes) != 0) { @@ -58,6 +60,7 @@ int32_t init_env() { pRes = taos_query(pConn, "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(10)) tags(t1 int)"); if (taos_errno(pRes) != 0) { + assert(0); printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes)); return -1; } @@ -265,10 +268,11 @@ void perf_loop(tmq_t* tmq, tmq_list_t* topics) { } int main(int argc, char* argv[]) { - int code; if (argc > 1) { printf("env init\n"); - code = init_env(); + if (init_env() < 0) { + return -1; + } create_topic(); } tmq_t* tmq = build_consumer(); diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index a354605a21..71e2e4fba3 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -238,10 +238,16 @@ static FORCE_INLINE int32_t blockCompressColData(SColumnInfoData* pColRes, int32 static FORCE_INLINE void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, int8_t needCompress) { - int32_t* colSizes = (int32_t*)data; + int32_t* actualLen = (int32_t*) data; + data += sizeof(int32_t); + uint64_t* groupId = (uint64_t*) data; + data += sizeof(uint64_t); + + int32_t* colSizes = (int32_t*)data; data += numOfCols * sizeof(int32_t); - *dataLen = (numOfCols * sizeof(int32_t)); + + *dataLen = (numOfCols * sizeof(int32_t) + sizeof(uint64_t) + sizeof(int32_t)); int32_t numOfRows = pBlock->info.rows; for (int32_t col = 0; col < numOfCols; ++col) { @@ -273,6 +279,9 @@ static FORCE_INLINE void blockCompressEncode(const SSDataBlock* pBlock, char* da colSizes[col] = htonl(colSizes[col]); } + + *actualLen = *dataLen; + *groupId = pBlock->info.groupId; } #ifdef __cplusplus diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 757fa8e74b..5a2dd502c1 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -1314,7 +1314,6 @@ typedef struct { } SMqConsumerLostMsg; typedef struct { - int32_t topicNum; int64_t consumerId; char cgroup[TSDB_CGROUP_LEN]; SArray* topicNames; // SArray @@ -1322,22 +1321,27 @@ typedef struct { static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubscribeReq* pReq) { int32_t tlen = 0; - tlen += taosEncodeFixedI32(buf, pReq->topicNum); tlen += taosEncodeFixedI64(buf, pReq->consumerId); tlen += taosEncodeString(buf, pReq->cgroup); - for (int32_t i = 0; i < pReq->topicNum; i++) { + int32_t topicNum = taosArrayGetSize(pReq->topicNames); + tlen += taosEncodeFixedI32(buf, topicNum); + + for (int32_t i = 0; i < topicNum; i++) { tlen += taosEncodeString(buf, (char*)taosArrayGetP(pReq->topicNames, i)); } return tlen; } static FORCE_INLINE void* tDeserializeSCMSubscribeReq(void* buf, SCMSubscribeReq* pReq) { - buf = taosDecodeFixedI32(buf, &pReq->topicNum); buf = taosDecodeFixedI64(buf, &pReq->consumerId); buf = taosDecodeStringTo(buf, pReq->cgroup); - pReq->topicNames = taosArrayInit(pReq->topicNum, sizeof(void*)); - for (int32_t i = 0; i < pReq->topicNum; i++) { + + int32_t topicNum; + buf = taosDecodeFixedI32(buf, &topicNum); + + pReq->topicNames = taosArrayInit(topicNum, sizeof(void*)); + for (int32_t i = 0; i < topicNum; i++) { char* name; buf = taosDecodeString(buf, &name); taosArrayPush(pReq->topicNames, &name); @@ -1969,7 +1973,6 @@ typedef struct { int8_t withTbName; int8_t withSchema; int8_t withTag; - int8_t withTagSchema; char* qmsg; } SMqRebVgReq; @@ -1984,7 +1987,6 @@ static FORCE_INLINE int32_t tEncodeSMqRebVgReq(void** buf, const SMqRebVgReq* pR tlen += taosEncodeFixedI8(buf, pReq->withTbName); tlen += taosEncodeFixedI8(buf, pReq->withSchema); tlen += taosEncodeFixedI8(buf, pReq->withTag); - tlen += taosEncodeFixedI8(buf, pReq->withTagSchema); if (pReq->subType == TOPIC_SUB_TYPE__TABLE) { tlen += taosEncodeString(buf, pReq->qmsg); } @@ -2001,7 +2003,6 @@ static FORCE_INLINE void* tDecodeSMqRebVgReq(const void* buf, SMqRebVgReq* pReq) buf = taosDecodeFixedI8(buf, &pReq->withTbName); buf = taosDecodeFixedI8(buf, &pReq->withSchema); buf = taosDecodeFixedI8(buf, &pReq->withTag); - buf = taosDecodeFixedI8(buf, &pReq->withTagSchema); if (pReq->subType == TOPIC_SUB_TYPE__TABLE) { buf = taosDecodeString(buf, &pReq->qmsg); } @@ -2590,7 +2591,6 @@ typedef struct { int8_t withTbName; int8_t withSchema; int8_t withTag; - int8_t withTagSchema; SArray* blockDataLen; // SArray SArray* blockData; // SArray SArray* blockTbName; // SArray @@ -2609,7 +2609,6 @@ static FORCE_INLINE int32_t tEncodeSMqDataBlkRsp(void** buf, const SMqDataBlkRsp tlen += taosEncodeFixedI8(buf, pRsp->withTbName); tlen += taosEncodeFixedI8(buf, pRsp->withSchema); tlen += taosEncodeFixedI8(buf, pRsp->withTag); - tlen += taosEncodeFixedI8(buf, pRsp->withTagSchema); for (int32_t i = 0; i < pRsp->blockNum; i++) { int32_t bLen = *(int32_t*)taosArrayGet(pRsp->blockDataLen, i); @@ -2632,7 +2631,6 @@ static FORCE_INLINE void* tDecodeSMqDataBlkRsp(const void* buf, SMqDataBlkRsp* p buf = taosDecodeFixedI8(buf, &pRsp->withTbName); buf = taosDecodeFixedI8(buf, &pRsp->withSchema); buf = taosDecodeFixedI8(buf, &pRsp->withTag); - buf = taosDecodeFixedI8(buf, &pRsp->withTagSchema); for (int32_t i = 0; i < pRsp->blockNum; i++) { int32_t bLen = 0; diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index b88afcb39d..30d1bd0a51 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -51,7 +51,7 @@ typedef struct SMetaData { SArray *pTableMeta; // STableMeta array SArray *pVgroupInfo; // SVgroupInfo list SArray *pUdfList; // udf info list - SArray *pEpSetList; // qnode epset list, SArray + SArray *pQnodeList; // qnode list, SArray } SMetaData; typedef struct SCatalogCfg { diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 004d834287..ef32533e5f 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -37,7 +37,7 @@ typedef struct SFuncExecEnv { typedef bool (*FExecGetEnv)(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); typedef bool (*FExecInit)(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); typedef int32_t (*FExecProcess)(struct SqlFunctionCtx *pCtx); -typedef int32_t (*FExecFinalize)(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock, int32_t slotId); +typedef int32_t (*FExecFinalize)(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock); typedef int32_t (*FScalarExecProcess)(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); typedef struct SScalarFuncExecFuncs { @@ -141,8 +141,7 @@ struct SResultRowEntryInfo; //for selectivity query, the corresponding tag value is assigned if the data is qualified typedef struct SSubsidiaryResInfo { - int16_t bufLen; // keep the tags data for top/bottom query result - int16_t numOfCols; + int16_t num; struct SqlFunctionCtx **pCtx; } SSubsidiaryResInfo; @@ -187,8 +186,8 @@ typedef struct SqlFunctionCtx { uint8_t currentStage; // record current running step, default: 0 bool isAggSet; int64_t startTs; // timestamp range of current query when function is executed on a specific data block, TODO remove it - ///////////////////////////////////////////////////////////////// bool stableQuery; + ///////////////////////////////////////////////////////////////// int16_t functionId; // function id char * pOutput; // final result output buffer, point to sdata->data int32_t numOfParams; @@ -198,11 +197,15 @@ typedef struct SqlFunctionCtx { int32_t offset; SVariant tag; struct SResultRowEntryInfo *resultInfo; - SSubsidiaryResInfo subsidiaryRes; - SPoint1 start; - SPoint1 end; - SFuncExecFuncs fpSet; - SScalarFuncExecFuncs sfp; + SSubsidiaryResInfo subsidiaries; + SPoint1 start; + SPoint1 end; + SFuncExecFuncs fpSet; + SScalarFuncExecFuncs sfp; + SExprInfo *pExpr; + struct SDiskbasedBuf *pBuf; + struct SSDataBlock *pSrcBlock; + int32_t curBufPage; } SqlFunctionCtx; enum { @@ -319,6 +322,20 @@ struct SUdfInfo; void qAddUdfInfo(uint64_t id, struct SUdfInfo* pUdfInfo); void qRemoveUdfInfo(uint64_t id, struct SUdfInfo* pUdfInfo); +/** + * create udfd proxy, called once in process that call setupUdf/callUdfxxx/teardownUdf + * @return error code + */ +int32_t udfcOpen(); + +/** + * destroy udfd proxy + * @return error code + */ +int32_t udfcClose(); + +typedef void *UdfcFuncHandle; + #ifdef __cplusplus } #endif diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index d41e797536..7b0d70b769 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -192,7 +192,13 @@ int32_t walEndSnapshot(SWal *); SWalReadHandle *walOpenReadHandle(SWal *); void walCloseReadHandle(SWalReadHandle *); int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver); -int32_t walReadWithHandle_s(SWalReadHandle *pRead, int64_t ver, SWalReadHead **ppHead); + +// only for tq usage +// int32_t walReadWithHandle_s(SWalReadHandle *pRead, int64_t ver, SWalReadHead **ppHead); +void walSetReaderCapacity(SWalReadHandle *pRead, int32_t capacity); +int32_t walFetchHead(SWalReadHandle *pRead, int64_t ver, SWalHead *pHead); +int32_t walFetchBody(SWalReadHandle *pRead, SWalHead **ppHead); +int32_t walSkipFetchBody(SWalReadHandle *pRead, const SWalHead *pHead); // deprecated #if 0 diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 3ac440346c..2545b33574 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -252,7 +252,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_COLUMN_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03AE) // mnode-infoSchema -#define TSDB_CODE_MND_INVALID_INFOS_TBL TAOS_DEF_ERROR_CODE(0, 0x03B0) +#define TSDB_CODE_MND_INVALID_SYS_TABLENAME TAOS_DEF_ERROR_CODE(0, 0x03B0) // mnode-func #define TSDB_CODE_MND_FUNC_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03C0) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 367baef53f..2ae9a11803 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -105,9 +105,9 @@ TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass, epSet.epSet.eps[0].port = port; } - char* key = getClusterKey(user, secretEncrypt, ip, port); - SAppInstInfo** pInst = NULL; + char* key = getClusterKey(user, secretEncrypt, ip, port); + SAppInstInfo** pInst = NULL; taosThreadMutexLock(&appInfo.mutex); pInst = taosHashGet(appInfo.pInstMap, key, strlen(key)); @@ -840,10 +840,21 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32 return code; } - int32_t* colLength = (int32_t*)pResultInfo->pData; - char* pStart = ((char*)pResultInfo->pData) + sizeof(int32_t) * numOfCols; + char* p = (char*) pResultInfo->pData; + + int32_t dataLen = *(int32_t*) p; + p += sizeof(int32_t); + + uint64_t groupId = *(uint64_t*) p; + p += sizeof(uint64_t); + + int32_t* colLength = (int32_t*)p; + p += sizeof(int32_t) * numOfCols; + + char* pStart = p; for (int32_t i = 0; i < numOfCols; ++i) { colLength[i] = htonl(colLength[i]); + ASSERT(colLength[i] < dataLen); if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) { pResultInfo->pCol[i].offset = (int32_t*)pStart; diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index a33351373e..8096ce395a 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -13,13 +13,13 @@ * along with this program. If not, see . */ -#include "os.h" -#include "tdef.h" -#include "tname.h" +#include "catalog.h" #include "clientInt.h" #include "clientLog.h" -#include "catalog.h" +#include "os.h" #include "query.h" +#include "tdef.h" +#include "tname.h" int32_t (*handleRequestRspFp[TDMT_MAX])(void*, const SDataBuf* pMsg, int32_t code); @@ -50,7 +50,13 @@ int32_t processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) { SConnectRsp connectRsp = {0}; tDeserializeSConnectRsp(pMsg->pData, pMsg->len, &connectRsp); - assert(connectRsp.epSet.numOfEps > 0); + /*assert(connectRsp.epSet.numOfEps > 0);*/ + if (connectRsp.epSet.numOfEps == 0) { + taosMemoryFree(pMsg->pData); + setErrno(pRequest, TSDB_CODE_MND_APP_ERROR); + tsem_post(&pRequest->body.rspSem); + return code; + } if (!isEpsetEqual(&pTscObj->pAppInfo->mgmtEp.epSet, &connectRsp.epSet)) { updateEpSet_s(&pTscObj->pAppInfo->mgmtEp, &connectRsp.epSet); @@ -82,18 +88,20 @@ int32_t processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) { return 0; } -SMsgSendInfo* buildMsgInfoImpl(SRequestObj *pRequest) { +SMsgSendInfo* buildMsgInfoImpl(SRequestObj* pRequest) { SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); pMsgSendInfo->requestObjRefId = pRequest->self; - pMsgSendInfo->requestId = pRequest->requestId; - pMsgSendInfo->param = pRequest; - pMsgSendInfo->msgType = pRequest->type; + pMsgSendInfo->requestId = pRequest->requestId; + pMsgSendInfo->param = pRequest; + pMsgSendInfo->msgType = pRequest->type; assert(pRequest != NULL); pMsgSendInfo->msgInfo = pRequest->body.requestMsg; - pMsgSendInfo->fp = (handleRequestRspFp[TMSG_INDEX(pRequest->type)] == NULL)? genericRspCallback:handleRequestRspFp[TMSG_INDEX(pRequest->type)]; + pMsgSendInfo->fp = (handleRequestRspFp[TMSG_INDEX(pRequest->type)] == NULL) + ? genericRspCallback + : handleRequestRspFp[TMSG_INDEX(pRequest->type)]; return pMsgSendInfo; } @@ -114,7 +122,7 @@ int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { if (TSDB_CODE_MND_DB_NOT_EXIST == code) { SUseDbRsp usedbRsp = {0}; tDeserializeSUseDbRsp(pMsg->pData, pMsg->len, &usedbRsp); - struct SCatalog *pCatalog = NULL; + struct SCatalog* pCatalog = NULL; if (usedbRsp.vgVersion >= 0) { int32_t code1 = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 699fa7ecf5..d576f28cb4 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -382,12 +382,9 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { pTmq->pTscObj = taos_connect_internal(conf->ip, user, pass, NULL, conf->db, conf->port, CONN_TYPE__TMQ); if (pTmq->pTscObj == NULL) return NULL; - /*pTmq->inWaiting = 0;*/ pTmq->status = 0; pTmq->pollCnt = 0; pTmq->epoch = 0; - /*pTmq->waitingRequest = 0;*/ - /*pTmq->readyRequest = 0;*/ pTmq->epStatus = 0; pTmq->epSkipCnt = 0; // set conf @@ -509,7 +506,6 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) { tmq->clientTopics = taosArrayInit(sz, sizeof(SMqClientTopic)); SCMSubscribeReq req; - req.topicNum = sz; req.consumerId = tmq->consumerId; strcpy(req.cgroup, tmq->groupId); req.topicNames = taosArrayInit(sz, sizeof(void*)); @@ -519,12 +515,16 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) { char* topicName = taosArrayGetP(container, i); SName name = {0}; +#if 0 char* dbName = getDbOfConnection(tmq->pTscObj); if (dbName == NULL) { return TMQ_RESP_ERR__FAIL; } - tNameSetDbName(&name, tmq->pTscObj->acctId, dbName, strlen(dbName)); +#endif + tNameSetDbName(&name, tmq->pTscObj->acctId, topicName, strlen(topicName)); +#if 0 tNameFromString(&name, topicName, T_NAME_TABLE); +#endif char* topicFname = taosMemoryCalloc(1, TSDB_TOPIC_FNAME_LEN); if (topicFname == NULL) { @@ -542,7 +542,9 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) { topic.vgs = taosArrayInit(0, sizeof(SMqClientVg)); taosArrayPush(tmq->clientTopics, &topic); taosArrayPush(req.topicNames, &topicFname); +#if 0 taosMemoryFree(dbName); +#endif } int tlen = tSerializeSCMSubscribeReq(NULL, &req); diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 800bcd70ce..09452b584d 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -644,12 +644,12 @@ size_t blockDataGetRowSize(SSDataBlock* pBlock) { /** * @refitem blockDataToBuf for the meta size - * * @param pBlock * @return */ size_t blockDataGetSerialMetaSize(const SSDataBlock* pBlock) { - return sizeof(int32_t) + pBlock->info.numOfCols * sizeof(int32_t); + // | total rows/total length | block group id | each column length | + return sizeof(int32_t) + sizeof(uint64_t) + pBlock->info.numOfCols * sizeof(int32_t); } double blockDataGetSerialRowSize(const SSDataBlock* pBlock) { @@ -1219,7 +1219,6 @@ void colDataDestroy(SColumnInfoData* pColData) { taosMemoryFree(pColData->pData); } - static void doShiftBitmap(char* nullBitmap, size_t n, size_t total) { int32_t len = BitmapLen(total); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index eb7c9a763f..efd4cbfcec 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -126,7 +126,6 @@ int32_t tDecodeSQueryNodeAddr(SCoder *pDecoder, SQueryNodeAddr *pAddr) { return 0; } - int32_t taosEncodeSEpSet(void **buf, const SEpSet *pEp) { int32_t tlen = 0; tlen += taosEncodeFixedI8(buf, pEp->inUse); @@ -1568,13 +1567,8 @@ int32_t tSerializeSCreateFuncReq(void *buf, int32_t bufLen, SCreateFuncReq *pReq if (tEncodeI32(&encoder, pReq->codeLen) < 0) return -1; if (tEncodeI64(&encoder, pReq->signature) < 0) return -1; - int32_t codeSize = 0; if (pReq->pCode != NULL) { - codeSize = strlen(pReq->pCode) + 1; - } - if (tEncodeI32(&encoder, codeSize) < 0) return -1; - if (pReq->pCode != NULL) { - if (tEncodeCStr(&encoder, pReq->pCode) < 0) return -1; + if (tEncodeBinary(&encoder, pReq->pCode, pReq->codeLen) < 0) return -1; } int32_t commentSize = 0; @@ -1608,10 +1602,8 @@ int32_t tDeserializeSCreateFuncReq(void *buf, int32_t bufLen, SCreateFuncReq *pR if (tDecodeI32(&decoder, &pReq->codeLen) < 0) return -1; if (tDecodeI64(&decoder, &pReq->signature) < 0) return -1; - int32_t codeSize = 0; - if (tDecodeI32(&decoder, &codeSize) < 0) return -1; - if (codeSize > 0) { - pReq->pCode = taosMemoryCalloc(1, codeSize); + if (pReq->codeLen > 0) { + pReq->pCode = taosMemoryCalloc(1, pReq->codeLen); if (pReq->pCode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -1734,7 +1726,7 @@ int32_t tSerializeSRetrieveFuncRsp(void *buf, int32_t bufLen, SRetrieveFuncRsp * if (tEncodeI32(&encoder, pInfo->codeSize) < 0) return -1; if (tEncodeI32(&encoder, pInfo->commentSize) < 0) return -1; if (pInfo->codeSize) { - if (tEncodeCStr(&encoder, pInfo->pCode) < 0) return -1; + if (tEncodeBinary(&encoder, pInfo->pCode, pInfo->codeSize) < 0) return -1; } if (pInfo->commentSize) { if (tEncodeCStr(&encoder, pInfo->pComment) < 0) return -1; @@ -2091,10 +2083,15 @@ int32_t tDeserializeSQnodeListRsp(void *buf, int32_t bufLen, SQnodeListRsp *pRsp if (tStartDecode(&decoder) < 0) return -1; int32_t num = 0; if (tDecodeI32(&decoder, &num) < 0) return -1; - pRsp->addrsList = taosArrayInit(num, sizeof(SQueryNodeAddr)); - if (NULL == pRsp->addrsList) return -1; + if (NULL == pRsp->addrsList) { + pRsp->addrsList = taosArrayInit(num, sizeof(SQueryNodeAddr)); + if (NULL == pRsp->addrsList) return -1; + } + for (int32_t i = 0; i < num; ++i) { - if (tDecodeSQueryNodeAddr(&decoder, TARRAY_GET_ELEM(pRsp->addrsList, i)) < 0) return -1; + SQueryNodeAddr addr = {0}; + if (tDecodeSQueryNodeAddr(&decoder, &addr) < 0) return -1; + taosArrayPush(pRsp->addrsList, &addr); } tEndDecode(&decoder); @@ -2747,11 +2744,11 @@ int32_t tSerializeSCMCreateTopicReq(void *buf, int32_t bufLen, const SCMCreateTo if (tEncodeI8(&encoder, pReq->withTbName) < 0) return -1; if (tEncodeI8(&encoder, pReq->withSchema) < 0) return -1; if (tEncodeI8(&encoder, pReq->withTag) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->subscribeDbName) < 0) return -1; if (tEncodeI32(&encoder, sqlLen) < 0) return -1; if (tEncodeI32(&encoder, astLen) < 0) return -1; if (sqlLen > 0 && tEncodeCStr(&encoder, pReq->sql) < 0) return -1; if (astLen > 0 && tEncodeCStr(&encoder, pReq->ast) < 0) return -1; - if (0 == astLen && tEncodeCStr(&encoder, pReq->subscribeDbName) < 0) return -1; tEndEncode(&encoder); @@ -2773,6 +2770,7 @@ int32_t tDeserializeSCMCreateTopicReq(void *buf, int32_t bufLen, SCMCreateTopicR if (tDecodeI8(&decoder, &pReq->withTbName) < 0) return -1; if (tDecodeI8(&decoder, &pReq->withSchema) < 0) return -1; if (tDecodeI8(&decoder, &pReq->withTag) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->subscribeDbName) < 0) return -1; if (tDecodeI32(&decoder, &sqlLen) < 0) return -1; if (tDecodeI32(&decoder, &astLen) < 0) return -1; @@ -2787,7 +2785,6 @@ int32_t tDeserializeSCMCreateTopicReq(void *buf, int32_t bufLen, SCMCreateTopicR if (pReq->ast == NULL) return -1; if (tDecodeCStrTo(&decoder, pReq->ast) < 0) return -1; } else { - if (tDecodeCStrTo(&decoder, pReq->subscribeDbName) < 0) return -1; } tEndDecode(&decoder); diff --git a/source/dnode/mgmt/implement/src/dmHandle.c b/source/dnode/mgmt/implement/src/dmHandle.c index e76d7fcd4f..ca1b943fb2 100644 --- a/source/dnode/mgmt/implement/src/dmHandle.c +++ b/source/dnode/mgmt/implement/src/dmHandle.c @@ -311,6 +311,9 @@ static void dmWatchUdfd(void *args) { } static int32_t dmStartUdfd(SDnode *pDnode) { + char dnodeId[8] = {0}; + snprintf(dnodeId, sizeof(dnodeId), "%d", pDnode->data.dnodeId); + uv_os_setenv("DNODE_ID", dnodeId); SUdfdData *pData = &pDnode->udfdData; if (pData->startCalled) { dInfo("dnode-mgmt start udfd already called"); @@ -320,8 +323,17 @@ static int32_t dmStartUdfd(SDnode *pDnode) { uv_barrier_init(&pData->barrier, 2); uv_thread_create(&pData->thread, dmWatchUdfd, pDnode); uv_barrier_wait(&pData->barrier); - pData->needCleanUp = true; - return pData->spawnErr; + int32_t err = atomic_load_32(&pData->spawnErr); + if (err != 0) { + uv_barrier_destroy(&pData->barrier); + uv_async_send(&pData->stopAsync); + uv_thread_join(&pData->thread); + pData->needCleanUp = false; + dInfo("dnode-mgmt udfd cleaned up after spawn err"); + } else { + pData->needCleanUp = true; + } + return err; } static int32_t dmStopUdfd(SDnode *pDnode) { @@ -336,7 +348,7 @@ static int32_t dmStopUdfd(SDnode *pDnode) { uv_barrier_destroy(&pData->barrier); uv_async_send(&pData->stopAsync); uv_thread_join(&pData->thread); - + dInfo("dnode-mgmt udfd cleaned up"); return 0; } @@ -371,9 +383,9 @@ static int32_t dmInitMgmt(SMgmtWrapper *pWrapper) { } dmReportStartup(pDnode, "dnode-transport", "initialized"); -// if (dmStartUdfd(pDnode) != 0) { -// dError("failed to start udfd"); -// } + if (dmStartUdfd(pDnode) != 0) { + dError("failed to start udfd"); + } dInfo("dnode-mgmt is initialized"); return 0; diff --git a/source/dnode/mgmt/interface/src/dmInt.c b/source/dnode/mgmt/interface/src/dmInt.c index 491eff5e17..2d15a7a008 100644 --- a/source/dnode/mgmt/interface/src/dmInt.c +++ b/source/dnode/mgmt/interface/src/dmInt.c @@ -173,9 +173,13 @@ static void dmGetServerStatus(SDnode *pDnode, SServerStatusRsp *pStatus) { void dmProcessNettestReq(SDnode *pDnode, SRpcMsg *pRpc) { dDebug("net test req is received"); - SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = 0}; + SRpcMsg rsp = {.handle = pRpc->handle, .refId = pRpc->refId, .ahandle = pRpc->ahandle, .code = 0}; rsp.pCont = rpcMallocCont(pRpc->contLen); - rsp.contLen = pRpc->contLen; + if (rsp.pCont == NULL) { + rsp.code = TSDB_CODE_OUT_OF_MEMORY; + } else { + rsp.contLen = pRpc->contLen; + } rpcSendResponse(&rsp); } diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index b434cf102e..afe57e3d8f 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -177,6 +177,7 @@ void mmInitMsgHandle(SMgmtWrapper *pWrapper) { dmSetMsgHandle(pWrapper, TDMT_MND_DROP_MNODE, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_MND_CREATE_QNODE, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_MND_DROP_QNODE, mmProcessWriteMsg, DEFAULT_HANDLE); + dmSetMsgHandle(pWrapper, TDMT_MND_QNODE_LIST, mmProcessReadMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_MND_CREATE_SNODE, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_MND_DROP_SNODE, mmProcessWriteMsg, DEFAULT_HANDLE); dmSetMsgHandle(pWrapper, TDMT_MND_CREATE_BNODE, mmProcessWriteMsg, DEFAULT_HANDLE); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index e784272581..687a799c57 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "vmInt.h" +#include "libs/function/function.h" SVnodeObj *vmAcquireVnode(SVnodesMgmt *pMgmt, int32_t vgId) { SVnodeObj *pVnode = NULL; @@ -275,7 +276,7 @@ static void vmCleanup(SMgmtWrapper *pWrapper) { pWrapper->pMgmt = NULL; // syncCleanUp(); - + udfcClose(); dInfo("vnode-mgmt is cleaned up"); } @@ -339,6 +340,10 @@ static int32_t vmInit(SMgmtWrapper *pWrapper) { } dmReportStartup(pDnode, "vnode-vnodes", "initialized"); + if (udfcOpen() != 0) { + dError("failed to open udfc in dnode"); + } + code = 0; _OVER: diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index bf994e12a3..cf1cd58540 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -450,7 +450,6 @@ typedef struct { int8_t withTbName; int8_t withSchema; int8_t withTag; - int8_t withTagSchema; SRWLatch lock; int32_t sqlLen; int32_t astLen; @@ -517,7 +516,6 @@ typedef struct { int8_t withTbName; int8_t withSchema; int8_t withTag; - int8_t withTagSchema; SHashObj* consumerHash; // consumerId -> SMqConsumerEpInSub // TODO put -1 into unassignVgs // SArray* unassignedVgs; diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index 78d54d273d..800a013c4f 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -237,7 +237,6 @@ SMqSubscribeObj *tCloneSubscribeObj(const SMqSubscribeObj *pSub) { pSubNew->withTbName = pSub->withTbName; pSubNew->withSchema = pSub->withSchema; pSubNew->withTag = pSub->withTag; - pSubNew->withTagSchema = pSub->withTagSchema; pSubNew->vgNum = pSub->vgNum; pSubNew->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); @@ -270,7 +269,6 @@ int32_t tEncodeSubscribeObj(void **buf, const SMqSubscribeObj *pSub) { tlen += taosEncodeFixedI8(buf, pSub->withTbName); tlen += taosEncodeFixedI8(buf, pSub->withSchema); tlen += taosEncodeFixedI8(buf, pSub->withTag); - tlen += taosEncodeFixedI8(buf, pSub->withTagSchema); void *pIter = NULL; int32_t sz = taosHashGetSize(pSub->consumerHash); @@ -297,7 +295,6 @@ void *tDecodeSubscribeObj(const void *buf, SMqSubscribeObj *pSub) { buf = taosDecodeFixedI8(buf, &pSub->withTbName); buf = taosDecodeFixedI8(buf, &pSub->withSchema); buf = taosDecodeFixedI8(buf, &pSub->withTag); - buf = taosDecodeFixedI8(buf, &pSub->withTagSchema); int32_t sz; buf = taosDecodeFixedI32(buf, &sz); diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index 156d894a44..09d0587019 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -309,10 +309,10 @@ static int32_t mndProcessCreateFuncReq(SNodeMsg *pReq) { goto _OVER; } - if (createReq.pCode[0] == 0) { - terrno = TSDB_CODE_MND_INVALID_FUNC_CODE; - goto _OVER; - } + if (createReq.codeLen <= 1) { + terrno = TSDB_CODE_MND_INVALID_FUNC_CODE; + goto _OVER; + } if (createReq.bufSize <= 0 || createReq.bufSize > TSDB_FUNC_BUF_SIZE) { terrno = TSDB_CODE_MND_INVALID_FUNC_BUFSIZE; diff --git a/source/dnode/mnode/impl/src/mndInfoSchema.c b/source/dnode/mnode/impl/src/mndInfoSchema.c index 5dbbd27930..2b46fc9274 100644 --- a/source/dnode/mnode/impl/src/mndInfoSchema.c +++ b/source/dnode/mnode/impl/src/mndInfoSchema.c @@ -325,7 +325,7 @@ static int32_t mndInsInitMeta(SHashObj *hash) { return -1; } - if (taosHashPut(hash, meta.tbName, strlen(meta.tbName) + 1, &meta, sizeof(meta))) { + if (taosHashPut(hash, meta.tbName, strlen(meta.tbName), &meta, sizeof(meta))) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } @@ -340,10 +340,10 @@ int32_t mndBuildInsTableSchema(SMnode *pMnode, const char *dbFName, const char * return -1; } - STableMetaRsp *pMeta = taosHashGet(pMnode->infosMeta, tbName, strlen(tbName) + 1); + STableMetaRsp *pMeta = taosHashGet(pMnode->infosMeta, tbName, strlen(tbName)); if (NULL == pMeta) { mError("invalid information schema table name:%s", tbName); - terrno = TSDB_CODE_MND_INVALID_INFOS_TBL; + terrno = TSDB_CODE_MND_INVALID_SYS_TABLENAME; return -1; } diff --git a/source/dnode/mnode/impl/src/mndPerfSchema.c b/source/dnode/mnode/impl/src/mndPerfSchema.c index 737068a2dd..a0ecbe9ae4 100644 --- a/source/dnode/mnode/impl/src/mndPerfSchema.c +++ b/source/dnode/mnode/impl/src/mndPerfSchema.c @@ -128,7 +128,7 @@ int32_t mndBuildPerfsTableSchema(SMnode *pMnode, const char *dbFName, const char STableMetaRsp *meta = (STableMetaRsp *)taosHashGet(pMnode->perfsMeta, tbName, strlen(tbName)); if (NULL == meta) { mError("invalid performance schema table name:%s", tbName); - terrno = TSDB_CODE_MND_INVALID_INFOS_TBL; + terrno = TSDB_CODE_MND_INVALID_SYS_TABLENAME; return -1; } diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index 1c03ca30f4..cec5933ba3 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -451,8 +451,9 @@ static int32_t mndProcessQnodeListReq(SNodeMsg *pReq) { goto _OVER; } + void *pIter = NULL; while (1) { - void *pIter = sdbFetch(pSdb, SDB_QNODE, NULL, (void **)&pObj); + pIter = sdbFetch(pSdb, SDB_QNODE, pIter, (void **)&pObj); if (pIter == NULL) break; SQueryNodeAddr nodeAddr = {0}; @@ -472,7 +473,7 @@ static int32_t mndProcessQnodeListReq(SNodeMsg *pReq) { } int32_t rspLen = tSerializeSQnodeListRsp(NULL, 0, &qlistRsp); - void *pRsp = taosMemoryMalloc(rspLen); + void *pRsp = rpcMallocCont(rspLen); if (pRsp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; goto _OVER; diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index 1ab25f795f..94366a241d 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -100,6 +100,8 @@ static int32_t convertToRetrieveType(char* name, int32_t len) { type = TSDB_MGMT_TABLE_QUERIES; } else if (strncasecmp(name, TSDB_INS_TABLE_VNODES, len) == 0) { type = TSDB_MGMT_TABLE_VNODES; + } else if (strncasecmp(name, TSDB_PERFS_TABLE_TOPICS, len) == 0) { + type = TSDB_MGMT_TABLE_TOPICS; } else { // ASSERT(0); } @@ -187,11 +189,14 @@ static int32_t mndProcessRetrieveSysTableReq(SNodeMsg *pReq) { } if (retrieveReq.showId == 0) { - STableMetaRsp *pMeta = (STableMetaRsp *)taosHashGet(pMnode->infosMeta, retrieveReq.tb, strlen(retrieveReq.tb) + 1); + STableMetaRsp *pMeta = (STableMetaRsp *)taosHashGet(pMnode->infosMeta, retrieveReq.tb, strlen(retrieveReq.tb)); if (pMeta == NULL) { - terrno = TSDB_CODE_MND_INVALID_INFOS_TBL; - mError("failed to process show-retrieve req:%p since %s", pShow, terrstr()); - return -1; + pMeta = (STableMetaRsp *)taosHashGet(pMnode->perfsMeta, retrieveReq.tb, strlen(retrieveReq.tb)); + if (pMeta == NULL) { + terrno = TSDB_CODE_MND_INVALID_SYS_TABLENAME; + mError("failed to process show-retrieve req:%p since %s", pShow, terrstr()); + return -1; + } } pShow = mndCreateShowObj(pMnode, &retrieveReq); diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index e37bd60e12..6a1994d7b8 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -35,11 +35,6 @@ #define MND_SUBSCRIBE_REBALANCE_CNT 3 -enum { - MQ_SUBSCRIBE_STATUS__ACTIVE = 1, - MQ_SUBSCRIBE_STATUS__DELETED, -}; - static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *); static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw); static int32_t mndSubActionInsert(SSdb *pSdb, SMqSubscribeObj *); @@ -89,7 +84,6 @@ static SMqSubscribeObj *mndCreateSub(SMnode *pMnode, const SMqTopicObj *pTopic, pSub->withTbName = pTopic->withTbName; pSub->withSchema = pTopic->withSchema; pSub->withTag = pTopic->withTag; - pSub->withTagSchema = pTopic->withTagSchema; ASSERT(taosHashGetSize(pSub->consumerHash) == 1); @@ -115,7 +109,6 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, const SMqSubscri req.withTbName = pSub->withTbName; req.withSchema = pSub->withSchema; req.withTag = pSub->withTag; - req.withTagSchema = pSub->withTagSchema; strncpy(req.subKey, pSub->key, TSDB_SUBSCRIBE_KEY_LEN); int32_t tlen = sizeof(SMsgHead) + tEncodeSMqRebVgReq(NULL, &req); @@ -514,9 +507,11 @@ static int32_t mndProcessRebalanceReq(SNodeMsg *pMsg) { // TODO replace assert with error check ASSERT(mndDoRebalance(pMnode, &rebInput, &rebOutput) == 0); + // if add more consumer to balanced subscribe, // possibly no vg is changed /*ASSERT(taosArrayGetSize(rebOutput.rebVgs) != 0);*/ + ASSERT(mndPersistRebResult(pMnode, pMsg, &rebOutput) == 0); if (rebInput.pTopic) { @@ -673,177 +668,7 @@ void mndReleaseSubscribe(SMnode *pMnode, SMqSubscribeObj *pSub) { sdbRelease(pSdb, pSub); } -#if 0 -static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg) { - SMnode *pMnode = pMsg->pNode; - char *msgStr = pMsg->rpcMsg.pCont; - SCMSubscribeReq subscribe; - tDeserializeSCMSubscribeReq(msgStr, &subscribe); - int64_t consumerId = subscribe.consumerId; - char *cgroup = subscribe.consumerGroup; - - SArray *newSub = subscribe.topicNames; - int32_t newTopicNum = subscribe.topicNum; - - taosArraySortString(newSub, taosArrayCompareString); - - SArray *oldSub = NULL; - int32_t oldTopicNum = 0; - bool createConsumer = false; - // create consumer if not exist - SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, consumerId); - if (pConsumer == NULL) { - // create consumer - pConsumer = mndCreateConsumer(consumerId, cgroup); - createConsumer = true; - } else { - pConsumer->epoch++; - oldSub = pConsumer->currentTopics; - } - pConsumer->currentTopics = newSub; - - if (oldSub != NULL) { - oldTopicNum = taosArrayGetSize(oldSub); - } - - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_SUBSCRIBE, &pMsg->rpcMsg); - if (pTrans == NULL) { - // TODO: free memory - return -1; - } - - int32_t i = 0, j = 0; - while (i < newTopicNum || j < oldTopicNum) { - char *newTopicName = NULL; - char *oldTopicName = NULL; - if (i >= newTopicNum) { - // encode unset topic msg to all vnodes related to that topic - oldTopicName = taosArrayGetP(oldSub, j); - j++; - } else if (j >= oldTopicNum) { - newTopicName = taosArrayGetP(newSub, i); - i++; - } else { - newTopicName = taosArrayGetP(newSub, i); - oldTopicName = taosArrayGetP(oldSub, j); - - int32_t comp = compareLenPrefixedStr(newTopicName, oldTopicName); - if (comp == 0) { - // do nothing - oldTopicName = newTopicName = NULL; - i++; - j++; - continue; - } else if (comp < 0) { - oldTopicName = NULL; - i++; - } else { - newTopicName = NULL; - j++; - } - } - - if (oldTopicName != NULL) { - ASSERT(newTopicName == NULL); - - // cancel subscribe of old topic - SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, cgroup, oldTopicName); - ASSERT(pSub); - int32_t csz = taosArrayGetSize(pSub->consumers); - for (int32_t ci = 0; ci < csz; ci++) { - SMqSubConsumer *pSubConsumer = taosArrayGet(pSub->consumers, ci); - if (pSubConsumer->consumerId == consumerId) { - int32_t vgsz = taosArrayGetSize(pSubConsumer->vgInfo); - for (int32_t vgi = 0; vgi < vgsz; vgi++) { - SMqConsumerEp *pConsumerEp = taosArrayGet(pSubConsumer->vgInfo, vgi); - mndPersistCancelConnReq(pMnode, pTrans, pConsumerEp, oldTopicName); - taosArrayPush(pSub->unassignedVg, pConsumerEp); - } - taosArrayRemove(pSub->consumers, ci); - break; - } - } - char *oldTopicNameDup = strdup(oldTopicName); - taosArrayPush(pConsumer->recentRemovedTopics, &oldTopicNameDup); - atomic_store_32(&pConsumer->status, MQ_CONSUMER_STATUS__MODIFY); - /*pSub->status = MQ_SUBSCRIBE_STATUS__DELETED;*/ - } else if (newTopicName != NULL) { - ASSERT(oldTopicName == NULL); - - SMqTopicObj *pTopic = mndAcquireTopic(pMnode, newTopicName); - if (pTopic == NULL) { - mError("topic being subscribed not exist: %s", newTopicName); - continue; - } - - SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, cgroup, newTopicName); - bool createSub = false; - if (pSub == NULL) { - mDebug("create new subscription by consumer %" PRId64 ", group: %s, topic %s", consumerId, cgroup, - newTopicName); - pSub = mndCreateSubscription(pMnode, pTopic, cgroup); - createSub = true; - - mndCreateOffset(pTrans, cgroup, newTopicName, pSub->unassignedVg); - } - - SMqSubConsumer mqSubConsumer; - mqSubConsumer.consumerId = consumerId; - mqSubConsumer.vgInfo = taosArrayInit(0, sizeof(SMqConsumerEp)); - taosArrayPush(pSub->consumers, &mqSubConsumer); - - // if have un assigned vg, assign one to the consumer - if (taosArrayGetSize(pSub->unassignedVg) > 0) { - SMqConsumerEp *pConsumerEp = taosArrayPop(pSub->unassignedVg); - pConsumerEp->oldConsumerId = pConsumerEp->consumerId; - pConsumerEp->consumerId = consumerId; - taosArrayPush(mqSubConsumer.vgInfo, pConsumerEp); - if (pConsumerEp->oldConsumerId == -1) { - mInfo("mq set conn: assign vgroup %d of topic %s to consumer %" PRId64 "", pConsumerEp->vgId, newTopicName, - pConsumerEp->consumerId); - mndPersistMqSetConnReq(pMnode, pTrans, pTopic, cgroup, pConsumerEp); - } else { - mndPersistRebalanceMsg(pMnode, pTrans, pConsumerEp, newTopicName); - } - // to trigger rebalance at once, do not set status active - /*atomic_store_32(&pConsumer->status, MQ_CONSUMER_STATUS__ACTIVE);*/ - } - - SSdbRaw *pRaw = mndSubActionEncode(pSub); - sdbSetRawStatus(pRaw, SDB_STATUS_READY); - mndTransAppendRedolog(pTrans, pRaw); - - if (!createSub) mndReleaseSubscribe(pMnode, pSub); - mndReleaseTopic(pMnode, pTopic); - } - } - - /*if (oldSub) taosArrayDestroyEx(oldSub, (void (*)(void *))taosMemoryFree);*/ - - // persist consumerObj - SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pConsumer); - sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY); - mndTransAppendRedolog(pTrans, pConsumerRaw); - - if (mndTransPrepare(pMnode, pTrans) != 0) { - mError("mq-subscribe-trans:%d, failed to prepare since %s", pTrans->id, terrstr()); - mndTransDrop(pTrans); - if (!createConsumer) mndReleaseConsumer(pMnode, pConsumer); - return -1; - } - - mndTransDrop(pTrans); - if (!createConsumer) mndReleaseConsumer(pMnode, pConsumer); - return TSDB_CODE_MND_ACTION_IN_PROGRESS; -} -#endif - static int32_t mndProcessSubscribeInternalRsp(SNodeMsg *pRsp) { mndTransProcessRsp(pRsp); return 0; } - -static void mndCancelGetNextConsumer(SMnode *pMnode, void *pIter) { - SSdb *pSdb = pMnode->pSdb; - sdbCancelFetch(pSdb, pIter); -} diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index a4b98ba01a..2516c43860 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -82,7 +82,6 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) { SDB_SET_INT8(pRaw, dataPos, pTopic->withTbName, TOPIC_ENCODE_OVER); SDB_SET_INT8(pRaw, dataPos, pTopic->withSchema, TOPIC_ENCODE_OVER); SDB_SET_INT8(pRaw, dataPos, pTopic->withTag, TOPIC_ENCODE_OVER); - SDB_SET_INT8(pRaw, dataPos, pTopic->withTagSchema, TOPIC_ENCODE_OVER); SDB_SET_INT32(pRaw, dataPos, pTopic->sqlLen, TOPIC_ENCODE_OVER); SDB_SET_BINARY(pRaw, dataPos, pTopic->sql, pTopic->sqlLen, TOPIC_ENCODE_OVER); SDB_SET_INT32(pRaw, dataPos, pTopic->astLen, TOPIC_ENCODE_OVER); @@ -146,7 +145,6 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { SDB_GET_INT8(pRaw, dataPos, &pTopic->withTbName, TOPIC_DECODE_OVER); SDB_GET_INT8(pRaw, dataPos, &pTopic->withSchema, TOPIC_DECODE_OVER); SDB_GET_INT8(pRaw, dataPos, &pTopic->withTag, TOPIC_DECODE_OVER); - SDB_GET_INT8(pRaw, dataPos, &pTopic->withTagSchema, TOPIC_DECODE_OVER); SDB_GET_INT32(pRaw, dataPos, &pTopic->sqlLen, TOPIC_DECODE_OVER); pTopic->sql = taosMemoryCalloc(pTopic->sqlLen, sizeof(char)); @@ -234,6 +232,7 @@ void mndReleaseTopic(SMnode *pMnode, SMqTopicObj *pTopic) { sdbRelease(pSdb, pTopic); } +#if 0 static SDbObj *mndAcquireDbByTopic(SMnode *pMnode, char *topicName) { SName name = {0}; tNameFromString(&name, topicName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); @@ -243,6 +242,7 @@ static SDbObj *mndAcquireDbByTopic(SMnode *pMnode, char *topicName) { return mndAcquireDb(pMnode, db); } +#endif static SDDropTopicReq *mndBuildDropTopicMsg(SMnode *pMnode, SVgObj *pVgroup, SMqTopicObj *pTopic) { int32_t contLen = sizeof(SDDropTopicReq); @@ -262,15 +262,11 @@ static SDDropTopicReq *mndBuildDropTopicMsg(SMnode *pMnode, SVgObj *pVgroup, SMq } static int32_t mndCheckCreateTopicReq(SCMCreateTopicReq *pCreate) { - if (pCreate->name[0] == 0 || pCreate->sql == NULL || pCreate->sql[0] == 0) { + if (pCreate->name[0] == 0 || pCreate->sql == NULL || pCreate->sql[0] == 0 || pCreate->subscribeDbName[0] == 0) { terrno = TSDB_CODE_MND_INVALID_TOPIC_OPTION; return -1; } - if ((pCreate->ast == NULL || pCreate->ast[0] == 0) && pCreate->subscribeDbName[0] == 0) { - terrno = TSDB_CODE_MND_INVALID_TOPIC_OPTION; - return -1; - } return 0; } @@ -386,7 +382,7 @@ static int32_t mndProcessCreateTopicReq(SNodeMsg *pReq) { goto CREATE_TOPIC_OVER; } - pDb = mndAcquireDbByTopic(pMnode, createTopicReq.name); + pDb = mndAcquireDb(pMnode, createTopicReq.subscribeDbName); if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_SELECTED; goto CREATE_TOPIC_OVER; @@ -524,8 +520,11 @@ static int32_t mndRetrieveTopic(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pB int32_t cols = 0; char topicName[TSDB_TOPIC_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - tstrncpy(&topicName[VARSTR_HEADER_SIZE], pTopic->name, TSDB_TOPIC_NAME_LEN); - varDataSetLen(topicName, strlen(&topicName[VARSTR_HEADER_SIZE])); + + SName n; + tNameFromString(&n, pTopic->name, T_NAME_ACCT|T_NAME_DB); + tNameGetDbName(&n, varDataVal(topicName)); + varDataSetLen(topicName, strlen(varDataVal(topicName))); SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)topicName, false); @@ -539,7 +538,7 @@ static int32_t mndRetrieveTopic(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pB varDataSetLen(sql, strlen(&sql[VARSTR_HEADER_SIZE])); colDataAppend(pColInfo, numOfRows, (const char *)sql, false); - taosMemoryFree(sql); +// taosMemoryFree(sql); numOfRows++; sdbRelease(pSdb, pTopic); diff --git a/source/dnode/mnode/impl/test/func/func.cpp b/source/dnode/mnode/impl/test/func/func.cpp index 4db9411e87..0473fa375e 100644 --- a/source/dnode/mnode/impl/test/func/func.cpp +++ b/source/dnode/mnode/impl/test/func/func.cpp @@ -22,17 +22,16 @@ class MndTestFunc : public ::testing::Test { void SetUp() override {} void TearDown() override {} - void SetCode(SCreateFuncReq* pReq, const char* pCode); + void SetCode(SCreateFuncReq* pReq, const char* pCode, int32_t size); void SetComment(SCreateFuncReq* pReq, const char* pComment); }; Testbase MndTestFunc::test; -void MndTestFunc::SetCode(SCreateFuncReq* pReq, const char* pCode) { - int32_t len = strlen(pCode); - pReq->pCode = (char*)taosMemoryCalloc(1, len + 1); - strcpy(pReq->pCode, pCode); - pReq->codeLen = len; +void MndTestFunc::SetCode(SCreateFuncReq *pReq, const char *pCode, int32_t size) { + pReq->pCode = (char*)taosMemoryMalloc(size); + memcpy(pReq->pCode, pCode, size); + pReq->codeLen = size; } void MndTestFunc::SetComment(SCreateFuncReq* pReq, const char* pComment) { @@ -79,7 +78,7 @@ TEST_F(MndTestFunc, 02_Create_Func) { { SCreateFuncReq createReq = {0}; strcpy(createReq.name, "f1"); - SetCode(&createReq, ""); + SetCode(&createReq, "", 1); SetComment(&createReq, "comment1"); int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); @@ -95,7 +94,7 @@ TEST_F(MndTestFunc, 02_Create_Func) { { SCreateFuncReq createReq = {0}; strcpy(createReq.name, "f1"); - SetCode(&createReq, "code1"); + SetCode(&createReq, "code1", 6); SetComment(&createReq, "comment1"); int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); @@ -111,7 +110,7 @@ TEST_F(MndTestFunc, 02_Create_Func) { { SCreateFuncReq createReq = {0}; strcpy(createReq.name, "f1"); - SetCode(&createReq, "code1"); + SetCode(&createReq, "code1", 6); SetComment(&createReq, "comment1"); createReq.bufSize = TSDB_FUNC_BUF_SIZE + 1; @@ -128,7 +127,7 @@ TEST_F(MndTestFunc, 02_Create_Func) { for (int32_t i = 0; i < 3; ++i) { SCreateFuncReq createReq = {0}; strcpy(createReq.name, "f1"); - SetCode(&createReq, "code1"); + SetCode(&createReq, "code1", 6); SetComment(&createReq, "comment1"); createReq.bufSize = TSDB_FUNC_BUF_SIZE + 1; createReq.igExists = 0; @@ -253,7 +252,7 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) { createReq.outputLen = 24; createReq.bufSize = 6; createReq.signature = 18; - SetCode(&createReq, "code2"); + SetCode(&createReq, "code2", 6); SetComment(&createReq, "comment2"); int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); @@ -439,3 +438,70 @@ TEST_F(MndTestFunc, 04_Drop_Func) { test.SendShowReq(TSDB_MGMT_TABLE_FUNC, "user_functions", ""); EXPECT_EQ(test.GetShowRows(), 1); } + +TEST_F(MndTestFunc, 05_Actual_code) { + { + SCreateFuncReq createReq = {0}; + strcpy(createReq.name, "udf1"); + char code[300] = {0}; + for (int32_t i = 0; i < sizeof(code); ++i) { + code[i] = (i) % 20; + } + SetCode(&createReq, code, 300); + SetComment(&createReq, "comment1"); + createReq.bufSize = 8; + createReq.igExists = 0; + createReq.funcType = 1; + createReq.scriptType = 2; + createReq.outputType = TSDB_DATA_TYPE_SMALLINT; + createReq.outputLen = 12; + createReq.bufSize = 4; + createReq.signature = 5; + + int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateFuncReq(pReq, contLen, &createReq); + tFreeSCreateFuncReq(&createReq); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + } + + { + SRetrieveFuncReq retrieveReq = {0}; + retrieveReq.numOfFuncs = 1; + retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN); + taosArrayPush(retrieveReq.pFuncNames, "udf1"); + + int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSRetrieveFuncReq(pReq, contLen, &retrieveReq); + tFreeSRetrieveFuncReq(&retrieveReq); + + SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + + SRetrieveFuncRsp retrieveRsp = {0}; + tDeserializeSRetrieveFuncRsp(pRsp->pCont, pRsp->contLen, &retrieveRsp); + EXPECT_EQ(retrieveRsp.numOfFuncs, 1); + EXPECT_EQ(retrieveRsp.numOfFuncs, (int32_t)taosArrayGetSize(retrieveRsp.pFuncInfos)); + + SFuncInfo* pFuncInfo = (SFuncInfo*)taosArrayGet(retrieveRsp.pFuncInfos, 0); + + EXPECT_STREQ(pFuncInfo->name, "udf1"); + EXPECT_EQ(pFuncInfo->funcType, 1); + EXPECT_EQ(pFuncInfo->scriptType, 2); + EXPECT_EQ(pFuncInfo->outputType, TSDB_DATA_TYPE_SMALLINT); + EXPECT_EQ(pFuncInfo->outputLen, 12); + EXPECT_EQ(pFuncInfo->bufSize, 4); + EXPECT_EQ(pFuncInfo->signature, 5); + EXPECT_STREQ("comment1", pFuncInfo->pComment); + for (int32_t i = 0; i < 300; ++i) { + EXPECT_EQ(pFuncInfo->pCode[i], (i) % 20); + } + tFreeSRetrieveFuncRsp(&retrieveRsp); + } + +} \ No newline at end of file diff --git a/source/dnode/qnode/src/qnode.c b/source/dnode/qnode/src/qnode.c index 7b299f1f3c..907fddaec2 100644 --- a/source/dnode/qnode/src/qnode.c +++ b/source/dnode/qnode/src/qnode.c @@ -17,6 +17,7 @@ #include "qndInt.h" #include "query.h" #include "qworker.h" +//#include "tudf.h" SQnode *qndOpen(const SQnodeOpt *pOption) { SQnode *pQnode = taosMemoryCalloc(1, sizeof(SQnode)); @@ -25,6 +26,8 @@ SQnode *qndOpen(const SQnodeOpt *pOption) { return NULL; } + //udfcOpen(); + if (qWorkerInit(NODE_TYPE_QNODE, pQnode->qndId, NULL, (void **)&pQnode->pQuery, &pOption->msgCb)) { taosMemoryFreeClear(pQnode); return NULL; @@ -37,13 +40,15 @@ SQnode *qndOpen(const SQnodeOpt *pOption) { void qndClose(SQnode *pQnode) { qWorkerDestroy((void **)&pQnode->pQuery); + //udfcClose(); + taosMemoryFree(pQnode); } int32_t qndGetLoad(SQnode *pQnode, SQnodeLoad *pLoad) { return 0; } int32_t qndProcessQueryMsg(SQnode *pQnode, SRpcMsg *pMsg) { - qTrace("message in query queue is processing"); + qTrace("message in qnode query queue is processing"); SReadHandle handle = {0}; switch (pMsg->msgType) { diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 6e004a89fc..347d28f1ea 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -159,7 +159,6 @@ typedef struct { int8_t withTbName; int8_t withSchema; int8_t withTag; - int8_t withTagSchema; char* qmsg; STqPushHandle pushHandle; // SRWLatch lock; diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 0b3feb5010..434c5bc5bb 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -31,6 +31,7 @@ STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal) { pTq->path = strdup(path); pTq->pVnode = pVnode; pTq->pWal = pWal; + #if 0 pTq->tqMeta = tqStoreOpen(pTq, path, (FTqSerialize)tqSerializeConsumer, (FTqDeserialize)tqDeserializeConsumer, (FTqDelete)taosMemoryFree, 0); @@ -401,6 +402,13 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { consumerEpoch = atomic_val_compare_exchange_32(&pExec->epoch, consumerEpoch, reqEpoch); } + SWalHead* pHeadWithCkSum = taosMemoryMalloc(sizeof(SWalHead) + 2048); + if (pHeadWithCkSum == NULL) { + return -1; + } + + walSetReaderCapacity(pExec->pWalReader, 2048); + SMqDataBlkRsp rsp = {0}; rsp.reqOffset = pReq->currentOffset; rsp.blockData = taosArrayInit(0, sizeof(void*)); @@ -414,6 +422,26 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { break; } + taosThreadMutexLock(&pExec->pWalReader->mutex); + + if (walFetchHead(pExec->pWalReader, fetchOffset, pHeadWithCkSum) < 0) { + vDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, no more log to return", consumerId, pReq->epoch, + TD_VID(pTq->pVnode), fetchOffset); + taosThreadMutexUnlock(&pExec->pWalReader->mutex); + break; + } + + if (pHeadWithCkSum->head.msgType != TDMT_VND_SUBMIT) { + walSkipFetchBody(pExec->pWalReader, pHeadWithCkSum); + } else { + walFetchBody(pExec->pWalReader, &pHeadWithCkSum); + } + + SWalReadHead* pHead = &pHeadWithCkSum->head; + + taosThreadMutexUnlock(&pExec->pWalReader->mutex); + +#if 0 SWalReadHead* pHead; if (walReadWithHandle_s(pExec->pWalReader, fetchOffset, &pHead) < 0) { // TODO: no more log, set timer to wait blocking time @@ -443,14 +471,16 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { return 0; #endif - break; - } + break; + } +#endif vDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d offset %ld msgType %d", consumerId, pReq->epoch, TD_VID(pTq->pVnode), fetchOffset, pHead->msgType); if (pHead->msgType == TDMT_VND_SUBMIT) { SSubmitReq* pCont = (SSubmitReq*)&pHead->body; + // table subscribe if (pExec->subType == TOPIC_SUB_TYPE__TABLE) { qTaskInfo_t task = pExec->task[workerId]; ASSERT(task); @@ -484,6 +514,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { taosArrayPush(rsp.blockData, &buf); rsp.blockNum++; } + // db subscribe } else if (pExec->subType == TOPIC_SUB_TYPE__DB) { STqReadHandle* pReader = pExec->pExecReader[workerId]; tqReadHandleSetMsg(pReader, pCont, 0); @@ -789,7 +820,6 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) { pExec->withTbName = req.withTbName; pExec->withSchema = req.withSchema; pExec->withTag = req.withTag; - pExec->withTagSchema = req.withTagSchema; pExec->qmsg = req.qmsg; req.qmsg = NULL; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index f70a4478b3..098b86975b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -3223,8 +3223,13 @@ void tsdbRetrieveDataBlockInfo(tsdbReaderT* pTsdbReadHandle, SDataBlockInfo* pDa tsdbDebug("data block generated, uid:%" PRIu64 " numOfRows:%d, tsrange:%" PRId64 " - %" PRId64 " %s", uid, cur->rows, cur->win.skey, cur->win.ekey, pHandle->idStr); - // pDataBlockInfo->uid = uid; // block Id may be over write by assigning uid fro this data block. Do NOT assign - // the table uid + pDataBlockInfo->uid = uid; + +#if 0 + // for multi-group data query processing test purpose + pDataBlockInfo->groupId = uid; +#endif + pDataBlockInfo->rows = cur->rows; pDataBlockInfo->window = cur->win; pDataBlockInfo->numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pHandle)); diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index aa90a04478..e3c504c93d 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -135,7 +135,7 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg } int vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) { - vTrace("message in query queue is processing"); + vTrace("message in vnode query queue is processing"); SReadHandle handle = {.reader = pVnode->pTsdb, .meta = pVnode->pMeta, .config = &pVnode->config}; switch (pMsg->msgType) { diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 410527c9e6..272c370ca3 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -494,7 +494,7 @@ _return: return TSDB_CODE_SUCCESS; } -int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, SArray **out) { +int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, SArray *out) { char *msg = NULL; int32_t msgLen = 0; @@ -526,7 +526,7 @@ int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmt CTG_ERR_RET(code); } - ctgDebug("Got qnode list from mnode, listNum:%d", (int32_t)taosArrayGetSize(*out)); + ctgDebug("Got qnode list from mnode, listNum:%d", (int32_t)taosArrayGetSize(out)); return TSDB_CODE_SUCCESS; } @@ -2778,7 +2778,8 @@ int32_t catalogGetAllMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps, } if (pReq->qNodeRequired) { - CTG_ERR_JRET(ctgGetQnodeListFromMnode(pCtg, pTrans, pMgmtEps, &pRsp->pEpSetList)); + pRsp->pQnodeList = taosArrayInit(10, sizeof(SQueryNodeAddr)); + CTG_ERR_JRET(ctgGetQnodeListFromMnode(pCtg, pTrans, pMgmtEps, pRsp->pQnodeList)); } CTG_API_LEAVE(TSDB_CODE_SUCCESS); @@ -2807,7 +2808,7 @@ int32_t catalogGetQnodeList(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } - CTG_ERR_JRET(ctgGetQnodeListFromMnode(pCtg, pRpc, pMgmtEps, &pQnodeList)); + CTG_ERR_JRET(ctgGetQnodeListFromMnode(pCtg, pRpc, pMgmtEps, pQnodeList)); _return: diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index 75084e7610..d17a5a59a4 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -304,8 +304,8 @@ int32_t qExplainResAppendRow(SExplainCtx *ctx, char *tbuf, int32_t len, int32_t memcpy(row.buf, tbuf, len); row.level = level; - row.len = len; - ctx->dataSize += len; + row.len = len; + ctx->dataSize += row.len; if (NULL == taosArrayPush(ctx->rows, &row)) { qError("taosArrayPush row to explain res rows failed"); @@ -756,7 +756,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { } int32_t colNum = 1; - int32_t rspSize = sizeof(SRetrieveTableRsp) + sizeof(int32_t) * colNum + sizeof(int32_t) * rowNum + pCtx->dataSize; + int32_t rspSize = sizeof(SRetrieveTableRsp) + sizeof(int32_t) + sizeof(uint64_t) + sizeof(int32_t) * colNum + sizeof(int32_t) * rowNum + pCtx->dataSize; SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, rspSize); if (NULL == rsp) { qError("malloc SRetrieveTableRsp failed, size:%d", rspSize); @@ -766,29 +766,38 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { rsp->completed = 1; rsp->numOfRows = htonl(rowNum); - *(int32_t *)rsp->data = htonl(pCtx->dataSize); + // payload length + *(int32_t *)rsp->data = sizeof(int32_t) + sizeof(uint64_t) + sizeof(int32_t) * colNum + sizeof(int32_t) * rowNum + pCtx->dataSize; - int32_t *offset = (int32_t *)((char *)rsp->data + sizeof(int32_t)); + // group id + *(uint64_t*)(rsp->data + sizeof(int32_t)) = 0; + + // column length + int32_t* colLength = (int32_t *)(rsp->data + sizeof(int32_t) + sizeof(uint64_t)); + + // varchar column offset segment + int32_t *offset = (int32_t *)((char *)colLength + sizeof(int32_t)); + + // varchar data real payload char *data = (char *)(offset + rowNum); - int32_t tOffset = 0; - + + char* start = data; for (int32_t i = 0; i < rowNum; ++i) { SQueryExplainRowInfo *row = taosArrayGet(pCtx->rows, i); - *offset = tOffset; - tOffset += row->len; + offset[i] = data - start; - memcpy(data, row->buf, row->len); - - ++offset; + varDataCopy(data, row->buf); + ASSERT(varDataTLen(row->buf) == row->len); data += row->len; } - *pRsp = rsp; + *colLength = htonl(data - start); + rsp->compLen = htonl(rspSize); + *pRsp = rsp; return TSDB_CODE_SUCCESS; } - int32_t qExplainPrepareCtx(SQueryPlan *pDag, SExplainCtx **pCtx) { int32_t code = 0; SNodeListNode *plans = NULL; @@ -895,9 +904,7 @@ int32_t qExplainAppendPlanRows(SExplainCtx *pCtx) { int32_t qExplainGenerateRsp(SExplainCtx *pCtx, SRetrieveTableRsp **pRsp) { QRY_ERR_RET(qExplainAppendGroupResRows(pCtx, pCtx->rootGroupId, 0)); - QRY_ERR_RET(qExplainAppendPlanRows(pCtx)); - QRY_ERR_RET(qExplainGetRspFromCtx(pCtx, pRsp)); return TSDB_CODE_SUCCESS; @@ -967,13 +974,10 @@ int32_t qExecStaticExplain(SQueryPlan *pDag, SRetrieveTableRsp **pRsp) { SExplainCtx *pCtx = NULL; QRY_ERR_RET(qExplainPrepareCtx(pDag, &pCtx)); - QRY_ERR_JRET(qExplainGenerateRsp(pCtx, pRsp)); _return: - qExplainFreeCtx(pCtx); - QRY_RET(code); } diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 834d37927a..f0cb1c1107 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -40,8 +40,6 @@ #define GET_TASKID(_t) (((SExecTaskInfo*)(_t))->id.str) -#define curTimeWindowIndex(_winres) ((_winres)->curIndex) - typedef struct SGroupResInfo { int32_t totalGroup; int32_t currentGroup; @@ -68,11 +66,16 @@ typedef struct SResultRowPosition { int32_t offset; } SResultRowPosition; +typedef struct SResKeyPos { + SResultRowPosition pos; + uint64_t groupId; + char key[]; +} SResKeyPos; + typedef struct SResultRowInfo { SResultRowPosition *pPosition; int32_t size; // number of result set int32_t capacity; // max capacity -// int32_t curPos; // current active result row index of pResult list SResultRowPosition cur; } SResultRowInfo; @@ -135,7 +138,7 @@ typedef struct { int32_t colId; } SStddevInterResult; -void initGroupResInfo(SGroupResInfo* pGroupResInfo, SResultRowInfo* pResultInfo); +void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, bool sortGroupResult); void initMultiResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList); void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index c6d209e706..db0dd1ffbb 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -347,10 +347,11 @@ typedef struct STableScanInfo { } STableScanInfo; typedef struct STagScanInfo { - SColumnInfo* pCols; - SSDataBlock* pRes; + SColumnInfo *pCols; + SSDataBlock *pRes; int32_t totalTables; int32_t curPos; + void *pReader; } STagScanInfo; typedef struct SStreamBlockScanInfo { @@ -376,13 +377,11 @@ typedef struct SSysTableScanInfo { SEpSet epSet; tsem_t ready; - int32_t accountId; - bool showRewrite; - SNode* pCondition; // db_name filter condition, to discard data that are not in current database - void* pCur; // cursor for iterate the local table meta store. - SArray* scanCols; // SArray scan column id list - -// int32_t type; // show type, TODO remove it + int32_t accountId; + bool showRewrite; + SNode* pCondition; // db_name filter condition, to discard data that are not in current database + void* pCur; // cursor for iterate the local table meta store. + SArray* scanCols; // SArray scan column id list SName name; SSDataBlock* pRes; int32_t capacity; @@ -628,7 +627,7 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfCols, int32_t dataLoadFlag, int32_t repeatTime, +SOperatorInfo* createTableScanOperatorInfo(void* pReaderHandle, int32_t order, int32_t numOfCols, int32_t dataLoadFlag, int32_t repeatTime, int32_t reverseTime, SArray* pColMatchInfo, SSDataBlock* pResBlock, SNode* pCondition, SInterval* pInterval, double ratio, SExecTaskInfo* pTaskInfo); SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SExprInfo* pScalarExprInfo, @@ -668,12 +667,12 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SExprInfo* SSDataBlock* pResultBlock, SExecTaskInfo* pTaskInfo); SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SNode* pOnCondition, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createTagScanOperatorInfo(void* pReaderHandle, SExprInfo* pExpr, int32_t numOfOutput, SExecTaskInfo* pTaskInfo); #if 0 SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv); SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createTagScanOperatorInfo(SReaderHandle* pReaderHandle, SExprInfo* pExpr, int32_t numOfOutput); #endif void projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* pSrcBlock, SqlFunctionCtx* pCtx, diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index e897bc8892..a5ce01dba2 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -64,10 +64,10 @@ static bool needCompress(const SSDataBlock* pData, int32_t numOfCols) { } // data format: -// +----------------+--------------------------------------+-------------+-----------+-------------+-----------+ -// |SDataCacheEntry | column#1 length, column#2 length ... | col1 bitmap | col1 data | col2 bitmap | col2 data | .... -// | | sizeof(int32_t) * numOfCols | actual size | | actual size | | -// +----------------+--------------------------------------+-------------+-----------+-------------+-----------+ +// +----------------+--------------+----------+--------------------------------------+-------------+-----------+-------------+-----------+ +// |SDataCacheEntry | total length | group id | column#1 length, column#2 length ... | col1 bitmap | col1 data | col2 bitmap | col2 data | .... +// | | (4 bytes) |(8 bytes) | sizeof(int32_t) * numOfCols | actual size | | actual size | | +// +----------------+--------------+----------+--------------------------------------+-------------+-----------+-------------+-----------+ // The length of bitmap is decided by number of rows of this data block, and the length of each column data is // recorded in the first segment, next to the struct header static void toDataCacheEntry(const SDataDispatchHandle* pHandle, const SInputData* pInput, SDataDispatchBuf* pBuf) { diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 98b72cf4c2..c3fa777779 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -186,12 +186,50 @@ void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo) { pGroupResInfo->index = 0; } -void initGroupResInfo(SGroupResInfo* pGroupResInfo, SResultRowInfo* pResultInfo) { +static int32_t resultrowCompar1(const void* p1, const void* p2) { + SResKeyPos* pp1 = *(SResKeyPos**) p1; + SResKeyPos* pp2 = *(SResKeyPos**) p2; + + if (pp1->groupId == pp2->groupId) { + int64_t pts1 = *(int64_t*) pp1->key; + int64_t pts2 = *(int64_t*) pp2->key; + + if (pts1 == pts2) { + return 0; + } else { + return pts1 < pts2? -1:1; + } + } else { + return pp1->groupId < pp2->groupId? -1:1; + } +} + +void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, bool sortGroupResult) { if (pGroupResInfo->pRows != NULL) { taosArrayDestroy(pGroupResInfo->pRows); } - pGroupResInfo->pRows = taosArrayFromList(pResultInfo->pPosition, pResultInfo->size, sizeof(SResultRowPosition)); + // extract the result rows information from the hash map + void* pData = NULL; + pGroupResInfo->pRows = taosArrayInit(10, POINTER_BYTES); + + size_t keyLen = 0; + while((pData = taosHashIterate(pHashmap, pData)) != NULL) { + void* key = taosHashGetKey(pData, &keyLen); + + SResKeyPos* p = taosMemoryMalloc(keyLen + sizeof(SResultRowPosition)); + + p->groupId = *(uint64_t*) key; + p->pos = *(SResultRowPosition*) pData; + memcpy(p->key, key + sizeof(uint64_t), keyLen - sizeof(uint64_t)); + + taosArrayPush(pGroupResInfo->pRows, &p); + } + + if (sortGroupResult) { + qsort(pGroupResInfo->pRows->pData, taosArrayGetSize(pGroupResInfo->pRows), POINTER_BYTES, resultrowCompar1); + } + pGroupResInfo->index = 0; assert(pGroupResInfo->index <= getNumOfTotalRes(pGroupResInfo)); } diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 32c3a60b0f..6a5f5b2dc3 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -235,7 +235,6 @@ static int32_t doCopyToSDataBlock(SSDataBlock* pBlock, SExprInfo* pExprInfo, SDi static void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size); static void setResultBufSize(STaskAttr* pQueryAttr, SResultInfo* pResultInfo); -static void setCtxTagForJoin(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable); static void doSetTableGroupOutputBuf(SAggOperatorInfo* pAggInfo, int32_t numOfOutput, uint64_t groupId, SExecTaskInfo* pTaskInfo); @@ -298,26 +297,6 @@ SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode) { return pBlock; } -static bool isSelectivityWithTagsQuery(SqlFunctionCtx* pCtx, int32_t numOfOutput) { - return true; - // bool hasTags = false; - // int32_t numOfSelectivity = 0; - // - // for (int32_t i = 0; i < numOfOutput; ++i) { - // int32_t functId = pCtx[i].functionId; - // if (functId == FUNCTION_TAG_DUMMY || functId == FUNCTION_TS_DUMMY) { - // hasTags = true; - // continue; - // } - // - // if ((aAggs[functId].status & FUNCSTATE_SELECTIVITY) != 0) { - // numOfSelectivity++; - // } - // } - // - // return (numOfSelectivity > 0 && hasTags); -} - static bool hasNull(SColumn* pColumn, SColumnDataAgg* pStatis) { if (TSDB_COL_IS_TAG(pColumn->flag) || TSDB_COL_IS_UD_COL(pColumn->flag) || pColumn->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { @@ -433,6 +412,13 @@ SResultRow* getNewResultRow_rv(SDiskbasedBuf* pResultBuf, int64_t tableGroupId, return pResultRow; } +/** + * the struct of key in hash table + * +----------+---------------+ + * | group id | key data | + * | 8 bytes | actual length | + * +----------+---------------+ + */ static SResultRow* doSetResultOutBufByKey_rv(SDiskbasedBuf* pResultBuf, SResultRowInfo* pResultRowInfo, int64_t uid, char* pData, int16_t bytes, bool masterscan, uint64_t groupId, SExecTaskInfo* pTaskInfo, bool isIntervalQuery, SAggSupporter* pSup) { @@ -1098,8 +1084,9 @@ static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCt int32_t code = TSDB_CODE_SUCCESS; for (int32_t i = 0; i < pOperator->numOfOutput; ++i) { - pCtx[i].order = order; - pCtx[i].size = pBlock->info.rows; + pCtx[i].order = order; + pCtx[i].size = pBlock->info.rows; + pCtx[i].pSrcBlock = pBlock; pCtx[i].currentStage = MAIN_SCAN; SInputColumnInfoData* pInput = &pCtx[i].input; @@ -1474,7 +1461,7 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe SArray* pUpdated = NULL; if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) { - pUpdated = taosArrayInit(4, sizeof(SResultRowPosition)); + pUpdated = taosArrayInit(4, POINTER_BYTES); } int32_t step = 1; @@ -1486,8 +1473,6 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe if (pSDataBlock->pDataBlock != NULL) { SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); tsCols = (int64_t*)pColDataInfo->pData; - // assert(tsCols[0] == pSDataBlock->info.window.skey && tsCols[pSDataBlock->info.rows - 1] == - // pSDataBlock->info.window.ekey); } int32_t startPos = ascScan ? 0 : (pSDataBlock->info.rows - 1); @@ -1506,7 +1491,11 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe } if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) { - SResultRowPosition pos = {.pageId = pResult->pageId, .offset = pResult->offset}; + SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t)); + pos->groupId = tableGroupId; + pos->pos = (SResultRowPosition) {.pageId = pResult->pageId, .offset = pResult->offset}; + *(int64_t*) pos->key = pResult->win.skey; + taosArrayPush(pUpdated, &pos); } @@ -1580,7 +1569,11 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe } if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) { - SResultRowPosition pos = {.pageId = pResult->pageId, .offset = pResult->offset}; + SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t)); + pos->groupId = tableGroupId; + pos->pos = (SResultRowPosition) {.pageId = pResult->pageId, .offset = pResult->offset}; + *(int64_t*) pos->key = pResult->win.skey; + taosArrayPush(pUpdated, &pos); } @@ -1844,10 +1837,6 @@ void setBlockStatisInfo(SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, SSDataBlock* // set the output buffer for the selectivity + tag query static int32_t setCtxTagColumnInfo(SqlFunctionCtx* pCtx, int32_t numOfOutput) { - if (!isSelectivityWithTagsQuery(pCtx, numOfOutput)) { - return TSDB_CODE_SUCCESS; - } - int32_t num = 0; int16_t tagLen = 0; @@ -1874,9 +1863,8 @@ static int32_t setCtxTagColumnInfo(SqlFunctionCtx* pCtx, int32_t numOfOutput) { } } if (p != NULL) { - p->subsidiaryRes.pCtx = pTagCtx; - p->subsidiaryRes.numOfCols = num; - p->subsidiaryRes.bufLen = tagLen; + p->subsidiaries.pCtx = pTagCtx; + p->subsidiaries.num = num; } else { taosMemoryFreeClear(pTagCtx); } @@ -1903,6 +1891,9 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, SqlFunctionCtx* pCtx = &pFuncCtx[i]; pCtx->functionId = -1; + pCtx->curBufPage = -1; + pCtx->pExpr = pExpr; + if (pExpr->pExpr->nodeType == QUERY_NODE_FUNCTION) { SFuncExecEnv env = {0}; pCtx->functionId = pExpr->pExpr->_function.pFunctNode->funcId; @@ -1930,9 +1921,9 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, pCtx->pTsOutput = NULL; pCtx->resDataInfo.bytes = pFunct->resSchema.bytes; pCtx->resDataInfo.type = pFunct->resSchema.type; - pCtx->order = TSDB_ORDER_ASC; + pCtx->order = TSDB_ORDER_ASC; pCtx->start.key = INT64_MIN; - pCtx->end.key = INT64_MIN; + pCtx->end.key = INT64_MIN; pCtx->numOfParams = pExpr->base.numOfParams; pCtx->param = pFunct->pParam; @@ -1992,7 +1983,7 @@ static void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput) { } taosVariantDestroy(&pCtx[i].tag); - taosMemoryFreeClear(pCtx[i].subsidiaryRes.pCtx); + taosMemoryFreeClear(pCtx[i].subsidiaries.pCtx); } taosMemoryFreeClear(pCtx); @@ -2095,25 +2086,6 @@ static int32_t updateBlockLoadStatus(STaskAttr* pQuery, int32_t status) { return status; } -static void doUpdateLastKey(STaskAttr* pQueryAttr) { - STimeWindow* win = &pQueryAttr->window; - - size_t num = taosArrayGetSize(pQueryAttr->tableGroupInfo.pGroupList); - for (int32_t i = 0; i < num; ++i) { - SArray* p1 = taosArrayGetP(pQueryAttr->tableGroupInfo.pGroupList, i); - - size_t len = taosArrayGetSize(p1); - for (int32_t j = 0; j < len; ++j) { - // STableKeyInfo* pInfo = taosArrayGet(p1, j); - // - // // update the new lastkey if it is equalled to the value of the old skey - // if (pInfo->lastKey == win->ekey) { - // pInfo->lastKey = win->skey; - // } - } - } -} - // static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableReq* pQueryMsg, bool stableQuery) { // STaskAttr* pQueryAttr = pQInfo->runtimeEnv.pQueryAttr; // @@ -2848,6 +2820,7 @@ void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status) { } } +// todo merged with the build group result. void finalizeMultiTupleQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset) { for (int32_t i = 0; i < pResultRowInfo->size; ++i) { @@ -2882,40 +2855,36 @@ void finalizeMultiTupleQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SD } } +// todo merged with the build group result. void finalizeUpdatedResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SDiskbasedBuf* pBuf, SArray* pUpdateList, int32_t* rowCellInfoOffset) { size_t num = taosArrayGetSize(pUpdateList); for (int32_t i = 0; i < num; ++i) { - SResultRowPosition* pPos = taosArrayGet(pUpdateList, i); - - SFilePage* bufPage = getBufPage(pBuf, pPos->pageId); - SResultRow* pRow = (SResultRow*)((char*)bufPage + pPos->offset); + SResKeyPos * pPos = taosArrayGetP(pUpdateList, i); + SFilePage* bufPage = getBufPage(pBuf, pPos->pos.pageId); + SResultRow* pRow = (SResultRow*)((char*)bufPage + pPos->pos.offset); +// for (int32_t j = 0; j < numOfOutput; ++j) { pCtx[j].resultInfo = getResultCell(pRow, j, rowCellInfoOffset); - +// struct SResultRowEntryInfo* pResInfo = pCtx[j].resultInfo; - if (isRowEntryCompleted(pResInfo) && isRowEntryInitialized(pResInfo)) { - continue; - } - - if (pCtx[j].fpSet.process) { // TODO set the dummy function. - // pCtx[j].fpSet.finalize(&pCtx[j]); - pResInfo->initialized = true; - } - +// if (isRowEntryCompleted(pResInfo) && isRowEntryInitialized(pResInfo)) { +// continue; +// } +// +// if (pCtx[j].fpSet.process) { // TODO set the dummy function. +//// pCtx[j].fpSet.finalize(&pCtx[j]); +// pResInfo->initialized = true; +// } +// if (pRow->numOfRows < pResInfo->numOfRes) { pRow->numOfRows = pResInfo->numOfRes; } } releaseBufPage(pBuf, bufPage); - /* - * set the number of output results for group by normal columns, the number of output rows usually is 1 except - * the top and bottom query - */ - // buf->numOfRows = (uint16_t)getNumOfResult(pCtx, numOfOutput); } } @@ -3062,33 +3031,6 @@ void setExecutionContext(int32_t numOfOutput, uint64_t groupId, SExecTaskInfo* p pAggInfo->groupId = groupId; } -void setCtxTagForJoin(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable) { - STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - - SExprBasicInfo* pExpr = &pExprInfo->base; - // if (pQueryAttr->stableQuery && (pRuntimeEnv->pTsBuf != NULL) && - // (pExpr->functionId == FUNCTION_TS || pExpr->functionId == FUNCTION_PRJ) && - // (pExpr->colInfo.colIndex == PRIMARYKEY_TIMESTAMP_COL_ID)) { - // assert(pExpr->numOfParams == 1); - // - // int16_t tagColId = (int16_t)pExprInfo->base.param[0].i; - // SColumnInfo* pColInfo = doGetTagColumnInfoById(pQueryAttr->tagColList, pQueryAttr->numOfTags, tagColId); - // - // doSetTagValueInParam(pTable, tagColId, &pCtx->tag, pColInfo->type, pColInfo->bytes); - // - // int16_t tagType = pCtx[0].tag.nType; - // if (tagType == TSDB_DATA_TYPE_BINARY || tagType == TSDB_DATA_TYPE_NCHAR) { - // //qDebug("QInfo:0x%"PRIx64" set tag value for join comparison, colId:%" PRId64 ", val:%s", - // GET_TASKID(pRuntimeEnv), - //// pExprInfo->base.param[0].i, pCtx[0].tag.pz); - // } else { - // //qDebug("QInfo:0x%"PRIx64" set tag value for join comparison, colId:%" PRId64 ", val:%" PRId64, - // GET_TASKID(pRuntimeEnv), - //// pExprInfo->base.param[0].i, pCtx[0].tag.i); - // } - // } -} - /* * There are two cases to handle: * @@ -3131,7 +3073,6 @@ void setIntervalQueryRange(STableQueryInfo* pTableQueryInfo, TSKEY key, STimeWin } /** - * copyToOutputBuf support copy data in ascending/descending order * For interval query of both super table and table, copy the data in ascending order, since the output results are * ordered in SWindowResutl already. While handling the group by query for both table and super table, * all group result are completed already. @@ -3159,10 +3100,10 @@ int32_t doCopyToSDataBlock(SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbased } for (int32_t i = start; (i < numOfRows) && (i >= 0); i += step) { - SResultRowPosition* pPos = taosArrayGet(pGroupResInfo->pRows, i); - SFilePage* page = getBufPage(pBuf, pPos->pageId); + SResKeyPos *pPos = taosArrayGetP(pGroupResInfo->pRows, i); + SFilePage* page = getBufPage(pBuf, pPos->pos.pageId); - SResultRow* pRow = (SResultRow*)((char*)page + pPos->offset); + SResultRow* pRow = (SResultRow*)((char*)page + pPos->pos.offset); if (pRow->numOfRows == 0) { pGroupResInfo->index += 1; continue; @@ -3181,7 +3122,7 @@ int32_t doCopyToSDataBlock(SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbased pCtx[j].resultInfo = getResultCell(pRow, j, rowCellOffset); if (pCtx[j].fpSet.process) { - pCtx[j].fpSet.finalize(&pCtx[j], pBlock, slotId); + pCtx[j].fpSet.finalize(&pCtx[j], pBlock); } else { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId); @@ -3806,9 +3747,15 @@ int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadI blockDataEnsureCapacity(pRes, numOfRows); if (pColList == NULL) { // data from other sources - int32_t* colLen = (int32_t*)pData; - char* pStart = pData + sizeof(int32_t) * numOfOutput; + int32_t dataLen = *(int32_t*) pData; + pData += sizeof(int32_t); + pRes->info.groupId = *(uint64_t*) pData; + pData += sizeof(uint64_t); + + int32_t* colLen = (int32_t*)pData; + + char* pStart = pData + sizeof(int32_t) * numOfOutput; for (int32_t i = 0; i < numOfOutput; ++i) { colLen[i] = htonl(colLen[i]); ASSERT(colLen[i] > 0); @@ -3861,7 +3808,11 @@ int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadI blockDataEnsureCapacity(&block, numOfRows); - int32_t* colLen = (int32_t*)pStart; + int32_t dataLen = *(int32_t*) pStart; + uint64_t groupId = *(uint64_t*) (pStart + sizeof(int32_t)); + pStart += sizeof(int32_t) + sizeof(uint64_t); + + int32_t* colLen = (int32_t*) (pStart); pStart += sizeof(int32_t) * numOfCols; for (int32_t i = 0; i < numOfCols; ++i) { @@ -3903,6 +3854,7 @@ int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadI } pRes->info.rows = numOfRows; + blockDataUpdateTsWindow(pRes); int64_t el = taosGetTimestampUs() - startTs; @@ -4318,18 +4270,6 @@ static void assignExprInfo(SExprInfo* dst, const SExprInfo* src) { // } } -static SExprInfo* exprArrayDup(SArray* pExprList) { - size_t numOfOutput = taosArrayGetSize(pExprList); - - SExprInfo* p = taosMemoryCalloc(numOfOutput, sizeof(SExprInfo)); - for (int32_t i = 0; i < numOfOutput; ++i) { - SExprInfo* pExpr = taosArrayGetP(pExprList, i); - assignExprInfo(&p[i], pExpr); - } - - return p; -} - // TODO merge aggregate super table static void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle) { for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { @@ -4822,7 +4762,7 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { finalizeMultiTupleQueryResult(pAggInfo->binfo.pCtx, pOperator->numOfOutput, pAggInfo->aggSup.pResultBuf, &pAggInfo->binfo.resultRowInfo, pAggInfo->binfo.rowCellInfoOffset); - initGroupResInfo(&pAggInfo->groupResInfo, &pAggInfo->binfo.resultRowInfo); + initGroupedResultInfo(&pAggInfo->groupResInfo, pAggInfo->aggSup.pResultRowHashTable, false); OPTR_SET_OPENED(pOperator); return TSDB_CODE_SUCCESS; } @@ -5125,7 +5065,7 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) { STableQueryInfo* pTableQueryInfo = pInfo->pCurrent; setIntervalQueryRange(pTableQueryInfo, pBlock->info.window.skey, &pTaskInfo->window); - hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, 0); + hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, pBlock->info.groupId); #if 0 // test for encode/decode result info if(pOperator->encodeResultRow){ @@ -5147,7 +5087,7 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) { finalizeMultiTupleQueryResult(pInfo->binfo.pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset); - initGroupResInfo(&pInfo->groupResInfo, &pInfo->binfo.resultRowInfo); + initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true); OPTR_SET_OPENED(pOperator); return TSDB_CODE_SUCCESS; } @@ -5276,7 +5216,7 @@ static SSDataBlock* doAllIntervalAgg(SOperatorInfo* pOperator, bool* newgroup) { setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); // finalizeQueryResult(pSliceInfo->binfo.pCtx, pOperator->numOfOutput); - initGroupResInfo(&pSliceInfo->groupResInfo, &pSliceInfo->binfo.resultRowInfo); +// initGroupedResultInfo(&pSliceInfo->groupResInfo, &pSliceInfo->binfo.resultRowInfo); // doBuildResultDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pSliceInfo->pRes); if (pSliceInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pSliceInfo->groupResInfo)) { @@ -5327,7 +5267,7 @@ static SSDataBlock* doSTableIntervalAgg(SOperatorInfo* pOperator, bool* newgroup finalizeMultiTupleQueryResult(pInfo->binfo.pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset); - initGroupResInfo(&pInfo->groupResInfo, &pInfo->binfo.resultRowInfo); +// initGroupedResultInfo(&pInfo->groupResInfo, &pInfo->binfo.resultRowInfo); OPTR_SET_OPENED(pOperator); blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); @@ -5458,7 +5398,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator, bool* newgroup) { finalizeMultiTupleQueryResult(pBInfo->pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, &pBInfo->resultRowInfo, pBInfo->rowCellInfoOffset); - initGroupResInfo(&pInfo->groupResInfo, &pBInfo->resultRowInfo); + initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true); blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); doBuildResultDatablock(pBInfo->pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pBInfo->rowCellInfoOffset, pInfo->binfo.pCtx); @@ -5510,7 +5450,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator, bool* newgroup) finalizeMultiTupleQueryResult(pBInfo->pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, &pBInfo->resultRowInfo, pBInfo->rowCellInfoOffset); - initGroupResInfo(&pInfo->groupResInfo, &pBInfo->resultRowInfo); + initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true); blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity); doBuildResultDatablock(pBInfo->pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pBInfo->rowCellInfoOffset, pInfo->binfo.pCtx); @@ -5699,6 +5639,11 @@ int32_t initAggInfo(SOptrBasicInfo* pBasicInfo, SAggSupporter* pAggSup, SExprInf pBasicInfo->pRes = pResultBlock; doInitAggInfoSup(pAggSup, pBasicInfo->pCtx, numOfCols, keyBufSize, pkey); + + for(int32_t i = 0; i < numOfCols; ++i) { + pBasicInfo->pCtx[i].pBuf = pAggSup->pResultBuf; + } + return TSDB_CODE_SUCCESS; } @@ -5847,11 +5792,6 @@ static void destroyProjectOperatorInfo(void* param, int32_t numOfOutput) { doDestroyBasicInfo(&pInfo->binfo, numOfOutput); } -static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) { - STagScanInfo* pInfo = (STagScanInfo*)param; - pInfo->pRes = blockDataDestroy(pInfo->pRes); -} - static void destroyOrderOperatorInfo(void* param, int32_t numOfOutput) { SSortOperatorInfo* pInfo = (SSortOperatorInfo*)param; pInfo->pDataBlock = blockDataDestroy(pInfo->pDataBlock); @@ -5941,11 +5881,12 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* goto _error; } - pInfo->order = TSDB_ORDER_ASC; - pInfo->interval = *pInterval; - pInfo->execModel = pTaskInfo->execModel; - pInfo->win = pTaskInfo->window; - pInfo->twAggSup = *pTwAggSupp; + pInfo->order = TSDB_ORDER_ASC; + pInfo->interval = *pInterval; +// pInfo->execModel = OPTR_EXEC_MODEL_STREAM; + pInfo->execModel = pTaskInfo->execModel; + pInfo->win = pTaskInfo->window; + pInfo->twAggSup = *pTwAggSupp; pInfo->primaryTsIndex = primaryTsSlotId; int32_t numOfRows = 4096; @@ -6204,157 +6145,6 @@ _error: return NULL; } -static SSDataBlock* doTagScan(SOperatorInfo* pOperator, bool* newgroup) { -#if 0 - SOperatorInfo* pOperator = (SOperatorInfo*) param; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - int32_t maxNumOfTables = (int32_t)pResultInfo->capacity; - - STagScanInfo *pInfo = pOperator->info; - SSDataBlock *pRes = pInfo->pRes; - *newgroup = false; - - int32_t count = 0; - SArray* pa = GET_TABLEGROUP(pRuntimeEnv, 0); - - int32_t functionId = getExprFunctionId(&pOperator->pExpr[0]); - if (functionId == FUNCTION_TID_TAG) { // return the tags & table Id - assert(pQueryAttr->numOfOutput == 1); - - SExprInfo* pExprInfo = &pOperator->pExpr[0]; - int32_t rsize = pExprInfo->base.resSchema.bytes; - - count = 0; - - int16_t bytes = pExprInfo->base.resSchema.bytes; - int16_t type = pExprInfo->base.resSchema.type; - - for(int32_t i = 0; i < pQueryAttr->numOfTags; ++i) { - if (pQueryAttr->tagColList[i].colId == pExprInfo->base.pColumns->info.colId) { - bytes = pQueryAttr->tagColList[i].bytes; - type = pQueryAttr->tagColList[i].type; - break; - } - } - - SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, 0); - - while(pInfo->curPos < pInfo->totalTables && count < maxNumOfTables) { - int32_t i = pInfo->curPos++; - STableQueryInfo *item = taosArrayGetP(pa, i); - - char *output = pColInfo->pData + count * rsize; - varDataSetLen(output, rsize - VARSTR_HEADER_SIZE); - - output = varDataVal(output); - STableId* id = TSDB_TABLEID(item->pTable); - - *(int16_t *)output = 0; - output += sizeof(int16_t); - - *(int64_t *)output = id->uid; // memory align problem, todo serialize - output += sizeof(id->uid); - - *(int32_t *)output = id->tid; - output += sizeof(id->tid); - - *(int32_t *)output = pQueryAttr->vgId; - output += sizeof(pQueryAttr->vgId); - - char* data = NULL; - if (pExprInfo->base.pColumns->info.colId == TSDB_TBNAME_COLUMN_INDEX) { - data = tsdbGetTableName(item->pTable); - } else { - data = tsdbGetTableTagVal(item->pTable, pExprInfo->base.pColumns->info.colId, type, bytes); - } - - doSetTagValueToResultBuf(output, data, type, bytes); - count += 1; - } - - //qDebug("QInfo:0x%"PRIx64" create (tableId, tag) info completed, rows:%d", GET_TASKID(pRuntimeEnv), count); - } else if (functionId == FUNCTION_COUNT) {// handle the "count(tbname)" query - SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, 0); - *(int64_t*)pColInfo->pData = pInfo->totalTables; - count = 1; - - pOperator->status = OP_EXEC_DONE; - //qDebug("QInfo:0x%"PRIx64" create count(tbname) query, res:%d rows:1", GET_TASKID(pRuntimeEnv), count); - } else { // return only the tags|table name etc. - SExprInfo* pExprInfo = &pOperator->pExpr[0]; // todo use the column list instead of exprinfo - - count = 0; - while(pInfo->curPos < pInfo->totalTables && count < maxNumOfTables) { - int32_t i = pInfo->curPos++; - - STableQueryInfo* item = taosArrayGetP(pa, i); - - char *data = NULL, *dst = NULL; - int16_t type = 0, bytes = 0; - for(int32_t j = 0; j < pOperator->numOfOutput; ++j) { - // not assign value in case of user defined constant output column - if (TSDB_COL_IS_UD_COL(pExprInfo[j].base.pColumns->flag)) { - continue; - } - - SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, j); - type = pExprInfo[j].base.resSchema.type; - bytes = pExprInfo[j].base.resSchema.bytes; - - if (pExprInfo[j].base.pColumns->info.colId == TSDB_TBNAME_COLUMN_INDEX) { - data = tsdbGetTableName(item->pTable); - } else { - data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.pColumns->info.colId, type, bytes); - } - - dst = pColInfo->pData + count * pExprInfo[j].base.resSchema.bytes; - doSetTagValueToResultBuf(dst, data, type, bytes); - } - - count += 1; - } - - if (pInfo->curPos >= pInfo->totalTables) { - pOperator->status = OP_EXEC_DONE; - } - - //qDebug("QInfo:0x%"PRIx64" create tag values results completed, rows:%d", GET_TASKID(pRuntimeEnv), count); - } - - if (pOperator->status == OP_EXEC_DONE) { - setTaskStatus(pOperator->pRuntimeEnv, TASK_COMPLETED); - } - - pRes->info.rows = count; - return (pRes->info.rows == 0)? NULL:pInfo->pRes; - -#endif - return TSDB_CODE_SUCCESS; -} - -SOperatorInfo* createTagScanOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput) { - STagScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STagScanInfo)); - size_t numOfGroup = GET_NUM_OF_TABLEGROUP(pRuntimeEnv); - assert(numOfGroup == 0 || numOfGroup == 1); - - pInfo->curPos = 0; - - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - pOperator->name = "SeqTableTagScan"; - pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN; - pOperator->blockingOptr = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->getNextFn = doTagScan; - pOperator->pExpr = pExpr; - pOperator->numOfOutput = numOfOutput; - pOperator->closeFn = destroyTagScanOperatorInfo; - - return pOperator; -} static int32_t getColumnIndexInSource(SQueriedTableInfo* pTableInfo, SExprBasicInfo* pExpr, SColumnInfo* pTagCols) { int32_t j = 0; @@ -6562,6 +6352,9 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo int32_t numOfCols = 0; tsdbReaderT pDataReader = doCreateDataReader(pTableScanNode, pHandle, pTableGroupInfo, (uint64_t)queryId, taskId); + if (pDataReader == NULL) { + return NULL; + } SArray* pColList = extractColMatchInfo(pScanPhyNode->pScanCols, pScanPhyNode->node.pOutputDataBlockDesc, &numOfCols); @@ -6576,8 +6369,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo }; return createTableScanOperatorInfo(pDataReader, pTableScanNode->scanSeq[0] > 0 ? TSDB_ORDER_ASC : TSDB_ORDER_DESC, - numOfCols, pTableScanNode->dataRequired, pTableScanNode->scanSeq[0], pTableScanNode->scanSeq[1], pColList, - pResBlock, pScanPhyNode->node.pConditions, &interval, pTableScanNode->ratio, pTaskInfo); + numOfCols, pTableScanNode->dataRequired, pTableScanNode->scanSeq[0], pTableScanNode->scanSeq[1], pColList, + pResBlock, pScanPhyNode->node.pConditions, &interval, pTableScanNode->ratio, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_EXCHANGE == type) { SExchangePhysiNode* pExchange = (SExchangePhysiNode*)pPhyNode; SSDataBlock* pResBlock = createResDataBlock(pExchange->node.pOutputDataBlockDesc); @@ -6585,17 +6378,14 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == type) { SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; // simple child table. - int32_t code = doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableGroupInfo, - queryId, taskId); + int32_t code = doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableGroupInfo, queryId, taskId); SArray* tableIdList = extractTableIdList(pTableGroupInfo); SSDataBlock* pResBlock = createResDataBlock(pScanPhyNode->node.pOutputDataBlockDesc); int32_t numOfCols = 0; - SArray* pColList = - extractColMatchInfo(pScanPhyNode->pScanCols, pScanPhyNode->node.pOutputDataBlockDesc, &numOfCols); - SOperatorInfo* pOperator = - createStreamScanOperatorInfo(pHandle->reader, pResBlock, pColList, tableIdList, pTaskInfo); + SArray* pCols = extractColMatchInfo(pScanPhyNode->pScanCols, pScanPhyNode->node.pOutputDataBlockDesc, &numOfCols); + SOperatorInfo* pOperator = createStreamScanOperatorInfo(pHandle->reader, pResBlock, pCols, tableIdList, pTaskInfo); taosArrayDestroy(tableIdList); return pOperator; } else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) { @@ -6621,6 +6411,9 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo for (int32_t i = 0; i < size; ++i) { SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, i); ops[i] = createOperatorTree(pChildNode, pTaskInfo, pHandle, queryId, taskId, pTableGroupInfo); + if (ops[i] == NULL) { + return NULL; + } } SOperatorInfo* pOptr = NULL; @@ -6988,8 +6781,7 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead goto _complete; } - STableGroupInfo group = {0}; - (*pTaskInfo)->pRoot = createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId, &group); + (*pTaskInfo)->pRoot = createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId, &(*pTaskInfo)->tableqinfoGroupInfo); if (NULL == (*pTaskInfo)->pRoot) { code = terrno; goto _complete; diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 4793600710..52bdbea8a5 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -308,7 +308,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator, bool* newgrou // } blockDataEnsureCapacity(pRes, pOperator->resultInfo.capacity); - initGroupResInfo(&pInfo->groupResInfo, &pInfo->binfo.resultRowInfo); + initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, false); while(1) { doBuildResultDatablock(pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset, pInfo->binfo.pCtx); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 3a9742d48a..a70e402871 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -271,9 +271,7 @@ static void setupEnvForReverseScan(STableScanInfo* pTableScanInfo, SqlFunctionCt static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator, bool* newgroup) { STableScanInfo* pTableScanInfo = pOperator->info; - SSDataBlock* pBlock = pTableScanInfo->pResBlock; - STableGroupInfo* pTableGroupInfo = &pOperator->pTaskInfo->tableqinfoGroupInfo; - + SSDataBlock* pBlock = pTableScanInfo->pResBlock; *newgroup = false; while (tsdbNextDataBlock(pTableScanInfo->dataReader)) { @@ -284,18 +282,6 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator, bool* newgroup) { pTableScanInfo->numOfBlocks += 1; tsdbRetrieveDataBlockInfo(pTableScanInfo->dataReader, &pBlock->info); - // todo opt - // if (pTableGroupInfo->numOfTables > 1 || (pRuntimeEnv->current == NULL && pTableGroupInfo->numOfTables == 1)) { - // STableQueryInfo** pTableQueryInfo = - // (STableQueryInfo**)taosHashGet(pTableGroupInfo->map, &pBlock->info.uid, sizeof(pBlock->info.uid)); - // if (pTableQueryInfo == NULL) { - // break; - // } - // - // doTableQueryInfoTimeWindowCheck(pTaskInfo, *pTableQueryInfo, pTableScanInfo->order); - // } - - // this function never returns error? uint32_t status = 0; int32_t code = loadDataBlock(pOperator, pTableScanInfo, pBlock, &status); // int32_t code = loadDataBlockOnDemand(pOperator->pRuntimeEnv, pTableScanInfo, pBlock, &status); @@ -308,6 +294,8 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator, bool* newgroup) { continue; } + // reset the block to be 0 by default, this blockId is assigned by physical plan and is used by direct upstream operator. + pBlock->info.blockId = 0; return pBlock; } @@ -405,11 +393,11 @@ SOperatorInfo* createTableScanOperatorInfo(void* pDataReader, int32_t order, int pOperator->name = "TableScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN; pOperator->blockingOptr = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->numOfOutput = numOfOutput; - pOperator->getNextFn = doTableScan; - pOperator->pTaskInfo = pTaskInfo; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->numOfOutput = numOfOutput; + pOperator->getNextFn = doTableScan; + pOperator->pTaskInfo = pTaskInfo; static int32_t cost = 0; pOperator->cost.openCost = ++cost; @@ -824,12 +812,12 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator, bool* newgroup) { int32_t tableNameSlotId = 1; SColumnInfoData* pTableNameCol = taosArrayGet(pInfo->pRes->pDataBlock, tableNameSlotId); - char* name = NULL; + char* tb = NULL; int32_t numOfRows = 0; char n[TSDB_TABLE_NAME_LEN] = {0}; - while ((name = metaTbCursorNext(pInfo->pCur)) != NULL) { - STR_TO_VARSTR(n, name); + while ((tb = metaTbCursorNext(pInfo->pCur)) != NULL) { + STR_TO_VARSTR(n, tb); colDataAppend(pTableNameCol, numOfRows, n, false); numOfRows += 1; if (numOfRows >= pInfo->capacity) { @@ -992,3 +980,167 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataB return pOperator; } + +static SSDataBlock* doTagScan(SOperatorInfo* pOperator, bool* newgroup) { +#if 0 + SOperatorInfo* pOperator = (SOperatorInfo*) param; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + int32_t maxNumOfTables = (int32_t)pResultInfo->capacity; + + STagScanInfo *pInfo = pOperator->info; + SSDataBlock *pRes = pInfo->pRes; + *newgroup = false; + + int32_t count = 0; + SArray* pa = GET_TABLEGROUP(pRuntimeEnv, 0); + + int32_t functionId = getExprFunctionId(&pOperator->pExpr[0]); + if (functionId == FUNCTION_TID_TAG) { // return the tags & table Id + assert(pQueryAttr->numOfOutput == 1); + + SExprInfo* pExprInfo = &pOperator->pExpr[0]; + int32_t rsize = pExprInfo->base.resSchema.bytes; + + count = 0; + + int16_t bytes = pExprInfo->base.resSchema.bytes; + int16_t type = pExprInfo->base.resSchema.type; + + for(int32_t i = 0; i < pQueryAttr->numOfTags; ++i) { + if (pQueryAttr->tagColList[i].colId == pExprInfo->base.pColumns->info.colId) { + bytes = pQueryAttr->tagColList[i].bytes; + type = pQueryAttr->tagColList[i].type; + break; + } + } + + SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, 0); + + while(pInfo->curPos < pInfo->totalTables && count < maxNumOfTables) { + int32_t i = pInfo->curPos++; + STableQueryInfo *item = taosArrayGetP(pa, i); + + char *output = pColInfo->pData + count * rsize; + varDataSetLen(output, rsize - VARSTR_HEADER_SIZE); + + output = varDataVal(output); + STableId* id = TSDB_TABLEID(item->pTable); + + *(int16_t *)output = 0; + output += sizeof(int16_t); + + *(int64_t *)output = id->uid; // memory align problem, todo serialize + output += sizeof(id->uid); + + *(int32_t *)output = id->tid; + output += sizeof(id->tid); + + *(int32_t *)output = pQueryAttr->vgId; + output += sizeof(pQueryAttr->vgId); + + char* data = NULL; + if (pExprInfo->base.pColumns->info.colId == TSDB_TBNAME_COLUMN_INDEX) { + data = tsdbGetTableName(item->pTable); + } else { + data = tsdbGetTableTagVal(item->pTable, pExprInfo->base.pColumns->info.colId, type, bytes); + } + + doSetTagValueToResultBuf(output, data, type, bytes); + count += 1; + } + + //qDebug("QInfo:0x%"PRIx64" create (tableId, tag) info completed, rows:%d", GET_TASKID(pRuntimeEnv), count); + } else if (functionId == FUNCTION_COUNT) {// handle the "count(tbname)" query + SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, 0); + *(int64_t*)pColInfo->pData = pInfo->totalTables; + count = 1; + + pOperator->status = OP_EXEC_DONE; + //qDebug("QInfo:0x%"PRIx64" create count(tbname) query, res:%d rows:1", GET_TASKID(pRuntimeEnv), count); + } else { // return only the tags|table name etc. + SExprInfo* pExprInfo = &pOperator->pExpr[0]; // todo use the column list instead of exprinfo + + count = 0; + while(pInfo->curPos < pInfo->totalTables && count < maxNumOfTables) { + int32_t i = pInfo->curPos++; + + STableQueryInfo* item = taosArrayGetP(pa, i); + + char *data = NULL, *dst = NULL; + int16_t type = 0, bytes = 0; + for(int32_t j = 0; j < pOperator->numOfOutput; ++j) { + // not assign value in case of user defined constant output column + if (TSDB_COL_IS_UD_COL(pExprInfo[j].base.pColumns->flag)) { + continue; + } + + SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, j); + type = pExprInfo[j].base.resSchema.type; + bytes = pExprInfo[j].base.resSchema.bytes; + + if (pExprInfo[j].base.pColumns->info.colId == TSDB_TBNAME_COLUMN_INDEX) { + data = tsdbGetTableName(item->pTable); + } else { + data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.pColumns->info.colId, type, bytes); + } + + dst = pColInfo->pData + count * pExprInfo[j].base.resSchema.bytes; + doSetTagValueToResultBuf(dst, data, type, bytes); + } + + count += 1; + } + + if (pInfo->curPos >= pInfo->totalTables) { + pOperator->status = OP_EXEC_DONE; + } + + //qDebug("QInfo:0x%"PRIx64" create tag values results completed, rows:%d", GET_TASKID(pRuntimeEnv), count); + } + + if (pOperator->status == OP_EXEC_DONE) { + setTaskStatus(pOperator->pRuntimeEnv, TASK_COMPLETED); + } + + pRes->info.rows = count; + return (pRes->info.rows == 0)? NULL:pInfo->pRes; + +#endif + return TSDB_CODE_SUCCESS; +} + +static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) { + STagScanInfo* pInfo = (STagScanInfo*)param; + pInfo->pRes = blockDataDestroy(pInfo->pRes); +} + +SOperatorInfo* createTagScanOperatorInfo(void* pReaderHandle, SExprInfo* pExpr, int32_t numOfOutput, SExecTaskInfo* pTaskInfo) { + STagScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STagScanInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } + + pInfo->pReader = pReaderHandle; + pInfo->curPos = 0; + pOperator->name = "TagScanOperator"; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN; + pOperator->blockingOptr = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->getNextFn = doTagScan; + pOperator->pExpr = pExpr; + pOperator->numOfOutput = numOfOutput; + pOperator->pTaskInfo = pTaskInfo; + pOperator->closeFn = destroyTagScanOperatorInfo; + + return pOperator; + _error: + taosMemoryFree(pInfo); + taosMemoryFree(pOperator); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; +} diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index 40393a4eab..0ad3730b5a 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -24,7 +24,7 @@ extern "C" { #include "functionMgt.h" bool functionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); -int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId); +int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); EFuncDataRequired countDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow); bool getCountFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); @@ -43,17 +43,17 @@ int32_t maxFunction(SqlFunctionCtx *pCtx); bool getAvgFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool avgFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); int32_t avgFunction(SqlFunctionCtx* pCtx); -int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId); +int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); bool getStddevFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool stddevFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); int32_t stddevFunction(SqlFunctionCtx* pCtx); -int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId); +int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); bool getPercentileFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool percentileFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo); int32_t percentileFunction(SqlFunctionCtx *pCtx); -int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId); +int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); bool getDiffFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool diffFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo); @@ -65,7 +65,7 @@ int32_t lastFunction(SqlFunctionCtx *pCtx); bool getTopBotFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv); int32_t topFunction(SqlFunctionCtx *pCtx); -int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId); +int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); #ifdef __cplusplus } diff --git a/source/libs/function/inc/tudf.h b/source/libs/function/inc/tudf.h index 5f4f96c4cc..b72905b872 100644 --- a/source/libs/function/inc/tudf.h +++ b/source/libs/function/inc/tudf.h @@ -29,29 +29,20 @@ extern "C" { #define UDF_LISTEN_PIPE_NAME_LEN 32 #define UDF_LISTEN_PIPE_NAME_PREFIX "udfd.sock." +#define UDF_DNODE_ID_ENV_NAME "DNODE_ID" //====================================================================================== //begin API to taosd and qworker enum { UDFC_CODE_STOPPING = -1, - UDFC_CODE_PIPE_READ_ERR = -3, + UDFC_CODE_PIPE_READ_ERR = -2, + UDFC_CODE_CONNECT_PIPE_ERR = -3, + UDFC_CODE_LOAD_UDF_FAILURE = -4, + UDFC_CODE_INVALID_STATE = -5 }; -typedef void *UdfcHandle; -typedef void *UdfcFuncHandle; -/** - * create udfd proxy, called once in process that call setupUdf/callUdfxxx/teardownUdf - * @return error code - */ -int32_t udfcOpen(int32_t dnodeId, UdfcHandle* proxyHandle); - -/** - * destroy udfd proxy - * @return error code - */ -int32_t udfcClose(UdfcHandle proxyhandle); /** @@ -60,7 +51,7 @@ int32_t udfcClose(UdfcHandle proxyhandle); * @param handle, out * @return error code */ -int32_t setupUdf(UdfcHandle proxyHandle, char udfName[], SEpSet *epSet, UdfcFuncHandle *handle); +int32_t setupUdf(char udfName[], UdfcFuncHandle *handle); typedef struct SUdfColumnMeta { int16_t type; diff --git a/source/libs/function/inc/tudfInt.h b/source/libs/function/inc/tudfInt.h index 2c16afbd0d..4e7178f7fd 100644 --- a/source/libs/function/inc/tudfInt.h +++ b/source/libs/function/inc/tudfInt.h @@ -39,7 +39,6 @@ enum { typedef struct SUdfSetupRequest { char udfName[TSDB_FUNC_NAME_LEN]; - SEpSet epSet; } SUdfSetupRequest; typedef struct SUdfSetupResponse { @@ -112,6 +111,7 @@ void freeUdfDataDataBlock(SUdfDataBlock *block); int32_t convertDataBlockToUdfDataBlock(SSDataBlock *block, SUdfDataBlock *udfBlock); int32_t convertUdfColumnToDataBlock(SUdfColumn *udfCol, SSDataBlock *block); +int32_t getUdfdPipeName(char* pipeName, int32_t size); #ifdef __cplusplus } #endif diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 63af2fb972..0b9765ef15 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -14,7 +14,7 @@ */ #include "builtinsimpl.h" -#include +#include "function.h" #include "querynodes.h" #include "taggfunction.h" #include "tdatablock.h" @@ -44,8 +44,6 @@ typedef struct STopBotResItem { } STopBotResItem; typedef struct STopBotRes { - int32_t pageId; -// int32_t num; STopBotResItem *pItems; } STopBotRes; @@ -92,18 +90,6 @@ typedef struct SDiffInfo { } \ } while (0); -#define DO_UPDATE_SUBSID_RES(ctx, ts) \ - do { \ - for (int32_t _i = 0; _i < (ctx)->subsidiaryRes.numOfCols; ++_i) { \ - SqlFunctionCtx *__ctx = (ctx)->subsidiaryRes.pCtx[_i]; \ - if (__ctx->functionId == FUNCTION_TS_DUMMY) { \ - __ctx->tag.i = (ts); \ - __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; \ - } \ - __ctx->fpSet.process(__ctx); \ - } \ - } while (0) - #define UPDATE_DATA(ctx, left, right, num, sign, _ts) \ do { \ if (((left) < (right)) ^ (sign)) { \ @@ -139,7 +125,8 @@ bool functionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) { return true; } -int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId) { +int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { + int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); @@ -406,7 +393,7 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) { return TSDB_CODE_SUCCESS; } -int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId) { +int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SInputColumnInfoData* pInput = &pCtx->input; int32_t type = pInput->pData[0]->info.type; SAvgRes* pAvgRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); @@ -416,7 +403,7 @@ int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId) { pAvgRes->result = pAvgRes->sum.dsum / ((double) pAvgRes->count); } - return functionFinalize(pCtx, pBlock, slotId); + return functionFinalize(pCtx, pBlock); } EFuncDataRequired statisDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow){ @@ -521,6 +508,49 @@ bool getMinmaxFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { return true; } +#define GET_TS_LIST(x) ((TSKEY*)((x)->ptsList)) +#define GET_TS_DATA(x, y) (GET_TS_LIST(x)[(y)]) + +#define DO_UPDATE_TAG_COLUMNS_WITHOUT_TS(ctx) \ + do { \ + for (int32_t _i = 0; _i < (ctx)->tagInfo.numOfTagCols; ++_i) { \ + SqlFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[_i]; \ + __ctx->fpSet.process(__ctx); \ + } \ + } while (0); + +#define DO_UPDATE_SUBSID_RES(ctx, ts) \ + do { \ + for (int32_t _i = 0; _i < (ctx)->subsidiaries.num; ++_i) { \ + SqlFunctionCtx* __ctx = (ctx)->subsidiaries.pCtx[_i]; \ + if (__ctx->functionId == FUNCTION_TS_DUMMY) { \ + __ctx->tag.i = (ts); \ + __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; \ + } \ + __ctx->fpSet.process(__ctx); \ + } \ + } while (0) + +#define UPDATE_DATA(ctx, left, right, num, sign, _ts) \ + do { \ + if (((left) < (right)) ^ (sign)) { \ + (left) = (right); \ + DO_UPDATE_SUBSID_RES(ctx, _ts); \ + (num) += 1; \ + } \ + } while (0) + +#define LOOPCHECK_N(val, _col, ctx, _t, _nrow, _start, sign, num) \ + do { \ + _t *d = (_t *)((_col)->pData); \ + for (int32_t i = (_start); i < (_nrow) + (_start); ++i) { \ + if (((_col)->hasNull) && colDataIsNull_f((_col)->nullbitmap, i)) { \ + continue; \ + } \ + TSKEY ts = (ctx)->ptsList != NULL ? GET_TS_DATA(ctx, i) : 0; \ + UPDATE_DATA(ctx, val, d[i], num, sign, ts); \ + } \ + } while (0) int32_t doMinMaxHelper(SqlFunctionCtx *pCtx, int32_t isMinFunc) { int32_t numOfElems = 0; @@ -564,8 +594,8 @@ int32_t doMinMaxHelper(SqlFunctionCtx *pCtx, int32_t isMinFunc) { int64_t val = GET_INT64_VAL(tval); if ((prev < val) ^ isMinFunc) { *(int64_t*) buf = val; - for (int32_t i = 0; i < (pCtx)->subsidiaryRes.numOfCols; ++i) { - SqlFunctionCtx* __ctx = pCtx->subsidiaryRes.pCtx[i]; + for (int32_t i = 0; i < (pCtx)->subsidiaries.num; ++i) { + SqlFunctionCtx* __ctx = pCtx->subsidiaries.pCtx[i]; if (__ctx->functionId == FUNCTION_TS_DUMMY) { // TODO refactor __ctx->tag.i = key; __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; @@ -581,8 +611,8 @@ int32_t doMinMaxHelper(SqlFunctionCtx *pCtx, int32_t isMinFunc) { uint64_t val = GET_UINT64_VAL(tval); if ((prev < val) ^ isMinFunc) { *(uint64_t*) buf = val; - for (int32_t i = 0; i < (pCtx)->subsidiaryRes.numOfCols; ++i) { - SqlFunctionCtx* __ctx = pCtx->subsidiaryRes.pCtx[i]; + for (int32_t i = 0; i < (pCtx)->subsidiaries.num; ++i) { + SqlFunctionCtx* __ctx = pCtx->subsidiaries.pCtx[i]; if (__ctx->functionId == FUNCTION_TS_DUMMY) { // TODO refactor __ctx->tag.i = key; __ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; @@ -797,7 +827,7 @@ int32_t stddevFunction(SqlFunctionCtx* pCtx) { return TSDB_CODE_SUCCESS; } -int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId) { +int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SInputColumnInfoData* pInput = &pCtx->input; int32_t type = pInput->pData[0]->info.type; SStddevRes* pStddevRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); @@ -810,7 +840,7 @@ int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId pStddevRes->result = sqrt(pStddevRes->quadraticDSum/((double)pStddevRes->count) - avg*avg); } - return functionFinalize(pCtx, pBlock, slotId); + return functionFinalize(pCtx, pBlock); } bool getPercentileFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { @@ -923,7 +953,7 @@ int32_t percentileFunction(SqlFunctionCtx *pCtx) { return TSDB_CODE_SUCCESS; } -int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId) { +int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SVariant* pVal = &pCtx->param[1].param; double v = pVal->nType == TSDB_DATA_TYPE_INT ? pVal->i : pVal->d; @@ -936,7 +966,7 @@ int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t sl } tMemBucketDestroy(pMemBucket); - return functionFinalize(pCtx, pBlock, slotId); + return functionFinalize(pCtx, pBlock); } bool getFirstLastFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { @@ -1353,15 +1383,16 @@ static STopBotRes *getTopBotOutputInfo(SqlFunctionCtx *pCtx) { return pRes; } -static void doAddIntoResult(STopBotRes* pRes, int32_t maxSize, void* pData, int32_t rowIndex, SSDataBlock* pSrcBlock, +static void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSDataBlock* pSrcBlock, uint16_t type, uint64_t uid, SResultRowEntryInfo* pEntryInfo); +static void saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STopBotResItem* pItem); +static void copyTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STopBotResItem* pItem); + int32_t topFunction(SqlFunctionCtx *pCtx) { int32_t numOfElems = 0; SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx); - STopBotRes *pRes = getTopBotOutputInfo(pCtx); - // if ((void *)pRes->res[0] != (void *)((char *)pRes + sizeof(STopBotRes) + POINTER_BYTES * pCtx->param[0].i)) { // buildTopBotStruct(pRes, pCtx); // } @@ -1381,7 +1412,7 @@ int32_t topFunction(SqlFunctionCtx *pCtx) { numOfElems++; char* data = colDataGetData(pCol, i); - doAddIntoResult(pRes, pCtx->param[1].param.i, data, i, NULL, type, pInput->uid, pResInfo); + doAddIntoResult(pCtx, data, i, pCtx->pSrcBlock, type, pInput->uid, pResInfo); } return TSDB_CODE_SUCCESS; @@ -1414,9 +1445,11 @@ static int32_t topBotResComparFn(const void *p1, const void *p2, const void *par return (val1->v.d > val2->v.d) ? 1 : -1; } +void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSDataBlock* pSrcBlock, uint16_t type, + uint64_t uid, SResultRowEntryInfo* pEntryInfo) { + STopBotRes *pRes = getTopBotOutputInfo(pCtx); + int32_t maxSize = pCtx->param[1].param.i; -void doAddIntoResult(STopBotRes *pRes, int32_t maxSize, void *pData, int32_t rowIndex, SSDataBlock* pSrcBlock, uint16_t type, - uint64_t uid, SResultRowEntryInfo* pEntryInfo) { SVariant val = {0}; taosVariantCreateFromBinary(&val, pData, tDataTypes[type].bytes, type); @@ -1428,22 +1461,9 @@ void doAddIntoResult(STopBotRes *pRes, int32_t maxSize, void *pData, int32_t row STopBotResItem* pItem = &pItems[pEntryInfo->numOfRes]; pItem->v = val; pItem->uid = uid; - pItem->tuplePos.pageId = -1; // todo set the corresponding tuple data in the disk-based buffer - if (pRes->pageId == -1) { - SFilePage* pPage = getNewBufPage(NULL, 0, &pRes->pageId); - pPage->num = sizeof(SFilePage); - - // keep the current row data - for(int32_t i = 0; i < pSrcBlock->info.numOfCols; ++i) { - SColumnInfoData* pCol = taosArrayGet(pSrcBlock->pDataBlock, i); - bool isNull = colDataIsNull_s(pCol, rowIndex); - - - colDataGetData(pCol, rowIndex); - } - - } + // save the data of this tuple + saveTupleData(pCtx, rowIndex, pSrcBlock, pItem); // allocate the buffer and keep the data of this row into the new allocated buffer pEntryInfo->numOfRes++; @@ -1452,22 +1472,100 @@ void doAddIntoResult(STopBotRes *pRes, int32_t maxSize, void *pData, int32_t row if ((IS_SIGNED_NUMERIC_TYPE(type) && val.i > pItems[0].v.i) || (IS_UNSIGNED_NUMERIC_TYPE(type) && val.u > pItems[0].v.u) || (IS_FLOAT_TYPE(type) && val.d > pItems[0].v.d)) { + // replace the old data and the coresponding tuple data STopBotResItem* pItem = &pItems[0]; pItem->v = val; pItem->uid = uid; - pItem->tuplePos.pageId = -1; // todo set the corresponding tuple data in the disk-based buffer + + // save the data of this tuple by over writing the old data + copyTupleData(pCtx, rowIndex, pSrcBlock, pItem); taosheapadjust((void *) pItems, sizeof(STopBotResItem), 0, pEntryInfo->numOfRes - 1, (const void *) &type, topBotResComparFn, NULL, false); } } } -int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId) { +void saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STopBotResItem* pItem) { + SFilePage* pPage = NULL; + + int32_t completeRowSize = pSrcBlock->info.rowSize + pSrcBlock->info.numOfCols * sizeof(bool); + + if (pCtx->curBufPage == -1) { + pPage = getNewBufPage(pCtx->pBuf, 0, &pCtx->curBufPage); + pPage->num = sizeof(SFilePage); + } else { + pPage = getBufPage(pCtx->pBuf, pCtx->curBufPage); + if (pPage->num + completeRowSize > getBufPageSize(pCtx->pBuf)) { + pPage = getNewBufPage(pCtx->pBuf, 0, &pCtx->curBufPage); + pPage->num = sizeof(SFilePage); + } + } + + pItem->tuplePos.pageId = pCtx->curBufPage; + + // keep the current row data, extract method + int32_t offset = 0; + bool* nullList = (bool*)((char*)pPage + pPage->num); + char* pStart = (char*)(nullList + sizeof(bool) * pSrcBlock->info.numOfCols); + for (int32_t i = 0; i < pSrcBlock->info.numOfCols; ++i) { + SColumnInfoData* pCol = taosArrayGet(pSrcBlock->pDataBlock, i); + bool isNull = colDataIsNull_s(pCol, rowIndex); + if (isNull) { + nullList[i] = true; + continue; + } + + char* p = colDataGetData(pCol, rowIndex); + if (IS_VAR_DATA_TYPE(pCol->info.type)) { + memcpy(pStart + offset, p, varDataTLen(p)); + } else { + memcpy(pStart + offset, p, pCol->info.bytes); + } + + offset += pCol->info.bytes; + } + + pItem->tuplePos.offset = pPage->num; + pPage->num += completeRowSize; + + setBufPageDirty(pPage, true); + releaseBufPage(pCtx->pBuf, pPage); +} + +void copyTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STopBotResItem* pItem) { + SFilePage* pPage = getBufPage(pCtx->pBuf, pItem->tuplePos.pageId); + + bool* nullList = (bool*)((char*)pPage + pItem->tuplePos.offset); + char* pStart = (char*)(nullList + pSrcBlock->info.numOfCols * sizeof(bool)); + + int32_t offset = 0; + for(int32_t i = 0; i < pSrcBlock->info.numOfCols; ++i) { + SColumnInfoData* pCol = taosArrayGet(pSrcBlock->pDataBlock, i); + if ((nullList[i] = colDataIsNull_s(pCol, rowIndex)) == true) { + continue; + } + + char* p = colDataGetData(pCol, rowIndex); + if (IS_VAR_DATA_TYPE(pCol->info.type)) { + memcpy(pStart + offset, p, varDataTLen(p)); + } else { + memcpy(pStart + offset, p, pCol->info.bytes); + } + + offset += pCol->info.bytes; + } + + setBufPageDirty(pPage, true); + releaseBufPage(pCtx->pBuf, pPage); +} + +int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SResultRowEntryInfo *pEntryInfo = GET_RES_INFO(pCtx); STopBotRes* pRes = GET_ROWCELL_INTERBUF(pEntryInfo); pEntryInfo->complete = true; int32_t type = pCtx->input.pData[0]->info.type; + int32_t slotId = pCtx->pExpr->base.resSchema.slotId; SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); // todo assign the tag value and the corresponding row data @@ -1476,19 +1574,45 @@ int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId case TSDB_DATA_TYPE_INT: { for (int32_t i = 0; i < pEntryInfo->numOfRes; ++i) { STopBotResItem* pItem = &pRes->pItems[i]; - colDataAppendInt32(pCol, currentRow++, (int32_t*)&pItem->v.i); + colDataAppendInt32(pCol, currentRow, (int32_t*)&pItem->v.i); int32_t pageId = pItem->tuplePos.pageId; int32_t offset = pItem->tuplePos.offset; - if (pageId != -1) { - // todo + if (pItem->tuplePos.pageId != -1) { + SFilePage* pPage = getBufPage(pCtx->pBuf, pageId); + + bool* nullList = (bool*)((char*)pPage + offset); + char* pStart = (char*)(nullList + pCtx->pSrcBlock->info.numOfCols * sizeof(bool)); + + // todo set the offset value to optimize the performance. + for (int32_t j = 0; j < pCtx->subsidiaries.num; ++j) { + SqlFunctionCtx* pc = pCtx->subsidiaries.pCtx[j]; + + SFunctParam *pFuncParam = &pc->pExpr->base.pParam[0]; + int32_t srcSlotId = pFuncParam->pCol->slotId; + int32_t dstSlotId = pCtx->pExpr->base.resSchema.slotId; + + int32_t ps = 0; + for(int32_t k = 0; k < srcSlotId; ++k) { + SColumnInfoData* pSrcCol = taosArrayGet(pCtx->pSrcBlock->pDataBlock, k); + ps += pSrcCol->info.bytes; + } + + SColumnInfoData* pDstCol = taosArrayGet(pBlock->pDataBlock, dstSlotId); + if (nullList[srcSlotId]) { + colDataAppendNULL(pDstCol, currentRow); + } else { + colDataAppend(pDstCol, currentRow, (pStart + ps), false); + } + } } + + currentRow += 1; } + break; } } return pEntryInfo->numOfRes; - -// return functionFinalize(pCtx, pBlock, slotId); } diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 317339af04..f8a7e77814 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -124,7 +124,7 @@ enum { int64_t gUdfTaskSeqNum = 0; typedef struct SUdfdProxy { - int32_t dnodeId; + char udfdPipeName[UDF_LISTEN_PIPE_NAME_LEN]; uv_barrier_t gUdfInitBarrier; uv_loop_t gUdfdLoop; @@ -137,11 +137,11 @@ typedef struct SUdfdProxy { int8_t gUdfcState; QUEUE gUdfTaskQueue; QUEUE gUvProcTaskQueue; - // int8_t gUdfcState = UDFC_STATE_INITAL; - // QUEUE gUdfTaskQueue = {0}; - // QUEUE gUvProcTaskQueue = {0}; + + int8_t initialized; } SUdfdProxy; +SUdfdProxy gUdfdProxy = {0}; typedef struct SUdfUvSession { SUdfdProxy *udfc; @@ -209,19 +209,27 @@ enum { UDFC_STATE_STARTNG, // starting after udfcOpen UDFC_STATE_READY, // started and begin to receive quests UDFC_STATE_STOPPING, // stopping after udfcClose - UDFC_STATUS_FINAL, // stopped }; +int32_t getUdfdPipeName(char* pipeName, int32_t size) { + char dnodeId[8] = {0}; + size_t dnodeIdSize; + int32_t err = uv_os_getenv(UDF_DNODE_ID_ENV_NAME, dnodeId, &dnodeIdSize); + if (err != 0) { + dnodeId[0] = '1'; + } + snprintf(pipeName, size, "%s%s", UDF_LISTEN_PIPE_NAME_PREFIX, dnodeId); + return 0; +} + int32_t encodeUdfSetupRequest(void **buf, const SUdfSetupRequest *setup) { int32_t len = 0; len += taosEncodeBinary(buf, setup->udfName, TSDB_FUNC_NAME_LEN); - len += taosEncodeSEpSet(buf, &setup->epSet); return len; } void* decodeUdfSetupRequest(const void* buf, SUdfSetupRequest *request) { buf = taosDecodeBinaryTo(buf, request->udfName, TSDB_FUNC_NAME_LEN); - buf = taosDecodeSEpSet((void*)buf, &request->epSet); return (void*)buf; } @@ -604,7 +612,7 @@ void onUdfcPipeClose(uv_handle_t *handle) { } int32_t udfcGetUvTaskResponseResult(SClientUdfTask *task, SClientUvTaskNode *uvTask) { - debugPrint("%s", "get uv task result"); + fnDebug("udfc get uv task result. task: %p", task); if (uvTask->type == UV_TASK_REQ_RSP) { if (uvTask->rspBuf.base != NULL) { SUdfResponse rsp; @@ -647,7 +655,6 @@ int32_t udfcGetUvTaskResponseResult(SClientUdfTask *task, SClientUvTaskNode *uvT } void udfcAllocateBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) { - debugPrint("%s", "client allocate buffer to receive from pipe"); SClientUvConn *conn = handle->data; SClientConnBuf *connBuf = &conn->readBuf; @@ -662,7 +669,7 @@ void udfcAllocateBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf buf->base = connBuf->buf; buf->len = connBuf->cap; } else { - //TODO: log error + fnError("udfc allocate buffer failure. size: %d", msgHeadSize); buf->base = NULL; buf->len = 0; } @@ -674,13 +681,13 @@ void udfcAllocateBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf buf->base = connBuf->buf + connBuf->len; buf->len = connBuf->cap - connBuf->len; } else { - //TODO: log error free connBuf->buf + fnError("udfc re-allocate buffer failure. size: %d", connBuf->cap); buf->base = NULL; buf->len = 0; } } - debugPrint("\tconn buf cap - len - total : %d - %d - %d", connBuf->cap, connBuf->len, connBuf->total); + fnTrace("conn buf cap - len - total : %d - %d - %d", connBuf->cap, connBuf->len, connBuf->total); } @@ -689,6 +696,7 @@ bool isUdfcUvMsgComplete(SClientConnBuf *connBuf) { connBuf->total = *(int32_t *) (connBuf->buf); } if (connBuf->len == connBuf->cap && connBuf->total == connBuf->cap) { + fnTrace("udfc complete message is received, now handle it"); return true; } return false; @@ -696,10 +704,10 @@ bool isUdfcUvMsgComplete(SClientConnBuf *connBuf) { void udfcUvHandleRsp(SClientUvConn *conn) { SClientConnBuf *connBuf = &conn->readBuf; - int64_t seqNum = *(int64_t *) (connBuf->buf + sizeof(int32_t)); // msglen int32_t then seqnum + int64_t seqNum = *(int64_t *) (connBuf->buf + sizeof(int32_t)); // msglen then seqnum if (QUEUE_EMPTY(&conn->taskQueue)) { - //LOG error + fnError("udfc no task waiting for response on connection"); return; } bool found = false; @@ -713,7 +721,7 @@ void udfcUvHandleRsp(SClientUvConn *conn) { found = true; taskFound = task; } else { - //LOG error; + fnError("udfc more than one task waiting for the same response"); continue; } } @@ -727,7 +735,7 @@ void udfcUvHandleRsp(SClientUvConn *conn) { uv_sem_post(&taskFound->taskSem); QUEUE_REMOVE(&taskFound->procTaskQueue); } else { - //TODO: LOG error + fnError("no task is waiting for the response."); } connBuf->buf = NULL; connBuf->total = -1; @@ -751,7 +759,7 @@ void udfcUvHandleError(SClientUvConn *conn) { } void onUdfcRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { - debugPrint("%s, nread: %zd", "client read from pipe", nread); + fnTrace("udfc client %p, client read from pipe. nread: %zd", client, nread); if (nread == 0) return; SClientUvConn *conn = client->data; @@ -764,9 +772,9 @@ void onUdfcRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { } if (nread < 0) { - debugPrint("\tclient read error: %s", uv_strerror(nread)); + fnError("udfc client pipe %p read error: %s", client, uv_strerror(nread)); if (nread == UV_EOF) { - //TODO: + fnError("udfc client pipe %p closed", client); } udfcUvHandleError(conn); } @@ -774,16 +782,15 @@ void onUdfcRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { } void onUdfClientWrite(uv_write_t *write, int status) { - debugPrint("%s", "after writing to pipe"); SClientUvTaskNode *uvTask = write->data; + uv_pipe_t *pipe = uvTask->pipe; if (status == 0) { - uv_pipe_t *pipe = uvTask->pipe; SClientUvConn *conn = pipe->data; QUEUE_INSERT_TAIL(&conn->taskQueue, &uvTask->connTaskQueue); } else { - //TODO Log error; + fnError("udfc client %p write error.", pipe); } - debugPrint("\tlength:%zu", uvTask->reqBuf.len); + fnTrace("udfc client %p write length:%zu", pipe, uvTask->reqBuf.len); taosMemoryFree(write); taosMemoryFree(uvTask->reqBuf.base); } @@ -841,7 +848,7 @@ int32_t createUdfcUvTask(SClientUdfTask *task, int8_t uvTaskType, SClientUvTaskN } int32_t queueUvUdfTask(SClientUvTaskNode *uvTask) { - debugPrint("%s, %d", "queue uv task", uvTask->type); + fnTrace("queue uv task to event loop, task: %d, %p", uvTask->type, uvTask); SUdfdProxy *udfc = uvTask->udfc; uv_mutex_lock(&udfc->gUdfTaskQueueMutex); QUEUE_INSERT_TAIL(&udfc->gUdfTaskQueue, &uvTask->recvTaskQueue); @@ -855,7 +862,7 @@ int32_t queueUvUdfTask(SClientUvTaskNode *uvTask) { } int32_t startUvUdfTask(SClientUvTaskNode *uvTask) { - debugPrint("%s, type %d", "start uv task ", uvTask->type); + fnTrace("event loop start uv task. task: %d, %p", uvTask->type, uvTask); switch (uvTask->type) { case UV_TASK_CONNECT: { uv_pipe_t *pipe = taosMemoryMalloc(sizeof(uv_pipe_t)); @@ -874,8 +881,7 @@ int32_t startUvUdfTask(SClientUvTaskNode *uvTask) { uv_connect_t *connReq = taosMemoryMalloc(sizeof(uv_connect_t)); connReq->data = uvTask; - - uv_pipe_connect(connReq, pipe, "udf.sock", onUdfClientConnect); + uv_pipe_connect(connReq, pipe, uvTask->udfc->udfdPipeName, onUdfClientConnect); break; } case UV_TASK_REQ_RSP: { @@ -971,27 +977,37 @@ void constructUdfService(void *argsThread) { uv_loop_close(&udfc->gUdfdLoop); } -int32_t udfcOpen(int32_t dnodeId, UdfcHandle *udfc) { - SUdfdProxy *proxy = taosMemoryCalloc(1, sizeof(SUdfdProxy)); - proxy->dnodeId = dnodeId; +int32_t udfcOpen() { + int8_t old = atomic_val_compare_exchange_8(&gUdfdProxy.initialized, 0, 1); + if (old == 1) { + return 0; + } + SUdfdProxy *proxy = &gUdfdProxy; + getUdfdPipeName(proxy->udfdPipeName, UDF_LISTEN_PIPE_NAME_LEN); proxy->gUdfcState = UDFC_STATE_STARTNG; uv_barrier_init(&proxy->gUdfInitBarrier, 2); uv_thread_create(&proxy->gUdfLoopThread, constructUdfService, proxy); - uv_barrier_wait(&proxy->gUdfInitBarrier); + atomic_store_8(&proxy->gUdfcState, UDFC_STATE_READY); proxy->gUdfcState = UDFC_STATE_READY; - *udfc = proxy; + uv_barrier_wait(&proxy->gUdfInitBarrier); + fnInfo("udfc initialized") return 0; } -int32_t udfcClose(UdfcHandle udfcHandle) { - SUdfdProxy *udfc = udfcHandle; +int32_t udfcClose() { + int8_t old = atomic_val_compare_exchange_8(&gUdfdProxy.initialized, 1, 0); + if (old == 0) { + return 0; + } + + SUdfdProxy *udfc = &gUdfdProxy; udfc->gUdfcState = UDFC_STATE_STOPPING; uv_async_send(&udfc->gUdfLoopStopAsync); uv_thread_join(&udfc->gUdfLoopThread); uv_mutex_destroy(&udfc->gUdfTaskQueueMutex); uv_barrier_destroy(&udfc->gUdfInitBarrier); - udfc->gUdfcState = UDFC_STATUS_FINAL; - taosMemoryFree(udfc); + udfc->gUdfcState = UDFC_STATE_INITAL; + fnInfo("udfc cleaned up"); return 0; } @@ -1009,12 +1025,15 @@ int32_t udfcRunUvTask(SClientUdfTask *task, int8_t uvTaskType) { return task->errCode; } -int32_t setupUdf(UdfcHandle udfc, char udfName[], SEpSet *epSet, UdfcFuncHandle *funcHandle) { - debugPrint("%s", "client setup udf"); +int32_t setupUdf(char udfName[], UdfcFuncHandle *funcHandle) { + fnInfo("udfc setup udf. udfName: %s", udfName); + if (gUdfdProxy.gUdfcState != UDFC_STATE_READY) { + return UDFC_CODE_INVALID_STATE; + } SClientUdfTask *task = taosMemoryMalloc(sizeof(SClientUdfTask)); task->errCode = 0; task->session = taosMemoryMalloc(sizeof(SUdfUvSession)); - task->session->udfc = udfc; + task->session->udfc = &gUdfdProxy; task->type = UDF_TASK_SETUP; SUdfSetupRequest *req = &task->_setup.req; @@ -1022,15 +1041,20 @@ int32_t setupUdf(UdfcHandle udfc, char udfName[], SEpSet *epSet, UdfcFuncHandle int32_t errCode = udfcRunUvTask(task, UV_TASK_CONNECT); if (errCode != 0) { - //TODO: log error - return -1; + fnError("failed to connect to pipe. udfName: %s, pipe: %s", udfName, (&gUdfdProxy)->udfdPipeName); + return UDFC_CODE_CONNECT_PIPE_ERR; } udfcRunUvTask(task, UV_TASK_REQ_RSP); SUdfSetupResponse *rsp = &task->_setup.rsp; task->session->severHandle = rsp->udfHandle; - *funcHandle = task->session; + if (task->errCode != 0) { + fnError("failed to setup udf. err: %d", task->errCode) + } else { + fnInfo("sucessfully setup udf func handle. handle: %p", task->session); + *funcHandle = task->session; + } int32_t err = task->errCode; taosMemoryFree(task); return err; @@ -1038,7 +1062,7 @@ int32_t setupUdf(UdfcHandle udfc, char udfName[], SEpSet *epSet, UdfcFuncHandle int32_t callUdf(UdfcFuncHandle handle, int8_t callType, SSDataBlock *input, SUdfInterBuf *state, SUdfInterBuf *state2, SSDataBlock* output, SUdfInterBuf *newState) { - debugPrint("%s", "client call udf"); + fnTrace("udfc call udf. callType: %d, funcHandle: %p", callType, handle); SClientUdfTask *task = taosMemoryMalloc(sizeof(SClientUdfTask)); task->errCode = 0; @@ -1076,35 +1100,37 @@ int32_t callUdf(UdfcFuncHandle handle, int8_t callType, SSDataBlock *input, SUdf udfcRunUvTask(task, UV_TASK_REQ_RSP); - SUdfCallResponse *rsp = &task->_call.rsp; - switch (callType) { - case TSDB_UDF_CALL_AGG_INIT: { - *newState = rsp->resultBuf; - break; - } - case TSDB_UDF_CALL_AGG_PROC: { - *newState = rsp->resultBuf; - break; - } - case TSDB_UDF_CALL_AGG_MERGE: { - *newState = rsp->resultBuf; - break; - } - case TSDB_UDF_CALL_AGG_FIN: { - *newState = rsp->resultBuf; - break; - } - case TSDB_UDF_CALL_SCALA_PROC: { - *output = rsp->resultData; - break; + if (task->errCode != 0) { + fnError("call udf failure. err: %d", task->errCode); + } else { + SUdfCallResponse *rsp = &task->_call.rsp; + switch (callType) { + case TSDB_UDF_CALL_AGG_INIT: { + *newState = rsp->resultBuf; + break; + } + case TSDB_UDF_CALL_AGG_PROC: { + *newState = rsp->resultBuf; + break; + } + case TSDB_UDF_CALL_AGG_MERGE: { + *newState = rsp->resultBuf; + break; + } + case TSDB_UDF_CALL_AGG_FIN: { + *newState = rsp->resultBuf; + break; + } + case TSDB_UDF_CALL_SCALA_PROC: { + *output = rsp->resultData; + break; + } } } - taosMemoryFree(task); return task->errCode; } -//TODO: translate these calls to callUdf int32_t callUdfAggInit(UdfcFuncHandle handle, SUdfInterBuf *interBuf) { int8_t callType = TSDB_UDF_CALL_AGG_INIT; @@ -1148,7 +1174,7 @@ int32_t callUdfScalarFunc(UdfcFuncHandle handle, SScalarParam *input, int32_t nu } int32_t teardownUdf(UdfcFuncHandle handle) { - debugPrint("%s", "client teardown udf"); + fnInfo("tear down udf. udf func handle: %p", handle); SClientUdfTask *task = taosMemoryMalloc(sizeof(SClientUdfTask)); task->errCode = 0; @@ -1160,7 +1186,6 @@ int32_t teardownUdf(UdfcFuncHandle handle) { udfcRunUvTask(task, UV_TASK_REQ_RSP); - SUdfTeardownResponse *rsp = &task->_teardown.rsp; int32_t err = task->errCode; diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index 6540851758..e4c4cb4893 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -20,6 +20,7 @@ #include "tudf.h" #include "tudfInt.h" +#include "tdatablock.h" #include "tdataformat.h" #include "tglobal.h" #include "tmsg.h" @@ -31,8 +32,9 @@ typedef struct SUdfdContext { uv_signal_t intrSignal; char listenPipeName[UDF_LISTEN_PIPE_NAME_LEN]; uv_pipe_t listeningPipe; - void *clientRpc; + void *clientRpc; + SCorEpSet mgmtEp; uv_mutex_t udfsMutex; SHashObj *udfsHash; @@ -63,8 +65,13 @@ typedef struct SUdf { uv_mutex_t lock; uv_cond_t condReady; - char name[16]; - int8_t type; + char name[TSDB_FUNC_NAME_LEN]; + int8_t funcType; + int8_t scriptType; + int8_t outputType; + int32_t outputLen; + int32_t bufSize; + char path[PATH_MAX]; uv_lib_t lib; @@ -78,17 +85,17 @@ typedef struct SUdfcFuncHandle { SUdf *udf; } SUdfcFuncHandle; -int32_t udfdFillUdfInfoFromMNode(void *clientRpc, SEpSet *pEpSet, char *udfName, SUdf *udf); +int32_t udfdFillUdfInfoFromMNode(void *clientRpc, char *udfName, SUdf *udf); -int32_t udfdLoadUdf(char *udfName, SEpSet *pEpSet, SUdf *udf) { +int32_t udfdLoadUdf(char *udfName, SUdf *udf) { strcpy(udf->name, udfName); - udfdFillUdfInfoFromMNode(global.clientRpc, pEpSet, udf->name, udf); - + udfdFillUdfInfoFromMNode(global.clientRpc, udf->name, udf); + //strcpy(udf->path, "/home/slzhou/TDengine/debug/build/lib/libudf1.so"); int err = uv_dlopen(udf->path, &udf->lib); if (err != 0) { fnError("can not load library %s. error: %s", udf->path, uv_strerror(err)); - // TODO set error + return UDFC_CODE_LOAD_UDF_FAILURE; } // TODO: find all the functions char normalFuncName[TSDB_FUNC_NAME_LEN] = {0}; @@ -115,8 +122,8 @@ void udfdProcessRequest(uv_work_t *req) { SUdf *udf = NULL; uv_mutex_lock(&global.udfsMutex); - SUdf **udfInHash = taosHashGet(global.udfsHash, request.setup.udfName, TSDB_FUNC_NAME_LEN); - if (*udfInHash) { + SUdf **udfInHash = taosHashGet(global.udfsHash, request.setup.udfName, strlen(request.setup.udfName)); + if (udfInHash) { ++(*udfInHash)->refCount; udf = *udfInHash; uv_mutex_unlock(&global.udfsMutex); @@ -128,14 +135,14 @@ void udfdProcessRequest(uv_work_t *req) { uv_mutex_init(&udfNew->lock); uv_cond_init(&udfNew->condReady); udf = udfNew; - taosHashPut(global.udfsHash, request.setup.udfName, TSDB_FUNC_NAME_LEN, &udfNew, sizeof(&udfNew)); + taosHashPut(global.udfsHash, request.setup.udfName, strlen(request.setup.udfName), &udfNew, sizeof(&udfNew)); uv_mutex_unlock(&global.udfsMutex); } uv_mutex_lock(&udf->lock); if (udf->state == UDF_STATE_INIT) { udf->state = UDF_STATE_LOADING; - udfdLoadUdf(setup->udfName, &setup->epSet, udf); + udfdLoadUdf(setup->udfName, udf); udf->state = UDF_STATE_READY; uv_cond_broadcast(&udf->condReady); uv_mutex_unlock(&udf->lock); @@ -214,7 +221,7 @@ void udfdProcessRequest(uv_work_t *req) { udf->refCount--; if (udf->refCount == 0) { unloadUdf = true; - taosHashRemove(global.udfsHash, udf->name, TSDB_FUNC_NAME_LEN); + taosHashRemove(global.udfsHash, udf->name, strlen(udf->name)); } uv_mutex_unlock(&global.udfsMutex); if (unloadUdf) { @@ -393,7 +400,48 @@ void udfdIntrSignalHandler(uv_signal_t *handle, int signum) { void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { return; } -int32_t udfdFillUdfInfoFromMNode(void *clientRpc, SEpSet *pEpSet, char *udfName, SUdf *udf) { +int initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet) { + pEpSet->version = 0; + + // init mnode ip set + SEpSet* mgmtEpSet = &(pEpSet->epSet); + mgmtEpSet->numOfEps = 0; + mgmtEpSet->inUse = 0; + + if (firstEp && firstEp[0] != 0) { + if (strlen(firstEp) >= TSDB_EP_LEN) { + terrno = TSDB_CODE_TSC_INVALID_FQDN; + return -1; + } + + int32_t code = taosGetFqdnPortFromEp(firstEp, &mgmtEpSet->eps[0]); + if (code != TSDB_CODE_SUCCESS) { + terrno = TSDB_CODE_TSC_INVALID_FQDN; + return terrno; + } + + mgmtEpSet->numOfEps++; + } + + if (secondEp && secondEp[0] != 0) { + if (strlen(secondEp) >= TSDB_EP_LEN) { + terrno = TSDB_CODE_TSC_INVALID_FQDN; + return -1; + } + + taosGetFqdnPortFromEp(secondEp, &mgmtEpSet->eps[mgmtEpSet->numOfEps]); + mgmtEpSet->numOfEps++; + } + + if (mgmtEpSet->numOfEps == 0) { + terrno = TSDB_CODE_TSC_INVALID_FQDN; + return -1; + } + + return 0; +} + +int32_t udfdFillUdfInfoFromMNode(void *clientRpc, char *udfName, SUdf *udf) { SRetrieveFuncReq retrieveReq = {0}; retrieveReq.numOfFuncs = 1; retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN); @@ -410,15 +458,21 @@ int32_t udfdFillUdfInfoFromMNode(void *clientRpc, SEpSet *pEpSet, char *udfName, rpcMsg.msgType = TDMT_MND_RETRIEVE_FUNC; SRpcMsg rpcRsp = {0}; - rpcSendRecv(clientRpc, pEpSet, &rpcMsg, &rpcRsp); + rpcSendRecv(clientRpc, &global.mgmtEp.epSet, &rpcMsg, &rpcRsp); SRetrieveFuncRsp retrieveRsp = {0}; tDeserializeSRetrieveFuncRsp(rpcRsp.pCont, rpcRsp.contLen, &retrieveRsp); SFuncInfo *pFuncInfo = (SFuncInfo *)taosArrayGet(retrieveRsp.pFuncInfos, 0); + udf->funcType = pFuncInfo->funcType; + udf->scriptType = pFuncInfo->scriptType; + udf->outputType = pFuncInfo->funcType; + udf->outputLen = pFuncInfo->outputLen; + udf->bufSize = pFuncInfo->bufSize; + char path[PATH_MAX] = {0}; - taosGetTmpfilePath("/tmp", "libudf", path); - TdFilePtr file = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC); + snprintf(path, sizeof(path), "%s/lib%s.so", "/tmp", udfName); + TdFilePtr file = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC | TD_FILE_AUTO_DEL); // TODO check for failure of flush to disk taosWriteFile(file, pFuncInfo->pCode, pFuncInfo->codeSize); taosCloseFile(&file); @@ -531,15 +585,7 @@ static int32_t udfdUvInit() { uv_pipe_open(&global.ctrlPipe, 0); uv_read_start((uv_stream_t *)&global.ctrlPipe, udfdCtrlAllocBufCb, udfdCtrlReadCb); - char dnodeId[8] = {0}; - size_t dnodeIdSize; - int32_t err = uv_os_getenv("DNODE_ID", dnodeId, &dnodeIdSize); - if (err != 0) { - dnodeId[0] = '1'; - } - char listenPipeName[32] = {0}; - snprintf(listenPipeName, sizeof(listenPipeName), "%s%s", UDF_LISTEN_PIPE_NAME_PREFIX, dnodeId); - strcpy(global.listenPipeName, listenPipeName); + getUdfdPipeName(global.listenPipeName, UDF_LISTEN_PIPE_NAME_LEN); removeListeningPipe(); @@ -550,7 +596,7 @@ static int32_t udfdUvInit() { int r; fnInfo("bind to pipe %s", global.listenPipeName); - if ((r = uv_pipe_bind(&global.listeningPipe, listenPipeName))) { + if ((r = uv_pipe_bind(&global.listeningPipe, global.listenPipeName))) { fnError("Bind error %s", uv_err_name(r)); removeListeningPipe(); return -1; @@ -580,7 +626,7 @@ static int32_t udfdRun() { fnInfo("start the udfd"); int code = uv_run(global.loop, UV_RUN_DEFAULT); - fnInfo("udfd stopped. result: %s", uv_err_name(code)); + fnInfo("udfd stopped. result: %s, code: %d", uv_err_name(code), code); int codeClose = uv_loop_close(global.loop); fnDebug("uv loop close. result: %s", uv_err_name(codeClose)); udfdCloseClientRpc(); @@ -615,5 +661,6 @@ int main(int argc, char *argv[]) { return -1; } + initEpSetFromCfg(tsFirst, tsSecond, &global.mgmtEp); return udfdRun(); } diff --git a/source/libs/function/test/runUdf.c b/source/libs/function/test/runUdf.c index 0727a4a1d2..8e5eac538e 100644 --- a/source/libs/function/test/runUdf.c +++ b/source/libs/function/test/runUdf.c @@ -1,61 +1,84 @@ #include #include #include - #include "uv.h" + +#include "fnLog.h" #include "os.h" -#include "tudf.h" #include "tdatablock.h" +#include "tglobal.h" +#include "tudf.h" + +static int32_t parseArgs(int32_t argc, char *argv[]) { + for (int32_t i = 1; i < argc; ++i) { + if (strcmp(argv[i], "-c") == 0) { + if (i < argc - 1) { + if (strlen(argv[++i]) >= PATH_MAX) { + printf("config file path overflow"); + return -1; + } + tstrncpy(configDir, argv[i], PATH_MAX); + } else { + printf("'-c' requires a parameter, default is %s\n", configDir); + return -1; + } + } + } + + return 0; +} + +static int32_t initLog() { + char logName[12] = {0}; + snprintf(logName, sizeof(logName), "%slog", "udfc"); + return taosCreateLog(logName, 1, configDir, NULL, NULL, NULL, 0); +} int main(int argc, char *argv[]) { - UdfcHandle udfc; - udfcOpen(1, &udfc); - uv_sleep(1000); - char path[256] = {0}; - size_t cwdSize = 256; - int err = uv_cwd(path, &cwdSize); - if (err != 0) { - fprintf(stderr, "err cwd: %s\n", uv_strerror(err)); - return err; + parseArgs(argc, argv); + initLog(); + if (taosInitCfg(configDir, NULL, NULL, NULL, 0) != 0) { + fnError("failed to start since read config error"); + return -1; + } + + udfcOpen(); + uv_sleep(1000); + + UdfcFuncHandle handle; + + setupUdf("udf1", &handle); + + SSDataBlock block = {0}; + SSDataBlock *pBlock = █ + pBlock->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); + pBlock->info.numOfCols = 1; + pBlock->info.rows = 4; + char data[16] = {0}; + char bitmap[4] = {0}; + for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + SColumnInfoData colInfo = {0}; + colInfo.info.type = TSDB_DATA_TYPE_INT; + colInfo.info.bytes = sizeof(int32_t); + colInfo.info.colId = 1; + colInfo.pData = data; + colInfo.nullbitmap = bitmap; + for (int32_t j = 0; j < pBlock->info.rows; ++j) { + colDataAppendInt32(&colInfo, j, &j); } - fprintf(stdout, "current working directory:%s\n", path); - strcat(path, "/libudf1.so"); + taosArrayPush(pBlock->pDataBlock, &colInfo); + } - UdfcFuncHandle handle; - SEpSet epSet; - setupUdf(udfc, "udf1", &epSet, &handle); + SScalarParam input = {0}; + input.numOfRows = pBlock->info.rows; + input.columnData = taosArrayGet(pBlock->pDataBlock, 0); + SScalarParam output = {0}; + callUdfScalarFunc(handle, &input, 1, &output); - SSDataBlock block = {0}; - SSDataBlock* pBlock = █ - pBlock->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); - pBlock->info.numOfCols = 1; - pBlock->info.rows = 4; - char data[16] = {0}; - char bitmap[4] = {0}; - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { - SColumnInfoData colInfo = {0}; - colInfo.info.type = TSDB_DATA_TYPE_INT; - colInfo.info.bytes = sizeof(int32_t); - colInfo.info.colId = 1; - colInfo.pData = data; - colInfo.nullbitmap = bitmap; - for (int32_t j = 0; j < pBlock->info.rows; ++j) { - colDataAppendInt32(&colInfo, j, &j); - } - taosArrayPush(pBlock->pDataBlock, &colInfo); - } - - SScalarParam input = {0}; - input.numOfRows = pBlock->info.rows; - input.columnData = taosArrayGet(pBlock->pDataBlock, 0); - SScalarParam output = {0}; - callUdfScalarFunc(handle, &input, 1 , &output); - - SColumnInfoData *col = output.columnData; - for (int32_t i = 0; i < output.numOfRows; ++i) { - fprintf(stderr, "%d\t%d\n" , i, *(int32_t*)(col->pData + i *sizeof(int32_t))); - } - teardownUdf(handle); - - udfcClose(udfc); + SColumnInfoData *col = output.columnData; + for (int32_t i = 0; i < output.numOfRows; ++i) { + fprintf(stderr, "%d\t%d\n", i, *(int32_t *)(col->pData + i * sizeof(int32_t))); + } + teardownUdf(handle); + udfcClose(); } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index a3f8c3aed5..e74a5ad6a9 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -198,7 +198,7 @@ static int32_t getDBVgVersion(STranslateContext* pCxt, const char* pDbFName, int static int32_t getDBCfg(STranslateContext* pCxt, const char* pDbName, SDbCfgInfo* pInfo) { SParseContext* pParCxt = pCxt->pParseCxt; - SName name; + SName name; tNameSetDbName(&name, pCxt->pParseCxt->acctId, pDbName, strlen(pDbName)); char dbFname[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(&name, dbFname); @@ -560,8 +560,7 @@ static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) { pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes; } } else if (nodesIsComparisonOp(pOp)) { - if (TSDB_DATA_TYPE_BLOB == ldt.type || TSDB_DATA_TYPE_JSON == rdt.type || - TSDB_DATA_TYPE_BLOB == rdt.type) { + if (TSDB_DATA_TYPE_BLOB == ldt.type || TSDB_DATA_TYPE_JSON == rdt.type || TSDB_DATA_TYPE_BLOB == rdt.type) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); } if (OP_TYPE_IN == pOp->opType || OP_TYPE_NOT_IN == pOp->opType) { @@ -569,7 +568,7 @@ static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) { } pOp->node.resType.type = TSDB_DATA_TYPE_BOOL; pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; - } else if (nodesIsJsonOp(pOp)){ + } else if (nodesIsJsonOp(pOp)) { if (TSDB_DATA_TYPE_JSON != ldt.type || TSDB_DATA_TYPE_BINARY != rdt.type) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); } @@ -588,7 +587,9 @@ static EDealRes haveAggFunction(SNode* pNode, void* pContext) { } static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode* pFunc) { - SFmGetFuncInfoParam param = { .pCtg = pCxt->pParseCxt->pCatalog, .pRpc = pCxt->pParseCxt->pTransporter, .pMgmtEps = &pCxt->pParseCxt->mgmtEpSet}; + SFmGetFuncInfoParam param = {.pCtg = pCxt->pParseCxt->pCatalog, + .pRpc = pCxt->pParseCxt->pTransporter, + .pMgmtEps = &pCxt->pParseCxt->mgmtEpSet}; if (TSDB_CODE_SUCCESS != fmGetFuncInfo(¶m, pFunc->functionName, &pFunc->funcId, &pFunc->funcType)) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_FUNTION, pFunc->functionName); } @@ -1268,7 +1269,7 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* static EDealRes checkStateExpr(SNode* pNode, void* pContext) { if (QUERY_NODE_COLUMN == nodeType(pNode)) { STranslateContext* pCxt = pContext; - SColumnNode* pCol = (SColumnNode*)pNode; + SColumnNode* pCol = (SColumnNode*)pNode; if (!IS_INTEGER_TYPE(pCol->node.resType.type)) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_STATE_WIN_TYPE); } @@ -1345,7 +1346,8 @@ static int32_t translateFrom(STranslateContext* pCxt, SSelectStmt* pSelect) { } static int32_t checkLimit(STranslateContext* pCxt, SSelectStmt* pSelect) { - if ((NULL != pSelect->pLimit && pSelect->pLimit->offset < 0) || (NULL != pSelect->pSlimit && pSelect->pSlimit->offset < 0)) { + if ((NULL != pSelect->pLimit && pSelect->pLimit->offset < 0) || + (NULL != pSelect->pSlimit && pSelect->pSlimit->offset < 0)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_OFFSET_LESS_ZERO); } @@ -1438,7 +1440,7 @@ static int32_t translateSetOperatorImpl(STranslateContext* pCxt, SSetOperator* p SExprNode* pLeftExpr = (SExprNode*)pLeft; SExprNode* pRightExpr = (SExprNode*)pRight; if (!dataTypeEqual(&pLeftExpr->resType, &pRightExpr->resType)) { - SNode* pRightFunc = NULL; + SNode* pRightFunc = NULL; int32_t code = createCastFunc(pCxt, pRight, pLeftExpr->resType, &pRightFunc); if (TSDB_CODE_SUCCESS != code) { return code; @@ -1650,7 +1652,8 @@ static int32_t checkKeepOption(STranslateContext* pCxt, SNodeList* pKeep) { (TIME_UNIT_MINUTE != pKeep1->unit && TIME_UNIT_HOUR != pKeep1->unit && TIME_UNIT_DAY != pKeep1->unit)) || (pKeep2->isDuration && (TIME_UNIT_MINUTE != pKeep2->unit && TIME_UNIT_HOUR != pKeep2->unit && TIME_UNIT_DAY != pKeep2->unit))) { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_KEEP_UNIT, pKeep0->unit, pKeep1->unit, pKeep2->unit); + return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_KEEP_UNIT, pKeep0->unit, pKeep1->unit, + pKeep2->unit); } int32_t daysToKeep0 = getBigintFromValueNode(pKeep0); @@ -1659,7 +1662,7 @@ static int32_t checkKeepOption(STranslateContext* pCxt, SNodeList* pKeep) { if (daysToKeep0 < TSDB_MIN_KEEP || daysToKeep1 < TSDB_MIN_KEEP || daysToKeep2 < TSDB_MIN_KEEP || daysToKeep0 > TSDB_MAX_KEEP || daysToKeep1 > TSDB_MAX_KEEP || daysToKeep2 > TSDB_MAX_KEEP) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_KEEP_VALUE, daysToKeep0, daysToKeep1, daysToKeep2, - TSDB_MIN_KEEP, TSDB_MAX_KEEP); + TSDB_MIN_KEEP, TSDB_MAX_KEEP); } if (!((daysToKeep0 <= daysToKeep1) && (daysToKeep1 <= daysToKeep2))) { @@ -1691,7 +1694,8 @@ static int32_t checkDbRetentionsOption(STranslateContext* pCxt, SNodeList* pRete return TSDB_CODE_SUCCESS; } -static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions, bool alter) { +static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions, + bool alter) { if (NULL == pOptions->pDaysPerFile && NULL == pOptions->pKeep) { return TSDB_CODE_SUCCESS; } @@ -1699,7 +1703,7 @@ static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbNa int64_t daysToKeep0 = GET_OPTION_VAL(nodesListGetNode(pOptions->pKeep, 0), alter ? -1 : TSDB_DEFAULT_KEEP); if (alter && (-1 == daysPerFile || -1 == daysToKeep0)) { SDbCfgInfo dbCfg; - int32_t code = getDBCfg(pCxt, pDbName, &dbCfg); + int32_t code = getDBCfg(pCxt, pDbName, &dbCfg); if (TSDB_CODE_SUCCESS != code) { return code; } @@ -1712,7 +1716,8 @@ static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbNa return TSDB_CODE_SUCCESS; } -static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions, bool alter) { +static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions, + bool alter) { int32_t code = checkRangeOption(pCxt, "totalBlocks", pOptions->pNumOfBlocks, TSDB_MIN_TOTAL_BLOCKS, TSDB_MAX_TOTAL_BLOCKS); if (TSDB_CODE_SUCCESS == code) { @@ -1918,7 +1923,7 @@ static int32_t checTableFactorOption(STranslateContext* pCxt, SValueNode* pVal) } if (pVal->datum.d < TSDB_MIN_DB_FILE_FACTOR || pVal->datum.d > TSDB_MAX_DB_FILE_FACTOR) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_F_RANGE_OPTION, "file_factor", pVal->datum.d, - TSDB_MIN_DB_FILE_FACTOR, TSDB_MAX_DB_FILE_FACTOR); + TSDB_MIN_DB_FILE_FACTOR, TSDB_MAX_DB_FILE_FACTOR); } } return TSDB_CODE_SUCCESS; @@ -1945,7 +1950,7 @@ static int32_t checkTableTags(STranslateContext* pCxt, SCreateTableStmt* pStmt) SNode* pNode; FOREACH(pNode, pStmt->pTags) { SColumnDefNode* pCol = (SColumnDefNode*)pNode; - if(pCol->dataType.type == TSDB_DATA_TYPE_JSON && LIST_LENGTH(pStmt->pTags) > 1){ + if (pCol->dataType.type == TSDB_DATA_TYPE_JSON && LIST_LENGTH(pStmt->pTags) > 1) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_ONLY_ONE_JSON_TAG); } } @@ -1960,8 +1965,10 @@ static int32_t checkTableRollupOption(STranslateContext* pCxt, SNodeList* pFuncs if (1 != LIST_LENGTH(pFuncs)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ROLLUP_OPTION); } - SFunctionNode* pFunc = nodesListGetNode(pFuncs, 0); - SFmGetFuncInfoParam param = { .pCtg = pCxt->pParseCxt->pCatalog, .pRpc = pCxt->pParseCxt->pTransporter, .pMgmtEps = &pCxt->pParseCxt->mgmtEpSet}; + SFunctionNode* pFunc = nodesListGetNode(pFuncs, 0); + SFmGetFuncInfoParam param = {.pCtg = pCxt->pParseCxt->pCatalog, + .pRpc = pCxt->pParseCxt->pTransporter, + .pMgmtEps = &pCxt->pParseCxt->mgmtEpSet}; if (TSDB_CODE_SUCCESS != fmGetFuncInfo(¶m, pFunc->functionName, &pFunc->funcId, &pFunc->funcType)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_FUNTION, pFunc->functionName); } @@ -2016,10 +2023,10 @@ static void toSchema(const SColumnDefNode* pCol, col_id_t colId, SSchema* pSchem typedef struct SSampleAstInfo { const char* pDbName; const char* pTableName; - SNodeList* pFuncs; - SNode* pInterval; - SNode* pOffset; - SNode* pSliding; + SNodeList* pFuncs; + SNode* pInterval; + SNode* pOffset; + SNode* pSliding; STableMeta* pRollupTableMeta; } SSampleAstInfo; @@ -2089,8 +2096,8 @@ static SNode* makeIntervalVal(SRetention* pRetension, int8_t precision) { return NULL; } int64_t timeVal = convertTimeFromPrecisionToUnit(pRetension->freq, precision, pRetension->freqUnit); - char buf[20] = {0}; - int32_t len = snprintf(buf, sizeof(buf), "%"PRId64"%c", timeVal, pRetension->freqUnit); + char buf[20] = {0}; + int32_t len = snprintf(buf, sizeof(buf), "%" PRId64 "%c", timeVal, pRetension->freqUnit); pVal->literal = strndup(buf, len); if (NULL == pVal->literal) { nodesDestroyNode(pVal); @@ -2133,7 +2140,7 @@ static SNodeList* createRollupFuncs(SCreateTableStmt* pStmt) { SNode* pFunc = NULL; FOREACH(pFunc, pStmt->pOptions->pFuncs) { SNode* pCol = NULL; - bool primaryKey = true; + bool primaryKey = true; FOREACH(pCol, pStmt->pCols) { if (primaryKey) { primaryKey = false; @@ -2150,7 +2157,7 @@ static SNodeList* createRollupFuncs(SCreateTableStmt* pStmt) { } static STableMeta* createRollupTableMeta(SCreateTableStmt* pStmt, int8_t precision) { - int32_t numOfField = LIST_LENGTH(pStmt->pCols) + LIST_LENGTH(pStmt->pTags); + int32_t numOfField = LIST_LENGTH(pStmt->pCols) + LIST_LENGTH(pStmt->pTags); STableMeta* pMeta = taosMemoryCalloc(1, sizeof(STableMeta) + numOfField * sizeof(SSchema)); if (NULL == pMeta) { return NULL; @@ -2161,7 +2168,7 @@ static STableMeta* createRollupTableMeta(SCreateTableStmt* pStmt, int8_t precisi pMeta->tableInfo.numOfColumns = LIST_LENGTH(pStmt->pCols); int32_t index = 0; - SNode* pCol = NULL; + SNode* pCol = NULL; FOREACH(pCol, pStmt->pCols) { toSchema((SColumnDefNode*)pCol, index + 1, pMeta->schema + index); ++index; @@ -2175,8 +2182,8 @@ static STableMeta* createRollupTableMeta(SCreateTableStmt* pStmt, int8_t precisi return pMeta; } -static int32_t buildSampleAstInfoByTable(STranslateContext* pCxt, - SCreateTableStmt* pStmt, SRetention* pRetension, int8_t precision, SSampleAstInfo* pInfo) { +static int32_t buildSampleAstInfoByTable(STranslateContext* pCxt, SCreateTableStmt* pStmt, SRetention* pRetension, + int8_t precision, SSampleAstInfo* pInfo) { pInfo->pDbName = pStmt->dbName; pInfo->pTableName = pStmt->tableName; pInfo->pFuncs = createRollupFuncs(pStmt); @@ -2188,10 +2195,10 @@ static int32_t buildSampleAstInfoByTable(STranslateContext* pCxt, return TSDB_CODE_SUCCESS; } -static int32_t getRollupAst(STranslateContext* pCxt, - SCreateTableStmt* pStmt, SRetention* pRetension, int8_t precision, char** pAst, int32_t* pLen) { +static int32_t getRollupAst(STranslateContext* pCxt, SCreateTableStmt* pStmt, SRetention* pRetension, int8_t precision, + char** pAst, int32_t* pLen) { SSampleAstInfo info = {0}; - int32_t code = buildSampleAstInfoByTable(pCxt, pStmt, pRetension, precision, &info); + int32_t code = buildSampleAstInfoByTable(pCxt, pStmt, pRetension, precision, &info); if (TSDB_CODE_SUCCESS == code) { code = buildSampleAst(pCxt, &info, pAst, pLen); } @@ -2201,17 +2208,17 @@ static int32_t getRollupAst(STranslateContext* pCxt, static int32_t buildRollupAst(STranslateContext* pCxt, SCreateTableStmt* pStmt, SMCreateStbReq* pReq) { SDbCfgInfo dbCfg = {0}; - int32_t code = getDBCfg(pCxt, pStmt->dbName, &dbCfg); - int32_t num = taosArrayGetSize(dbCfg.pRetensions); + int32_t code = getDBCfg(pCxt, pStmt->dbName, &dbCfg); + int32_t num = taosArrayGetSize(dbCfg.pRetensions); if (TSDB_CODE_SUCCESS != code || num < 2) { return code; } for (int32_t i = 1; i < num; ++i) { - SRetention *pRetension = taosArrayGet(dbCfg.pRetensions, i); + SRetention* pRetension = taosArrayGet(dbCfg.pRetensions, i); STranslateContext cxt = {0}; initTranslateContext(pCxt->pParseCxt, &cxt); - code = getRollupAst(&cxt, pStmt, pRetension, dbCfg.precision, - 1 == i ? &pReq->pAst1 : &pReq->pAst2, 1 == i ? &pReq->ast1Len : &pReq->ast2Len); + code = getRollupAst(&cxt, pStmt, pRetension, dbCfg.precision, 1 == i ? &pReq->pAst1 : &pReq->pAst2, + 1 == i ? &pReq->ast1Len : &pReq->ast2Len); destroyTranslateContext(&cxt); if (TSDB_CODE_SUCCESS != code) { break; @@ -2245,7 +2252,7 @@ static int32_t buildCreateStbReq(STranslateContext* pCxt, SCreateTableStmt* pStm static int32_t translateCreateSuperTable(STranslateContext* pCxt, SCreateTableStmt* pStmt) { SMCreateStbReq createReq = {0}; - int32_t code = checkCreateTable(pCxt, pStmt); + int32_t code = checkCreateTable(pCxt, pStmt); if (TSDB_CODE_SUCCESS == code) { code = buildCreateStbReq(pCxt, pStmt, &createReq); } @@ -2289,7 +2296,8 @@ static int32_t translateDropTable(STranslateContext* pCxt, SDropTableStmt* pStmt static int32_t translateDropSuperTable(STranslateContext* pCxt, SDropSuperTableStmt* pStmt) { SName tableName; - return doTranslateDropSuperTable(pCxt, toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tableName), pStmt->ignoreNotExists); + return doTranslateDropSuperTable(pCxt, toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tableName), + pStmt->ignoreNotExists); } static int32_t setAlterTableField(SAlterTableStmt* pStmt, SMAltertbReq* pAlterReq) { @@ -2421,7 +2429,7 @@ static int32_t nodeTypeToShowType(ENodeType nt) { case QUERY_NODE_SHOW_QUERIES_STMT: return TSDB_MGMT_TABLE_QUERIES; case QUERY_NODE_SHOW_VARIABLE_STMT: - return 0; // todo + return 0; // todo default: break; } @@ -2463,8 +2471,8 @@ static int32_t buildSampleAstInfoByIndex(STranslateContext* pCxt, SCreateIndexSt pInfo->pOffset = nodesCloneNode(pStmt->pOptions->pOffset); pInfo->pSliding = nodesCloneNode(pStmt->pOptions->pSliding); if (NULL == pInfo->pFuncs || NULL == pInfo->pInterval || - (NULL != pStmt->pOptions->pOffset && NULL == pInfo->pOffset) || - (NULL != pStmt->pOptions->pSliding && NULL == pInfo->pSliding)) { + (NULL != pStmt->pOptions->pOffset && NULL == pInfo->pOffset) || + (NULL != pStmt->pOptions->pSliding && NULL == pInfo->pSliding)) { return TSDB_CODE_OUT_OF_MEMORY; } return TSDB_CODE_SUCCESS; @@ -2472,7 +2480,7 @@ static int32_t buildSampleAstInfoByIndex(STranslateContext* pCxt, SCreateIndexSt static int32_t getSmaIndexAst(STranslateContext* pCxt, SCreateIndexStmt* pStmt, char** pAst, int32_t* pLen) { SSampleAstInfo info = {0}; - int32_t code = buildSampleAstInfoByIndex(pCxt, pStmt, &info); + int32_t code = buildSampleAstInfoByIndex(pCxt, pStmt, &info); if (TSDB_CODE_SUCCESS == code) { code = buildSampleAst(pCxt, &info, pAst, pLen); } @@ -2618,9 +2626,9 @@ static int32_t translateDropComponentNode(STranslateContext* pCxt, SDropComponen static int32_t buildCreateTopicReq(STranslateContext* pCxt, SCreateTopicStmt* pStmt, SCMCreateTopicReq* pReq) { SName name; - // tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->topicName, strlen(pStmt->topicName)); - // tNameGetFullDbName(&name, pReq->name); - tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pCxt->pParseCxt->db, pStmt->topicName, &name), pReq->name); + tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->topicName, strlen(pStmt->topicName)); + tNameGetFullDbName(&name, pReq->name); + /*tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pCxt->pParseCxt->db, pStmt->topicName, &name), pReq->name);*/ pReq->igExists = pStmt->ignoreExists; pReq->withTbName = pStmt->pOptions->withTable; pReq->withSchema = pStmt->pOptions->withSchema; @@ -2633,16 +2641,19 @@ static int32_t buildCreateTopicReq(STranslateContext* pCxt, SCreateTopicStmt* pS int32_t code = TSDB_CODE_SUCCESS; + const char* dbName; if (NULL != pStmt->pQuery) { - strcpy(pReq->subscribeDbName, ((SRealTableNode*)(((SSelectStmt*)pStmt->pQuery)->pFromTable))->table.dbName); + dbName = ((SRealTableNode*)(((SSelectStmt*)pStmt->pQuery)->pFromTable))->table.dbName; pCxt->pParseCxt->topicQuery = true; code = translateQuery(pCxt, pStmt->pQuery); if (TSDB_CODE_SUCCESS == code) { code = nodesNodeToString(pStmt->pQuery, false, &pReq->ast, NULL); } } else { - strcpy(pReq->subscribeDbName, pStmt->subscribeDbName); + dbName = pStmt->subscribeDbName; } + tNameSetDbName(&name, pCxt->pParseCxt->acctId, dbName, strlen(dbName)); + tNameGetFullDbName(&name, pReq->subscribeDbName); return code; } @@ -2654,8 +2665,9 @@ static int32_t checkCreateTopic(STranslateContext* pCxt, SCreateTopicStmt* pStmt if (QUERY_NODE_SELECT_STMT == nodeType(pStmt->pQuery)) { SSelectStmt* pSelect = (SSelectStmt*)pStmt->pQuery; - if (!pSelect->isDistinct && QUERY_NODE_REAL_TABLE == nodeType(pSelect->pFromTable) && NULL == pSelect->pGroupByList && - NULL == pSelect->pLimit && NULL == pSelect->pSlimit && NULL == pSelect->pOrderByList && NULL == pSelect->pPartitionByList) { + if (!pSelect->isDistinct && QUERY_NODE_REAL_TABLE == nodeType(pSelect->pFromTable) && + NULL == pSelect->pGroupByList && NULL == pSelect->pLimit && NULL == pSelect->pSlimit && + NULL == pSelect->pOrderByList && NULL == pSelect->pPartitionByList) { return TSDB_CODE_SUCCESS; } } @@ -2665,7 +2677,7 @@ static int32_t checkCreateTopic(STranslateContext* pCxt, SCreateTopicStmt* pStmt static int32_t translateCreateTopic(STranslateContext* pCxt, SCreateTopicStmt* pStmt) { SCMCreateTopicReq createReq = {0}; - int32_t code = checkCreateTopic(pCxt, pStmt); + int32_t code = checkCreateTopic(pCxt, pStmt); if (TSDB_CODE_SUCCESS == code) { code = buildCreateTopicReq(pCxt, pStmt, &createReq); } @@ -2763,7 +2775,7 @@ static int32_t translateDropStream(STranslateContext* pCxt, SDropStreamStmt* pSt return TSDB_CODE_SUCCESS; } -static int32_t readFromFile(char* pName, int32_t *len, char **buf) { +static int32_t readFromFile(char* pName, int32_t* len, char** buf) { int64_t filesize = 0; if (taosStatFile(pName, &filesize, NULL) < 0) { return TAOS_SYSTEM_ERROR(errno); @@ -3408,8 +3420,8 @@ static void addCreateTbReqIntoVgroup(int32_t acctId, SHashObj* pVgroupHashmap, c static int32_t addValToKVRow(STranslateContext* pCxt, SValueNode* pVal, const SSchema* pSchema, SKVRowBuilder* pBuilder) { - if(pSchema->type == TSDB_DATA_TYPE_JSON){ - if(pVal->literal && strlen(pVal->literal) > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){ + if (pSchema->type == TSDB_DATA_TYPE_JSON) { + if (pVal->literal && strlen(pVal->literal) > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) { return buildSyntaxErrMsg(&pCxt->msgBuf, "json string too long than 4095", pVal->literal); } @@ -3420,10 +3432,11 @@ static int32_t addValToKVRow(STranslateContext* pCxt, SValueNode* pVal, const SS return pCxt->errCode; } - if(pVal->node.resType.type == TSDB_DATA_TYPE_NULL){ + if (pVal->node.resType.type == TSDB_DATA_TYPE_NULL) { // todo - }else{ - tdAddColToKVRow(pBuilder, pSchema->colId, &(pVal->datum.p), IS_VAR_DATA_TYPE(pSchema->type) ? varDataTLen(pVal->datum.p) : TYPE_BYTES[pSchema->type]); + } else { + tdAddColToKVRow(pBuilder, pSchema->colId, &(pVal->datum.p), + IS_VAR_DATA_TYPE(pSchema->type) ? varDataTLen(pVal->datum.p) : TYPE_BYTES[pSchema->type]); } return TSDB_CODE_SUCCESS; @@ -3660,7 +3673,9 @@ static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) { return TSDB_CODE_OUT_OF_MEMORY; } - pQuery->precision = extractResultTsPrecision((SSelectStmt*)pQuery->pRoot); + if (nodeType(pQuery->pRoot) == QUERY_NODE_SELECT_STMT) { + pQuery->precision = extractResultTsPrecision((SSelectStmt*)pQuery->pRoot); + } } if (NULL != pCxt->pDbs) { @@ -3692,7 +3707,7 @@ static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) { int32_t translate(SParseContext* pParseCxt, SQuery* pQuery) { STranslateContext cxt = {0}; - int32_t code = initTranslateContext(pParseCxt, &cxt); + int32_t code = initTranslateContext(pParseCxt, &cxt); if (TSDB_CODE_SUCCESS == code) { code = fmFuncMgtInit(); } diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index de805d2c77..fd7c831399 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -342,28 +342,20 @@ PROCESS_META_OVER: int32_t queryProcessQnodeListRsp(void *output, char *msg, int32_t msgSize) { SQnodeListRsp out = {0}; - int32_t code = -1; + int32_t code = 0; if (NULL == output || NULL == msg || msgSize <= 0) { code = TSDB_CODE_TSC_INVALID_INPUT; - goto PROCESS_QLIST_OVER; + return code; } + out.addrsList = (SArray *)output; if (tDeserializeSQnodeListRsp(msg, msgSize, &out) != 0) { qError("invalid qnode list rsp msg, msgSize:%d", msgSize); code = TSDB_CODE_INVALID_MSG; - goto PROCESS_QLIST_OVER; + return code; } -PROCESS_QLIST_OVER: - - if (code != 0) { - tFreeSQnodeListRsp(&out); - out.addrsList = NULL; - } - - *(SArray **)output = out.addrsList; - return code; } diff --git a/source/libs/qworker/src/qworkerMsg.c b/source/libs/qworker/src/qworkerMsg.c index faf5182e26..64df4f02bf 100644 --- a/source/libs/qworker/src/qworkerMsg.c +++ b/source/libs/qworker/src/qworkerMsg.c @@ -46,7 +46,7 @@ int32_t qwBuildAndSendQueryRsp(SQWConnInfo *pConn, int32_t code) { SQueryTableRsp rsp = {.code = code}; int32_t contLen = tSerializeSQueryTableRsp(NULL, 0, &rsp); - void * msg = rpcMallocCont(contLen); + void *msg = rpcMallocCont(contLen); tSerializeSQueryTableRsp(msg, contLen, &rsp); SRpcMsg rpcRsp = { @@ -87,7 +87,7 @@ int32_t qwBuildAndSendExplainRsp(SQWConnInfo *pConn, SExplainExecInfo *execInfo, SExplainRsp rsp = {.numOfPlans = num, .subplanInfo = execInfo}; int32_t contLen = tSerializeSExplainRsp(NULL, 0, &rsp); - void * pRsp = rpcMallocCont(contLen); + void *pRsp = rpcMallocCont(contLen); tSerializeSExplainRsp(pRsp, contLen, &rsp); SRpcMsg rpcRsp = { @@ -107,7 +107,7 @@ int32_t qwBuildAndSendExplainRsp(SQWConnInfo *pConn, SExplainExecInfo *execInfo, int32_t qwBuildAndSendHbRsp(SQWConnInfo *pConn, SSchedulerHbRsp *pStatus, int32_t code) { int32_t contLen = tSerializeSSchedulerHbRsp(NULL, 0, pStatus); - void * pRsp = rpcMallocCont(contLen); + void *pRsp = rpcMallocCont(contLen); tSerializeSSchedulerHbRsp(pRsp, contLen, pStatus); SRpcMsg rpcRsp = { @@ -223,7 +223,7 @@ int32_t qwBuildAndSendShowRsp(SRpcMsg *pMsg, int32_t code) { showRsp.tableMeta.numOfColumns = cols; int32_t bufLen = tSerializeSShowRsp(NULL, 0, &showRsp); - void * pBuf = rpcMallocCont(bufLen); + void *pBuf = rpcMallocCont(bufLen); tSerializeSShowRsp(pBuf, bufLen, &showRsp); SRpcMsg rpcMsg = { @@ -403,8 +403,8 @@ int32_t qWorkerProcessCQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { bool queryDone = false; SQueryContinueReq *msg = (SQueryContinueReq *)pMsg->pCont; bool needStop = false; - SQWTaskCtx * handles = NULL; - SQWorkerMgmt * mgmt = (SQWorkerMgmt *)qWorkerMgmt; + SQWTaskCtx *handles = NULL; + SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt; if (NULL == msg || pMsg->contLen < sizeof(*msg)) { QW_ELOG("invalid cquery msg, msg:%p, msgLen:%d", msg, pMsg->contLen); @@ -538,7 +538,7 @@ int32_t qWorkerProcessCancelMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { return TSDB_CODE_QRY_INVALID_INPUT; } - SQWorkerMgmt * mgmt = (SQWorkerMgmt *)qWorkerMgmt; + SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt; int32_t code = 0; STaskCancelReq *msg = pMsg->pCont; if (NULL == msg || pMsg->contLen < sizeof(*msg)) { @@ -620,7 +620,7 @@ int32_t qWorkerProcessHbMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { int32_t code = 0; SSchedulerHbReq req = {0}; - SQWorkerMgmt * mgmt = (SQWorkerMgmt *)qWorkerMgmt; + SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt; if (NULL == pMsg->pCont) { QW_ELOG("invalid hb msg, msg:%p, msgLen:%d", pMsg->pCont, pMsg->contLen); diff --git a/source/libs/scalar/inc/sclInt.h b/source/libs/scalar/inc/sclInt.h index 3d59ffd93d..9142cc41c4 100644 --- a/source/libs/scalar/inc/sclInt.h +++ b/source/libs/scalar/inc/sclInt.h @@ -32,6 +32,9 @@ typedef struct SScalarCtx { #define SCL_DATA_TYPE_DUMMY_HASH 9000 #define SCL_DEFAULT_OP_NUM 10 +#define SCL_IS_CONST_NODE(_node) ((NULL == (_node)) || (QUERY_NODE_VALUE == (_node)->type) || (QUERY_NODE_NODE_LIST == (_node)->type)) +#define SCL_IS_CONST_CALC(_ctx) (NULL == (_ctx)->pBlockList) + #define sclFatal(...) qFatal(__VA_ARGS__) #define sclError(...) qError(__VA_ARGS__) #define sclWarn(...) qWarn(__VA_ARGS__) diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 145afbe984..820a4894b5 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -244,23 +244,53 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t return TSDB_CODE_SUCCESS; } -int32_t sclInitParamList(SScalarParam **pParams, SNodeList* pParamList, SScalarCtx *ctx, int32_t *rowNum) { +int32_t sclInitParamList(SScalarParam **pParams, SNodeList* pParamList, SScalarCtx *ctx, int32_t *paramNum, int32_t *rowNum) { int32_t code = 0; - SScalarParam *paramList = taosMemoryCalloc(pParamList->length, sizeof(SScalarParam)); + if (NULL == pParamList) { + if (ctx->pBlockList) { + SSDataBlock *pBlock = taosArrayGet(ctx->pBlockList, 0); + *rowNum = pBlock->info.rows; + } else { + *rowNum = 1; + } + + *paramNum = 1; + } else { + *paramNum = pParamList->length; + } + + SScalarParam *paramList = taosMemoryCalloc(*paramNum, sizeof(SScalarParam)); if (NULL == paramList) { - sclError("calloc %d failed", (int32_t)(pParamList->length * sizeof(SScalarParam))); + sclError("calloc %d failed", (int32_t)((*paramNum) * sizeof(SScalarParam))); SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - SListCell *cell = pParamList->pHead; - for (int32_t i = 0; i < pParamList->length; ++i) { - if (NULL == cell || NULL == cell->pNode) { - sclError("invalid cell, cell:%p, pNode:%p", cell, cell->pNode); - SCL_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); + if (pParamList) { + SNode *tnode = NULL; + int32_t i = 0; + if (SCL_IS_CONST_CALC(ctx)) { + WHERE_EACH (tnode, pParamList) { + if (!SCL_IS_CONST_NODE(tnode)) { + WHERE_NEXT; + } else { + SCL_ERR_JRET(sclInitParam(tnode, ¶mList[i], ctx, rowNum)); + ERASE_NODE(pParamList); + } + + ++i; + } + } else { + FOREACH(tnode, pParamList) { + SCL_ERR_JRET(sclInitParam(tnode, ¶mList[i], ctx, rowNum)); + ++i; + } } + } else { + paramList[0].numOfRows = *rowNum; + } - SCL_ERR_JRET(sclInitParam(cell->pNode, ¶mList[i], ctx, rowNum)); - cell = cell->pNext; + if (0 == *rowNum) { + taosMemoryFreeClear(paramList); } *pParams = paramList; @@ -299,37 +329,45 @@ _return: } int32_t sclExecFunction(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *output) { - if (NULL == node->pParameterList || node->pParameterList->length <= 0) { - sclError("invalid function parameter list, list:%p, paramNum:%d", node->pParameterList, node->pParameterList ? node->pParameterList->length : 0); - SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); - } - - SScalarFuncExecFuncs ffpSet = {0}; - int32_t code = fmGetScalarFuncExecFuncs(node->funcId, &ffpSet); - if (code) { - sclError("fmGetFuncExecFuncs failed, funcId:%d, code:%s", node->funcId, tstrerror(code)); - SCL_ERR_RET(code); - } - SScalarParam *params = NULL; int32_t rowNum = 0; - SCL_ERR_RET(sclInitParamList(¶ms, node->pParameterList, ctx, &rowNum)); + int32_t paramNum = 0; + int32_t code = 0; + SCL_ERR_RET(sclInitParamList(¶ms, node->pParameterList, ctx, ¶mNum, &rowNum)); - output->columnData = createColumnInfoData(&node->node.resType, rowNum); - if (output->columnData == NULL) { - sclError("calloc %d failed", (int32_t)(rowNum * output->columnData->info.bytes)); - SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - code = (*ffpSet.process)(params, node->pParameterList->length, output); - if (code) { - sclError("scalar function exec failed, funcId:%d, code:%s", node->funcId, tstrerror(code)); + if (fmIsUserDefinedFunc(node->funcId)) { +#if 0 + UdfcFuncHandle udfHandle = NULL; + + SCL_ERR_JRET(setupUdf(node->functionName, &udfHandle)); + code = callUdfScalarFunc(udfHandle, params, paramNum, output); + teardownUdf(udfHandle); SCL_ERR_JRET(code); +#endif + } else { + SScalarFuncExecFuncs ffpSet = {0}; + code = fmGetScalarFuncExecFuncs(node->funcId, &ffpSet); + if (code) { + sclError("fmGetFuncExecFuncs failed, funcId:%d, code:%s", node->funcId, tstrerror(code)); + SCL_ERR_JRET(code); + } + + output->columnData = createColumnInfoData(&node->node.resType, rowNum); + if (output->columnData == NULL) { + sclError("calloc %d failed", (int32_t)(rowNum * output->columnData->info.bytes)); + SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + code = (*ffpSet.process)(params, paramNum, output); + if (code) { + sclError("scalar function exec failed, funcId:%d, code:%s", node->funcId, tstrerror(code)); + SCL_ERR_JRET(code); + } } _return: - for (int32_t i = 0; i < node->pParameterList->length; ++i) { + for (int32_t i = 0; i < paramNum; ++i) { // sclFreeParamNoData(params + i); } @@ -355,8 +393,13 @@ int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *o SScalarParam *params = NULL; int32_t rowNum = 0; + int32_t paramNum = 0; int32_t code = 0; - SCL_ERR_RET(sclInitParamList(¶ms, node->pParameterList, ctx, &rowNum)); + SCL_ERR_RET(sclInitParamList(¶ms, node->pParameterList, ctx, ¶mNum, &rowNum)); + if (NULL == params) { + output->numOfRows = 0; + return TSDB_CODE_SUCCESS; + } int32_t type = node->node.resType.type; output->numOfRows = rowNum; @@ -369,25 +412,41 @@ int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *o } bool value = false; + bool complete = true; for (int32_t i = 0; i < rowNum; ++i) { - for (int32_t m = 0; m < node->pParameterList->length; ++m) { + complete = true; + for (int32_t m = 0; m < paramNum; ++m) { + if (NULL == params[m].columnData) { + complete = false; + continue; + } char* p = colDataGetData(params[m].columnData, i); GET_TYPED_DATA(value, bool, params[m].columnData->info.type, p); if (LOGIC_COND_TYPE_AND == node->condType && (false == value)) { + complete = true; break; } else if (LOGIC_COND_TYPE_OR == node->condType && value) { + complete = true; break; } else if (LOGIC_COND_TYPE_NOT == node->condType) { value = !value; } } - colDataAppend(output->columnData, i, (char*) &value, false); + if (complete) { + colDataAppend(output->columnData, i, (char*) &value, false); + } + } + + if (SCL_IS_CONST_CALC(ctx) && (false == complete)) { + sclFreeParam(output); + output->numOfRows = 0; } _return: - for (int32_t i = 0; i < node->pParameterList->length; ++i) { + + for (int32_t i = 0; i < paramNum; ++i) { // sclFreeParamNoData(params + i); } @@ -426,6 +485,17 @@ _return: EDealRes sclRewriteFunction(SNode** pNode, SScalarCtx *ctx) { SFunctionNode *node = (SFunctionNode *)*pNode; + SNode* tnode = NULL; + if (fmIsUserDefinedFunc(node->funcId)) { + return DEAL_RES_CONTINUE; + } + + FOREACH(tnode, node->pParameterList) { + if (!SCL_IS_CONST_NODE(tnode)) { + return DEAL_RES_CONTINUE; + } + } + SScalarParam output = {0}; ctx->code = sclExecFunction(node, ctx, &output); @@ -470,6 +540,10 @@ EDealRes sclRewriteLogic(SNode** pNode, SScalarCtx *ctx) { return DEAL_RES_ERROR; } + if (0 == output.numOfRows) { + return DEAL_RES_CONTINUE; + } + SValueNode *res = (SValueNode *)nodesMakeNode(QUERY_NODE_VALUE); if (NULL == res) { sclError("make value node failed"); @@ -498,6 +572,14 @@ EDealRes sclRewriteLogic(SNode** pNode, SScalarCtx *ctx) { EDealRes sclRewriteOperator(SNode** pNode, SScalarCtx *ctx) { SOperatorNode *node = (SOperatorNode *)*pNode; + if (!SCL_IS_CONST_NODE(node->pLeft)) { + return DEAL_RES_CONTINUE; + } + + if (!SCL_IS_CONST_NODE(node->pRight)) { + return DEAL_RES_CONTINUE; + } + SScalarParam output = {.columnData = taosMemoryCalloc(1, sizeof(SColumnInfoData))}; ctx->code = sclExecOperator(node, ctx, &output); if (ctx->code) { @@ -530,7 +612,7 @@ EDealRes sclRewriteOperator(SNode** pNode, SScalarCtx *ctx) { } EDealRes sclConstantsRewriter(SNode** pNode, void* pContext) { - if (QUERY_NODE_VALUE == nodeType(*pNode) || QUERY_NODE_NODE_LIST == nodeType(*pNode)) { + if (QUERY_NODE_VALUE == nodeType(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode) || QUERY_NODE_NODE_LIST == nodeType(*pNode)) { return DEAL_RES_CONTINUE; } diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index 04ab26d8a4..ed52e48f6d 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -137,6 +137,11 @@ void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in rnode->node.resType.bytes = dataBytes; rnode->dataBlockId = 0; + if (NULL == block) { + *pNode = (SNode *)rnode; + return; + } + if (NULL == *block) { SSDataBlock *res = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock)); res->info.numOfCols = 3; @@ -889,6 +894,8 @@ TEST(constantTest, int_greater_int_is_true2) { } TEST(constantTest, greater_and_lower) { + scltInitLogFile(); + SNode *pval1 = NULL, *pval2 = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode = NULL, *res = NULL; bool eRes[5] = {false, false, true, true, true}; int64_t v1 = 333, v2 = 222, v3 = -10, v4 = 20; @@ -913,6 +920,115 @@ TEST(constantTest, greater_and_lower) { nodesDestroyNode(res); } +TEST(constantTest, column_and_value1) { + scltInitLogFile(); + + SNode *pval1 = NULL, *pval2 = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode = NULL, *res = NULL; + bool eRes[5] = {false, false, true, true, true}; + int64_t v1 = 333, v2 = 222, v3 = -10, v4 = 20; + SNode *list[2] = {0}; + scltMakeValueNode(&pval1, TSDB_DATA_TYPE_BIGINT, &v1); + scltMakeValueNode(&pval2, TSDB_DATA_TYPE_BIGINT, &v2); + scltMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pval1, pval2); + scltMakeValueNode(&pval1, TSDB_DATA_TYPE_BIGINT, &v3); + scltMakeColumnNode(&pval2, NULL, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), 0, NULL); + scltMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pval1, pval2); + list[0] = opNode1; + list[1] = opNode2; + scltMakeLogicNode(&logicNode, LOGIC_COND_TYPE_AND, list, 2); + + int32_t code = scalarCalculateConstants(logicNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_LOGIC_CONDITION); + SLogicConditionNode *v = (SLogicConditionNode *)res; + ASSERT_EQ(v->condType, LOGIC_COND_TYPE_AND); + ASSERT_EQ(v->pParameterList->length, 1); + nodesDestroyNode(res); +} + +TEST(constantTest, column_and_value2) { + scltInitLogFile(); + + SNode *pval1 = NULL, *pval2 = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode = NULL, *res = NULL; + bool eRes[5] = {false, false, true, true, true}; + int64_t v1 = 333, v2 = 222, v3 = -10, v4 = 20; + SNode *list[2] = {0}; + scltMakeValueNode(&pval1, TSDB_DATA_TYPE_BIGINT, &v1); + scltMakeValueNode(&pval2, TSDB_DATA_TYPE_BIGINT, &v2); + scltMakeOpNode(&opNode1, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pval1, pval2); + scltMakeValueNode(&pval1, TSDB_DATA_TYPE_BIGINT, &v3); + scltMakeColumnNode(&pval2, NULL, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), 0, NULL); + scltMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pval1, pval2); + list[0] = opNode1; + list[1] = opNode2; + scltMakeLogicNode(&logicNode, LOGIC_COND_TYPE_AND, list, 2); + + int32_t code = scalarCalculateConstants(logicNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, false); + nodesDestroyNode(res); +} + +TEST(constantTest, column_and_value3) { + scltInitLogFile(); + + SNode *pval1 = NULL, *pval2 = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode = NULL, *res = NULL; + bool eRes[5] = {false, false, true, true, true}; + int64_t v1 = 333, v2 = 222, v3 = -10, v4 = 20; + SNode *list[2] = {0}; + scltMakeValueNode(&pval1, TSDB_DATA_TYPE_BIGINT, &v1); + scltMakeValueNode(&pval2, TSDB_DATA_TYPE_BIGINT, &v2); + scltMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pval1, pval2); + scltMakeValueNode(&pval1, TSDB_DATA_TYPE_BIGINT, &v3); + scltMakeColumnNode(&pval2, NULL, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), 0, NULL); + scltMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pval1, pval2); + list[0] = opNode1; + list[1] = opNode2; + scltMakeLogicNode(&logicNode, LOGIC_COND_TYPE_OR, list, 2); + + int32_t code = scalarCalculateConstants(logicNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE); + SValueNode *v = (SValueNode *)res; + ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL); + ASSERT_EQ(v->datum.b, true); + nodesDestroyNode(res); +} + +TEST(constantTest, column_and_value4) { + scltInitLogFile(); + + SNode *pval1 = NULL, *pval2 = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode = NULL, *res = NULL; + bool eRes[5] = {false, false, true, true, true}; + int64_t v1 = 333, v2 = 222, v3 = -10, v4 = 20; + SNode *list[2] = {0}; + scltMakeValueNode(&pval1, TSDB_DATA_TYPE_BIGINT, &v1); + scltMakeValueNode(&pval2, TSDB_DATA_TYPE_BIGINT, &v2); + scltMakeOpNode(&opNode1, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pval1, pval2); + scltMakeValueNode(&pval1, TSDB_DATA_TYPE_BIGINT, &v3); + scltMakeColumnNode(&pval2, NULL, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), 0, NULL); + scltMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pval1, pval2); + list[0] = opNode1; + list[1] = opNode2; + scltMakeLogicNode(&logicNode, LOGIC_COND_TYPE_OR, list, 2); + + int32_t code = scalarCalculateConstants(logicNode, &res); + ASSERT_EQ(code, 0); + ASSERT_TRUE(res); + ASSERT_EQ(nodeType(res), QUERY_NODE_LOGIC_CONDITION); + SLogicConditionNode *v = (SLogicConditionNode *)res; + ASSERT_EQ(v->condType, LOGIC_COND_TYPE_OR); + ASSERT_EQ(v->pParameterList->length, 1); + nodesDestroyNode(res); +} + + void makeJsonArrow(SSDataBlock **src, SNode **opNode, void *json, char *key){ char keyVar[32] = {0}; memcpy(varDataVal(keyVar), key, strlen(key)); diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 6d598699c5..70a3559cd9 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -138,6 +138,91 @@ static int32_t walReadSeekVer(SWalReadHandle *pRead, int64_t ver) { return 0; } +void walSetReaderCapacity(SWalReadHandle *pRead, int32_t capacity) { pRead->capacity = capacity; } + +int32_t walFetchHead(SWalReadHandle *pRead, int64_t ver, SWalHead *pHead) { + int32_t code; + // TODO: valid ver + + if (pRead->curVersion != ver) { + code = walReadSeekVer(pRead, ver); + if (code < 0) return -1; + } + + if (!taosValidFile(pRead->pReadLogTFile)) { + return -1; + } + + code = taosReadFile(pRead->pReadLogTFile, pHead, sizeof(SWalHead)); + if (code != sizeof(SWalHead)) { + return -1; + } + + code = walValidHeadCksum(pHead); + + if (code != 0) { + wError("unexpected wal log version: % " PRId64 ", since head checksum not passed", ver); + terrno = TSDB_CODE_WAL_FILE_CORRUPTED; + return -1; + } + + return 0; +} + +int32_t walSkipFetchBody(SWalReadHandle *pRead, const SWalHead *pHead) { + int32_t code; + + ASSERT(pRead->curVersion == pHead->head.version); + + code = taosLSeekFile(pRead->pReadLogTFile, pHead->head.bodyLen, SEEK_CUR); + if (code < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + pRead->curVersion = -1; + return -1; + } + + pRead->curVersion++; + + return 0; +} + +int32_t walFetchBody(SWalReadHandle *pRead, SWalHead **ppHead) { + SWalReadHead *pReadHead = &((*ppHead)->head); + int64_t ver = pReadHead->version; + + if (pRead->capacity < pReadHead->bodyLen) { + void *ptr = taosMemoryRealloc(*ppHead, sizeof(SWalHead) + pReadHead->bodyLen); + if (ptr == NULL) { + terrno = TSDB_CODE_WAL_OUT_OF_MEMORY; + return -1; + } + *ppHead = ptr; + pRead->capacity = pReadHead->bodyLen; + } + + if (pReadHead->bodyLen != taosReadFile(pRead->pReadLogTFile, pReadHead->body, pReadHead->bodyLen)) { + return -1; + } + + if (pReadHead->version != ver) { + wError("unexpected wal log version: %" PRId64 ", read request version:%" PRId64 "", pRead->pHead->head.version, + ver); + pRead->curVersion = -1; + terrno = TSDB_CODE_WAL_FILE_CORRUPTED; + return -1; + } + + if (walValidBodyCksum(*ppHead) != 0) { + wError("unexpected wal log version: % " PRId64 ", since body checksum not passed", ver); + pRead->curVersion = -1; + terrno = TSDB_CODE_WAL_FILE_CORRUPTED; + return -1; + } + + pRead->curVersion = ver + 1; + return 0; +} + int32_t walReadWithHandle_s(SWalReadHandle *pRead, int64_t ver, SWalReadHead **ppHead) { taosThreadMutexLock(&pRead->mutex); if (walReadWithHandle(pRead, ver) < 0) { @@ -172,12 +257,14 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { if (code != sizeof(SWalHead)) { return -1; } + code = walValidHeadCksum(pRead->pHead); if (code != 0) { wError("unexpected wal log version: % " PRId64 ", since head checksum not passed", ver); terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; } + if (pRead->capacity < pRead->pHead->head.bodyLen) { void *ptr = taosMemoryRealloc(pRead->pHead, sizeof(SWalHead) + pRead->pHead->head.bodyLen); if (ptr == NULL) { diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 530930c261..207b7d8421 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -13,8 +13,6 @@ * along with this program. If not, see . */ -#define _DEFAULT_SOURCE - #include "os.h" #include "taoserror.h" #include "tchecksum.h" @@ -298,14 +296,14 @@ int64_t walWriteWithSyncInfo(SWal *pWal, int64_t index, tmsg_t msgType, SSyncLog pWal->writeHead.head.bodyLen = bodyLen; pWal->writeHead.head.msgType = msgType; - // sync info + // sync info for sync module pWal->writeHead.head.syncMeta = syncMeta; pWal->writeHead.cksumHead = walCalcHeadCksum(&pWal->writeHead); pWal->writeHead.cksumBody = walCalcBodyCksum(body, bodyLen); if (taosWriteFile(pWal->pWriteLogTFile, &pWal->writeHead, sizeof(SWalHead)) != sizeof(SWalHead)) { - // ftruncate + // TODO ftruncate terrno = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(errno)); @@ -313,7 +311,7 @@ int64_t walWriteWithSyncInfo(SWal *pWal, int64_t index, tmsg_t msgType, SSyncLog } if (taosWriteFile(pWal->pWriteLogTFile, (char *)body, bodyLen) != bodyLen) { - // ftruncate + // TODO ftruncate terrno = TAOS_SYSTEM_ERROR(errno); wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal), strerror(errno)); diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index 1c655fc2bf..2d741b18f6 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -476,6 +476,7 @@ void* taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t return (void*)buf; } +// todo remove it // order array void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* param) { taosArrayGetSize(pArray) > 8 ? taosArrayQuickSort(pArray, fn, param) : taosArrayInsertSort(pArray, fn, param); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 9332cb481e..50c924bbb9 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -258,7 +258,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_COLUMN_ALREADY_EXIST, "Column already exists TAOS_DEFINE_ERROR(TSDB_CODE_MND_COLUMN_NOT_EXIST, "Column does not exist") // mnode-infoSchema -TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_INFOS_TBL, "Invalid information schema table name") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_SYS_TABLENAME, "Invalid system table name") // mnode-func diff --git a/tests/script/tsim/parser/groupby-basic.sim b/tests/script/tsim/parser/groupby-basic.sim index f073200a05..4d6b33612f 100644 --- a/tests/script/tsim/parser/groupby-basic.sim +++ b/tests/script/tsim/parser/groupby-basic.sim @@ -115,19 +115,19 @@ endi if $data00 != 10 then return -1 endi -if $data01 != 0 then - return -1 -endi if $data10 != 10 then return -1 endi -if $data11 != 1 then - return -1 -endi if $data90 != 10 then return -1 endi -if $data91 != 9 then +if $data01 != 7 then + return -1 +endi +if $data11 != 6 then + return -1 +endi +if $data91 != 3 then return -1 endi @@ -143,16 +143,16 @@ if $row != 10 then return -1 endi -if $data00 != @22-01-01 00:00:00.000@ then +if $data00 != @22-01-01 00:00:00.007@ then return -1 endi -if $data01 != 0 then +if $data01 != 7 then return -1 endi -if $data90 != @22-01-01 00:00:00.009@ then +if $data90 != @22-01-01 00:00:00.003@ then return -1 endi -if $data91 != 9 then +if $data91 != 3 then return -1 endi diff --git a/tests/script/tsim/stable/disk.sim b/tests/script/tsim/stable/disk.sim index 97ef779ff2..c1ced6ae10 100644 --- a/tests/script/tsim/stable/disk.sim +++ b/tests/script/tsim/stable/disk.sim @@ -53,6 +53,7 @@ endi sql select count(tbcol) from $mt print select count(tbcol) from $mt ===> $data00 if $data00 != $totalNum then + print expect $totalNum , actual: $data00 return -1 endi diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 455d70aac7..2954358feb 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -4,4 +4,4 @@ set -e #python3 ./test.py -f 2-query/between.py #python3 ./test.py -f 2-query/distinct.py python3 ./test.py -f 2-query/varchar.py -python3 ./test.py -f 2-query/cast.py +#python3 ./test.py -f 2-query/cast.py diff --git a/tools/shell/src/shellNettest.c b/tools/shell/src/shellNettest.c index 307049dd38..c8ec31c48b 100644 --- a/tools/shell/src/shellNettest.c +++ b/tools/shell/src/shellNettest.c @@ -67,7 +67,7 @@ static void shellWorkAsClient() { printf("request is sent, size:%d\n", rpcMsg.contLen); rpcSendRecv(clientRpc, &epSet, &rpcMsg, &rpcRsp); - if (rpcRsp.code == 0 &&rpcRsp.contLen == rpcMsg.contLen) { + if (rpcRsp.code == 0 && rpcRsp.contLen == rpcMsg.contLen) { printf("response is received, size:%d\n", rpcMsg.contLen); if (rpcRsp.code == 0) totalSucc++; } else { @@ -97,8 +97,12 @@ static void shellProcessMsg(void *p, SRpcMsg *pRpc, SEpSet *pEpSet) { printf("request is received, size:%d\n", pRpc->contLen); fflush(stdout); SRpcMsg rsp = {.handle = pRpc->handle, .refId = pRpc->refId, .ahandle = pRpc->ahandle, .code = 0}; - rsp.pCont = rpcMallocCont(shell.args.pktLen); - rsp.contLen = shell.args.pktLen; + rsp.pCont = rpcMallocCont(pRpc->contLen); + if (rsp.pCont == NULL) { + rsp.code = TSDB_CODE_OUT_OF_MEMORY; + } else { + rsp.contLen = pRpc->contLen; + } rpcSendResponse(&rsp); }