xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' of github.com:taosdata/TDengine into feature/udf

This commit is contained in:
shenglian zhou 2022-04-25 13:28:36 +08:00
parent 9e411e9f92 815c70c604
commit 1952decc2f
66 changed files with 1576 additions and 1163 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Jenkinsfile vendored
View File

@ -113,7 +113,7 @@ pipeline {
'''
sh'''
cd ${WKC}/debug
ctest
ctest -VV
'''
}
}

2
Jenkinsfile2
View File

@ -121,7 +121,7 @@ pipeline {
pre_test()
sh'''
cd ${WKC}/debug
ctest
ctest -VV
'''
sh'''
export LD_LIBRARY_PATH=${WKC}/debug/build/lib

95
contrib/CMakeLists.txt
View File

@ -14,24 +14,9 @@ if(${BUILD_PTHREAD})
cat("${TD_SUPPORT_DIR}/pthread_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
endif()
# iconv
if(${BUILD_WITH_ICONV})
cat("${TD_SUPPORT_DIR}/iconv_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
endif()
# msvc regex
if(${BUILD_MSVCREGEX})
cat("${TD_SUPPORT_DIR}/msvcregex_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
endif()
# wcwidth
if(${BUILD_WCWIDTH})
cat("${TD_SUPPORT_DIR}/wcwidth_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
endif()
# wingetopt
if(${BUILD_WINGETOPT})
cat("${TD_SUPPORT_DIR}/wingetopt_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
# gnu regex
if(${BUILD_GNUREGEX})
cat("${TD_SUPPORT_DIR}/gnuregex_CMakeLists.txt.in" ${CONTRIB_TMP_FILE})
endif()
# googletest
@ -114,27 +99,8 @@ if(${BUILD_TEST})
target_include_directories(
gtest
PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src>
PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src_linux>
)
if(${TD_WINDOWS})
target_include_directories(
gtest
PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src_win>
)
endif(${TD_WINDOWS})
if(${TD_LINUX})
target_include_directories(
gtest
PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src_linux>
)
endif(${TD_LINUX})
if(${TD_DARWIN})
target_include_directories(
gtest
PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/cpp-stub/src_darwin>
)
endif(${TD_DARWIN})
endif(${BUILD_TEST})
# cJson
@ -216,53 +182,6 @@ if(${BUILD_WITH_NURAFT})
add_subdirectory(nuraft)
endif(${BUILD_WITH_NURAFT})
# pthread
if(${BUILD_PTHREAD})
set(CMAKE_BUILD_TYPE release)
add_definitions(-DPTW32_STATIC_LIB)
add_subdirectory(pthread)
set_target_properties(libpthreadVC3 PROPERTIES OUTPUT_NAME pthread)
add_library(pthread STATIC IMPORTED GLOBAL)
SET_PROPERTY(TARGET pthread PROPERTY IMPORTED_LOCATION ${LIBRARY_OUTPUT_PATH}/pthread.lib)
endif()
# iconv
if(${BUILD_WITH_ICONV})
add_subdirectory(iconv)
endif(${BUILD_WITH_ICONV})
# wingetopt
if(${BUILD_WINGETOPT})
add_subdirectory(wingetopt)
endif(${BUILD_WINGETOPT})
# msvcregex
if(${BUILD_MSVCREGEX})
add_library(msvcregex STATIC "")
target_sources(msvcregex
PRIVATE "msvcregex/regex.c"
)
target_include_directories(msvcregex
PRIVATE "msvcregex"
)
target_link_libraries(msvcregex
INTERFACE Shell32
)
SET_TARGET_PROPERTIES(msvcregex PROPERTIES OUTPUT_NAME msvcregex)
endif(${BUILD_MSVCREGEX})
# msvcregex
if(${BUILD_WCWIDTH})
add_library(wcwidth STATIC "")
target_sources(wcwidth
PRIVATE "wcwidth/wcwidth.c"
)
target_include_directories(wcwidth
PRIVATE "wcwidth"
)
SET_TARGET_PROPERTIES(wcwidth PROPERTIES OUTPUT_NAME wcwidth)
endif(${BUILD_WCWIDTH})
# CRAFT
if(${BUILD_WITH_CRAFT})
add_library(craft STATIC IMPORTED GLOBAL)
@ -319,12 +238,8 @@ if(${BUILD_WITH_SQLITE})
target_link_libraries(sqlite
INTERFACE m
INTERFACE pthread
INTERFACE dl
)
if(NOT TD_WINDOWS)
target_link_libraries(sqlite
INTERFACE dl
)
endif(NOT TD_WINDOWS)
endif(${BUILD_WITH_SQLITE})
# pthread

9
contrib/test/craft/CMakeLists.txt
View File

@ -1,9 +1,2 @@
add_executable(simulate_vnode "simulate_vnode.c")
target_link_libraries(simulate_vnode PUBLIC craft lz4 uv_a)
if(${BUILD_WINGETOPT})
target_link_libraries(simulate_vnode PUBLIC wingetopt)
target_include_directories(
simulate_vnode
PUBLIC "${TD_SOURCE_DIR}/contrib/wingetopt/src"
)
endif()
target_link_libraries(simulate_vnode PUBLIC craft lz4 uv_a)

72
contrib/test/tdev/src/main.c
View File

@ -6,39 +6,43 @@
#define POINTER_SHIFT(ptr, s) ((void *)(((char *)ptr) + (s)))
#define POINTER_DISTANCE(pa, pb) ((char *)(pb) - (char *)(pa))
static inline void tPutA(void **buf, uint64_t val) {
memcpy(buf, &val, sizeof(val));
*buf = POINTER_SHIFT(buf, sizeof(val));
}
#define tPutA(buf, val) \
({ \
memcpy(buf, &val, sizeof(val)); \
POINTER_SHIFT(buf, sizeof(val)); \
})
static inline void tPutB(void **buf, uint64_t val) {
((uint8_t *)buf)[7] = ((val) >> 56) & 0xff;
((uint8_t *)buf)[6] = ((val) >> 48) & 0xff;
((uint8_t *)buf)[5] = ((val) >> 40) & 0xff;
((uint8_t *)buf)[4] = ((val) >> 32) & 0xff;
((uint8_t *)buf)[3] = ((val) >> 24) & 0xff;
((uint8_t *)buf)[2] = ((val) >> 16) & 0xff;
((uint8_t *)buf)[1] = ((val) >> 8) & 0xff;
((uint8_t *)buf)[0] = (val)&0xff;
*buf = POINTER_SHIFT(buf, sizeof(val));
}
#define tPutB(buf, val) \
({ \
((uint8_t *)buf)[7] = ((val) >> 56) & 0xff; \
((uint8_t *)buf)[6] = ((val) >> 48) & 0xff; \
((uint8_t *)buf)[5] = ((val) >> 40) & 0xff; \
((uint8_t *)buf)[4] = ((val) >> 32) & 0xff; \
((uint8_t *)buf)[3] = ((val) >> 24) & 0xff; \
((uint8_t *)buf)[2] = ((val) >> 16) & 0xff; \
((uint8_t *)buf)[1] = ((val) >> 8) & 0xff; \
((uint8_t *)buf)[0] = (val)&0xff; \
POINTER_SHIFT(buf, sizeof(val)); \
})
static inline void tPutC(void **buf, uint64_t val) {
if (buf) {
((uint64_t *)buf)[0] = (val);
POINTER_SHIFT(buf, sizeof(val));
}
*buf = NULL;
}
#define tPutC(buf, val) \
({ \
if (buf) { \
((uint64_t *)buf)[0] = (val); \
POINTER_SHIFT(buf, sizeof(val)); \
} \
NULL; \
})
static inline void tPutD(void **buf, uint64_t val) {
uint64_t tmp = val;
for (size_t i = 0; i < sizeof(val); i++) {
((uint8_t *)buf)[i] = tmp & 0xff;
tmp >>= 8;
}
*buf = POINTER_SHIFT(buf, sizeof(val));
}
#define tPutD(buf, val) \
({ \
uint64_t tmp = val; \
for (size_t i = 0; i < sizeof(val); i++) { \
((uint8_t *)buf)[i] = tmp & 0xff; \
tmp >>= 8; \
} \
POINTER_SHIFT(buf, sizeof(val)); \
})
static inline void tPutE(void **buf, uint64_t val) {
if (buf) {
@ -57,7 +61,7 @@ static void func(T t) {
switch (t) {
case A:
for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) {
tPutA(pBuf, val);
pBuf = tPutA(pBuf, val);
if (POINTER_DISTANCE(buf, pBuf) == 1024) {
pBuf = buf;
}
@ -65,7 +69,7 @@ static void func(T t) {
break;
case B:
for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) {
tPutB(pBuf, val);
pBuf = tPutB(pBuf, val);
if (POINTER_DISTANCE(buf, pBuf) == 1024) {
pBuf = buf;
}
@ -73,7 +77,7 @@ static void func(T t) {
break;
case C:
for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) {
tPutC(pBuf, val);
pBuf = tPutC(pBuf, val);
if (POINTER_DISTANCE(buf, pBuf) == 1024) {
pBuf = buf;
}
@ -81,7 +85,7 @@ static void func(T t) {
break;
case D:
for (size_t i = 0; i < 10 * 1024l * 1024l * 1024l; i++) {
tPutD(pBuf, val);
pBuf = tPutD(pBuf, val);
if (POINTER_DISTANCE(buf, pBuf) == 1024) {
pBuf = buf;
}

8
example/src/tmq.c
View File

@ -17,6 +17,7 @@
#include <stdio.h>
#include <string.h>
#include <time.h>
#include <unistd.h>
#include "taos.h"
static int running = 1;
@ -47,6 +48,7 @@ int32_t init_env() {
return -1;
}
taos_free_result(pRes);
sleep(1);
pRes = taos_query(pConn, "use abc1");
if (taos_errno(pRes) != 0) {
@ -58,6 +60,7 @@ int32_t init_env() {
pRes =
taos_query(pConn, "create stable if not exists st1 (ts timestamp, c1 int, c2 float, c3 binary(10)) tags(t1 int)");
if (taos_errno(pRes) != 0) {
assert(0);
printf("failed to create super table st1, reason:%s\n", taos_errstr(pRes));
return -1;
}
@ -265,10 +268,11 @@ void perf_loop(tmq_t* tmq, tmq_list_t* topics) {
}
int main(int argc, char* argv[]) {
int code;
if (argc > 1) {
printf("env init\n");
code = init_env();
if (init_env() < 0) {
return -1;
}
create_topic();
}
tmq_t* tmq = build_consumer();

13
include/common/tdatablock.h
View File

@ -238,10 +238,16 @@ static FORCE_INLINE int32_t blockCompressColData(SColumnInfoData* pColRes, int32
static FORCE_INLINE void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols,
int8_t needCompress) {
int32_t* colSizes = (int32_t*)data;
int32_t* actualLen = (int32_t*) data;
data += sizeof(int32_t);
uint64_t* groupId = (uint64_t*) data;
data += sizeof(uint64_t);
int32_t* colSizes = (int32_t*)data;
data += numOfCols * sizeof(int32_t);
*dataLen = (numOfCols * sizeof(int32_t));
*dataLen = (numOfCols * sizeof(int32_t) + sizeof(uint64_t) + sizeof(int32_t));
int32_t numOfRows = pBlock->info.rows;
for (int32_t col = 0; col < numOfCols; ++col) {
@ -273,6 +279,9 @@ static FORCE_INLINE void blockCompressEncode(const SSDataBlock* pBlock, char* da
colSizes[col] = htonl(colSizes[col]);
}
*actualLen = *dataLen;
*groupId = pBlock->info.groupId;
}
#ifdef __cplusplus

22
include/common/tmsg.h
View File

@ -1314,7 +1314,6 @@ typedef struct {
} SMqConsumerLostMsg;
typedef struct {
int32_t topicNum;
int64_t consumerId;
char cgroup[TSDB_CGROUP_LEN];
SArray* topicNames; // SArray<char*>
@ -1322,22 +1321,27 @@ typedef struct {
static FORCE_INLINE int32_t tSerializeSCMSubscribeReq(void** buf, const SCMSubscribeReq* pReq) {
int32_t tlen = 0;
tlen += taosEncodeFixedI32(buf, pReq->topicNum);
tlen += taosEncodeFixedI64(buf, pReq->consumerId);
tlen += taosEncodeString(buf, pReq->cgroup);
for (int32_t i = 0; i < pReq->topicNum; i++) {
int32_t topicNum = taosArrayGetSize(pReq->topicNames);
tlen += taosEncodeFixedI32(buf, topicNum);
for (int32_t i = 0; i < topicNum; i++) {
tlen += taosEncodeString(buf, (char*)taosArrayGetP(pReq->topicNames, i));
}
return tlen;
}
static FORCE_INLINE void* tDeserializeSCMSubscribeReq(void* buf, SCMSubscribeReq* pReq) {
buf = taosDecodeFixedI32(buf, &pReq->topicNum);
buf = taosDecodeFixedI64(buf, &pReq->consumerId);
buf = taosDecodeStringTo(buf, pReq->cgroup);
pReq->topicNames = taosArrayInit(pReq->topicNum, sizeof(void*));
for (int32_t i = 0; i < pReq->topicNum; i++) {
int32_t topicNum;
buf = taosDecodeFixedI32(buf, &topicNum);
pReq->topicNames = taosArrayInit(topicNum, sizeof(void*));
for (int32_t i = 0; i < topicNum; i++) {
char* name;
buf = taosDecodeString(buf, &name);
taosArrayPush(pReq->topicNames, &name);
@ -1969,7 +1973,6 @@ typedef struct {
int8_t withTbName;
int8_t withSchema;
int8_t withTag;
int8_t withTagSchema;
char* qmsg;
} SMqRebVgReq;
@ -1984,7 +1987,6 @@ static FORCE_INLINE int32_t tEncodeSMqRebVgReq(void** buf, const SMqRebVgReq* pR
tlen += taosEncodeFixedI8(buf, pReq->withTbName);
tlen += taosEncodeFixedI8(buf, pReq->withSchema);
tlen += taosEncodeFixedI8(buf, pReq->withTag);
tlen += taosEncodeFixedI8(buf, pReq->withTagSchema);
if (pReq->subType == TOPIC_SUB_TYPE__TABLE) {
tlen += taosEncodeString(buf, pReq->qmsg);
}
@ -2001,7 +2003,6 @@ static FORCE_INLINE void* tDecodeSMqRebVgReq(const void* buf, SMqRebVgReq* pReq)
buf = taosDecodeFixedI8(buf, &pReq->withTbName);
buf = taosDecodeFixedI8(buf, &pReq->withSchema);
buf = taosDecodeFixedI8(buf, &pReq->withTag);
buf = taosDecodeFixedI8(buf, &pReq->withTagSchema);
if (pReq->subType == TOPIC_SUB_TYPE__TABLE) {
buf = taosDecodeString(buf, &pReq->qmsg);
}
@ -2590,7 +2591,6 @@ typedef struct {
int8_t withTbName;
int8_t withSchema;
int8_t withTag;
int8_t withTagSchema;
SArray* blockDataLen; // SArray<int32_t>
SArray* blockData; // SArray<SRetrieveTableRsp*>
SArray* blockTbName; // SArray<char*>
@ -2609,7 +2609,6 @@ static FORCE_INLINE int32_t tEncodeSMqDataBlkRsp(void** buf, const SMqDataBlkRsp
tlen += taosEncodeFixedI8(buf, pRsp->withTbName);
tlen += taosEncodeFixedI8(buf, pRsp->withSchema);
tlen += taosEncodeFixedI8(buf, pRsp->withTag);
tlen += taosEncodeFixedI8(buf, pRsp->withTagSchema);
for (int32_t i = 0; i < pRsp->blockNum; i++) {
int32_t bLen = *(int32_t*)taosArrayGet(pRsp->blockDataLen, i);
@ -2632,7 +2631,6 @@ static FORCE_INLINE void* tDecodeSMqDataBlkRsp(const void* buf, SMqDataBlkRsp* p
buf = taosDecodeFixedI8(buf, &pRsp->withTbName);
buf = taosDecodeFixedI8(buf, &pRsp->withSchema);
buf = taosDecodeFixedI8(buf, &pRsp->withTag);
buf = taosDecodeFixedI8(buf, &pRsp->withTagSchema);
for (int32_t i = 0; i < pRsp->blockNum; i++) {
int32_t bLen = 0;

2
include/libs/catalog/catalog.h
View File

@ -51,7 +51,7 @@ typedef struct SMetaData {
SArray *pTableMeta; // STableMeta array
SArray *pVgroupInfo; // SVgroupInfo list
SArray *pUdfList; // udf info list
SArray *pEpSetList; // qnode epset list, SArray<SEpSet>
SArray *pQnodeList; // qnode list, SArray<SQueryNodeAddr>
} SMetaData;
typedef struct SCatalogCfg {

35
include/libs/function/function.h
View File

@ -37,7 +37,7 @@ typedef struct SFuncExecEnv {
typedef bool (*FExecGetEnv)(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
typedef bool (*FExecInit)(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo);
typedef int32_t (*FExecProcess)(struct SqlFunctionCtx *pCtx);
typedef int32_t (*FExecFinalize)(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock, int32_t slotId);
typedef int32_t (*FExecFinalize)(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock);
typedef int32_t (*FScalarExecProcess)(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
typedef struct SScalarFuncExecFuncs {
@ -141,8 +141,7 @@ struct SResultRowEntryInfo;
//for selectivity query, the corresponding tag value is assigned if the data is qualified
typedef struct SSubsidiaryResInfo {
int16_t bufLen; // keep the tags data for top/bottom query result
int16_t numOfCols;
int16_t num;
struct SqlFunctionCtx **pCtx;
} SSubsidiaryResInfo;
@ -187,8 +186,8 @@ typedef struct SqlFunctionCtx {
uint8_t currentStage; // record current running step, default: 0
bool isAggSet;
int64_t startTs; // timestamp range of current query when function is executed on a specific data block, TODO remove it
/////////////////////////////////////////////////////////////////
bool stableQuery;
/////////////////////////////////////////////////////////////////
int16_t functionId; // function id
char * pOutput; // final result output buffer, point to sdata->data
int32_t numOfParams;
@ -198,11 +197,15 @@ typedef struct SqlFunctionCtx {
int32_t offset;
SVariant tag;
struct SResultRowEntryInfo *resultInfo;
SSubsidiaryResInfo subsidiaryRes;
SPoint1 start;
SPoint1 end;
SFuncExecFuncs fpSet;
SScalarFuncExecFuncs sfp;
SSubsidiaryResInfo subsidiaries;
SPoint1 start;
SPoint1 end;
SFuncExecFuncs fpSet;
SScalarFuncExecFuncs sfp;
SExprInfo *pExpr;
struct SDiskbasedBuf *pBuf;
struct SSDataBlock *pSrcBlock;
int32_t curBufPage;
} SqlFunctionCtx;
enum {
@ -319,6 +322,20 @@ struct SUdfInfo;
void qAddUdfInfo(uint64_t id, struct SUdfInfo* pUdfInfo);
void qRemoveUdfInfo(uint64_t id, struct SUdfInfo* pUdfInfo);
/**
* create udfd proxy, called once in process that call setupUdf/callUdfxxx/teardownUdf
* @return error code
*/
int32_t udfcOpen();
/**
* destroy udfd proxy
* @return error code
*/
int32_t udfcClose();
typedef void *UdfcFuncHandle;
#ifdef __cplusplus
}
#endif

8
include/libs/wal/wal.h
View File

@ -192,7 +192,13 @@ int32_t walEndSnapshot(SWal *);
SWalReadHandle *walOpenReadHandle(SWal *);
void walCloseReadHandle(SWalReadHandle *);
int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver);
int32_t walReadWithHandle_s(SWalReadHandle *pRead, int64_t ver, SWalReadHead **ppHead);
// only for tq usage
// int32_t walReadWithHandle_s(SWalReadHandle *pRead, int64_t ver, SWalReadHead **ppHead);
void walSetReaderCapacity(SWalReadHandle *pRead, int32_t capacity);
int32_t walFetchHead(SWalReadHandle *pRead, int64_t ver, SWalHead *pHead);
int32_t walFetchBody(SWalReadHandle *pRead, SWalHead **ppHead);
int32_t walSkipFetchBody(SWalReadHandle *pRead, const SWalHead *pHead);
// deprecated
#if 0

2
include/util/taoserror.h
View File

@ -252,7 +252,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_MND_COLUMN_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03AE)
// mnode-infoSchema
#define TSDB_CODE_MND_INVALID_INFOS_TBL TAOS_DEF_ERROR_CODE(0, 0x03B0)
#define TSDB_CODE_MND_INVALID_SYS_TABLENAME TAOS_DEF_ERROR_CODE(0, 0x03B0)
// mnode-func
#define TSDB_CODE_MND_FUNC_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03C0)

19
source/client/src/clientImpl.c
View File

@ -105,9 +105,9 @@ TAOS* taos_connect_internal(const char* ip, const char* user, const char* pass,
epSet.epSet.eps[0].port = port;
}
char* key = getClusterKey(user, secretEncrypt, ip, port);
SAppInstInfo** pInst = NULL;
char* key = getClusterKey(user, secretEncrypt, ip, port);
SAppInstInfo** pInst = NULL;
taosThreadMutexLock(&appInfo.mutex);
pInst = taosHashGet(appInfo.pInstMap, key, strlen(key));
@ -840,10 +840,21 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32
return code;
}
int32_t* colLength = (int32_t*)pResultInfo->pData;
char* pStart = ((char*)pResultInfo->pData) + sizeof(int32_t) * numOfCols;
char* p = (char*) pResultInfo->pData;
int32_t dataLen = *(int32_t*) p;
p += sizeof(int32_t);
uint64_t groupId = *(uint64_t*) p;
p += sizeof(uint64_t);
int32_t* colLength = (int32_t*)p;
p += sizeof(int32_t) * numOfCols;
char* pStart = p;
for (int32_t i = 0; i < numOfCols; ++i) {
colLength[i] = htonl(colLength[i]);
ASSERT(colLength[i] < dataLen);
if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) {
pResultInfo->pCol[i].offset = (int32_t*)pStart;

30
source/client/src/clientMsgHandler.c
View File

@ -13,13 +13,13 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "os.h"
#include "tdef.h"
#include "tname.h"
#include "catalog.h"
#include "clientInt.h"
#include "clientLog.h"
#include "catalog.h"
#include "os.h"
#include "query.h"
#include "tdef.h"
#include "tname.h"
int32_t (*handleRequestRspFp[TDMT_MAX])(void*, const SDataBuf* pMsg, int32_t code);
@ -50,7 +50,13 @@ int32_t processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) {
SConnectRsp connectRsp = {0};
tDeserializeSConnectRsp(pMsg->pData, pMsg->len, &connectRsp);
assert(connectRsp.epSet.numOfEps > 0);
/*assert(connectRsp.epSet.numOfEps > 0);*/
if (connectRsp.epSet.numOfEps == 0) {
taosMemoryFree(pMsg->pData);
setErrno(pRequest, TSDB_CODE_MND_APP_ERROR);
tsem_post(&pRequest->body.rspSem);
return code;
}
if (!isEpsetEqual(&pTscObj->pAppInfo->mgmtEp.epSet, &connectRsp.epSet)) {
updateEpSet_s(&pTscObj->pAppInfo->mgmtEp, &connectRsp.epSet);
@ -82,18 +88,20 @@ int32_t processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) {
return 0;
}
SMsgSendInfo* buildMsgInfoImpl(SRequestObj *pRequest) {
SMsgSendInfo* buildMsgInfoImpl(SRequestObj* pRequest) {
SMsgSendInfo* pMsgSendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo));
pMsgSendInfo->requestObjRefId = pRequest->self;
pMsgSendInfo->requestId = pRequest->requestId;
pMsgSendInfo->param = pRequest;
pMsgSendInfo->msgType = pRequest->type;
pMsgSendInfo->requestId = pRequest->requestId;
pMsgSendInfo->param = pRequest;
pMsgSendInfo->msgType = pRequest->type;
assert(pRequest != NULL);
pMsgSendInfo->msgInfo = pRequest->body.requestMsg;
pMsgSendInfo->fp = (handleRequestRspFp[TMSG_INDEX(pRequest->type)] == NULL)? genericRspCallback:handleRequestRspFp[TMSG_INDEX(pRequest->type)];
pMsgSendInfo->fp = (handleRequestRspFp[TMSG_INDEX(pRequest->type)] == NULL)
? genericRspCallback
: handleRequestRspFp[TMSG_INDEX(pRequest->type)];
return pMsgSendInfo;
}
@ -114,7 +122,7 @@ int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) {
if (TSDB_CODE_MND_DB_NOT_EXIST == code) {
SUseDbRsp usedbRsp = {0};
tDeserializeSUseDbRsp(pMsg->pData, pMsg->len, &usedbRsp);
struct SCatalog *pCatalog = NULL;
struct SCatalog* pCatalog = NULL;
if (usedbRsp.vgVersion >= 0) {
int32_t code1 = catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog);

12
source/client/src/tmq.c
View File

@ -382,12 +382,9 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
pTmq->pTscObj = taos_connect_internal(conf->ip, user, pass, NULL, conf->db, conf->port, CONN_TYPE__TMQ);
if (pTmq->pTscObj == NULL) return NULL;
/*pTmq->inWaiting = 0;*/
pTmq->status = 0;
pTmq->pollCnt = 0;
pTmq->epoch = 0;
/*pTmq->waitingRequest = 0;*/
/*pTmq->readyRequest = 0;*/
pTmq->epStatus = 0;
pTmq->epSkipCnt = 0;
// set conf
@ -509,7 +506,6 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) {
tmq->clientTopics = taosArrayInit(sz, sizeof(SMqClientTopic));
SCMSubscribeReq req;
req.topicNum = sz;
req.consumerId = tmq->consumerId;
strcpy(req.cgroup, tmq->groupId);
req.topicNames = taosArrayInit(sz, sizeof(void*));
@ -519,12 +515,16 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) {
char* topicName = taosArrayGetP(container, i);
SName name = {0};
#if 0
char* dbName = getDbOfConnection(tmq->pTscObj);
if (dbName == NULL) {
return TMQ_RESP_ERR__FAIL;
}
tNameSetDbName(&name, tmq->pTscObj->acctId, dbName, strlen(dbName));
#endif
tNameSetDbName(&name, tmq->pTscObj->acctId, topicName, strlen(topicName));
#if 0
tNameFromString(&name, topicName, T_NAME_TABLE);
#endif
char* topicFname = taosMemoryCalloc(1, TSDB_TOPIC_FNAME_LEN);
if (topicFname == NULL) {
@ -542,7 +542,9 @@ tmq_resp_err_t tmq_subscribe(tmq_t* tmq, tmq_list_t* topic_list) {
topic.vgs = taosArrayInit(0, sizeof(SMqClientVg));
taosArrayPush(tmq->clientTopics, &topic);
taosArrayPush(req.topicNames, &topicFname);
#if 0
taosMemoryFree(dbName);
#endif
}
int tlen = tSerializeSCMSubscribeReq(NULL, &req);

5
source/common/src/tdatablock.c
View File

@ -644,12 +644,12 @@ size_t blockDataGetRowSize(SSDataBlock* pBlock) {
/**
* @refitem blockDataToBuf for the meta size
*
* @param pBlock
* @return
*/
size_t blockDataGetSerialMetaSize(const SSDataBlock* pBlock) {
return sizeof(int32_t) + pBlock->info.numOfCols * sizeof(int32_t);
// | total rows/total length | block group id | each column length |
return sizeof(int32_t) + sizeof(uint64_t) + pBlock->info.numOfCols * sizeof(int32_t);
}
double blockDataGetSerialRowSize(const SSDataBlock* pBlock) {
@ -1219,7 +1219,6 @@ void colDataDestroy(SColumnInfoData* pColData) {
taosMemoryFree(pColData->pData);
}
static void doShiftBitmap(char* nullBitmap, size_t n, size_t total) {
int32_t len = BitmapLen(total);

31
source/common/src/tmsg.c
View File

@ -126,7 +126,6 @@ int32_t tDecodeSQueryNodeAddr(SCoder *pDecoder, SQueryNodeAddr *pAddr) {
return 0;
}
int32_t taosEncodeSEpSet(void **buf, const SEpSet *pEp) {
int32_t tlen = 0;
tlen += taosEncodeFixedI8(buf, pEp->inUse);
@ -1568,13 +1567,8 @@ int32_t tSerializeSCreateFuncReq(void *buf, int32_t bufLen, SCreateFuncReq *pReq
if (tEncodeI32(&encoder, pReq->codeLen) < 0) return -1;
if (tEncodeI64(&encoder, pReq->signature) < 0) return -1;
int32_t codeSize = 0;
if (pReq->pCode != NULL) {
codeSize = strlen(pReq->pCode) + 1;
}
if (tEncodeI32(&encoder, codeSize) < 0) return -1;
if (pReq->pCode != NULL) {
if (tEncodeCStr(&encoder, pReq->pCode) < 0) return -1;
if (tEncodeBinary(&encoder, pReq->pCode, pReq->codeLen) < 0) return -1;
}
int32_t commentSize = 0;
@ -1608,10 +1602,8 @@ int32_t tDeserializeSCreateFuncReq(void *buf, int32_t bufLen, SCreateFuncReq *pR
if (tDecodeI32(&decoder, &pReq->codeLen) < 0) return -1;
if (tDecodeI64(&decoder, &pReq->signature) < 0) return -1;
int32_t codeSize = 0;
if (tDecodeI32(&decoder, &codeSize) < 0) return -1;
if (codeSize > 0) {
pReq->pCode = taosMemoryCalloc(1, codeSize);
if (pReq->codeLen > 0) {
pReq->pCode = taosMemoryCalloc(1, pReq->codeLen);
if (pReq->pCode == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
@ -1734,7 +1726,7 @@ int32_t tSerializeSRetrieveFuncRsp(void *buf, int32_t bufLen, SRetrieveFuncRsp *
if (tEncodeI32(&encoder, pInfo->codeSize) < 0) return -1;
if (tEncodeI32(&encoder, pInfo->commentSize) < 0) return -1;
if (pInfo->codeSize) {
if (tEncodeCStr(&encoder, pInfo->pCode) < 0) return -1;
if (tEncodeBinary(&encoder, pInfo->pCode, pInfo->codeSize) < 0) return -1;
}
if (pInfo->commentSize) {
if (tEncodeCStr(&encoder, pInfo->pComment) < 0) return -1;
@ -2091,10 +2083,15 @@ int32_t tDeserializeSQnodeListRsp(void *buf, int32_t bufLen, SQnodeListRsp *pRsp
if (tStartDecode(&decoder) < 0) return -1;
int32_t num = 0;
if (tDecodeI32(&decoder, &num) < 0) return -1;
pRsp->addrsList = taosArrayInit(num, sizeof(SQueryNodeAddr));
if (NULL == pRsp->addrsList) return -1;
if (NULL == pRsp->addrsList) {
pRsp->addrsList = taosArrayInit(num, sizeof(SQueryNodeAddr));
if (NULL == pRsp->addrsList) return -1;
}
for (int32_t i = 0; i < num; ++i) {
if (tDecodeSQueryNodeAddr(&decoder, TARRAY_GET_ELEM(pRsp->addrsList, i)) < 0) return -1;
SQueryNodeAddr addr = {0};
if (tDecodeSQueryNodeAddr(&decoder, &addr) < 0) return -1;
taosArrayPush(pRsp->addrsList, &addr);
}
tEndDecode(&decoder);
@ -2747,11 +2744,11 @@ int32_t tSerializeSCMCreateTopicReq(void *buf, int32_t bufLen, const SCMCreateTo
if (tEncodeI8(&encoder, pReq->withTbName) < 0) return -1;
if (tEncodeI8(&encoder, pReq->withSchema) < 0) return -1;
if (tEncodeI8(&encoder, pReq->withTag) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->subscribeDbName) < 0) return -1;
if (tEncodeI32(&encoder, sqlLen) < 0) return -1;
if (tEncodeI32(&encoder, astLen) < 0) return -1;
if (sqlLen > 0 && tEncodeCStr(&encoder, pReq->sql) < 0) return -1;
if (astLen > 0 && tEncodeCStr(&encoder, pReq->ast) < 0) return -1;
if (0 == astLen && tEncodeCStr(&encoder, pReq->subscribeDbName) < 0) return -1;
tEndEncode(&encoder);
@ -2773,6 +2770,7 @@ int32_t tDeserializeSCMCreateTopicReq(void *buf, int32_t bufLen, SCMCreateTopicR
if (tDecodeI8(&decoder, &pReq->withTbName) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->withSchema) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->withTag) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->subscribeDbName) < 0) return -1;
if (tDecodeI32(&decoder, &sqlLen) < 0) return -1;
if (tDecodeI32(&decoder, &astLen) < 0) return -1;
@ -2787,7 +2785,6 @@ int32_t tDeserializeSCMCreateTopicReq(void *buf, int32_t bufLen, SCMCreateTopicR
if (pReq->ast == NULL) return -1;
if (tDecodeCStrTo(&decoder, pReq->ast) < 0) return -1;
} else {
if (tDecodeCStrTo(&decoder, pReq->subscribeDbName) < 0) return -1;
}
tEndDecode(&decoder);

24
source/dnode/mgmt/implement/src/dmHandle.c
View File

@ -311,6 +311,9 @@ static void dmWatchUdfd(void *args) {
}
static int32_t dmStartUdfd(SDnode *pDnode) {
char dnodeId[8] = {0};
snprintf(dnodeId, sizeof(dnodeId), "%d", pDnode->data.dnodeId);
uv_os_setenv("DNODE_ID", dnodeId);
SUdfdData *pData = &pDnode->udfdData;
if (pData->startCalled) {
dInfo("dnode-mgmt start udfd already called");
@ -320,8 +323,17 @@ static int32_t dmStartUdfd(SDnode *pDnode) {
uv_barrier_init(&pData->barrier, 2);
uv_thread_create(&pData->thread, dmWatchUdfd, pDnode);
uv_barrier_wait(&pData->barrier);
pData->needCleanUp = true;
return pData->spawnErr;
int32_t err = atomic_load_32(&pData->spawnErr);
if (err != 0) {
uv_barrier_destroy(&pData->barrier);
uv_async_send(&pData->stopAsync);
uv_thread_join(&pData->thread);
pData->needCleanUp = false;
dInfo("dnode-mgmt udfd cleaned up after spawn err");
} else {
pData->needCleanUp = true;
}
return err;
}
static int32_t dmStopUdfd(SDnode *pDnode) {
@ -336,7 +348,7 @@ static int32_t dmStopUdfd(SDnode *pDnode) {
uv_barrier_destroy(&pData->barrier);
uv_async_send(&pData->stopAsync);
uv_thread_join(&pData->thread);
dInfo("dnode-mgmt udfd cleaned up");
return 0;
}
@ -371,9 +383,9 @@ static int32_t dmInitMgmt(SMgmtWrapper *pWrapper) {
}
dmReportStartup(pDnode, "dnode-transport", "initialized");
// if (dmStartUdfd(pDnode) != 0) {
// dError("failed to start udfd");
// }
if (dmStartUdfd(pDnode) != 0) {
dError("failed to start udfd");
}
dInfo("dnode-mgmt is initialized");
return 0;

8
source/dnode/mgmt/interface/src/dmInt.c
View File

@ -173,9 +173,13 @@ static void dmGetServerStatus(SDnode *pDnode, SServerStatusRsp *pStatus) {
void dmProcessNettestReq(SDnode *pDnode, SRpcMsg *pRpc) {
dDebug("net test req is received");
SRpcMsg rsp = {.handle = pRpc->handle, .ahandle = pRpc->ahandle, .code = 0};
SRpcMsg rsp = {.handle = pRpc->handle, .refId = pRpc->refId, .ahandle = pRpc->ahandle, .code = 0};
rsp.pCont = rpcMallocCont(pRpc->contLen);
rsp.contLen = pRpc->contLen;
if (rsp.pCont == NULL) {
rsp.code = TSDB_CODE_OUT_OF_MEMORY;
} else {
rsp.contLen = pRpc->contLen;
}
rpcSendResponse(&rsp);
}

1
source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
View File

@ -177,6 +177,7 @@ void mmInitMsgHandle(SMgmtWrapper *pWrapper) {
dmSetMsgHandle(pWrapper, TDMT_MND_DROP_MNODE, mmProcessWriteMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_MND_CREATE_QNODE, mmProcessWriteMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_MND_DROP_QNODE, mmProcessWriteMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_MND_QNODE_LIST, mmProcessReadMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_MND_CREATE_SNODE, mmProcessWriteMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_MND_DROP_SNODE, mmProcessWriteMsg, DEFAULT_HANDLE);
dmSetMsgHandle(pWrapper, TDMT_MND_CREATE_BNODE, mmProcessWriteMsg, DEFAULT_HANDLE);

7
source/dnode/mgmt/mgmt_vnode/src/vmInt.c
View File

@ -15,6 +15,7 @@
#define _DEFAULT_SOURCE
#include "vmInt.h"
#include "libs/function/function.h"
SVnodeObj *vmAcquireVnode(SVnodesMgmt *pMgmt, int32_t vgId) {
SVnodeObj *pVnode = NULL;
@ -275,7 +276,7 @@ static void vmCleanup(SMgmtWrapper *pWrapper) {
pWrapper->pMgmt = NULL;
// syncCleanUp();
udfcClose();
dInfo("vnode-mgmt is cleaned up");
}
@ -339,6 +340,10 @@ static int32_t vmInit(SMgmtWrapper *pWrapper) {
}
dmReportStartup(pDnode, "vnode-vnodes", "initialized");
if (udfcOpen() != 0) {
dError("failed to open udfc in dnode");
}
code = 0;
_OVER:

2
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -450,7 +450,6 @@ typedef struct {
int8_t withTbName;
int8_t withSchema;
int8_t withTag;
int8_t withTagSchema;
SRWLatch lock;
int32_t sqlLen;
int32_t astLen;
@ -517,7 +516,6 @@ typedef struct {
int8_t withTbName;
int8_t withSchema;
int8_t withTag;
int8_t withTagSchema;
SHashObj* consumerHash; // consumerId -> SMqConsumerEpInSub
// TODO put -1 into unassignVgs
// SArray* unassignedVgs;

3
source/dnode/mnode/impl/src/mndDef.c
View File

@ -237,7 +237,6 @@ SMqSubscribeObj *tCloneSubscribeObj(const SMqSubscribeObj *pSub) {
pSubNew->withTbName = pSub->withTbName;
pSubNew->withSchema = pSub->withSchema;
pSubNew->withTag = pSub->withTag;
pSubNew->withTagSchema = pSub->withTagSchema;
pSubNew->vgNum = pSub->vgNum;
pSubNew->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
@ -270,7 +269,6 @@ int32_t tEncodeSubscribeObj(void **buf, const SMqSubscribeObj *pSub) {
tlen += taosEncodeFixedI8(buf, pSub->withTbName);
tlen += taosEncodeFixedI8(buf, pSub->withSchema);
tlen += taosEncodeFixedI8(buf, pSub->withTag);
tlen += taosEncodeFixedI8(buf, pSub->withTagSchema);
void *pIter = NULL;
int32_t sz = taosHashGetSize(pSub->consumerHash);
@ -297,7 +295,6 @@ void *tDecodeSubscribeObj(const void *buf, SMqSubscribeObj *pSub) {
buf = taosDecodeFixedI8(buf, &pSub->withTbName);
buf = taosDecodeFixedI8(buf, &pSub->withSchema);
buf = taosDecodeFixedI8(buf, &pSub->withTag);
buf = taosDecodeFixedI8(buf, &pSub->withTagSchema);
int32_t sz;
buf = taosDecodeFixedI32(buf, &sz);

8
source/dnode/mnode/impl/src/mndFunc.c
View File

@ -309,10 +309,10 @@ static int32_t mndProcessCreateFuncReq(SNodeMsg *pReq) {
goto _OVER;
}
if (createReq.pCode[0] == 0) {
terrno = TSDB_CODE_MND_INVALID_FUNC_CODE;
goto _OVER;
}
if (createReq.codeLen <= 1) {
terrno = TSDB_CODE_MND_INVALID_FUNC_CODE;
goto _OVER;
}
if (createReq.bufSize <= 0 || createReq.bufSize > TSDB_FUNC_BUF_SIZE) {
terrno = TSDB_CODE_MND_INVALID_FUNC_BUFSIZE;

6
source/dnode/mnode/impl/src/mndInfoSchema.c
View File

@ -325,7 +325,7 @@ static int32_t mndInsInitMeta(SHashObj *hash) {
return -1;
}
if (taosHashPut(hash, meta.tbName, strlen(meta.tbName) + 1, &meta, sizeof(meta))) {
if (taosHashPut(hash, meta.tbName, strlen(meta.tbName), &meta, sizeof(meta))) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
@ -340,10 +340,10 @@ int32_t mndBuildInsTableSchema(SMnode *pMnode, const char *dbFName, const char *
return -1;
}
STableMetaRsp *pMeta = taosHashGet(pMnode->infosMeta, tbName, strlen(tbName) + 1);
STableMetaRsp *pMeta = taosHashGet(pMnode->infosMeta, tbName, strlen(tbName));
if (NULL == pMeta) {
mError("invalid information schema table name:%s", tbName);
terrno = TSDB_CODE_MND_INVALID_INFOS_TBL;
terrno = TSDB_CODE_MND_INVALID_SYS_TABLENAME;
return -1;
}

2
source/dnode/mnode/impl/src/mndPerfSchema.c
View File

@ -128,7 +128,7 @@ int32_t mndBuildPerfsTableSchema(SMnode *pMnode, const char *dbFName, const char
STableMetaRsp *meta = (STableMetaRsp *)taosHashGet(pMnode->perfsMeta, tbName, strlen(tbName));
if (NULL == meta) {
mError("invalid performance schema table name:%s", tbName);
terrno = TSDB_CODE_MND_INVALID_INFOS_TBL;
terrno = TSDB_CODE_MND_INVALID_SYS_TABLENAME;
return -1;
}

5
source/dnode/mnode/impl/src/mndQnode.c
View File

@ -451,8 +451,9 @@ static int32_t mndProcessQnodeListReq(SNodeMsg *pReq) {
goto _OVER;
}
void *pIter = NULL;
while (1) {
void *pIter = sdbFetch(pSdb, SDB_QNODE, NULL, (void **)&pObj);
pIter = sdbFetch(pSdb, SDB_QNODE, pIter, (void **)&pObj);
if (pIter == NULL) break;
SQueryNodeAddr nodeAddr = {0};
@ -472,7 +473,7 @@ static int32_t mndProcessQnodeListReq(SNodeMsg *pReq) {
}
int32_t rspLen = tSerializeSQnodeListRsp(NULL, 0, &qlistRsp);
void *pRsp = taosMemoryMalloc(rspLen);
void *pRsp = rpcMallocCont(rspLen);
if (pRsp == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;

13
source/dnode/mnode/impl/src/mndShow.c
View File

@ -100,6 +100,8 @@ static int32_t convertToRetrieveType(char* name, int32_t len) {
type = TSDB_MGMT_TABLE_QUERIES;
} else if (strncasecmp(name, TSDB_INS_TABLE_VNODES, len) == 0) {
type = TSDB_MGMT_TABLE_VNODES;
} else if (strncasecmp(name, TSDB_PERFS_TABLE_TOPICS, len) == 0) {
type = TSDB_MGMT_TABLE_TOPICS;
} else {
// ASSERT(0);
}
@ -187,11 +189,14 @@ static int32_t mndProcessRetrieveSysTableReq(SNodeMsg *pReq) {
}
if (retrieveReq.showId == 0) {
STableMetaRsp *pMeta = (STableMetaRsp *)taosHashGet(pMnode->infosMeta, retrieveReq.tb, strlen(retrieveReq.tb) + 1);
STableMetaRsp *pMeta = (STableMetaRsp *)taosHashGet(pMnode->infosMeta, retrieveReq.tb, strlen(retrieveReq.tb));
if (pMeta == NULL) {
terrno = TSDB_CODE_MND_INVALID_INFOS_TBL;
mError("failed to process show-retrieve req:%p since %s", pShow, terrstr());
return -1;
pMeta = (STableMetaRsp *)taosHashGet(pMnode->perfsMeta, retrieveReq.tb, strlen(retrieveReq.tb));
if (pMeta == NULL) {
terrno = TSDB_CODE_MND_INVALID_SYS_TABLENAME;
mError("failed to process show-retrieve req:%p since %s", pShow, terrstr());
return -1;
}
}
pShow = mndCreateShowObj(pMnode, &retrieveReq);

179
source/dnode/mnode/impl/src/mndSubscribe.c
View File

@ -35,11 +35,6 @@
#define MND_SUBSCRIBE_REBALANCE_CNT 3
enum {
MQ_SUBSCRIBE_STATUS__ACTIVE = 1,
MQ_SUBSCRIBE_STATUS__DELETED,
};
static SSdbRaw *mndSubActionEncode(SMqSubscribeObj *);
static SSdbRow *mndSubActionDecode(SSdbRaw *pRaw);
static int32_t mndSubActionInsert(SSdb *pSdb, SMqSubscribeObj *);
@ -89,7 +84,6 @@ static SMqSubscribeObj *mndCreateSub(SMnode *pMnode, const SMqTopicObj *pTopic,
pSub->withTbName = pTopic->withTbName;
pSub->withSchema = pTopic->withSchema;
pSub->withTag = pTopic->withTag;
pSub->withTagSchema = pTopic->withTagSchema;
ASSERT(taosHashGetSize(pSub->consumerHash) == 1);
@ -115,7 +109,6 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, const SMqSubscri
req.withTbName = pSub->withTbName;
req.withSchema = pSub->withSchema;
req.withTag = pSub->withTag;
req.withTagSchema = pSub->withTagSchema;
strncpy(req.subKey, pSub->key, TSDB_SUBSCRIBE_KEY_LEN);
int32_t tlen = sizeof(SMsgHead) + tEncodeSMqRebVgReq(NULL, &req);
@ -514,9 +507,11 @@ static int32_t mndProcessRebalanceReq(SNodeMsg *pMsg) {
// TODO replace assert with error check
ASSERT(mndDoRebalance(pMnode, &rebInput, &rebOutput) == 0);
// if add more consumer to balanced subscribe,
// possibly no vg is changed
/*ASSERT(taosArrayGetSize(rebOutput.rebVgs) != 0);*/
ASSERT(mndPersistRebResult(pMnode, pMsg, &rebOutput) == 0);
if (rebInput.pTopic) {
@ -673,177 +668,7 @@ void mndReleaseSubscribe(SMnode *pMnode, SMqSubscribeObj *pSub) {
sdbRelease(pSdb, pSub);
}
#if 0
static int32_t mndProcessSubscribeReq(SNodeMsg *pMsg) {
SMnode *pMnode = pMsg->pNode;
char *msgStr = pMsg->rpcMsg.pCont;
SCMSubscribeReq subscribe;
tDeserializeSCMSubscribeReq(msgStr, &subscribe);
int64_t consumerId = subscribe.consumerId;
char *cgroup = subscribe.consumerGroup;
SArray *newSub = subscribe.topicNames;
int32_t newTopicNum = subscribe.topicNum;
taosArraySortString(newSub, taosArrayCompareString);
SArray *oldSub = NULL;
int32_t oldTopicNum = 0;
bool createConsumer = false;
// create consumer if not exist
SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, consumerId);
if (pConsumer == NULL) {
// create consumer
pConsumer = mndCreateConsumer(consumerId, cgroup);
createConsumer = true;
} else {
pConsumer->epoch++;
oldSub = pConsumer->currentTopics;
}
pConsumer->currentTopics = newSub;
if (oldSub != NULL) {
oldTopicNum = taosArrayGetSize(oldSub);
}
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_TYPE_SUBSCRIBE, &pMsg->rpcMsg);
if (pTrans == NULL) {
// TODO: free memory
return -1;
}
int32_t i = 0, j = 0;
while (i < newTopicNum || j < oldTopicNum) {
char *newTopicName = NULL;
char *oldTopicName = NULL;
if (i >= newTopicNum) {
// encode unset topic msg to all vnodes related to that topic
oldTopicName = taosArrayGetP(oldSub, j);
j++;
} else if (j >= oldTopicNum) {
newTopicName = taosArrayGetP(newSub, i);
i++;
} else {
newTopicName = taosArrayGetP(newSub, i);
oldTopicName = taosArrayGetP(oldSub, j);
int32_t comp = compareLenPrefixedStr(newTopicName, oldTopicName);
if (comp == 0) {
// do nothing
oldTopicName = newTopicName = NULL;
i++;
j++;
continue;
} else if (comp < 0) {
oldTopicName = NULL;
i++;
} else {
newTopicName = NULL;
j++;
}
}
if (oldTopicName != NULL) {
ASSERT(newTopicName == NULL);
// cancel subscribe of old topic
SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, cgroup, oldTopicName);
ASSERT(pSub);
int32_t csz = taosArrayGetSize(pSub->consumers);
for (int32_t ci = 0; ci < csz; ci++) {
SMqSubConsumer *pSubConsumer = taosArrayGet(pSub->consumers, ci);
if (pSubConsumer->consumerId == consumerId) {
int32_t vgsz = taosArrayGetSize(pSubConsumer->vgInfo);
for (int32_t vgi = 0; vgi < vgsz; vgi++) {
SMqConsumerEp *pConsumerEp = taosArrayGet(pSubConsumer->vgInfo, vgi);
mndPersistCancelConnReq(pMnode, pTrans, pConsumerEp, oldTopicName);
taosArrayPush(pSub->unassignedVg, pConsumerEp);
}
taosArrayRemove(pSub->consumers, ci);
break;
}
}
char *oldTopicNameDup = strdup(oldTopicName);
taosArrayPush(pConsumer->recentRemovedTopics, &oldTopicNameDup);
atomic_store_32(&pConsumer->status, MQ_CONSUMER_STATUS__MODIFY);
/*pSub->status = MQ_SUBSCRIBE_STATUS__DELETED;*/
} else if (newTopicName != NULL) {
ASSERT(oldTopicName == NULL);
SMqTopicObj *pTopic = mndAcquireTopic(pMnode, newTopicName);
if (pTopic == NULL) {
mError("topic being subscribed not exist: %s", newTopicName);
continue;
}
SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, cgroup, newTopicName);
bool createSub = false;
if (pSub == NULL) {
mDebug("create new subscription by consumer %" PRId64 ", group: %s, topic %s", consumerId, cgroup,
newTopicName);
pSub = mndCreateSubscription(pMnode, pTopic, cgroup);
createSub = true;
mndCreateOffset(pTrans, cgroup, newTopicName, pSub->unassignedVg);
}
SMqSubConsumer mqSubConsumer;
mqSubConsumer.consumerId = consumerId;
mqSubConsumer.vgInfo = taosArrayInit(0, sizeof(SMqConsumerEp));
taosArrayPush(pSub->consumers, &mqSubConsumer);
// if have un assigned vg, assign one to the consumer
if (taosArrayGetSize(pSub->unassignedVg) > 0) {
SMqConsumerEp *pConsumerEp = taosArrayPop(pSub->unassignedVg);
pConsumerEp->oldConsumerId = pConsumerEp->consumerId;
pConsumerEp->consumerId = consumerId;
taosArrayPush(mqSubConsumer.vgInfo, pConsumerEp);
if (pConsumerEp->oldConsumerId == -1) {
mInfo("mq set conn: assign vgroup %d of topic %s to consumer %" PRId64 "", pConsumerEp->vgId, newTopicName,
pConsumerEp->consumerId);
mndPersistMqSetConnReq(pMnode, pTrans, pTopic, cgroup, pConsumerEp);
} else {
mndPersistRebalanceMsg(pMnode, pTrans, pConsumerEp, newTopicName);
}
// to trigger rebalance at once, do not set status active
/*atomic_store_32(&pConsumer->status, MQ_CONSUMER_STATUS__ACTIVE);*/
}
SSdbRaw *pRaw = mndSubActionEncode(pSub);
sdbSetRawStatus(pRaw, SDB_STATUS_READY);
mndTransAppendRedolog(pTrans, pRaw);
if (!createSub) mndReleaseSubscribe(pMnode, pSub);
mndReleaseTopic(pMnode, pTopic);
}
}
/*if (oldSub) taosArrayDestroyEx(oldSub, (void (*)(void *))taosMemoryFree);*/
// persist consumerObj
SSdbRaw *pConsumerRaw = mndConsumerActionEncode(pConsumer);
sdbSetRawStatus(pConsumerRaw, SDB_STATUS_READY);
mndTransAppendRedolog(pTrans, pConsumerRaw);
if (mndTransPrepare(pMnode, pTrans) != 0) {
mError("mq-subscribe-trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
if (!createConsumer) mndReleaseConsumer(pMnode, pConsumer);
return -1;
}
mndTransDrop(pTrans);
if (!createConsumer) mndReleaseConsumer(pMnode, pConsumer);
return TSDB_CODE_MND_ACTION_IN_PROGRESS;
}
#endif
static int32_t mndProcessSubscribeInternalRsp(SNodeMsg *pRsp) {
mndTransProcessRsp(pRsp);
return 0;
}
static void mndCancelGetNextConsumer(SMnode *pMnode, void *pIter) {
SSdb *pSdb = pMnode->pSdb;
sdbCancelFetch(pSdb, pIter);
}

21
source/dnode/mnode/impl/src/mndTopic.c
View File

@ -82,7 +82,6 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) {
SDB_SET_INT8(pRaw, dataPos, pTopic->withTbName, TOPIC_ENCODE_OVER);
SDB_SET_INT8(pRaw, dataPos, pTopic->withSchema, TOPIC_ENCODE_OVER);
SDB_SET_INT8(pRaw, dataPos, pTopic->withTag, TOPIC_ENCODE_OVER);
SDB_SET_INT8(pRaw, dataPos, pTopic->withTagSchema, TOPIC_ENCODE_OVER);
SDB_SET_INT32(pRaw, dataPos, pTopic->sqlLen, TOPIC_ENCODE_OVER);
SDB_SET_BINARY(pRaw, dataPos, pTopic->sql, pTopic->sqlLen, TOPIC_ENCODE_OVER);
SDB_SET_INT32(pRaw, dataPos, pTopic->astLen, TOPIC_ENCODE_OVER);
@ -146,7 +145,6 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) {
SDB_GET_INT8(pRaw, dataPos, &pTopic->withTbName, TOPIC_DECODE_OVER);
SDB_GET_INT8(pRaw, dataPos, &pTopic->withSchema, TOPIC_DECODE_OVER);
SDB_GET_INT8(pRaw, dataPos, &pTopic->withTag, TOPIC_DECODE_OVER);
SDB_GET_INT8(pRaw, dataPos, &pTopic->withTagSchema, TOPIC_DECODE_OVER);
SDB_GET_INT32(pRaw, dataPos, &pTopic->sqlLen, TOPIC_DECODE_OVER);
pTopic->sql = taosMemoryCalloc(pTopic->sqlLen, sizeof(char));
@ -234,6 +232,7 @@ void mndReleaseTopic(SMnode *pMnode, SMqTopicObj *pTopic) {
sdbRelease(pSdb, pTopic);
}
#if 0
static SDbObj *mndAcquireDbByTopic(SMnode *pMnode, char *topicName) {
SName name = {0};
tNameFromString(&name, topicName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
@ -243,6 +242,7 @@ static SDbObj *mndAcquireDbByTopic(SMnode *pMnode, char *topicName) {
return mndAcquireDb(pMnode, db);
}
#endif
static SDDropTopicReq *mndBuildDropTopicMsg(SMnode *pMnode, SVgObj *pVgroup, SMqTopicObj *pTopic) {
int32_t contLen = sizeof(SDDropTopicReq);
@ -262,15 +262,11 @@ static SDDropTopicReq *mndBuildDropTopicMsg(SMnode *pMnode, SVgObj *pVgroup, SMq
}
static int32_t mndCheckCreateTopicReq(SCMCreateTopicReq *pCreate) {
if (pCreate->name[0] == 0 || pCreate->sql == NULL || pCreate->sql[0] == 0) {
if (pCreate->name[0] == 0 || pCreate->sql == NULL || pCreate->sql[0] == 0 || pCreate->subscribeDbName[0] == 0) {
terrno = TSDB_CODE_MND_INVALID_TOPIC_OPTION;
return -1;
}
if ((pCreate->ast == NULL || pCreate->ast[0] == 0) && pCreate->subscribeDbName[0] == 0) {
terrno = TSDB_CODE_MND_INVALID_TOPIC_OPTION;
return -1;
}
return 0;
}
@ -386,7 +382,7 @@ static int32_t mndProcessCreateTopicReq(SNodeMsg *pReq) {
goto CREATE_TOPIC_OVER;
}
pDb = mndAcquireDbByTopic(pMnode, createTopicReq.name);
pDb = mndAcquireDb(pMnode, createTopicReq.subscribeDbName);
if (pDb == NULL) {
terrno = TSDB_CODE_MND_DB_NOT_SELECTED;
goto CREATE_TOPIC_OVER;
@ -524,8 +520,11 @@ static int32_t mndRetrieveTopic(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pB
int32_t cols = 0;
char topicName[TSDB_TOPIC_NAME_LEN + VARSTR_HEADER_SIZE] = {0};
tstrncpy(&topicName[VARSTR_HEADER_SIZE], pTopic->name, TSDB_TOPIC_NAME_LEN);
varDataSetLen(topicName, strlen(&topicName[VARSTR_HEADER_SIZE]));
SName n;
tNameFromString(&n, pTopic->name, T_NAME_ACCT|T_NAME_DB);
tNameGetDbName(&n, varDataVal(topicName));
varDataSetLen(topicName, strlen(varDataVal(topicName)));
SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++);
colDataAppend(pColInfo, numOfRows, (const char *)topicName, false);
@ -539,7 +538,7 @@ static int32_t mndRetrieveTopic(SNodeMsg *pReq, SShowObj *pShow, SSDataBlock *pB
varDataSetLen(sql, strlen(&sql[VARSTR_HEADER_SIZE]));
colDataAppend(pColInfo, numOfRows, (const char *)sql, false);
taosMemoryFree(sql);
// taosMemoryFree(sql);
numOfRows++;
sdbRelease(pSdb, pTopic);

88
source/dnode/mnode/impl/test/func/func.cpp
View File

@ -22,17 +22,16 @@ class MndTestFunc : public ::testing::Test {
void SetUp() override {}
void TearDown() override {}
void SetCode(SCreateFuncReq* pReq, const char* pCode);
void SetCode(SCreateFuncReq* pReq, const char* pCode, int32_t size);
void SetComment(SCreateFuncReq* pReq, const char* pComment);
};
Testbase MndTestFunc::test;
void MndTestFunc::SetCode(SCreateFuncReq* pReq, const char* pCode) {
int32_t len = strlen(pCode);
pReq->pCode = (char*)taosMemoryCalloc(1, len + 1);
strcpy(pReq->pCode, pCode);
pReq->codeLen = len;
void MndTestFunc::SetCode(SCreateFuncReq *pReq, const char *pCode, int32_t size) {
pReq->pCode = (char*)taosMemoryMalloc(size);
memcpy(pReq->pCode, pCode, size);
pReq->codeLen = size;
}
void MndTestFunc::SetComment(SCreateFuncReq* pReq, const char* pComment) {
@ -79,7 +78,7 @@ TEST_F(MndTestFunc, 02_Create_Func) {
{
SCreateFuncReq createReq = {0};
strcpy(createReq.name, "f1");
SetCode(&createReq, "");
SetCode(&createReq, "", 1);
SetComment(&createReq, "comment1");
int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq);
@ -95,7 +94,7 @@ TEST_F(MndTestFunc, 02_Create_Func) {
{
SCreateFuncReq createReq = {0};
strcpy(createReq.name, "f1");
SetCode(&createReq, "code1");
SetCode(&createReq, "code1", 6);
SetComment(&createReq, "comment1");
int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq);
@ -111,7 +110,7 @@ TEST_F(MndTestFunc, 02_Create_Func) {
{
SCreateFuncReq createReq = {0};
strcpy(createReq.name, "f1");
SetCode(&createReq, "code1");
SetCode(&createReq, "code1", 6);
SetComment(&createReq, "comment1");
createReq.bufSize = TSDB_FUNC_BUF_SIZE + 1;
@ -128,7 +127,7 @@ TEST_F(MndTestFunc, 02_Create_Func) {
for (int32_t i = 0; i < 3; ++i) {
SCreateFuncReq createReq = {0};
strcpy(createReq.name, "f1");
SetCode(&createReq, "code1");
SetCode(&createReq, "code1", 6);
SetComment(&createReq, "comment1");
createReq.bufSize = TSDB_FUNC_BUF_SIZE + 1;
createReq.igExists = 0;
@ -253,7 +252,7 @@ TEST_F(MndTestFunc, 03_Retrieve_Func) {
createReq.outputLen = 24;
createReq.bufSize = 6;
createReq.signature = 18;
SetCode(&createReq, "code2");
SetCode(&createReq, "code2", 6);
SetComment(&createReq, "comment2");
int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq);
@ -439,3 +438,70 @@ TEST_F(MndTestFunc, 04_Drop_Func) {
test.SendShowReq(TSDB_MGMT_TABLE_FUNC, "user_functions", "");
EXPECT_EQ(test.GetShowRows(), 1);
}
TEST_F(MndTestFunc, 05_Actual_code) {
{
SCreateFuncReq createReq = {0};
strcpy(createReq.name, "udf1");
char code[300] = {0};
for (int32_t i = 0; i < sizeof(code); ++i) {
code[i] = (i) % 20;
}
SetCode(&createReq, code, 300);
SetComment(&createReq, "comment1");
createReq.bufSize = 8;
createReq.igExists = 0;
createReq.funcType = 1;
createReq.scriptType = 2;
createReq.outputType = TSDB_DATA_TYPE_SMALLINT;
createReq.outputLen = 12;
createReq.bufSize = 4;
createReq.signature = 5;
int32_t contLen = tSerializeSCreateFuncReq(NULL, 0, &createReq);
void* pReq = rpcMallocCont(contLen);
tSerializeSCreateFuncReq(pReq, contLen, &createReq);
tFreeSCreateFuncReq(&createReq);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_FUNC, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0);
}
{
SRetrieveFuncReq retrieveReq = {0};
retrieveReq.numOfFuncs = 1;
retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN);
taosArrayPush(retrieveReq.pFuncNames, "udf1");
int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq);
void* pReq = rpcMallocCont(contLen);
tSerializeSRetrieveFuncReq(pReq, contLen, &retrieveReq);
tFreeSRetrieveFuncReq(&retrieveReq);
SRpcMsg* pRsp = test.SendReq(TDMT_MND_RETRIEVE_FUNC, pReq, contLen);
ASSERT_NE(pRsp, nullptr);
ASSERT_EQ(pRsp->code, 0);
SRetrieveFuncRsp retrieveRsp = {0};
tDeserializeSRetrieveFuncRsp(pRsp->pCont, pRsp->contLen, &retrieveRsp);
EXPECT_EQ(retrieveRsp.numOfFuncs, 1);
EXPECT_EQ(retrieveRsp.numOfFuncs, (int32_t)taosArrayGetSize(retrieveRsp.pFuncInfos));
SFuncInfo* pFuncInfo = (SFuncInfo*)taosArrayGet(retrieveRsp.pFuncInfos, 0);
EXPECT_STREQ(pFuncInfo->name, "udf1");
EXPECT_EQ(pFuncInfo->funcType, 1);
EXPECT_EQ(pFuncInfo->scriptType, 2);
EXPECT_EQ(pFuncInfo->outputType, TSDB_DATA_TYPE_SMALLINT);
EXPECT_EQ(pFuncInfo->outputLen, 12);
EXPECT_EQ(pFuncInfo->bufSize, 4);
EXPECT_EQ(pFuncInfo->signature, 5);
EXPECT_STREQ("comment1", pFuncInfo->pComment);
for (int32_t i = 0; i < 300; ++i) {
EXPECT_EQ(pFuncInfo->pCode[i], (i) % 20);
}
tFreeSRetrieveFuncRsp(&retrieveRsp);
}
}

7
source/dnode/qnode/src/qnode.c
View File

@ -17,6 +17,7 @@
#include "qndInt.h"
#include "query.h"
#include "qworker.h"
//#include "tudf.h"
SQnode *qndOpen(const SQnodeOpt *pOption) {
SQnode *pQnode = taosMemoryCalloc(1, sizeof(SQnode));
@ -25,6 +26,8 @@ SQnode *qndOpen(const SQnodeOpt *pOption) {
return NULL;
}
//udfcOpen();
if (qWorkerInit(NODE_TYPE_QNODE, pQnode->qndId, NULL, (void **)&pQnode->pQuery, &pOption->msgCb)) {
taosMemoryFreeClear(pQnode);
return NULL;
@ -37,13 +40,15 @@ SQnode *qndOpen(const SQnodeOpt *pOption) {
void qndClose(SQnode *pQnode) {
qWorkerDestroy((void **)&pQnode->pQuery);
//udfcClose();
taosMemoryFree(pQnode);
}
int32_t qndGetLoad(SQnode *pQnode, SQnodeLoad *pLoad) { return 0; }
int32_t qndProcessQueryMsg(SQnode *pQnode, SRpcMsg *pMsg) {
qTrace("message in query queue is processing");
qTrace("message in qnode query queue is processing");
SReadHandle handle = {0};
switch (pMsg->msgType) {

1
source/dnode/vnode/src/inc/tq.h
View File

@ -159,7 +159,6 @@ typedef struct {
int8_t withTbName;
int8_t withSchema;
int8_t withTag;
int8_t withTagSchema;
char* qmsg;
STqPushHandle pushHandle;
// SRWLatch lock;

36
source/dnode/vnode/src/tq/tq.c
View File

@ -31,6 +31,7 @@ STQ* tqOpen(const char* path, SVnode* pVnode, SWal* pWal) {
pTq->path = strdup(path);
pTq->pVnode = pVnode;
pTq->pWal = pWal;
#if 0
pTq->tqMeta = tqStoreOpen(pTq, path, (FTqSerialize)tqSerializeConsumer, (FTqDeserialize)tqDeserializeConsumer,
(FTqDelete)taosMemoryFree, 0);
@ -401,6 +402,13 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
consumerEpoch = atomic_val_compare_exchange_32(&pExec->epoch, consumerEpoch, reqEpoch);
}
SWalHead* pHeadWithCkSum = taosMemoryMalloc(sizeof(SWalHead) + 2048);
if (pHeadWithCkSum == NULL) {
return -1;
}
walSetReaderCapacity(pExec->pWalReader, 2048);
SMqDataBlkRsp rsp = {0};
rsp.reqOffset = pReq->currentOffset;
rsp.blockData = taosArrayInit(0, sizeof(void*));
@ -414,6 +422,26 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
break;
}
taosThreadMutexLock(&pExec->pWalReader->mutex);
if (walFetchHead(pExec->pWalReader, fetchOffset, pHeadWithCkSum) < 0) {
vDebug("tmq poll: consumer %ld (epoch %d) vg %d offset %ld, no more log to return", consumerId, pReq->epoch,
TD_VID(pTq->pVnode), fetchOffset);
taosThreadMutexUnlock(&pExec->pWalReader->mutex);
break;
}
if (pHeadWithCkSum->head.msgType != TDMT_VND_SUBMIT) {
walSkipFetchBody(pExec->pWalReader, pHeadWithCkSum);
} else {
walFetchBody(pExec->pWalReader, &pHeadWithCkSum);
}
SWalReadHead* pHead = &pHeadWithCkSum->head;
taosThreadMutexUnlock(&pExec->pWalReader->mutex);
#if 0
SWalReadHead* pHead;
if (walReadWithHandle_s(pExec->pWalReader, fetchOffset, &pHead) < 0) {
// TODO: no more log, set timer to wait blocking time
@ -443,14 +471,16 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
return 0;
#endif
break;
}
break;
}
#endif
vDebug("tmq poll: consumer %ld (epoch %d) iter log, vg %d offset %ld msgType %d", consumerId, pReq->epoch,
TD_VID(pTq->pVnode), fetchOffset, pHead->msgType);
if (pHead->msgType == TDMT_VND_SUBMIT) {
SSubmitReq* pCont = (SSubmitReq*)&pHead->body;
// table subscribe
if (pExec->subType == TOPIC_SUB_TYPE__TABLE) {
qTaskInfo_t task = pExec->task[workerId];
ASSERT(task);
@ -484,6 +514,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) {
taosArrayPush(rsp.blockData, &buf);
rsp.blockNum++;
}
// db subscribe
} else if (pExec->subType == TOPIC_SUB_TYPE__DB) {
STqReadHandle* pReader = pExec->pExecReader[workerId];
tqReadHandleSetMsg(pReader, pCont, 0);
@ -789,7 +820,6 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) {
pExec->withTbName = req.withTbName;
pExec->withSchema = req.withSchema;
pExec->withTag = req.withTag;
pExec->withTagSchema = req.withTagSchema;
pExec->qmsg = req.qmsg;
req.qmsg = NULL;

9
source/dnode/vnode/src/tsdb/tsdbRead.c
View File

@ -3223,8 +3223,13 @@ void tsdbRetrieveDataBlockInfo(tsdbReaderT* pTsdbReadHandle, SDataBlockInfo* pDa
tsdbDebug("data block generated, uid:%" PRIu64 " numOfRows:%d, tsrange:%" PRId64 " - %" PRId64 " %s", uid, cur->rows,
cur->win.skey, cur->win.ekey, pHandle->idStr);
// pDataBlockInfo->uid = uid; // block Id may be over write by assigning uid fro this data block. Do NOT assign
// the table uid
pDataBlockInfo->uid = uid;
#if 0
// for multi-group data query processing test purpose
pDataBlockInfo->groupId = uid;
#endif
pDataBlockInfo->rows = cur->rows;
pDataBlockInfo->window = cur->win;
pDataBlockInfo->numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pHandle));

2
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -135,7 +135,7 @@ int vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRpcMsg
}
int vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg) {
vTrace("message in query queue is processing");
vTrace("message in vnode query queue is processing");
SReadHandle handle = {.reader = pVnode->pTsdb, .meta = pVnode->pMeta, .config = &pVnode->config};
switch (pMsg->msgType) {

9
source/libs/catalog/src/catalog.c
View File

@ -494,7 +494,7 @@ _return:
return TSDB_CODE_SUCCESS;
}
int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, SArray **out) {
int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps, SArray *out) {
char *msg = NULL;
int32_t msgLen = 0;
@ -526,7 +526,7 @@ int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmt
CTG_ERR_RET(code);
}
ctgDebug("Got qnode list from mnode, listNum:%d", (int32_t)taosArrayGetSize(*out));
ctgDebug("Got qnode list from mnode, listNum:%d", (int32_t)taosArrayGetSize(out));
return TSDB_CODE_SUCCESS;
}
@ -2778,7 +2778,8 @@ int32_t catalogGetAllMeta(SCatalog* pCtg, void *pTrans, const SEpSet* pMgmtEps,
}
if (pReq->qNodeRequired) {
CTG_ERR_JRET(ctgGetQnodeListFromMnode(pCtg, pTrans, pMgmtEps, &pRsp->pEpSetList));
pRsp->pQnodeList = taosArrayInit(10, sizeof(SQueryNodeAddr));
CTG_ERR_JRET(ctgGetQnodeListFromMnode(pCtg, pTrans, pMgmtEps, pRsp->pQnodeList));
}
CTG_API_LEAVE(TSDB_CODE_SUCCESS);
@ -2807,7 +2808,7 @@ int32_t catalogGetQnodeList(SCatalog* pCtg, void *pRpc, const SEpSet* pMgmtEps,
CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
}
CTG_ERR_JRET(ctgGetQnodeListFromMnode(pCtg, pRpc, pMgmtEps, &pQnodeList));
CTG_ERR_JRET(ctgGetQnodeListFromMnode(pCtg, pRpc, pMgmtEps, pQnodeList));
_return:

42
source/libs/command/src/explain.c
View File

@ -304,8 +304,8 @@ int32_t qExplainResAppendRow(SExplainCtx *ctx, char *tbuf, int32_t len, int32_t
memcpy(row.buf, tbuf, len);
row.level = level;
row.len = len;
ctx->dataSize += len;
row.len = len;
ctx->dataSize += row.len;
if (NULL == taosArrayPush(ctx->rows, &row)) {
qError("taosArrayPush row to explain res rows failed");
@ -756,7 +756,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) {
}
int32_t colNum = 1;
int32_t rspSize = sizeof(SRetrieveTableRsp) + sizeof(int32_t) * colNum + sizeof(int32_t) * rowNum + pCtx->dataSize;
int32_t rspSize = sizeof(SRetrieveTableRsp) + sizeof(int32_t) + sizeof(uint64_t) + sizeof(int32_t) * colNum + sizeof(int32_t) * rowNum + pCtx->dataSize;
SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, rspSize);
if (NULL == rsp) {
qError("malloc SRetrieveTableRsp failed, size:%d", rspSize);
@ -766,29 +766,38 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) {
rsp->completed = 1;
rsp->numOfRows = htonl(rowNum);
*(int32_t *)rsp->data = htonl(pCtx->dataSize);
// payload length
*(int32_t *)rsp->data = sizeof(int32_t) + sizeof(uint64_t) + sizeof(int32_t) * colNum + sizeof(int32_t) * rowNum + pCtx->dataSize;
int32_t *offset = (int32_t *)((char *)rsp->data + sizeof(int32_t));
// group id
*(uint64_t*)(rsp->data + sizeof(int32_t)) = 0;
// column length
int32_t* colLength = (int32_t *)(rsp->data + sizeof(int32_t) + sizeof(uint64_t));
// varchar column offset segment
int32_t *offset = (int32_t *)((char *)colLength + sizeof(int32_t));
// varchar data real payload
char *data = (char *)(offset + rowNum);
int32_t tOffset = 0;
char* start = data;
for (int32_t i = 0; i < rowNum; ++i) {
SQueryExplainRowInfo *row = taosArrayGet(pCtx->rows, i);
*offset = tOffset;
tOffset += row->len;
offset[i] = data - start;
memcpy(data, row->buf, row->len);
++offset;
varDataCopy(data, row->buf);
ASSERT(varDataTLen(row->buf) == row->len);
data += row->len;
}
*pRsp = rsp;
*colLength = htonl(data - start);
rsp->compLen = htonl(rspSize);
*pRsp = rsp;
return TSDB_CODE_SUCCESS;
}
int32_t qExplainPrepareCtx(SQueryPlan *pDag, SExplainCtx **pCtx) {
int32_t code = 0;
SNodeListNode *plans = NULL;
@ -895,9 +904,7 @@ int32_t qExplainAppendPlanRows(SExplainCtx *pCtx) {
int32_t qExplainGenerateRsp(SExplainCtx *pCtx, SRetrieveTableRsp **pRsp) {
QRY_ERR_RET(qExplainAppendGroupResRows(pCtx, pCtx->rootGroupId, 0));
QRY_ERR_RET(qExplainAppendPlanRows(pCtx));
QRY_ERR_RET(qExplainGetRspFromCtx(pCtx, pRsp));
return TSDB_CODE_SUCCESS;
@ -967,13 +974,10 @@ int32_t qExecStaticExplain(SQueryPlan *pDag, SRetrieveTableRsp **pRsp) {
SExplainCtx *pCtx = NULL;
QRY_ERR_RET(qExplainPrepareCtx(pDag, &pCtx));
QRY_ERR_JRET(qExplainGenerateRsp(pCtx, pRsp));
_return:
qExplainFreeCtx(pCtx);
QRY_RET(code);
}

11
source/libs/executor/inc/executil.h
View File

@ -40,8 +40,6 @@
#define GET_TASKID(_t) (((SExecTaskInfo*)(_t))->id.str)
#define curTimeWindowIndex(_winres) ((_winres)->curIndex)
typedef struct SGroupResInfo {
int32_t totalGroup;
int32_t currentGroup;
@ -68,11 +66,16 @@ typedef struct SResultRowPosition {
int32_t offset;
} SResultRowPosition;
typedef struct SResKeyPos {
SResultRowPosition pos;
uint64_t groupId;
char key[];
} SResKeyPos;
typedef struct SResultRowInfo {
SResultRowPosition *pPosition;
int32_t size; // number of result set
int32_t capacity; // max capacity
// int32_t curPos; // current active result row index of pResult list
SResultRowPosition cur;
} SResultRowInfo;
@ -135,7 +138,7 @@ typedef struct {
int32_t colId;
} SStddevInterResult;
void initGroupResInfo(SGroupResInfo* pGroupResInfo, SResultRowInfo* pResultInfo);
void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, bool sortGroupResult);
void initMultiResInfoFromArrayList(SGroupResInfo* pGroupResInfo, SArray* pArrayList);
void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo);

21
source/libs/executor/inc/executorimpl.h
View File

@ -347,10 +347,11 @@ typedef struct STableScanInfo {
} STableScanInfo;
typedef struct STagScanInfo {
SColumnInfo* pCols;
SSDataBlock* pRes;
SColumnInfo *pCols;
SSDataBlock *pRes;
int32_t totalTables;
int32_t curPos;
void *pReader;
} STagScanInfo;
typedef struct SStreamBlockScanInfo {
@ -376,13 +377,11 @@ typedef struct SSysTableScanInfo {
SEpSet epSet;
tsem_t ready;
int32_t accountId;
bool showRewrite;
SNode* pCondition; // db_name filter condition, to discard data that are not in current database
void* pCur; // cursor for iterate the local table meta store.
SArray* scanCols; // SArray<int16_t> scan column id list
// int32_t type; // show type, TODO remove it
int32_t accountId;
bool showRewrite;
SNode* pCondition; // db_name filter condition, to discard data that are not in current database
void* pCur; // cursor for iterate the local table meta store.
SArray* scanCols; // SArray<int16_t> scan column id list
SName name;
SSDataBlock* pRes;
int32_t capacity;
@ -628,7 +627,7 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput,
SOperatorInfo* createExchangeOperatorInfo(const SNodeList* pSources, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfCols, int32_t dataLoadFlag, int32_t repeatTime,
SOperatorInfo* createTableScanOperatorInfo(void* pReaderHandle, int32_t order, int32_t numOfCols, int32_t dataLoadFlag, int32_t repeatTime,
int32_t reverseTime, SArray* pColMatchInfo, SSDataBlock* pResBlock, SNode* pCondition,
SInterval* pInterval, double ratio, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SExprInfo* pScalarExprInfo,
@ -668,12 +667,12 @@ SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SExprInfo*
SSDataBlock* pResultBlock, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SNode* pOnCondition, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createTagScanOperatorInfo(void* pReaderHandle, SExprInfo* pExpr, int32_t numOfOutput, SExecTaskInfo* pTaskInfo);
#if 0
SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv);
SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream,
SExprInfo* pExpr, int32_t numOfOutput);
SOperatorInfo* createTagScanOperatorInfo(SReaderHandle* pReaderHandle, SExprInfo* pExpr, int32_t numOfOutput);
#endif
void projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* pSrcBlock, SqlFunctionCtx* pCtx,

8
source/libs/executor/src/dataDispatcher.c
View File

@ -64,10 +64,10 @@ static bool needCompress(const SSDataBlock* pData, int32_t numOfCols) {
}
// data format:
// +----------------+--------------------------------------+-------------+-----------+-------------+-----------+
// |SDataCacheEntry | column#1 length, column#2 length ... | col1 bitmap | col1 data | col2 bitmap | col2 data | ....
// | | sizeof(int32_t) * numOfCols | actual size | | actual size | |
// +----------------+--------------------------------------+-------------+-----------+-------------+-----------+
// +----------------+--------------+----------+--------------------------------------+-------------+-----------+-------------+-----------+
// |SDataCacheEntry | total length | group id | column#1 length, column#2 length ... | col1 bitmap | col1 data | col2 bitmap | col2 data | ....
// | | (4 bytes) |(8 bytes) | sizeof(int32_t) * numOfCols | actual size | | actual size | |
// +----------------+--------------+----------+--------------------------------------+-------------+-----------+-------------+-----------+
// The length of bitmap is decided by number of rows of this data block, and the length of each column data is
// recorded in the first segment, next to the struct header
static void toDataCacheEntry(const SDataDispatchHandle* pHandle, const SInputData* pInput, SDataDispatchBuf* pBuf) {

42
source/libs/executor/src/executil.c
View File

@ -186,12 +186,50 @@ void cleanupGroupResInfo(SGroupResInfo* pGroupResInfo) {
pGroupResInfo->index = 0;
}
void initGroupResInfo(SGroupResInfo* pGroupResInfo, SResultRowInfo* pResultInfo) {
static int32_t resultrowCompar1(const void* p1, const void* p2) {
SResKeyPos* pp1 = *(SResKeyPos**) p1;
SResKeyPos* pp2 = *(SResKeyPos**) p2;
if (pp1->groupId == pp2->groupId) {
int64_t pts1 = *(int64_t*) pp1->key;
int64_t pts2 = *(int64_t*) pp2->key;
if (pts1 == pts2) {
return 0;
} else {
return pts1 < pts2? -1:1;
}
} else {
return pp1->groupId < pp2->groupId? -1:1;
}
}
void initGroupedResultInfo(SGroupResInfo* pGroupResInfo, SHashObj* pHashmap, bool sortGroupResult) {
if (pGroupResInfo->pRows != NULL) {
taosArrayDestroy(pGroupResInfo->pRows);
}
pGroupResInfo->pRows = taosArrayFromList(pResultInfo->pPosition, pResultInfo->size, sizeof(SResultRowPosition));
// extract the result rows information from the hash map
void* pData = NULL;
pGroupResInfo->pRows = taosArrayInit(10, POINTER_BYTES);
size_t keyLen = 0;
while((pData = taosHashIterate(pHashmap, pData)) != NULL) {
void* key = taosHashGetKey(pData, &keyLen);
SResKeyPos* p = taosMemoryMalloc(keyLen + sizeof(SResultRowPosition));
p->groupId = *(uint64_t*) key;
p->pos = *(SResultRowPosition*) pData;
memcpy(p->key, key + sizeof(uint64_t), keyLen - sizeof(uint64_t));
taosArrayPush(pGroupResInfo->pRows, &p);
}
if (sortGroupResult) {
qsort(pGroupResInfo->pRows->pData, taosArrayGetSize(pGroupResInfo->pRows), POINTER_BYTES, resultrowCompar1);
}
pGroupResInfo->index = 0;
assert(pGroupResInfo->index <= getNumOfTotalRes(pGroupResInfo));
}

394
source/libs/executor/src/executorimpl.c
View File

@ -235,7 +235,6 @@ static int32_t doCopyToSDataBlock(SSDataBlock* pBlock, SExprInfo* pExprInfo, SDi
static void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size);
static void setResultBufSize(STaskAttr* pQueryAttr, SResultInfo* pResultInfo);
static void setCtxTagForJoin(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable);
static void doSetTableGroupOutputBuf(SAggOperatorInfo* pAggInfo, int32_t numOfOutput, uint64_t groupId,
SExecTaskInfo* pTaskInfo);
@ -298,26 +297,6 @@ SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode) {
return pBlock;
}
static bool isSelectivityWithTagsQuery(SqlFunctionCtx* pCtx, int32_t numOfOutput) {
return true;
// bool hasTags = false;
// int32_t numOfSelectivity = 0;
//
// for (int32_t i = 0; i < numOfOutput; ++i) {
// int32_t functId = pCtx[i].functionId;
// if (functId == FUNCTION_TAG_DUMMY || functId == FUNCTION_TS_DUMMY) {
// hasTags = true;
// continue;
// }
//
// if ((aAggs[functId].status & FUNCSTATE_SELECTIVITY) != 0) {
// numOfSelectivity++;
// }
// }
//
// return (numOfSelectivity > 0 && hasTags);
}
static bool hasNull(SColumn* pColumn, SColumnDataAgg* pStatis) {
if (TSDB_COL_IS_TAG(pColumn->flag) || TSDB_COL_IS_UD_COL(pColumn->flag) ||
pColumn->colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
@ -433,6 +412,13 @@ SResultRow* getNewResultRow_rv(SDiskbasedBuf* pResultBuf, int64_t tableGroupId,
return pResultRow;
}
/**
* the struct of key in hash table
* +----------+---------------+
* | group id | key data |
* | 8 bytes | actual length |
* +----------+---------------+
*/
static SResultRow* doSetResultOutBufByKey_rv(SDiskbasedBuf* pResultBuf, SResultRowInfo* pResultRowInfo, int64_t uid,
char* pData, int16_t bytes, bool masterscan, uint64_t groupId,
SExecTaskInfo* pTaskInfo, bool isIntervalQuery, SAggSupporter* pSup) {
@ -1098,8 +1084,9 @@ static int32_t doSetInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCt
int32_t code = TSDB_CODE_SUCCESS;
for (int32_t i = 0; i < pOperator->numOfOutput; ++i) {
pCtx[i].order = order;
pCtx[i].size = pBlock->info.rows;
pCtx[i].order = order;
pCtx[i].size = pBlock->info.rows;
pCtx[i].pSrcBlock = pBlock;
pCtx[i].currentStage = MAIN_SCAN;
SInputColumnInfoData* pInput = &pCtx[i].input;
@ -1474,7 +1461,7 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe
SArray* pUpdated = NULL;
if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) {
pUpdated = taosArrayInit(4, sizeof(SResultRowPosition));
pUpdated = taosArrayInit(4, POINTER_BYTES);
}
int32_t step = 1;
@ -1486,8 +1473,6 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe
if (pSDataBlock->pDataBlock != NULL) {
SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex);
tsCols = (int64_t*)pColDataInfo->pData;
// assert(tsCols[0] == pSDataBlock->info.window.skey && tsCols[pSDataBlock->info.rows - 1] ==
// pSDataBlock->info.window.ekey);
}
int32_t startPos = ascScan ? 0 : (pSDataBlock->info.rows - 1);
@ -1506,7 +1491,11 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe
}
if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) {
SResultRowPosition pos = {.pageId = pResult->pageId, .offset = pResult->offset};
SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t));
pos->groupId = tableGroupId;
pos->pos = (SResultRowPosition) {.pageId = pResult->pageId, .offset = pResult->offset};
*(int64_t*) pos->key = pResult->win.skey;
taosArrayPush(pUpdated, &pos);
}
@ -1580,7 +1569,11 @@ static SArray* hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pRe
}
if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM) {
SResultRowPosition pos = {.pageId = pResult->pageId, .offset = pResult->offset};
SResKeyPos* pos = taosMemoryMalloc(sizeof(SResKeyPos) + sizeof(uint64_t));
pos->groupId = tableGroupId;
pos->pos = (SResultRowPosition) {.pageId = pResult->pageId, .offset = pResult->offset};
*(int64_t*) pos->key = pResult->win.skey;
taosArrayPush(pUpdated, &pos);
}
@ -1844,10 +1837,6 @@ void setBlockStatisInfo(SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, SSDataBlock*
// set the output buffer for the selectivity + tag query
static int32_t setCtxTagColumnInfo(SqlFunctionCtx* pCtx, int32_t numOfOutput) {
if (!isSelectivityWithTagsQuery(pCtx, numOfOutput)) {
return TSDB_CODE_SUCCESS;
}
int32_t num = 0;
int16_t tagLen = 0;
@ -1874,9 +1863,8 @@ static int32_t setCtxTagColumnInfo(SqlFunctionCtx* pCtx, int32_t numOfOutput) {
}
}
if (p != NULL) {
p->subsidiaryRes.pCtx = pTagCtx;
p->subsidiaryRes.numOfCols = num;
p->subsidiaryRes.bufLen = tagLen;
p->subsidiaries.pCtx = pTagCtx;
p->subsidiaries.num = num;
} else {
taosMemoryFreeClear(pTagCtx);
}
@ -1903,6 +1891,9 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput,
SqlFunctionCtx* pCtx = &pFuncCtx[i];
pCtx->functionId = -1;
pCtx->curBufPage = -1;
pCtx->pExpr = pExpr;
if (pExpr->pExpr->nodeType == QUERY_NODE_FUNCTION) {
SFuncExecEnv env = {0};
pCtx->functionId = pExpr->pExpr->_function.pFunctNode->funcId;
@ -1930,9 +1921,9 @@ SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput,
pCtx->pTsOutput = NULL;
pCtx->resDataInfo.bytes = pFunct->resSchema.bytes;
pCtx->resDataInfo.type = pFunct->resSchema.type;
pCtx->order = TSDB_ORDER_ASC;
pCtx->order = TSDB_ORDER_ASC;
pCtx->start.key = INT64_MIN;
pCtx->end.key = INT64_MIN;
pCtx->end.key = INT64_MIN;
pCtx->numOfParams = pExpr->base.numOfParams;
pCtx->param = pFunct->pParam;
@ -1992,7 +1983,7 @@ static void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput) {
}
taosVariantDestroy(&pCtx[i].tag);
taosMemoryFreeClear(pCtx[i].subsidiaryRes.pCtx);
taosMemoryFreeClear(pCtx[i].subsidiaries.pCtx);
}
taosMemoryFreeClear(pCtx);
@ -2095,25 +2086,6 @@ static int32_t updateBlockLoadStatus(STaskAttr* pQuery, int32_t status) {
return status;
}
static void doUpdateLastKey(STaskAttr* pQueryAttr) {
STimeWindow* win = &pQueryAttr->window;
size_t num = taosArrayGetSize(pQueryAttr->tableGroupInfo.pGroupList);
for (int32_t i = 0; i < num; ++i) {
SArray* p1 = taosArrayGetP(pQueryAttr->tableGroupInfo.pGroupList, i);
size_t len = taosArrayGetSize(p1);
for (int32_t j = 0; j < len; ++j) {
// STableKeyInfo* pInfo = taosArrayGet(p1, j);
//
// // update the new lastkey if it is equalled to the value of the old skey
// if (pInfo->lastKey == win->ekey) {
// pInfo->lastKey = win->skey;
// }
}
}
}
// static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableReq* pQueryMsg, bool stableQuery) {
// STaskAttr* pQueryAttr = pQInfo->runtimeEnv.pQueryAttr;
//
@ -2848,6 +2820,7 @@ void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status) {
}
}
// todo merged with the build group result.
void finalizeMultiTupleQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SDiskbasedBuf* pBuf,
SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset) {
for (int32_t i = 0; i < pResultRowInfo->size; ++i) {
@ -2882,40 +2855,36 @@ void finalizeMultiTupleQueryResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SD
}
}
// todo merged with the build group result.
void finalizeUpdatedResult(SqlFunctionCtx* pCtx, int32_t numOfOutput, SDiskbasedBuf* pBuf, SArray* pUpdateList,
int32_t* rowCellInfoOffset) {
size_t num = taosArrayGetSize(pUpdateList);
for (int32_t i = 0; i < num; ++i) {
SResultRowPosition* pPos = taosArrayGet(pUpdateList, i);
SFilePage* bufPage = getBufPage(pBuf, pPos->pageId);
SResultRow* pRow = (SResultRow*)((char*)bufPage + pPos->offset);
SResKeyPos * pPos = taosArrayGetP(pUpdateList, i);
SFilePage* bufPage = getBufPage(pBuf, pPos->pos.pageId);
SResultRow* pRow = (SResultRow*)((char*)bufPage + pPos->pos.offset);
//
for (int32_t j = 0; j < numOfOutput; ++j) {
pCtx[j].resultInfo = getResultCell(pRow, j, rowCellInfoOffset);
//
struct SResultRowEntryInfo* pResInfo = pCtx[j].resultInfo;
if (isRowEntryCompleted(pResInfo) && isRowEntryInitialized(pResInfo)) {
continue;
}
if (pCtx[j].fpSet.process) { // TODO set the dummy function.
// pCtx[j].fpSet.finalize(&pCtx[j]);
pResInfo->initialized = true;
}
// if (isRowEntryCompleted(pResInfo) && isRowEntryInitialized(pResInfo)) {
// continue;
// }
//
// if (pCtx[j].fpSet.process) { // TODO set the dummy function.
//// pCtx[j].fpSet.finalize(&pCtx[j]);
// pResInfo->initialized = true;
// }
//
if (pRow->numOfRows < pResInfo->numOfRes) {
pRow->numOfRows = pResInfo->numOfRes;
}
}
releaseBufPage(pBuf, bufPage);
/*
* set the number of output results for group by normal columns, the number of output rows usually is 1 except
* the top and bottom query
*/
// buf->numOfRows = (uint16_t)getNumOfResult(pCtx, numOfOutput);
}
}
@ -3062,33 +3031,6 @@ void setExecutionContext(int32_t numOfOutput, uint64_t groupId, SExecTaskInfo* p
pAggInfo->groupId = groupId;
}
void setCtxTagForJoin(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable) {
STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr;
SExprBasicInfo* pExpr = &pExprInfo->base;
// if (pQueryAttr->stableQuery && (pRuntimeEnv->pTsBuf != NULL) &&
// (pExpr->functionId == FUNCTION_TS || pExpr->functionId == FUNCTION_PRJ) &&
// (pExpr->colInfo.colIndex == PRIMARYKEY_TIMESTAMP_COL_ID)) {
// assert(pExpr->numOfParams == 1);
//
// int16_t tagColId = (int16_t)pExprInfo->base.param[0].i;
// SColumnInfo* pColInfo = doGetTagColumnInfoById(pQueryAttr->tagColList, pQueryAttr->numOfTags, tagColId);
//
// doSetTagValueInParam(pTable, tagColId, &pCtx->tag, pColInfo->type, pColInfo->bytes);
//
// int16_t tagType = pCtx[0].tag.nType;
// if (tagType == TSDB_DATA_TYPE_BINARY || tagType == TSDB_DATA_TYPE_NCHAR) {
// //qDebug("QInfo:0x%"PRIx64" set tag value for join comparison, colId:%" PRId64 ", val:%s",
// GET_TASKID(pRuntimeEnv),
//// pExprInfo->base.param[0].i, pCtx[0].tag.pz);
// } else {
// //qDebug("QInfo:0x%"PRIx64" set tag value for join comparison, colId:%" PRId64 ", val:%" PRId64,
// GET_TASKID(pRuntimeEnv),
//// pExprInfo->base.param[0].i, pCtx[0].tag.i);
// }
// }
}
/*
* There are two cases to handle:
*
@ -3131,7 +3073,6 @@ void setIntervalQueryRange(STableQueryInfo* pTableQueryInfo, TSKEY key, STimeWin
}
/**
* copyToOutputBuf support copy data in ascending/descending order
* For interval query of both super table and table, copy the data in ascending order, since the output results are
* ordered in SWindowResutl already. While handling the group by query for both table and super table,
* all group result are completed already.
@ -3159,10 +3100,10 @@ int32_t doCopyToSDataBlock(SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbased
}
for (int32_t i = start; (i < numOfRows) && (i >= 0); i += step) {
SResultRowPosition* pPos = taosArrayGet(pGroupResInfo->pRows, i);
SFilePage* page = getBufPage(pBuf, pPos->pageId);
SResKeyPos *pPos = taosArrayGetP(pGroupResInfo->pRows, i);
SFilePage* page = getBufPage(pBuf, pPos->pos.pageId);
SResultRow* pRow = (SResultRow*)((char*)page + pPos->offset);
SResultRow* pRow = (SResultRow*)((char*)page + pPos->pos.offset);
if (pRow->numOfRows == 0) {
pGroupResInfo->index += 1;
continue;
@ -3181,7 +3122,7 @@ int32_t doCopyToSDataBlock(SSDataBlock* pBlock, SExprInfo* pExprInfo, SDiskbased
pCtx[j].resultInfo = getResultCell(pRow, j, rowCellOffset);
if (pCtx[j].fpSet.process) {
pCtx[j].fpSet.finalize(&pCtx[j], pBlock, slotId);
pCtx[j].fpSet.finalize(&pCtx[j], pBlock);
} else {
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId);
@ -3806,9 +3747,15 @@ int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadI
blockDataEnsureCapacity(pRes, numOfRows);
if (pColList == NULL) { // data from other sources
int32_t* colLen = (int32_t*)pData;
char* pStart = pData + sizeof(int32_t) * numOfOutput;
int32_t dataLen = *(int32_t*) pData;
pData += sizeof(int32_t);
pRes->info.groupId = *(uint64_t*) pData;
pData += sizeof(uint64_t);
int32_t* colLen = (int32_t*)pData;
char* pStart = pData + sizeof(int32_t) * numOfOutput;
for (int32_t i = 0; i < numOfOutput; ++i) {
colLen[i] = htonl(colLen[i]);
ASSERT(colLen[i] > 0);
@ -3861,7 +3808,11 @@ int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadI
blockDataEnsureCapacity(&block, numOfRows);
int32_t* colLen = (int32_t*)pStart;
int32_t dataLen = *(int32_t*) pStart;
uint64_t groupId = *(uint64_t*) (pStart + sizeof(int32_t));
pStart += sizeof(int32_t) + sizeof(uint64_t);
int32_t* colLen = (int32_t*) (pStart);
pStart += sizeof(int32_t) * numOfCols;
for (int32_t i = 0; i < numOfCols; ++i) {
@ -3903,6 +3854,7 @@ int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLoadI
}
pRes->info.rows = numOfRows;
blockDataUpdateTsWindow(pRes);
int64_t el = taosGetTimestampUs() - startTs;
@ -4318,18 +4270,6 @@ static void assignExprInfo(SExprInfo* dst, const SExprInfo* src) {
// }
}
static SExprInfo* exprArrayDup(SArray* pExprList) {
size_t numOfOutput = taosArrayGetSize(pExprList);
SExprInfo* p = taosMemoryCalloc(numOfOutput, sizeof(SExprInfo));
for (int32_t i = 0; i < numOfOutput; ++i) {
SExprInfo* pExpr = taosArrayGetP(pExprList, i);
assignExprInfo(&p[i], pExpr);
}
return p;
}
// TODO merge aggregate super table
static void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle) {
for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
@ -4822,7 +4762,7 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) {
finalizeMultiTupleQueryResult(pAggInfo->binfo.pCtx, pOperator->numOfOutput, pAggInfo->aggSup.pResultBuf,
&pAggInfo->binfo.resultRowInfo, pAggInfo->binfo.rowCellInfoOffset);
initGroupResInfo(&pAggInfo->groupResInfo, &pAggInfo->binfo.resultRowInfo);
initGroupedResultInfo(&pAggInfo->groupResInfo, pAggInfo->aggSup.pResultRowHashTable, false);
OPTR_SET_OPENED(pOperator);
return TSDB_CODE_SUCCESS;
}
@ -5125,7 +5065,7 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) {
STableQueryInfo* pTableQueryInfo = pInfo->pCurrent;
setIntervalQueryRange(pTableQueryInfo, pBlock->info.window.skey, &pTaskInfo->window);
hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, 0);
hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, pBlock->info.groupId);
#if 0 // test for encode/decode result info
if(pOperator->encodeResultRow){
@ -5147,7 +5087,7 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) {
finalizeMultiTupleQueryResult(pInfo->binfo.pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf,
&pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset);
initGroupResInfo(&pInfo->groupResInfo, &pInfo->binfo.resultRowInfo);
initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true);
OPTR_SET_OPENED(pOperator);
return TSDB_CODE_SUCCESS;
}
@ -5276,7 +5216,7 @@ static SSDataBlock* doAllIntervalAgg(SOperatorInfo* pOperator, bool* newgroup) {
setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED);
// finalizeQueryResult(pSliceInfo->binfo.pCtx, pOperator->numOfOutput);
initGroupResInfo(&pSliceInfo->groupResInfo, &pSliceInfo->binfo.resultRowInfo);
// initGroupedResultInfo(&pSliceInfo->groupResInfo, &pSliceInfo->binfo.resultRowInfo);
// doBuildResultDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pSliceInfo->pRes);
if (pSliceInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pSliceInfo->groupResInfo)) {
@ -5327,7 +5267,7 @@ static SSDataBlock* doSTableIntervalAgg(SOperatorInfo* pOperator, bool* newgroup
finalizeMultiTupleQueryResult(pInfo->binfo.pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf,
&pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset);
initGroupResInfo(&pInfo->groupResInfo, &pInfo->binfo.resultRowInfo);
// initGroupedResultInfo(&pInfo->groupResInfo, &pInfo->binfo.resultRowInfo);
OPTR_SET_OPENED(pOperator);
blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
@ -5458,7 +5398,7 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator, bool* newgroup) {
finalizeMultiTupleQueryResult(pBInfo->pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, &pBInfo->resultRowInfo,
pBInfo->rowCellInfoOffset);
initGroupResInfo(&pInfo->groupResInfo, &pBInfo->resultRowInfo);
initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true);
blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
doBuildResultDatablock(pBInfo->pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf,
pBInfo->rowCellInfoOffset, pInfo->binfo.pCtx);
@ -5510,7 +5450,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator, bool* newgroup)
finalizeMultiTupleQueryResult(pBInfo->pCtx, pOperator->numOfOutput, pInfo->aggSup.pResultBuf, &pBInfo->resultRowInfo,
pBInfo->rowCellInfoOffset);
initGroupResInfo(&pInfo->groupResInfo, &pBInfo->resultRowInfo);
initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, true);
blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
doBuildResultDatablock(pBInfo->pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf,
pBInfo->rowCellInfoOffset, pInfo->binfo.pCtx);
@ -5699,6 +5639,11 @@ int32_t initAggInfo(SOptrBasicInfo* pBasicInfo, SAggSupporter* pAggSup, SExprInf
pBasicInfo->pRes = pResultBlock;
doInitAggInfoSup(pAggSup, pBasicInfo->pCtx, numOfCols, keyBufSize, pkey);
for(int32_t i = 0; i < numOfCols; ++i) {
pBasicInfo->pCtx[i].pBuf = pAggSup->pResultBuf;
}
return TSDB_CODE_SUCCESS;
}
@ -5847,11 +5792,6 @@ static void destroyProjectOperatorInfo(void* param, int32_t numOfOutput) {
doDestroyBasicInfo(&pInfo->binfo, numOfOutput);
}
static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) {
STagScanInfo* pInfo = (STagScanInfo*)param;
pInfo->pRes = blockDataDestroy(pInfo->pRes);
}
static void destroyOrderOperatorInfo(void* param, int32_t numOfOutput) {
SSortOperatorInfo* pInfo = (SSortOperatorInfo*)param;
pInfo->pDataBlock = blockDataDestroy(pInfo->pDataBlock);
@ -5941,11 +5881,12 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo*
goto _error;
}
pInfo->order = TSDB_ORDER_ASC;
pInfo->interval = *pInterval;
pInfo->execModel = pTaskInfo->execModel;
pInfo->win = pTaskInfo->window;
pInfo->twAggSup = *pTwAggSupp;
pInfo->order = TSDB_ORDER_ASC;
pInfo->interval = *pInterval;
// pInfo->execModel = OPTR_EXEC_MODEL_STREAM;
pInfo->execModel = pTaskInfo->execModel;
pInfo->win = pTaskInfo->window;
pInfo->twAggSup = *pTwAggSupp;
pInfo->primaryTsIndex = primaryTsSlotId;
int32_t numOfRows = 4096;
@ -6204,157 +6145,6 @@ _error:
return NULL;
}
static SSDataBlock* doTagScan(SOperatorInfo* pOperator, bool* newgroup) {
#if 0
SOperatorInfo* pOperator = (SOperatorInfo*) param;
if (pOperator->status == OP_EXEC_DONE) {
return NULL;
}
int32_t maxNumOfTables = (int32_t)pResultInfo->capacity;
STagScanInfo *pInfo = pOperator->info;
SSDataBlock *pRes = pInfo->pRes;
*newgroup = false;
int32_t count = 0;
SArray* pa = GET_TABLEGROUP(pRuntimeEnv, 0);
int32_t functionId = getExprFunctionId(&pOperator->pExpr[0]);
if (functionId == FUNCTION_TID_TAG) { // return the tags & table Id
assert(pQueryAttr->numOfOutput == 1);
SExprInfo* pExprInfo = &pOperator->pExpr[0];
int32_t rsize = pExprInfo->base.resSchema.bytes;
count = 0;
int16_t bytes = pExprInfo->base.resSchema.bytes;
int16_t type = pExprInfo->base.resSchema.type;
for(int32_t i = 0; i < pQueryAttr->numOfTags; ++i) {
if (pQueryAttr->tagColList[i].colId == pExprInfo->base.pColumns->info.colId) {
bytes = pQueryAttr->tagColList[i].bytes;
type = pQueryAttr->tagColList[i].type;
break;
}
}
SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, 0);
while(pInfo->curPos < pInfo->totalTables && count < maxNumOfTables) {
int32_t i = pInfo->curPos++;
STableQueryInfo *item = taosArrayGetP(pa, i);
char *output = pColInfo->pData + count * rsize;
varDataSetLen(output, rsize - VARSTR_HEADER_SIZE);
output = varDataVal(output);
STableId* id = TSDB_TABLEID(item->pTable);
*(int16_t *)output = 0;
output += sizeof(int16_t);
*(int64_t *)output = id->uid; // memory align problem, todo serialize
output += sizeof(id->uid);
*(int32_t *)output = id->tid;
output += sizeof(id->tid);
*(int32_t *)output = pQueryAttr->vgId;
output += sizeof(pQueryAttr->vgId);
char* data = NULL;
if (pExprInfo->base.pColumns->info.colId == TSDB_TBNAME_COLUMN_INDEX) {
data = tsdbGetTableName(item->pTable);
} else {
data = tsdbGetTableTagVal(item->pTable, pExprInfo->base.pColumns->info.colId, type, bytes);
}
doSetTagValueToResultBuf(output, data, type, bytes);
count += 1;
}
//qDebug("QInfo:0x%"PRIx64" create (tableId, tag) info completed, rows:%d", GET_TASKID(pRuntimeEnv), count);
} else if (functionId == FUNCTION_COUNT) {// handle the "count(tbname)" query
SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, 0);
*(int64_t*)pColInfo->pData = pInfo->totalTables;
count = 1;
pOperator->status = OP_EXEC_DONE;
//qDebug("QInfo:0x%"PRIx64" create count(tbname) query, res:%d rows:1", GET_TASKID(pRuntimeEnv), count);
} else { // return only the tags|table name etc.
SExprInfo* pExprInfo = &pOperator->pExpr[0]; // todo use the column list instead of exprinfo
count = 0;
while(pInfo->curPos < pInfo->totalTables && count < maxNumOfTables) {
int32_t i = pInfo->curPos++;
STableQueryInfo* item = taosArrayGetP(pa, i);
char *data = NULL, *dst = NULL;
int16_t type = 0, bytes = 0;
for(int32_t j = 0; j < pOperator->numOfOutput; ++j) {
// not assign value in case of user defined constant output column
if (TSDB_COL_IS_UD_COL(pExprInfo[j].base.pColumns->flag)) {
continue;
}
SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, j);
type = pExprInfo[j].base.resSchema.type;
bytes = pExprInfo[j].base.resSchema.bytes;
if (pExprInfo[j].base.pColumns->info.colId == TSDB_TBNAME_COLUMN_INDEX) {
data = tsdbGetTableName(item->pTable);
} else {
data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.pColumns->info.colId, type, bytes);
}
dst = pColInfo->pData + count * pExprInfo[j].base.resSchema.bytes;
doSetTagValueToResultBuf(dst, data, type, bytes);
}
count += 1;
}
if (pInfo->curPos >= pInfo->totalTables) {
pOperator->status = OP_EXEC_DONE;
}
//qDebug("QInfo:0x%"PRIx64" create tag values results completed, rows:%d", GET_TASKID(pRuntimeEnv), count);
}
if (pOperator->status == OP_EXEC_DONE) {
setTaskStatus(pOperator->pRuntimeEnv, TASK_COMPLETED);
}
pRes->info.rows = count;
return (pRes->info.rows == 0)? NULL:pInfo->pRes;
#endif
return TSDB_CODE_SUCCESS;
}
SOperatorInfo* createTagScanOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput) {
STagScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STagScanInfo));
size_t numOfGroup = GET_NUM_OF_TABLEGROUP(pRuntimeEnv);
assert(numOfGroup == 0 || numOfGroup == 1);
pInfo->curPos = 0;
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
pOperator->name = "SeqTableTagScan";
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN;
pOperator->blockingOptr = false;
pOperator->status = OP_NOT_OPENED;
pOperator->info = pInfo;
pOperator->getNextFn = doTagScan;
pOperator->pExpr = pExpr;
pOperator->numOfOutput = numOfOutput;
pOperator->closeFn = destroyTagScanOperatorInfo;
return pOperator;
}
static int32_t getColumnIndexInSource(SQueriedTableInfo* pTableInfo, SExprBasicInfo* pExpr, SColumnInfo* pTagCols) {
int32_t j = 0;
@ -6562,6 +6352,9 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
int32_t numOfCols = 0;
tsdbReaderT pDataReader = doCreateDataReader(pTableScanNode, pHandle, pTableGroupInfo, (uint64_t)queryId, taskId);
if (pDataReader == NULL) {
return NULL;
}
SArray* pColList =
extractColMatchInfo(pScanPhyNode->pScanCols, pScanPhyNode->node.pOutputDataBlockDesc, &numOfCols);
@ -6576,8 +6369,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
};
return createTableScanOperatorInfo(pDataReader, pTableScanNode->scanSeq[0] > 0 ? TSDB_ORDER_ASC : TSDB_ORDER_DESC,
numOfCols, pTableScanNode->dataRequired, pTableScanNode->scanSeq[0], pTableScanNode->scanSeq[1], pColList,
pResBlock, pScanPhyNode->node.pConditions, &interval, pTableScanNode->ratio, pTaskInfo);
numOfCols, pTableScanNode->dataRequired, pTableScanNode->scanSeq[0], pTableScanNode->scanSeq[1], pColList,
pResBlock, pScanPhyNode->node.pConditions, &interval, pTableScanNode->ratio, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_EXCHANGE == type) {
SExchangePhysiNode* pExchange = (SExchangePhysiNode*)pPhyNode;
SSDataBlock* pResBlock = createResDataBlock(pExchange->node.pOutputDataBlockDesc);
@ -6585,17 +6378,14 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
} else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN == type) {
SScanPhysiNode* pScanPhyNode = (SScanPhysiNode*)pPhyNode; // simple child table.
int32_t code = doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableGroupInfo,
queryId, taskId);
int32_t code = doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableGroupInfo, queryId, taskId);
SArray* tableIdList = extractTableIdList(pTableGroupInfo);
SSDataBlock* pResBlock = createResDataBlock(pScanPhyNode->node.pOutputDataBlockDesc);
int32_t numOfCols = 0;
SArray* pColList =
extractColMatchInfo(pScanPhyNode->pScanCols, pScanPhyNode->node.pOutputDataBlockDesc, &numOfCols);
SOperatorInfo* pOperator =
createStreamScanOperatorInfo(pHandle->reader, pResBlock, pColList, tableIdList, pTaskInfo);
SArray* pCols = extractColMatchInfo(pScanPhyNode->pScanCols, pScanPhyNode->node.pOutputDataBlockDesc, &numOfCols);
SOperatorInfo* pOperator = createStreamScanOperatorInfo(pHandle->reader, pResBlock, pCols, tableIdList, pTaskInfo);
taosArrayDestroy(tableIdList);
return pOperator;
} else if (QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN == type) {
@ -6621,6 +6411,9 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
for (int32_t i = 0; i < size; ++i) {
SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, i);
ops[i] = createOperatorTree(pChildNode, pTaskInfo, pHandle, queryId, taskId, pTableGroupInfo);
if (ops[i] == NULL) {
return NULL;
}
}
SOperatorInfo* pOptr = NULL;
@ -6988,8 +6781,7 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead
goto _complete;
}
STableGroupInfo group = {0};
(*pTaskInfo)->pRoot = createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId, &group);
(*pTaskInfo)->pRoot = createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId, &(*pTaskInfo)->tableqinfoGroupInfo);
if (NULL == (*pTaskInfo)->pRoot) {
code = terrno;
goto _complete;

2
source/libs/executor/src/groupoperator.c
View File

@ -308,7 +308,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator, bool* newgrou
// }
blockDataEnsureCapacity(pRes, pOperator->resultInfo.capacity);
initGroupResInfo(&pInfo->groupResInfo, &pInfo->binfo.resultRowInfo);
initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, false);
while(1) {
doBuildResultDatablock(pRes, &pInfo->groupResInfo, pOperator->pExpr, pInfo->aggSup.pResultBuf, pInfo->binfo.rowCellInfoOffset, pInfo->binfo.pCtx);

198
source/libs/executor/src/scanoperator.c
View File

@ -271,9 +271,7 @@ static void setupEnvForReverseScan(STableScanInfo* pTableScanInfo, SqlFunctionCt
static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator, bool* newgroup) {
STableScanInfo* pTableScanInfo = pOperator->info;
SSDataBlock* pBlock = pTableScanInfo->pResBlock;
STableGroupInfo* pTableGroupInfo = &pOperator->pTaskInfo->tableqinfoGroupInfo;
SSDataBlock* pBlock = pTableScanInfo->pResBlock;
*newgroup = false;
while (tsdbNextDataBlock(pTableScanInfo->dataReader)) {
@ -284,18 +282,6 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator, bool* newgroup) {
pTableScanInfo->numOfBlocks += 1;
tsdbRetrieveDataBlockInfo(pTableScanInfo->dataReader, &pBlock->info);
// todo opt
// if (pTableGroupInfo->numOfTables > 1 || (pRuntimeEnv->current == NULL && pTableGroupInfo->numOfTables == 1)) {
// STableQueryInfo** pTableQueryInfo =
// (STableQueryInfo**)taosHashGet(pTableGroupInfo->map, &pBlock->info.uid, sizeof(pBlock->info.uid));
// if (pTableQueryInfo == NULL) {
// break;
// }
//
// doTableQueryInfoTimeWindowCheck(pTaskInfo, *pTableQueryInfo, pTableScanInfo->order);
// }
// this function never returns error?
uint32_t status = 0;
int32_t code = loadDataBlock(pOperator, pTableScanInfo, pBlock, &status);
// int32_t code = loadDataBlockOnDemand(pOperator->pRuntimeEnv, pTableScanInfo, pBlock, &status);
@ -308,6 +294,8 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator, bool* newgroup) {
continue;
}
// reset the block to be 0 by default, this blockId is assigned by physical plan and is used by direct upstream operator.
pBlock->info.blockId = 0;
return pBlock;
}
@ -405,11 +393,11 @@ SOperatorInfo* createTableScanOperatorInfo(void* pDataReader, int32_t order, int
pOperator->name = "TableScanOperator";
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN;
pOperator->blockingOptr = false;
pOperator->status = OP_NOT_OPENED;
pOperator->info = pInfo;
pOperator->numOfOutput = numOfOutput;
pOperator->getNextFn = doTableScan;
pOperator->pTaskInfo = pTaskInfo;
pOperator->status = OP_NOT_OPENED;
pOperator->info = pInfo;
pOperator->numOfOutput = numOfOutput;
pOperator->getNextFn = doTableScan;
pOperator->pTaskInfo = pTaskInfo;
static int32_t cost = 0;
pOperator->cost.openCost = ++cost;
@ -824,12 +812,12 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator, bool* newgroup) {
int32_t tableNameSlotId = 1;
SColumnInfoData* pTableNameCol = taosArrayGet(pInfo->pRes->pDataBlock, tableNameSlotId);
char* name = NULL;
char* tb = NULL;
int32_t numOfRows = 0;
char n[TSDB_TABLE_NAME_LEN] = {0};
while ((name = metaTbCursorNext(pInfo->pCur)) != NULL) {
STR_TO_VARSTR(n, name);
while ((tb = metaTbCursorNext(pInfo->pCur)) != NULL) {
STR_TO_VARSTR(n, tb);
colDataAppend(pTableNameCol, numOfRows, n, false);
numOfRows += 1;
if (numOfRows >= pInfo->capacity) {
@ -992,3 +980,167 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataB
return pOperator;
}
static SSDataBlock* doTagScan(SOperatorInfo* pOperator, bool* newgroup) {
#if 0
SOperatorInfo* pOperator = (SOperatorInfo*) param;
if (pOperator->status == OP_EXEC_DONE) {
return NULL;
}
int32_t maxNumOfTables = (int32_t)pResultInfo->capacity;
STagScanInfo *pInfo = pOperator->info;
SSDataBlock *pRes = pInfo->pRes;
*newgroup = false;
int32_t count = 0;
SArray* pa = GET_TABLEGROUP(pRuntimeEnv, 0);
int32_t functionId = getExprFunctionId(&pOperator->pExpr[0]);
if (functionId == FUNCTION_TID_TAG) { // return the tags & table Id
assert(pQueryAttr->numOfOutput == 1);
SExprInfo* pExprInfo = &pOperator->pExpr[0];
int32_t rsize = pExprInfo->base.resSchema.bytes;
count = 0;
int16_t bytes = pExprInfo->base.resSchema.bytes;
int16_t type = pExprInfo->base.resSchema.type;
for(int32_t i = 0; i < pQueryAttr->numOfTags; ++i) {
if (pQueryAttr->tagColList[i].colId == pExprInfo->base.pColumns->info.colId) {
bytes = pQueryAttr->tagColList[i].bytes;
type = pQueryAttr->tagColList[i].type;
break;
}
}
SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, 0);
while(pInfo->curPos < pInfo->totalTables && count < maxNumOfTables) {
int32_t i = pInfo->curPos++;
STableQueryInfo *item = taosArrayGetP(pa, i);
char *output = pColInfo->pData + count * rsize;
varDataSetLen(output, rsize - VARSTR_HEADER_SIZE);
output = varDataVal(output);
STableId* id = TSDB_TABLEID(item->pTable);
*(int16_t *)output = 0;
output += sizeof(int16_t);
*(int64_t *)output = id->uid; // memory align problem, todo serialize
output += sizeof(id->uid);
*(int32_t *)output = id->tid;
output += sizeof(id->tid);
*(int32_t *)output = pQueryAttr->vgId;
output += sizeof(pQueryAttr->vgId);
char* data = NULL;
if (pExprInfo->base.pColumns->info.colId == TSDB_TBNAME_COLUMN_INDEX) {
data = tsdbGetTableName(item->pTable);
} else {
data = tsdbGetTableTagVal(item->pTable, pExprInfo->base.pColumns->info.colId, type, bytes);
}
doSetTagValueToResultBuf(output, data, type, bytes);
count += 1;
}
//qDebug("QInfo:0x%"PRIx64" create (tableId, tag) info completed, rows:%d", GET_TASKID(pRuntimeEnv), count);
} else if (functionId == FUNCTION_COUNT) {// handle the "count(tbname)" query
SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, 0);
*(int64_t*)pColInfo->pData = pInfo->totalTables;
count = 1;
pOperator->status = OP_EXEC_DONE;
//qDebug("QInfo:0x%"PRIx64" create count(tbname) query, res:%d rows:1", GET_TASKID(pRuntimeEnv), count);
} else { // return only the tags|table name etc.
SExprInfo* pExprInfo = &pOperator->pExpr[0]; // todo use the column list instead of exprinfo
count = 0;
while(pInfo->curPos < pInfo->totalTables && count < maxNumOfTables) {
int32_t i = pInfo->curPos++;
STableQueryInfo* item = taosArrayGetP(pa, i);
char *data = NULL, *dst = NULL;
int16_t type = 0, bytes = 0;
for(int32_t j = 0; j < pOperator->numOfOutput; ++j) {
// not assign value in case of user defined constant output column
if (TSDB_COL_IS_UD_COL(pExprInfo[j].base.pColumns->flag)) {
continue;
}
SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, j);
type = pExprInfo[j].base.resSchema.type;
bytes = pExprInfo[j].base.resSchema.bytes;
if (pExprInfo[j].base.pColumns->info.colId == TSDB_TBNAME_COLUMN_INDEX) {
data = tsdbGetTableName(item->pTable);
} else {
data = tsdbGetTableTagVal(item->pTable, pExprInfo[j].base.pColumns->info.colId, type, bytes);
}
dst = pColInfo->pData + count * pExprInfo[j].base.resSchema.bytes;
doSetTagValueToResultBuf(dst, data, type, bytes);
}
count += 1;
}
if (pInfo->curPos >= pInfo->totalTables) {
pOperator->status = OP_EXEC_DONE;
}
//qDebug("QInfo:0x%"PRIx64" create tag values results completed, rows:%d", GET_TASKID(pRuntimeEnv), count);
}
if (pOperator->status == OP_EXEC_DONE) {
setTaskStatus(pOperator->pRuntimeEnv, TASK_COMPLETED);
}
pRes->info.rows = count;
return (pRes->info.rows == 0)? NULL:pInfo->pRes;
#endif
return TSDB_CODE_SUCCESS;
}
static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) {
STagScanInfo* pInfo = (STagScanInfo*)param;
pInfo->pRes = blockDataDestroy(pInfo->pRes);
}
SOperatorInfo* createTagScanOperatorInfo(void* pReaderHandle, SExprInfo* pExpr, int32_t numOfOutput, SExecTaskInfo* pTaskInfo) {
STagScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STagScanInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
goto _error;
}
pInfo->pReader = pReaderHandle;
pInfo->curPos = 0;
pOperator->name = "TagScanOperator";
pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN;
pOperator->blockingOptr = false;
pOperator->status = OP_NOT_OPENED;
pOperator->info = pInfo;
pOperator->getNextFn = doTagScan;
pOperator->pExpr = pExpr;
pOperator->numOfOutput = numOfOutput;
pOperator->pTaskInfo = pTaskInfo;
pOperator->closeFn = destroyTagScanOperatorInfo;
return pOperator;
_error:
taosMemoryFree(pInfo);
taosMemoryFree(pOperator);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}

10
source/libs/function/inc/builtinsimpl.h
View File

@ -24,7 +24,7 @@ extern "C" {
#include "functionMgt.h"
bool functionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId);
int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
EFuncDataRequired countDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow);
bool getCountFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
@ -43,17 +43,17 @@ int32_t maxFunction(SqlFunctionCtx *pCtx);
bool getAvgFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool avgFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
int32_t avgFunction(SqlFunctionCtx* pCtx);
int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId);
int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
bool getStddevFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool stddevFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
int32_t stddevFunction(SqlFunctionCtx* pCtx);
int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId);
int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
bool getPercentileFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool percentileFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo);
int32_t percentileFunction(SqlFunctionCtx *pCtx);
int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId);
int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
bool getDiffFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv);
bool diffFunctionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo);
@ -65,7 +65,7 @@ int32_t lastFunction(SqlFunctionCtx *pCtx);
bool getTopBotFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv);
int32_t topFunction(SqlFunctionCtx *pCtx);
int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId);
int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock);
#ifdef __cplusplus
}

21
source/libs/function/inc/tudf.h
View File

@ -29,29 +29,20 @@ extern "C" {
#define UDF_LISTEN_PIPE_NAME_LEN 32
#define UDF_LISTEN_PIPE_NAME_PREFIX "udfd.sock."
#define UDF_DNODE_ID_ENV_NAME "DNODE_ID"
//======================================================================================
//begin API to taosd and qworker
enum {
UDFC_CODE_STOPPING = -1,
UDFC_CODE_PIPE_READ_ERR = -3,
UDFC_CODE_PIPE_READ_ERR = -2,
UDFC_CODE_CONNECT_PIPE_ERR = -3,
UDFC_CODE_LOAD_UDF_FAILURE = -4,
UDFC_CODE_INVALID_STATE = -5
};
typedef void *UdfcHandle;
typedef void *UdfcFuncHandle;
/**
* create udfd proxy, called once in process that call setupUdf/callUdfxxx/teardownUdf
* @return error code
*/
int32_t udfcOpen(int32_t dnodeId, UdfcHandle* proxyHandle);
/**
* destroy udfd proxy
* @return error code
*/
int32_t udfcClose(UdfcHandle proxyhandle);
/**
@ -60,7 +51,7 @@ int32_t udfcClose(UdfcHandle proxyhandle);
* @param handle, out
* @return error code
*/
int32_t setupUdf(UdfcHandle proxyHandle, char udfName[], SEpSet *epSet, UdfcFuncHandle *handle);
int32_t setupUdf(char udfName[], UdfcFuncHandle *handle);
typedef struct SUdfColumnMeta {
int16_t type;

2
source/libs/function/inc/tudfInt.h
View File

@ -39,7 +39,6 @@ enum {
typedef struct SUdfSetupRequest {
char udfName[TSDB_FUNC_NAME_LEN];
SEpSet epSet;
} SUdfSetupRequest;
typedef struct SUdfSetupResponse {
@ -112,6 +111,7 @@ void freeUdfDataDataBlock(SUdfDataBlock *block);
int32_t convertDataBlockToUdfDataBlock(SSDataBlock *block, SUdfDataBlock *udfBlock);
int32_t convertUdfColumnToDataBlock(SUdfColumn *udfCol, SSDataBlock *block);
int32_t getUdfdPipeName(char* pipeName, int32_t size);
#ifdef __cplusplus
}
#endif

232
source/libs/function/src/builtinsimpl.c
View File

@ -14,7 +14,7 @@
*/
#include "builtinsimpl.h"
#include <libs/nodes/querynodes.h>
#include "function.h"
#include "querynodes.h"
#include "taggfunction.h"
#include "tdatablock.h"
@ -44,8 +44,6 @@ typedef struct STopBotResItem {
} STopBotResItem;
typedef struct STopBotRes {
int32_t pageId;
// int32_t num;
STopBotResItem *pItems;
} STopBotRes;
@ -92,18 +90,6 @@ typedef struct SDiffInfo {
} \
} while (0);
#define DO_UPDATE_SUBSID_RES(ctx, ts) \
do { \
for (int32_t _i = 0; _i < (ctx)->subsidiaryRes.numOfCols; ++_i) { \
SqlFunctionCtx *__ctx = (ctx)->subsidiaryRes.pCtx[_i]; \
if (__ctx->functionId == FUNCTION_TS_DUMMY) { \
__ctx->tag.i = (ts); \
__ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; \
} \
__ctx->fpSet.process(__ctx); \
} \
} while (0)
#define UPDATE_DATA(ctx, left, right, num, sign, _ts) \
do { \
if (((left) < (right)) ^ (sign)) { \
@ -139,7 +125,8 @@ bool functionSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResultInfo) {
return true;
}
int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId) {
int32_t functionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
int32_t slotId = pCtx->pExpr->base.resSchema.slotId;
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId);
SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
@ -406,7 +393,7 @@ int32_t avgFunction(SqlFunctionCtx* pCtx) {
return TSDB_CODE_SUCCESS;
}
int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId) {
int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
SInputColumnInfoData* pInput = &pCtx->input;
int32_t type = pInput->pData[0]->info.type;
SAvgRes* pAvgRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
@ -416,7 +403,7 @@ int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId) {
pAvgRes->result = pAvgRes->sum.dsum / ((double) pAvgRes->count);
}
return functionFinalize(pCtx, pBlock, slotId);
return functionFinalize(pCtx, pBlock);
}
EFuncDataRequired statisDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow){
@ -521,6 +508,49 @@ bool getMinmaxFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
return true;
}
#define GET_TS_LIST(x) ((TSKEY*)((x)->ptsList))
#define GET_TS_DATA(x, y) (GET_TS_LIST(x)[(y)])
#define DO_UPDATE_TAG_COLUMNS_WITHOUT_TS(ctx) \
do { \
for (int32_t _i = 0; _i < (ctx)->tagInfo.numOfTagCols; ++_i) { \
SqlFunctionCtx *__ctx = (ctx)->tagInfo.pTagCtxList[_i]; \
__ctx->fpSet.process(__ctx); \
} \
} while (0);
#define DO_UPDATE_SUBSID_RES(ctx, ts) \
do { \
for (int32_t _i = 0; _i < (ctx)->subsidiaries.num; ++_i) { \
SqlFunctionCtx* __ctx = (ctx)->subsidiaries.pCtx[_i]; \
if (__ctx->functionId == FUNCTION_TS_DUMMY) { \
__ctx->tag.i = (ts); \
__ctx->tag.nType = TSDB_DATA_TYPE_BIGINT; \
} \
__ctx->fpSet.process(__ctx); \
} \
} while (0)
#define UPDATE_DATA(ctx, left, right, num, sign, _ts) \
do { \
if (((left) < (right)) ^ (sign)) { \
(left) = (right); \
DO_UPDATE_SUBSID_RES(ctx, _ts); \
(num) += 1; \
} \
} while (0)
#define LOOPCHECK_N(val, _col, ctx, _t, _nrow, _start, sign, num) \
do { \
_t *d = (_t *)((_col)->pData); \
for (int32_t i = (_start); i < (_nrow) + (_start); ++i) { \
if (((_col)->hasNull) && colDataIsNull_f((_col)->nullbitmap, i)) { \
continue; \
} \
TSKEY ts = (ctx)->ptsList != NULL ? GET_TS_DATA(ctx, i) : 0; \
UPDATE_DATA(ctx, val, d[i], num, sign, ts); \
} \
} while (0)
int32_t doMinMaxHelper(SqlFunctionCtx *pCtx, int32_t isMinFunc) {
int32_t numOfElems = 0;
@ -564,8 +594,8 @@ int32_t doMinMaxHelper(SqlFunctionCtx *pCtx, int32_t isMinFunc) {
int64_t val = GET_INT64_VAL(tval);
if ((prev < val) ^ isMinFunc) {
*(int64_t*) buf = val;
for (int32_t i = 0; i < (pCtx)->subsidiaryRes.numOfCols; ++i) {
SqlFunctionCtx* __ctx = pCtx->subsidiaryRes.pCtx[i];
for (int32_t i = 0; i < (pCtx)->subsidiaries.num; ++i) {
SqlFunctionCtx* __ctx = pCtx->subsidiaries.pCtx[i];
if (__ctx->functionId == FUNCTION_TS_DUMMY) { // TODO refactor
__ctx->tag.i = key;
__ctx->tag.nType = TSDB_DATA_TYPE_BIGINT;
@ -581,8 +611,8 @@ int32_t doMinMaxHelper(SqlFunctionCtx *pCtx, int32_t isMinFunc) {
uint64_t val = GET_UINT64_VAL(tval);
if ((prev < val) ^ isMinFunc) {
*(uint64_t*) buf = val;
for (int32_t i = 0; i < (pCtx)->subsidiaryRes.numOfCols; ++i) {
SqlFunctionCtx* __ctx = pCtx->subsidiaryRes.pCtx[i];
for (int32_t i = 0; i < (pCtx)->subsidiaries.num; ++i) {
SqlFunctionCtx* __ctx = pCtx->subsidiaries.pCtx[i];
if (__ctx->functionId == FUNCTION_TS_DUMMY) { // TODO refactor
__ctx->tag.i = key;
__ctx->tag.nType = TSDB_DATA_TYPE_BIGINT;
@ -797,7 +827,7 @@ int32_t stddevFunction(SqlFunctionCtx* pCtx) {
return TSDB_CODE_SUCCESS;
}
int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId) {
int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
SInputColumnInfoData* pInput = &pCtx->input;
int32_t type = pInput->pData[0]->info.type;
SStddevRes* pStddevRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
@ -810,7 +840,7 @@ int32_t stddevFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId
pStddevRes->result = sqrt(pStddevRes->quadraticDSum/((double)pStddevRes->count) - avg*avg);
}
return functionFinalize(pCtx, pBlock, slotId);
return functionFinalize(pCtx, pBlock);
}
bool getPercentileFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) {
@ -923,7 +953,7 @@ int32_t percentileFunction(SqlFunctionCtx *pCtx) {
return TSDB_CODE_SUCCESS;
}
int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId) {
int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
SVariant* pVal = &pCtx->param[1].param;
double v = pVal->nType == TSDB_DATA_TYPE_INT ? pVal->i : pVal->d;
@ -936,7 +966,7 @@ int32_t percentileFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t sl
}
tMemBucketDestroy(pMemBucket);
return functionFinalize(pCtx, pBlock, slotId);
return functionFinalize(pCtx, pBlock);
}
bool getFirstLastFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) {
@ -1353,15 +1383,16 @@ static STopBotRes *getTopBotOutputInfo(SqlFunctionCtx *pCtx) {
return pRes;
}
static void doAddIntoResult(STopBotRes* pRes, int32_t maxSize, void* pData, int32_t rowIndex, SSDataBlock* pSrcBlock,
static void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSDataBlock* pSrcBlock,
uint16_t type, uint64_t uid, SResultRowEntryInfo* pEntryInfo);
static void saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STopBotResItem* pItem);
static void copyTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STopBotResItem* pItem);
int32_t topFunction(SqlFunctionCtx *pCtx) {
int32_t numOfElems = 0;
SResultRowEntryInfo *pResInfo = GET_RES_INFO(pCtx);
STopBotRes *pRes = getTopBotOutputInfo(pCtx);
// if ((void *)pRes->res[0] != (void *)((char *)pRes + sizeof(STopBotRes) + POINTER_BYTES * pCtx->param[0].i)) {
// buildTopBotStruct(pRes, pCtx);
// }
@ -1381,7 +1412,7 @@ int32_t topFunction(SqlFunctionCtx *pCtx) {
numOfElems++;
char* data = colDataGetData(pCol, i);
doAddIntoResult(pRes, pCtx->param[1].param.i, data, i, NULL, type, pInput->uid, pResInfo);
doAddIntoResult(pCtx, data, i, pCtx->pSrcBlock, type, pInput->uid, pResInfo);
}
return TSDB_CODE_SUCCESS;
@ -1414,9 +1445,11 @@ static int32_t topBotResComparFn(const void *p1, const void *p2, const void *par
return (val1->v.d > val2->v.d) ? 1 : -1;
}
void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSDataBlock* pSrcBlock, uint16_t type,
uint64_t uid, SResultRowEntryInfo* pEntryInfo) {
STopBotRes *pRes = getTopBotOutputInfo(pCtx);
int32_t maxSize = pCtx->param[1].param.i;
void doAddIntoResult(STopBotRes *pRes, int32_t maxSize, void *pData, int32_t rowIndex, SSDataBlock* pSrcBlock, uint16_t type,
uint64_t uid, SResultRowEntryInfo* pEntryInfo) {
SVariant val = {0};
taosVariantCreateFromBinary(&val, pData, tDataTypes[type].bytes, type);
@ -1428,22 +1461,9 @@ void doAddIntoResult(STopBotRes *pRes, int32_t maxSize, void *pData, int32_t row
STopBotResItem* pItem = &pItems[pEntryInfo->numOfRes];
pItem->v = val;
pItem->uid = uid;
pItem->tuplePos.pageId = -1; // todo set the corresponding tuple data in the disk-based buffer
if (pRes->pageId == -1) {
SFilePage* pPage = getNewBufPage(NULL, 0, &pRes->pageId);
pPage->num = sizeof(SFilePage);
// keep the current row data
for(int32_t i = 0; i < pSrcBlock->info.numOfCols; ++i) {
SColumnInfoData* pCol = taosArrayGet(pSrcBlock->pDataBlock, i);
bool isNull = colDataIsNull_s(pCol, rowIndex);
colDataGetData(pCol, rowIndex);
}
}
// save the data of this tuple
saveTupleData(pCtx, rowIndex, pSrcBlock, pItem);
// allocate the buffer and keep the data of this row into the new allocated buffer
pEntryInfo->numOfRes++;
@ -1452,22 +1472,100 @@ void doAddIntoResult(STopBotRes *pRes, int32_t maxSize, void *pData, int32_t row
if ((IS_SIGNED_NUMERIC_TYPE(type) && val.i > pItems[0].v.i) ||
(IS_UNSIGNED_NUMERIC_TYPE(type) && val.u > pItems[0].v.u) ||
(IS_FLOAT_TYPE(type) && val.d > pItems[0].v.d)) {
// replace the old data and the coresponding tuple data
STopBotResItem* pItem = &pItems[0];
pItem->v = val;
pItem->uid = uid;
pItem->tuplePos.pageId = -1; // todo set the corresponding tuple data in the disk-based buffer
// save the data of this tuple by over writing the old data
copyTupleData(pCtx, rowIndex, pSrcBlock, pItem);
taosheapadjust((void *) pItems, sizeof(STopBotResItem), 0, pEntryInfo->numOfRes - 1, (const void *) &type, topBotResComparFn, NULL, false);
}
}
}
int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId) {
void saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STopBotResItem* pItem) {
SFilePage* pPage = NULL;
int32_t completeRowSize = pSrcBlock->info.rowSize + pSrcBlock->info.numOfCols * sizeof(bool);
if (pCtx->curBufPage == -1) {
pPage = getNewBufPage(pCtx->pBuf, 0, &pCtx->curBufPage);
pPage->num = sizeof(SFilePage);
} else {
pPage = getBufPage(pCtx->pBuf, pCtx->curBufPage);
if (pPage->num + completeRowSize > getBufPageSize(pCtx->pBuf)) {
pPage = getNewBufPage(pCtx->pBuf, 0, &pCtx->curBufPage);
pPage->num = sizeof(SFilePage);
}
}
pItem->tuplePos.pageId = pCtx->curBufPage;
// keep the current row data, extract method
int32_t offset = 0;
bool* nullList = (bool*)((char*)pPage + pPage->num);
char* pStart = (char*)(nullList + sizeof(bool) * pSrcBlock->info.numOfCols);
for (int32_t i = 0; i < pSrcBlock->info.numOfCols; ++i) {
SColumnInfoData* pCol = taosArrayGet(pSrcBlock->pDataBlock, i);
bool isNull = colDataIsNull_s(pCol, rowIndex);
if (isNull) {
nullList[i] = true;
continue;
}
char* p = colDataGetData(pCol, rowIndex);
if (IS_VAR_DATA_TYPE(pCol->info.type)) {
memcpy(pStart + offset, p, varDataTLen(p));
} else {
memcpy(pStart + offset, p, pCol->info.bytes);
}
offset += pCol->info.bytes;
}
pItem->tuplePos.offset = pPage->num;
pPage->num += completeRowSize;
setBufPageDirty(pPage, true);
releaseBufPage(pCtx->pBuf, pPage);
}
void copyTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STopBotResItem* pItem) {
SFilePage* pPage = getBufPage(pCtx->pBuf, pItem->tuplePos.pageId);
bool* nullList = (bool*)((char*)pPage + pItem->tuplePos.offset);
char* pStart = (char*)(nullList + pSrcBlock->info.numOfCols * sizeof(bool));
int32_t offset = 0;
for(int32_t i = 0; i < pSrcBlock->info.numOfCols; ++i) {
SColumnInfoData* pCol = taosArrayGet(pSrcBlock->pDataBlock, i);
if ((nullList[i] = colDataIsNull_s(pCol, rowIndex)) == true) {
continue;
}
char* p = colDataGetData(pCol, rowIndex);
if (IS_VAR_DATA_TYPE(pCol->info.type)) {
memcpy(pStart + offset, p, varDataTLen(p));
} else {
memcpy(pStart + offset, p, pCol->info.bytes);
}
offset += pCol->info.bytes;
}
setBufPageDirty(pPage, true);
releaseBufPage(pCtx->pBuf, pPage);
}
int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
SResultRowEntryInfo *pEntryInfo = GET_RES_INFO(pCtx);
STopBotRes* pRes = GET_ROWCELL_INTERBUF(pEntryInfo);
pEntryInfo->complete = true;
int32_t type = pCtx->input.pData[0]->info.type;
int32_t slotId = pCtx->pExpr->base.resSchema.slotId;
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId);
// todo assign the tag value and the corresponding row data
@ -1476,19 +1574,45 @@ int32_t topBotFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t slotId
case TSDB_DATA_TYPE_INT: {
for (int32_t i = 0; i < pEntryInfo->numOfRes; ++i) {
STopBotResItem* pItem = &pRes->pItems[i];
colDataAppendInt32(pCol, currentRow++, (int32_t*)&pItem->v.i);
colDataAppendInt32(pCol, currentRow, (int32_t*)&pItem->v.i);
int32_t pageId = pItem->tuplePos.pageId;
int32_t offset = pItem->tuplePos.offset;
if (pageId != -1) {
// todo
if (pItem->tuplePos.pageId != -1) {
SFilePage* pPage = getBufPage(pCtx->pBuf, pageId);
bool* nullList = (bool*)((char*)pPage + offset);
char* pStart = (char*)(nullList + pCtx->pSrcBlock->info.numOfCols * sizeof(bool));
// todo set the offset value to optimize the performance.
for (int32_t j = 0; j < pCtx->subsidiaries.num; ++j) {
SqlFunctionCtx* pc = pCtx->subsidiaries.pCtx[j];
SFunctParam *pFuncParam = &pc->pExpr->base.pParam[0];
int32_t srcSlotId = pFuncParam->pCol->slotId;
int32_t dstSlotId = pCtx->pExpr->base.resSchema.slotId;
int32_t ps = 0;
for(int32_t k = 0; k < srcSlotId; ++k) {
SColumnInfoData* pSrcCol = taosArrayGet(pCtx->pSrcBlock->pDataBlock, k);
ps += pSrcCol->info.bytes;
}
SColumnInfoData* pDstCol = taosArrayGet(pBlock->pDataBlock, dstSlotId);
if (nullList[srcSlotId]) {
colDataAppendNULL(pDstCol, currentRow);
} else {
colDataAppend(pDstCol, currentRow, (pStart + ps), false);
}
}
}
currentRow += 1;
}
break;
}
}
return pEntryInfo->numOfRes;
// return functionFinalize(pCtx, pBlock, slotId);
}

161
source/libs/function/src/tudf.c
View File

@ -124,7 +124,7 @@ enum {
int64_t gUdfTaskSeqNum = 0;
typedef struct SUdfdProxy {
int32_t dnodeId;
char udfdPipeName[UDF_LISTEN_PIPE_NAME_LEN];
uv_barrier_t gUdfInitBarrier;
uv_loop_t gUdfdLoop;
@ -137,11 +137,11 @@ typedef struct SUdfdProxy {
int8_t gUdfcState;
QUEUE gUdfTaskQueue;
QUEUE gUvProcTaskQueue;
// int8_t gUdfcState = UDFC_STATE_INITAL;
// QUEUE gUdfTaskQueue = {0};
// QUEUE gUvProcTaskQueue = {0};
int8_t initialized;
} SUdfdProxy;
SUdfdProxy gUdfdProxy = {0};
typedef struct SUdfUvSession {
SUdfdProxy *udfc;
@ -209,19 +209,27 @@ enum {
UDFC_STATE_STARTNG, // starting after udfcOpen
UDFC_STATE_READY, // started and begin to receive quests
UDFC_STATE_STOPPING, // stopping after udfcClose
UDFC_STATUS_FINAL, // stopped
};
int32_t getUdfdPipeName(char* pipeName, int32_t size) {
char dnodeId[8] = {0};
size_t dnodeIdSize;
int32_t err = uv_os_getenv(UDF_DNODE_ID_ENV_NAME, dnodeId, &dnodeIdSize);
if (err != 0) {
dnodeId[0] = '1';
}
snprintf(pipeName, size, "%s%s", UDF_LISTEN_PIPE_NAME_PREFIX, dnodeId);
return 0;
}
int32_t encodeUdfSetupRequest(void **buf, const SUdfSetupRequest *setup) {
int32_t len = 0;
len += taosEncodeBinary(buf, setup->udfName, TSDB_FUNC_NAME_LEN);
len += taosEncodeSEpSet(buf, &setup->epSet);
return len;
}
void* decodeUdfSetupRequest(const void* buf, SUdfSetupRequest *request) {
buf = taosDecodeBinaryTo(buf, request->udfName, TSDB_FUNC_NAME_LEN);
buf = taosDecodeSEpSet((void*)buf, &request->epSet);
return (void*)buf;
}
@ -604,7 +612,7 @@ void onUdfcPipeClose(uv_handle_t *handle) {
}
int32_t udfcGetUvTaskResponseResult(SClientUdfTask *task, SClientUvTaskNode *uvTask) {
debugPrint("%s", "get uv task result");
fnDebug("udfc get uv task result. task: %p", task);
if (uvTask->type == UV_TASK_REQ_RSP) {
if (uvTask->rspBuf.base != NULL) {
SUdfResponse rsp;
@ -647,7 +655,6 @@ int32_t udfcGetUvTaskResponseResult(SClientUdfTask *task, SClientUvTaskNode *uvT
}
void udfcAllocateBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) {
debugPrint("%s", "client allocate buffer to receive from pipe");
SClientUvConn *conn = handle->data;
SClientConnBuf *connBuf = &conn->readBuf;
@ -662,7 +669,7 @@ void udfcAllocateBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf
buf->base = connBuf->buf;
buf->len = connBuf->cap;
} else {
//TODO: log error
fnError("udfc allocate buffer failure. size: %d", msgHeadSize);
buf->base = NULL;
buf->len = 0;
}
@ -674,13 +681,13 @@ void udfcAllocateBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf
buf->base = connBuf->buf + connBuf->len;
buf->len = connBuf->cap - connBuf->len;
} else {
//TODO: log error free connBuf->buf
fnError("udfc re-allocate buffer failure. size: %d", connBuf->cap);
buf->base = NULL;
buf->len = 0;
}
}
debugPrint("\tconn buf cap - len - total : %d - %d - %d", connBuf->cap, connBuf->len, connBuf->total);
fnTrace("conn buf cap - len - total : %d - %d - %d", connBuf->cap, connBuf->len, connBuf->total);
}
@ -689,6 +696,7 @@ bool isUdfcUvMsgComplete(SClientConnBuf *connBuf) {
connBuf->total = *(int32_t *) (connBuf->buf);
}
if (connBuf->len == connBuf->cap && connBuf->total == connBuf->cap) {
fnTrace("udfc complete message is received, now handle it");
return true;
}
return false;
@ -696,10 +704,10 @@ bool isUdfcUvMsgComplete(SClientConnBuf *connBuf) {
void udfcUvHandleRsp(SClientUvConn *conn) {
SClientConnBuf *connBuf = &conn->readBuf;
int64_t seqNum = *(int64_t *) (connBuf->buf + sizeof(int32_t)); // msglen int32_t then seqnum
int64_t seqNum = *(int64_t *) (connBuf->buf + sizeof(int32_t)); // msglen then seqnum
if (QUEUE_EMPTY(&conn->taskQueue)) {
//LOG error
fnError("udfc no task waiting for response on connection");
return;
}
bool found = false;
@ -713,7 +721,7 @@ void udfcUvHandleRsp(SClientUvConn *conn) {
found = true;
taskFound = task;
} else {
//LOG error;
fnError("udfc more than one task waiting for the same response");
continue;
}
}
@ -727,7 +735,7 @@ void udfcUvHandleRsp(SClientUvConn *conn) {
uv_sem_post(&taskFound->taskSem);
QUEUE_REMOVE(&taskFound->procTaskQueue);
} else {
//TODO: LOG error
fnError("no task is waiting for the response.");
}
connBuf->buf = NULL;
connBuf->total = -1;
@ -751,7 +759,7 @@ void udfcUvHandleError(SClientUvConn *conn) {
}
void onUdfcRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) {
debugPrint("%s, nread: %zd", "client read from pipe", nread);
fnTrace("udfc client %p, client read from pipe. nread: %zd", client, nread);
if (nread == 0) return;
SClientUvConn *conn = client->data;
@ -764,9 +772,9 @@ void onUdfcRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) {
}
if (nread < 0) {
debugPrint("\tclient read error: %s", uv_strerror(nread));
fnError("udfc client pipe %p read error: %s", client, uv_strerror(nread));
if (nread == UV_EOF) {
//TODO:
fnError("udfc client pipe %p closed", client);
}
udfcUvHandleError(conn);
}
@ -774,16 +782,15 @@ void onUdfcRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) {
}
void onUdfClientWrite(uv_write_t *write, int status) {
debugPrint("%s", "after writing to pipe");
SClientUvTaskNode *uvTask = write->data;
uv_pipe_t *pipe = uvTask->pipe;
if (status == 0) {
uv_pipe_t *pipe = uvTask->pipe;
SClientUvConn *conn = pipe->data;
QUEUE_INSERT_TAIL(&conn->taskQueue, &uvTask->connTaskQueue);
} else {
//TODO Log error;
fnError("udfc client %p write error.", pipe);
}
debugPrint("\tlength:%zu", uvTask->reqBuf.len);
fnTrace("udfc client %p write length:%zu", pipe, uvTask->reqBuf.len);
taosMemoryFree(write);
taosMemoryFree(uvTask->reqBuf.base);
}
@ -841,7 +848,7 @@ int32_t createUdfcUvTask(SClientUdfTask *task, int8_t uvTaskType, SClientUvTaskN
}
int32_t queueUvUdfTask(SClientUvTaskNode *uvTask) {
debugPrint("%s, %d", "queue uv task", uvTask->type);
fnTrace("queue uv task to event loop, task: %d, %p", uvTask->type, uvTask);
SUdfdProxy *udfc = uvTask->udfc;
uv_mutex_lock(&udfc->gUdfTaskQueueMutex);
QUEUE_INSERT_TAIL(&udfc->gUdfTaskQueue, &uvTask->recvTaskQueue);
@ -855,7 +862,7 @@ int32_t queueUvUdfTask(SClientUvTaskNode *uvTask) {
}
int32_t startUvUdfTask(SClientUvTaskNode *uvTask) {
debugPrint("%s, type %d", "start uv task ", uvTask->type);
fnTrace("event loop start uv task. task: %d, %p", uvTask->type, uvTask);
switch (uvTask->type) {
case UV_TASK_CONNECT: {
uv_pipe_t *pipe = taosMemoryMalloc(sizeof(uv_pipe_t));
@ -874,8 +881,7 @@ int32_t startUvUdfTask(SClientUvTaskNode *uvTask) {
uv_connect_t *connReq = taosMemoryMalloc(sizeof(uv_connect_t));
connReq->data = uvTask;
uv_pipe_connect(connReq, pipe, "udf.sock", onUdfClientConnect);
uv_pipe_connect(connReq, pipe, uvTask->udfc->udfdPipeName, onUdfClientConnect);
break;
}
case UV_TASK_REQ_RSP: {
@ -971,27 +977,37 @@ void constructUdfService(void *argsThread) {
uv_loop_close(&udfc->gUdfdLoop);
}
int32_t udfcOpen(int32_t dnodeId, UdfcHandle *udfc) {
SUdfdProxy *proxy = taosMemoryCalloc(1, sizeof(SUdfdProxy));
proxy->dnodeId = dnodeId;
int32_t udfcOpen() {
int8_t old = atomic_val_compare_exchange_8(&gUdfdProxy.initialized, 0, 1);
if (old == 1) {
return 0;
}
SUdfdProxy *proxy = &gUdfdProxy;
getUdfdPipeName(proxy->udfdPipeName, UDF_LISTEN_PIPE_NAME_LEN);
proxy->gUdfcState = UDFC_STATE_STARTNG;
uv_barrier_init(&proxy->gUdfInitBarrier, 2);
uv_thread_create(&proxy->gUdfLoopThread, constructUdfService, proxy);
uv_barrier_wait(&proxy->gUdfInitBarrier);
atomic_store_8(&proxy->gUdfcState, UDFC_STATE_READY);
proxy->gUdfcState = UDFC_STATE_READY;
*udfc = proxy;
uv_barrier_wait(&proxy->gUdfInitBarrier);
fnInfo("udfc initialized")
return 0;
}
int32_t udfcClose(UdfcHandle udfcHandle) {
SUdfdProxy *udfc = udfcHandle;
int32_t udfcClose() {
int8_t old = atomic_val_compare_exchange_8(&gUdfdProxy.initialized, 1, 0);
if (old == 0) {
return 0;
}
SUdfdProxy *udfc = &gUdfdProxy;
udfc->gUdfcState = UDFC_STATE_STOPPING;
uv_async_send(&udfc->gUdfLoopStopAsync);
uv_thread_join(&udfc->gUdfLoopThread);
uv_mutex_destroy(&udfc->gUdfTaskQueueMutex);
uv_barrier_destroy(&udfc->gUdfInitBarrier);
udfc->gUdfcState = UDFC_STATUS_FINAL;
taosMemoryFree(udfc);
udfc->gUdfcState = UDFC_STATE_INITAL;
fnInfo("udfc cleaned up");
return 0;
}
@ -1009,12 +1025,15 @@ int32_t udfcRunUvTask(SClientUdfTask *task, int8_t uvTaskType) {
return task->errCode;
}
int32_t setupUdf(UdfcHandle udfc, char udfName[], SEpSet *epSet, UdfcFuncHandle *funcHandle) {
debugPrint("%s", "client setup udf");
int32_t setupUdf(char udfName[], UdfcFuncHandle *funcHandle) {
fnInfo("udfc setup udf. udfName: %s", udfName);
if (gUdfdProxy.gUdfcState != UDFC_STATE_READY) {
return UDFC_CODE_INVALID_STATE;
}
SClientUdfTask *task = taosMemoryMalloc(sizeof(SClientUdfTask));
task->errCode = 0;
task->session = taosMemoryMalloc(sizeof(SUdfUvSession));
task->session->udfc = udfc;
task->session->udfc = &gUdfdProxy;
task->type = UDF_TASK_SETUP;
SUdfSetupRequest *req = &task->_setup.req;
@ -1022,15 +1041,20 @@ int32_t setupUdf(UdfcHandle udfc, char udfName[], SEpSet *epSet, UdfcFuncHandle
int32_t errCode = udfcRunUvTask(task, UV_TASK_CONNECT);
if (errCode != 0) {
//TODO: log error
return -1;
fnError("failed to connect to pipe. udfName: %s, pipe: %s", udfName, (&gUdfdProxy)->udfdPipeName);
return UDFC_CODE_CONNECT_PIPE_ERR;
}
udfcRunUvTask(task, UV_TASK_REQ_RSP);
SUdfSetupResponse *rsp = &task->_setup.rsp;
task->session->severHandle = rsp->udfHandle;
*funcHandle = task->session;
if (task->errCode != 0) {
fnError("failed to setup udf. err: %d", task->errCode)
} else {
fnInfo("sucessfully setup udf func handle. handle: %p", task->session);
*funcHandle = task->session;
}
int32_t err = task->errCode;
taosMemoryFree(task);
return err;
@ -1038,7 +1062,7 @@ int32_t setupUdf(UdfcHandle udfc, char udfName[], SEpSet *epSet, UdfcFuncHandle
int32_t callUdf(UdfcFuncHandle handle, int8_t callType, SSDataBlock *input, SUdfInterBuf *state, SUdfInterBuf *state2,
SSDataBlock* output, SUdfInterBuf *newState) {
debugPrint("%s", "client call udf");
fnTrace("udfc call udf. callType: %d, funcHandle: %p", callType, handle);
SClientUdfTask *task = taosMemoryMalloc(sizeof(SClientUdfTask));
task->errCode = 0;
@ -1076,35 +1100,37 @@ int32_t callUdf(UdfcFuncHandle handle, int8_t callType, SSDataBlock *input, SUdf
udfcRunUvTask(task, UV_TASK_REQ_RSP);
SUdfCallResponse *rsp = &task->_call.rsp;
switch (callType) {
case TSDB_UDF_CALL_AGG_INIT: {
*newState = rsp->resultBuf;
break;
}
case TSDB_UDF_CALL_AGG_PROC: {
*newState = rsp->resultBuf;
break;
}
case TSDB_UDF_CALL_AGG_MERGE: {
*newState = rsp->resultBuf;
break;
}
case TSDB_UDF_CALL_AGG_FIN: {
*newState = rsp->resultBuf;
break;
}
case TSDB_UDF_CALL_SCALA_PROC: {
*output = rsp->resultData;
break;
if (task->errCode != 0) {
fnError("call udf failure. err: %d", task->errCode);
} else {
SUdfCallResponse *rsp = &task->_call.rsp;
switch (callType) {
case TSDB_UDF_CALL_AGG_INIT: {
*newState = rsp->resultBuf;
break;
}
case TSDB_UDF_CALL_AGG_PROC: {
*newState = rsp->resultBuf;
break;
}
case TSDB_UDF_CALL_AGG_MERGE: {
*newState = rsp->resultBuf;
break;
}
case TSDB_UDF_CALL_AGG_FIN: {
*newState = rsp->resultBuf;
break;
}
case TSDB_UDF_CALL_SCALA_PROC: {
*output = rsp->resultData;
break;
}
}
}
taosMemoryFree(task);
return task->errCode;
}
//TODO: translate these calls to callUdf
int32_t callUdfAggInit(UdfcFuncHandle handle, SUdfInterBuf *interBuf) {
int8_t callType = TSDB_UDF_CALL_AGG_INIT;
@ -1148,7 +1174,7 @@ int32_t callUdfScalarFunc(UdfcFuncHandle handle, SScalarParam *input, int32_t nu
}
int32_t teardownUdf(UdfcFuncHandle handle) {
debugPrint("%s", "client teardown udf");
fnInfo("tear down udf. udf func handle: %p", handle);
SClientUdfTask *task = taosMemoryMalloc(sizeof(SClientUdfTask));
task->errCode = 0;
@ -1160,7 +1186,6 @@ int32_t teardownUdf(UdfcFuncHandle handle) {
udfcRunUvTask(task, UV_TASK_REQ_RSP);
SUdfTeardownResponse *rsp = &task->_teardown.rsp;
int32_t err = task->errCode;

103
source/libs/function/src/udfd.c
View File

@ -20,6 +20,7 @@
#include "tudf.h"
#include "tudfInt.h"
#include "tdatablock.h"
#include "tdataformat.h"
#include "tglobal.h"
#include "tmsg.h"
@ -31,8 +32,9 @@ typedef struct SUdfdContext {
uv_signal_t intrSignal;
char listenPipeName[UDF_LISTEN_PIPE_NAME_LEN];
uv_pipe_t listeningPipe;
void *clientRpc;
void *clientRpc;
SCorEpSet mgmtEp;
uv_mutex_t udfsMutex;
SHashObj *udfsHash;
@ -63,8 +65,13 @@ typedef struct SUdf {
uv_mutex_t lock;
uv_cond_t condReady;
char name[16];
int8_t type;
char name[TSDB_FUNC_NAME_LEN];
int8_t funcType;
int8_t scriptType;
int8_t outputType;
int32_t outputLen;
int32_t bufSize;
char path[PATH_MAX];
uv_lib_t lib;
@ -78,17 +85,17 @@ typedef struct SUdfcFuncHandle {
SUdf *udf;
} SUdfcFuncHandle;
int32_t udfdFillUdfInfoFromMNode(void *clientRpc, SEpSet *pEpSet, char *udfName, SUdf *udf);
int32_t udfdFillUdfInfoFromMNode(void *clientRpc, char *udfName, SUdf *udf);
int32_t udfdLoadUdf(char *udfName, SEpSet *pEpSet, SUdf *udf) {
int32_t udfdLoadUdf(char *udfName, SUdf *udf) {
strcpy(udf->name, udfName);
udfdFillUdfInfoFromMNode(global.clientRpc, pEpSet, udf->name, udf);
udfdFillUdfInfoFromMNode(global.clientRpc, udf->name, udf);
//strcpy(udf->path, "/home/slzhou/TDengine/debug/build/lib/libudf1.so");
int err = uv_dlopen(udf->path, &udf->lib);
if (err != 0) {
fnError("can not load library %s. error: %s", udf->path, uv_strerror(err));
// TODO set error
return UDFC_CODE_LOAD_UDF_FAILURE;
}
// TODO: find all the functions
char normalFuncName[TSDB_FUNC_NAME_LEN] = {0};
@ -115,8 +122,8 @@ void udfdProcessRequest(uv_work_t *req) {
SUdf *udf = NULL;
uv_mutex_lock(&global.udfsMutex);
SUdf **udfInHash = taosHashGet(global.udfsHash, request.setup.udfName, TSDB_FUNC_NAME_LEN);
if (*udfInHash) {
SUdf **udfInHash = taosHashGet(global.udfsHash, request.setup.udfName, strlen(request.setup.udfName));
if (udfInHash) {
++(*udfInHash)->refCount;
udf = *udfInHash;
uv_mutex_unlock(&global.udfsMutex);
@ -128,14 +135,14 @@ void udfdProcessRequest(uv_work_t *req) {
uv_mutex_init(&udfNew->lock);
uv_cond_init(&udfNew->condReady);
udf = udfNew;
taosHashPut(global.udfsHash, request.setup.udfName, TSDB_FUNC_NAME_LEN, &udfNew, sizeof(&udfNew));
taosHashPut(global.udfsHash, request.setup.udfName, strlen(request.setup.udfName), &udfNew, sizeof(&udfNew));
uv_mutex_unlock(&global.udfsMutex);
}
uv_mutex_lock(&udf->lock);
if (udf->state == UDF_STATE_INIT) {
udf->state = UDF_STATE_LOADING;
udfdLoadUdf(setup->udfName, &setup->epSet, udf);
udfdLoadUdf(setup->udfName, udf);
udf->state = UDF_STATE_READY;
uv_cond_broadcast(&udf->condReady);
uv_mutex_unlock(&udf->lock);
@ -214,7 +221,7 @@ void udfdProcessRequest(uv_work_t *req) {
udf->refCount--;
if (udf->refCount == 0) {
unloadUdf = true;
taosHashRemove(global.udfsHash, udf->name, TSDB_FUNC_NAME_LEN);
taosHashRemove(global.udfsHash, udf->name, strlen(udf->name));
}
uv_mutex_unlock(&global.udfsMutex);
if (unloadUdf) {
@ -393,7 +400,48 @@ void udfdIntrSignalHandler(uv_signal_t *handle, int signum) {
void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { return; }
int32_t udfdFillUdfInfoFromMNode(void *clientRpc, SEpSet *pEpSet, char *udfName, SUdf *udf) {
int initEpSetFromCfg(const char* firstEp, const char* secondEp, SCorEpSet* pEpSet) {
pEpSet->version = 0;
// init mnode ip set
SEpSet* mgmtEpSet = &(pEpSet->epSet);
mgmtEpSet->numOfEps = 0;
mgmtEpSet->inUse = 0;
if (firstEp && firstEp[0] != 0) {
if (strlen(firstEp) >= TSDB_EP_LEN) {
terrno = TSDB_CODE_TSC_INVALID_FQDN;
return -1;
}
int32_t code = taosGetFqdnPortFromEp(firstEp, &mgmtEpSet->eps[0]);
if (code != TSDB_CODE_SUCCESS) {
terrno = TSDB_CODE_TSC_INVALID_FQDN;
return terrno;
}
mgmtEpSet->numOfEps++;
}
if (secondEp && secondEp[0] != 0) {
if (strlen(secondEp) >= TSDB_EP_LEN) {
terrno = TSDB_CODE_TSC_INVALID_FQDN;
return -1;
}
taosGetFqdnPortFromEp(secondEp, &mgmtEpSet->eps[mgmtEpSet->numOfEps]);
mgmtEpSet->numOfEps++;
}
if (mgmtEpSet->numOfEps == 0) {
terrno = TSDB_CODE_TSC_INVALID_FQDN;
return -1;
}
return 0;
}
int32_t udfdFillUdfInfoFromMNode(void *clientRpc, char *udfName, SUdf *udf) {
SRetrieveFuncReq retrieveReq = {0};
retrieveReq.numOfFuncs = 1;
retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN);
@ -410,15 +458,21 @@ int32_t udfdFillUdfInfoFromMNode(void *clientRpc, SEpSet *pEpSet, char *udfName,
rpcMsg.msgType = TDMT_MND_RETRIEVE_FUNC;
SRpcMsg rpcRsp = {0};
rpcSendRecv(clientRpc, pEpSet, &rpcMsg, &rpcRsp);
rpcSendRecv(clientRpc, &global.mgmtEp.epSet, &rpcMsg, &rpcRsp);
SRetrieveFuncRsp retrieveRsp = {0};
tDeserializeSRetrieveFuncRsp(rpcRsp.pCont, rpcRsp.contLen, &retrieveRsp);
SFuncInfo *pFuncInfo = (SFuncInfo *)taosArrayGet(retrieveRsp.pFuncInfos, 0);
udf->funcType = pFuncInfo->funcType;
udf->scriptType = pFuncInfo->scriptType;
udf->outputType = pFuncInfo->funcType;
udf->outputLen = pFuncInfo->outputLen;
udf->bufSize = pFuncInfo->bufSize;
char path[PATH_MAX] = {0};
taosGetTmpfilePath("/tmp", "libudf", path);
TdFilePtr file = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC);
snprintf(path, sizeof(path), "%s/lib%s.so", "/tmp", udfName);
TdFilePtr file = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC | TD_FILE_AUTO_DEL);
// TODO check for failure of flush to disk
taosWriteFile(file, pFuncInfo->pCode, pFuncInfo->codeSize);
taosCloseFile(&file);
@ -531,15 +585,7 @@ static int32_t udfdUvInit() {
uv_pipe_open(&global.ctrlPipe, 0);
uv_read_start((uv_stream_t *)&global.ctrlPipe, udfdCtrlAllocBufCb, udfdCtrlReadCb);
char dnodeId[8] = {0};
size_t dnodeIdSize;
int32_t err = uv_os_getenv("DNODE_ID", dnodeId, &dnodeIdSize);
if (err != 0) {
dnodeId[0] = '1';
}
char listenPipeName[32] = {0};
snprintf(listenPipeName, sizeof(listenPipeName), "%s%s", UDF_LISTEN_PIPE_NAME_PREFIX, dnodeId);
strcpy(global.listenPipeName, listenPipeName);
getUdfdPipeName(global.listenPipeName, UDF_LISTEN_PIPE_NAME_LEN);
removeListeningPipe();
@ -550,7 +596,7 @@ static int32_t udfdUvInit() {
int r;
fnInfo("bind to pipe %s", global.listenPipeName);
if ((r = uv_pipe_bind(&global.listeningPipe, listenPipeName))) {
if ((r = uv_pipe_bind(&global.listeningPipe, global.listenPipeName))) {
fnError("Bind error %s", uv_err_name(r));
removeListeningPipe();
return -1;
@ -580,7 +626,7 @@ static int32_t udfdRun() {
fnInfo("start the udfd");
int code = uv_run(global.loop, UV_RUN_DEFAULT);
fnInfo("udfd stopped. result: %s", uv_err_name(code));
fnInfo("udfd stopped. result: %s, code: %d", uv_err_name(code), code);
int codeClose = uv_loop_close(global.loop);
fnDebug("uv loop close. result: %s", uv_err_name(codeClose));
udfdCloseClientRpc();
@ -615,5 +661,6 @@ int main(int argc, char *argv[]) {
return -1;
}
initEpSetFromCfg(tsFirst, tsSecond, &global.mgmtEp);
return udfdRun();
}

121
source/libs/function/test/runUdf.c
View File

@ -1,61 +1,84 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "uv.h"
#include "fnLog.h"
#include "os.h"
#include "tudf.h"
#include "tdatablock.h"
#include "tglobal.h"
#include "tudf.h"
static int32_t parseArgs(int32_t argc, char *argv[]) {
for (int32_t i = 1; i < argc; ++i) {
if (strcmp(argv[i], "-c") == 0) {
if (i < argc - 1) {
if (strlen(argv[++i]) >= PATH_MAX) {
printf("config file path overflow");
return -1;
}
tstrncpy(configDir, argv[i], PATH_MAX);
} else {
printf("'-c' requires a parameter, default is %s\n", configDir);
return -1;
}
}
}
return 0;
}
static int32_t initLog() {
char logName[12] = {0};
snprintf(logName, sizeof(logName), "%slog", "udfc");
return taosCreateLog(logName, 1, configDir, NULL, NULL, NULL, 0);
}
int main(int argc, char *argv[]) {
UdfcHandle udfc;
udfcOpen(1, &udfc);
uv_sleep(1000);
char path[256] = {0};
size_t cwdSize = 256;
int err = uv_cwd(path, &cwdSize);
if (err != 0) {
fprintf(stderr, "err cwd: %s\n", uv_strerror(err));
return err;
parseArgs(argc, argv);
initLog();
if (taosInitCfg(configDir, NULL, NULL, NULL, 0) != 0) {
fnError("failed to start since read config error");
return -1;
}
udfcOpen();
uv_sleep(1000);
UdfcFuncHandle handle;
setupUdf("udf1", &handle);
SSDataBlock block = {0};
SSDataBlock *pBlock = &block;
pBlock->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData));
pBlock->info.numOfCols = 1;
pBlock->info.rows = 4;
char data[16] = {0};
char bitmap[4] = {0};
for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
SColumnInfoData colInfo = {0};
colInfo.info.type = TSDB_DATA_TYPE_INT;
colInfo.info.bytes = sizeof(int32_t);
colInfo.info.colId = 1;
colInfo.pData = data;
colInfo.nullbitmap = bitmap;
for (int32_t j = 0; j < pBlock->info.rows; ++j) {
colDataAppendInt32(&colInfo, j, &j);
}
fprintf(stdout, "current working directory:%s\n", path);
strcat(path, "/libudf1.so");
taosArrayPush(pBlock->pDataBlock, &colInfo);
}
UdfcFuncHandle handle;
SEpSet epSet;
setupUdf(udfc, "udf1", &epSet, &handle);
SScalarParam input = {0};
input.numOfRows = pBlock->info.rows;
input.columnData = taosArrayGet(pBlock->pDataBlock, 0);
SScalarParam output = {0};
callUdfScalarFunc(handle, &input, 1, &output);
SSDataBlock block = {0};
SSDataBlock* pBlock = &block;
pBlock->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData));
pBlock->info.numOfCols = 1;
pBlock->info.rows = 4;
char data[16] = {0};
char bitmap[4] = {0};
for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) {
SColumnInfoData colInfo = {0};
colInfo.info.type = TSDB_DATA_TYPE_INT;
colInfo.info.bytes = sizeof(int32_t);
colInfo.info.colId = 1;
colInfo.pData = data;
colInfo.nullbitmap = bitmap;
for (int32_t j = 0; j < pBlock->info.rows; ++j) {
colDataAppendInt32(&colInfo, j, &j);
}
taosArrayPush(pBlock->pDataBlock, &colInfo);
}
SScalarParam input = {0};
input.numOfRows = pBlock->info.rows;
input.columnData = taosArrayGet(pBlock->pDataBlock, 0);
SScalarParam output = {0};
callUdfScalarFunc(handle, &input, 1 , &output);
SColumnInfoData *col = output.columnData;
for (int32_t i = 0; i < output.numOfRows; ++i) {
fprintf(stderr, "%d\t%d\n" , i, *(int32_t*)(col->pData + i *sizeof(int32_t)));
}
teardownUdf(handle);
udfcClose(udfc);
SColumnInfoData *col = output.columnData;
for (int32_t i = 0; i < output.numOfRows; ++i) {
fprintf(stderr, "%d\t%d\n", i, *(int32_t *)(col->pData + i * sizeof(int32_t)));
}
teardownUdf(handle);
udfcClose();
}

131
source/libs/parser/src/parTranslater.c
View File

@ -198,7 +198,7 @@ static int32_t getDBVgVersion(STranslateContext* pCxt, const char* pDbFName, int
static int32_t getDBCfg(STranslateContext* pCxt, const char* pDbName, SDbCfgInfo* pInfo) {
SParseContext* pParCxt = pCxt->pParseCxt;
SName name;
SName name;
tNameSetDbName(&name, pCxt->pParseCxt->acctId, pDbName, strlen(pDbName));
char dbFname[TSDB_DB_FNAME_LEN] = {0};
tNameGetFullDbName(&name, dbFname);
@ -560,8 +560,7 @@ static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) {
pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes;
}
} else if (nodesIsComparisonOp(pOp)) {
if (TSDB_DATA_TYPE_BLOB == ldt.type || TSDB_DATA_TYPE_JSON == rdt.type ||
TSDB_DATA_TYPE_BLOB == rdt.type) {
if (TSDB_DATA_TYPE_BLOB == ldt.type || TSDB_DATA_TYPE_JSON == rdt.type || TSDB_DATA_TYPE_BLOB == rdt.type) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName);
}
if (OP_TYPE_IN == pOp->opType || OP_TYPE_NOT_IN == pOp->opType) {
@ -569,7 +568,7 @@ static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) {
}
pOp->node.resType.type = TSDB_DATA_TYPE_BOOL;
pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes;
} else if (nodesIsJsonOp(pOp)){
} else if (nodesIsJsonOp(pOp)) {
if (TSDB_DATA_TYPE_JSON != ldt.type || TSDB_DATA_TYPE_BINARY != rdt.type) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName);
}
@ -588,7 +587,9 @@ static EDealRes haveAggFunction(SNode* pNode, void* pContext) {
}
static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode* pFunc) {
SFmGetFuncInfoParam param = { .pCtg = pCxt->pParseCxt->pCatalog, .pRpc = pCxt->pParseCxt->pTransporter, .pMgmtEps = &pCxt->pParseCxt->mgmtEpSet};
SFmGetFuncInfoParam param = {.pCtg = pCxt->pParseCxt->pCatalog,
.pRpc = pCxt->pParseCxt->pTransporter,
.pMgmtEps = &pCxt->pParseCxt->mgmtEpSet};
if (TSDB_CODE_SUCCESS != fmGetFuncInfo(&param, pFunc->functionName, &pFunc->funcId, &pFunc->funcType)) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_FUNTION, pFunc->functionName);
}
@ -1268,7 +1269,7 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode*
static EDealRes checkStateExpr(SNode* pNode, void* pContext) {
if (QUERY_NODE_COLUMN == nodeType(pNode)) {
STranslateContext* pCxt = pContext;
SColumnNode* pCol = (SColumnNode*)pNode;
SColumnNode* pCol = (SColumnNode*)pNode;
if (!IS_INTEGER_TYPE(pCol->node.resType.type)) {
return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_STATE_WIN_TYPE);
}
@ -1345,7 +1346,8 @@ static int32_t translateFrom(STranslateContext* pCxt, SSelectStmt* pSelect) {
}
static int32_t checkLimit(STranslateContext* pCxt, SSelectStmt* pSelect) {
if ((NULL != pSelect->pLimit && pSelect->pLimit->offset < 0) || (NULL != pSelect->pSlimit && pSelect->pSlimit->offset < 0)) {
if ((NULL != pSelect->pLimit && pSelect->pLimit->offset < 0) ||
(NULL != pSelect->pSlimit && pSelect->pSlimit->offset < 0)) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_OFFSET_LESS_ZERO);
}
@ -1438,7 +1440,7 @@ static int32_t translateSetOperatorImpl(STranslateContext* pCxt, SSetOperator* p
SExprNode* pLeftExpr = (SExprNode*)pLeft;
SExprNode* pRightExpr = (SExprNode*)pRight;
if (!dataTypeEqual(&pLeftExpr->resType, &pRightExpr->resType)) {
SNode* pRightFunc = NULL;
SNode* pRightFunc = NULL;
int32_t code = createCastFunc(pCxt, pRight, pLeftExpr->resType, &pRightFunc);
if (TSDB_CODE_SUCCESS != code) {
return code;
@ -1650,7 +1652,8 @@ static int32_t checkKeepOption(STranslateContext* pCxt, SNodeList* pKeep) {
(TIME_UNIT_MINUTE != pKeep1->unit && TIME_UNIT_HOUR != pKeep1->unit && TIME_UNIT_DAY != pKeep1->unit)) ||
(pKeep2->isDuration &&
(TIME_UNIT_MINUTE != pKeep2->unit && TIME_UNIT_HOUR != pKeep2->unit && TIME_UNIT_DAY != pKeep2->unit))) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_KEEP_UNIT, pKeep0->unit, pKeep1->unit, pKeep2->unit);
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_KEEP_UNIT, pKeep0->unit, pKeep1->unit,
pKeep2->unit);
}
int32_t daysToKeep0 = getBigintFromValueNode(pKeep0);
@ -1659,7 +1662,7 @@ static int32_t checkKeepOption(STranslateContext* pCxt, SNodeList* pKeep) {
if (daysToKeep0 < TSDB_MIN_KEEP || daysToKeep1 < TSDB_MIN_KEEP || daysToKeep2 < TSDB_MIN_KEEP ||
daysToKeep0 > TSDB_MAX_KEEP || daysToKeep1 > TSDB_MAX_KEEP || daysToKeep2 > TSDB_MAX_KEEP) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_KEEP_VALUE, daysToKeep0, daysToKeep1, daysToKeep2,
TSDB_MIN_KEEP, TSDB_MAX_KEEP);
TSDB_MIN_KEEP, TSDB_MAX_KEEP);
}
if (!((daysToKeep0 <= daysToKeep1) && (daysToKeep1 <= daysToKeep2))) {
@ -1691,7 +1694,8 @@ static int32_t checkDbRetentionsOption(STranslateContext* pCxt, SNodeList* pRete
return TSDB_CODE_SUCCESS;
}
static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions, bool alter) {
static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions,
bool alter) {
if (NULL == pOptions->pDaysPerFile && NULL == pOptions->pKeep) {
return TSDB_CODE_SUCCESS;
}
@ -1699,7 +1703,7 @@ static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbNa
int64_t daysToKeep0 = GET_OPTION_VAL(nodesListGetNode(pOptions->pKeep, 0), alter ? -1 : TSDB_DEFAULT_KEEP);
if (alter && (-1 == daysPerFile || -1 == daysToKeep0)) {
SDbCfgInfo dbCfg;
int32_t code = getDBCfg(pCxt, pDbName, &dbCfg);
int32_t code = getDBCfg(pCxt, pDbName, &dbCfg);
if (TSDB_CODE_SUCCESS != code) {
return code;
}
@ -1712,7 +1716,8 @@ static int32_t checkOptionsDependency(STranslateContext* pCxt, const char* pDbNa
return TSDB_CODE_SUCCESS;
}
static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions, bool alter) {
static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName, SDatabaseOptions* pOptions,
bool alter) {
int32_t code =
checkRangeOption(pCxt, "totalBlocks", pOptions->pNumOfBlocks, TSDB_MIN_TOTAL_BLOCKS, TSDB_MAX_TOTAL_BLOCKS);
if (TSDB_CODE_SUCCESS == code) {
@ -1918,7 +1923,7 @@ static int32_t checTableFactorOption(STranslateContext* pCxt, SValueNode* pVal)
}
if (pVal->datum.d < TSDB_MIN_DB_FILE_FACTOR || pVal->datum.d > TSDB_MAX_DB_FILE_FACTOR) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_F_RANGE_OPTION, "file_factor", pVal->datum.d,
TSDB_MIN_DB_FILE_FACTOR, TSDB_MAX_DB_FILE_FACTOR);
TSDB_MIN_DB_FILE_FACTOR, TSDB_MAX_DB_FILE_FACTOR);
}
}
return TSDB_CODE_SUCCESS;
@ -1945,7 +1950,7 @@ static int32_t checkTableTags(STranslateContext* pCxt, SCreateTableStmt* pStmt)
SNode* pNode;
FOREACH(pNode, pStmt->pTags) {
SColumnDefNode* pCol = (SColumnDefNode*)pNode;
if(pCol->dataType.type == TSDB_DATA_TYPE_JSON && LIST_LENGTH(pStmt->pTags) > 1){
if (pCol->dataType.type == TSDB_DATA_TYPE_JSON && LIST_LENGTH(pStmt->pTags) > 1) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_ONLY_ONE_JSON_TAG);
}
}
@ -1960,8 +1965,10 @@ static int32_t checkTableRollupOption(STranslateContext* pCxt, SNodeList* pFuncs
if (1 != LIST_LENGTH(pFuncs)) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ROLLUP_OPTION);
}
SFunctionNode* pFunc = nodesListGetNode(pFuncs, 0);
SFmGetFuncInfoParam param = { .pCtg = pCxt->pParseCxt->pCatalog, .pRpc = pCxt->pParseCxt->pTransporter, .pMgmtEps = &pCxt->pParseCxt->mgmtEpSet};
SFunctionNode* pFunc = nodesListGetNode(pFuncs, 0);
SFmGetFuncInfoParam param = {.pCtg = pCxt->pParseCxt->pCatalog,
.pRpc = pCxt->pParseCxt->pTransporter,
.pMgmtEps = &pCxt->pParseCxt->mgmtEpSet};
if (TSDB_CODE_SUCCESS != fmGetFuncInfo(&param, pFunc->functionName, &pFunc->funcId, &pFunc->funcType)) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_FUNTION, pFunc->functionName);
}
@ -2016,10 +2023,10 @@ static void toSchema(const SColumnDefNode* pCol, col_id_t colId, SSchema* pSchem
typedef struct SSampleAstInfo {
const char* pDbName;
const char* pTableName;
SNodeList* pFuncs;
SNode* pInterval;
SNode* pOffset;
SNode* pSliding;
SNodeList* pFuncs;
SNode* pInterval;
SNode* pOffset;
SNode* pSliding;
STableMeta* pRollupTableMeta;
} SSampleAstInfo;
@ -2089,8 +2096,8 @@ static SNode* makeIntervalVal(SRetention* pRetension, int8_t precision) {
return NULL;
}
int64_t timeVal = convertTimeFromPrecisionToUnit(pRetension->freq, precision, pRetension->freqUnit);
char buf[20] = {0};
int32_t len = snprintf(buf, sizeof(buf), "%"PRId64"%c", timeVal, pRetension->freqUnit);
char buf[20] = {0};
int32_t len = snprintf(buf, sizeof(buf), "%" PRId64 "%c", timeVal, pRetension->freqUnit);
pVal->literal = strndup(buf, len);
if (NULL == pVal->literal) {
nodesDestroyNode(pVal);
@ -2133,7 +2140,7 @@ static SNodeList* createRollupFuncs(SCreateTableStmt* pStmt) {
SNode* pFunc = NULL;
FOREACH(pFunc, pStmt->pOptions->pFuncs) {
SNode* pCol = NULL;
bool primaryKey = true;
bool primaryKey = true;
FOREACH(pCol, pStmt->pCols) {
if (primaryKey) {
primaryKey = false;
@ -2150,7 +2157,7 @@ static SNodeList* createRollupFuncs(SCreateTableStmt* pStmt) {
}
static STableMeta* createRollupTableMeta(SCreateTableStmt* pStmt, int8_t precision) {
int32_t numOfField = LIST_LENGTH(pStmt->pCols) + LIST_LENGTH(pStmt->pTags);
int32_t numOfField = LIST_LENGTH(pStmt->pCols) + LIST_LENGTH(pStmt->pTags);
STableMeta* pMeta = taosMemoryCalloc(1, sizeof(STableMeta) + numOfField * sizeof(SSchema));
if (NULL == pMeta) {
return NULL;
@ -2161,7 +2168,7 @@ static STableMeta* createRollupTableMeta(SCreateTableStmt* pStmt, int8_t precisi
pMeta->tableInfo.numOfColumns = LIST_LENGTH(pStmt->pCols);
int32_t index = 0;
SNode* pCol = NULL;
SNode* pCol = NULL;
FOREACH(pCol, pStmt->pCols) {
toSchema((SColumnDefNode*)pCol, index + 1, pMeta->schema + index);
++index;
@ -2175,8 +2182,8 @@ static STableMeta* createRollupTableMeta(SCreateTableStmt* pStmt, int8_t precisi
return pMeta;
}
static int32_t buildSampleAstInfoByTable(STranslateContext* pCxt,
SCreateTableStmt* pStmt, SRetention* pRetension, int8_t precision, SSampleAstInfo* pInfo) {
static int32_t buildSampleAstInfoByTable(STranslateContext* pCxt, SCreateTableStmt* pStmt, SRetention* pRetension,
int8_t precision, SSampleAstInfo* pInfo) {
pInfo->pDbName = pStmt->dbName;
pInfo->pTableName = pStmt->tableName;
pInfo->pFuncs = createRollupFuncs(pStmt);
@ -2188,10 +2195,10 @@ static int32_t buildSampleAstInfoByTable(STranslateContext* pCxt,
return TSDB_CODE_SUCCESS;
}
static int32_t getRollupAst(STranslateContext* pCxt,
SCreateTableStmt* pStmt, SRetention* pRetension, int8_t precision, char** pAst, int32_t* pLen) {
static int32_t getRollupAst(STranslateContext* pCxt, SCreateTableStmt* pStmt, SRetention* pRetension, int8_t precision,
char** pAst, int32_t* pLen) {
SSampleAstInfo info = {0};
int32_t code = buildSampleAstInfoByTable(pCxt, pStmt, pRetension, precision, &info);
int32_t code = buildSampleAstInfoByTable(pCxt, pStmt, pRetension, precision, &info);
if (TSDB_CODE_SUCCESS == code) {
code = buildSampleAst(pCxt, &info, pAst, pLen);
}
@ -2201,17 +2208,17 @@ static int32_t getRollupAst(STranslateContext* pCxt,
static int32_t buildRollupAst(STranslateContext* pCxt, SCreateTableStmt* pStmt, SMCreateStbReq* pReq) {
SDbCfgInfo dbCfg = {0};
int32_t code = getDBCfg(pCxt, pStmt->dbName, &dbCfg);
int32_t num = taosArrayGetSize(dbCfg.pRetensions);
int32_t code = getDBCfg(pCxt, pStmt->dbName, &dbCfg);
int32_t num = taosArrayGetSize(dbCfg.pRetensions);
if (TSDB_CODE_SUCCESS != code || num < 2) {
return code;
}
for (int32_t i = 1; i < num; ++i) {
SRetention *pRetension = taosArrayGet(dbCfg.pRetensions, i);
SRetention* pRetension = taosArrayGet(dbCfg.pRetensions, i);
STranslateContext cxt = {0};
initTranslateContext(pCxt->pParseCxt, &cxt);
code = getRollupAst(&cxt, pStmt, pRetension, dbCfg.precision,
1 == i ? &pReq->pAst1 : &pReq->pAst2, 1 == i ? &pReq->ast1Len : &pReq->ast2Len);
code = getRollupAst(&cxt, pStmt, pRetension, dbCfg.precision, 1 == i ? &pReq->pAst1 : &pReq->pAst2,
1 == i ? &pReq->ast1Len : &pReq->ast2Len);
destroyTranslateContext(&cxt);
if (TSDB_CODE_SUCCESS != code) {
break;
@ -2245,7 +2252,7 @@ static int32_t buildCreateStbReq(STranslateContext* pCxt, SCreateTableStmt* pStm
static int32_t translateCreateSuperTable(STranslateContext* pCxt, SCreateTableStmt* pStmt) {
SMCreateStbReq createReq = {0};
int32_t code = checkCreateTable(pCxt, pStmt);
int32_t code = checkCreateTable(pCxt, pStmt);
if (TSDB_CODE_SUCCESS == code) {
code = buildCreateStbReq(pCxt, pStmt, &createReq);
}
@ -2289,7 +2296,8 @@ static int32_t translateDropTable(STranslateContext* pCxt, SDropTableStmt* pStmt
static int32_t translateDropSuperTable(STranslateContext* pCxt, SDropSuperTableStmt* pStmt) {
SName tableName;
return doTranslateDropSuperTable(pCxt, toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tableName), pStmt->ignoreNotExists);
return doTranslateDropSuperTable(pCxt, toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tableName),
pStmt->ignoreNotExists);
}
static int32_t setAlterTableField(SAlterTableStmt* pStmt, SMAltertbReq* pAlterReq) {
@ -2421,7 +2429,7 @@ static int32_t nodeTypeToShowType(ENodeType nt) {
case QUERY_NODE_SHOW_QUERIES_STMT:
return TSDB_MGMT_TABLE_QUERIES;
case QUERY_NODE_SHOW_VARIABLE_STMT:
return 0; // todo
return 0; // todo
default:
break;
}
@ -2463,8 +2471,8 @@ static int32_t buildSampleAstInfoByIndex(STranslateContext* pCxt, SCreateIndexSt
pInfo->pOffset = nodesCloneNode(pStmt->pOptions->pOffset);
pInfo->pSliding = nodesCloneNode(pStmt->pOptions->pSliding);
if (NULL == pInfo->pFuncs || NULL == pInfo->pInterval ||
(NULL != pStmt->pOptions->pOffset && NULL == pInfo->pOffset) ||
(NULL != pStmt->pOptions->pSliding && NULL == pInfo->pSliding)) {
(NULL != pStmt->pOptions->pOffset && NULL == pInfo->pOffset) ||
(NULL != pStmt->pOptions->pSliding && NULL == pInfo->pSliding)) {
return TSDB_CODE_OUT_OF_MEMORY;
}
return TSDB_CODE_SUCCESS;
@ -2472,7 +2480,7 @@ static int32_t buildSampleAstInfoByIndex(STranslateContext* pCxt, SCreateIndexSt
static int32_t getSmaIndexAst(STranslateContext* pCxt, SCreateIndexStmt* pStmt, char** pAst, int32_t* pLen) {
SSampleAstInfo info = {0};
int32_t code = buildSampleAstInfoByIndex(pCxt, pStmt, &info);
int32_t code = buildSampleAstInfoByIndex(pCxt, pStmt, &info);
if (TSDB_CODE_SUCCESS == code) {
code = buildSampleAst(pCxt, &info, pAst, pLen);
}
@ -2618,9 +2626,9 @@ static int32_t translateDropComponentNode(STranslateContext* pCxt, SDropComponen
static int32_t buildCreateTopicReq(STranslateContext* pCxt, SCreateTopicStmt* pStmt, SCMCreateTopicReq* pReq) {
SName name;
// tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->topicName, strlen(pStmt->topicName));
// tNameGetFullDbName(&name, pReq->name);
tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pCxt->pParseCxt->db, pStmt->topicName, &name), pReq->name);
tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->topicName, strlen(pStmt->topicName));
tNameGetFullDbName(&name, pReq->name);
/*tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pCxt->pParseCxt->db, pStmt->topicName, &name), pReq->name);*/
pReq->igExists = pStmt->ignoreExists;
pReq->withTbName = pStmt->pOptions->withTable;
pReq->withSchema = pStmt->pOptions->withSchema;
@ -2633,16 +2641,19 @@ static int32_t buildCreateTopicReq(STranslateContext* pCxt, SCreateTopicStmt* pS
int32_t code = TSDB_CODE_SUCCESS;
const char* dbName;
if (NULL != pStmt->pQuery) {
strcpy(pReq->subscribeDbName, ((SRealTableNode*)(((SSelectStmt*)pStmt->pQuery)->pFromTable))->table.dbName);
dbName = ((SRealTableNode*)(((SSelectStmt*)pStmt->pQuery)->pFromTable))->table.dbName;
pCxt->pParseCxt->topicQuery = true;
code = translateQuery(pCxt, pStmt->pQuery);
if (TSDB_CODE_SUCCESS == code) {
code = nodesNodeToString(pStmt->pQuery, false, &pReq->ast, NULL);
}
} else {
strcpy(pReq->subscribeDbName, pStmt->subscribeDbName);
dbName = pStmt->subscribeDbName;
}
tNameSetDbName(&name, pCxt->pParseCxt->acctId, dbName, strlen(dbName));
tNameGetFullDbName(&name, pReq->subscribeDbName);
return code;
}
@ -2654,8 +2665,9 @@ static int32_t checkCreateTopic(STranslateContext* pCxt, SCreateTopicStmt* pStmt
if (QUERY_NODE_SELECT_STMT == nodeType(pStmt->pQuery)) {
SSelectStmt* pSelect = (SSelectStmt*)pStmt->pQuery;
if (!pSelect->isDistinct && QUERY_NODE_REAL_TABLE == nodeType(pSelect->pFromTable) && NULL == pSelect->pGroupByList &&
NULL == pSelect->pLimit && NULL == pSelect->pSlimit && NULL == pSelect->pOrderByList && NULL == pSelect->pPartitionByList) {
if (!pSelect->isDistinct && QUERY_NODE_REAL_TABLE == nodeType(pSelect->pFromTable) &&
NULL == pSelect->pGroupByList && NULL == pSelect->pLimit && NULL == pSelect->pSlimit &&
NULL == pSelect->pOrderByList && NULL == pSelect->pPartitionByList) {
return TSDB_CODE_SUCCESS;
}
}
@ -2665,7 +2677,7 @@ static int32_t checkCreateTopic(STranslateContext* pCxt, SCreateTopicStmt* pStmt
static int32_t translateCreateTopic(STranslateContext* pCxt, SCreateTopicStmt* pStmt) {
SCMCreateTopicReq createReq = {0};
int32_t code = checkCreateTopic(pCxt, pStmt);
int32_t code = checkCreateTopic(pCxt, pStmt);
if (TSDB_CODE_SUCCESS == code) {
code = buildCreateTopicReq(pCxt, pStmt, &createReq);
}
@ -2763,7 +2775,7 @@ static int32_t translateDropStream(STranslateContext* pCxt, SDropStreamStmt* pSt
return TSDB_CODE_SUCCESS;
}
static int32_t readFromFile(char* pName, int32_t *len, char **buf) {
static int32_t readFromFile(char* pName, int32_t* len, char** buf) {
int64_t filesize = 0;
if (taosStatFile(pName, &filesize, NULL) < 0) {
return TAOS_SYSTEM_ERROR(errno);
@ -3408,8 +3420,8 @@ static void addCreateTbReqIntoVgroup(int32_t acctId, SHashObj* pVgroupHashmap, c
static int32_t addValToKVRow(STranslateContext* pCxt, SValueNode* pVal, const SSchema* pSchema,
SKVRowBuilder* pBuilder) {
if(pSchema->type == TSDB_DATA_TYPE_JSON){
if(pVal->literal && strlen(pVal->literal) > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE){
if (pSchema->type == TSDB_DATA_TYPE_JSON) {
if (pVal->literal && strlen(pVal->literal) > (TSDB_MAX_JSON_TAG_LEN - VARSTR_HEADER_SIZE) / TSDB_NCHAR_SIZE) {
return buildSyntaxErrMsg(&pCxt->msgBuf, "json string too long than 4095", pVal->literal);
}
@ -3420,10 +3432,11 @@ static int32_t addValToKVRow(STranslateContext* pCxt, SValueNode* pVal, const SS
return pCxt->errCode;
}
if(pVal->node.resType.type == TSDB_DATA_TYPE_NULL){
if (pVal->node.resType.type == TSDB_DATA_TYPE_NULL) {
// todo
}else{
tdAddColToKVRow(pBuilder, pSchema->colId, &(pVal->datum.p), IS_VAR_DATA_TYPE(pSchema->type) ? varDataTLen(pVal->datum.p) : TYPE_BYTES[pSchema->type]);
} else {
tdAddColToKVRow(pBuilder, pSchema->colId, &(pVal->datum.p),
IS_VAR_DATA_TYPE(pSchema->type) ? varDataTLen(pVal->datum.p) : TYPE_BYTES[pSchema->type]);
}
return TSDB_CODE_SUCCESS;
@ -3660,7 +3673,9 @@ static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pQuery->precision = extractResultTsPrecision((SSelectStmt*)pQuery->pRoot);
if (nodeType(pQuery->pRoot) == QUERY_NODE_SELECT_STMT) {
pQuery->precision = extractResultTsPrecision((SSelectStmt*)pQuery->pRoot);
}
}
if (NULL != pCxt->pDbs) {
@ -3692,7 +3707,7 @@ static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) {
int32_t translate(SParseContext* pParseCxt, SQuery* pQuery) {
STranslateContext cxt = {0};
int32_t code = initTranslateContext(pParseCxt, &cxt);
int32_t code = initTranslateContext(pParseCxt, &cxt);
if (TSDB_CODE_SUCCESS == code) {
code = fmFuncMgtInit();
}

16
source/libs/qcom/src/querymsg.c
View File

@ -342,28 +342,20 @@ PROCESS_META_OVER:
int32_t queryProcessQnodeListRsp(void *output, char *msg, int32_t msgSize) {
SQnodeListRsp out = {0};
int32_t code = -1;
int32_t code = 0;
if (NULL == output || NULL == msg || msgSize <= 0) {
code = TSDB_CODE_TSC_INVALID_INPUT;
goto PROCESS_QLIST_OVER;
return code;
}
out.addrsList = (SArray *)output;
if (tDeserializeSQnodeListRsp(msg, msgSize, &out) != 0) {
qError("invalid qnode list rsp msg, msgSize:%d", msgSize);
code = TSDB_CODE_INVALID_MSG;
goto PROCESS_QLIST_OVER;
return code;
}
PROCESS_QLIST_OVER:
if (code != 0) {
tFreeSQnodeListRsp(&out);
out.addrsList = NULL;
}
*(SArray **)output = out.addrsList;
return code;
}

16
source/libs/qworker/src/qworkerMsg.c
View File

@ -46,7 +46,7 @@ int32_t qwBuildAndSendQueryRsp(SQWConnInfo *pConn, int32_t code) {
SQueryTableRsp rsp = {.code = code};
int32_t contLen = tSerializeSQueryTableRsp(NULL, 0, &rsp);
void * msg = rpcMallocCont(contLen);
void *msg = rpcMallocCont(contLen);
tSerializeSQueryTableRsp(msg, contLen, &rsp);
SRpcMsg rpcRsp = {
@ -87,7 +87,7 @@ int32_t qwBuildAndSendExplainRsp(SQWConnInfo *pConn, SExplainExecInfo *execInfo,
SExplainRsp rsp = {.numOfPlans = num, .subplanInfo = execInfo};
int32_t contLen = tSerializeSExplainRsp(NULL, 0, &rsp);
void * pRsp = rpcMallocCont(contLen);
void *pRsp = rpcMallocCont(contLen);
tSerializeSExplainRsp(pRsp, contLen, &rsp);
SRpcMsg rpcRsp = {
@ -107,7 +107,7 @@ int32_t qwBuildAndSendExplainRsp(SQWConnInfo *pConn, SExplainExecInfo *execInfo,
int32_t qwBuildAndSendHbRsp(SQWConnInfo *pConn, SSchedulerHbRsp *pStatus, int32_t code) {
int32_t contLen = tSerializeSSchedulerHbRsp(NULL, 0, pStatus);
void * pRsp = rpcMallocCont(contLen);
void *pRsp = rpcMallocCont(contLen);
tSerializeSSchedulerHbRsp(pRsp, contLen, pStatus);
SRpcMsg rpcRsp = {
@ -223,7 +223,7 @@ int32_t qwBuildAndSendShowRsp(SRpcMsg *pMsg, int32_t code) {
showRsp.tableMeta.numOfColumns = cols;
int32_t bufLen = tSerializeSShowRsp(NULL, 0, &showRsp);
void * pBuf = rpcMallocCont(bufLen);
void *pBuf = rpcMallocCont(bufLen);
tSerializeSShowRsp(pBuf, bufLen, &showRsp);
SRpcMsg rpcMsg = {
@ -403,8 +403,8 @@ int32_t qWorkerProcessCQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
bool queryDone = false;
SQueryContinueReq *msg = (SQueryContinueReq *)pMsg->pCont;
bool needStop = false;
SQWTaskCtx * handles = NULL;
SQWorkerMgmt * mgmt = (SQWorkerMgmt *)qWorkerMgmt;
SQWTaskCtx *handles = NULL;
SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt;
if (NULL == msg || pMsg->contLen < sizeof(*msg)) {
QW_ELOG("invalid cquery msg, msg:%p, msgLen:%d", msg, pMsg->contLen);
@ -538,7 +538,7 @@ int32_t qWorkerProcessCancelMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
return TSDB_CODE_QRY_INVALID_INPUT;
}
SQWorkerMgmt * mgmt = (SQWorkerMgmt *)qWorkerMgmt;
SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt;
int32_t code = 0;
STaskCancelReq *msg = pMsg->pCont;
if (NULL == msg || pMsg->contLen < sizeof(*msg)) {
@ -620,7 +620,7 @@ int32_t qWorkerProcessHbMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) {
int32_t code = 0;
SSchedulerHbReq req = {0};
SQWorkerMgmt * mgmt = (SQWorkerMgmt *)qWorkerMgmt;
SQWorkerMgmt *mgmt = (SQWorkerMgmt *)qWorkerMgmt;
if (NULL == pMsg->pCont) {
QW_ELOG("invalid hb msg, msg:%p, msgLen:%d", pMsg->pCont, pMsg->contLen);

3
source/libs/scalar/inc/sclInt.h
View File

@ -32,6 +32,9 @@ typedef struct SScalarCtx {
#define SCL_DATA_TYPE_DUMMY_HASH 9000
#define SCL_DEFAULT_OP_NUM 10
#define SCL_IS_CONST_NODE(_node) ((NULL == (_node)) || (QUERY_NODE_VALUE == (_node)->type) || (QUERY_NODE_NODE_LIST == (_node)->type))
#define SCL_IS_CONST_CALC(_ctx) (NULL == (_ctx)->pBlockList)
#define sclFatal(...) qFatal(__VA_ARGS__)
#define sclError(...) qError(__VA_ARGS__)
#define sclWarn(...) qWarn(__VA_ARGS__)

158
source/libs/scalar/src/scalar.c
View File

@ -244,23 +244,53 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t
return TSDB_CODE_SUCCESS;
}
int32_t sclInitParamList(SScalarParam **pParams, SNodeList* pParamList, SScalarCtx *ctx, int32_t *rowNum) {
int32_t sclInitParamList(SScalarParam **pParams, SNodeList* pParamList, SScalarCtx *ctx, int32_t *paramNum, int32_t *rowNum) {
int32_t code = 0;
SScalarParam *paramList = taosMemoryCalloc(pParamList->length, sizeof(SScalarParam));
if (NULL == pParamList) {
if (ctx->pBlockList) {
SSDataBlock *pBlock = taosArrayGet(ctx->pBlockList, 0);
*rowNum = pBlock->info.rows;
} else {
*rowNum = 1;
}
*paramNum = 1;
} else {
*paramNum = pParamList->length;
}
SScalarParam *paramList = taosMemoryCalloc(*paramNum, sizeof(SScalarParam));
if (NULL == paramList) {
sclError("calloc %d failed", (int32_t)(pParamList->length * sizeof(SScalarParam)));
sclError("calloc %d failed", (int32_t)((*paramNum) * sizeof(SScalarParam)));
SCL_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
SListCell *cell = pParamList->pHead;
for (int32_t i = 0; i < pParamList->length; ++i) {
if (NULL == cell || NULL == cell->pNode) {
sclError("invalid cell, cell:%p, pNode:%p", cell, cell->pNode);
SCL_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT);
if (pParamList) {
SNode *tnode = NULL;
int32_t i = 0;
if (SCL_IS_CONST_CALC(ctx)) {
WHERE_EACH (tnode, pParamList) {
if (!SCL_IS_CONST_NODE(tnode)) {
WHERE_NEXT;
} else {
SCL_ERR_JRET(sclInitParam(tnode, &paramList[i], ctx, rowNum));
ERASE_NODE(pParamList);
}
++i;
}
} else {
FOREACH(tnode, pParamList) {
SCL_ERR_JRET(sclInitParam(tnode, &paramList[i], ctx, rowNum));
++i;
}
}
} else {
paramList[0].numOfRows = *rowNum;
}
SCL_ERR_JRET(sclInitParam(cell->pNode, &paramList[i], ctx, rowNum));
cell = cell->pNext;
if (0 == *rowNum) {
taosMemoryFreeClear(paramList);
}
*pParams = paramList;
@ -299,37 +329,45 @@ _return:
}
int32_t sclExecFunction(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *output) {
if (NULL == node->pParameterList || node->pParameterList->length <= 0) {
sclError("invalid function parameter list, list:%p, paramNum:%d", node->pParameterList, node->pParameterList ? node->pParameterList->length : 0);
SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT);
}
SScalarFuncExecFuncs ffpSet = {0};
int32_t code = fmGetScalarFuncExecFuncs(node->funcId, &ffpSet);
if (code) {
sclError("fmGetFuncExecFuncs failed, funcId:%d, code:%s", node->funcId, tstrerror(code));
SCL_ERR_RET(code);
}
SScalarParam *params = NULL;
int32_t rowNum = 0;
SCL_ERR_RET(sclInitParamList(&params, node->pParameterList, ctx, &rowNum));
int32_t paramNum = 0;
int32_t code = 0;
SCL_ERR_RET(sclInitParamList(&params, node->pParameterList, ctx, &paramNum, &rowNum));
output->columnData = createColumnInfoData(&node->node.resType, rowNum);
if (output->columnData == NULL) {
sclError("calloc %d failed", (int32_t)(rowNum * output->columnData->info.bytes));
SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
code = (*ffpSet.process)(params, node->pParameterList->length, output);
if (code) {
sclError("scalar function exec failed, funcId:%d, code:%s", node->funcId, tstrerror(code));
if (fmIsUserDefinedFunc(node->funcId)) {
#if 0
UdfcFuncHandle udfHandle = NULL;
SCL_ERR_JRET(setupUdf(node->functionName, &udfHandle));
code = callUdfScalarFunc(udfHandle, params, paramNum, output);
teardownUdf(udfHandle);
SCL_ERR_JRET(code);
#endif
} else {
SScalarFuncExecFuncs ffpSet = {0};
code = fmGetScalarFuncExecFuncs(node->funcId, &ffpSet);
if (code) {
sclError("fmGetFuncExecFuncs failed, funcId:%d, code:%s", node->funcId, tstrerror(code));
SCL_ERR_JRET(code);
}
output->columnData = createColumnInfoData(&node->node.resType, rowNum);
if (output->columnData == NULL) {
sclError("calloc %d failed", (int32_t)(rowNum * output->columnData->info.bytes));
SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
code = (*ffpSet.process)(params, paramNum, output);
if (code) {
sclError("scalar function exec failed, funcId:%d, code:%s", node->funcId, tstrerror(code));
SCL_ERR_JRET(code);
}
}
_return:
for (int32_t i = 0; i < node->pParameterList->length; ++i) {
for (int32_t i = 0; i < paramNum; ++i) {
// sclFreeParamNoData(params + i);
}
@ -355,8 +393,13 @@ int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *o
SScalarParam *params = NULL;
int32_t rowNum = 0;
int32_t paramNum = 0;
int32_t code = 0;
SCL_ERR_RET(sclInitParamList(&params, node->pParameterList, ctx, &rowNum));
SCL_ERR_RET(sclInitParamList(&params, node->pParameterList, ctx, &paramNum, &rowNum));
if (NULL == params) {
output->numOfRows = 0;
return TSDB_CODE_SUCCESS;
}
int32_t type = node->node.resType.type;
output->numOfRows = rowNum;
@ -369,25 +412,41 @@ int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *o
}
bool value = false;
bool complete = true;
for (int32_t i = 0; i < rowNum; ++i) {
for (int32_t m = 0; m < node->pParameterList->length; ++m) {
complete = true;
for (int32_t m = 0; m < paramNum; ++m) {
if (NULL == params[m].columnData) {
complete = false;
continue;
}
char* p = colDataGetData(params[m].columnData, i);
GET_TYPED_DATA(value, bool, params[m].columnData->info.type, p);
if (LOGIC_COND_TYPE_AND == node->condType && (false == value)) {
complete = true;
break;
} else if (LOGIC_COND_TYPE_OR == node->condType && value) {
complete = true;
break;
} else if (LOGIC_COND_TYPE_NOT == node->condType) {
value = !value;
}
}
colDataAppend(output->columnData, i, (char*) &value, false);
if (complete) {
colDataAppend(output->columnData, i, (char*) &value, false);
}
}
if (SCL_IS_CONST_CALC(ctx) && (false == complete)) {
sclFreeParam(output);
output->numOfRows = 0;
}
_return:
for (int32_t i = 0; i < node->pParameterList->length; ++i) {
for (int32_t i = 0; i < paramNum; ++i) {
// sclFreeParamNoData(params + i);
}
@ -426,6 +485,17 @@ _return:
EDealRes sclRewriteFunction(SNode** pNode, SScalarCtx *ctx) {
SFunctionNode *node = (SFunctionNode *)*pNode;
SNode* tnode = NULL;
if (fmIsUserDefinedFunc(node->funcId)) {
return DEAL_RES_CONTINUE;
}
FOREACH(tnode, node->pParameterList) {
if (!SCL_IS_CONST_NODE(tnode)) {
return DEAL_RES_CONTINUE;
}
}
SScalarParam output = {0};
ctx->code = sclExecFunction(node, ctx, &output);
@ -470,6 +540,10 @@ EDealRes sclRewriteLogic(SNode** pNode, SScalarCtx *ctx) {
return DEAL_RES_ERROR;
}
if (0 == output.numOfRows) {
return DEAL_RES_CONTINUE;
}
SValueNode *res = (SValueNode *)nodesMakeNode(QUERY_NODE_VALUE);
if (NULL == res) {
sclError("make value node failed");
@ -498,6 +572,14 @@ EDealRes sclRewriteLogic(SNode** pNode, SScalarCtx *ctx) {
EDealRes sclRewriteOperator(SNode** pNode, SScalarCtx *ctx) {
SOperatorNode *node = (SOperatorNode *)*pNode;
if (!SCL_IS_CONST_NODE(node->pLeft)) {
return DEAL_RES_CONTINUE;
}
if (!SCL_IS_CONST_NODE(node->pRight)) {
return DEAL_RES_CONTINUE;
}
SScalarParam output = {.columnData = taosMemoryCalloc(1, sizeof(SColumnInfoData))};
ctx->code = sclExecOperator(node, ctx, &output);
if (ctx->code) {
@ -530,7 +612,7 @@ EDealRes sclRewriteOperator(SNode** pNode, SScalarCtx *ctx) {
}
EDealRes sclConstantsRewriter(SNode** pNode, void* pContext) {
if (QUERY_NODE_VALUE == nodeType(*pNode) || QUERY_NODE_NODE_LIST == nodeType(*pNode)) {
if (QUERY_NODE_VALUE == nodeType(*pNode) || QUERY_NODE_COLUMN == nodeType(*pNode) || QUERY_NODE_NODE_LIST == nodeType(*pNode)) {
return DEAL_RES_CONTINUE;
}

116
source/libs/scalar/test/scalar/scalarTests.cpp
View File

@ -137,6 +137,11 @@ void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in
rnode->node.resType.bytes = dataBytes;
rnode->dataBlockId = 0;
if (NULL == block) {
*pNode = (SNode *)rnode;
return;
}
if (NULL == *block) {
SSDataBlock *res = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock));
res->info.numOfCols = 3;
@ -889,6 +894,8 @@ TEST(constantTest, int_greater_int_is_true2) {
}
TEST(constantTest, greater_and_lower) {
scltInitLogFile();
SNode *pval1 = NULL, *pval2 = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode = NULL, *res = NULL;
bool eRes[5] = {false, false, true, true, true};
int64_t v1 = 333, v2 = 222, v3 = -10, v4 = 20;
@ -913,6 +920,115 @@ TEST(constantTest, greater_and_lower) {
nodesDestroyNode(res);
}
TEST(constantTest, column_and_value1) {
scltInitLogFile();
SNode *pval1 = NULL, *pval2 = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode = NULL, *res = NULL;
bool eRes[5] = {false, false, true, true, true};
int64_t v1 = 333, v2 = 222, v3 = -10, v4 = 20;
SNode *list[2] = {0};
scltMakeValueNode(&pval1, TSDB_DATA_TYPE_BIGINT, &v1);
scltMakeValueNode(&pval2, TSDB_DATA_TYPE_BIGINT, &v2);
scltMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pval1, pval2);
scltMakeValueNode(&pval1, TSDB_DATA_TYPE_BIGINT, &v3);
scltMakeColumnNode(&pval2, NULL, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), 0, NULL);
scltMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pval1, pval2);
list[0] = opNode1;
list[1] = opNode2;
scltMakeLogicNode(&logicNode, LOGIC_COND_TYPE_AND, list, 2);
int32_t code = scalarCalculateConstants(logicNode, &res);
ASSERT_EQ(code, 0);
ASSERT_TRUE(res);
ASSERT_EQ(nodeType(res), QUERY_NODE_LOGIC_CONDITION);
SLogicConditionNode *v = (SLogicConditionNode *)res;
ASSERT_EQ(v->condType, LOGIC_COND_TYPE_AND);
ASSERT_EQ(v->pParameterList->length, 1);
nodesDestroyNode(res);
}
TEST(constantTest, column_and_value2) {
scltInitLogFile();
SNode *pval1 = NULL, *pval2 = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode = NULL, *res = NULL;
bool eRes[5] = {false, false, true, true, true};
int64_t v1 = 333, v2 = 222, v3 = -10, v4 = 20;
SNode *list[2] = {0};
scltMakeValueNode(&pval1, TSDB_DATA_TYPE_BIGINT, &v1);
scltMakeValueNode(&pval2, TSDB_DATA_TYPE_BIGINT, &v2);
scltMakeOpNode(&opNode1, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pval1, pval2);
scltMakeValueNode(&pval1, TSDB_DATA_TYPE_BIGINT, &v3);
scltMakeColumnNode(&pval2, NULL, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), 0, NULL);
scltMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pval1, pval2);
list[0] = opNode1;
list[1] = opNode2;
scltMakeLogicNode(&logicNode, LOGIC_COND_TYPE_AND, list, 2);
int32_t code = scalarCalculateConstants(logicNode, &res);
ASSERT_EQ(code, 0);
ASSERT_TRUE(res);
ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE);
SValueNode *v = (SValueNode *)res;
ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL);
ASSERT_EQ(v->datum.b, false);
nodesDestroyNode(res);
}
TEST(constantTest, column_and_value3) {
scltInitLogFile();
SNode *pval1 = NULL, *pval2 = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode = NULL, *res = NULL;
bool eRes[5] = {false, false, true, true, true};
int64_t v1 = 333, v2 = 222, v3 = -10, v4 = 20;
SNode *list[2] = {0};
scltMakeValueNode(&pval1, TSDB_DATA_TYPE_BIGINT, &v1);
scltMakeValueNode(&pval2, TSDB_DATA_TYPE_BIGINT, &v2);
scltMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pval1, pval2);
scltMakeValueNode(&pval1, TSDB_DATA_TYPE_BIGINT, &v3);
scltMakeColumnNode(&pval2, NULL, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), 0, NULL);
scltMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pval1, pval2);
list[0] = opNode1;
list[1] = opNode2;
scltMakeLogicNode(&logicNode, LOGIC_COND_TYPE_OR, list, 2);
int32_t code = scalarCalculateConstants(logicNode, &res);
ASSERT_EQ(code, 0);
ASSERT_TRUE(res);
ASSERT_EQ(nodeType(res), QUERY_NODE_VALUE);
SValueNode *v = (SValueNode *)res;
ASSERT_EQ(v->node.resType.type, TSDB_DATA_TYPE_BOOL);
ASSERT_EQ(v->datum.b, true);
nodesDestroyNode(res);
}
TEST(constantTest, column_and_value4) {
scltInitLogFile();
SNode *pval1 = NULL, *pval2 = NULL, *opNode1 = NULL, *opNode2 = NULL, *logicNode = NULL, *res = NULL;
bool eRes[5] = {false, false, true, true, true};
int64_t v1 = 333, v2 = 222, v3 = -10, v4 = 20;
SNode *list[2] = {0};
scltMakeValueNode(&pval1, TSDB_DATA_TYPE_BIGINT, &v1);
scltMakeValueNode(&pval2, TSDB_DATA_TYPE_BIGINT, &v2);
scltMakeOpNode(&opNode1, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pval1, pval2);
scltMakeValueNode(&pval1, TSDB_DATA_TYPE_BIGINT, &v3);
scltMakeColumnNode(&pval2, NULL, TSDB_DATA_TYPE_BIGINT, sizeof(int64_t), 0, NULL);
scltMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pval1, pval2);
list[0] = opNode1;
list[1] = opNode2;
scltMakeLogicNode(&logicNode, LOGIC_COND_TYPE_OR, list, 2);
int32_t code = scalarCalculateConstants(logicNode, &res);
ASSERT_EQ(code, 0);
ASSERT_TRUE(res);
ASSERT_EQ(nodeType(res), QUERY_NODE_LOGIC_CONDITION);
SLogicConditionNode *v = (SLogicConditionNode *)res;
ASSERT_EQ(v->condType, LOGIC_COND_TYPE_OR);
ASSERT_EQ(v->pParameterList->length, 1);
nodesDestroyNode(res);
}
void makeJsonArrow(SSDataBlock **src, SNode **opNode, void *json, char *key){
char keyVar[32] = {0};
memcpy(varDataVal(keyVar), key, strlen(key));

87
source/libs/wal/src/walRead.c
View File

@ -138,6 +138,91 @@ static int32_t walReadSeekVer(SWalReadHandle *pRead, int64_t ver) {
return 0;
}
void walSetReaderCapacity(SWalReadHandle *pRead, int32_t capacity) { pRead->capacity = capacity; }
int32_t walFetchHead(SWalReadHandle *pRead, int64_t ver, SWalHead *pHead) {
int32_t code;
// TODO: valid ver
if (pRead->curVersion != ver) {
code = walReadSeekVer(pRead, ver);
if (code < 0) return -1;
}
if (!taosValidFile(pRead->pReadLogTFile)) {
return -1;
}
code = taosReadFile(pRead->pReadLogTFile, pHead, sizeof(SWalHead));
if (code != sizeof(SWalHead)) {
return -1;
}
code = walValidHeadCksum(pHead);
if (code != 0) {
wError("unexpected wal log version: % " PRId64 ", since head checksum not passed", ver);
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
return -1;
}
return 0;
}
int32_t walSkipFetchBody(SWalReadHandle *pRead, const SWalHead *pHead) {
int32_t code;
ASSERT(pRead->curVersion == pHead->head.version);
code = taosLSeekFile(pRead->pReadLogTFile, pHead->head.bodyLen, SEEK_CUR);
if (code < 0) {
terrno = TAOS_SYSTEM_ERROR(errno);
pRead->curVersion = -1;
return -1;
}
pRead->curVersion++;
return 0;
}
int32_t walFetchBody(SWalReadHandle *pRead, SWalHead **ppHead) {
SWalReadHead *pReadHead = &((*ppHead)->head);
int64_t ver = pReadHead->version;
if (pRead->capacity < pReadHead->bodyLen) {
void *ptr = taosMemoryRealloc(*ppHead, sizeof(SWalHead) + pReadHead->bodyLen);
if (ptr == NULL) {
terrno = TSDB_CODE_WAL_OUT_OF_MEMORY;
return -1;
}
*ppHead = ptr;
pRead->capacity = pReadHead->bodyLen;
}
if (pReadHead->bodyLen != taosReadFile(pRead->pReadLogTFile, pReadHead->body, pReadHead->bodyLen)) {
return -1;
}
if (pReadHead->version != ver) {
wError("unexpected wal log version: %" PRId64 ", read request version:%" PRId64 "", pRead->pHead->head.version,
ver);
pRead->curVersion = -1;
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
return -1;
}
if (walValidBodyCksum(*ppHead) != 0) {
wError("unexpected wal log version: % " PRId64 ", since body checksum not passed", ver);
pRead->curVersion = -1;
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
return -1;
}
pRead->curVersion = ver + 1;
return 0;
}
int32_t walReadWithHandle_s(SWalReadHandle *pRead, int64_t ver, SWalReadHead **ppHead) {
taosThreadMutexLock(&pRead->mutex);
if (walReadWithHandle(pRead, ver) < 0) {
@ -172,12 +257,14 @@ int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) {
if (code != sizeof(SWalHead)) {
return -1;
}
code = walValidHeadCksum(pRead->pHead);
if (code != 0) {
wError("unexpected wal log version: % " PRId64 ", since head checksum not passed", ver);
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
return -1;
}
if (pRead->capacity < pRead->pHead->head.bodyLen) {
void *ptr = taosMemoryRealloc(pRead->pHead, sizeof(SWalHead) + pRead->pHead->head.bodyLen);
if (ptr == NULL) {

8
source/libs/wal/src/walWrite.c
View File

@ -13,8 +13,6 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "os.h"
#include "taoserror.h"
#include "tchecksum.h"
@ -298,14 +296,14 @@ int64_t walWriteWithSyncInfo(SWal *pWal, int64_t index, tmsg_t msgType, SSyncLog
pWal->writeHead.head.bodyLen = bodyLen;
pWal->writeHead.head.msgType = msgType;
// sync info
// sync info for sync module
pWal->writeHead.head.syncMeta = syncMeta;
pWal->writeHead.cksumHead = walCalcHeadCksum(&pWal->writeHead);
pWal->writeHead.cksumBody = walCalcBodyCksum(body, bodyLen);
if (taosWriteFile(pWal->pWriteLogTFile, &pWal->writeHead, sizeof(SWalHead)) != sizeof(SWalHead)) {
// ftruncate
// TODO ftruncate
terrno = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal),
strerror(errno));
@ -313,7 +311,7 @@ int64_t walWriteWithSyncInfo(SWal *pWal, int64_t index, tmsg_t msgType, SSyncLog
}
if (taosWriteFile(pWal->pWriteLogTFile, (char *)body, bodyLen) != bodyLen) {
// ftruncate
// TODO ftruncate
terrno = TAOS_SYSTEM_ERROR(errno);
wError("vgId:%d, file:%" PRId64 ".log, failed to write since %s", pWal->cfg.vgId, walGetLastFileFirstVer(pWal),
strerror(errno));

1
source/util/src/tarray.c
View File

@ -476,6 +476,7 @@ void* taosDecodeArray(const void* buf, SArray** pArray, FDecode decode, int32_t
return (void*)buf;
}
// todo remove it
// order array<type *>
void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* param) {
taosArrayGetSize(pArray) > 8 ? taosArrayQuickSort(pArray, fn, param) : taosArrayInsertSort(pArray, fn, param);

2
source/util/src/terror.c
View File

@ -258,7 +258,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_COLUMN_ALREADY_EXIST, "Column already exists
TAOS_DEFINE_ERROR(TSDB_CODE_MND_COLUMN_NOT_EXIST, "Column does not exist")
// mnode-infoSchema
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_INFOS_TBL, "Invalid information schema table name")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_SYS_TABLENAME, "Invalid system table name")
// mnode-func

22
tests/script/tsim/parser/groupby-basic.sim
View File

@ -115,19 +115,19 @@ endi
if $data00 != 10 then
return -1
endi
if $data01 != 0 then
return -1
endi
if $data10 != 10 then
return -1
endi
if $data11 != 1 then
return -1
endi
if $data90 != 10 then
return -1
endi
if $data91 != 9 then
if $data01 != 7 then
return -1
endi
if $data11 != 6 then
return -1
endi
if $data91 != 3 then
return -1
endi
@ -143,16 +143,16 @@ if $row != 10 then
return -1
endi
if $data00 != @22-01-01 00:00:00.000@ then
if $data00 != @22-01-01 00:00:00.007@ then
return -1
endi
if $data01 != 0 then
if $data01 != 7 then
return -1
endi
if $data90 != @22-01-01 00:00:00.009@ then
if $data90 != @22-01-01 00:00:00.003@ then
return -1
endi
if $data91 != 9 then
if $data91 != 3 then
return -1
endi

1
tests/script/tsim/stable/disk.sim
View File

@ -53,6 +53,7 @@ endi
sql select count(tbcol) from $mt
print select count(tbcol) from $mt ===> $data00
if $data00 != $totalNum then
print expect $totalNum , actual: $data00
return -1
endi

2
tests/system-test/fulltest.sh
View File

@ -4,4 +4,4 @@ set -e
#python3 ./test.py -f 2-query/between.py
#python3 ./test.py -f 2-query/distinct.py
python3 ./test.py -f 2-query/varchar.py
python3 ./test.py -f 2-query/cast.py
#python3 ./test.py -f 2-query/cast.py

10
tools/shell/src/shellNettest.c
View File

@ -67,7 +67,7 @@ static void shellWorkAsClient() {
printf("request is sent, size:%d\n", rpcMsg.contLen);
rpcSendRecv(clientRpc, &epSet, &rpcMsg, &rpcRsp);
if (rpcRsp.code == 0 &&rpcRsp.contLen == rpcMsg.contLen) {
if (rpcRsp.code == 0 && rpcRsp.contLen == rpcMsg.contLen) {
printf("response is received, size:%d\n", rpcMsg.contLen);
if (rpcRsp.code == 0) totalSucc++;
} else {
@ -97,8 +97,12 @@ static void shellProcessMsg(void *p, SRpcMsg *pRpc, SEpSet *pEpSet) {
printf("request is received, size:%d\n", pRpc->contLen);
fflush(stdout);
SRpcMsg rsp = {.handle = pRpc->handle, .refId = pRpc->refId, .ahandle = pRpc->ahandle, .code = 0};
rsp.pCont = rpcMallocCont(shell.args.pktLen);
rsp.contLen = shell.args.pktLen;
rsp.pCont = rpcMallocCont(pRpc->contLen);
if (rsp.pCont == NULL) {
rsp.code = TSDB_CODE_OUT_OF_MEMORY;
} else {
rsp.contLen = pRpc->contLen;
}
rpcSendResponse(&rsp);
}