xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/3.0' into fix/refactorTqBackend

This commit is contained in:
yihaoDeng 2024-07-29 08:50:09 +08:00
parent eb7fde9d98 5e51e4149a
commit f58fabb15b
111 changed files with 2928 additions and 1437 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
include/common/tdatablock.h
View File

@ -41,6 +41,15 @@ typedef struct SBlockOrderInfo {
#define BMCharPos(bm_, r_) ((bm_)[(r_) >> NBIT])
#define colDataIsNull_f(bm_, r_) ((BMCharPos(bm_, r_) & (1u << (7u - BitPos(r_)))) == (1u << (7u - BitPos(r_))))
#define QRY_OPTR_CHECK(_o) \
do { \
if ((_o) == NULL) { \
return TSDB_CODE_INVALID_PARA; \
} else { \
*(_o) = NULL; \
} \
} while(0)
#define colDataSetNull_f(bm_, r_) \
do { \
BMCharPos(bm_, r_) |= (1u << (7u - BitPos(r_))); \
@ -222,8 +231,7 @@ int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startInd
int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock);
int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf);
int32_t blockDataFromBuf1(SSDataBlock* pBlock, const char* buf, size_t capacity);
SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int32_t rowCount);
int32_t blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int32_t rowCount, SSDataBlock** pResBlock);
size_t blockDataGetSize(const SSDataBlock* pBlock);
size_t blockDataGetRowSize(SSDataBlock* pBlock);
@ -254,24 +262,24 @@ void blockDataKeepFirstNRows(SSDataBlock* pBlock, size_t n);
int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src);
int32_t copyDataBlock(SSDataBlock* pDst, const SSDataBlock* pSrc);
SSDataBlock* createDataBlock();
void blockDataDestroy(SSDataBlock* pBlock);
void blockDataFreeRes(SSDataBlock* pBlock);
SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData);
SSDataBlock* createSpecialDataBlock(EStreamType type);
int32_t createDataBlock(SSDataBlock** pResBlock);
void blockDataDestroy(SSDataBlock* pBlock);
void blockDataFreeRes(SSDataBlock* pBlock);
int32_t createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData, SSDataBlock** pResBlock);
int32_t createSpecialDataBlock(EStreamType type, SSDataBlock** pBlock);
SSDataBlock* blockCopyOneRow(const SSDataBlock* pDataBlock, int32_t rowIdx);
int32_t blockDataAppendColInfo(SSDataBlock* pBlock, SColumnInfoData* pColInfoData);
int32_t blockCopyOneRow(const SSDataBlock* pDataBlock, int32_t rowIdx, SSDataBlock** pResBlock);
int32_t blockDataAppendColInfo(SSDataBlock* pBlock, SColumnInfoData* pColInfoData);
SColumnInfoData createColumnInfoData(int16_t type, int32_t bytes, int16_t colId);
SColumnInfoData* bdGetColumnInfoData(const SSDataBlock* pBlock, int32_t index);
SColumnInfoData createColumnInfoData(int16_t type, int32_t bytes, int16_t colId);
int32_t bdGetColumnInfoData(const SSDataBlock* pBlock, int32_t index, SColumnInfoData** pColInfoData);
int32_t blockGetEncodeSize(const SSDataBlock* pBlock);
int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols);
const char* blockDecode(SSDataBlock* pBlock, const char* pData);
int32_t blockGetEncodeSize(const SSDataBlock* pBlock);
int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols);
int32_t blockDecode(SSDataBlock* pBlock, const char* pData, const char** pEndPos);
// for debug
char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** dumpBuf, const char* taskIdStr);
int32_t dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** dumpBuf, const char* taskIdStr);
int32_t buildSubmitReqFromDataBlock(SSubmitReq2** pReq, const SSDataBlock* pDataBlocks, const STSchema* pTSchema,
int64_t uid, int32_t vgId, tb_uid_t suid);
@ -279,10 +287,10 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq2** pReq, const SSDataBlock* pData
bool alreadyAddGroupId(char* ctbName, int64_t groupId);
bool isAutoTableName(char* ctbName);
void buildCtbNameAddGroupId(const char* stbName, char* ctbName, uint64_t groupId);
char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId);
int32_t buildCtbNameByGroupId(const char* stbName, uint64_t groupId, char** pName);
int32_t buildCtbNameByGroupIdImpl(const char* stbName, uint64_t groupId, char* pBuf);
void trimDataBlock(SSDataBlock* pBlock, int32_t totalRows, const bool* pBoolList);
int32_t trimDataBlock(SSDataBlock* pBlock, int32_t totalRows, const bool* pBoolList);
void copyPkVal(SDataBlockInfo* pDst, const SDataBlockInfo* pSrc);

9
include/common/tmsg.h
View File

@ -2899,7 +2899,6 @@ static FORCE_INLINE SMqRebInfo* tNewSMqRebSubscribe(const char* key) {
}
pRebInfo->newConsumers = taosArrayInit(0, sizeof(int64_t));
if (pRebInfo->newConsumers == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
return pRebInfo;
@ -3455,7 +3454,9 @@ static FORCE_INLINE void* taosDecodeSMqTopicInfoMsg(void* buf, SMqTopicInfo* pTo
buf = taosDecodeStringTo(buf, pTopicInfo->name);
int32_t sz;
buf = taosDecodeFixedI32(buf, &sz);
pTopicInfo->pVgInfo = taosArrayInit(sz, sizeof(SMqReportVgInfo));
if ((pTopicInfo->pVgInfo = taosArrayInit(sz, sizeof(SMqReportVgInfo))) == NULL) {
return NULL;
}
for (int32_t i = 0; i < sz; i++) {
SMqReportVgInfo vgInfo;
buf = taosDecodeSMqVgInfo(buf, &vgInfo);
@ -3493,7 +3494,9 @@ static FORCE_INLINE void* taosDecodeSMqReportMsg(void* buf, SMqReportReq* pMsg)
buf = taosDecodeFixedI64(buf, &pMsg->consumerId);
int32_t sz;
buf = taosDecodeFixedI32(buf, &sz);
pMsg->pTopics = taosArrayInit(sz, sizeof(SMqTopicInfo));
if ((pMsg->pTopics = taosArrayInit(sz, sizeof(SMqTopicInfo))) == NULL) {
return NULL;
}
for (int32_t i = 0; i < sz; i++) {
SMqTopicInfo topicInfo;
buf = taosDecodeSMqTopicInfoMsg(buf, &topicInfo);

2
include/common/ttime.h
View File

@ -64,7 +64,7 @@ static FORCE_INLINE int64_t taosGetTimestampToday(int32_t precision) {
: 1000000000;
time_t t = taosTime(NULL);
struct tm tm;
taosLocalTime(&t, &tm, NULL);
(void) taosLocalTime(&t, &tm, NULL);
tm.tm_hour = 0;
tm.tm_min = 0;
tm.tm_sec = 0;

24
include/libs/function/taosudf.h
View File

@ -33,6 +33,15 @@ extern "C" {
#else
#define FORCE_INLINE
#endif
#define TAOS_UDF_CHECK_RETURN(CMD) \
do { \
int32_t code = (CMD); \
if (code != TSDB_CODE_SUCCESS) { \
return (CMD); \
} \
} while (0)
typedef struct SUdfColumnMeta {
int16_t type;
int32_t bytes;
@ -192,25 +201,28 @@ static FORCE_INLINE int32_t udfColEnsureCapacity(SUdfColumn *pColumn, int32_t ne
return TSDB_CODE_SUCCESS;
}
static FORCE_INLINE void udfColDataSetNull(SUdfColumn *pColumn, int32_t row) {
udfColEnsureCapacity(pColumn, row + 1);
static FORCE_INLINE int32_t udfColDataSetNull(SUdfColumn *pColumn, int32_t row) {
int32_t code = udfColEnsureCapacity(pColumn, row + 1);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
if (IS_VAR_DATA_TYPE(pColumn->colMeta.type)) {
udfColDataSetNull_var(pColumn, row);
} else {
udfColDataSetNull_f(pColumn, row);
}
pColumn->hasNull = true;
pColumn->colData.numOfRows =
((int32_t)(row + 1) > pColumn->colData.numOfRows) ? (int32_t)(row + 1) : pColumn->colData.numOfRows;
pColumn->colData.numOfRows = ((int32_t)(row + 1) > pColumn->colData.numOfRows) ? (int32_t)(row + 1) : pColumn->colData.numOfRows;
return 0;
}
static FORCE_INLINE int32_t udfColDataSet(SUdfColumn *pColumn, uint32_t currentRow, const char *pData, bool isNull) {
SUdfColumnMeta *meta = &pColumn->colMeta;
SUdfColumnData *data = &pColumn->colData;
udfColEnsureCapacity(pColumn, currentRow + 1);
TAOS_UDF_CHECK_RETURN(udfColEnsureCapacity(pColumn, currentRow + 1));
bool isVarCol = IS_VAR_DATA_TYPE(meta->type);
if (isNull) {
udfColDataSetNull(pColumn, currentRow);
TAOS_UDF_CHECK_RETURN(udfColDataSetNull(pColumn, currentRow));
} else {
if (!isVarCol) {
udfColDataSetNotNull_f(pColumn, currentRow);

23
include/libs/function/tudf.h
View File

@ -43,6 +43,25 @@ extern "C" {
#endif
#define UDF_DNODE_ID_ENV_NAME "DNODE_ID"
#define TAOS_UV_LIB_ERROR_RET(ret) \
do { \
if (0 != ret) { \
terrno = TSDB_CODE_UDF_UV_EXEC_FAILURE; \
return TSDB_CODE_UDF_UV_EXEC_FAILURE; \
} \
} while(0)
#define TAOS_UV_CHECK_ERRNO(CODE) \
do { \
if (0 != CODE) { \
terrln = __LINE__; \
terrno = (CODE); \
goto _exit; \
} \
} while (0)
// low level APIs
/**
* setup udf
@ -109,13 +128,13 @@ int32_t udfStartUdfd(int32_t startDnodeId);
* stop udfd
* @return
*/
int32_t udfStopUdfd();
void udfStopUdfd();
/**
* get udfd pid
*
*/
int32_t udfGetUdfdPid(int32_t* pUdfdPid);
// int32_t udfGetUdfdPid(int32_t* pUdfdPid);
#ifdef __cplusplus
}

2
include/os/osDir.h
View File

@ -79,6 +79,8 @@ extern "C" {
typedef struct TdDir *TdDirPtr;
typedef struct TdDirEntry *TdDirEntryPtr;
#define TAOS_DIRNAME(name) ((void)taosDirName(name))
void taosRemoveDir(const char *dirname);
bool taosDirExist(const char *dirname);
int32_t taosMkDir(const char *dirname);

2
include/os/osSysinfo.h
View File

@ -39,7 +39,7 @@ int64_t taosGetOsUptime();
int32_t taosGetEmail(char *email, int32_t maxLen);
int32_t taosGetOsReleaseName(char *releaseName, char* sName, char* ver, int32_t maxLen);
int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores);
int32_t taosGetCpuCores(float *numOfCores, bool physical);
void taosGetCpuCores(float *numOfCores, bool physical);
void taosGetCpuUsage(double *cpu_system, double *cpu_engine);
int32_t taosGetCpuInstructions(char* sse42, char* avx, char* avx2, char* fma, char* avx512);
int32_t taosGetTotalMemory(int64_t *totalKB);

1
include/util/taoserror.h
View File

@ -886,6 +886,7 @@ int32_t taosGetErrSize();
#define TSDB_CODE_UDF_INVALID_OUTPUT_TYPE TAOS_DEF_ERROR_CODE(0, 0x2908)
#define TSDB_CODE_UDF_SCRIPT_NOT_SUPPORTED TAOS_DEF_ERROR_CODE(0, 0x2909)
#define TSDB_CODE_UDF_FUNC_EXEC_FAILURE TAOS_DEF_ERROR_CODE(0, 0x290A)
#define TSDB_CODE_UDF_UV_EXEC_FAILURE TAOS_DEF_ERROR_CODE(0, 0x290B)
// sml
#define TSDB_CODE_SML_INVALID_PROTOCOL_TYPE TAOS_DEF_ERROR_CODE(0, 0x3000)

6
include/util/tbuffer.h
View File

@ -27,9 +27,9 @@ typedef struct SBufferReader SBufferReader;
// SBuffer
#define BUFFER_INITIALIZER ((SBuffer){0, 0, NULL})
static int32_t tBufferInit(SBuffer *buffer);
static int32_t tBufferDestroy(SBuffer *buffer);
static int32_t tBufferClear(SBuffer *buffer);
static void tBufferInit(SBuffer *buffer);
static void tBufferDestroy(SBuffer *buffer);
static void tBufferClear(SBuffer *buffer);
static int32_t tBufferEnsureCapacity(SBuffer *buffer, uint32_t capacity);
static int32_t tBufferPut(SBuffer *buffer, const void *data, uint32_t size);
static int32_t tBufferPutAt(SBuffer *buffer, uint32_t offset, const void *data, uint32_t size);

11
include/util/tbuffer.inc
View File

@ -29,27 +29,22 @@ struct SBufferReader {
};
// SBuffer
static FORCE_INLINE int32_t tBufferInit(SBuffer *buffer) {
static FORCE_INLINE void tBufferInit(SBuffer *buffer) {
buffer->size = 0;
buffer->capacity = 0;
buffer->data = NULL;
return 0;
}
static FORCE_INLINE int32_t tBufferDestroy(SBuffer *buffer) {
static FORCE_INLINE void tBufferDestroy(SBuffer *buffer) {
buffer->size = 0;
buffer->capacity = 0;
if (buffer->data) {
taosMemoryFree(buffer->data);
buffer->data = NULL;
}
return 0;
}
static FORCE_INLINE int32_t tBufferClear(SBuffer *buffer) {
buffer->size = 0;
return 0;
}
static FORCE_INLINE void tBufferClear(SBuffer *buffer) { buffer->size = 0; }
static FORCE_INLINE int32_t tBufferEnsureCapacity(SBuffer *buffer, uint32_t capacity) {
if (buffer->capacity < capacity) {

5
include/util/tsimplehash.h
View File

@ -17,15 +17,12 @@
#define TDENGINE_TSIMPLEHASH_H
#include "tarray.h"
#include "thash.h"
#ifdef __cplusplus
extern "C" {
#endif
typedef uint32_t (*_hash_fn_t)(const char *, uint32_t);
typedef int32_t (*_equal_fn_t)(const void *, const void *, size_t len);
typedef void (*_hash_free_fn_t)(void *);
/**
* @brief single thread hash
*

14
include/util/tutil.h
View File

@ -16,12 +16,10 @@
#ifndef _TD_UTIL_UTIL_H_
#define _TD_UTIL_UTIL_H_
#include "os.h"
#include "tcrc32c.h"
#include "tdef.h"
#include "thash.h"
#include "tmd5.h"
#include "tutil.h"
#ifdef __cplusplus
extern "C" {
@ -160,12 +158,12 @@ static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen,
return (terrno = (CODE)); \
} while (0)
#define TAOS_CHECK_RETURN(CMD) \
do { \
int32_t code = (CMD); \
if (code != TSDB_CODE_SUCCESS) { \
TAOS_RETURN(code); \
} \
#define TAOS_CHECK_RETURN(CMD) \
do { \
int32_t __c = (CMD); \
if (__c != TSDB_CODE_SUCCESS) { \
TAOS_RETURN(__c); \
} \
} while (0)
#define TAOS_CHECK_GOTO(CMD, LINO, LABEL) \

12
source/client/src/clientImpl.c
View File

@ -1017,15 +1017,13 @@ void returnToUser(SRequestObj* pRequest) {
}
static int32_t createResultBlock(TAOS_RES* pRes, int32_t numOfRows, SSDataBlock**pBlock) {
int64_t lastTs = 0;
int32_t code = TSDB_CODE_SUCCESS;
int64_t lastTs = 0;
TAOS_FIELD* pResFields = taos_fetch_fields(pRes);
int32_t numOfFields = taos_num_fields(pRes);
int32_t numOfFields = taos_num_fields(pRes);
*pBlock = createDataBlock();
if (NULL == *pBlock) {
return terrno;
int32_t code = createDataBlock(pBlock);
if (code) {
return code;
}
for(int32_t i = 0; i < numOfFields; ++i) {

619
source/common/src/tdatablock.c
View File

File diff suppressed because it is too large Load Diff

6
source/common/src/tdataformat.c
View File

@ -459,7 +459,7 @@ int32_t tRowBuildFromBind(SBindInfo *infos, int32_t numOfInfos, bool infoSorted,
SColVal colVal;
if ((colValArray = taosArrayInit(numOfInfos, sizeof(SColVal))) == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
for (int32_t iRow = 0; iRow < numOfRows; iRow++) {
@ -670,7 +670,7 @@ static int32_t tRowMergeImpl(SArray *aRowP, STSchema *pTSchema, int32_t iStart,
// merge
aColVal = taosArrayInit(pTSchema->numOfCols, sizeof(SColVal));
if (aColVal == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
code = terrno;
goto _exit;
}
@ -1748,7 +1748,7 @@ int32_t tTagToValArray(const STag *pTag, SArray **ppArray) {
(*ppArray) = taosArrayInit(pTag->nTag + 1, sizeof(STagVal));
if (*ppArray == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
code = terrno;
goto _err;
}

3
source/common/src/tname.c
View File

@ -298,6 +298,7 @@ int32_t buildChildTableName(RandTableName* rName) {
if (sb.buf == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
taosArraySort(rName->tags, compareKv);
for (int j = 0; j < taosArrayGetSize(rName->tags); ++j) {
taosStringBuilderAppendChar(&sb, ',');
@ -305,6 +306,7 @@ int32_t buildChildTableName(RandTableName* rName) {
if (tagKv == NULL) {
return TSDB_CODE_SML_INVALID_DATA;
}
taosStringBuilderAppendStringLen(&sb, tagKv->key, tagKv->keyLen);
taosStringBuilderAppendChar(&sb, '=');
if (IS_VAR_DATA_TYPE(tagKv->type)) {
@ -313,6 +315,7 @@ int32_t buildChildTableName(RandTableName* rName) {
taosStringBuilderAppendStringLen(&sb, (char*)(&(tagKv->value)), tagKv->length);
}
}
size_t len = 0;
char* keyJoined = taosStringBuilderGetResult(&sb, &len);
T_MD5_CTX context;

8
source/common/test/commonTests.cpp
View File

@ -236,7 +236,9 @@ TEST(testCase, toInteger_test) {
}
TEST(testCase, Datablock_test) {
SSDataBlock* b = createDataBlock();
SSDataBlock* b = NULL;
int32_t code = createDataBlock(&b);
ASSERT(code == 0);
SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 1);
taosArrayPush(b->pDataBlock, &infoData);
@ -361,7 +363,9 @@ TEST(testCase, non_var_dataBlock_split_test) {
TEST(testCase, var_dataBlock_split_test) {
int32_t numOfRows = 1000000;
SSDataBlock* b = createDataBlock();
SSDataBlock* b = NULL;
int32_t code = createDataBlock(&b);
ASSERT(code == 0);
SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 1);
blockDataAppendColInfo(b, &infoData);

14
source/dnode/mgmt/node_mgmt/src/dmEnv.c
View File

@ -260,7 +260,7 @@ static int32_t dmProcessCreateNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) {
return code;
}
taosThreadMutexLock(&pDnode->mutex);
(void)taosThreadMutexLock(&pDnode->mutex);
SMgmtInputOpt input = dmBuildMgmtInputOpt(pWrapper);
dInfo("node:%s, start to create", pWrapper->name);
@ -277,7 +277,7 @@ static int32_t dmProcessCreateNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) {
pWrapper->required = true;
}
taosThreadMutexUnlock(&pDnode->mutex);
(void)taosThreadMutexUnlock(&pDnode->mutex);
return code;
}
@ -316,7 +316,7 @@ static int32_t dmProcessAlterNodeTypeReq(EDndNodeType ntype, SRpcMsg *pMsg) {
dInfo("node:%s, catched up leader, continue to process alter-node-type-request", pWrapper->name);
taosThreadMutexLock(&pDnode->mutex);
(void)taosThreadMutexLock(&pDnode->mutex);
dInfo("node:%s, stopping node", pWrapper->name);
dmStopNode(pWrapper);
@ -325,7 +325,7 @@ static int32_t dmProcessAlterNodeTypeReq(EDndNodeType ntype, SRpcMsg *pMsg) {
pWrapper = &pDnode->wrappers[ntype];
if (taosMkDir(pWrapper->path) != 0) {
taosThreadMutexUnlock(&pDnode->mutex);
(void)taosThreadMutexUnlock(&pDnode->mutex);
code = terrno;
dError("failed to create dir:%s since %s", pWrapper->path, tstrerror(code));
return code;
@ -347,7 +347,7 @@ static int32_t dmProcessAlterNodeTypeReq(EDndNodeType ntype, SRpcMsg *pMsg) {
pWrapper->required = true;
}
taosThreadMutexUnlock(&pDnode->mutex);
(void)taosThreadMutexUnlock(&pDnode->mutex);
return code;
}
@ -375,7 +375,7 @@ static int32_t dmProcessDropNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) {
return terrno = code;
}
taosThreadMutexLock(&pDnode->mutex);
(void)taosThreadMutexLock(&pDnode->mutex);
SMgmtInputOpt input = dmBuildMgmtInputOpt(pWrapper);
dInfo("node:%s, start to drop", pWrapper->name);
@ -395,7 +395,7 @@ static int32_t dmProcessDropNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) {
dmCloseNode(pWrapper);
taosRemoveDir(pWrapper->path);
}
taosThreadMutexUnlock(&pDnode->mutex);
(void)taosThreadMutexUnlock(&pDnode->mutex);
return code;
}

7
source/dnode/mnode/impl/src/mndShow.c
View File

@ -289,7 +289,12 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) {
int32_t numOfCols = pShow->pMeta->numOfColumns;
SSDataBlock *pBlock = createDataBlock();
SSDataBlock *pBlock = NULL;
code = createDataBlock(&pBlock);
if (code) {
TAOS_RETURN(code);
}
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData idata = {0};

19
source/dnode/mnode/impl/src/mndStream.c
View File

@ -806,14 +806,16 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
}
// schedule stream task for stream obj
if ((code = mndScheduleStream(pMnode, &streamObj, createReq.lastTs, createReq.pVgroupVerList)) < 0) {
code = mndScheduleStream(pMnode, &streamObj, createReq.lastTs, createReq.pVgroupVerList);
if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("stream:%s, failed to schedule since %s", createReq.name, terrstr());
mndTransDrop(pTrans);
goto _OVER;
}
// add stream to trans
if ((code = mndPersistStream(pTrans, &streamObj)) < 0) {
code = mndPersistStream(pTrans, &streamObj);
if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("stream:%s, failed to persist since %s", createReq.name, terrstr());
mndTransDrop(pTrans);
goto _OVER;
@ -837,7 +839,8 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
streamMutexUnlock(&execInfo.lock);
// execute creation
if ((code = mndTransPrepare(pMnode, pTrans)) != 0) {
code = mndTransPrepare(pMnode, pTrans);
if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr());
mndTransDrop(pTrans);
goto _OVER;
@ -848,12 +851,14 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
SName dbname = {0};
code = tNameFromString(&dbname, createReq.sourceDB, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
if (code) {
mError("invalid source dbname:%s in create stream, code:%s", createReq.sourceDB, tstrerror(code));
goto _OVER;
}
SName name = {0};
code = tNameFromString(&name, createReq.name, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE);
code = tNameFromString(&name, createReq.name, T_NAME_ACCT | T_NAME_TABLE);
if (code) {
mError("invalid stream name:%s in create strem, code:%s", createReq.name, tstrerror(code));
goto _OVER;
}
@ -868,7 +873,9 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
_OVER:
if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("stream:%s, failed to create since %s", createReq.name, terrstr());
mError("stream:%s, failed to create since %s", createReq.name, terrstr(code));
} else {
mDebug("stream:%s create stream completed", createReq.name);
}
mndReleaseStream(pMnode, pStream);
@ -3133,4 +3140,4 @@ int32_t mndCreateStreamChkptInfoUpdateTrans(SMnode *pMnode, SStreamObj *pStream,
mndTransDrop(pTrans);
return TSDB_CODE_ACTION_IN_PROGRESS;
}
}

5
source/dnode/mnode/impl/src/mndTrans.c
View File

@ -1001,7 +1001,9 @@ static int32_t mndTransCheckCommitActions(SMnode *pMnode, STrans *pTrans) {
int32_t mndTransPrepare(SMnode *pMnode, STrans *pTrans) {
int32_t code = 0;
if (pTrans == NULL) return -1;
if (pTrans == NULL) {
return TSDB_CODE_INVALID_PARA;
}
TAOS_CHECK_RETURN(mndTransCheckConflict(pMnode, pTrans));
@ -1583,6 +1585,7 @@ _OVER:
static bool mndTransPerformRedoActionStage(SMnode *pMnode, STrans *pTrans, bool topHalf) {
bool continueExec = true;
int32_t code = 0;
terrno = 0;
if (pTrans->exec == TRN_EXEC_SERIAL) {
code = mndTransExecuteRedoActionsSerial(pMnode, pTrans, topHalf);

49
source/dnode/mnode/impl/test/stream/stream.cpp
View File

@ -49,7 +49,9 @@ SRpcMsg buildHbReq() {
entry.stage = 4;
}
taosArrayPush(msg.pTaskStatus, &entry);
void* px = taosArrayPush(msg.pTaskStatus, &entry);
ASSERT(px != NULL);
}
// (p->checkpointId != 0) && p->checkpointFailed
@ -65,7 +67,8 @@ SRpcMsg buildHbReq() {
entry.checkpointInfo.activeId = 1;
entry.checkpointInfo.failed = true;
taosArrayPush(msg.pTaskStatus, &entry);
void* px = taosArrayPush(msg.pTaskStatus, &entry);
ASSERT(px != NULL);
}
int32_t tlen = 0;
@ -122,8 +125,11 @@ void setTask(SStreamTask* pTask, int32_t nodeId, int64_t streamId, int32_t taskI
entry.stage = 1;
entry.status = TASK_STATUS__READY;
taosHashPut(pExecNode->pTaskMap, &id, sizeof(id), &entry, sizeof(entry));
taosArrayPush(pExecNode->pTaskList, &id);
int32_t code = taosHashPut(pExecNode->pTaskMap, &id, sizeof(id), &entry, sizeof(entry));
ASSERT(code == 0);
void* px = taosArrayPush(pExecNode->pTaskList, &id);
ASSERT(px != NULL);
}
void initStreamExecInfo() {
@ -141,7 +147,8 @@ void initNodeInfo() {
SNodeEntry entry = {0};
entry.nodeId = 2;
entry.stageUpdated = true;
taosArrayPush(execInfo.pNodeList, &entry);
void* px = taosArrayPush(execInfo.pNodeList, &entry);
ASSERT(px != NULL);
}
} // namespace
@ -149,15 +156,17 @@ class StreamTest : public testing::Test { // 继承了 testing::Test
protected:
static void SetUpTestSuite() {
mndInitExecInfo();
int32_t code = mndInitExecInfo();
ASSERT(code == 0);
initStreamExecInfo();
initNodeInfo();
std::cout<<"setup env for streamTest suite"<<std::endl;
(void) printf("setup env for streamTest suite");
}
static void TearDownTestSuite() {
std::cout<<"tearDown env for streamTest suite"<<std::endl;
(void) printf("tearDown env for streamTest suite");
}
virtual void SetUp() override {
@ -182,7 +191,8 @@ TEST_F(StreamTest, handle_error_in_hb) {
TEST_F(StreamTest, kill_checkpoint_trans) {
STrans trans;
trans.id = 100;
mndStreamRegisterTrans(&trans, MND_STREAM_CHECKPOINT_NAME, defStreamId);
int32_t code = mndStreamRegisterTrans(&trans, MND_STREAM_CHECKPOINT_NAME, defStreamId);
ASSERT(code == 0);
SMnode* pMnode = static_cast<SMnode*>(taosMemoryCalloc(1, sizeof(SMnode)));
{// init sdb
@ -192,7 +202,8 @@ TEST_F(StreamTest, kill_checkpoint_trans) {
opt.pWal = pMnode->pWal;
pMnode->pSdb = sdbInit(&opt);
taosThreadMutexInit(&pMnode->syncMgmt.lock, NULL);
int32_t code = taosThreadMutexInit(&pMnode->syncMgmt.lock, NULL);
ASSERT(code == 0);
}
SVgroupChangeInfo info;
@ -202,7 +213,8 @@ TEST_F(StreamTest, kill_checkpoint_trans) {
const char* pDbName = "test_db_name";
int32_t len = strlen(pDbName);
taosHashPut(info.pDBMap, pDbName, len, NULL, 0);
code = taosHashPut(info.pDBMap, pDbName, len, NULL, 0);
ASSERT(code == 0);
killAllCheckpointTrans(pMnode, &info);
@ -224,12 +236,17 @@ TEST_F(StreamTest, kill_checkpoint_trans) {
pTask->id.streamId = defStreamId;
pTask->id.taskId = 1;
pTask->exec.qmsg = (char*)taosMemoryCalloc(1,1);
taosThreadMutexInit(&pTask->lock, NULL);
code = taosThreadMutexInit(&pTask->lock, NULL);
ASSERT(code == 0);
taosArrayPush(pLevel, &pTask);
void* px = taosArrayPush(pLevel, &pTask);
ASSERT(px != NULL);
taosArrayPush(pStream->tasks, &pLevel);
mndCreateStreamResetStatusTrans(pMnode, pStream);
px = taosArrayPush(pStream->tasks, &pLevel);
ASSERT(px != NULL);
code = mndCreateStreamResetStatusTrans(pMnode, pStream);
ASSERT(code != 0);
tFreeStreamObj(pStream);
sdbCleanup(pMnode->pSdb);
@ -248,7 +265,7 @@ TEST_F(StreamTest, plan_Test) {
if (taosCreateLog("taoslog", 10, "/etc/taos", NULL, NULL, NULL, NULL, 1) != 0) {
// ignore create log failed, only print
printf(" WARING: Create failed:%s. configDir\n", strerror(errno));
(void) printf(" WARING: Create failed:%s. configDir\n", strerror(errno));
}
if (nodesStringToNode(ast, &pAst) < 0) {

36
source/dnode/vnode/src/meta/metaCache.c
View File

@ -530,12 +530,12 @@ int32_t metaGetCachedTableUidList(void* pVnode, tb_uid_t suid, const uint8_t* pK
uint64_t key[4];
initCacheKey(key, pTableMap, suid, (const char*)pKey, keyLen);
taosThreadMutexLock(pLock);
(void)taosThreadMutexLock(pLock);
pMeta->pCache->sTagFilterResCache.accTimes += 1;
LRUHandle* pHandle = taosLRUCacheLookup(pCache, key, TAG_FILTER_RES_KEY_LEN);
if (pHandle == NULL) {
taosThreadMutexUnlock(pLock);
(void)taosThreadMutexUnlock(pLock);
return TSDB_CODE_SUCCESS;
}
@ -565,7 +565,7 @@ int32_t metaGetCachedTableUidList(void* pVnode, tb_uid_t suid, const uint8_t* pK
taosLRUCacheRelease(pCache, pHandle, false);
// unlock meta
taosThreadMutexUnlock(pLock);
(void)taosThreadMutexUnlock(pLock);
return TSDB_CODE_SUCCESS;
}
@ -652,7 +652,7 @@ int32_t metaUidFilterCachePut(void* pVnode, uint64_t suid, const void* pKey, int
uint64_t key[4] = {0};
initCacheKey(key, pTableEntry, suid, pKey, keyLen);
taosThreadMutexLock(pLock);
(void)taosThreadMutexLock(pLock);
STagFilterResEntry** pEntry = taosHashGet(pTableEntry, &suid, sizeof(uint64_t));
if (pEntry == NULL) {
code = addNewEntry(pTableEntry, pKey, keyLen, suid);
@ -668,7 +668,7 @@ int32_t metaUidFilterCachePut(void* pVnode, uint64_t suid, const void* pKey, int
uint64_t* p = (uint64_t*)pNode->data;
if (p[1] == ((uint64_t*)pKey)[1] && p[0] == ((uint64_t*)pKey)[0]) {
// we have already found the existed items, no need to added to cache anymore.
taosThreadMutexUnlock(pLock);
(void)taosThreadMutexUnlock(pLock);
return TSDB_CODE_SUCCESS;
} else { // not equal, append it
tdListAppend(&(*pEntry)->list, pKey);
@ -680,7 +680,7 @@ int32_t metaUidFilterCachePut(void* pVnode, uint64_t suid, const void* pKey, int
(void)taosLRUCacheInsert(pCache, key, TAG_FILTER_RES_KEY_LEN, pPayload, payloadLen, freeUidCachePayload, NULL,
TAOS_LRU_PRIORITY_LOW, NULL);
_end:
taosThreadMutexUnlock(pLock);
(void)taosThreadMutexUnlock(pLock);
metaDebug("vgId:%d, suid:%" PRIu64 " list cache added into cache, total:%d, tables:%d", vgId, suid,
(int32_t)taosLRUCacheGetUsage(pCache), taosHashGetSize(pTableEntry));
@ -697,11 +697,11 @@ int32_t metaUidCacheClear(SMeta* pMeta, uint64_t suid) {
initCacheKey(p, pEntryHashMap, suid, (char*)&dummy[0], 16);
TdThreadMutex* pLock = &pMeta->pCache->sTagFilterResCache.lock;
taosThreadMutexLock(pLock);
(void)taosThreadMutexLock(pLock);
STagFilterResEntry** pEntry = taosHashGet(pEntryHashMap, &suid, sizeof(uint64_t));
if (pEntry == NULL || listNEles(&(*pEntry)->list) == 0) {
taosThreadMutexUnlock(pLock);
(void)taosThreadMutexUnlock(pLock);
return TSDB_CODE_SUCCESS;
}
@ -717,7 +717,7 @@ int32_t metaUidCacheClear(SMeta* pMeta, uint64_t suid) {
}
tdListEmpty(&(*pEntry)->list);
taosThreadMutexUnlock(pLock);
(void)taosThreadMutexUnlock(pLock);
metaDebug("vgId:%d suid:%" PRId64 " cached related tag filter uid list cleared", vgId, suid);
return TSDB_CODE_SUCCESS;
@ -736,12 +736,12 @@ int32_t metaGetCachedTbGroup(void* pVnode, tb_uid_t suid, const uint8_t* pKey, i
uint64_t key[4];
initCacheKey(key, pTableMap, suid, (const char*)pKey, keyLen);
taosThreadMutexLock(pLock);
(void)taosThreadMutexLock(pLock);
pMeta->pCache->STbGroupResCache.accTimes += 1;
LRUHandle* pHandle = taosLRUCacheLookup(pCache, key, TAG_FILTER_RES_KEY_LEN);
if (pHandle == NULL) {
taosThreadMutexUnlock(pLock);
(void)taosThreadMutexUnlock(pLock);
return TSDB_CODE_SUCCESS;
}
@ -764,7 +764,7 @@ int32_t metaGetCachedTbGroup(void* pVnode, tb_uid_t suid, const uint8_t* pKey, i
taosLRUCacheRelease(pCache, pHandle, false);
// unlock meta
taosThreadMutexUnlock(pLock);
(void)taosThreadMutexUnlock(pLock);
return TSDB_CODE_SUCCESS;
}
@ -829,7 +829,7 @@ int32_t metaPutTbGroupToCache(void* pVnode, uint64_t suid, const void* pKey, int
uint64_t key[4] = {0};
initCacheKey(key, pTableEntry, suid, pKey, keyLen);
taosThreadMutexLock(pLock);
(void)taosThreadMutexLock(pLock);
STagFilterResEntry** pEntry = taosHashGet(pTableEntry, &suid, sizeof(uint64_t));
if (pEntry == NULL) {
code = addNewEntry(pTableEntry, pKey, keyLen, suid);
@ -845,7 +845,7 @@ int32_t metaPutTbGroupToCache(void* pVnode, uint64_t suid, const void* pKey, int
uint64_t* p = (uint64_t*)pNode->data;
if (p[1] == ((uint64_t*)pKey)[1] && p[0] == ((uint64_t*)pKey)[0]) {
// we have already found the existed items, no need to added to cache anymore.
taosThreadMutexUnlock(pLock);
(void)taosThreadMutexUnlock(pLock);
return TSDB_CODE_SUCCESS;
} else { // not equal, append it
tdListAppend(&(*pEntry)->list, pKey);
@ -857,7 +857,7 @@ int32_t metaPutTbGroupToCache(void* pVnode, uint64_t suid, const void* pKey, int
taosLRUCacheInsert(pCache, key, TAG_FILTER_RES_KEY_LEN, pPayload, payloadLen, freeTbGroupCachePayload, NULL,
TAOS_LRU_PRIORITY_LOW, NULL);
_end:
taosThreadMutexUnlock(pLock);
(void)taosThreadMutexUnlock(pLock);
metaDebug("vgId:%d, suid:%" PRIu64 " tb group added into cache, total:%d, tables:%d", vgId, suid,
(int32_t)taosLRUCacheGetUsage(pCache), taosHashGetSize(pTableEntry));
@ -874,11 +874,11 @@ int32_t metaTbGroupCacheClear(SMeta* pMeta, uint64_t suid) {
initCacheKey(p, pEntryHashMap, suid, (char*)&dummy[0], 16);
TdThreadMutex* pLock = &pMeta->pCache->STbGroupResCache.lock;
taosThreadMutexLock(pLock);
(void)taosThreadMutexLock(pLock);
STagFilterResEntry** pEntry = taosHashGet(pEntryHashMap, &suid, sizeof(uint64_t));
if (pEntry == NULL || listNEles(&(*pEntry)->list) == 0) {
taosThreadMutexUnlock(pLock);
(void)taosThreadMutexUnlock(pLock);
return TSDB_CODE_SUCCESS;
}
@ -894,7 +894,7 @@ int32_t metaTbGroupCacheClear(SMeta* pMeta, uint64_t suid) {
}
tdListEmpty(&(*pEntry)->list);
taosThreadMutexUnlock(pLock);
(void)taosThreadMutexUnlock(pLock);
metaDebug("vgId:%d suid:%" PRId64 " cached related tb group cleared", vgId, suid);
return TSDB_CODE_SUCCESS;

25
source/dnode/vnode/src/meta/metaQuery.c
View File

@ -1069,9 +1069,12 @@ int32_t metaFilterCreateTime(void *pVnode, SMetaFltParam *arg, SArray *pUids) {
if (count > TRY_ERROR_LIMIT) break;
int32_t cmp = (*param->filterFunc)((void *)&p->btime, (void *)&pBtimeKey->btime, param->type);
if (cmp == 0)
taosArrayPush(pUids, &p->uid);
else {
if (cmp == 0) {
if (taosArrayPush(pUids, &p->uid) == NULL) {
ret = terrno;
break;
}
} else {
if (param->equal == true) {
if (count > TRY_ERROR_LIMIT) break;
count++;
@ -1132,7 +1135,10 @@ int32_t metaFilterTableName(void *pVnode, SMetaFltParam *arg, SArray *pUids) {
cmp = (*param->filterFunc)(pTableKey, pName, pCursor->type);
if (cmp == 0) {
tb_uid_t tuid = *(tb_uid_t *)pEntryVal;
taosArrayPush(pUids, &tuid);
if (taosArrayPush(pUids, &tuid) == NULL) {
ret = terrno;
goto END;
}
} else {
if (param->equal == true) {
if (count > TRY_ERROR_LIMIT) break;
@ -1328,7 +1334,10 @@ int32_t metaFilterTableIds(void *pVnode, SMetaFltParam *arg, SArray *pUids) {
} else {
tuid = *(tb_uid_t *)(p->data + tDataTypes[pCursor->type].bytes);
}
taosArrayPush(pUids, &tuid);
if (taosArrayPush(pUids, &tuid) == NULL) {
ret = terrno;
break;
}
found = true;
} else {
if (param->equal == true) {
@ -1432,7 +1441,11 @@ int32_t metaGetTableTags(void *pVnode, uint64_t suid, SArray *pUidTagInfo) {
STUidTagInfo info = {.uid = uid, .pTagVal = pCur->pVal};
info.pTagVal = taosMemoryMalloc(pCur->vLen);
memcpy(info.pTagVal, pCur->pVal, pCur->vLen);
taosArrayPush(pUidTagInfo, &info);
if (taosArrayPush(pUidTagInfo, &info) == NULL) {
metaCloseCtbCursor(pCur);
taosHashCleanup(pSepecifiedUidMap);
return TSDB_CODE_OUT_OF_MEMORY;
}
}
} else { // only the specified tables need to be added
while (1) {

14
source/dnode/vnode/src/meta/metaTable.c
View File

@ -341,7 +341,7 @@ int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq, SArray *tb
break;
}
taosArrayPush(tbUidList, &(((SCtbIdxKey *)pKey)->uid));
(void)taosArrayPush(tbUidList, &(((SCtbIdxKey *)pKey)->uid));
}
tdbTbcClose(pCtbIdxc);
@ -405,7 +405,7 @@ static void metaGetSubtables(SMeta *pMeta, int64_t suid, SArray *uids) {
break;
}
taosArrayPush(uids, &(((SCtbIdxKey *)pKey)->uid));
(void)taosArrayPush(uids, &(((SCtbIdxKey *)pKey)->uid));
}
tdbFree(pKey);
@ -1033,7 +1033,7 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUi
}
if ((type == TSDB_CHILD_TABLE || type == TSDB_NORMAL_TABLE) && tbUids) {
taosArrayPush(tbUids, &uid);
(void)taosArrayPush(tbUids, &uid);
if (!TSDB_CACHE_NO(pMeta->pVnode->config)) {
tsdbCacheDropTable(pMeta->pVnode->pTsdb, uid, suid, NULL);
@ -1135,7 +1135,7 @@ static int32_t metaFilterTableByHash(SMeta *pMeta, SArray *uidList) {
tbFName[TSDB_TABLE_FNAME_LEN] = '\0';
int32_t ret = vnodeValidateTableHash(pMeta->pVnode, tbFName);
if (ret < 0 && terrno == TSDB_CODE_VND_HASH_MISMATCH) {
taosArrayPush(uidList, &me.uid);
(void)taosArrayPush(uidList, &me.uid);
}
}
tDecoderClear(&dc);
@ -1783,11 +1783,11 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
} else {
memcpy(&val.i64, pAlterTbReq->pTagVal, pAlterTbReq->nTagVal);
}
taosArrayPush(pTagArray, &val);
(void)taosArrayPush(pTagArray, &val);
} else {
STagVal val = {.cid = pCol->colId};
if (tTagGet(pOldTag, &val)) {
taosArrayPush(pTagArray, &val);
(void)taosArrayPush(pTagArray, &val);
}
}
}
@ -2171,7 +2171,7 @@ static int metaDropTagIndex(SMeta *pMeta, int64_t version, SVAlterTbReq *pAlterT
}
SMetaPair pair = {.key = pKey, nKey = nKey};
taosArrayPush(tagIdxList, &pair);
(void)taosArrayPush(tagIdxList, &pair);
}
tdbTbcClose(pTagIdxc);

8
source/dnode/vnode/src/tq/tqRead.c
View File

@ -289,7 +289,13 @@ STqReader* tqReaderOpen(SVnode* pVnode) {
pReader->cachedSchemaSuid = 0;
pReader->pSchemaWrapper = NULL;
pReader->tbIdHash = NULL;
pReader->pResBlock = createDataBlock();
pReader->pResBlock = NULL;
int32_t code = createDataBlock(&pReader->pResBlock);
if (code) {
terrno = code;
}
return pReader;
}

9
source/dnode/vnode/src/tq/tqScan.c
View File

@ -117,8 +117,13 @@ int32_t tqScanData(STQ* pTq, STqHandle* pHandle, SMqDataRsp* pRsp, STqOffsetVal*
}
STqOffsetVal offset = {0};
qStreamExtractOffset(task, &offset);
pHandle->block = createOneDataBlock(pDataBlock, true);
TSDB_CHECK_NULL(pDataBlock, code, line, END, terrno);
pHandle->block = NULL;
code = createOneDataBlock(pDataBlock, true, &pHandle->block);
if (code) {
return code;
}
pHandle->blockTime = offset.ts;
tOffsetDestroy(&offset);
code = getDataBlock(task, pHandle, vgId, &pDataBlock);

45
source/dnode/vnode/src/tq/tqSink.c
View File

@ -41,8 +41,8 @@ static int32_t doRemoveFromCache(SSHashObj* pSinkTableMap, uint64_t groupId, con
static bool isValidDstChildTable(SMetaReader* pReader, int32_t vgId, const char* ctbName, int64_t suid);
static int32_t initCreateTableMsg(SVCreateTbReq* pCreateTableReq, uint64_t suid, const char* stbFullName,
int32_t numOfTags);
static int32_t createDefaultTagColName(SArray** pList);
static void setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* pDataBlock, const char* stbFullName,
static int32_t createDefaultTagColName(SArray** pColNameList);
static int32_t setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* pDataBlock, const char* stbFullName,
int64_t gid, bool newSubTableRule);
static int32_t doCreateSinkInfo(const char* pDstTableName, STableSinkInfo** pInfo);
@ -70,14 +70,25 @@ int32_t tqBuildDeleteReq(STQ* pTq, const char* stbFullName, const SSDataBlock* p
if (varTbName != NULL && varTbName != (void*)-1) {
name = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN);
if (name == NULL) {
return terrno;
}
memcpy(name, varDataVal(varTbName), varDataLen(varTbName));
if (newSubTableRule && !isAutoTableName(name) && !alreadyAddGroupId(name, groupId) && groupId != 0 && stbFullName) {
buildCtbNameAddGroupId(stbFullName, name, groupId);
}
} else if (stbFullName) {
name = buildCtbNameByGroupId(stbFullName, groupId);
int32_t code = buildCtbNameByGroupId(stbFullName, groupId, &name);
if (code) {
return code;
}
} else {
originName = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE);
if (originName == NULL) {
return terrno;
}
if (metaGetTableNameByUid(pTq->pVnode, groupId, originName) == 0) {
name = varDataVal(originName);
}
@ -205,23 +216,33 @@ int32_t createDefaultTagColName(SArray** pColNameList) {
return TSDB_CODE_SUCCESS;
}
void setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* pDataBlock, const char* stbFullName,
int32_t setCreateTableMsgTableName(SVCreateTbReq* pCreateTableReq, SSDataBlock* pDataBlock, const char* stbFullName,
int64_t gid, bool newSubTableRule) {
if (pDataBlock->info.parTbName[0]) {
if (newSubTableRule && !isAutoTableName(pDataBlock->info.parTbName) &&
!alreadyAddGroupId(pDataBlock->info.parTbName, gid) && gid != 0 && stbFullName) {
pCreateTableReq->name = taosMemoryCalloc(1, TSDB_TABLE_NAME_LEN);
if (pCreateTableReq->name == NULL) {
return terrno;
}
strcpy(pCreateTableReq->name, pDataBlock->info.parTbName);
buildCtbNameAddGroupId(stbFullName, pCreateTableReq->name, gid);
// tqDebug("gen name from:%s", pDataBlock->info.parTbName);
} else {
pCreateTableReq->name = taosStrdup(pDataBlock->info.parTbName);
if (pCreateTableReq->name == NULL) {
return terrno;
}
// tqDebug("copy name:%s", pDataBlock->info.parTbName);
}
} else {
pCreateTableReq->name = buildCtbNameByGroupId(stbFullName, gid);
int32_t code = buildCtbNameByGroupId(stbFullName, gid, &pCreateTableReq->name);
return code;
// tqDebug("gen name from stbFullName:%s gid:%"PRId64, stbFullName, gid);
}
return 0;
}
static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, SSDataBlock* pDataBlock,
@ -310,8 +331,11 @@ static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, S
ASSERT(gid == *(int64_t*)pGpIdData);
}
setCreateTableMsgTableName(pCreateTbReq, pDataBlock, stbFullName, gid,
code = setCreateTableMsgTableName(pCreateTbReq, pDataBlock, stbFullName, gid,
pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && pTask->subtableWithoutMd5 != 1);
if (code) {
goto _end;
}
void* p = taosArrayPush(reqs.pArray, pCreateTbReq);
if (p == NULL) {
@ -534,9 +558,16 @@ int32_t buildAutoCreateTableReq(const char* stbFullName, int64_t suid, int32_t n
}
code = createDefaultTagColName(&pCreateTbReq->ctb.tagName);
if (code) {
return code;
}
// set table name
setCreateTableMsgTableName(pCreateTbReq, pDataBlock, stbFullName, pDataBlock->info.id.groupId, newSubTableRule);
code = setCreateTableMsgTableName(pCreateTbReq, pDataBlock, stbFullName, pDataBlock->info.id.groupId, newSubTableRule);
if (code) {
return code;
}
*pReq = pCreateTbReq;
return code;
}

6
source/dnode/vnode/src/tq/tqUtil.c
View File

@ -576,8 +576,10 @@ int32_t tqExtractDelDataBlock(const void* pData, int32_t len, int64_t ver, void*
goto END;
}
SSDataBlock* pDelBlock = createSpecialDataBlock(STREAM_DELETE_DATA);
TSDB_CHECK_NULL(pDelBlock, code, line, END, terrno)
SSDataBlock* pDelBlock = NULL;
code = createSpecialDataBlock(STREAM_DELETE_DATA, &pDelBlock);
TSDB_CHECK_CODE(code, line, END);
code = blockDataEnsureCapacity(pDelBlock, numOfTables);
TSDB_CHECK_CODE(code, line, END);

6
source/dnode/vnode/src/tsdb/tsdbCache.c
View File

@ -2419,7 +2419,7 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie
if (!state->pBrinBlock) {
state->pBrinBlock = &state->brinBlock;
} else {
tBrinBlockClear(&state->brinBlock);
(void)tBrinBlockClear(&state->brinBlock);
}
TAOS_CHECK_GOTO(tsdbDataFileReadBrinBlock(state->pr->pFileReader, pBrinBlk, &state->brinBlock), &lino, _err);
@ -2431,7 +2431,7 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie
if (SFSNEXTROW_BRINBLOCK == state->state) {
_next_brinrecord:
if (state->iBrinRecord < 0) { // empty brin block, goto _next_brinindex
tBrinBlockClear(&state->brinBlock);
(void)tBrinBlockClear(&state->brinBlock);
goto _next_brinindex;
}
@ -2691,7 +2691,7 @@ int32_t clearNextRowFromFS(void *iter) {
}
if (state->pBrinBlock) {
tBrinBlockDestroy(state->pBrinBlock);
(void)tBrinBlockDestroy(state->pBrinBlock);
state->pBrinBlock = NULL;
}

26
source/dnode/vnode/src/tsdb/tsdbCommit2.c
View File

@ -540,7 +540,7 @@ static int32_t tsdbCommitInfoBuild(STsdb *tsdb) {
}
}
taosThreadMutexLock(&tsdb->mutex);
(void)taosThreadMutexLock(&tsdb->mutex);
// scan tomb data
if (tsdb->imem->nDel > 0) {
@ -572,7 +572,7 @@ static int32_t tsdbCommitInfoBuild(STsdb *tsdb) {
} else {
hasDataToCommit = true;
if ((code = tsdbCommitInfoAdd(tsdb, fset->fid))) {
taosThreadMutexUnlock(&tsdb->mutex);
(void)taosThreadMutexUnlock(&tsdb->mutex);
TSDB_CHECK_CODE(code, lino, _exit);
}
break;
@ -593,13 +593,13 @@ static int32_t tsdbCommitInfoBuild(STsdb *tsdb) {
if (fset) {
code = tsdbTFileSetInitCopy(tsdb, fset, &info->fset);
if (code) {
taosThreadMutexUnlock(&tsdb->mutex);
(void)taosThreadMutexUnlock(&tsdb->mutex);
TAOS_CHECK_GOTO(code, &lino, _exit);
}
}
}
taosThreadMutexUnlock(&tsdb->mutex);
(void)taosThreadMutexUnlock(&tsdb->mutex);
_exit:
if (code) {
@ -667,11 +667,11 @@ _exit:
}
int32_t tsdbPreCommit(STsdb *tsdb) {
taosThreadMutexLock(&tsdb->mutex);
(void)taosThreadMutexLock(&tsdb->mutex);
ASSERT_CORE(tsdb->imem == NULL, "imem should be null to commit mem");
tsdb->imem = tsdb->mem;
tsdb->mem = NULL;
taosThreadMutexUnlock(&tsdb->mutex);
(void)taosThreadMutexUnlock(&tsdb->mutex);
return 0;
}
@ -686,9 +686,9 @@ int32_t tsdbCommitBegin(STsdb *tsdb, SCommitInfo *info) {
int64_t nDel = imem->nDel;
if (nRow == 0 && nDel == 0) {
taosThreadMutexLock(&tsdb->mutex);
(void)taosThreadMutexLock(&tsdb->mutex);
tsdb->imem = NULL;
taosThreadMutexUnlock(&tsdb->mutex);
(void)taosThreadMutexUnlock(&tsdb->mutex);
tsdbUnrefMemTable(imem, NULL, true);
} else {
SCommitter2 committer = {0};
@ -719,10 +719,10 @@ int32_t tsdbCommitCommit(STsdb *tsdb) {
if (tsdb->imem) {
SMemTable *pMemTable = tsdb->imem;
taosThreadMutexLock(&tsdb->mutex);
(void)taosThreadMutexLock(&tsdb->mutex);
if ((code = tsdbFSEditCommit(tsdb->pFS))) {
taosThreadMutexUnlock(&tsdb->mutex);
(void)taosThreadMutexUnlock(&tsdb->mutex);
TSDB_CHECK_CODE(code, lino, _exit);
}
tsdb->imem = NULL;
@ -734,7 +734,7 @@ int32_t tsdbCommitCommit(STsdb *tsdb) {
}
}
taosThreadMutexUnlock(&tsdb->mutex);
(void)taosThreadMutexUnlock(&tsdb->mutex);
tsdbCommitInfoDestroy(tsdb);
tsdbUnrefMemTable(pMemTable, NULL, true);
@ -757,14 +757,14 @@ int32_t tsdbCommitAbort(STsdb *pTsdb) {
TAOS_CHECK_GOTO(tsdbFSEditAbort(pTsdb->pFS), &lino, _exit);
taosThreadMutexLock(&pTsdb->mutex);
(void)taosThreadMutexLock(&pTsdb->mutex);
for (int32_t i = 0; i < taosArrayGetSize(pTsdb->commitInfo->arr); i++) {
SFileSetCommitInfo *info = *(SFileSetCommitInfo **)taosArrayGet(pTsdb->commitInfo->arr, i);
if (info->fset) {
tsdbFinishTaskOnFileSet(pTsdb, info->fid);
}
}
taosThreadMutexUnlock(&pTsdb->mutex);
(void)taosThreadMutexUnlock(&pTsdb->mutex);
tsdbCommitInfoDestroy(pTsdb);
_exit:

18
source/dnode/vnode/src/tsdb/tsdbDataFileRW.c
View File

@ -230,7 +230,7 @@ int32_t tsdbDataFileReadBrinBlock(SDataFileReader *reader, const SBrinBlk *brinB
// decode brin block
SBufferReader br = BUFFER_READER_INITIALIZER(0, buffer);
tBrinBlockClear(brinBlock);
(void)tBrinBlockClear(brinBlock);
brinBlock->numOfPKs = brinBlk->numOfPKs;
brinBlock->numOfRecords = brinBlk->numRec;
for (int32_t i = 0; i < 10; i++) { // int64_t
@ -673,12 +673,12 @@ static int32_t tsdbDataFileWriterDoClose(SDataFileWriter *writer) {
tTombBlockDestroy(writer->tombBlock);
TARRAY2_DESTROY(writer->tombBlkArray, NULL);
tBlockDataDestroy(writer->blockData);
tBrinBlockDestroy(writer->brinBlock);
(void)tBrinBlockDestroy(writer->brinBlock);
TARRAY2_DESTROY(writer->brinBlkArray, NULL);
tTombBlockDestroy(writer->ctx->tombBlock);
tBlockDataDestroy(writer->ctx->blockData);
tBrinBlockDestroy(writer->ctx->brinBlock);
(void)tBrinBlockDestroy(writer->ctx->brinBlock);
for (int32_t i = 0; i < ARRAY_SIZE(writer->local); ++i) {
tBufferDestroy(writer->local + i);
@ -838,7 +838,7 @@ int32_t tsdbFileWriteBrinBlock(STsdbFD *fd, SBrinBlock *brinBlock, uint32_t cmpr
for (int i = 0; i < brinBlock->numOfRecords; i++) {
SBrinRecord record;
tBrinBlockGet(brinBlock, i, &record);
(void)tBrinBlockGet(brinBlock, i, &record);
if (i == 0) {
brinBlk.minTbid.suid = record.suid;
brinBlk.minTbid.uid = record.uid;
@ -918,7 +918,7 @@ int32_t tsdbFileWriteBrinBlock(STsdbFD *fd, SBrinBlock *brinBlock, uint32_t cmpr
// append to brinBlkArray
TAOS_CHECK_RETURN(TARRAY2_APPEND_PTR(brinBlkArray, &brinBlk));
tBrinBlockClear(brinBlock);
(void)tBrinBlockClear(brinBlock);
return 0;
}
@ -1147,7 +1147,7 @@ static int32_t tsdbDataFileDoWriteTableOldData(SDataFileWriter *writer, const ST
for (; writer->ctx->brinBlockIdx < writer->ctx->brinBlock->numOfRecords; writer->ctx->brinBlockIdx++) {
SBrinRecord record;
tBrinBlockGet(writer->ctx->brinBlock, writer->ctx->brinBlockIdx, &record);
(void)tBrinBlockGet(writer->ctx->brinBlock, writer->ctx->brinBlockIdx, &record);
if (record.uid != writer->ctx->tbid->uid) {
writer->ctx->tbHasOldData = false;
goto _exit;
@ -1157,7 +1157,7 @@ static int32_t tsdbDataFileDoWriteTableOldData(SDataFileWriter *writer, const ST
goto _exit;
} else {
SBrinRecord record[1];
tBrinBlockGet(writer->ctx->brinBlock, writer->ctx->brinBlockIdx, record);
(void)tBrinBlockGet(writer->ctx->brinBlock, writer->ctx->brinBlockIdx, record);
if (tsdbRowKeyCmprNullAsLargest(key, &record->lastKey) > 0) { // key > record->lastKey
if (writer->blockData->nRow > 0) {
TAOS_CHECK_GOTO(tsdbDataFileDoWriteBlockData(writer, writer->blockData), &lino, _exit);
@ -1351,7 +1351,7 @@ int32_t tsdbFileWriteTombBlock(STsdbFD *fd, STombBlock *tombBlock, int8_t cmprAl
};
for (int i = 0; i < TOMB_BLOCK_SIZE(tombBlock); i++) {
STombRecord record;
tTombBlockGet(tombBlock, i, &record);
TAOS_UNUSED(tTombBlockGet(tombBlock, i, &record));
if (i == 0) {
tombBlk.minTbid.suid = record.suid;
@ -1506,7 +1506,7 @@ static int32_t tsdbDataFileDoWriteTombRecord(SDataFileWriter *writer, const STom
while (writer->ctx->hasOldTomb) {
for (; writer->ctx->tombBlockIdx < TOMB_BLOCK_SIZE(writer->ctx->tombBlock); writer->ctx->tombBlockIdx++) {
STombRecord record1[1];
tTombBlockGet(writer->ctx->tombBlock, writer->ctx->tombBlockIdx, record1);
TAOS_UNUSED(tTombBlockGet(writer->ctx->tombBlock, writer->ctx->tombBlockIdx, record1));
int32_t c = tTombRecordCompare(record, record1);
if (c < 0) {

42
source/dnode/vnode/src/tsdb/tsdbFS2.c
View File

@ -761,7 +761,7 @@ int32_t tsdbDisableAndCancelAllBgTask(STsdb *pTsdb) {
return TSDB_CODE_OUT_OF_MEMORY;
}
taosThreadMutexLock(&pTsdb->mutex);
(void)taosThreadMutexLock(&pTsdb->mutex);
// disable
pTsdb->bgTaskDisabled = true;
@ -770,7 +770,11 @@ int32_t tsdbDisableAndCancelAllBgTask(STsdb *pTsdb) {
STFileSet *fset;
TARRAY2_FOREACH(fs->fSetArr, fset) {
if (fset->channelOpened) {
taosArrayPush(channelArray, &fset->channel);
if (taosArrayPush(channelArray, &fset->channel) == NULL) {
taosArrayDestroy(channelArray);
(void)taosThreadMutexUnlock(&pTsdb->mutex);
return terrno;
}
fset->channel = (SVAChannelID){0};
fset->mergeScheduled = false;
tsdbFSSetBlockCommit(fset, false);
@ -778,7 +782,7 @@ int32_t tsdbDisableAndCancelAllBgTask(STsdb *pTsdb) {
}
}
taosThreadMutexUnlock(&pTsdb->mutex);
(void)taosThreadMutexUnlock(&pTsdb->mutex);
// destroy all channels
for (int32_t i = 0; i < taosArrayGetSize(channelArray); i++) {
@ -794,9 +798,9 @@ int32_t tsdbDisableAndCancelAllBgTask(STsdb *pTsdb) {
}
int32_t tsdbEnableBgTask(STsdb *pTsdb) {
taosThreadMutexLock(&pTsdb->mutex);
(void)taosThreadMutexLock(&pTsdb->mutex);
pTsdb->bgTaskDisabled = false;
taosThreadMutexUnlock(&pTsdb->mutex);
(void)taosThreadMutexUnlock(&pTsdb->mutex);
return 0;
}
@ -810,16 +814,16 @@ int32_t tsdbCloseFS(STFileSystem **fs) {
}
int64_t tsdbFSAllocEid(STFileSystem *fs) {
taosThreadMutexLock(&fs->tsdb->mutex);
(void)taosThreadMutexLock(&fs->tsdb->mutex);
int64_t cid = ++fs->neid;
taosThreadMutexUnlock(&fs->tsdb->mutex);
(void)taosThreadMutexUnlock(&fs->tsdb->mutex);
return cid;
}
void tsdbFSUpdateEid(STFileSystem *fs, int64_t cid) {
taosThreadMutexLock(&fs->tsdb->mutex);
(void)taosThreadMutexLock(&fs->tsdb->mutex);
fs->neid = TMAX(fs->neid, cid);
taosThreadMutexUnlock(&fs->tsdb->mutex);
(void)taosThreadMutexUnlock(&fs->tsdb->mutex);
}
int32_t tsdbFSEditBegin(STFileSystem *fs, const TFileOpArray *opArray, EFEditT etype) {
@ -867,7 +871,7 @@ static int32_t tsdbFSSetBlockCommit(STFileSet *fset, bool block) {
}
int32_t tsdbFSCheckCommit(STsdb *tsdb, int32_t fid) {
taosThreadMutexLock(&tsdb->mutex);
(void)taosThreadMutexLock(&tsdb->mutex);
STFileSet *fset;
tsdbFSGetFSet(tsdb->pFS, fid, &fset);
if (fset) {
@ -877,7 +881,7 @@ int32_t tsdbFSCheckCommit(STsdb *tsdb, int32_t fid) {
fset->numWaitCommit--;
}
}
taosThreadMutexUnlock(&tsdb->mutex);
(void)taosThreadMutexUnlock(&tsdb->mutex);
return 0;
}
@ -969,7 +973,7 @@ int32_t tsdbFSCreateCopySnapshot(STFileSystem *fs, TFileSetArray **fsetArr) {
TARRAY2_INIT(fsetArr[0]);
taosThreadMutexLock(&fs->tsdb->mutex);
(void)taosThreadMutexLock(&fs->tsdb->mutex);
TARRAY2_FOREACH(fs->fSetArr, fset) {
code = tsdbTFileSetInitCopy(fs->tsdb, fset, &fset1);
if (code) break;
@ -977,7 +981,7 @@ int32_t tsdbFSCreateCopySnapshot(STFileSystem *fs, TFileSetArray **fsetArr) {
code = TARRAY2_APPEND(fsetArr[0], fset1);
if (code) break;
}
taosThreadMutexUnlock(&fs->tsdb->mutex);
(void)taosThreadMutexUnlock(&fs->tsdb->mutex);
if (code) {
TARRAY2_DESTROY(fsetArr[0], tsdbTFileSetClear);
@ -997,9 +1001,9 @@ int32_t tsdbFSDestroyCopySnapshot(TFileSetArray **fsetArr) {
}
int32_t tsdbFSCreateRefSnapshot(STFileSystem *fs, TFileSetArray **fsetArr) {
taosThreadMutexLock(&fs->tsdb->mutex);
(void)taosThreadMutexLock(&fs->tsdb->mutex);
int32_t code = tsdbFSCreateRefSnapshotWithoutLock(fs, fsetArr);
taosThreadMutexUnlock(&fs->tsdb->mutex);
(void)taosThreadMutexUnlock(&fs->tsdb->mutex);
return code;
}
@ -1071,7 +1075,7 @@ int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TFileSetRangeArray *pRa
}
}
taosThreadMutexLock(&fs->tsdb->mutex);
(void)taosThreadMutexLock(&fs->tsdb->mutex);
TARRAY2_FOREACH(fs->fSetArr, fset) {
int64_t ever = VERSION_MAX;
if (pHash) {
@ -1088,7 +1092,7 @@ int32_t tsdbFSCreateCopyRangedSnapshot(STFileSystem *fs, TFileSetRangeArray *pRa
code = TARRAY2_APPEND(fsetArr[0], fset1);
if (code) break;
}
taosThreadMutexUnlock(&fs->tsdb->mutex);
(void)taosThreadMutexUnlock(&fs->tsdb->mutex);
_out:
if (code) {
@ -1127,7 +1131,7 @@ int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ev
}
}
taosThreadMutexLock(&fs->tsdb->mutex);
(void)taosThreadMutexLock(&fs->tsdb->mutex);
TARRAY2_FOREACH(fs->fSetArr, fset) {
int64_t sver1 = sver;
int64_t ever1 = ever;
@ -1156,7 +1160,7 @@ int32_t tsdbFSCreateRefRangedSnapshot(STFileSystem *fs, int64_t sver, int64_t ev
fsr1 = NULL;
}
taosThreadMutexUnlock(&fs->tsdb->mutex);
(void)taosThreadMutexUnlock(&fs->tsdb->mutex);
if (code) {
tsdbTFileSetRangeClear(&fsr1);

20
source/dnode/vnode/src/tsdb/tsdbFile2.c
View File

@ -230,11 +230,11 @@ int32_t tsdbTFileObjInit(STsdb *pTsdb, const STFile *f, STFileObj **fobj) {
return terrno;
}
taosThreadMutexInit(&fobj[0]->mutex, NULL);
(void)taosThreadMutexInit(&fobj[0]->mutex, NULL);
fobj[0]->f[0] = f[0];
fobj[0]->state = TSDB_FSTATE_LIVE;
fobj[0]->ref = 1;
tsdbTFileName(pTsdb, f, fobj[0]->fname);
(void)tsdbTFileName(pTsdb, f, fobj[0]->fname);
// fobj[0]->nlevel = tfsGetLevel(pTsdb->pVnode->pTfs);
fobj[0]->nlevel = vnodeNodeId(pTsdb->pVnode);
return 0;
@ -242,18 +242,18 @@ int32_t tsdbTFileObjInit(STsdb *pTsdb, const STFile *f, STFileObj **fobj) {
int32_t tsdbTFileObjRef(STFileObj *fobj) {
int32_t nRef;
taosThreadMutexLock(&fobj->mutex);
(void)(void)taosThreadMutexLock(&fobj->mutex);
ASSERT(fobj->ref > 0 && fobj->state == TSDB_FSTATE_LIVE);
nRef = ++fobj->ref;
taosThreadMutexUnlock(&fobj->mutex);
(void)(void)taosThreadMutexUnlock(&fobj->mutex);
tsdbTrace("ref file %s, fobj:%p ref %d", fobj->fname, fobj, nRef);
return 0;
}
int32_t tsdbTFileObjUnref(STFileObj *fobj) {
taosThreadMutexLock(&fobj->mutex);
(void)(void)taosThreadMutexLock(&fobj->mutex);
int32_t nRef = --fobj->ref;
taosThreadMutexUnlock(&fobj->mutex);
(void)(void)taosThreadMutexUnlock(&fobj->mutex);
ASSERT(nRef >= 0);
tsdbTrace("unref file %s, fobj:%p ref %d", fobj->fname, fobj, nRef);
if (nRef == 0) {
@ -318,11 +318,11 @@ static void tsdbTFileObjRemoveLC(STFileObj *fobj, bool remove_all) {
}
int32_t tsdbTFileObjRemove(STFileObj *fobj) {
taosThreadMutexLock(&fobj->mutex);
(void)taosThreadMutexLock(&fobj->mutex);
ASSERT(fobj->state == TSDB_FSTATE_LIVE && fobj->ref > 0);
fobj->state = TSDB_FSTATE_DEAD;
int32_t nRef = --fobj->ref;
taosThreadMutexUnlock(&fobj->mutex);
(void)taosThreadMutexUnlock(&fobj->mutex);
tsdbTrace("remove unref file %s, fobj:%p ref %d", fobj->fname, fobj, nRef);
if (nRef == 0) {
tsdbTFileObjRemoveLC(fobj, true);
@ -332,11 +332,11 @@ int32_t tsdbTFileObjRemove(STFileObj *fobj) {
}
int32_t tsdbTFileObjRemoveUpdateLC(STFileObj *fobj) {
taosThreadMutexLock(&fobj->mutex);
(void)taosThreadMutexLock(&fobj->mutex);
ASSERT(fobj->state == TSDB_FSTATE_LIVE && fobj->ref > 0);
fobj->state = TSDB_FSTATE_DEAD;
int32_t nRef = --fobj->ref;
taosThreadMutexUnlock(&fobj->mutex);
(void)taosThreadMutexUnlock(&fobj->mutex);
tsdbTrace("remove unref file %s, fobj:%p ref %d", fobj->fname, fobj, nRef);
if (nRef == 0) {
tsdbTFileObjRemoveLC(fobj, false);

12
source/dnode/vnode/src/tsdb/tsdbIter.c
View File

@ -153,7 +153,7 @@ static int32_t tsdbDataIterNext(STsdbIter *iter, const TABLEID *tbid) {
for (; iter->dataData->brinBlockIdx < iter->dataData->brinBlock->numOfRecords; iter->dataData->brinBlockIdx++) {
SBrinRecord record[1];
tBrinBlockGet(iter->dataData->brinBlock, iter->dataData->brinBlockIdx, record);
(void)tBrinBlockGet(iter->dataData->brinBlock, iter->dataData->brinBlockIdx, record);
if (iter->filterByVersion && (record->maxVer < iter->range[0] || record->minVer > iter->range[1])) {
continue;
@ -254,7 +254,8 @@ _exit:
static int32_t tsdbDataTombIterNext(STsdbIter *iter, const TABLEID *tbid) {
while (!iter->noMoreData) {
for (; iter->dataTomb->tombBlockIdx < TOMB_BLOCK_SIZE(iter->dataTomb->tombBlock); iter->dataTomb->tombBlockIdx++) {
tTombBlockGet(iter->dataTomb->tombBlock, iter->dataTomb->tombBlockIdx, iter->record);
int32_t code = tTombBlockGet(iter->dataTomb->tombBlock, iter->dataTomb->tombBlockIdx, iter->record);
if (code) return code;
if (iter->filterByVersion && (iter->record->version < iter->range[0] || iter->record->version > iter->range[1])) {
continue;
@ -372,7 +373,7 @@ static int32_t tsdbDataIterOpen(STsdbIter *iter) {
iter->dataData->brinBlkArrayIdx = 0;
// SBrinBlock
tBrinBlockInit(iter->dataData->brinBlock);
(void)tBrinBlockInit(iter->dataData->brinBlock);
iter->dataData->brinBlockIdx = 0;
// SBlockData
@ -429,7 +430,7 @@ static int32_t tsdbMemTombIterOpen(STsdbIter *iter) {
}
static int32_t tsdbDataIterClose(STsdbIter *iter) {
tBrinBlockDestroy(iter->dataData->brinBlock);
(void)tBrinBlockDestroy(iter->dataData->brinBlock);
tBlockDataDestroy(iter->dataData->blockData);
return 0;
}
@ -439,7 +440,8 @@ static int32_t tsdbMemTableIterClose(STsdbIter *iter) { return 0; }
static int32_t tsdbSttTombIterNext(STsdbIter *iter, const TABLEID *tbid) {
while (!iter->noMoreData) {
for (; iter->sttTomb->tombBlockIdx < TOMB_BLOCK_SIZE(iter->sttTomb->tombBlock); iter->sttTomb->tombBlockIdx++) {
tTombBlockGet(iter->sttTomb->tombBlock, iter->sttTomb->tombBlockIdx, iter->record);
int32_t code = tTombBlockGet(iter->sttTomb->tombBlock, iter->sttTomb->tombBlockIdx, iter->record);
if (code) return code;
if (iter->filterByVersion && (iter->record->version < iter->range[0] || iter->record->version > iter->range[1])) {
continue;

14
source/dnode/vnode/src/tsdb/tsdbMerge.c
View File

@ -375,13 +375,13 @@ static int32_t tsdbMergeFileSetEnd(SMerger *merger) {
// edit file system
TAOS_CHECK_GOTO(tsdbFSEditBegin(merger->tsdb->pFS, merger->fopArr, TSDB_FEDIT_MERGE), &lino, _exit);
taosThreadMutexLock(&merger->tsdb->mutex);
(void)taosThreadMutexLock(&merger->tsdb->mutex);
code = tsdbFSEditCommit(merger->tsdb->pFS);
if (code) {
taosThreadMutexUnlock(&merger->tsdb->mutex);
(void)taosThreadMutexUnlock(&merger->tsdb->mutex);
TSDB_CHECK_CODE(code, lino, _exit);
}
taosThreadMutexUnlock(&merger->tsdb->mutex);
(void)taosThreadMutexUnlock(&merger->tsdb->mutex);
_exit:
if (code) {
@ -478,10 +478,10 @@ _exit:
static int32_t tsdbMergeGetFSet(SMerger *merger) {
STFileSet *fset;
taosThreadMutexLock(&merger->tsdb->mutex);
(void)taosThreadMutexLock(&merger->tsdb->mutex);
tsdbFSGetFSet(merger->tsdb->pFS, merger->fid, &fset);
if (fset == NULL) {
taosThreadMutexUnlock(&merger->tsdb->mutex);
(void)taosThreadMutexUnlock(&merger->tsdb->mutex);
return 0;
}
@ -489,10 +489,10 @@ static int32_t tsdbMergeGetFSet(SMerger *merger) {
int32_t code = tsdbTFileSetInitCopy(merger->tsdb, fset, &merger->fset);
if (code) {
taosThreadMutexUnlock(&merger->tsdb->mutex);
(void)taosThreadMutexUnlock(&merger->tsdb->mutex);
return code;
}
taosThreadMutexUnlock(&merger->tsdb->mutex);
(void)taosThreadMutexUnlock(&merger->tsdb->mutex);
return 0;
}

16
source/dnode/vnode/src/tsdb/tsdbMergeTree.c
View File

@ -63,8 +63,8 @@ int32_t tCreateSttBlockLoadInfo(STSchema *pSchema, int16_t *colList, int32_t num
return code;
}
static void freeItem(void* pValue) {
SValue* p = (SValue*) pValue;
static void freeItem(void *pValue) {
SValue *p = (SValue *)pValue;
if (IS_VAR_DATA_TYPE(p->type)) {
taosMemoryFree(p->pData);
}
@ -120,7 +120,7 @@ void destroySttBlockReader(SArray *pLDataIterArray, SSttBlockLoadCostInfo *pLoad
continue;
}
SSttBlockLoadCostInfo* pCost = &pIter->pBlockLoadInfo->cost;
SSttBlockLoadCostInfo *pCost = &pIter->pBlockLoadInfo->cost;
if (pLoadCost != NULL) {
pLoadCost->loadBlocks += pCost->loadBlocks;
pLoadCost->loadStatisBlocks += pCost->loadStatisBlocks;
@ -380,7 +380,7 @@ static int32_t loadSttStatisticsBlockData(SSttFileReader *pSttFileReader, SSttBl
pBlockLoadInfo->cost.loadStatisBlocks += num;
STbStatisBlock block;
tStatisBlockInit(&block);
TAOS_UNUSED(tStatisBlockInit(&block));
int64_t st = taosGetTimestampUs();
@ -436,7 +436,7 @@ static int32_t loadSttStatisticsBlockData(SSttFileReader *pSttFileReader, SSttBl
}
} else {
SValue vFirst = {0};
for(int32_t j = 0; j < size; ++j) {
for (int32_t j = 0; j < size; ++j) {
taosArrayPush(pBlockLoadInfo->info.pFirstKey, &vFirst);
taosArrayPush(pBlockLoadInfo->info.pLastKey, &vFirst);
}
@ -445,7 +445,7 @@ static int32_t loadSttStatisticsBlockData(SSttFileReader *pSttFileReader, SSttBl
STbStatisRecord record = {0};
while (i < rows) {
tStatisBlockGet(&block, i, &record);
(void)tStatisBlockGet(&block, i, &record);
if (record.suid != suid) {
break;
}
@ -476,7 +476,7 @@ static int32_t loadSttStatisticsBlockData(SSttFileReader *pSttFileReader, SSttBl
}
}
tStatisBlockDestroy(&block);
(void)tStatisBlockDestroy(&block);
double el = (taosGetTimestampUs() - st) / 1000.0;
pBlockLoadInfo->cost.statisElapsedTime += el;
@ -712,7 +712,7 @@ static void findNextValidRow(SLDataIter *pIter, const char *idStr) {
}
int64_t ts = pData->aTSKEY[i];
if (!pIter->backward) { // asc
if (!pIter->backward) { // asc
if (ts > pIter->timeWindow.ekey) { // no more data
break;
} else {

4
source/dnode/vnode/src/tsdb/tsdbOpen.c
View File

@ -105,10 +105,10 @@ int32_t tsdbClose(STsdb **pTsdb) {
tsdbDebug("vgId:%d, tsdb is close at %s, days:%d, keep:%d,%d,%d, keepTimeOffset:%d", TD_VID(pdb->pVnode), pdb->path,
pdb->keepCfg.days, pdb->keepCfg.keep0, pdb->keepCfg.keep1, pdb->keepCfg.keep2,
pdb->keepCfg.keepTimeOffset);
taosThreadMutexLock(&(*pTsdb)->mutex);
(void)taosThreadMutexLock(&(*pTsdb)->mutex);
tsdbMemTableDestroy((*pTsdb)->mem, true);
(*pTsdb)->mem = NULL;
taosThreadMutexUnlock(&(*pTsdb)->mutex);
(void)taosThreadMutexUnlock(&(*pTsdb)->mutex);
tsdbCloseFS(&(*pTsdb)->pFS);
tsdbCloseCache(*pTsdb);

20
source/dnode/vnode/src/tsdb/tsdbRead2.c
View File

@ -402,28 +402,30 @@ static void initReaderStatus(SReaderStatus* pStatus) {
}
static int32_t createResBlock(SQueryTableDataCond* pCond, int32_t capacity, SSDataBlock** pResBlock) {
*pResBlock = createDataBlock();
if (*pResBlock == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
QRY_OPTR_CHECK(pResBlock);
SSDataBlock* pBlock = NULL;
int32_t code = createDataBlock(&pBlock);
if (code != 0) {
return code;
}
for (int32_t i = 0; i < pCond->numOfCols; ++i) {
SColumnInfoData colInfo = {0};
colInfo.info = pCond->colList[i];
int32_t code = blockDataAppendColInfo(*pResBlock, &colInfo);
code = blockDataAppendColInfo(pBlock, &colInfo);
if (code != TSDB_CODE_SUCCESS) {
taosMemoryFree(*pResBlock);
*pResBlock = NULL;
taosMemoryFree(pBlock);
return code;
}
}
int32_t code = blockDataEnsureCapacity(*pResBlock, capacity);
code = blockDataEnsureCapacity(pBlock, capacity);
if (code != TSDB_CODE_SUCCESS) {
taosMemoryFree(*pResBlock);
*pResBlock = NULL;
taosMemoryFree(pBlock);
}
*pResBlock = pBlock;
return code;
}

87
source/dnode/vnode/src/tsdb/tsdbReadUtil.c
View File

@ -88,6 +88,9 @@ int32_t ensureBlockScanInfoBuf(SBlockInfoBuf* pBuf, int32_t numOfTables) {
int32_t remainder = (numOfTables - pBuf->numOfTables) % pBuf->numPerBucket;
if (pBuf->pData == NULL) {
pBuf->pData = taosArrayInit(num + 1, POINTER_BYTES);
if (pBuf->pData == NULL) {
return terrno;
}
}
for (int32_t i = 0; i < num; ++i) {
@ -163,22 +166,22 @@ int32_t initRowKey(SRowKey* pKey, int64_t ts, int32_t numOfPks, int32_t type, in
if (IS_NUMERIC_TYPE(type)) {
if (asc) {
switch(type) {
switch (type) {
case TSDB_DATA_TYPE_BIGINT: {
pKey->pks[0].val = INT64_MIN;
break;
}
case TSDB_DATA_TYPE_INT:{
case TSDB_DATA_TYPE_INT: {
int32_t min = INT32_MIN;
(void)memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes);
break;
}
case TSDB_DATA_TYPE_SMALLINT:{
case TSDB_DATA_TYPE_SMALLINT: {
int16_t min = INT16_MIN;
(void)memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes);
break;
}
case TSDB_DATA_TYPE_TINYINT:{
case TSDB_DATA_TYPE_TINYINT: {
int8_t min = INT8_MIN;
(void)memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes);
break;
@ -194,15 +197,31 @@ int32_t initRowKey(SRowKey* pKey, int64_t ts, int32_t numOfPks, int32_t type, in
ASSERT(0);
}
} else {
switch(type) {
case TSDB_DATA_TYPE_BIGINT:pKey->pks[0].val = INT64_MAX;break;
case TSDB_DATA_TYPE_INT:pKey->pks[0].val = INT32_MAX;break;
case TSDB_DATA_TYPE_SMALLINT:pKey->pks[0].val = INT16_MAX;break;
case TSDB_DATA_TYPE_TINYINT:pKey->pks[0].val = INT8_MAX;break;
case TSDB_DATA_TYPE_UBIGINT:pKey->pks[0].val = UINT64_MAX;break;
case TSDB_DATA_TYPE_UINT:pKey->pks[0].val = UINT32_MAX;break;
case TSDB_DATA_TYPE_USMALLINT:pKey->pks[0].val = UINT16_MAX;break;
case TSDB_DATA_TYPE_UTINYINT:pKey->pks[0].val = UINT8_MAX;break;
switch (type) {
case TSDB_DATA_TYPE_BIGINT:
pKey->pks[0].val = INT64_MAX;
break;
case TSDB_DATA_TYPE_INT:
pKey->pks[0].val = INT32_MAX;
break;
case TSDB_DATA_TYPE_SMALLINT:
pKey->pks[0].val = INT16_MAX;
break;
case TSDB_DATA_TYPE_TINYINT:
pKey->pks[0].val = INT8_MAX;
break;
case TSDB_DATA_TYPE_UBIGINT:
pKey->pks[0].val = UINT64_MAX;
break;
case TSDB_DATA_TYPE_UINT:
pKey->pks[0].val = UINT32_MAX;
break;
case TSDB_DATA_TYPE_USMALLINT:
pKey->pks[0].val = UINT16_MAX;
break;
case TSDB_DATA_TYPE_UTINYINT:
pKey->pks[0].val = UINT8_MAX;
break;
default:
ASSERT(0);
}
@ -232,7 +251,7 @@ void clearRowKey(SRowKey* pKey) {
taosMemoryFreeClear(pKey->pks[0].pData);
}
static int32_t initLastProcKey(STableBlockScanInfo *pScanInfo, STsdbReader* pReader) {
static int32_t initLastProcKey(STableBlockScanInfo* pScanInfo, STsdbReader* pReader) {
int32_t code = 0;
int32_t numOfPks = pReader->suppInfo.numOfPks;
bool asc = ASCENDING_TRAVERSE(pReader->info.order);
@ -448,8 +467,8 @@ void cleanupInfoForNextFileset(SSHashObj* pTableMap) {
// brin records iterator
void initBrinRecordIter(SBrinRecordIter* pIter, SDataFileReader* pReader, SArray* pList) {
(void) memset(&pIter->block, 0, sizeof(SBrinBlock));
(void) memset(&pIter->record, 0, sizeof(SBrinRecord));
(void)memset(&pIter->block, 0, sizeof(SBrinBlock));
(void)memset(&pIter->record, 0, sizeof(SBrinRecord));
pIter->blockIndex = -1;
pIter->recordIndex = -1;
@ -471,7 +490,7 @@ int32_t getNextBrinRecord(SBrinRecordIter* pIter, SBrinRecord** pRecord) {
return TSDB_CODE_INVALID_PARA;
}
(void) tBrinBlockClear(&pIter->block);
(void)tBrinBlockClear(&pIter->block);
int32_t code = tsdbDataFileReadBrinBlock(pIter->pReader, pIter->pCurrentBlk, &pIter->block);
if (code != TSDB_CODE_SUCCESS) {
tsdbError("failed to read brinBlock from file, code:%s", tstrerror(code));
@ -488,7 +507,7 @@ int32_t getNextBrinRecord(SBrinRecordIter* pIter, SBrinRecord** pRecord) {
return code;
}
void clearBrinBlockIter(SBrinRecordIter* pIter) { (void) tBrinBlockDestroy(&pIter->block); }
void clearBrinBlockIter(SBrinRecordIter* pIter) { (void)tBrinBlockDestroy(&pIter->block); }
// initialize the file block access order
// sort the file blocks according to the offset of each data block in the files
@ -658,7 +677,7 @@ int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIter, int3
STableBlockScanInfo* pTableScanInfo = taosArrayGetP(pTableList, 0);
for (int32_t i = 0; i < numOfBlocks; ++i) {
STableDataBlockIdx tableDataBlockIdx = {.globalIndex = i};
void* px = taosArrayPush(pTableScanInfo->pBlockIdxList, &tableDataBlockIdx);
void* px = taosArrayPush(pTableScanInfo->pBlockIdxList, &tableDataBlockIdx);
if (px == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
@ -774,6 +793,9 @@ static int32_t doCheckTombBlock(STombBlock* pBlock, STsdbReader* pReader, int32_
if (pScanInfo->pFileDelData == NULL) {
pScanInfo->pFileDelData = taosArrayInit(4, sizeof(SDelData));
if (pScanInfo->pFileDelData == NULL) {
return terrno;
}
}
for (int32_t k = 0; k < pBlock->numOfRecords; ++k) {
@ -810,6 +832,9 @@ static int32_t doCheckTombBlock(STombBlock* pBlock, STsdbReader* pReader, int32_
if (pScanInfo->pFileDelData == NULL) {
pScanInfo->pFileDelData = taosArrayInit(4, sizeof(SDelData));
if (pScanInfo->pFileDelData == NULL) {
return terrno;
}
}
}
@ -821,7 +846,7 @@ static int32_t doCheckTombBlock(STombBlock* pBlock, STsdbReader* pReader, int32_
if (record.version <= pReader->info.verRange.maxVer) {
SDelData delData = {.version = record.version, .sKey = record.skey, .eKey = record.ekey};
void* px = taosArrayPush(pScanInfo->pFileDelData, &delData);
void* px = taosArrayPush(pScanInfo->pFileDelData, &delData);
if (px == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
@ -878,7 +903,7 @@ static int32_t doLoadTombDataFromTombBlk(const TTombBlkArray* pTombBlkArray, STs
ETombBlkCheckEnum ret = 0;
code = doCheckTombBlock(&block, pReader, numOfTables, &j, &ret);
(void) tTombBlockDestroy(&block);
(void)tTombBlockDestroy(&block);
if (code != TSDB_CODE_SUCCESS || ret == BLK_CHECK_QUIT) {
return code;
}
@ -977,7 +1002,7 @@ int32_t getNumOfRowsInSttBlock(SSttFileReader* pSttFileReader, SSttBlockLoadInfo
SStatisBlk* p = &pStatisBlkArray->data[i];
STbStatisBlock* pStatisBlock = taosMemoryCalloc(1, sizeof(STbStatisBlock));
(void) tStatisBlockInit(pStatisBlock);
(void)tStatisBlockInit(pStatisBlock);
int64_t st = taosGetTimestampMs();
int32_t code = tsdbSttFileReadStatisBlock(pSttFileReader, p, pStatisBlock);
@ -995,7 +1020,7 @@ int32_t getNumOfRowsInSttBlock(SSttFileReader* pSttFileReader, SSttBlockLoadInfo
}
if (index >= pStatisBlock->numOfRecords) {
(void) tStatisBlockDestroy(pStatisBlock);
(void)tStatisBlockDestroy(pStatisBlock);
taosMemoryFreeClear(pStatisBlock);
return num;
}
@ -1005,7 +1030,7 @@ int32_t getNumOfRowsInSttBlock(SSttFileReader* pSttFileReader, SSttBlockLoadInfo
while (i < TARRAY2_SIZE(pStatisBlkArray) && uidIndex < numOfTables) {
p = &pStatisBlkArray->data[i];
if (p->minTbid.suid > suid) {
(void) tStatisBlockDestroy(pStatisBlock);
(void)tStatisBlockDestroy(pStatisBlock);
taosMemoryFreeClear(pStatisBlock);
return num;
}
@ -1025,7 +1050,7 @@ int32_t getNumOfRowsInSttBlock(SSttFileReader* pSttFileReader, SSttBlockLoadInfo
}
}
(void) tStatisBlockDestroy(pStatisBlock);
(void)tStatisBlockDestroy(pStatisBlock);
taosMemoryFreeClear(pStatisBlock);
return num;
}
@ -1037,7 +1062,7 @@ static void loadNextStatisticsBlock(SSttFileReader* pSttFileReader, STbStatisBlo
(*i) += 1;
(*j) = 0;
if ((*i) < TARRAY2_SIZE(pStatisBlkArray)) {
(void) tsdbSttFileReadStatisBlock(pSttFileReader, &pStatisBlkArray->data[(*i)], pStatisBlock);
(void)tsdbSttFileReadStatisBlock(pSttFileReader, &pStatisBlkArray->data[(*i)], pStatisBlock);
}
}
}
@ -1049,7 +1074,7 @@ int32_t doAdjustValidDataIters(SArray* pLDIterList, int32_t numOfFileObj) {
int32_t inc = numOfFileObj - size;
for (int32_t k = 0; k < inc; ++k) {
SLDataIter* pIter = taosMemoryCalloc(1, sizeof(SLDataIter));
void* px = taosArrayPush(pLDIterList, &pIter);
void* px = taosArrayPush(pLDIterList, &pIter);
if (px == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
@ -1073,6 +1098,9 @@ int32_t adjustSttDataIters(SArray* pSttFileBlockIterArray, STFileSet* pFileSet)
// add the list/iter placeholder
while (taosArrayGetSize(pSttFileBlockIterArray) < numOfLevels) {
SArray* pList = taosArrayInit(4, POINTER_BYTES);
if (pList == NULL) {
return terrno;
}
void* px = taosArrayPush(pSttFileBlockIterArray, &pList);
if (px == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
@ -1210,8 +1238,7 @@ static int32_t sortUidComparFn(const void* p1, const void* p2) {
return ret;
}
bool isCleanSttBlock(SArray* pKeyRangeList, STimeWindow* pQueryWindow, STableBlockScanInfo* pScanInfo,
int32_t order) {
bool isCleanSttBlock(SArray* pKeyRangeList, STimeWindow* pQueryWindow, STableBlockScanInfo* pScanInfo, int32_t order) {
// check if it overlap with del skyline
taosArraySort(pKeyRangeList, sortUidComparFn);
@ -1242,7 +1269,7 @@ bool isCleanSttBlock(SArray* pKeyRangeList, STimeWindow* pQueryWindow, STableBlo
}
STimeWindow w2 = {.skey = p2->skey.ts, .ekey = p2->ekey.ts};
bool overlap = overlapWithTimeWindow(&w2, pQueryWindow, pScanInfo, order);
bool overlap = overlapWithTimeWindow(&w2, pQueryWindow, pScanInfo, order);
if (overlap) {
return false;
}

24
source/dnode/vnode/src/tsdb/tsdbRetention.c
View File

@ -209,15 +209,15 @@ static int32_t tsdbDoRetentionEnd(SRTNer *rtner) {
if (TARRAY2_SIZE(&rtner->fopArr) > 0) {
TAOS_CHECK_GOTO(tsdbFSEditBegin(rtner->tsdb->pFS, &rtner->fopArr, TSDB_FEDIT_RETENTION), &lino, _exit);
taosThreadMutexLock(&rtner->tsdb->mutex);
(void)taosThreadMutexLock(&rtner->tsdb->mutex);
code = tsdbFSEditCommit(rtner->tsdb->pFS);
if (code) {
taosThreadMutexUnlock(&rtner->tsdb->mutex);
(void)taosThreadMutexUnlock(&rtner->tsdb->mutex);
TSDB_CHECK_CODE(code, lino, _exit);
}
taosThreadMutexUnlock(&rtner->tsdb->mutex);
(void)taosThreadMutexUnlock(&rtner->tsdb->mutex);
TARRAY2_DESTROY(&rtner->fopArr, NULL);
}
@ -315,13 +315,13 @@ static int32_t tsdbRetention(void *arg) {
};
// begin task
taosThreadMutexLock(&pTsdb->mutex);
(void)taosThreadMutexLock(&pTsdb->mutex);
tsdbBeginTaskOnFileSet(pTsdb, rtnArg->fid, &fset);
if (fset && (code = tsdbTFileSetInitCopy(pTsdb, fset, &rtner.fset))) {
taosThreadMutexUnlock(&pTsdb->mutex);
(void)taosThreadMutexUnlock(&pTsdb->mutex);
TSDB_CHECK_CODE(code, lino, _exit);
}
taosThreadMutexUnlock(&pTsdb->mutex);
(void)taosThreadMutexUnlock(&pTsdb->mutex);
// do retention
if (rtner.fset) {
@ -336,9 +336,9 @@ static int32_t tsdbRetention(void *arg) {
_exit:
if (rtner.fset) {
taosThreadMutexLock(&pTsdb->mutex);
(void)taosThreadMutexLock(&pTsdb->mutex);
tsdbFinishTaskOnFileSet(pTsdb, rtnArg->fid);
taosThreadMutexUnlock(&pTsdb->mutex);
(void)taosThreadMutexUnlock(&pTsdb->mutex);
}
// clear resources
@ -387,9 +387,9 @@ _exit:
int32_t tsdbAsyncRetention(STsdb *tsdb, int64_t now) {
int32_t code = 0;
taosThreadMutexLock(&tsdb->mutex);
(void)taosThreadMutexLock(&tsdb->mutex);
code = tsdbAsyncRetentionImpl(tsdb, now, false);
taosThreadMutexUnlock(&tsdb->mutex);
(void)taosThreadMutexUnlock(&tsdb->mutex);
return code;
}
@ -735,9 +735,9 @@ int32_t tsdbAsyncS3Migrate(STsdb *tsdb, int64_t now) {
return 0;
}
taosThreadMutexLock(&tsdb->mutex);
(void)taosThreadMutexLock(&tsdb->mutex);
code = tsdbAsyncRetentionImpl(tsdb, now, true);
taosThreadMutexUnlock(&tsdb->mutex);
(void)taosThreadMutexUnlock(&tsdb->mutex);
if (code) {
tsdbError("vgId:%d, %s failed, reason:%s", TD_VID(tsdb->pVnode), __func__, tstrerror(code));

4
source/dnode/vnode/src/tsdb/tsdbSnapInfo.c
View File

@ -351,7 +351,7 @@ static STsdbFSetPartList* tsdbSnapGetFSetPartList(STFileSystem* fs) {
}
int32_t code = 0;
taosThreadMutexLock(&fs->tsdb->mutex);
(void)taosThreadMutexLock(&fs->tsdb->mutex);
STFileSet* fset;
TARRAY2_FOREACH(fs->fSetArr, fset) {
STsdbFSetPartition* pItem = NULL;
@ -364,7 +364,7 @@ static STsdbFSetPartList* tsdbSnapGetFSetPartList(STFileSystem* fs) {
code = TARRAY2_SORT_INSERT(pList, pItem, tsdbFSetPartCmprFn);
ASSERT(code == 0);
}
taosThreadMutexUnlock(&fs->tsdb->mutex);
(void)taosThreadMutexUnlock(&fs->tsdb->mutex);
if (code) {
TARRAY2_DESTROY(pList, tsdbFSetPartitionClear);

12
source/dnode/vnode/src/tsdb/tsdbSnapshot.c
View File

@ -1022,7 +1022,8 @@ static int32_t tsdbSnapWriteTombData(STsdbSnapWriter* writer, SSnapDataHdr* hdr)
code = tsdbSnapWriteDecmprTombBlock(hdr, tombBlock);
TSDB_CHECK_CODE(code, lino, _exit);
tTombBlockGet(tombBlock, 0, &record);
code = tTombBlockGet(tombBlock, 0, &record);
TSDB_CHECK_CODE(code, lino, _exit);
int32_t fid = tsdbKeyFid(record.skey, writer->minutes, writer->precision);
if (!writer->ctx->fsetWriteBegin || fid != writer->ctx->fid) {
code = tsdbSnapWriteFileSetEnd(writer);
@ -1045,7 +1046,8 @@ static int32_t tsdbSnapWriteTombData(STsdbSnapWriter* writer, SSnapDataHdr* hdr)
ASSERT(writer->ctx->hasData == false);
for (int32_t i = 0; i < TOMB_BLOCK_SIZE(tombBlock); ++i) {
tTombBlockGet(tombBlock, i, &record);
code = tTombBlockGet(tombBlock, i, &record);
TSDB_CHECK_CODE(code, lino, _exit);
code = tsdbSnapWriteTombRecord(writer, &record);
TSDB_CHECK_CODE(code, lino, _exit);
@ -1125,17 +1127,17 @@ int32_t tsdbSnapWriterClose(STsdbSnapWriter** writer, int8_t rollback) {
code = tsdbFSEditAbort(writer[0]->tsdb->pFS);
TSDB_CHECK_CODE(code, lino, _exit);
} else {
taosThreadMutexLock(&writer[0]->tsdb->mutex);
(void)taosThreadMutexLock(&writer[0]->tsdb->mutex);
code = tsdbFSEditCommit(writer[0]->tsdb->pFS);
if (code) {
taosThreadMutexUnlock(&writer[0]->tsdb->mutex);
(void)taosThreadMutexUnlock(&writer[0]->tsdb->mutex);
TSDB_CHECK_CODE(code, lino, _exit);
}
writer[0]->tsdb->pFS->fsstate = TSDB_FS_STATE_NORMAL;
taosThreadMutexUnlock(&writer[0]->tsdb->mutex);
(void)taosThreadMutexUnlock(&writer[0]->tsdb->mutex);
}
tsdbIterMergerClose(&writer[0]->ctx->tombIterMerger);

6
source/dnode/vnode/src/tsdb/tsdbSnapshotRAW.c
View File

@ -485,17 +485,17 @@ int32_t tsdbSnapRAWWriterClose(STsdbSnapRAWWriter** writer, int8_t rollback) {
code = tsdbFSEditAbort(writer[0]->tsdb->pFS);
TSDB_CHECK_CODE(code, lino, _exit);
} else {
taosThreadMutexLock(&writer[0]->tsdb->mutex);
(void)taosThreadMutexLock(&writer[0]->tsdb->mutex);
code = tsdbFSEditCommit(writer[0]->tsdb->pFS);
if (code) {
taosThreadMutexUnlock(&writer[0]->tsdb->mutex);
(void)taosThreadMutexUnlock(&writer[0]->tsdb->mutex);
TSDB_CHECK_CODE(code, lino, _exit);
}
writer[0]->tsdb->pFS->fsstate = TSDB_FS_STATE_NORMAL;
taosThreadMutexUnlock(&writer[0]->tsdb->mutex);
(void)taosThreadMutexUnlock(&writer[0]->tsdb->mutex);
}
TARRAY2_DESTROY(writer[0]->fopArr, NULL);

10
source/dnode/vnode/src/tsdb/tsdbSttFileRW.c
View File

@ -401,7 +401,7 @@ int32_t tsdbSttFileReadStatisBlock(SSttFileReader *reader, const SStatisBlk *sta
&lino, _exit);
// decode data
tStatisBlockClear(statisBlock);
TAOS_UNUSED(tStatisBlockClear(statisBlock));
statisBlock->numOfPKs = statisBlk->numOfPKs;
statisBlock->numOfRecords = statisBlk->numRec;
SBufferReader br = BUFFER_READER_INITIALIZER(0, buffer0);
@ -574,11 +574,11 @@ static int32_t tsdbSttFileDoWriteStatisBlock(SSttFileWriter *writer) {
statisBlk.cmprAlg = writer->config->cmprAlg;
statisBlk.numOfPKs = statisBlock->numOfPKs;
tStatisBlockGet(statisBlock, 0, &record);
(void)tStatisBlockGet(statisBlock, 0, &record);
statisBlk.minTbid.suid = record.suid;
statisBlk.minTbid.uid = record.uid;
tStatisBlockGet(statisBlock, statisBlock->numOfRecords - 1, &record);
(void)tStatisBlockGet(statisBlock, statisBlock->numOfRecords - 1, &record);
statisBlk.maxTbid.suid = record.suid;
statisBlk.maxTbid.uid = record.uid;
@ -636,7 +636,7 @@ static int32_t tsdbSttFileDoWriteStatisBlock(SSttFileWriter *writer) {
TAOS_CHECK_GOTO(TARRAY2_APPEND_PTR(writer->statisBlkArray, &statisBlk), &lino, _exit);
tStatisBlockClear(writer->staticBlock);
TAOS_UNUSED(tStatisBlockClear(writer->staticBlock));
_exit:
if (code) {
@ -822,7 +822,7 @@ static void tsdbSttFWriterDoClose(SSttFileWriter *writer) {
tDestroyTSchema(writer->skmRow->pTSchema);
tDestroyTSchema(writer->skmTb->pTSchema);
tTombBlockDestroy(writer->tombBlock);
tStatisBlockDestroy(writer->staticBlock);
(void)tStatisBlockDestroy(writer->staticBlock);
tBlockDataDestroy(writer->blockData);
TARRAY2_DESTROY(writer->tombBlkArray, NULL);
TARRAY2_DESTROY(writer->statisBlkArray, NULL);

2
source/dnode/vnode/src/tsdb/tsdbUpgrade.c
View File

@ -167,7 +167,7 @@ _exit:
tsdbError("vgId:%d %s failed at %s:%d since %s", TD_VID(tsdb->pVnode), __func__, __FILE__, lino, tstrerror(code));
}
TARRAY2_DESTROY(ctx->brinBlkArray, NULL);
tBrinBlockDestroy(ctx->brinBlock);
(void)tBrinBlockDestroy(ctx->brinBlock);
tBlockDataDestroy(ctx->blockData);
tMapDataClear(ctx->mDataBlk);
taosArrayDestroy(ctx->aBlockIdx);

31
source/dnode/vnode/src/tsdb/tsdbUtil2.c
View File

@ -16,35 +16,26 @@
#include "tsdbUtil2.h"
// SDelBlock ----------
int32_t tTombBlockInit(STombBlock *tombBlock) {
int32_t code;
void tTombBlockInit(STombBlock *tombBlock) {
tombBlock->numOfRecords = 0;
for (int32_t i = 0; i < TOMB_RECORD_ELEM_NUM; ++i) {
TAOS_CHECK_GOTO(tBufferInit(&tombBlock->buffers[i]), NULL, _exit);
tBufferInit(&tombBlock->buffers[i]);
}
_exit:
if (code) {
TAOS_UNUSED(tTombBlockDestroy(tombBlock));
}
return code;
return;
}
int32_t tTombBlockDestroy(STombBlock *tombBlock) {
void tTombBlockDestroy(STombBlock *tombBlock) {
tombBlock->numOfRecords = 0;
for (int32_t i = 0; i < TOMB_RECORD_ELEM_NUM; ++i) {
TAOS_UNUSED(tBufferDestroy(&tombBlock->buffers[i]));
tBufferDestroy(&tombBlock->buffers[i]);
}
return 0;
}
int32_t tTombBlockClear(STombBlock *tombBlock) {
void tTombBlockClear(STombBlock *tombBlock) {
tombBlock->numOfRecords = 0;
for (int32_t i = 0; i < TOMB_RECORD_ELEM_NUM; ++i) {
TAOS_UNUSED(tBufferClear(&tombBlock->buffers[i]));
tBufferClear(&tombBlock->buffers[i]);
}
return 0;
}
int32_t tTombBlockPut(STombBlock *tombBlock, const STombRecord *record) {
@ -79,12 +70,12 @@ int32_t tTombRecordCompare(const STombRecord *r1, const STombRecord *r2) {
// STbStatisBlock ----------
int32_t tStatisBlockInit(STbStatisBlock *statisBlock) {
int32_t code;
int32_t code = 0;
statisBlock->numOfPKs = 0;
statisBlock->numOfRecords = 0;
for (int32_t i = 0; i < ARRAY_SIZE(statisBlock->buffers); ++i) {
TAOS_CHECK_GOTO(tBufferInit(&statisBlock->buffers[i]), NULL, _exit);
tBufferInit(&statisBlock->buffers[i]);
}
for (int32_t i = 0; i < TD_MAX_PK_COLS; ++i) {
TAOS_CHECK_GOTO(tValueColumnInit(&statisBlock->firstKeyPKs[i]), NULL, _exit);
@ -244,7 +235,7 @@ int32_t tBrinBlockInit(SBrinBlock *brinBlock) {
brinBlock->numOfPKs = 0;
brinBlock->numOfRecords = 0;
for (int32_t i = 0; i < ARRAY_SIZE(brinBlock->buffers); ++i) {
TAOS_CHECK_GOTO(tBufferInit(&brinBlock->buffers[i]), NULL, _exit);
tBufferInit(&brinBlock->buffers[i]);
}
for (int32_t i = 0; i < TD_MAX_PK_COLS; ++i) {
TAOS_CHECK_GOTO(tValueColumnInit(&brinBlock->firstKeyPKs[i]), NULL, _exit);
@ -253,7 +244,7 @@ int32_t tBrinBlockInit(SBrinBlock *brinBlock) {
_exit:
if (code) {
TAOS_UNUSED(tBrinBlockDestroy(brinBlock));
(void)tBrinBlockDestroy(brinBlock);
}
return code;
}

6
source/dnode/vnode/src/tsdb/tsdbUtil2.h
View File

@ -65,9 +65,9 @@ typedef TARRAY2(STombBlk) TTombBlkArray;
#define TOMB_BLOCK_SIZE(db) ((db)->numOfRecords)
int32_t tTombBlockInit(STombBlock *tombBlock);
int32_t tTombBlockDestroy(STombBlock *tombBlock);
int32_t tTombBlockClear(STombBlock *tombBlock);
void tTombBlockInit(STombBlock *tombBlock);
void tTombBlockDestroy(STombBlock *tombBlock);
void tTombBlockClear(STombBlock *tombBlock);
int32_t tTombBlockPut(STombBlock *tombBlock, const STombRecord *record);
int32_t tTombBlockGet(STombBlock *tombBlock, int32_t idx, STombRecord *record);
int32_t tTombRecordCompare(const STombRecord *record1, const STombRecord *record2);

44
source/dnode/vnode/src/vnd/vnodeAsync.c
View File

@ -213,7 +213,7 @@ static void *vnodeAsyncLoop(void *arg) {
setThreadName(async->label);
for (;;) {
taosThreadMutexLock(&async->mutex);
(void)taosThreadMutexLock(&async->mutex);
// finish last running task
if (worker->runningTask != NULL) {
@ -228,7 +228,7 @@ static void *vnodeAsyncLoop(void *arg) {
}
worker->state = EVA_WORKER_STATE_STOP;
async->numLaunchWorkers--;
taosThreadMutexUnlock(&async->mutex);
(void)taosThreadMutexUnlock(&async->mutex);
goto _exit;
}
@ -268,7 +268,7 @@ static void *vnodeAsyncLoop(void *arg) {
}
}
taosThreadMutexUnlock(&async->mutex);
(void)taosThreadMutexUnlock(&async->mutex);
// do run the task
worker->runningTask->execute(worker->runningTask->arg);
@ -387,16 +387,16 @@ static int32_t vnodeAsyncDestroy(SVAsync **async) {
}
// set stop and broadcast
taosThreadMutexLock(&(*async)->mutex);
(void)taosThreadMutexLock(&(*async)->mutex);
(*async)->stop = true;
taosThreadCondBroadcast(&(*async)->hasTask);
taosThreadMutexUnlock(&(*async)->mutex);
(void)taosThreadMutexUnlock(&(*async)->mutex);
// join all workers
for (int32_t i = 0; i < VNODE_ASYNC_MAX_WORKERS; i++) {
taosThreadMutexLock(&(*async)->mutex);
(void)taosThreadMutexLock(&(*async)->mutex);
EVWorkerState state = (*async)->workers[i].state;
taosThreadMutexUnlock(&(*async)->mutex);
(void)taosThreadMutexUnlock(&(*async)->mutex);
if (state == EVA_WORKER_STATE_UINIT) {
continue;
@ -504,7 +504,7 @@ int32_t vnodeAsync(SVAChannelID *channelID, EVAPriority priority, int32_t (*exec
taosThreadCondInit(&task->waitCond, NULL);
// schedule task
taosThreadMutexLock(&async->mutex);
(void)taosThreadMutexLock(&async->mutex);
if (channelID->id == 0) {
task->channel = NULL;
@ -514,7 +514,7 @@ int32_t vnodeAsync(SVAChannelID *channelID, EVAPriority priority, int32_t (*exec
};
vHashGet(async->channelTable, &channel, (void **)&task->channel);
if (task->channel == NULL) {
taosThreadMutexUnlock(&async->mutex);
(void)taosThreadMutexUnlock(&async->mutex);
taosThreadCondDestroy(&task->waitCond);
taosMemoryFree(task);
return TSDB_CODE_INVALID_PARA;
@ -526,7 +526,7 @@ int32_t vnodeAsync(SVAChannelID *channelID, EVAPriority priority, int32_t (*exec
// add task to hash table
int32_t ret = vHashPut(async->taskTable, task);
if (ret != 0) {
taosThreadMutexUnlock(&async->mutex);
(void)taosThreadMutexUnlock(&async->mutex);
taosThreadCondDestroy(&task->waitCond);
taosMemoryFree(task);
return ret;
@ -580,7 +580,7 @@ int32_t vnodeAsync(SVAChannelID *channelID, EVAPriority priority, int32_t (*exec
task->prev->next = task;
}
taosThreadMutexUnlock(&async->mutex);
(void)taosThreadMutexUnlock(&async->mutex);
if (taskID != NULL) {
taskID->async = channelID->async;
@ -601,7 +601,7 @@ int32_t vnodeAWait(SVATaskID *taskID) {
.taskId = taskID->id,
};
taosThreadMutexLock(&async->mutex);
(void)taosThreadMutexLock(&async->mutex);
vHashGet(async->taskTable, &task2, (void **)&task);
if (task) {
@ -615,7 +615,7 @@ int32_t vnodeAWait(SVATaskID *taskID) {
}
}
taosThreadMutexUnlock(&async->mutex);
(void)taosThreadMutexUnlock(&async->mutex);
return 0;
}
@ -634,7 +634,7 @@ int32_t vnodeACancel(SVATaskID *taskID) {
void (*cancel)(void *) = NULL;
void *arg = NULL;
taosThreadMutexLock(&async->mutex);
(void)taosThreadMutexLock(&async->mutex);
vHashGet(async->taskTable, &task2, (void **)&task);
if (task) {
@ -649,7 +649,7 @@ int32_t vnodeACancel(SVATaskID *taskID) {
}
}
taosThreadMutexUnlock(&async->mutex);
(void)taosThreadMutexUnlock(&async->mutex);
if (cancel) {
cancel(arg);
@ -663,12 +663,12 @@ int32_t vnodeAsyncSetWorkers(int64_t asyncID, int32_t numWorkers) {
return TSDB_CODE_INVALID_PARA;
}
SVAsync *async = vnodeAsyncs[asyncID];
taosThreadMutexLock(&async->mutex);
(void)taosThreadMutexLock(&async->mutex);
async->numWorkers = numWorkers;
if (async->numIdleWorkers > 0) {
taosThreadCondBroadcast(&async->hasTask);
}
taosThreadMutexUnlock(&async->mutex);
(void)taosThreadMutexUnlock(&async->mutex);
return 0;
}
@ -693,14 +693,14 @@ int32_t vnodeAChannelInit(int64_t asyncID, SVAChannelID *channelID) {
channel->scheduled = NULL;
// register channel
taosThreadMutexLock(&async->mutex);
(void)taosThreadMutexLock(&async->mutex);
channel->channelId = channelID->id = ++async->nextChannelId;
// add to hash table
int32_t ret = vHashPut(async->channelTable, channel);
if (ret != 0) {
taosThreadMutexUnlock(&async->mutex);
(void)taosThreadMutexUnlock(&async->mutex);
taosMemoryFree(channel);
return ret;
}
@ -713,7 +713,7 @@ int32_t vnodeAChannelInit(int64_t asyncID, SVAChannelID *channelID) {
async->numChannels++;
taosThreadMutexUnlock(&async->mutex);
(void)taosThreadMutexUnlock(&async->mutex);
channelID->async = asyncID;
return 0;
@ -734,7 +734,7 @@ int32_t vnodeAChannelDestroy(SVAChannelID *channelID, bool waitRunning) {
return TSDB_CODE_OUT_OF_MEMORY;
}
taosThreadMutexLock(&async->mutex);
(void)taosThreadMutexLock(&async->mutex);
vHashGet(async->channelTable, &channel2, (void **)&channel);
if (channel) {
@ -793,7 +793,7 @@ int32_t vnodeAChannelDestroy(SVAChannelID *channelID, bool waitRunning) {
}
}
taosThreadMutexUnlock(&async->mutex);
(void)taosThreadMutexUnlock(&async->mutex);
for (int32_t i = 0; i < taosArrayGetSize(cancelArray); i++) {
SVATaskCancelInfo *cancel = (SVATaskCancelInfo *)taosArrayGet(cancelArray, i);
cancel->cancel(cancel->arg);

24
source/dnode/vnode/src/vnd/vnodeBufPool.c
View File

@ -242,7 +242,9 @@ void vnodeBufPoolUnRef(SVBufPool *pPool, bool proactive) {
SVnode *pVnode = pPool->pVnode;
if (proactive) taosThreadMutexLock(&pVnode->mutex);
if (proactive) {
(void)taosThreadMutexLock(&pVnode->mutex);
}
if (atomic_sub_fetch_32(&pPool->nRef, 1) > 0) goto _exit;
@ -267,12 +269,14 @@ void vnodeBufPoolUnRef(SVBufPool *pPool, bool proactive) {
vnodeBufPoolAddToFreeList(pPool);
_exit:
if (proactive) taosThreadMutexUnlock(&pVnode->mutex);
if (proactive) {
(void)taosThreadMutexUnlock(&pVnode->mutex);
}
return;
}
int32_t vnodeBufPoolRegisterQuery(SVBufPool *pPool, SQueryNode *pQNode) {
taosThreadMutexLock(&pPool->mutex);
(void)taosThreadMutexLock(&pPool->mutex);
pQNode->pNext = pPool->qList.pNext;
pQNode->ppNext = &pPool->qList.pNext;
@ -280,20 +284,24 @@ int32_t vnodeBufPoolRegisterQuery(SVBufPool *pPool, SQueryNode *pQNode) {
pPool->qList.pNext = pQNode;
pPool->nQuery++;
taosThreadMutexUnlock(&pPool->mutex);
(void)taosThreadMutexUnlock(&pPool->mutex);
return 0;
}
void vnodeBufPoolDeregisterQuery(SVBufPool *pPool, SQueryNode *pQNode, bool proactive) {
int32_t code = 0;
if (proactive) taosThreadMutexLock(&pPool->mutex);
if (proactive) {
(void)taosThreadMutexLock(&pPool->mutex);
}
pQNode->pNext->ppNext = pQNode->ppNext;
*pQNode->ppNext = pQNode->pNext;
pPool->nQuery--;
if (proactive) taosThreadMutexUnlock(&pPool->mutex);
if (proactive) {
(void)taosThreadMutexUnlock(&pPool->mutex);
}
}
int32_t vnodeBufPoolRecycle(SVBufPool *pPool) {
@ -303,7 +311,7 @@ int32_t vnodeBufPoolRecycle(SVBufPool *pPool) {
vDebug("vgId:%d, recycle buffer pool %p of id %d", TD_VID(pVnode), pPool, pPool->id);
taosThreadMutexLock(&pPool->mutex);
(void)taosThreadMutexLock(&pPool->mutex);
SQueryNode *pNode = pPool->qList.pNext;
while (pNode != &pPool->qList) {
@ -319,6 +327,6 @@ int32_t vnodeBufPoolRecycle(SVBufPool *pPool) {
}
_exit:
taosThreadMutexUnlock(&pPool->mutex);
(void)taosThreadMutexUnlock(&pPool->mutex);
return code;
}

16
source/dnode/vnode/src/vnd/vnodeCommit.c
View File

@ -65,7 +65,7 @@ static int32_t vnodeGetBufPoolToUse(SVnode *pVnode) {
int32_t code = 0;
int32_t lino = 0;
taosThreadMutexLock(&pVnode->mutex);
(void)taosThreadMutexLock(&pVnode->mutex);
int32_t nTry = 0;
for (;;) {
@ -110,7 +110,7 @@ static int32_t vnodeGetBufPoolToUse(SVnode *pVnode) {
}
_exit:
taosThreadMutexUnlock(&pVnode->mutex);
(void)taosThreadMutexUnlock(&pVnode->mutex);
if (code) {
vError("vgId:%d, %s failed at line %d since %s", TD_VID(pVnode), __func__, lino, tstrerror(code));
}
@ -150,7 +150,7 @@ int vnodeShouldCommit(SVnode *pVnode, bool atExit) {
bool diskAvail = osDataSpaceAvailable();
bool needCommit = false;
taosThreadMutexLock(&pVnode->mutex);
(void)taosThreadMutexLock(&pVnode->mutex);
if (pVnode->inUse && diskAvail) {
needCommit = (pVnode->inUse->size > pVnode->inUse->node.size) ||
(atExit && (pVnode->inUse->size > 0 || pVnode->pMeta->changed ||
@ -162,7 +162,7 @@ int vnodeShouldCommit(SVnode *pVnode, bool atExit) {
TD_VID(pVnode), needCommit, diskAvail, pVnode->inUse ? pVnode->inUse->size : 0,
pVnode->inUse ? pVnode->inUse->node.size : 0, pVnode->pMeta->changed, pVnode->state.applied,
pVnode->state.committed);
taosThreadMutexUnlock(&pVnode->mutex);
(void)taosThreadMutexUnlock(&pVnode->mutex);
return needCommit;
}
@ -299,11 +299,11 @@ static int32_t vnodePrepareCommit(SVnode *pVnode, SCommitInfo *pInfo) {
code = smaPrepareAsyncCommit(pVnode->pSma);
TSDB_CHECK_CODE(code, lino, _exit);
taosThreadMutexLock(&pVnode->mutex);
(void)taosThreadMutexLock(&pVnode->mutex);
ASSERT(pVnode->onCommit == NULL);
pVnode->onCommit = pVnode->inUse;
pVnode->inUse = NULL;
taosThreadMutexUnlock(&pVnode->mutex);
(void)taosThreadMutexUnlock(&pVnode->mutex);
_exit:
if (code) {
@ -316,7 +316,7 @@ _exit:
return code;
}
static void vnodeReturnBufPool(SVnode *pVnode) {
taosThreadMutexLock(&pVnode->mutex);
(void)taosThreadMutexLock(&pVnode->mutex);
SVBufPool *pPool = pVnode->onCommit;
int32_t nRef = atomic_sub_fetch_32(&pPool->nRef, 1);
@ -340,7 +340,7 @@ static void vnodeReturnBufPool(SVnode *pVnode) {
ASSERT(0);
}
taosThreadMutexUnlock(&pVnode->mutex);
(void)taosThreadMutexUnlock(&pVnode->mutex);
}
static int32_t vnodeCommit(void *arg) {
int32_t code = 0;

41
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -1124,7 +1124,11 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq,
sprintf(tbName, "%s.%s", pVnode->config.dbname, pCreateReq->name);
if (vnodeValidateTableHash(pVnode, tbName) < 0) {
cRsp.code = TSDB_CODE_VND_HASH_MISMATCH;
taosArrayPush(rsp.pArray, &cRsp);
if (taosArrayPush(rsp.pArray, &cRsp) == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
rcode = -1;
goto _exit;
}
vError("vgId:%d create-table:%s failed due to hash value mismatch", TD_VID(pVnode), tbName);
continue;
}
@ -1139,11 +1143,19 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t ver, void *pReq,
} else {
cRsp.code = TSDB_CODE_SUCCESS;
tdFetchTbUidList(pVnode->pSma, &pStore, pCreateReq->ctb.suid, pCreateReq->uid);
taosArrayPush(tbUids, &pCreateReq->uid);
if (taosArrayPush(tbUids, &pCreateReq->uid) == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
rcode = -1;
goto _exit;
}
vnodeUpdateMetaRsp(pVnode, cRsp.pMeta);
}
taosArrayPush(rsp.pArray, &cRsp);
if (taosArrayPush(rsp.pArray, &cRsp) == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
rcode = -1;
goto _exit;
}
}
vDebug("vgId:%d, add %d new created tables into query table list", TD_VID(pVnode), (int32_t)taosArrayGetSize(tbUids));
@ -1375,12 +1387,20 @@ static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t ver, void *pReq, in
if (tbUid > 0) tdFetchTbUidList(pVnode->pSma, &pStore, pDropTbReq->suid, tbUid);
}
taosArrayPush(rsp.pArray, &dropTbRsp);
if (taosArrayPush(rsp.pArray, &dropTbRsp) == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
pRsp->code = terrno;
goto _exit;
}
if (tsEnableAuditCreateTable) {
char *str = taosMemoryCalloc(1, TSDB_TABLE_FNAME_LEN);
strcpy(str, pDropTbReq->name);
taosArrayPush(tbNames, &str);
if (taosArrayPush(tbNames, &str) == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
pRsp->code = terrno;
goto _exit;
}
}
}
@ -1499,11 +1519,13 @@ static int32_t vnodeResetTableCxt(SMeta *pMeta, SSubmitReqConvertCxt *pCxt) {
taosArrayDestroy(pCxt->pColValues);
pCxt->pColValues = taosArrayInit(pCxt->pTbSchema->numOfCols, sizeof(SColVal));
if (NULL == pCxt->pColValues) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
for (int32_t i = 0; i < pCxt->pTbSchema->numOfCols; ++i) {
SColVal val = COL_VAL_NONE(pCxt->pTbSchema->columns[i].colId, pCxt->pTbSchema->columns[i].type);
taosArrayPush(pCxt->pColValues, &val);
if (taosArrayPush(pCxt->pColValues, &val) == NULL) {
return terrno;
}
}
return TSDB_CODE_SUCCESS;
@ -1819,7 +1841,10 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t ver, void *pReq, in
goto _exit;
}
taosArrayPush(newTbUids, &pSubmitTbData->uid);
if (taosArrayPush(newTbUids, &pSubmitTbData->uid) == NULL) {
code = terrno;
goto _exit;
}
if (pCreateTbRsp->pMeta) {
vnodeUpdateMetaRsp(pVnode, pCreateTbRsp->pMeta);

28
source/dnode/vnode/src/vnd/vnodeSync.c
View File

@ -34,7 +34,7 @@ static inline void vnodeWaitBlockMsg(SVnode *pVnode, const SRpcMsg *pMsg) {
static inline void vnodePostBlockMsg(SVnode *pVnode, const SRpcMsg *pMsg) {
if (vnodeIsMsgBlock(pMsg->msgType)) {
const STraceId *trace = &pMsg->info.traceId;
taosThreadMutexLock(&pVnode->lock);
(void)taosThreadMutexLock(&pVnode->lock);
if (pVnode->blocked) {
vGTrace("vgId:%d, msg:%p post block, type:%s sec:%d seq:%" PRId64, pVnode->config.vgId, pMsg,
TMSG_INFO(pMsg->msgType), pVnode->blockSec, pVnode->blockSeq);
@ -43,7 +43,7 @@ static inline void vnodePostBlockMsg(SVnode *pVnode, const SRpcMsg *pMsg) {
pVnode->blockSeq = 0;
tsem_post(&pVnode->syncSem);
}
taosThreadMutexUnlock(&pVnode->lock);
(void)taosThreadMutexUnlock(&pVnode->lock);
}
}
@ -113,7 +113,7 @@ static void vnodeHandleProposeError(SVnode *pVnode, SRpcMsg *pMsg, int32_t code)
static int32_t inline vnodeProposeMsg(SVnode *pVnode, SRpcMsg *pMsg, bool isWeak) {
int64_t seq = 0;
taosThreadMutexLock(&pVnode->lock);
(void)taosThreadMutexLock(&pVnode->lock);
int32_t code = syncPropose(pVnode->sync, pMsg, isWeak, &seq);
bool wait = (code == 0 && vnodeIsMsgBlock(pMsg->msgType));
if (wait) {
@ -122,7 +122,7 @@ static int32_t inline vnodeProposeMsg(SVnode *pVnode, SRpcMsg *pMsg, bool isWeak
pVnode->blockSec = taosGetTimestampSec();
pVnode->blockSeq = seq;
}
taosThreadMutexUnlock(&pVnode->lock);
(void)taosThreadMutexUnlock(&pVnode->lock);
if (code > 0) {
vnodeHandleWriteMsg(pVnode, pMsg);
@ -171,14 +171,14 @@ static void inline vnodeProposeBatchMsg(SVnode *pVnode, SRpcMsg **pMsgArr, bool
if (*arrSize <= 0) return;
SRpcMsg *pLastMsg = pMsgArr[*arrSize - 1];
taosThreadMutexLock(&pVnode->lock);
(void)taosThreadMutexLock(&pVnode->lock);
int32_t code = syncProposeBatch(pVnode->sync, pMsgArr, pIsWeakArr, *arrSize);
bool wait = (code == 0 && vnodeIsBlockMsg(pLastMsg->msgType));
if (wait) {
ASSERT(!pVnode->blocked);
pVnode->blocked = true;
}
taosThreadMutexUnlock(&pVnode->lock);
(void)taosThreadMutexUnlock(&pVnode->lock);
if (code > 0) {
for (int32_t i = 0; i < *arrSize; ++i) {
@ -598,13 +598,13 @@ static void vnodeBecomeFollower(const SSyncFSM *pFsm) {
SVnode *pVnode = pFsm->data;
vInfo("vgId:%d, become follower", pVnode->config.vgId);
taosThreadMutexLock(&pVnode->lock);
(void)taosThreadMutexLock(&pVnode->lock);
if (pVnode->blocked) {
pVnode->blocked = false;
vDebug("vgId:%d, become follower and post block", pVnode->config.vgId);
tsem_post(&pVnode->syncSem);
}
taosThreadMutexUnlock(&pVnode->lock);
(void)taosThreadMutexUnlock(&pVnode->lock);
if (pVnode->pTq) {
tqUpdateNodeStage(pVnode->pTq, false);
@ -616,13 +616,13 @@ static void vnodeBecomeLearner(const SSyncFSM *pFsm) {
SVnode *pVnode = pFsm->data;
vInfo("vgId:%d, become learner", pVnode->config.vgId);
taosThreadMutexLock(&pVnode->lock);
(void)taosThreadMutexLock(&pVnode->lock);
if (pVnode->blocked) {
pVnode->blocked = false;
vDebug("vgId:%d, become learner and post block", pVnode->config.vgId);
tsem_post(&pVnode->syncSem);
}
taosThreadMutexUnlock(&pVnode->lock);
(void)taosThreadMutexUnlock(&pVnode->lock);
}
static void vnodeBecomeLeader(const SSyncFSM *pFsm) {
@ -746,13 +746,13 @@ void vnodeSyncPreClose(SVnode *pVnode) {
syncLeaderTransfer(pVnode->sync);
syncPreStop(pVnode->sync);
taosThreadMutexLock(&pVnode->lock);
(void)taosThreadMutexLock(&pVnode->lock);
if (pVnode->blocked) {
vInfo("vgId:%d, post block after close sync", pVnode->config.vgId);
pVnode->blocked = false;
tsem_post(&pVnode->syncSem);
}
taosThreadMutexUnlock(&pVnode->lock);
(void)taosThreadMutexUnlock(&pVnode->lock);
}
void vnodeSyncPostClose(SVnode *pVnode) {
@ -767,7 +767,7 @@ void vnodeSyncClose(SVnode *pVnode) {
void vnodeSyncCheckTimeout(SVnode *pVnode) {
vTrace("vgId:%d, check sync timeout msg", pVnode->config.vgId);
taosThreadMutexLock(&pVnode->lock);
(void)taosThreadMutexLock(&pVnode->lock);
if (pVnode->blocked) {
int32_t curSec = taosGetTimestampSec();
int32_t delta = curSec - pVnode->blockSec;
@ -788,7 +788,7 @@ void vnodeSyncCheckTimeout(SVnode *pVnode) {
tsem_post(&pVnode->syncSem);
}
}
taosThreadMutexUnlock(&pVnode->lock);
(void)taosThreadMutexUnlock(&pVnode->lock);
}
bool vnodeIsRoleLeader(SVnode *pVnode) {

56
source/libs/command/src/command.c
View File

@ -74,13 +74,16 @@ static int32_t getSchemaBytes(const SSchema* pSchema) {
}
static int32_t buildDescResultDataBlock(SSDataBlock** pOutput) {
SSDataBlock* pBlock = createDataBlock();
if (NULL == pBlock) {
return TSDB_CODE_OUT_OF_MEMORY;
QRY_OPTR_CHECK(pOutput);
SSDataBlock* pBlock = NULL;
int32_t code = createDataBlock(&pBlock);
if (code) {
return code;
}
SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, DESCRIBE_RESULT_FIELD_LEN, 1);
int32_t code = blockDataAppendColInfo(pBlock, &infoData);
code = blockDataAppendColInfo(pBlock, &infoData);
if (TSDB_CODE_SUCCESS == code) {
infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, DESCRIBE_RESULT_TYPE_LEN, 2);
code = blockDataAppendColInfo(pBlock, &infoData);
@ -229,13 +232,16 @@ static int32_t execDescribe(bool sysInfoUser, SNode* pStmt, SRetrieveTableRsp**
static int32_t execResetQueryCache() { return catalogClearCache(); }
static int32_t buildCreateDBResultDataBlock(SSDataBlock** pOutput) {
SSDataBlock* pBlock = createDataBlock();
if (NULL == pBlock) {
return TSDB_CODE_OUT_OF_MEMORY;
QRY_OPTR_CHECK(pOutput);
SSDataBlock* pBlock = NULL;
int32_t code = createDataBlock(&pBlock);
if (code) {
return code;
}
SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_DB_RESULT_COLS, 1);
int32_t code = blockDataAppendColInfo(pBlock, &infoData);
code = blockDataAppendColInfo(pBlock, &infoData);
if (TSDB_CODE_SUCCESS == code) {
infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_DB_RESULT_FIELD2_LEN, 2);
code = blockDataAppendColInfo(pBlock, &infoData);
@ -418,13 +424,16 @@ static int32_t execShowCreateDatabase(SShowCreateDatabaseStmt* pStmt, SRetrieveT
}
static int32_t buildCreateTbResultDataBlock(SSDataBlock** pOutput) {
SSDataBlock* pBlock = createDataBlock();
if (NULL == pBlock) {
return TSDB_CODE_OUT_OF_MEMORY;
QRY_OPTR_CHECK(pOutput);
SSDataBlock* pBlock = NULL;
int32_t code = createDataBlock(&pBlock);
if (code) {
return code;
}
SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_TB_RESULT_FIELD1_LEN, 1);
int32_t code = blockDataAppendColInfo(pBlock, &infoData);
code = blockDataAppendColInfo(pBlock, &infoData);
if (TSDB_CODE_SUCCESS == code) {
infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_TB_RESULT_FIELD2_LEN, 2);
code = blockDataAppendColInfo(pBlock, &infoData);
@ -439,13 +448,16 @@ static int32_t buildCreateTbResultDataBlock(SSDataBlock** pOutput) {
}
static int32_t buildCreateViewResultDataBlock(SSDataBlock** pOutput) {
SSDataBlock* pBlock = createDataBlock();
if (NULL == pBlock) {
return TSDB_CODE_OUT_OF_MEMORY;
QRY_OPTR_CHECK(pOutput);
SSDataBlock* pBlock = NULL;
int32_t code = createDataBlock(&pBlock);
if (code) {
return code;
}
SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_VIEW_RESULT_FIELD1_LEN, 1);
int32_t code = blockDataAppendColInfo(pBlock, &infoData);
code = blockDataAppendColInfo(pBlock, &infoData);
if (TSDB_CODE_SUCCESS == code) {
infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_VIEW_RESULT_FIELD2_LEN, 2);
code = blockDataAppendColInfo(pBlock, &infoData);
@ -892,9 +904,12 @@ static int32_t execShowLocalVariables(SRetrieveTableRsp** pRsp) {
}
static int32_t createSelectResultDataBlock(SNodeList* pProjects, SSDataBlock** pOutput) {
SSDataBlock* pBlock = createDataBlock();
if (NULL == pBlock) {
return TSDB_CODE_OUT_OF_MEMORY;
QRY_OPTR_CHECK(pOutput);
SSDataBlock* pBlock = NULL;
int32_t code = createDataBlock(&pBlock);
if (code) {
return code;
}
SNode* pProj = NULL;
@ -910,8 +925,9 @@ static int32_t createSelectResultDataBlock(SNodeList* pProjects, SSDataBlock** p
}
QRY_ERR_RET(blockDataAppendColInfo(pBlock, &infoData));
}
*pOutput = pBlock;
return TSDB_CODE_SUCCESS;
return code;
}
int32_t buildSelectResultDataBlock(SNodeList* pProjects, SSDataBlock* pBlock) {

7
source/libs/command/src/explain.c
View File

@ -1941,7 +1941,8 @@ _return:
}
int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) {
int32_t code = 0;
int32_t code = 0;
SSDataBlock *pBlock = NULL;
SExplainCtx *pCtx = (SExplainCtx *)ctx;
int32_t rowNum = taosArrayGetSize(pCtx->rows);
if (rowNum <= 0) {
@ -1949,7 +1950,9 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) {
QRY_ERR_RET(TSDB_CODE_APP_ERROR);
}
SSDataBlock *pBlock = createDataBlock();
code = createDataBlock(&pBlock);
QRY_ERR_JRET(code);
SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, TSDB_EXPLAIN_RESULT_ROW_SIZE, 1);
QRY_ERR_JRET(blockDataAppendColInfo(pBlock, &infoData));
QRY_ERR_JRET(blockDataEnsureCapacity(pBlock, rowNum));

2
source/libs/executor/inc/executil.h
View File

@ -26,7 +26,7 @@
#define T_LONG_JMP(_obj, _c) \
do { \
ASSERT((_c) != -1); \
ASSERT((_c) != 1); \
longjmp((_obj), (_c)); \
} while (0)

9
source/libs/executor/inc/operator.h
View File

@ -20,15 +20,6 @@
extern "C" {
#endif
#define QRY_OPTR_CHECK(_o) \
do { \
if ((_o) == NULL) { \
return TSDB_CODE_INVALID_PARA; \
} else { \
*(_o) = NULL; \
} \
} while(0)
typedef struct SOperatorCostInfo {
double openCost;
double totalCost;

11
source/libs/executor/inc/tsort.h
View File

@ -78,8 +78,8 @@ typedef int32_t (*_sort_merge_compar_fn_t)(const void* p1, const void* p2, void*
* @return
*/
int32_t tsortCreateSortHandle(SArray* pOrderInfo, int32_t type, int32_t pageSize, int32_t numOfPages,
SSDataBlock* pBlock, const char* idstr, uint64_t pqMaxRows, uint32_t pqMaxTupleLength,
uint32_t pqSortBufSize, SSortHandle** pHandle);
SSDataBlock* pBlock, const char* idstr, uint64_t pqMaxRows, uint32_t pqMaxTupleLength,
uint32_t pqSortBufSize, SSortHandle** pHandle);
void tsortSetForceUsePQSort(SSortHandle* pHandle);
@ -213,10 +213,11 @@ int32_t tsortCompAndBuildKeys(const SArray* pSortCols, char* keyBuf, int32_t* ke
/**
* @brief set the merge limit reached callback. it calls mergeLimitReached param with tableUid and param
*/
void tsortSetMergeLimitReachedFp(SSortHandle* pHandle, void (*mergeLimitReached)(uint64_t tableUid, void* param), void* param);
void tsortSetMergeLimitReachedFp(SSortHandle* pHandle, void (*mergeLimitReached)(uint64_t tableUid, void* param),
void* param);
int tsortComparBlockCell(SSDataBlock* pLeftBlock, SSDataBlock* pRightBlock, int32_t leftRowIndex, int32_t rightRowIndex,
void* pOrder);
int32_t tsortComparBlockCell(SSDataBlock* pLeftBlock, SSDataBlock* pRightBlock, int32_t leftRowIndex,
int32_t rightRowIndex, void* pOrder);
#ifdef __cplusplus
}
#endif

7
source/libs/executor/src/aggregateoperator.c
View File

@ -364,7 +364,12 @@ static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBloc
return TSDB_CODE_SUCCESS;
}
SSDataBlock* pBlock = createDataBlock();
SSDataBlock* pBlock = NULL;
code = createDataBlock(&pBlock);
if (code) {
return code;
}
pBlock->info.rows = 1;
pBlock->info.capacity = 0;

5
source/libs/executor/src/cachescanoperator.c
View File

@ -181,7 +181,10 @@ int32_t createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SReadHandl
capacity = TMIN(totalTables, 4096);
pInfo->pBufferedRes = createOneDataBlock(pInfo->pRes, false);
pInfo->pBufferedRes = NULL;
code = createOneDataBlock(pInfo->pRes, false, &pInfo->pBufferedRes);
QUERY_CHECK_CODE(code, lino, _error);
setColIdForCacheReadBlock(pInfo->pBufferedRes, pScanNode);
code = blockDataEnsureCapacity(pInfo->pBufferedRes, capacity);
QUERY_CHECK_CODE(code, lino, _error);

19
source/libs/executor/src/exchangeoperator.c
View File

@ -655,7 +655,10 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, SArray* pCo
int32_t lino = 0;
if (pColList == NULL) { // data from other sources
blockDataCleanup(pRes);
*pNextStart = (char*)blockDecode(pRes, pData);
code = blockDecode(pRes, pData, (const char**) pNextStart);
if (code) {
return code;
}
} else { // extract data according to pColList
char* pStart = pData;
@ -672,14 +675,20 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, SArray* pCo
pStart += sizeof(SSysTableSchema);
}
SSDataBlock* pBlock = createDataBlock();
SSDataBlock* pBlock = NULL;
code = createDataBlock(&pBlock);
QUERY_CHECK_CODE(code, lino, _end);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData idata = createColumnInfoData(pSchema[i].type, pSchema[i].bytes, pSchema[i].colId);
code = blockDataAppendColInfo(pBlock, &idata);
QUERY_CHECK_CODE(code, lino, _end);
}
(void)blockDecode(pBlock, pStart);
const char* pDummy = NULL;
code = blockDecode(pBlock, pStart, &pDummy);
QUERY_CHECK_CODE(code, lino, _end);
code = blockDataEnsureCapacity(pRes, pBlock->info.rows);
QUERY_CHECK_CODE(code, lino, _end);
@ -791,8 +800,8 @@ int32_t doExtractResultBlocks(SExchangeInfo* pExchangeInfo, SSourceDataInfo* pDa
pb = *(SSDataBlock**)taosArrayPop(pExchangeInfo->pRecycledBlocks);
blockDataCleanup(pb);
} else {
pb = createOneDataBlock(pExchangeInfo->pDummyBlock, false);
QUERY_CHECK_NULL(pb, code, lino, _end, terrno);
code = createOneDataBlock(pExchangeInfo->pDummyBlock, false, &pb);
QUERY_CHECK_NULL(pb, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
}
int32_t compLen = *(int32_t*)pStart;

36
source/libs/executor/src/executil.c
View File

@ -251,9 +251,13 @@ SArray* createSortInfo(SNodeList* pNodeList) {
}
SSDataBlock* createDataBlockFromDescNode(SDataBlockDescNode* pNode) {
int32_t numOfCols = LIST_LENGTH(pNode->pSlots);
SSDataBlock* pBlock = createDataBlock();
int32_t numOfCols = LIST_LENGTH(pNode->pSlots);
SSDataBlock* pBlock = NULL;
int32_t code = createDataBlock(&pBlock);
if (code) {
terrno = code;
return NULL;
}
pBlock->info.id.blockId = pNode->dataBlockId;
pBlock->info.type = STREAM_INVALID;
@ -267,7 +271,7 @@ SSDataBlock* createDataBlockFromDescNode(SDataBlockDescNode* pNode) {
idata.info.scale = pDescNode->dataType.scale;
idata.info.precision = pDescNode->dataType.precision;
int32_t code = blockDataAppendColInfo(pBlock, &idata);
code = blockDataAppendColInfo(pBlock, &idata);
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code));
blockDataDestroy(pBlock);
@ -1029,11 +1033,9 @@ SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, S
SStorageAPI* pStorageAPI) {
int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
SSDataBlock* pResBlock = createDataBlock();
if (pResBlock == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
SSDataBlock* pResBlock = NULL;
code = createDataBlock(&pResBlock);
QUERY_CHECK_CODE(code, lino, _end);
for (int32_t i = 0; i < taosArrayGetSize(pColList); ++i) {
SColumnInfoData colInfo = {0};
@ -2626,9 +2628,12 @@ void printDataBlock(SSDataBlock* pBlock, const char* flag, const char* taskIdStr
qDebug("%s===stream===%s: Block is Null or Empty", taskIdStr, flag);
return;
}
char* pBuf = NULL;
qDebug("%s", dumpBlockData(pBlock, flag, &pBuf, taskIdStr));
taosMemoryFree(pBuf);
char* pBuf = NULL;
int32_t code = dumpBlockData(pBlock, flag, &pBuf, taskIdStr);
if (code == 0) {
qDebug("%s", pBuf);
taosMemoryFree(pBuf);
}
}
void printSpecDataBlock(SSDataBlock* pBlock, const char* flag, const char* opStr, const char* taskIdStr) {
@ -2643,8 +2648,11 @@ void printSpecDataBlock(SSDataBlock* pBlock, const char* flag, const char* opStr
char* pBuf = NULL;
char flagBuf[64];
snprintf(flagBuf, sizeof(flagBuf), "%s %s", flag, opStr);
qDebug("%s", dumpBlockData(pBlock, flagBuf, &pBuf, taskIdStr));
taosMemoryFree(pBuf);
int32_t code = dumpBlockData(pBlock, flagBuf, &pBuf, taskIdStr);
if (code == 0) {
qDebug("%s", pBuf);
taosMemoryFree(pBuf);
}
}
}

9
source/libs/executor/src/executor.c
View File

@ -695,9 +695,12 @@ int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds, bo
while (pRes != NULL) {
SSDataBlock* p = NULL;
if (blockIndex >= taosArrayGetSize(pTaskInfo->pResultBlockList)) {
SSDataBlock* p1 = createOneDataBlock(pRes, true);
void* tmp = taosArrayPush(pTaskInfo->pResultBlockList, &p1);
QUERY_CHECK_NULL(tmp, code, lino, _end, terrno);
SSDataBlock* p1 = NULL;
code = createOneDataBlock(pRes, true, &p1);
QUERY_CHECK_CODE(code, lino, _end);
void* tmp = taosArrayPush(pTaskInfo->pResultBlockList, &p1);
QUERY_CHECK_NULL(tmp, code, lino, _end, TSDB_CODE_OUT_OF_MEMORY);
p = p1;
} else {
p = *(SSDataBlock**)taosArrayGet(pTaskInfo->pResultBlockList, blockIndex);

8
source/libs/executor/src/filloperator.c
View File

@ -513,7 +513,13 @@ int32_t createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* pPhyFi
goto _error;
}
pInfo->pFinalRes = createOneDataBlock(pInfo->pRes, false);
pInfo->pFinalRes = NULL;
code = createOneDataBlock(pInfo->pRes, false, &pInfo->pFinalRes);
if (code) {
goto _error;
}
code = blockDataEnsureCapacity(pInfo->pFinalRes, pOperator->resultInfo.capacity);
if (code != TSDB_CODE_SUCCESS) {
goto _error;

15
source/libs/executor/src/groupoperator.c
View File

@ -613,7 +613,10 @@ SSDataBlock* createBlockDataNotLoaded(const SOperatorInfo* pOperator, SSDataBloc
return NULL;
}
SSDataBlock* pDstBlock = createDataBlock();
SSDataBlock* pDstBlock = NULL;
code = createDataBlock(&pDstBlock);
QUERY_CHECK_CODE(code, lino, _end);
pDstBlock->info = pDataBlock->info;
pDstBlock->info.id.blockId = pOperator->resultDataBlockId;
pDstBlock->info.capacity = 0;
@ -1319,7 +1322,10 @@ int32_t appendCreateTableRow(void* pState, SExprSupp* pTableSup, SExprSupp* pTag
QUERY_CHECK_CODE(code, lino, _end);
if (winCode != TSDB_CODE_SUCCESS) {
SSDataBlock* pTmpBlock = blockCopyOneRow(pSrcBlock, rowId);
SSDataBlock* pTmpBlock = NULL;
code = blockCopyOneRow(pSrcBlock, rowId, &pTmpBlock);
QUERY_CHECK_CODE(code, lino, _end);
memset(pTmpBlock->info.parTbName, 0, TSDB_TABLE_NAME_LEN);
pTmpBlock->info.id.groupId = groupId;
char* tbName = pSrcBlock->info.parTbName;
@ -1708,8 +1714,9 @@ int32_t createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStreamPart
pInfo->pPartitions = taosHashInit(1024, hashFn, false, HASH_NO_LOCK);
taosHashSetFreeFp(pInfo->pPartitions, freePartItem);
pInfo->tsColIndex = 0;
pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
QUERY_CHECK_NULL(pInfo->pDelRes, code, lino, _error, terrno);
code = createSpecialDataBlock(STREAM_DELETE_RESULT, &pInfo->pDelRes);
QUERY_CHECK_CODE(code, lino, _error);
int32_t numOfCols = 0;
SExprInfo* pExprInfo = createExprInfo(pPartNode->part.pTargets, NULL, &numOfCols);

13
source/libs/executor/src/hashjoinoperator.c
View File

@ -1157,22 +1157,23 @@ int32_t hJoinInitResBlocks(SHJoinOperatorInfo* pJoin, SHashJoinPhysiNode* pJoinN
int32_t code = blockDataEnsureCapacity(pJoin->finBlk, hJoinGetFinBlkCapacity(pJoin, pJoinNode));
if (TSDB_CODE_SUCCESS != code) {
QRY_ERR_RET(terrno);
QRY_ERR_RET(code);
}
if (NULL != pJoin->pPreFilter) {
pJoin->midBlk = createOneDataBlock(pJoin->finBlk, false);
if (NULL == pJoin->finBlk) {
QRY_ERR_RET(terrno);
pJoin->midBlk = NULL;
code = createOneDataBlock(pJoin->finBlk, false, &pJoin->midBlk);
if (code) {
QRY_ERR_RET(code);
}
code = blockDataEnsureCapacity(pJoin->midBlk, pJoin->finBlk->info.capacity);
if (TSDB_CODE_SUCCESS != code) {
QRY_ERR_RET(terrno);
QRY_ERR_RET(code);
}
}
pJoin->blkThreshold = pJoin->finBlk->info.capacity * HJOIN_BLK_THRESHOLD_RATIO;
return TSDB_CODE_SUCCESS;
}

40
source/libs/executor/src/mergejoin.c
View File

@ -2248,10 +2248,12 @@ static int32_t mAsofBackwardRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInf
} while (true);
if (buildGot && NULL == pCtx->cache.outBlk) {
pCtx->cache.outBlk = createOneDataBlock(pJoin->build->blk, false);
if (NULL == pCtx->cache.outBlk) {
MJ_ERR_RET(terrno);
pCtx->cache.outBlk = NULL;
int32_t code = createOneDataBlock(pJoin->build->blk, false, &pCtx->cache.outBlk);
if (code) {
MJ_ERR_RET(code);
}
MJ_ERR_RET(blockDataEnsureCapacity(pCtx->cache.outBlk, pCtx->jLimit));
}
@ -2678,10 +2680,12 @@ static int32_t mAsofForwardRetrieve(SOperatorInfo* pOperator, SMJoinOperatorInfo
if (buildGot && pJoin->build->newBlk) {
if (NULL == pCtx->cache.outBlk) {
pCtx->cache.outBlk = createOneDataBlock(pJoin->build->blk, false);
if (NULL == pCtx->cache.outBlk) {
MJ_ERR_RET(terrno);
pCtx->cache.outBlk = NULL;
int32_t code = createOneDataBlock(pJoin->build->blk, false, &pCtx->cache.outBlk);
if (code) {
MJ_ERR_RET(code);
}
MJ_ERR_RET(blockDataEnsureCapacity(pCtx->cache.outBlk, pCtx->jLimit));
}
@ -2832,19 +2836,24 @@ static int32_t mWinJoinCloneCacheBlk(SMJoinWindowCtx* pCtx) {
}
if (!pGrp->clonedBlk) {
int32_t code = 0;
if (0 == pGrp->beginIdx) {
pGrp->blk = createOneDataBlock(pGrp->blk, true);
SSDataBlock* p = NULL;
code = createOneDataBlock(pGrp->blk, true, &p);
if (code) {
MJ_ERR_RET(code);
}
pGrp->blk = p;
} else {
pGrp->blk = blockDataExtractBlock(pGrp->blk, pGrp->beginIdx, pGrp->blk->info.rows - pGrp->beginIdx);
code = blockDataExtractBlock(pGrp->blk, pGrp->beginIdx, pGrp->blk->info.rows - pGrp->beginIdx, &pGrp->blk);
pGrp->endIdx -= pGrp->beginIdx;
pGrp->beginIdx = 0;
pGrp->readIdx = 0;
}
if (NULL == pGrp->blk) {
MJ_ERR_RET(terrno);
if (code) {
MJ_ERR_RET(code);
}
pGrp->clonedBlk = true;
}
@ -3672,9 +3681,10 @@ int32_t mJoinInitMergeCtx(SMJoinOperatorInfo* pJoin, SSortMergeJoinPhysiNode* pJ
MJ_ERR_RET(blockDataEnsureCapacity(pCtx->finBlk, mJoinGetFinBlkCapacity(pJoin, pJoinNode)));
if (pJoin->pFPreFilter) {
pCtx->midBlk = createOneDataBlock(pCtx->finBlk, false);
if (NULL == pCtx->midBlk) {
MJ_ERR_RET(terrno);
pCtx->midBlk = NULL;
int32_t code = createOneDataBlock(pCtx->finBlk, false, &pCtx->midBlk);
if (code) {
MJ_ERR_RET(code);
}
MJ_ERR_RET(blockDataEnsureCapacity(pCtx->midBlk, pCtx->finBlk->info.capacity));
}

15
source/libs/executor/src/mergejoinoperator.c
View File

@ -1331,10 +1331,12 @@ int32_t mJoinBuildEqGroups(SMJoinTableCtx* pTable, int64_t timestamp, bool* whol
}
if (0 == pGrp->beginIdx && pTable->multiEqGrpRows && 0 == pTable->eqRowLimit) {
pGrp->blk = createOneDataBlock(pTable->blk, true);
if (NULL == pGrp->blk) {
MJ_ERR_RET(terrno);
pGrp->blk = NULL;
code = createOneDataBlock(pTable->blk, true, &pGrp->blk);
if (code) {
MJ_ERR_RET(code);
}
if (NULL == taosArrayPush(pTable->createdBlks, &pGrp->blk)) {
MJ_ERR_RET(terrno);
}
@ -1358,10 +1360,9 @@ int32_t mJoinBuildEqGroups(SMJoinTableCtx* pTable, int64_t timestamp, bool* whol
if (keepGrp && rowNum > 0) {
pTable->eqRowNum += rowNum;
pGrp->blk = blockDataExtractBlock(pTable->blk, pGrp->beginIdx, rowNum);
if (NULL == pGrp->blk) {
MJ_ERR_RET(terrno);
code = blockDataExtractBlock(pTable->blk, pGrp->beginIdx, rowNum, &pGrp->blk);
if (code) {
MJ_ERR_RET(code);
}
pGrp->endIdx -= pGrp->beginIdx;

1
source/libs/executor/src/mergeoperator.c
View File

@ -207,7 +207,6 @@ int32_t doSortMerge(SOperatorInfo* pOperator, SSDataBlock** pResBlock) {
if (pSortMergeInfo->pIntermediateBlock == NULL) {
pSortMergeInfo->pIntermediateBlock = NULL;
code = tsortGetSortedDataBlock(pHandle, &pSortMergeInfo->pIntermediateBlock);
if (pSortMergeInfo->pIntermediateBlock == NULL || code != 0) {
return code;

2
source/libs/executor/src/operator.c
View File

@ -27,6 +27,8 @@
#include "querytask.h"
#include "storageapi.h"
#include "tdatablock.h"
SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t cleanup,
__optr_close_fn_t closeFn, __optr_reqBuf_fn_t reqBufFn,

10
source/libs/executor/src/projectoperator.c
View File

@ -114,7 +114,11 @@ int32_t createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhysiNode*
initLimitInfo(pProjPhyNode->node.pLimit, pProjPhyNode->node.pSlimit, &pInfo->limitInfo);
pInfo->binfo.pRes = pResBlock;
pInfo->pFinalRes = createOneDataBlock(pResBlock, false);
pInfo->pFinalRes = NULL;
code = createOneDataBlock(pResBlock, false, &pInfo->pFinalRes);
TSDB_CHECK_CODE(code, lino, _error);
pInfo->binfo.inputTsOrder = pProjPhyNode->node.inputTsOrder;
pInfo->binfo.outputTsOrder = pProjPhyNode->node.outputTsOrder;
pInfo->inputIgnoreGroup = pProjPhyNode->inputIgnoreGroup;
@ -366,8 +370,8 @@ int32_t doProjectOperation(SOperatorInfo* pOperator, SSDataBlock** pResBlock) {
pFinalRes->info.version = pRes->info.version;
// continue merge data, ignore the group id
code = blockDataMerge(pFinalRes, pRes);
if (code) {
int32_t ret = blockDataMerge(pFinalRes, pRes);
if (ret < 0) {
pTaskInfo->code = code;
return code;
}

59
source/libs/executor/src/scanoperator.c
View File

@ -1747,8 +1747,9 @@ static int32_t doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32_t ts
}
if (pInfo->partitionSup.needCalc) {
SSDataBlock* tmpBlock = createOneDataBlock(pResult, true);
QUERY_CHECK_NULL(tmpBlock, code, lino, _end, terrno);
SSDataBlock* tmpBlock = NULL;
code = createOneDataBlock(pResult, true, &tmpBlock);
QUERY_CHECK_CODE(code, lino, _end);
blockDataCleanup(pResult);
for (int32_t i = 0; i < tmpBlock->info.rows; i++) {
@ -2583,7 +2584,10 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock
bool colExists = false;
for (int32_t j = 0; j < blockDataGetNumOfCols(pBlock); ++j) {
SColumnInfoData* pResCol = bdGetColumnInfoData(pBlock, j);
SColumnInfoData* pResCol = NULL;
code = bdGetColumnInfoData(pBlock, j, &pResCol);
QUERY_CHECK_CODE(code, lino, _end);
if (pResCol->info.colId == pColMatchInfo->colId) {
SColumnInfoData* pDst = taosArrayGet(pInfo->pRes->pDataBlock, pColMatchInfo->dstSlotId);
code = colDataAssign(pDst, pResCol, pBlock->info.rows, &pInfo->pRes->info);
@ -3141,7 +3145,9 @@ FETCH_NEXT_BLOCK:
printSpecDataBlock(pBlock, getStreamOpName(pOperator->operatorType), "delete recv", GET_TASKID(pTaskInfo));
SSDataBlock* pDelBlock = NULL;
if (pInfo->tqReader) {
pDelBlock = createSpecialDataBlock(STREAM_DELETE_DATA);
code = createSpecialDataBlock(STREAM_DELETE_DATA, &pDelBlock);
QUERY_CHECK_CODE(code, lino, _end);
code = filterDelBlockByUid(pDelBlock, pBlock, pInfo);
QUERY_CHECK_CODE(code, lino, _end);
} else {
@ -3782,7 +3788,8 @@ _end:
}
int32_t createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, SNode* pTagCond,
STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo, SOperatorInfo** pOptrInfo) {
STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo,
SOperatorInfo** pOptrInfo) {
QRY_OPTR_CHECK(pOptrInfo);
int32_t code = TSDB_CODE_SUCCESS;
@ -3945,15 +3952,21 @@ int32_t createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode*
}
pInfo->pRes = createDataBlockFromDescNode(pDescNode);
pInfo->pUpdateRes = createSpecialDataBlock(STREAM_CLEAR);
code = createSpecialDataBlock(STREAM_CLEAR, &pInfo->pUpdateRes);
QUERY_CHECK_CODE(code, lino, _error);
pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE;
pInfo->windowSup = (SWindowSupporter){.pStreamAggSup = NULL, .gap = -1, .parentType = QUERY_NODE_PHYSICAL_PLAN};
pInfo->groupId = 0;
pInfo->pStreamScanOp = pOperator;
pInfo->deleteDataIndex = 0;
pInfo->pDeleteDataRes = createSpecialDataBlock(STREAM_DELETE_DATA);
code = createSpecialDataBlock(STREAM_DELETE_DATA, &pInfo->pDeleteDataRes);
QUERY_CHECK_CODE(code, lino, _error);
pInfo->updateWin = (STimeWindow){.skey = INT64_MAX, .ekey = INT64_MAX};
pInfo->pUpdateDataRes = createSpecialDataBlock(STREAM_CLEAR);
createSpecialDataBlock(STREAM_CLEAR, &pInfo->pUpdateDataRes);
QUERY_CHECK_CODE(code, lino, _error);
if (hasPrimaryKeyCol(pInfo)) {
code = addPrimaryKeyCol(pInfo->pUpdateDataRes, pkType.type, pkType.bytes);
QUERY_CHECK_CODE(code, lino, _error);
@ -3961,6 +3974,7 @@ int32_t createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode*
pInfo->pkColType = pkType.type;
pInfo->pkColLen = pkType.bytes;
}
pInfo->assignBlockUid = pTableScanNode->assignBlockUid;
pInfo->partitionSup.needCalc = false;
pInfo->igCheckUpdate = pTableScanNode->igCheckUpdate;
@ -3969,7 +3983,9 @@ int32_t createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode*
pInfo->pState = pTaskInfo->streamInfo.pState;
pInfo->stateStore = pTaskInfo->storageAPI.stateStore;
pInfo->readerFn = pTaskInfo->storageAPI.tqReaderFn;
pInfo->pCheckpointRes = createSpecialDataBlock(STREAM_CHECKPOINT);
code = createSpecialDataBlock(STREAM_CHECKPOINT, &pInfo->pCheckpointRes);
QUERY_CHECK_CODE(code, lino, _error);
// for stream
if (pTaskInfo->streamInfo.pState) {
@ -4770,10 +4786,16 @@ static int32_t initSubTableInputs(SOperatorInfo* pOperator, STableMergeScanInfo*
for (int32_t i = 0; i < pSubTblsInfo->numSubTables; ++i) {
STmsSubTableInput* pInput = pSubTblsInfo->aInputs + i;
pInput->type = SUB_TABLE_MEM_BLOCK;
code = dumpQueryTableCond(&pInfo->base.cond, &pInput->tblCond);
QUERY_CHECK_CODE(code, lino, _end);
pInput->pReaderBlock = createOneDataBlock(pInfo->pResBlock, false);
pInput->pPageBlock = createOneDataBlock(pInfo->pResBlock, false);
code = createOneDataBlock(pInfo->pResBlock, false, &pInput->pReaderBlock);
QUERY_CHECK_CODE(code, lino, _end);
code = createOneDataBlock(pInfo->pResBlock, false, &pInput->pPageBlock);
QUERY_CHECK_CODE(code, lino, _end);
STableKeyInfo* keyInfo = tableListGetInfo(pInfo->base.pTableListInfo, i + pInfo->tableStartIndex);
pInput->pKeyInfo = keyInfo;
@ -5183,7 +5205,12 @@ static SSDataBlock* getBlockForTableMergeScan(void* param) {
if (pInfo->bNextDurationBlockEvent || pInfo->bNewFilesetEvent) {
if (!bSkipped) {
pInfo->nextDurationBlocks[pInfo->numNextDurationBlocks] = createOneDataBlock(pBlock, true);
int32_t code = createOneDataBlock(pBlock, true, &pInfo->nextDurationBlocks[pInfo->numNextDurationBlocks]);
if (code) {
terrno = code;
return NULL;
}
++pInfo->numNextDurationBlocks;
if (pInfo->numNextDurationBlocks > 2) {
qError("%s table merge scan prefetch %d next duration blocks. end early.", GET_TASKID(pTaskInfo),
@ -5685,8 +5712,9 @@ int32_t createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SR
code = generateSortByTsPkInfo(pInfo->base.matchInfo.pList, pInfo->base.cond.order, &pInfo->pSortInfo);
QUERY_CHECK_CODE(code, lino, _error);
pInfo->pReaderBlock = createOneDataBlock(pInfo->pResBlock, false);
QUERY_CHECK_NULL(pInfo->pReaderBlock, code, lino, _error, terrno);
code = createOneDataBlock(pInfo->pResBlock, false, &pInfo->pReaderBlock);
QUERY_CHECK_CODE(code, lino, _error);
pInfo->needCountEmptyTable = tsCountAlwaysReturnValue && pTableScanNode->needCountEmptyTable;
@ -5696,7 +5724,8 @@ int32_t createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SR
pInfo->bufPageSize = getProperSortPageSize(rowSize, nCols);
// start one reader variable
pInfo->pSortInputBlock = createOneDataBlock(pInfo->pResBlock, false);
code = createOneDataBlock(pInfo->pResBlock, false, &pInfo->pSortInputBlock);
QUERY_CHECK_CODE(code, lino, _error);
if (!tsExperimental) {
pInfo->filesetDelimited = false;

9
source/libs/executor/src/streamcountwindowoperator.c
View File

@ -858,7 +858,10 @@ int32_t createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
pInfo->pStDeleted = tSimpleHashInit(64, hashFn);
pInfo->pDelIterator = NULL;
pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
code = createSpecialDataBlock(STREAM_DELETE_RESULT, &pInfo->pDelRes);
QUERY_CHECK_CODE(code, lino, _error);
pInfo->ignoreExpiredData = pCountNode->window.igExpired;
pInfo->ignoreExpiredDataSaved = false;
pInfo->pUpdated = NULL;
@ -870,7 +873,9 @@ int32_t createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
QUERY_CHECK_CODE(code, lino, _error);
}
pInfo->pCheckpointRes = createSpecialDataBlock(STREAM_CHECKPOINT);
code = createSpecialDataBlock(STREAM_CHECKPOINT, &pInfo->pCheckpointRes);
QUERY_CHECK_CODE(code, lino, _error);
pInfo->recvGetAll = false;
pInfo->pPkDeleted = tSimpleHashInit(64, hashFn);
pInfo->destHasPrimaryKey = pCountNode->window.destHasPrimayKey;

8
source/libs/executor/src/streameventwindowoperator.c
View File

@ -900,7 +900,9 @@ int32_t createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
pInfo->pSeDeleted = tSimpleHashInit(64, hashFn);
pInfo->pDelIterator = NULL;
pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
code = createSpecialDataBlock(STREAM_DELETE_RESULT, &pInfo->pDelRes);
QUERY_CHECK_CODE(code, lino, _error);
pInfo->pChildren = NULL;
pInfo->ignoreExpiredData = pEventNode->window.igExpired;
pInfo->ignoreExpiredDataSaved = false;
@ -922,8 +924,8 @@ int32_t createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
pInfo->pAllUpdated = NULL;
}
pInfo->pCheckpointRes = createSpecialDataBlock(STREAM_CHECKPOINT);
QUERY_CHECK_NULL(pInfo->pCheckpointRes, code, lino, _error, terrno);
code = createSpecialDataBlock(STREAM_CHECKPOINT, &pInfo->pCheckpointRes);
QUERY_CHECK_CODE(code, lino, _error);
pInfo->reCkBlock = false;
pInfo->recvGetAll = false;

7
source/libs/executor/src/streamfilloperator.c
View File

@ -1394,11 +1394,8 @@ int32_t createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFillPhysi
}
}
pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
if (!pInfo->pDelRes) {
code = TSDB_CODE_OUT_OF_MEMORY;
QUERY_CHECK_CODE(code, lino, _error);
}
code = createSpecialDataBlock(STREAM_DELETE_RESULT, &pInfo->pDelRes);
QUERY_CHECK_CODE(code, lino, _error);
code = blockDataEnsureCapacity(pInfo->pDelRes, pOperator->resultInfo.capacity);
QUERY_CHECK_CODE(code, lino, _error);

62
source/libs/executor/src/streamtimewindowoperator.c
View File

@ -1918,10 +1918,15 @@ int32_t createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiN
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
pInfo->pPullDataMap = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
pInfo->pFinalPullDataMap = taosHashInit(64, hashFn, true, HASH_NO_LOCK);
pInfo->pPullDataRes = createSpecialDataBlock(STREAM_RETRIEVE);
code = createSpecialDataBlock(STREAM_RETRIEVE, &pInfo->pPullDataRes);
QUERY_CHECK_CODE(code, lino, _error);
pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired;
pInfo->ignoreExpiredDataSaved = false;
pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
code = createSpecialDataBlock(STREAM_DELETE_RESULT, &pInfo->pDelRes);
QUERY_CHECK_CODE(code, lino, _error);
pInfo->delIndex = 0;
pInfo->pDelWins = taosArrayInit(4, sizeof(SWinKey));
pInfo->delKey.ts = INT64_MAX;
@ -1936,11 +1941,16 @@ int32_t createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiN
pInfo->dataVersion = 0;
pInfo->stateStore = pTaskInfo->storageAPI.stateStore;
pInfo->recvGetAll = false;
pInfo->pCheckpointRes = createSpecialDataBlock(STREAM_CHECKPOINT);
pInfo->recvRetrive = false;
pInfo->pMidRetriveRes = createSpecialDataBlock(STREAM_MID_RETRIEVE);
pInfo->recvPullover = false;
pInfo->pMidPulloverRes = createSpecialDataBlock(STREAM_MID_RETRIEVE);
pInfo->recvRetrive = false;
code = createSpecialDataBlock(STREAM_CHECKPOINT, &pInfo->pCheckpointRes);
QUERY_CHECK_CODE(code, lino, _error);
code = createSpecialDataBlock(STREAM_MID_RETRIEVE, &pInfo->pMidRetriveRes);
QUERY_CHECK_CODE(code, lino, _error);
code = createSpecialDataBlock(STREAM_MID_RETRIEVE, &pInfo->pMidPulloverRes);
QUERY_CHECK_CODE(code, lino, _error);
pInfo->clearState = false;
pInfo->pMidPullDatas = taosArrayInit(4, sizeof(SWinKey));
pInfo->pDeletedMap = tSimpleHashInit(4096, hashFn);
@ -2101,13 +2111,18 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SExprSupp* pExpSup, in
SReadHandle* pHandle, STimeWindowAggSupp* pTwAggSup, const char* taskIdStr,
SStorageAPI* pApi, int32_t tsIndex) {
pSup->resultRowSize = keySize + getResultRowSize(pExpSup->pCtx, numOfOutput);
pSup->pScanBlock = createSpecialDataBlock(STREAM_CLEAR);
int32_t code = createSpecialDataBlock(STREAM_CLEAR, &pSup->pScanBlock);
if (code) {
return code;
}
pSup->gap = gap;
pSup->stateKeySize = keySize;
pSup->stateKeyType = keyType;
pSup->pDummyCtx = (SqlFunctionCtx*)taosMemoryCalloc(numOfOutput, sizeof(SqlFunctionCtx));
if (pSup->pDummyCtx == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
pSup->stateStore = *pStore;
@ -2129,7 +2144,6 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SExprSupp* pExpSup, in
}
pSup->pSessionAPI = pApi;
return TSDB_CODE_SUCCESS;
}
@ -3718,7 +3732,9 @@ int32_t createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
pInfo->pStDeleted = tSimpleHashInit(64, hashFn);
pInfo->pDelIterator = NULL;
pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
code = createSpecialDataBlock(STREAM_DELETE_RESULT, &pInfo->pDelRes);
QUERY_CHECK_CODE(code, lino, _error);
pInfo->pChildren = NULL;
pInfo->pPhyNode = pPhyNode;
pInfo->ignoreExpiredData = pSessionNode->window.igExpired;
@ -3734,7 +3750,9 @@ int32_t createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode
pInfo->isHistoryOp = pHandle->fillHistory;
}
pInfo->pCheckpointRes = createSpecialDataBlock(STREAM_CHECKPOINT);
code = createSpecialDataBlock(STREAM_CHECKPOINT, &pInfo->pCheckpointRes);
QUERY_CHECK_CODE(code, lino, _error);
pInfo->clearState = false;
pInfo->recvGetAll = false;
pInfo->destHasPrimaryKey = pSessionNode->window.destHasPrimayKey;
@ -3771,6 +3789,7 @@ int32_t createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode
*pOptrInfo = pOperator;
return code;
_error:
if (pInfo != NULL) {
destroyStreamSessionAggOperatorInfo(pInfo);
@ -4847,7 +4866,10 @@ int32_t createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
pInfo->pSeDeleted = tSimpleHashInit(64, hashFn);
pInfo->pDelIterator = NULL;
pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
code = createSpecialDataBlock(STREAM_DELETE_RESULT, &pInfo->pDelRes);
QUERY_CHECK_CODE(code, lino, _error);
pInfo->pChildren = NULL;
pInfo->ignoreExpiredData = pStateNode->window.igExpired;
pInfo->ignoreExpiredDataSaved = false;
@ -4856,14 +4878,17 @@ int32_t createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
pInfo->dataVersion = 0;
pInfo->historyWins = taosArrayInit(4, sizeof(SSessionKey));
if (!pInfo->historyWins) {
code = TSDB_CODE_OUT_OF_MEMORY;
code = terrno;
QUERY_CHECK_CODE(code, lino, _error);
}
if (pHandle) {
pInfo->isHistoryOp = pHandle->fillHistory;
}
pInfo->pCheckpointRes = createSpecialDataBlock(STREAM_CHECKPOINT);
code = createSpecialDataBlock(STREAM_CHECKPOINT, &pInfo->pCheckpointRes);
QUERY_CHECK_CODE(code, lino, _error);
pInfo->recvGetAll = false;
pInfo->pPkDeleted = tSimpleHashInit(64, hashFn);
pInfo->destHasPrimaryKey = pStateNode->window.destHasPrimayKey;
@ -5157,7 +5182,10 @@ int32_t createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
pInfo->invertible = false;
pInfo->pDelWins = taosArrayInit(4, sizeof(SWinKey));
pInfo->delIndex = 0;
pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
createSpecialDataBlock(STREAM_DELETE_RESULT, &pInfo->pDelRes);
QUERY_CHECK_CODE(code, lino, _error);
initResultRowInfo(&pInfo->binfo.resultRowInfo);
pInfo->pPhyNode = NULL; // create new child
@ -5187,7 +5215,9 @@ int32_t createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
pInfo->stateStore = pTaskInfo->storageAPI.stateStore;
pInfo->recvGetAll = false;
pInfo->pCheckpointRes = createSpecialDataBlock(STREAM_CHECKPOINT);
code = createSpecialDataBlock(STREAM_CHECKPOINT, &pInfo->pCheckpointRes);
QUERY_CHECK_CODE(code, lino, _error);
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
pInfo->pDeletedMap = tSimpleHashInit(4096, hashFn);

10
source/libs/executor/src/sysscanoperator.c
View File

@ -1176,11 +1176,17 @@ static SSDataBlock* buildInfoSchemaTableMetaBlock(char* tableName) {
}
}
SSDataBlock* pBlock = createDataBlock();
SSDataBlock* pBlock = NULL;
int32_t code = createDataBlock(&pBlock);
if (code) {
terrno = code;
return NULL;
}
for (int32_t i = 0; i < pMeta[index].colNum; ++i) {
SColumnInfoData colInfoData =
createColumnInfoData(pMeta[index].schema[i].type, pMeta[index].schema[i].bytes, i + 1);
int32_t code = blockDataAppendColInfo(pBlock, &colInfoData);
code = blockDataAppendColInfo(pBlock, &colInfoData);
if (code != TSDB_CODE_SUCCESS) {
qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code));
blockDataDestroy(pBlock);

485
source/libs/executor/src/tsort.c
View File

@ -27,6 +27,9 @@
#include "tsimplehash.h"
#include "executil.h"
#define AllocatedTupleType 0
#define ReferencedTupleType 1 // tuple references to one row in pDataBlock
struct STupleHandle {
SSDataBlock* pBlock;
int32_t rowIndex;
@ -70,15 +73,15 @@ struct SSortHandle {
int64_t startTs;
uint64_t totalElapsed;
uint64_t pqMaxRows;
uint32_t pqMaxTupleLength;
uint32_t pqSortBufSize;
bool forceUsePQSort;
BoundedQueue* pBoundedQueue;
uint32_t tmpRowIdx;
uint64_t pqMaxRows;
uint32_t pqMaxTupleLength;
uint32_t pqSortBufSize;
bool forceUsePQSort;
BoundedQueue* pBoundedQueue;
uint32_t tmpRowIdx;
int64_t mergeLimit;
int64_t currMergeLimitTs;
int64_t mergeLimit;
int64_t currMergeLimitTs;
int32_t sourceId;
SSDataBlock* pDataBlock;
@ -102,14 +105,14 @@ struct SSortHandle {
bool (*abortCheckFn)(void* param);
void* abortCheckParam;
bool bSortByRowId;
bool bSortByRowId;
SSortMemFile* pExtRowsMemFile;
int32_t extRowBytes;
int32_t extRowsPageSize;
int32_t extRowsMemSize;
int32_t srcTsSlotId;
SArray* aExtRowsOrders;
bool bSortPk;
int32_t extRowBytes;
int32_t extRowsPageSize;
int32_t extRowsMemSize;
int32_t srcTsSlotId;
SArray* aExtRowsOrders;
bool bSortPk;
void (*mergeLimitReachedFn)(uint64_t tableUid, void* param);
void* mergeLimitReachedParam;
};
@ -133,6 +136,7 @@ static void* createTuple(uint32_t columnNum, uint32_t tupleLen) {
uint32_t totalLen = sizeof(uint32_t) * columnNum + BitmapLen(columnNum) + tupleLen;
return taosMemoryCalloc(1, totalLen);
}
static void destoryAllocatedTuple(void* t) { taosMemoryFree(t); }
#define tupleOffset(tuple, colIdx) ((uint32_t*)(tuple + sizeof(uint32_t) * colIdx))
@ -148,46 +152,43 @@ static void destoryAllocatedTuple(void* t) { taosMemoryFree(t); }
* @param colIndex the columnIndex, for setting null bitmap
* @return the next offset to add field
* */
static inline size_t tupleAddField(char** t, uint32_t colNum, uint32_t offset, uint32_t colIdx, void* data, size_t length,
bool isNull, uint32_t tupleLen) {
static inline size_t tupleAddField(char** t, uint32_t colNum, uint32_t offset, uint32_t colIdx, void* data,
size_t length, bool isNull, uint32_t tupleLen) {
tupleSetOffset(*t, colIdx, offset);
if (isNull) {
tupleSetNull(*t, colIdx, colNum);
} else {
if (offset + length > tupleLen + tupleGetDataStartOffset(colNum)) {
*t = taosMemoryRealloc(*t, offset + length);
void* px = taosMemoryRealloc(*t, offset + length);
if (px == NULL) {
return terrno;
}
*t = px;
}
tupleSetData(*t, offset, data, length);
}
return offset + length;
}
static void* tupleGetField(char* t, uint32_t colIdx, uint32_t colNum) {
if (tupleColIsNull(t, colIdx, colNum)) return NULL;
if (tupleColIsNull(t, colIdx, colNum)) {
return NULL;
}
return t + *tupleOffset(t, colIdx);
}
int32_t tsortGetSortedDataBlock(const SSortHandle* pSortHandle, SSDataBlock** pBlock) {
if (pBlock == NULL) {
*pBlock = NULL;
return TSDB_CODE_SUCCESS;
}
if (pSortHandle->pDataBlock == NULL) {
*pBlock = NULL;
return TSDB_CODE_SUCCESS;
}
*pBlock = createOneDataBlock(pSortHandle->pDataBlock, false);
if (*pBlock == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
} else {
return TSDB_CODE_SUCCESS;
}
return createOneDataBlock(pSortHandle->pDataBlock, false, pBlock);
}
#define AllocatedTupleType 0
#define ReferencedTupleType 1 // tuple references to one row in pDataBlock
typedef struct TupleDesc {
uint8_t type;
char* data; // if type is AllocatedTuple, then points to the created tuple, otherwise points to the DataBlock
@ -198,17 +199,26 @@ typedef struct ReferencedTuple {
size_t rowIndex;
} ReferencedTuple;
static TupleDesc* createAllocatedTuple(SSDataBlock* pBlock, size_t colNum, uint32_t tupleLen, size_t rowIdx) {
static int32_t createAllocatedTuple(SSDataBlock* pBlock, size_t colNum, uint32_t tupleLen, size_t rowIdx, TupleDesc** pDesc) {
TupleDesc* t = taosMemoryCalloc(1, sizeof(TupleDesc));
void* pTuple = createTuple(colNum, tupleLen);
if (t == NULL) {
return terrno;
}
void* pTuple = createTuple(colNum, tupleLen);
if (!pTuple) {
taosMemoryFree(t);
return NULL;
return terrno;
}
size_t colLen = 0;
uint32_t offset = tupleGetDataStartOffset(colNum);
for (size_t colIdx = 0; colIdx < colNum; ++colIdx) {
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, colIdx);
if (pCol == NULL) {
return terrno;
}
if (colDataIsNull_s(pCol, rowIdx)) {
offset = tupleAddField((char**)&pTuple, colNum, offset, colIdx, 0, 0, true, tupleLen);
} else {
@ -217,20 +227,34 @@ static TupleDesc* createAllocatedTuple(SSDataBlock* pBlock, size_t colNum, uint3
tupleAddField((char**)&pTuple, colNum, offset, colIdx, colDataGetData(pCol, rowIdx), colLen, false, tupleLen);
}
}
t->type = AllocatedTupleType;
t->data = pTuple;
return t;
*pDesc = t;
return 0;
}
void* tupleDescGetField(const TupleDesc* pDesc, int32_t colIdx, uint32_t colNum) {
int32_t tupleDescGetField(const TupleDesc* pDesc, int32_t colIdx, uint32_t colNum, void** pResult) {
*pResult = NULL;
if (pDesc->type == ReferencedTupleType) {
ReferencedTuple* pRefTuple = (ReferencedTuple*)pDesc;
SColumnInfoData* pCol = taosArrayGet(((SSDataBlock*)pDesc->data)->pDataBlock, colIdx);
if (colDataIsNull_s(pCol, pRefTuple->rowIndex)) return NULL;
return colDataGetData(pCol, pRefTuple->rowIndex);
if (pCol == NULL) {
return terrno;
}
if (colDataIsNull_s(pCol, pRefTuple->rowIndex)) {
return TSDB_CODE_SUCCESS;
}
*pResult = colDataGetData(pCol, pRefTuple->rowIndex);
} else {
return tupleGetField(pDesc->data, colIdx, colNum);
*pResult = tupleGetField(pDesc->data, colIdx, colNum);
}
return 0;
}
void destroyTuple(void* t) {
@ -271,9 +295,8 @@ int32_t tsortCreateSortHandle(SArray* pSortInfo, int32_t type, int32_t pageSize,
pSortHandle->forceUsePQSort = false;
if (pBlock != NULL) {
pSortHandle->pDataBlock = createOneDataBlock(pBlock, false);
if (pSortHandle->pDataBlock == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
code = createOneDataBlock(pBlock, false, &pSortHandle->pDataBlock);
if (code) {
goto _err;
}
}
@ -282,7 +305,7 @@ int32_t tsortCreateSortHandle(SArray* pSortInfo, int32_t type, int32_t pageSize,
pSortHandle->pOrderedSource = taosArrayInit(4, POINTER_BYTES);
if (pSortHandle->pOrderedSource == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
code = terrno;
goto _err;
}
@ -410,7 +433,7 @@ static int32_t doAddNewExternalMemSource(SDiskbasedBuf* pBuf, SArray* pAllSource
SSortSource* pSource = taosMemoryCalloc(1, sizeof(SSortSource));
if (pSource == NULL) {
taosArrayDestroy(pPageIdList);
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
pSource->src.pBlock = pBlock;
@ -453,6 +476,10 @@ static int32_t doAddToBuf(SSDataBlock* pDataBlock, SSortHandle* pHandle) {
}
SArray* pPageIdList = taosArrayInit(4, sizeof(int32_t));
if (pPageIdList == NULL) {
return terrno;
}
while (start < pDataBlock->info.rows) {
int32_t stop = 0;
@ -462,10 +489,11 @@ static int32_t doAddToBuf(SSDataBlock* pDataBlock, SSortHandle* pHandle) {
return code;
}
SSDataBlock* p = blockDataExtractBlock(pDataBlock, start, stop - start + 1);
if (p == NULL) {
SSDataBlock* p = NULL;
code = blockDataExtractBlock(pDataBlock, start, stop - start + 1, &p);
if (code) {
taosArrayDestroy(pPageIdList);
return terrno;
return code;
}
int32_t pageId = -1;
@ -500,7 +528,12 @@ static int32_t doAddToBuf(SSDataBlock* pDataBlock, SSortHandle* pHandle) {
blockDataCleanup(pDataBlock);
SSDataBlock* pBlock = createOneDataBlock(pDataBlock, false);
SSDataBlock* pBlock = NULL;
int32_t code = createOneDataBlock(pDataBlock, false, &pBlock);
if (code) {
return code;
}
return doAddNewExternalMemSource(pHandle->pBuf, pHandle->pOrderedSource, pBlock, &pHandle->sourceId, pPageIdList);
}
@ -512,8 +545,11 @@ static void setCurrentSourceDone(SSortSource* pSource, SSortHandle* pHandle) {
static int32_t sortComparInit(SMsortComparParam* pParam, SArray* pSources, int32_t startIndex, int32_t endIndex,
SSortHandle* pHandle) {
pParam->pSources = taosArrayGet(pSources, startIndex);
pParam->numOfSources = (endIndex - startIndex + 1);
if (pParam->pSources == NULL) {
return terrno;
}
pParam->numOfSources = (endIndex - startIndex + 1);
int32_t code = 0;
// multi-pass internal merge sort is required
@ -544,6 +580,9 @@ static int32_t sortComparInit(SMsortComparParam* pParam, SArray* pSources, int32
}
int32_t* pPgId = taosArrayGet(pSource->pageIdList, pSource->pageIndex);
if (pPgId == NULL) {
return terrno;
}
void* pPage = getBufPage(pHandle->pBuf, *pPgId);
if (NULL == pPage) {
@ -584,7 +623,14 @@ static int32_t appendOneRowToDataBlock(SSDataBlock* pBlock, const SSDataBlock* p
for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) {
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);
if (pColInfo == NULL) {
return terrno;
}
SColumnInfoData* pSrcColInfo = taosArrayGet(pSource->pDataBlock, i);
if (pSrcColInfo == NULL) {
return terrno;
}
bool isNull = colDataIsNull(pSrcColInfo, pSource->info.rows, *rowIndex, NULL);
if (isNull) {
@ -631,8 +677,11 @@ static int32_t adjustMergeTreeForNextTuple(SSortSource* pSource, SMultiwayMergeT
}
int32_t* pPgId = taosArrayGet(pSource->pageIdList, pSource->pageIndex);
if (pPgId == NULL) {
return terrno;
}
void* pPage = getBufPage(pHandle->pBuf, *pPgId);
void* pPage = getBufPage(pHandle->pBuf, *pPgId);
if (pPage == NULL) {
qError("failed to get buffer, code:%s", tstrerror(terrno));
return terrno;
@ -906,8 +955,10 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) {
size_t numOfSorted = taosArrayGetSize(pHandle->pOrderedSource);
for (int32_t t = 0; t < sortPass; ++t) {
int64_t st = taosGetTimestampUs();
SArray* pResList = taosArrayInit(4, POINTER_BYTES);
if (pResList == NULL) {
return terrno;
}
int32_t numOfInputSources = pHandle->numOfPages;
int32_t sortGroup = (numOfSorted + numOfInputSources - 1) / numOfInputSources;
@ -938,8 +989,12 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) {
}
int32_t nMergedRows = 0;
SArray* pPageIdList = taosArrayInit(4, sizeof(int32_t));
if (pPageIdList == NULL) {
taosArrayDestroy(pResList);
return terrno;
}
while (1) {
if (tsortIsClosed(pHandle) || (pHandle->abortCheckFn && pHandle->abortCheckFn(pHandle->abortCheckParam))) {
code = terrno = TSDB_CODE_TSC_QUERY_CANCELLED;
@ -994,7 +1049,14 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) {
tMergeTreeDestroy(&pHandle->pMergeTree);
pHandle->numOfCompletedSources = 0;
SSDataBlock* pBlock = createOneDataBlock(pHandle->pDataBlock, false);
SSDataBlock* pBlock = NULL;
code = createOneDataBlock(pHandle->pDataBlock, false, &pBlock);
if (code) {
taosArrayDestroy(pResList);
return code;
}
code = doAddNewExternalMemSource(pHandle->pBuf, pResList, pBlock, &pHandle->sourceId, pPageIdList);
if (code != TSDB_CODE_SUCCESS) {
taosArrayDestroy(pResList);
@ -1083,6 +1145,9 @@ int32_t tsortAppendTupleToBlock(SSortHandle* pHandle, SSDataBlock* pBlock, STupl
char* pStart = (char*)buf + sizeof(int8_t) * numOfCols;
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);
if (pColInfo == NULL) {
return terrno;
}
if (!isNull[i]) {
code = colDataSetVal(pColInfo, pBlock->info.rows, pStart, false);
@ -1124,7 +1189,11 @@ int32_t tsortAppendTupleToBlock(SSortHandle* pHandle, SSDataBlock* pBlock, STupl
} else {
for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) {
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);
bool isNull = tsortIsNullVal(pTupleHandle, i);
if (pColInfo == NULL) {
return terrno;
}
bool isNull = tsortIsNullVal(pTupleHandle, i);
if (isNull) {
colDataSetNULL(pColInfo, pBlock->info.rows);
} else {
@ -1157,6 +1226,10 @@ static int32_t blockRowToBuf(SSDataBlock* pBlock, int32_t rowIdx, char* buf) {
char* pStart = (char*)buf + sizeof(int8_t) * numOfCols;
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i);
if (pCol == NULL) {
return terrno;
}
if (colDataIsNull_s(pCol, rowIdx)) {
isNull[i] = 1;
continue;
@ -1198,11 +1271,15 @@ static int32_t getRowBufFromExtMemFile(SSortHandle* pHandle, int32_t regionId, i
char** ppRow, bool* pFreeRow) {
SSortMemFile* pMemFile = pHandle->pExtRowsMemFile;
SSortMemFileRegion* pRegion = taosArrayGet(pMemFile->aFileRegions, regionId);
if (pRegion == NULL) {
return terrno;
}
if (pRegion->buf == NULL) {
pRegion->bufRegOffset = 0;
pRegion->buf = taosMemoryMalloc(pMemFile->blockSize);
if (pRegion->buf == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
// todo
@ -1223,7 +1300,7 @@ static int32_t getRowBufFromExtMemFile(SSortHandle* pHandle, int32_t regionId, i
} else {
*ppRow = taosMemoryMalloc(rowLen);
if (*ppRow == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
int32_t szThisBlock = pRegion->bufLen - (tupleOffset - pRegion->bufRegOffset);
memcpy(*ppRow, pRegion->buf + tupleOffset - pRegion->bufRegOffset, szThisBlock);
@ -1277,7 +1354,7 @@ static int32_t createSortMemFile(SSortHandle* pHandle) {
pMemFile->writeBuf = taosMemoryMalloc(pMemFile->writeBufSize);
if (pMemFile->writeBuf == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
code = terrno;
}
}
@ -1285,7 +1362,7 @@ static int32_t createSortMemFile(SSortHandle* pHandle) {
pMemFile->cacheSize = pHandle->extRowsMemSize;
pMemFile->aFileRegions = taosArrayInit(64, sizeof(SSortMemFileRegion));
if (pMemFile->aFileRegions == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
code = terrno;
}
}
@ -1313,8 +1390,13 @@ static void destroySortMemFile(SSortHandle* pHandle) {
SSortMemFile* pMemFile = pHandle->pExtRowsMemFile;
for (int32_t i = 0; i < taosArrayGetSize(pMemFile->aFileRegions); ++i) {
SSortMemFileRegion* pRegion = taosArrayGet(pMemFile->aFileRegions, i);
if (pRegion == NULL) {
continue;
}
taosMemoryFree(pRegion->buf);
}
taosArrayDestroy(pMemFile->aFileRegions);
pMemFile->aFileRegions = NULL;
@ -1338,7 +1420,7 @@ static int32_t tsortOpenRegion(SSortHandle* pHandle) {
region.bufRegOffset = 0;
void* px = taosArrayPush(pMemFile->aFileRegions, &region);
if (px == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
code = terrno;
}
pMemFile->currRegionId = 0;
@ -1347,11 +1429,16 @@ static int32_t tsortOpenRegion(SSortHandle* pHandle) {
} else {
SSortMemFileRegion regionNew = {0};
SSortMemFileRegion* pRegion = taosArrayGet(pMemFile->aFileRegions, pMemFile->currRegionId);
if (pRegion == NULL) {
return terrno;
}
regionNew.fileOffset = pRegion->fileOffset + pRegion->regionSize;
regionNew.bufRegOffset = 0;
void* px = taosArrayPush(pMemFile->aFileRegions, &regionNew);
if (px == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
code = terrno;
}
++pMemFile->currRegionId;
pMemFile->currRegionOffset = 0;
@ -1363,6 +1450,10 @@ static int32_t tsortOpenRegion(SSortHandle* pHandle) {
static int32_t tsortCloseRegion(SSortHandle* pHandle) {
SSortMemFile* pMemFile = pHandle->pExtRowsMemFile;
SSortMemFileRegion* pRegion = taosArrayGet(pMemFile->aFileRegions, pMemFile->currRegionId);
if (pRegion == NULL) {
return terrno;
}
pRegion->regionSize = pMemFile->currRegionOffset;
int32_t writeBytes = pRegion->regionSize - (pMemFile->writeFileOffset - pRegion->fileOffset);
if (writeBytes > 0) {
@ -1390,7 +1481,7 @@ static int32_t tsortFinalizeRegions(SSortHandle* pHandle) {
for (int32_t i = 0; i < numRegions; ++i) {
SSortMemFileRegion* pRegion = taosArrayGet(pMemFile->aFileRegions, i);
if (pRegion == NULL) {
return TSDB_CODE_INVALID_PARA;
return terrno;
}
pRegion->bufRegOffset = 0;
@ -1406,6 +1497,10 @@ static int32_t saveBlockRowToExtRowsMemFile(SSortHandle* pHandle, SSDataBlock* p
SSortMemFile* pMemFile = pHandle->pExtRowsMemFile;
SSortMemFileRegion* pRegion = taosArrayGet(pMemFile->aFileRegions, pMemFile->currRegionId);
if (pRegion == NULL) {
return terrno;
}
{
if (pMemFile->currRegionOffset + pHandle->extRowBytes >= pMemFile->writeBufSize) {
int32_t writeBytes = pMemFile->currRegionOffset - (pMemFile->writeFileOffset - pRegion->fileOffset);
@ -1442,8 +1537,20 @@ static int32_t appendToRowIndexDataBlock(SSortHandle* pHandle, SSDataBlock* pSou
SSDataBlock* pBlock = pHandle->pDataBlock;
SBlockOrderInfo* extRowsTsOrder = taosArrayGet(pHandle->aExtRowsOrders, 0);
if (extRowsTsOrder == NULL) {
return terrno;
}
SColumnInfoData* pSrcTsCol = taosArrayGet(pSource->pDataBlock, extRowsTsOrder->slotId);
if (pSrcTsCol == NULL) {
return terrno;
}
SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, 0);
if (pTsCol == NULL) {
return terrno;
}
char* pData = colDataGetData(pSrcTsCol, *rowIndex);
code = colDataSetVal(pTsCol, pBlock->info.rows, pData, false);
if (code) {
@ -1451,18 +1558,42 @@ static int32_t appendToRowIndexDataBlock(SSortHandle* pHandle, SSDataBlock* pSou
}
SColumnInfoData* pRegionIdCol = taosArrayGet(pBlock->pDataBlock, 1);
if (pRegionIdCol == NULL) {
return terrno;
}
colDataSetInt32(pRegionIdCol, pBlock->info.rows, &pageId);
SColumnInfoData* pOffsetCol = taosArrayGet(pBlock->pDataBlock, 2);
if (pOffsetCol == NULL) {
return terrno;
}
colDataSetInt32(pOffsetCol, pBlock->info.rows, &offset);
SColumnInfoData* pLengthCol = taosArrayGet(pBlock->pDataBlock, 3);
if (pLengthCol == NULL) {
return terrno;
}
colDataSetInt32(pLengthCol, pBlock->info.rows, &length);
if (pHandle->bSortPk) {
SBlockOrderInfo* extRowsPkOrder = taosArrayGet(pHandle->aExtRowsOrders, 1);
if (extRowsPkOrder == NULL) {
return terrno;
}
SColumnInfoData* pSrcPkCol = taosArrayGet(pSource->pDataBlock, extRowsPkOrder->slotId);
if (pSrcPkCol == NULL) {
return terrno;
}
SColumnInfoData* pPkCol = taosArrayGet(pBlock->pDataBlock, 4);
if (pPkCol == NULL) {
return terrno;
}
if (colDataIsNull_s(pSrcPkCol, *rowIndex)) {
colDataSetNULL(pPkCol, pBlock->info.rows);
} else {
@ -1481,12 +1612,18 @@ static int32_t appendToRowIndexDataBlock(SSortHandle* pHandle, SSDataBlock* pSou
static int32_t initRowIdSort(SSortHandle* pHandle) {
SBlockOrderInfo* pkOrder = (pHandle->bSortPk) ? taosArrayGet(pHandle->aExtRowsOrders, 1) : NULL;
SColumnInfoData* extPkCol = (pHandle->bSortPk) ? taosArrayGet(pHandle->pDataBlock->pDataBlock, pkOrder->slotId) : NULL;
SColumnInfoData pkCol = {0};
SColumnInfoData* extPkCol =
(pHandle->bSortPk) ? taosArrayGet(pHandle->pDataBlock->pDataBlock, pkOrder->slotId) : NULL;
SColumnInfoData pkCol = {0};
SSDataBlock* pSortInput = NULL;
int32_t code = createDataBlock(&pSortInput);
if (code) {
return code;
}
SSDataBlock* pSortInput = createDataBlock();
SColumnInfoData tsCol = createColumnInfoData(TSDB_DATA_TYPE_TIMESTAMP, 8, 1);
int32_t code = blockDataAppendColInfo(pSortInput, &tsCol);
code = blockDataAppendColInfo(pSortInput, &tsCol);
if (code) {
blockDataDestroy(pSortInput);
return code;
@ -1499,14 +1636,14 @@ static int32_t initRowIdSort(SSortHandle* pHandle) {
return code;
}
SColumnInfoData offsetCol = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 3);
SColumnInfoData offsetCol = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 3);
code = blockDataAppendColInfo(pSortInput, &offsetCol);
if (code) {
blockDataDestroy(pSortInput);
return code;
}
SColumnInfoData lengthCol = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 4);
SColumnInfoData lengthCol = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 4);
code = blockDataAppendColInfo(pSortInput, &lengthCol);
if (code) {
blockDataDestroy(pSortInput);
@ -1525,14 +1662,14 @@ static int32_t initRowIdSort(SSortHandle* pHandle) {
blockDataDestroy(pHandle->pDataBlock);
pHandle->pDataBlock = pSortInput;
// int32_t rowSize = blockDataGetRowSize(pHandle->pDataBlock);
// size_t nCols = taosArrayGetSize(pHandle->pDataBlock->pDataBlock);
pHandle->pageSize = 256 * 1024; // 256k
// int32_t rowSize = blockDataGetRowSize(pHandle->pDataBlock);
// size_t nCols = taosArrayGetSize(pHandle->pDataBlock->pDataBlock);
pHandle->pageSize = 256 * 1024; // 256k
pHandle->numOfPages = 256;
SArray* pOrderInfoList = taosArrayInit(1, sizeof(SBlockOrderInfo));
if (pOrderInfoList == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
int32_t tsOrder = ((SBlockOrderInfo*)taosArrayGet(pHandle->pSortInfo, 0))->order;
@ -1544,7 +1681,7 @@ static int32_t initRowIdSort(SSortHandle* pHandle) {
biTs.compFn = getKeyComparFunc(TSDB_DATA_TYPE_TIMESTAMP, biTs.order);
void* p = taosArrayPush(pOrderInfoList, &biTs);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
if (pHandle->bSortPk) {
@ -1556,7 +1693,7 @@ static int32_t initRowIdSort(SSortHandle* pHandle) {
void* px = taosArrayPush(pOrderInfoList, &biPk);
if (px == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
}
@ -1648,7 +1785,7 @@ static int32_t appendDataBlockToPageBuf(SSortHandle* pHandle, SSDataBlock* blk,
void* px = taosArrayPush(aPgId, &pageId);
if (px == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
int32_t size = blockDataGetSize(blk) + sizeof(int32_t) + taosArrayGetSize(blk->pDataBlock) * sizeof(int32_t);
@ -1810,13 +1947,22 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SArray*
SBlockOrderInfo* pOrigBlockTsOrder =
(!pHandle->bSortByRowId) ? taosArrayGet(pHandle->pSortInfo, 0) : taosArrayGet(pHandle->aExtRowsOrders, 0);
if (pOrigBlockTsOrder == NULL) {
return terrno;
}
SBlockOrderInfo* pHandleBlockTsOrder = taosArrayGet(pHandle->pSortInfo, 0);
if (pHandleBlockTsOrder == NULL) {
return terrno;
}
SBlockOrderInfo* pOrigBlockPkOrder = NULL;
if (pHandle->bSortPk) {
pOrigBlockPkOrder =
(!pHandle->bSortByRowId) ? taosArrayGet(pHandle->pSortInfo, 1) : taosArrayGet(pHandle->aExtRowsOrders, 1);
if (pOrigBlockPkOrder == NULL) {
return terrno;
}
}
code = initMergeSup(&sup, aBlk, pOrigBlockTsOrder->order, pOrigBlockTsOrder->slotId, pOrigBlockPkOrder);
@ -1836,6 +1982,10 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SArray*
}
SArray* aPgId = taosArrayInit(8, sizeof(int32_t));
if (aPgId == NULL) {
return terrno;
}
int32_t nRows = 0;
int32_t nMergedRows = 0;
bool mergeLimitReached = false;
@ -1928,7 +2078,14 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SArray*
blockDataCleanup(pHandle->pDataBlock);
}
SSDataBlock* pMemSrcBlk = createOneDataBlock(pHandle->pDataBlock, false);
SSDataBlock* pMemSrcBlk = NULL;
code = createOneDataBlock(pHandle->pDataBlock, false, &pMemSrcBlk);
if (code) {
cleanupMergeSup(&sup);
tMergeTreeDestroy(&pTree);
return code;
}
code = doAddNewExternalMemSource(pHandle->pBuf, aExtSrc, pMemSrcBlk, &pHandle->sourceId, aPgId);
cleanupMergeSup(&sup);
@ -1975,7 +2132,11 @@ static int32_t getRowsBlockWithinMergeLimit(const SSortHandle* pHandle, SSHashOb
*pSkipBlock = false;
SSDataBlock* pBlock = NULL;
if (keepRows != pOrigBlk->info.rows) {
pBlock = blockDataExtractBlock(pOrigBlk, 0, keepRows);
code = blockDataExtractBlock(pOrigBlk, 0, keepRows, &pBlock);
if (code) {
return code;
}
*pExtractedBlock = true;
} else {
*pExtractedBlock = false;
@ -1987,11 +2148,14 @@ static int32_t getRowsBlockWithinMergeLimit(const SSortHandle* pHandle, SSHashOb
}
static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) {
int32_t szSort = 0;
size_t nSrc = taosArrayGetSize(pHandle->pOrderedSource);
SArray* aExtSrc = taosArrayInit(nSrc, POINTER_BYTES);
if (aExtSrc == NULL) {
return terrno;
}
size_t maxBufSize = (pHandle->bSortByRowId) ? pHandle->extRowsMemSize : (pHandle->numOfPages * pHandle->pageSize);
int32_t code = createPageBuf(pHandle);
if (code != TSDB_CODE_SUCCESS) {
taosArrayDestroy(aExtSrc);
@ -1999,10 +2163,17 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) {
}
SSortSource* pSrc = taosArrayGetP(pHandle->pOrderedSource, 0);
int32_t szSort = 0;
if (pSrc == NULL) {
taosArrayDestroy(aExtSrc);
return TSDB_CODE_INVALID_PARA;
}
SBlockOrderInfo* pOrigTsOrder = (!pHandle->bSortByRowId) ?
taosArrayGet(pHandle->pSortInfo, 0) : taosArrayGet(pHandle->aExtRowsOrders, 0);
if (pOrigTsOrder == NULL) {
return terrno;
}
if (pOrigTsOrder->order == TSDB_ORDER_ASC) {
pHandle->currMergeLimitTs = INT64_MAX;
} else {
@ -2010,13 +2181,28 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) {
}
SSHashObj* mTableNumRows = tSimpleHashInit(8192, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT));
SArray* aBlkSort = taosArrayInit(8, POINTER_BYTES);
SSHashObj* mUidBlk = tSimpleHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT));
while (1) {
SSDataBlock* pBlk = pHandle->fetchfp(pSrc->param);
if (mTableNumRows == NULL) {
return terrno;
}
SArray* aBlkSort = taosArrayInit(8, POINTER_BYTES);
if (aBlkSort == NULL) {
tSimpleHashCleanup(mTableNumRows);
return terrno;
}
SSHashObj* mUidBlk = tSimpleHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT));
if (mUidBlk == NULL) {
tSimpleHashCleanup(mTableNumRows);
taosArrayDestroy(aBlkSort);
return terrno;
}
while (1) {
bool bExtractedBlock = false;
bool bSkipBlock = false;
SSDataBlock* pBlk = pHandle->fetchfp(pSrc->param);
if (pBlk != NULL && pHandle->mergeLimit > 0) {
SSDataBlock* p = NULL;
code = getRowsBlockWithinMergeLimit(pHandle, mTableNumRows, pBlk, &bExtractedBlock, &bSkipBlock, &p);
@ -2029,7 +2215,11 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) {
if (pBlk != NULL) {
SColumnInfoData* tsCol = taosArrayGet(pBlk->pDataBlock, pOrigTsOrder->slotId);
int64_t firstRowTs = *(int64_t*)tsCol->pData;
if (tsCol == NULL) {
return terrno;
}
int64_t firstRowTs = *(int64_t*)tsCol->pData;
if ((pOrigTsOrder->order == TSDB_ORDER_ASC && firstRowTs > pHandle->currMergeLimitTs) ||
(pOrigTsOrder->order == TSDB_ORDER_DESC && firstRowTs < pHandle->currMergeLimitTs)) {
if (bExtractedBlock) {
@ -2044,17 +2234,19 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) {
void* ppBlk = tSimpleHashGet(mUidBlk, &pBlk->info.id.uid, sizeof(pBlk->info.id.uid));
if (ppBlk != NULL) {
SSDataBlock* tBlk = *(SSDataBlock**)(ppBlk);
code = blockDataMerge(tBlk, pBlk);
if (code) {
return code;
}
TAOS_CHECK_RETURN(blockDataMerge(tBlk, pBlk));
if (bExtractedBlock) {
blockDataDestroy(pBlk);
}
} else {
SSDataBlock* tBlk = (bExtractedBlock) ? pBlk : createOneDataBlock(pBlk, true);
SSDataBlock* tBlk = NULL;
if (bExtractedBlock) {
tBlk = pBlk;
} else {
TAOS_CHECK_RETURN(createOneDataBlock(pBlk, true, &tBlk));
}
code = tSimpleHashPut(mUidBlk, &pBlk->info.id.uid, sizeof(pBlk->info.id.uid), &tBlk, POINTER_BYTES);
if (code) {
return code;
@ -2062,7 +2254,7 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) {
void* px = taosArrayPush(aBlkSort, &tBlk);
if (px == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
}
}
@ -2072,10 +2264,7 @@ static int32_t createBlocksMergeSortInitialSources(SSortHandle* pHandle) {
int64_t p = taosGetTimestampUs();
if (pHandle->bSortByRowId) {
code = tsortOpenRegion(pHandle);
if (code) {
return code;
}
TAOS_CHECK_RETURN(tsortOpenRegion(pHandle));
}
code = sortBlocksToExtSource(pHandle, aBlkSort, aExtSrc);
@ -2153,10 +2342,13 @@ static void freeSSortSource(SSortSource* source) {
}
static int32_t createBlocksQuickSortInitialSources(SSortHandle* pHandle) {
int32_t code = 0;
size_t sortBufSize = pHandle->numOfPages * pHandle->pageSize;
int32_t code = 0;
size_t sortBufSize = pHandle->numOfPages * pHandle->pageSize;
SSortSource** pSource = taosArrayGet(pHandle->pOrderedSource, 0);
if (pSource == NULL) {
return terrno;
}
SSortSource* source = *pSource;
*pSource = NULL;
@ -2175,7 +2367,11 @@ static int32_t createBlocksQuickSortInitialSources(SSortHandle* pHandle) {
// todo, number of pages are set according to the total available sort buffer
pHandle->numOfPages = 1024;
sortBufSize = pHandle->numOfPages * pHandle->pageSize;
pHandle->pDataBlock = createOneDataBlock(pBlock, false);
code = createOneDataBlock(pBlock, false, &pHandle->pDataBlock);
if (code) {
freeSSortSource(source);
return code;
}
}
if (pHandle->beforeFp != NULL) {
@ -2250,6 +2446,7 @@ static int32_t createInitialSources(SSortHandle* pHandle) {
} else if (pHandle->type == SORT_BLOCK_TS_MERGE) {
code = createBlocksMergeSortInitialSources(pHandle);
}
qDebug("%zu sources created", taosArrayGetSize(pHandle->pOrderedSource));
return code;
}
@ -2411,14 +2608,31 @@ static int32_t tupleComparFn(const void* pLeft, const void* pRight, void* param)
uint32_t colNum = blockDataGetNumOfCols(pHandle->pDataBlock);
for (int32_t i = 0; i < orderInfo->size; ++i) {
SBlockOrderInfo* pOrder = TARRAY_GET_ELEM(orderInfo, i);
void *lData = tupleDescGetField(pLeftDesc, pOrder->slotId, colNum);
void *rData = tupleDescGetField(pRightDesc, pOrder->slotId, colNum);
if (!lData && !rData) continue;
void *lData = NULL, *rData = NULL;
int32_t ret1 = tupleDescGetField(pLeftDesc, pOrder->slotId, colNum, &lData);
int32_t ret2 = tupleDescGetField(pRightDesc, pOrder->slotId, colNum, &rData);
if (ret1) {
return ret1;
}
if (ret2) {
return ret2;
}
if ((!lData) && (!rData)) {
continue;
}
if (!lData) return pOrder->nullFirst ? -1 : 1;
if (!rData) return pOrder->nullFirst ? 1 : -1;
int32_t type = ((SColumnInfoData*)taosArrayGet(pHandle->pDataBlock->pDataBlock, pOrder->slotId))->info.type;
__compar_fn_t fn = getKeyComparFunc(type, pOrder->order);
SColumnInfoData* p = (SColumnInfoData*)taosArrayGet(pHandle->pDataBlock->pDataBlock, pOrder->slotId);
if (p == NULL) {
return terrno;
}
__compar_fn_t fn = getKeyComparFunc(p->info.type, pOrder->order);
int32_t ret = fn(lData, rData);
if (ret == 0) {
@ -2427,20 +2641,28 @@ static int32_t tupleComparFn(const void* pLeft, const void* pRight, void* param)
return ret;
}
}
return 0;
}
static int32_t tsortOpenForPQSort(SSortHandle* pHandle) {
pHandle->pBoundedQueue = createBoundedQueue(pHandle->pqMaxRows, tsortPQCompFn, destroyTuple, pHandle);
if (NULL == pHandle->pBoundedQueue) return TSDB_CODE_OUT_OF_MEMORY;
if (NULL == pHandle->pBoundedQueue) {
return TSDB_CODE_OUT_OF_MEMORY;
}
tsortSetComparFp(pHandle, tupleComparFn);
SSortSource** pSource = taosArrayGet(pHandle->pOrderedSource, 0);
SSortSource* source = *pSource;
if (pSource == NULL) {
return terrno;
}
pHandle->pDataBlock = NULL;
uint32_t tupleLen = 0;
SSortSource* source = *pSource;
uint32_t tupleLen = 0;
PriorityQueueNode pqNode;
pHandle->pDataBlock = NULL;
while (1) {
// fetch data
SSDataBlock* pBlock = pHandle->fetchfp(source->param);
@ -2453,11 +2675,10 @@ static int32_t tsortOpenForPQSort(SSortHandle* pHandle) {
}
if (pHandle->pDataBlock == NULL) {
pHandle->pDataBlock = createOneDataBlock(pBlock, false);
}
if (pHandle->pDataBlock == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
int32_t code = createOneDataBlock(pBlock, false, &pHandle->pDataBlock);
if (code) {
return code;
}
}
size_t colNum = blockDataGetNumOfCols(pBlock);
@ -2465,12 +2686,17 @@ static int32_t tsortOpenForPQSort(SSortHandle* pHandle) {
if (tupleLen == 0) {
for (size_t colIdx = 0; colIdx < colNum; ++colIdx) {
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, colIdx);
if (pCol == NULL) {
return terrno;
}
tupleLen += pCol->info.bytes;
if (IS_VAR_DATA_TYPE(pCol->info.type)) {
tupleLen += sizeof(VarDataLenT);
}
}
}
ReferencedTuple refTuple = {.desc.data = (char*)pBlock, .desc.type = ReferencedTupleType, .rowIndex = 0};
for (size_t rowIdx = 0; rowIdx < pBlock->info.rows; ++rowIdx) {
refTuple.rowIndex = rowIdx;
@ -2479,11 +2705,15 @@ static int32_t tsortOpenForPQSort(SSortHandle* pHandle) {
if (!pPushedNode) {
// do nothing if push failed
} else {
pPushedNode->data = createAllocatedTuple(pBlock, colNum, tupleLen, rowIdx);
if (pPushedNode->data == NULL) return TSDB_CODE_OUT_OF_MEMORY;
pPushedNode->data = NULL;
int32_t code = createAllocatedTuple(pBlock, colNum, tupleLen, rowIdx, (TupleDesc**)&pPushedNode->data);
if (code) {
return code;
}
}
}
}
return TSDB_CODE_SUCCESS;
}
@ -2517,13 +2747,14 @@ static int32_t tsortPQSortNextTuple(SSortHandle* pHandle, STupleHandle **pTupleH
for (uint32_t i = 0; i < colNum; ++i) {
void* pData = tupleGetField(pTuple, i, colNum);
SColumnInfoData* p = NULL;
TAOS_CHECK_RETURN(bdGetColumnInfoData(pHandle->pDataBlock, i, &p));
if (!pData) {
colDataSetNULL(bdGetColumnInfoData(pHandle->pDataBlock, i), 0);
colDataSetNULL(p, 0);
} else {
code = colDataSetVal(bdGetColumnInfoData(pHandle->pDataBlock, i), 0, pData, false);
if (code) {
return code;
}
TAOS_CHECK_RETURN(colDataSetVal(p, 0, pData, false));
}
}
pHandle->pDataBlock->info.rows++;
@ -2556,6 +2787,10 @@ static int32_t tsortSingleTableMergeNextTuple(SSortHandle* pHandle, STupleHandle
}
SSortSource** pSource = taosArrayGet(pHandle->pOrderedSource, 0);
if (pSource == NULL) {
return terrno;
}
SSortSource* source = *pSource;
SSDataBlock* pBlock = pHandle->fetchfp(source->param);
if (!pBlock || pBlock->info.rows == 0) {
@ -2577,7 +2812,7 @@ int32_t tsortOpen(SSortHandle* pHandle) {
}
if (pHandle->fetchfp == NULL || pHandle->comparFn == NULL) {
return -1;
return TSDB_CODE_INVALID_PARA;
}
pHandle->opened = true;
@ -2603,6 +2838,10 @@ int32_t tsortNextTuple(SSortHandle* pHandle, STupleHandle** pTupleHandle) {
bool tsortIsNullVal(STupleHandle* pVHandle, int32_t colIndex) {
SColumnInfoData* pColInfoSrc = taosArrayGet(pVHandle->pBlock->pDataBlock, colIndex);
if (pColInfoSrc == NULL) {
return true;
}
return colDataIsNull_s(pColInfoSrc, pVHandle->rowIndex);
}

36
source/libs/executor/test/executorTests.cpp
View File

@ -71,11 +71,16 @@ SSDataBlock* getDummyBlock(SOperatorInfo* pOperator) {
}
if (pInfo->pBlock == NULL) {
pInfo->pBlock = createDataBlock();
pInfo->pBlock = NULL;
int32_t code = createDataBlock(&pInfo->pBlock);
ASSERT(code == 0);
SColumnInfoData colInfo = createColumnInfoData(TSDB_DATA_TYPE_INT, sizeof(int32_t), 1);
blockDataAppendColInfo(pInfo->pBlock, &colInfo);
blockDataEnsureCapacity(pInfo->pBlock, pInfo->numOfRowsPerPage);
code = blockDataAppendColInfo(pInfo->pBlock, &colInfo);
ASSERT(code == 0);
code = blockDataEnsureCapacity(pInfo->pBlock, pInfo->numOfRowsPerPage);
ASSERT(code == 0);
// SColumnInfoData colInfo1 = {0};
// colInfo1.info.type = TSDB_DATA_TYPE_BINARY;
@ -107,7 +112,8 @@ SSDataBlock* getDummyBlock(SOperatorInfo* pOperator) {
v = taosRand();
}
colDataSetVal(pColInfo, i, reinterpret_cast<const char*>(&v), false);
int32_t code = colDataSetVal(pColInfo, i, reinterpret_cast<const char*>(&v), false);
ASSERT(code == 0);
// sprintf(buf, "this is %d row", i);
// STR_TO_VARSTR(b1, buf);
@ -129,15 +135,21 @@ SSDataBlock* get2ColsDummyBlock(SOperatorInfo* pOperator) {
}
if (pInfo->pBlock == NULL) {
pInfo->pBlock = createDataBlock();
pInfo->pBlock = NULL;
int32_t code = createDataBlock(&pInfo->pBlock);
ASSERT(code == 0);
SColumnInfoData colInfo = createColumnInfoData(TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 1);
blockDataAppendColInfo(pInfo->pBlock, &colInfo);
int32_t code = blockDataAppendColInfo(pInfo->pBlock, &colInfo);
ASSERT(code == 0);
SColumnInfoData colInfo1 = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 2);
blockDataAppendColInfo(pInfo->pBlock, &colInfo1);
code = blockDataAppendColInfo(pInfo->pBlock, &colInfo1);
ASSERT(code == 0);
blockDataEnsureCapacity(pInfo->pBlock, pInfo->numOfRowsPerPage);
code = blockDataEnsureCapacity(pInfo->pBlock, pInfo->numOfRowsPerPage);
ASSERT(code == 0);
} else {
blockDataCleanup(pInfo->pBlock);
}
@ -152,7 +164,8 @@ SSDataBlock* get2ColsDummyBlock(SOperatorInfo* pOperator) {
SColumnInfoData* pColInfo = static_cast<SColumnInfoData*>(TARRAY_GET_ELEM(pBlock->pDataBlock, 0));
ts = (++pInfo->tsStart);
colDataSetVal(pColInfo, i, reinterpret_cast<const char*>(&ts), false);
int32_t code = colDataSetVal(pColInfo, i, reinterpret_cast<const char*>(&ts), false);
ASSERT(code == 0);
SColumnInfoData* pColInfo1 = static_cast<SColumnInfoData*>(TARRAY_GET_ELEM(pBlock->pDataBlock, 1));
if (pInfo->type == data_desc) {
@ -163,7 +176,8 @@ SSDataBlock* get2ColsDummyBlock(SOperatorInfo* pOperator) {
v = taosRand();
}
colDataSetVal(pColInfo1, i, reinterpret_cast<const char*>(&v), false);
code = colDataSetVal(pColInfo1, i, reinterpret_cast<const char*>(&v), false);
ASSERT(code == 0);
// sprintf(buf, "this is %d row", i);
// STR_TO_VARSTR(b1, buf);
@ -177,7 +191,7 @@ SSDataBlock* get2ColsDummyBlock(SOperatorInfo* pOperator) {
pInfo->current += 1;
pBlock->info.dataLoad = 1;
blockDataUpdateTsWindow(pBlock, 0);
int32_t code = blockDataUpdateTsWindow(pBlock, 0);
return pBlock;
}

5
source/libs/executor/test/joinTests.cpp
View File

@ -928,8 +928,9 @@ SExecTaskInfo* createDummyTaskInfo(char* taskId) {
}
SSDataBlock* createDummyBlock(int32_t blkId) {
SSDataBlock* p = createDataBlock();
assert(p);
SSDataBlock* p = NULL;
int32_t code = createDataBlock(&p);
assert(code == 0);
p->info.id.blockId = blkId;
p->info.type = STREAM_INVALID;

15
source/libs/executor/test/lhashTests.cpp
View File

@ -42,9 +42,6 @@ TEST(testCase, linear_hash_Tests) {
int64_t et = taosGetTimestampUs();
for (int32_t i = 0; i < 1000000; ++i) {
if (i == 950000) {
printf("kf\n");
}
char* v = tHashGet(pHashObj, &i, sizeof(i));
if (v != NULL) {
// printf("find value: %d, key:%d\n", *(int32_t*) v, i);
@ -54,12 +51,16 @@ TEST(testCase, linear_hash_Tests) {
}
// tHashPrint(pHashObj, LINEAR_HASH_STATIS);
tHashCleanup(pHashObj);
int32_t code = tHashCleanup(pHashObj);
ASSERT(code == 0);
int64_t et1 = taosGetTimestampUs();
SHashObj* pHashObj1 = taosHashInit(1000, fn, false, HASH_NO_LOCK);
ASSERT(pHashObj1 != NULL);
for (int32_t i = 0; i < 1000000; ++i) {
taosHashPut(pHashObj1, &i, sizeof(i), &i, sizeof(i));
int32_t code = taosHashPut(pHashObj1, &i, sizeof(i), &i, sizeof(i));
ASSERT(code == 0);
}
for (int32_t i = 0; i < 1000000; ++i) {
@ -68,6 +69,6 @@ TEST(testCase, linear_hash_Tests) {
taosHashCleanup(pHashObj1);
int64_t et2 = taosGetTimestampUs();
printf("linear hash time:%.2f ms, buildHash:%.2f ms, hash:%.2f\n", (et1 - st) / 1000.0, (et - st) / 1000.0,
(et2 - et1) / 1000.0);
(void)printf("linear hash time:%.2f ms, buildHash:%.2f ms, hash:%.2f\n", (et1 - st) / 1000.0, (et - st) / 1000.0,
(et2 - et1) / 1000.0);
}

48
source/libs/executor/test/sortTests.cpp
View File

@ -60,7 +60,12 @@ SSDataBlock* getSingleColDummyBlock(void* param) {
return NULL;
}
SSDataBlock* pBlock = createDataBlock();
SSDataBlock* pBlock = NULL;
int32_t code = createDataBlock(&pBlock);
if (code) {
return NULL;
}
SColumnInfoData colInfo = {0};
colInfo.info.type = pInfo->type;
@ -73,8 +78,11 @@ SSDataBlock* getSingleColDummyBlock(void* param) {
}
colInfo.info.colId = 1;
blockDataAppendColInfo(pBlock, &colInfo);
blockDataEnsureCapacity(pBlock, pInfo->pageRows);
int32_t code = blockDataAppendColInfo(pBlock, &colInfo);
ASSERT(code == 0);
code = blockDataEnsureCapacity(pBlock, pInfo->pageRows);
ASSERT(code == 0);
for (int32_t i = 0; i < pInfo->pageRows; ++i) {
SColumnInfoData* pColInfo = static_cast<SColumnInfoData*>(TARRAY_GET_ELEM(pBlock->pDataBlock, 0));
@ -87,25 +95,31 @@ SSDataBlock* getSingleColDummyBlock(void* param) {
int32_t len = 0;
bool ret = taosMbsToUcs4(strOri, size, (TdUcs4*)varDataVal(str), size * TSDB_NCHAR_SIZE, &len);
if (!ret) {
printf("error\n");
(void) printf("error\n");
return NULL;
}
varDataSetLen(str, len);
colDataSetVal(pColInfo, i, reinterpret_cast<const char*>(str), false);
int32_t code = colDataSetVal(pColInfo, i, reinterpret_cast<const char*>(str), false);
ASSERT(code == 0);
pBlock->info.hasVarCol = true;
printf("nchar: %s\n", strOri);
(void) printf("nchar: %s\n", strOri);
} else if (pInfo->type == TSDB_DATA_TYPE_BINARY || pInfo->type == TSDB_DATA_TYPE_GEOMETRY) {
int32_t size = taosRand() % VARCOUNT;
char str[64] = {0};
taosRandStr(varDataVal(str), size);
varDataSetLen(str, size);
colDataSetVal(pColInfo, i, reinterpret_cast<const char*>(str), false);
code = colDataSetVal(pColInfo, i, reinterpret_cast<const char*>(str), false);
ASSERT(code == 0);
pBlock->info.hasVarCol = true;
printf("binary: %s\n", varDataVal(str));
(void) printf("binary: %s\n", varDataVal(str));
} else if (pInfo->type == TSDB_DATA_TYPE_DOUBLE || pInfo->type == TSDB_DATA_TYPE_FLOAT) {
double v = rand_f2();
colDataSetVal(pColInfo, i, reinterpret_cast<const char*>(&v), false);
printf("float: %f\n", v);
code = colDataSetVal(pColInfo, i, reinterpret_cast<const char*>(&v), false);
ASSERT(code == 0);
(void) printf("float: %f\n", v);
} else {
int64_t v = ++pInfo->startVal;
char* result = static_cast<char*>(taosMemoryCalloc(tDataTypes[pInfo->type].bytes, 1));
@ -115,8 +129,10 @@ SSDataBlock* getSingleColDummyBlock(void* param) {
memcpy(result, (char*)(&v) + sizeof(int64_t) - tDataTypes[pInfo->type].bytes, tDataTypes[pInfo->type].bytes);
}
colDataSetVal(pColInfo, i, result, false);
printf("int: %" PRId64 "\n", v);
code = colDataSetVal(pColInfo, i, result, false);
ASSERT(code == 0);
(void) printf("int: %" PRId64 "\n", v);
taosMemoryFree(result);
}
}
@ -348,10 +364,14 @@ TEST(testCase, ordered_merge_sort_Test) {
SArray* orderInfo = taosArrayInit(1, sizeof(SBlockOrderInfo));
taosArrayPush(orderInfo, &oi);
SSDataBlock* pBlock = createDataBlock();
SSDataBlock* pBlock = NULL;
int32_t code = createDataBlock(&pBlock);
ASSERT(code == 0);
for (int32_t i = 0; i < 1; ++i) {
SColumnInfoData colInfo = createColumnInfoData(TSDB_DATA_TYPE_INT, sizeof(int32_t), 1);
blockDataAppendColInfo(pBlock, &colInfo);
code = blockDataAppendColInfo(pBlock, &colInfo);
ASSERT(code == 0);
}
SSortHandle* phandle = tsortCreateSortHandle(orderInfo, SORT_MULTISOURCE_MERGE, 1024, 5, pBlock,"test_abc");

19
source/libs/executor/test/tSimpleHashTests.cpp
View File

@ -43,7 +43,9 @@ TEST(testCase, tSimpleHashTest_intKey) {
int64_t originKeySum = 0;
for (int64_t i = 1; i <= 100; ++i) {
originKeySum += i;
tSimpleHashPut(pHashObj, (const void *)&i, keyLen, (const void *)&i, dataLen);
code = tSimpleHashPut(pHashObj, (const void *)&i, keyLen, (const void *)&i, dataLen);
ASSERT(code == 0);
ASSERT_EQ(i, tSimpleHashGetSize(pHashObj));
}
@ -68,7 +70,9 @@ TEST(testCase, tSimpleHashTest_intKey) {
ASSERT_EQ(keySum, originKeySum);
for (int64_t i = 1; i <= 100; ++i) {
tSimpleHashRemove(pHashObj, (const void *)&i, keyLen);
code = tSimpleHashRemove(pHashObj, (const void *)&i, keyLen);
ASSERT(code == 0);
ASSERT_EQ(100 - i, tSimpleHashGetSize(pHashObj));
}
@ -95,7 +99,9 @@ TEST(testCase, tSimpleHashTest_binaryKey) {
for (int64_t i = 1; i <= 100; ++i) {
combineKey.suid = i;
combineKey.uid = i + 1;
tSimpleHashPut(pHashObj, (const void *)&combineKey, keyLen, (const void *)&i, dataLen);
code = tSimpleHashPut(pHashObj, (const void *)&combineKey, keyLen, (const void *)&i, dataLen);
ASSERT(code == 0);
originDataSum += i;
ASSERT_EQ(i, tSimpleHashGetSize(pHashObj));
}
@ -120,7 +126,8 @@ TEST(testCase, tSimpleHashTest_binaryKey) {
ASSERT_EQ(originDataSum, dataSum);
tSimpleHashRemove(pHashObj, (const void *)&combineKey, keyLen);
code = tSimpleHashRemove(pHashObj, (const void *)&combineKey, keyLen);
ASSERT(code == 0);
while ((data = tSimpleHashIterate(pHashObj, data, &iter))) {
void *key = tSimpleHashGetKey(data, &kLen);
@ -130,7 +137,9 @@ TEST(testCase, tSimpleHashTest_binaryKey) {
for (int64_t i = 1; i <= 99; ++i) {
combineKey.suid = i;
combineKey.uid = i + 1;
tSimpleHashRemove(pHashObj, (const void *)&combineKey, keyLen);
code = tSimpleHashRemove(pHashObj, (const void *)&combineKey, keyLen);
ASSERT(code == 0);
ASSERT_EQ(99 - i, tSimpleHashGetSize(pHashObj));
}

2
source/libs/function/inc/tudfInt.h
View File

@ -109,7 +109,7 @@ void freeUdfDataDataBlock(SUdfDataBlock *block);
int32_t convertDataBlockToUdfDataBlock(SSDataBlock *block, SUdfDataBlock *udfBlock);
int32_t convertUdfColumnToDataBlock(SUdfColumn *udfCol, SSDataBlock *block);
int32_t getUdfdPipeName(char *pipeName, int32_t size);
void getUdfdPipeName(char *pipeName, int32_t size);
#ifdef __cplusplus
}
#endif

17
source/libs/function/src/builtinsimpl.c
View File

@ -469,8 +469,12 @@ int32_t funcInputGetNextRowDescPk(SFuncInputRowIter* pIter, SFuncInputRow* pRow,
}
if (pIter->hasPrev) {
if (pIter->prevBlockTsEnd == pIter->tsList[pIter->inputEndIndex]) {
(void)blockDataDestroy(pIter->pPrevRowBlock);
pIter->pPrevRowBlock = blockDataExtractBlock(pIter->pSrcBlock, pIter->inputEndIndex, 1);
blockDataDestroy(pIter->pPrevRowBlock);
int32_t code = blockDataExtractBlock(pIter->pSrcBlock, pIter->inputEndIndex, 1, &pIter->pPrevRowBlock);
if (code) {
return code;
}
pIter->prevIsDataNull = colDataIsNull_f(pIter->pDataCol->nullbitmap, pIter->inputEndIndex);
pIter->pPrevData = taosMemoryMalloc(pIter->pDataCol->info.bytes);
@ -490,11 +494,10 @@ int32_t funcInputGetNextRowDescPk(SFuncInputRowIter* pIter, SFuncInputRow* pRow,
char* pkData = colDataGetData(pIter->pPkCol, pIter->inputEndIndex);
(void)memcpy(pIter->pPrevPk, pkData, pIter->pPkCol->info.bytes);
pIter->pPrevRowBlock = blockDataExtractBlock(pIter->pSrcBlock, pIter->inputEndIndex, 1);
code = blockDataExtractBlock(pIter->pSrcBlock, pIter->inputEndIndex, 1, &pIter->pPrevRowBlock);
pIter->hasPrev = true;
*res = false;
return TSDB_CODE_SUCCESS;
return code;
} else {
int32_t idx = pIter->rowIndex;
while (pIter->tsList[idx] == pIter->prevBlockTsEnd) {
@ -553,9 +556,9 @@ int32_t funcInputGetNextRowDescPk(SFuncInputRowIter* pIter, SFuncInputRow* pRow,
}
(void)memcpy(pIter->pPrevPk, colDataGetData(pIter->pPkCol, pIter->inputEndIndex), pIter->pPkCol->info.bytes);
pIter->pPrevRowBlock = blockDataExtractBlock(pIter->pSrcBlock, pIter->inputEndIndex, 1);
int32_t code = blockDataExtractBlock(pIter->pSrcBlock, pIter->inputEndIndex, 1, &pIter->pPrevRowBlock);
*res = false;
return TSDB_CODE_SUCCESS;
return code;
}
}
}

521
source/libs/function/src/tudf.c
View File

@ -51,11 +51,10 @@ typedef struct SUdfdData {
SUdfdData udfdGlobal = {0};
int32_t udfStartUdfd(int32_t startDnodeId);
int32_t udfStopUdfd();
void udfStopUdfd();
static int32_t udfSpawnUdfd(SUdfdData *pData);
void udfUdfdExit(uv_process_t *process, int64_t exitStatus, int termSignal);
static int32_t udfSpawnUdfd(SUdfdData *pData);
static void udfUdfdCloseWalkCb(uv_handle_t *handle, void *arg);
static void udfUdfdStopAsyncCb(uv_async_t *async);
static void udfWatchUdfd(void *args);
@ -67,7 +66,10 @@ void udfUdfdExit(uv_process_t *process, int64_t exitStatus, int termSignal) {
fnInfo("udfd process exit due to SIGINT or dnode-mgmt called stop");
} else {
fnInfo("udfd process restart");
udfSpawnUdfd(pData);
int32_t code = udfSpawnUdfd(pData);
if(code != 0) {
fnError("udfd process restart failed with code:%d", code);
}
}
}
@ -80,26 +82,26 @@ static int32_t udfSpawnUdfd(SUdfdData *pData) {
path[0] = '.';
#ifdef WINDOWS
GetModuleFileName(NULL, path, PATH_MAX);
taosDirName(path);
TAOS_DIRNAME(path);
#elif defined(_TD_DARWIN_64)
uint32_t pathSize = sizeof(path);
_NSGetExecutablePath(path, &pathSize);
taosDirName(path);
TAOS_DIRNAME(path);
#endif
} else {
strncpy(path, tsProcPath, PATH_MAX);
taosDirName(path);
TAOS_STRNCPY(path, tsProcPath, PATH_MAX);
TAOS_DIRNAME(path);
}
#ifdef WINDOWS
if (strlen(path) == 0) {
strcat(path, "C:\\TDengine");
TAOS_STRCAT(path, "C:\\TDengine");
}
strcat(path, "\\udfd.exe");
TAOS_STRCAT(path, "\\udfd.exe");
#else
if (strlen(path) == 0) {
strcat(path, "/usr/bin");
TAOS_STRCAT(path, "/usr/bin");
}
strcat(path, "/udfd");
TAOS_STRCAT(path, "/udfd");
#endif
char *argsUdfd[] = {path, "-c", configDir, NULL};
options.args = argsUdfd;
@ -107,7 +109,7 @@ static int32_t udfSpawnUdfd(SUdfdData *pData) {
options.exit_cb = udfUdfdExit;
uv_pipe_init(&pData->loop, &pData->ctrlPipe, 1);
TAOS_UV_LIB_ERROR_RET(uv_pipe_init(&pData->loop, &pData->ctrlPipe, 1));
uv_stdio_container_t child_stdio[3];
child_stdio[0].flags = UV_CREATE_PIPE | UV_READABLE_PIPE;
@ -156,7 +158,7 @@ static int32_t udfSpawnUdfd(SUdfdData *pData) {
taosFqdnEnvItem = taosMemoryMalloc(strlen("TAOS_FQDN=") + strlen(taosFqdn) + 1);
if (taosFqdnEnvItem != NULL) {
strcpy(taosFqdnEnvItem, "TAOS_FQDN=");
strcat(taosFqdnEnvItem, taosFqdn);
TAOS_STRCAT(taosFqdnEnvItem, taosFqdn);
fnInfo("[UDFD]Succsess to set TAOS_FQDN:%s", taosFqdn);
} else {
fnError("[UDFD]Failed to allocate memory for TAOS_FQDN");
@ -212,22 +214,37 @@ static void udfUdfdStopAsyncCb(uv_async_t *async) {
static void udfWatchUdfd(void *args) {
SUdfdData *pData = args;
uv_loop_init(&pData->loop);
uv_async_init(&pData->loop, &pData->stopAsync, udfUdfdStopAsyncCb);
TAOS_UV_CHECK_ERRNO(uv_loop_init(&pData->loop));
TAOS_UV_CHECK_ERRNO(uv_async_init(&pData->loop, &pData->stopAsync, udfUdfdStopAsyncCb));
pData->stopAsync.data = pData;
int32_t err = udfSpawnUdfd(pData);
atomic_store_32(&pData->spawnErr, err);
uv_barrier_wait(&pData->barrier);
uv_run(&pData->loop, UV_RUN_DEFAULT);
uv_loop_close(&pData->loop);
TAOS_UV_CHECK_ERRNO(udfSpawnUdfd(pData));
atomic_store_32(&pData->spawnErr, 0);
(void)uv_barrier_wait(&pData->barrier);
int num = uv_run(&pData->loop, UV_RUN_DEFAULT);
fnInfo("udfd loop exit with %d active handles, line:%d", num, __LINE__);
uv_walk(&pData->loop, udfUdfdCloseWalkCb, NULL);
uv_run(&pData->loop, UV_RUN_DEFAULT);
uv_loop_close(&pData->loop);
num = uv_run(&pData->loop, UV_RUN_DEFAULT);
fnInfo("udfd loop exit with %d active handles, line:%d", num, __LINE__);
if(uv_loop_close(&pData->loop) != 0) {
fnError("udfd loop close failed, lino:%d", __LINE__);
}
_exit:
if (terrno != 0) {
(void)uv_barrier_wait(&pData->barrier);
atomic_store_32(&pData->spawnErr, terrno);
if(uv_loop_close(&pData->loop) != 0) {
fnError("udfd loop close failed, lino:%d", __LINE__);
}
fnError("udfd thread exit with code:%d lino:%d", terrno, terrln);
terrno = TSDB_CODE_UDF_UV_EXEC_FAILURE;
}
return;
}
int32_t udfStartUdfd(int32_t startDnodeId) {
int32_t code = 0, lino = 0;
if (!tsStartUdfd) {
fnInfo("start udfd is disabled.") return 0;
}
@ -239,43 +256,58 @@ int32_t udfStartUdfd(int32_t startDnodeId) {
pData->startCalled = true;
char dnodeId[8] = {0};
snprintf(dnodeId, sizeof(dnodeId), "%d", startDnodeId);
uv_os_setenv("DNODE_ID", dnodeId);
TAOS_CHECK_GOTO(uv_os_setenv("DNODE_ID", dnodeId), &lino, _exit);
pData->dnodeId = startDnodeId;
uv_barrier_init(&pData->barrier, 2);
uv_thread_create(&pData->thread, udfWatchUdfd, pData);
uv_barrier_wait(&pData->barrier);
TAOS_CHECK_GOTO(uv_barrier_init(&pData->barrier, 2), &lino, _exit);
TAOS_CHECK_GOTO(uv_thread_create(&pData->thread, udfWatchUdfd, pData), &lino, _exit);
(void)uv_barrier_wait(&pData->barrier);
int32_t err = atomic_load_32(&pData->spawnErr);
if (err != 0) {
uv_barrier_destroy(&pData->barrier);
uv_async_send(&pData->stopAsync);
uv_thread_join(&pData->thread);
if(uv_async_send(&pData->stopAsync) != 0) {
fnError("start udfd: failed to send stop async");
}
if(uv_thread_join(&pData->thread)!= 0) {
fnError("start udfd: failed to join udfd thread");
}
pData->needCleanUp = false;
fnInfo("udfd is cleaned up after spawn err");
TAOS_CHECK_GOTO(err, &lino, _exit);
} else {
pData->needCleanUp = true;
}
return err;
_exit:
if (code != 0) {
fnError("udfd start failed with code:%d, lino:%d", code, lino);
}
return code;
}
int32_t udfStopUdfd() {
void udfStopUdfd() {
SUdfdData *pData = &udfdGlobal;
fnInfo("udfd start to stop, need cleanup:%d, spawn err:%d", pData->needCleanUp, pData->spawnErr);
if (!pData->needCleanUp || atomic_load_32(&pData->stopCalled)) {
return 0;
return;
}
atomic_store_32(&pData->stopCalled, 1);
pData->needCleanUp = false;
uv_barrier_destroy(&pData->barrier);
uv_async_send(&pData->stopAsync);
uv_thread_join(&pData->thread);
if(uv_async_send(&pData->stopAsync) != 0) {
fnError("stop udfd: failed to send stop async");
}
if(uv_thread_join(&pData->thread) != 0) {
fnError("stop udfd: failed to join udfd thread");
}
#ifdef WINDOWS
if (pData->jobHandle != NULL) CloseHandle(pData->jobHandle);
#endif
fnInfo("udfd is cleaned up");
return 0;
return;
}
/*
int32_t udfGetUdfdPid(int32_t* pUdfdPid) {
SUdfdData *pData = &udfdGlobal;
if (pData->spawnErr) {
@ -287,6 +319,7 @@ int32_t udfGetUdfdPid(int32_t* pUdfdPid) {
}
return TSDB_CODE_SUCCESS;
}
*/
//==============================================================================================
/* Copyright (c) 2013, Ben Noordhuis <info@bnoordhuis.nl>
@ -439,8 +472,6 @@ typedef struct SClientUdfTask {
SUdfcUvSession *session;
int32_t errCode;
union {
struct {
SUdfSetupRequest req;
@ -479,7 +510,7 @@ enum {
UDFC_STATE_STOPPING, // stopping after udfcClose
};
int32_t getUdfdPipeName(char *pipeName, int32_t size);
void getUdfdPipeName(char *pipeName, int32_t size);
int32_t encodeUdfSetupRequest(void **buf, const SUdfSetupRequest *setup);
void *decodeUdfSetupRequest(const void *buf, SUdfSetupRequest *request);
int32_t encodeUdfInterBuf(void **buf, const SUdfInterBuf *state);
@ -507,7 +538,7 @@ int32_t convertUdfColumnToDataBlock(SUdfColumn *udfCol, SSDataBlock *block);
int32_t convertScalarParamToDataBlock(SScalarParam *input, int32_t numOfCols, SSDataBlock *output);
int32_t convertDataBlockToScalarParm(SSDataBlock *input, SScalarParam *output);
int32_t getUdfdPipeName(char *pipeName, int32_t size) {
void getUdfdPipeName(char *pipeName, int32_t size) {
char dnodeId[8] = {0};
size_t dnodeIdSize = sizeof(dnodeId);
int32_t err = uv_os_getenv(UDF_DNODE_ID_ENV_NAME, dnodeId, &dnodeIdSize);
@ -522,7 +553,6 @@ int32_t getUdfdPipeName(char *pipeName, int32_t size) {
snprintf(pipeName, size, "%s/%s%s", tsDataDir, UDF_LISTEN_PIPE_NAME_PREFIX, dnodeId);
#endif
fnInfo("get dnodeId:%s from env, pipe path:%s", dnodeId, pipeName);
return 0;
}
int32_t encodeUdfSetupRequest(void **buf, const SUdfSetupRequest *setup) {
@ -712,16 +742,14 @@ void *decodeUdfTeardownResponse(const void *buf, SUdfTeardownResponse *teardownR
int32_t encodeUdfResponse(void **buf, const SUdfResponse *rsp) {
int32_t len = 0;
if (buf == NULL) {
len += sizeof(rsp->msgLen);
} else {
len += sizeof(rsp->msgLen);
if (buf != NULL) {
*(int32_t *)(*buf) = rsp->msgLen;
*buf = POINTER_SHIFT(*buf, sizeof(rsp->msgLen));
}
if (buf == NULL) {
len += sizeof(rsp->seqNum);
} else {
len += sizeof(rsp->seqNum);
if (buf != NULL) {
*(int64_t *)(*buf) = rsp->seqNum;
*buf = POINTER_SHIFT(*buf, sizeof(rsp->seqNum));
}
@ -810,6 +838,9 @@ int32_t convertDataBlockToUdfDataBlock(SSDataBlock *block, SUdfDataBlock *udfBlo
udfBlock->udfCols = taosMemoryCalloc(taosArrayGetSize(block->pDataBlock), sizeof(SUdfColumn *));
for (int32_t i = 0; i < udfBlock->numOfCols; ++i) {
udfBlock->udfCols[i] = taosMemoryCalloc(1, sizeof(SUdfColumn));
if(udfBlock->udfCols[i] == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
SColumnInfoData *col = (SColumnInfoData *)taosArrayGet(block->pDataBlock, i);
SUdfColumn *udfCol = udfBlock->udfCols[i];
udfCol->colMeta.type = col->info.type;
@ -821,9 +852,15 @@ int32_t convertDataBlockToUdfDataBlock(SSDataBlock *block, SUdfDataBlock *udfBlo
if (IS_VAR_DATA_TYPE(udfCol->colMeta.type)) {
udfCol->colData.varLenCol.varOffsetsLen = sizeof(int32_t) * udfBlock->numOfRows;
udfCol->colData.varLenCol.varOffsets = taosMemoryMalloc(udfCol->colData.varLenCol.varOffsetsLen);
if(udfCol->colData.varLenCol.varOffsets == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
memcpy(udfCol->colData.varLenCol.varOffsets, col->varmeta.offset, udfCol->colData.varLenCol.varOffsetsLen);
udfCol->colData.varLenCol.payloadLen = colDataGetLength(col, udfBlock->numOfRows);
udfCol->colData.varLenCol.payload = taosMemoryMalloc(udfCol->colData.varLenCol.payloadLen);
if(udfCol->colData.varLenCol.payload == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
if (col->reassigned) {
for (int32_t row = 0; row < udfCol->colData.numOfRows; ++row) {
char* pColData = col->pData + col->varmeta.offset[row];
@ -843,6 +880,9 @@ int32_t convertDataBlockToUdfDataBlock(SSDataBlock *block, SUdfDataBlock *udfBlo
udfCol->colData.fixLenCol.nullBitmapLen = BitmapLen(udfCol->colData.numOfRows);
int32_t bitmapLen = udfCol->colData.fixLenCol.nullBitmapLen;
udfCol->colData.fixLenCol.nullBitmap = taosMemoryMalloc(udfCol->colData.fixLenCol.nullBitmapLen);
if(udfCol->colData.fixLenCol.nullBitmap == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
char *bitmap = udfCol->colData.fixLenCol.nullBitmap;
memcpy(bitmap, col->nullbitmap, bitmapLen);
udfCol->colData.fixLenCol.dataLen = colDataGetLength(col, udfBlock->numOfRows);
@ -852,59 +892,44 @@ int32_t convertDataBlockToUdfDataBlock(SSDataBlock *block, SUdfDataBlock *udfBlo
memcpy(data, col->pData, dataLen);
}
}
return 0;
return TSDB_CODE_SUCCESS;
}
int32_t convertUdfColumnToDataBlock(SUdfColumn *udfCol, SSDataBlock *block) {
int32_t code = 0, lino = 0;
SUdfColumnMeta* meta = &udfCol->colMeta;
SColumnInfoData colInfoData = createColumnInfoData(meta->type, meta->bytes, 1);
blockDataAppendColInfo(block, &colInfoData);
blockDataEnsureCapacity(block, udfCol->colData.numOfRows);
code = blockDataAppendColInfo(block, &colInfoData);
TAOS_CHECK_GOTO(code, &lino, _exit);
code = blockDataEnsureCapacity(block, udfCol->colData.numOfRows);
TAOS_CHECK_GOTO(code, &lino, _exit);
SColumnInfoData *col = NULL;
code = bdGetColumnInfoData(block, 0, &col);
TAOS_CHECK_GOTO(code, &lino, _exit);
SColumnInfoData *col = bdGetColumnInfoData(block, 0);
for (int i = 0; i < udfCol->colData.numOfRows; ++i) {
if (udfColDataIsNull(udfCol, i)) {
colDataSetNULL(col, i);
} else {
char* data = udfColDataGetData(udfCol, i);
colDataSetVal(col, i, data, false);
code = colDataSetVal(col, i, data, false);
TAOS_CHECK_GOTO(code, &lino, _exit);
}
}
block->info.rows = udfCol->colData.numOfRows;
return 0;
}
int32_t convertUdfColumnToDataBlock2(SUdfColumn *udfCol, SSDataBlock *block) {
block->info.rows = udfCol->colData.numOfRows;
block->info.hasVarCol = IS_VAR_DATA_TYPE(udfCol->colMeta.type);
block->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData));
taosArrayPush(block->pDataBlock, &(SColumnInfoData){0});
SColumnInfoData *col = taosArrayGet(block->pDataBlock, 0);
SUdfColumnMeta *meta = &udfCol->colMeta;
col->info.precision = meta->precision;
col->info.bytes = meta->bytes;
col->info.scale = meta->scale;
col->info.type = meta->type;
col->hasNull = udfCol->hasNull;
SUdfColumnData *data = &udfCol->colData;
if (!IS_VAR_DATA_TYPE(meta->type)) {
col->nullbitmap = taosMemoryMalloc(data->fixLenCol.nullBitmapLen);
memcpy(col->nullbitmap, data->fixLenCol.nullBitmap, data->fixLenCol.nullBitmapLen);
col->pData = taosMemoryMalloc(data->fixLenCol.dataLen);
memcpy(col->pData, data->fixLenCol.data, data->fixLenCol.dataLen);
} else {
col->varmeta.offset = taosMemoryMalloc(data->varLenCol.varOffsetsLen);
memcpy(col->varmeta.offset, data->varLenCol.varOffsets, data->varLenCol.varOffsetsLen);
col->pData = taosMemoryMalloc(data->varLenCol.payloadLen);
memcpy(col->pData, data->varLenCol.payload, data->varLenCol.payloadLen);
_exit:
if (code != 0) {
fnError("failed to convert udf column to data block, code:%d, line:%d", code, lino);
}
return 0;
return TSDB_CODE_SUCCESS;
}
int32_t convertScalarParamToDataBlock(SScalarParam *input, int32_t numOfCols, SSDataBlock *output) {
int32_t code = 0, lino = 0;
int32_t numOfRows = 0;
for (int32_t i = 0; i < numOfCols; ++i) {
numOfRows = (input[i].numOfRows > numOfRows) ? input[i].numOfRows : numOfRows;
@ -916,16 +941,16 @@ int32_t convertScalarParamToDataBlock(SScalarParam *input, int32_t numOfCols, SS
SColumnInfoData d = {0};
d.info = pInfo->info;
blockDataAppendColInfo(output, &d);
TAOS_CHECK_GOTO(blockDataAppendColInfo(output, &d), &lino, _exit);
}
blockDataEnsureCapacity(output, numOfRows);
TAOS_CHECK_GOTO(blockDataEnsureCapacity(output, numOfRows), &lino, _exit);
for(int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pDest = taosArrayGet(output->pDataBlock, i);
SColumnInfoData* pColInfoData = input[i].columnData;
colDataAssign(pDest, pColInfoData, input[i].numOfRows, &output->info);
TAOS_CHECK_GOTO(colDataAssign(pDest, pColInfoData, input[i].numOfRows, &output->info), &lino, _exit);
if (input[i].numOfRows < numOfRows) {
int32_t startRow = input[i].numOfRows;
@ -936,26 +961,31 @@ int32_t convertScalarParamToDataBlock(SScalarParam *input, int32_t numOfCols, SS
} else {
char* src = colDataGetData(pColInfoData, (input + i)->numOfRows - 1);
for (int j = 0; j < expandRows; ++j) {
colDataSetVal(pDest, startRow+j, src, false);
TAOS_CHECK_GOTO(colDataSetVal(pDest, startRow+j, src, false), &lino, _exit);
}
//colDataSetNItems(pColInfoData, startRow, data, expandRows);
}
}
}
output->info.rows = numOfRows;
return 0;
_exit:
if (code != 0) {
fnError("failed to convert scalar param to data block, code:%d, line:%d", code, lino);
}
return code;
}
int32_t convertDataBlockToScalarParm(SSDataBlock *input, SScalarParam *output) {
if (taosArrayGetSize(input->pDataBlock) != 1) {
fnError("scalar function only support one column");
return -1;
return 0;
}
output->numOfRows = input->info.rows;
output->columnData = taosMemoryMalloc(sizeof(SColumnInfoData));
if(output->columnData == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
memcpy(output->columnData, taosArrayGet(input->pDataBlock, 0), sizeof(SColumnInfoData));
output->colAlloced = true;
@ -1024,7 +1054,7 @@ int compareUdfcFuncSub(const void *elem1, const void *elem2) {
}
int32_t acquireUdfFuncHandle(char *udfName, UdfcFuncHandle *pHandle) {
int32_t code = 0;
int32_t code = 0, line = 0;
uv_mutex_lock(&gUdfcProxy.udfStubsMutex);
SUdfcFuncStub key = {0};
strncpy(key.udfName, udfName, TSDB_FUNC_NAME_LEN);
@ -1048,7 +1078,10 @@ int32_t acquireUdfFuncHandle(char *udfName, UdfcFuncHandle *pHandle) {
} else {
fnInfo("udf handle expired for %s, will setup udf. move it to expired list", udfName);
taosArrayRemove(gUdfcProxy.udfStubs, stubIndex);
taosArrayPush(gUdfcProxy.expiredUdfStubs, foundStub);
if(taosArrayPush(gUdfcProxy.expiredUdfStubs, foundStub) == NULL) {
fnError("acquireUdfFuncHandle: failed to push udf stub to array");
goto _exit;
}
taosArraySort(gUdfcProxy.expiredUdfStubs, compareUdfcFuncSub);
}
}
@ -1060,12 +1093,16 @@ int32_t acquireUdfFuncHandle(char *udfName, UdfcFuncHandle *pHandle) {
stub.handle = *pHandle;
++stub.refCount;
stub.createTime = taosGetTimestampUs();
taosArrayPush(gUdfcProxy.udfStubs, &stub);
if(taosArrayPush(gUdfcProxy.udfStubs, &stub) == NULL) {
fnError("acquireUdfFuncHandle: failed to push udf stub to array");
goto _exit;
}
taosArraySort(gUdfcProxy.udfStubs, compareUdfcFuncSub);
} else {
*pHandle = NULL;
}
_exit:
uv_mutex_unlock(&gUdfcProxy.udfStubsMutex);
return code;
}
@ -1092,17 +1129,23 @@ void releaseUdfFuncHandle(char *udfName, UdfcFuncHandle handle) {
void cleanupExpiredUdfs() {
int32_t i = 0;
SArray *expiredUdfStubs = taosArrayInit(16, sizeof(SUdfcFuncStub));
if(expiredUdfStubs == NULL) {
fnError("cleanupExpiredUdfs: failed to init array");
return;
}
while (i < taosArrayGetSize(gUdfcProxy.expiredUdfStubs)) {
SUdfcFuncStub *stub = taosArrayGet(gUdfcProxy.expiredUdfStubs, i);
if (stub->refCount == 0) {
fnInfo("tear down udf. expired. udf name: %s, handle: %p, ref count: %d", stub->udfName, stub->handle, stub->refCount);
doTeardownUdf(stub->handle);
(void)doTeardownUdf(stub->handle);
} else {
fnInfo("udf still in use. expired. udf name: %s, ref count: %d, create time: %" PRId64 ", handle: %p", stub->udfName,
stub->refCount, stub->createTime, stub->handle);
UdfcFuncHandle handle = stub->handle;
if (handle != NULL && ((SUdfcUvSession *)handle)->udfUvPipe != NULL) {
taosArrayPush(expiredUdfStubs, stub);
if(taosArrayPush(expiredUdfStubs, stub) == NULL) {
fnError("cleanupExpiredUdfs: failed to push udf stub to array");
}
} else {
fnInfo("udf invalid handle for %s, expired. refCount: %d, create time: %" PRId64 ". remove it from cache",
stub->udfName, stub->refCount, stub->createTime);
@ -1121,16 +1164,18 @@ void cleanupNotExpiredUdfs() {
SUdfcFuncStub *stub = taosArrayGet(gUdfcProxy.udfStubs, i);
if (stub->refCount == 0) {
fnInfo("tear down udf. udf name: %s, handle: %p, ref count: %d", stub->udfName, stub->handle, stub->refCount);
doTeardownUdf(stub->handle);
(void)doTeardownUdf(stub->handle);
} else {
fnInfo("udf still in use. udf name: %s, ref count: %d, create time: %" PRId64 ", handle: %p", stub->udfName,
stub->refCount, stub->createTime, stub->handle);
UdfcFuncHandle handle = stub->handle;
if (handle != NULL && ((SUdfcUvSession *)handle)->udfUvPipe != NULL) {
taosArrayPush(udfStubs, stub);
if (taosArrayPush(udfStubs, stub) == NULL) {
fnError("cleanupNotExpiredUdfs: failed to push udf stub to array");
}
} else {
fnInfo("udf invalid handle for %s, refCount: %d, create time: %" PRId64 ". remove it from cache",
stub->udfName, stub->refCount, stub->createTime);
fnInfo("udf invalid handle for %s, refCount: %d, create time: %" PRId64 ". remove it from cache", stub->udfName,
stub->refCount, stub->createTime);
}
}
++i;
@ -1254,15 +1299,28 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) {
int32_t numOfCols = pInput->numOfInputCols;
int32_t start = pInput->startRowIndex;
int32_t numOfRows = pInput->numOfRows;
SSDataBlock *pTempBlock = NULL;
int32_t code = createDataBlock(&pTempBlock);
if (code) {
return code;
}
SSDataBlock *pTempBlock = createDataBlock();
pTempBlock->info.rows = pInput->totalRows;
pTempBlock->info.id.uid = pInput->uid;
for (int32_t i = 0; i < numOfCols; ++i) {
blockDataAppendColInfo(pTempBlock, pInput->pData[i]);
if ((udfCode = blockDataAppendColInfo(pTempBlock, pInput->pData[i])) != 0) {
fnError("udfAggProcess error. step blockDataAppendColInfo. udf code: %d", udfCode);
blockDataDestroy(pTempBlock);
return udfCode;
}
}
SSDataBlock *inputBlock = blockDataExtractBlock(pTempBlock, start, numOfRows);
SSDataBlock *inputBlock = NULL;
code = blockDataExtractBlock(pTempBlock, start, numOfRows, &inputBlock);
if (code) {
return code;
}
SUdfInterBuf state = {.buf = udfRes->interResBuf, .bufLen = udfRes->interResBufLen, .numOfResult = udfRes->interResNum};
SUdfInterBuf newState = {0};
@ -1358,12 +1416,16 @@ void onUdfcPipeClose(uv_handle_t *handle) {
}
int32_t udfcGetUdfTaskResultFromUvTask(SClientUdfTask *task, SClientUvTaskNode *uvTask) {
int32_t code = 0;
fnDebug("udfc get uv task result. task: %p, uvTask: %p", task, uvTask);
if (uvTask->type == UV_TASK_REQ_RSP) {
if (uvTask->rspBuf.base != NULL) {
SUdfResponse rsp = {0};
void *buf = decodeUdfResponse(uvTask->rspBuf.base, &rsp);
task->errCode = rsp.code;
code = rsp.code;
if(code != 0) {
fnError("udfc get udf task result failure. code: %d", code);
}
switch (task->type) {
case UDF_TASK_SETUP: {
@ -1386,14 +1448,23 @@ int32_t udfcGetUdfTaskResultFromUvTask(SClientUdfTask *task, SClientUvTaskNode *
// TODO: the call buffer is setup and freed by udf invocation
taosMemoryFree(uvTask->rspBuf.base);
} else {
task->errCode = uvTask->errCode;
code = uvTask->errCode;
if(code != 0) {
fnError("udfc get udf task result failure. code: %d, line:%d", code, __LINE__);
}
}
} else if (uvTask->type == UV_TASK_CONNECT) {
task->errCode = uvTask->errCode;
code = uvTask->errCode;
if(code != 0) {
fnError("udfc get udf task result failure. code: %d, line:%d", code, __LINE__);
}
} else if (uvTask->type == UV_TASK_DISCONNECT) {
task->errCode = uvTask->errCode;
code = uvTask->errCode;
if(code != 0) {
fnError("udfc get udf task result failure. code: %d, line:%d", code, __LINE__);
}
}
return 0;
return code;
}
void udfcAllocateBuffer(uv_handle_t *handle, size_t suggestedSize, uv_buf_t *buf) {
@ -1542,7 +1613,11 @@ void onUdfcPipeConnect(uv_connect_t *connect, int status) {
}
uvTask->errCode = status;
uv_read_start((uv_stream_t *)uvTask->pipe, udfcAllocateBuffer, onUdfcPipeRead);
int32_t code = uv_read_start((uv_stream_t *)uvTask->pipe, udfcAllocateBuffer, onUdfcPipeRead);
if(code != 0) {
fnError("udfc client connection %p read start failed. code: %d(%s)", uvTask->pipe, code, uv_strerror(code));
uvTask->errCode = code;
}
taosMemoryFree(connect);
QUEUE_REMOVE(&uvTask->procTaskQueue);
uv_sem_post(&uvTask->taskSem);
@ -1572,16 +1647,37 @@ int32_t udfcInitializeUvTask(SClientUdfTask *task, int8_t uvTaskType, SClientUvT
fnError("udfc create uv task, invalid task type : %d", task->type);
}
int32_t bufLen = encodeUdfRequest(NULL, &request);
if (bufLen <= 0) {
fnError("udfc create uv task, encode request failed. size: %d", bufLen);
return TSDB_CODE_UDF_UV_EXEC_FAILURE;
}
request.msgLen = bufLen;
void *bufBegin = taosMemoryMalloc(bufLen);
if(bufBegin == NULL) {
fnError("udfc create uv task, malloc buffer failed. size: %d", bufLen);
return TSDB_CODE_OUT_OF_MEMORY;
}
void *buf = bufBegin;
encodeUdfRequest(&buf, &request);
if(encodeUdfRequest(&buf, &request) <= 0)
{
fnError("udfc create uv task, encode request failed. size: %d", bufLen);
taosMemoryFree(bufBegin);
return TSDB_CODE_UDF_UV_EXEC_FAILURE;
}
uvTask->reqBuf = uv_buf_init(bufBegin, bufLen);
uvTask->seqNum = request.seqNum;
} else if (uvTaskType == UV_TASK_DISCONNECT) {
uvTask->pipe = task->session->udfUvPipe;
}
uv_sem_init(&uvTask->taskSem, 0);
if (uv_sem_init(&uvTask->taskSem, 0) != 0)
{
if (uvTaskType == UV_TASK_REQ_RSP) {
taosMemoryFree(uvTask->reqBuf.base);
}
fnError("udfc create uv task, init semaphore failed.");
return TSDB_CODE_UDF_UV_EXEC_FAILURE;
}
return 0;
}
@ -1592,7 +1688,11 @@ int32_t udfcQueueUvTask(SClientUvTaskNode *uvTask) {
uv_mutex_lock(&udfc->taskQueueMutex);
QUEUE_INSERT_TAIL(&udfc->taskQueue, &uvTask->recvTaskQueue);
uv_mutex_unlock(&udfc->taskQueueMutex);
uv_async_send(&udfc->loopTaskAync);
int32_t code = uv_async_send(&udfc->loopTaskAync);
if (code != 0) {
fnError("udfc queue uv task to event loop failed. code: %s", uv_strerror(code));
return TSDB_CODE_UDF_UV_EXEC_FAILURE;
}
uv_sem_wait(&uvTask->taskSem);
fnInfo("udfc uvTask finished. uvTask:%" PRId64 "-%d-%p", uvTask->seqNum, uvTask->type, uvTask);
@ -1608,10 +1708,23 @@ int32_t udfcStartUvTask(SClientUvTaskNode *uvTask) {
switch (uvTask->type) {
case UV_TASK_CONNECT: {
uv_pipe_t *pipe = taosMemoryMalloc(sizeof(uv_pipe_t));
uv_pipe_init(&uvTask->udfc->uvLoop, pipe, 0);
if(pipe == NULL) {
fnError("udfc event loop start connect task malloc pipe failed.");
return TSDB_CODE_OUT_OF_MEMORY;
}
if (uv_pipe_init(&uvTask->udfc->uvLoop, pipe, 0) != 0) {
fnError("udfc event loop start connect task uv_pipe_init failed.");
taosMemoryFree(pipe);
return TSDB_CODE_UDF_UV_EXEC_FAILURE;
}
uvTask->pipe = pipe;
SClientUvConn *conn = taosMemoryCalloc(1, sizeof(SClientUvConn));
if(conn == NULL) {
fnError("udfc event loop start connect task malloc conn failed.");
taosMemoryFree(pipe);
return TSDB_CODE_OUT_OF_MEMORY;
}
conn->pipe = pipe;
conn->readBuf.len = 0;
conn->readBuf.cap = 0;
@ -1622,6 +1735,12 @@ int32_t udfcStartUvTask(SClientUvTaskNode *uvTask) {
pipe->data = conn;
uv_connect_t *connReq = taosMemoryMalloc(sizeof(uv_connect_t));
if(connReq == NULL) {
fnError("udfc event loop start connect task malloc connReq failed.");
taosMemoryFree(pipe);
taosMemoryFree(conn);
return TSDB_CODE_OUT_OF_MEMORY;
}
connReq->data = uvTask;
uv_pipe_connect(connReq, pipe, uvTask->udfc->udfdPipeName, onUdfcPipeConnect);
code = 0;
@ -1633,6 +1752,10 @@ int32_t udfcStartUvTask(SClientUvTaskNode *uvTask) {
code = TSDB_CODE_UDF_PIPE_NOT_EXIST;
} else {
uv_write_t *write = taosMemoryMalloc(sizeof(uv_write_t));
if(write == NULL) {
fnError("udfc event loop start req_rsp task malloc write failed.");
return TSDB_CODE_OUT_OF_MEMORY;
}
write->data = pipe->data;
QUEUE *connTaskQueue = &((SClientUvConn *)pipe->data)->taskQueue;
QUEUE_INSERT_TAIL(connTaskQueue, &uvTask->connTaskQueue);
@ -1726,27 +1849,41 @@ void udfStopAsyncCb(uv_async_t *async) {
}
void constructUdfService(void *argsThread) {
int32_t code = 0, lino = 0;
SUdfcProxy *udfc = (SUdfcProxy *)argsThread;
uv_loop_init(&udfc->uvLoop);
code = uv_loop_init(&udfc->uvLoop);
TAOS_CHECK_GOTO(code, &lino, _exit);
uv_async_init(&udfc->uvLoop, &udfc->loopTaskAync, udfcAsyncTaskCb);
code = uv_async_init(&udfc->uvLoop, &udfc->loopTaskAync, udfcAsyncTaskCb);
TAOS_CHECK_GOTO(code, &lino, _exit);
udfc->loopTaskAync.data = udfc;
uv_async_init(&udfc->uvLoop, &udfc->loopStopAsync, udfStopAsyncCb);
code = uv_async_init(&udfc->uvLoop, &udfc->loopStopAsync, udfStopAsyncCb);
TAOS_CHECK_GOTO(code, &lino, _exit);
udfc->loopStopAsync.data = udfc;
uv_mutex_init(&udfc->taskQueueMutex);
code = uv_mutex_init(&udfc->taskQueueMutex);
TAOS_CHECK_GOTO(code, &lino, _exit);
QUEUE_INIT(&udfc->taskQueue);
QUEUE_INIT(&udfc->uvProcTaskQueue);
uv_barrier_wait(&udfc->initBarrier);
(void)uv_barrier_wait(&udfc->initBarrier);
// TODO return value of uv_run
uv_run(&udfc->uvLoop, UV_RUN_DEFAULT);
uv_loop_close(&udfc->uvLoop);
int num = uv_run(&udfc->uvLoop, UV_RUN_DEFAULT);
fnInfo("udfc uv loop exit. active handle num: %d", num);
(void)uv_loop_close(&udfc->uvLoop);
uv_walk(&udfc->uvLoop, udfUdfdCloseWalkCb, NULL);
uv_run(&udfc->uvLoop, UV_RUN_DEFAULT);
uv_loop_close(&udfc->uvLoop);
num = uv_run(&udfc->uvLoop, UV_RUN_DEFAULT);
fnInfo("udfc uv loop exit. active handle num: %d", num);
(void)uv_loop_close(&udfc->uvLoop);
_exit:
if (code != 0) {
fnError("udfc construct error. code: %d, line: %d", code, lino);
}
fnInfo("udfc construct finished");
}
int32_t udfcOpen() {
int32_t code = 0, lino = 0;
int8_t old = atomic_val_compare_exchange_8(&gUdfcProxy.initialized, 0, 1);
if (old == 1) {
return 0;
@ -1754,16 +1891,36 @@ int32_t udfcOpen() {
SUdfcProxy *proxy = &gUdfcProxy;
getUdfdPipeName(proxy->udfdPipeName, sizeof(proxy->udfdPipeName));
proxy->udfcState = UDFC_STATE_STARTNG;
uv_barrier_init(&proxy->initBarrier, 2);
uv_thread_create(&proxy->loopThread, constructUdfService, proxy);
code = uv_barrier_init(&proxy->initBarrier, 2);
TAOS_CHECK_GOTO(code, &lino, _exit);
code = uv_thread_create(&proxy->loopThread, constructUdfService, proxy);
TAOS_CHECK_GOTO(code, &lino, _exit);
atomic_store_8(&proxy->udfcState, UDFC_STATE_READY);
proxy->udfcState = UDFC_STATE_READY;
uv_barrier_wait(&proxy->initBarrier);
uv_mutex_init(&proxy->udfStubsMutex);
(void)uv_barrier_wait(&proxy->initBarrier);
TAOS_CHECK_GOTO(code, &lino, _exit);
code = uv_mutex_init(&proxy->udfStubsMutex);
TAOS_CHECK_GOTO(code, &lino, _exit);
proxy->udfStubs = taosArrayInit(8, sizeof(SUdfcFuncStub));
if(proxy->udfStubs == NULL) {
fnError("udfc init failed. udfStubs: %p", proxy->udfStubs);
return -1;
}
proxy->expiredUdfStubs = taosArrayInit(8, sizeof(SUdfcFuncStub));
uv_mutex_init(&proxy->udfcUvMutex);
fnInfo("udfc initialized") return 0;
if(proxy->expiredUdfStubs == NULL) {
taosArrayDestroy(proxy->udfStubs);
fnError("udfc init failed. expiredUdfStubs: %p", proxy->expiredUdfStubs);
return -1;
}
code = uv_mutex_init(&proxy->udfcUvMutex);
TAOS_CHECK_GOTO(code, &lino, _exit);
_exit:
if (code != 0) {
fnError("udfc open error. code: %d, line: %d", code, lino);
return TSDB_CODE_UDF_UV_EXEC_FAILURE;
}
fnInfo("udfc initialized");
return 0;
}
int32_t udfcClose() {
@ -1774,8 +1931,12 @@ int32_t udfcClose() {
SUdfcProxy *udfc = &gUdfcProxy;
udfc->udfcState = UDFC_STATE_STOPPING;
uv_async_send(&udfc->loopStopAsync);
uv_thread_join(&udfc->loopThread);
if(uv_async_send(&udfc->loopStopAsync) != 0) {
fnError("udfc close error to send stop async");
}
if(uv_thread_join(&udfc->loopThread) != 0 ) {
fnError("udfc close errir to join loop thread");
}
uv_mutex_destroy(&udfc->taskQueueMutex);
uv_barrier_destroy(&udfc->initBarrier);
taosArrayDestroy(udfc->expiredUdfStubs);
@ -1788,45 +1949,61 @@ int32_t udfcClose() {
}
int32_t udfcRunUdfUvTask(SClientUdfTask *task, int8_t uvTaskType) {
int32_t code = 0, lino = 0;
SClientUvTaskNode *uvTask = taosMemoryCalloc(1, sizeof(SClientUvTaskNode));
if(uvTask == NULL) {
fnError("udfc client task: %p failed to allocate memory for uvTask", task);
return TSDB_CODE_OUT_OF_MEMORY;
}
fnDebug("udfc client task: %p created uvTask: %p. pipe: %p", task, uvTask, task->session->udfUvPipe);
udfcInitializeUvTask(task, uvTaskType, uvTask);
udfcQueueUvTask(uvTask);
udfcGetUdfTaskResultFromUvTask(task, uvTask);
code = udfcInitializeUvTask(task, uvTaskType, uvTask);
TAOS_CHECK_GOTO(code, &lino, _exit);
code = udfcQueueUvTask(uvTask);
TAOS_CHECK_GOTO(code, &lino, _exit);
code = udfcGetUdfTaskResultFromUvTask(task, uvTask);
TAOS_CHECK_GOTO(code, &lino, _exit);
if (uvTaskType == UV_TASK_CONNECT) {
task->session->udfUvPipe = uvTask->pipe;
SClientUvConn *conn = uvTask->pipe->data;
conn->session = task->session;
}
_exit:
if (code != 0) {
fnError("udfc run udf uv task failure. task: %p, uvTask: %p, err: %d, line: %d", task, uvTask, code, lino);
}
taosMemoryFree(uvTask->reqBuf.base);
uvTask->reqBuf.base = NULL;
taosMemoryFree(uvTask);
fnDebug("udfc freed uvTask: %p", task);
uvTask = NULL;
return task->errCode;
return code;
}
int32_t doSetupUdf(char udfName[], UdfcFuncHandle *funcHandle) {
int32_t code = TSDB_CODE_SUCCESS, lino = 0;
SClientUdfTask *task = taosMemoryCalloc(1, sizeof(SClientUdfTask));
task->errCode = 0;
if(task == NULL) {
fnError("doSetupUdf, failed to allocate memory for task");
return TSDB_CODE_OUT_OF_MEMORY;
}
task->session = taosMemoryCalloc(1, sizeof(SUdfcUvSession));
if(task->session == NULL) {
fnError("doSetupUdf, failed to allocate memory for session");
taosMemoryFree(task);
return TSDB_CODE_OUT_OF_MEMORY;
}
task->session->udfc = &gUdfcProxy;
task->type = UDF_TASK_SETUP;
SUdfSetupRequest *req = &task->_setup.req;
strncpy(req->udfName, udfName, TSDB_FUNC_NAME_LEN);
int32_t errCode = udfcRunUdfUvTask(task, UV_TASK_CONNECT);
if (errCode != 0) {
fnError("failed to connect to pipe. udfName: %s, pipe: %s", udfName, (&gUdfcProxy)->udfdPipeName);
taosMemoryFree(task->session);
taosMemoryFree(task);
return TSDB_CODE_UDF_PIPE_CONNECT_ERR;
}
code = udfcRunUdfUvTask(task, UV_TASK_CONNECT);
TAOS_CHECK_GOTO(code, &lino, _exit);
udfcRunUdfUvTask(task, UV_TASK_REQ_RSP);
code = udfcRunUdfUvTask(task, UV_TASK_REQ_RSP);
TAOS_CHECK_GOTO(code, &lino, _exit);
SUdfSetupResponse *rsp = &task->_setup.rsp;
task->session->severHandle = rsp->udfHandle;
@ -1834,15 +2011,18 @@ int32_t doSetupUdf(char udfName[], UdfcFuncHandle *funcHandle) {
task->session->bytes = rsp->bytes;
task->session->bufSize = rsp->bufSize;
strncpy(task->session->udfName, udfName, TSDB_FUNC_NAME_LEN);
if (task->errCode != 0) {
fnError("failed to setup udf. udfname: %s, err: %d", udfName, task->errCode)
} else {
fnInfo("successfully setup udf func handle. udfName: %s, handle: %p", udfName, task->session);
*funcHandle = task->session;
}
int32_t err = task->errCode;
fnInfo("successfully setup udf func handle. udfName: %s, handle: %p", udfName, task->session);
*funcHandle = task->session;
taosMemoryFree(task);
return err;
return 0;
_exit:
if (code != 0) {
fnError("failed to setup udf. udfname: %s, err: %d line:%d", udfName, code, lino);
}
taosMemoryFree(task->session);
taosMemoryFree(task);
return code;
}
int32_t callUdf(UdfcFuncHandle handle, int8_t callType, SSDataBlock *input, SUdfInterBuf *state, SUdfInterBuf *state2,
@ -1854,7 +2034,10 @@ int32_t callUdf(UdfcFuncHandle handle, int8_t callType, SSDataBlock *input, SUdf
return TSDB_CODE_UDF_PIPE_NOT_EXIST;
}
SClientUdfTask *task = taosMemoryCalloc(1, sizeof(SClientUdfTask));
task->errCode = 0;
if(task == NULL) {
fnError("udfc call udf. failed to allocate memory for task");
return TSDB_CODE_OUT_OF_MEMORY;
}
task->session = (SUdfcUvSession *)handle;
task->type = UDF_TASK_CALL;
@ -1887,10 +2070,9 @@ int32_t callUdf(UdfcFuncHandle handle, int8_t callType, SSDataBlock *input, SUdf
}
}
udfcRunUdfUvTask(task, UV_TASK_REQ_RSP);
if (task->errCode != 0) {
fnError("call udf failure. err: %d", task->errCode);
int32_t code = udfcRunUdfUvTask(task, UV_TASK_REQ_RSP);
if (code != 0) {
fnError("call udf failure. udfcRunUdfUvTask err: %d", code);
} else {
SUdfCallResponse *rsp = &task->_call.rsp;
switch (callType) {
@ -1916,9 +2098,8 @@ int32_t callUdf(UdfcFuncHandle handle, int8_t callType, SSDataBlock *input, SUdf
}
}
};
int err = task->errCode;
taosMemoryFree(task);
return err;
return code;
}
int32_t doCallUdfAggInit(UdfcFuncHandle handle, SUdfInterBuf *interBuf) {
@ -1957,11 +2138,15 @@ int32_t doCallUdfAggFinalize(UdfcFuncHandle handle, SUdfInterBuf *interBuf, SUdf
int32_t doCallUdfScalarFunc(UdfcFuncHandle handle, SScalarParam *input, int32_t numOfCols, SScalarParam *output) {
int8_t callType = TSDB_UDF_CALL_SCALA_PROC;
SSDataBlock inputBlock = {0};
convertScalarParamToDataBlock(input, numOfCols, &inputBlock);
int32_t code = convertScalarParamToDataBlock(input, numOfCols, &inputBlock);
if(code != 0) {
fnError("doCallUdfScalarFunc, convertScalarParamToDataBlock failed. code: %d", code);
return code;
}
SSDataBlock resultBlock = {0};
int32_t err = callUdf(handle, callType, &inputBlock, NULL, NULL, &resultBlock, NULL);
if (err == 0) {
convertDataBlockToScalarParm(&resultBlock, output);
err = convertDataBlockToScalarParm(&resultBlock, output);
taosArrayDestroy(resultBlock.pDataBlock);
}
@ -1970,6 +2155,7 @@ int32_t doCallUdfScalarFunc(UdfcFuncHandle handle, SScalarParam *input, int32_t
}
int32_t doTeardownUdf(UdfcFuncHandle handle) {
int32_t code = TSDB_CODE_SUCCESS, lino = 0;;
SUdfcUvSession *session = (SUdfcUvSession *)handle;
if (session->udfUvPipe == NULL) {
@ -1979,18 +2165,22 @@ int32_t doTeardownUdf(UdfcFuncHandle handle) {
}
SClientUdfTask *task = taosMemoryCalloc(1, sizeof(SClientUdfTask));
task->errCode = 0;
if(task == NULL) {
fnError("doTeardownUdf, failed to allocate memory for task");
taosMemoryFree(session);
return TSDB_CODE_OUT_OF_MEMORY;
}
task->session = session;
task->type = UDF_TASK_TEARDOWN;
SUdfTeardownRequest *req = &task->_teardown.req;
req->udfHandle = task->session->severHandle;
udfcRunUdfUvTask(task, UV_TASK_REQ_RSP);
code = udfcRunUdfUvTask(task, UV_TASK_REQ_RSP);
TAOS_CHECK_GOTO(code, &lino, _exit);
int32_t err = task->errCode;
udfcRunUdfUvTask(task, UV_TASK_DISCONNECT);
code = udfcRunUdfUvTask(task, UV_TASK_DISCONNECT);
TAOS_CHECK_GOTO(code, &lino, _exit);
fnInfo("tear down udf. udf name: %s, udf func handle: %p", session->udfName, handle);
// TODO: synchronization refactor between libuv event loop and request thread
@ -2000,8 +2190,13 @@ int32_t doTeardownUdf(UdfcFuncHandle handle) {
conn->session = NULL;
}
uv_mutex_unlock(&gUdfcProxy.udfcUvMutex);
_exit:
if (code != 0) {
fnError("failed to teardown udf. udf name: %s, err: %d, line: %d", session->udfName, code, lino);
}
taosMemoryFree(session);
taosMemoryFree(task);
return err;
return code;
}

546
source/libs/function/src/udfd.c
View File

@ -54,38 +54,39 @@ int32_t udfdCPluginOpen(SScriptUdfEnvItem *items, int numItems) { return 0; }
int32_t udfdCPluginClose() { return 0; }
const char *udfdCPluginUdfInitLoadInitDestoryFuncs(SUdfCPluginCtx *udfCtx, const char *udfName) {
int32_t udfdCPluginUdfInitLoadInitDestoryFuncs(SUdfCPluginCtx *udfCtx, const char *udfName) {
char initFuncName[TSDB_FUNC_NAME_LEN + 6] = {0};
char *initSuffix = "_init";
snprintf(initFuncName, sizeof(initFuncName), "%s%s", udfName, initSuffix);
uv_dlsym(&udfCtx->lib, initFuncName, (void **)(&udfCtx->initFunc));
TAOS_CHECK_RETURN(uv_dlsym(&udfCtx->lib, initFuncName, (void **)(&udfCtx->initFunc)));
char destroyFuncName[TSDB_FUNC_NAME_LEN + 9] = {0};
char *destroySuffix = "_destroy";
snprintf(destroyFuncName, sizeof(destroyFuncName), "%s%s", udfName, destroySuffix);
uv_dlsym(&udfCtx->lib, destroyFuncName, (void **)(&udfCtx->destroyFunc));
return udfName;
TAOS_CHECK_RETURN(uv_dlsym(&udfCtx->lib, destroyFuncName, (void **)(&udfCtx->destroyFunc)));
return 0;
}
void udfdCPluginUdfInitLoadAggFuncs(SUdfCPluginCtx *udfCtx, const char *udfName) {
int32_t udfdCPluginUdfInitLoadAggFuncs(SUdfCPluginCtx *udfCtx, const char *udfName) {
char processFuncName[TSDB_FUNC_NAME_LEN] = {0};
snprintf(processFuncName, sizeof(processFuncName), "%s", udfName);
uv_dlsym(&udfCtx->lib, processFuncName, (void **)(&udfCtx->aggProcFunc));
snprintf(processFuncName, sizeof(processFuncName), "%s", udfName);
TAOS_CHECK_RETURN(uv_dlsym(&udfCtx->lib, processFuncName, (void **)(&udfCtx->aggProcFunc)));
char startFuncName[TSDB_FUNC_NAME_LEN + 7] = {0};
char *startSuffix = "_start";
snprintf(startFuncName, sizeof(startFuncName), "%s%s", processFuncName, startSuffix);
uv_dlsym(&udfCtx->lib, startFuncName, (void **)(&udfCtx->aggStartFunc));
TAOS_CHECK_RETURN(uv_dlsym(&udfCtx->lib, startFuncName, (void **)(&udfCtx->aggStartFunc)));
char finishFuncName[TSDB_FUNC_NAME_LEN + 8] = {0};
char *finishSuffix = "_finish";
snprintf(finishFuncName, sizeof(finishFuncName), "%s%s", processFuncName, finishSuffix);
uv_dlsym(&udfCtx->lib, finishFuncName, (void **)(&udfCtx->aggFinishFunc));
TAOS_CHECK_RETURN(uv_dlsym(&udfCtx->lib, finishFuncName, (void **)(&udfCtx->aggFinishFunc)));
char mergeFuncName[TSDB_FUNC_NAME_LEN + 7] = {0};
char *mergeSuffix = "_merge";
snprintf(mergeFuncName, sizeof(mergeFuncName), "%s%s", processFuncName, mergeSuffix);
uv_dlsym(&udfCtx->lib, mergeFuncName, (void **)(&udfCtx->aggMergeFunc));
(void)(uv_dlsym(&udfCtx->lib, mergeFuncName, (void **)(&udfCtx->aggMergeFunc)));
return 0;
}
int32_t udfdCPluginUdfInit(SScriptUdfInfo *udf, void **pUdfCtx) {
@ -99,27 +100,43 @@ int32_t udfdCPluginUdfInit(SScriptUdfInfo *udf, void **pUdfCtx) {
}
const char *udfName = udf->name;
udfdCPluginUdfInitLoadInitDestoryFuncs(udfCtx, udfName);
err = udfdCPluginUdfInitLoadInitDestoryFuncs(udfCtx, udfName);
if (err != 0) {
fnError("can not load init/destroy functions. error: %d", err);
err = TSDB_CODE_UDF_LOAD_UDF_FAILURE;
goto _exit;
}
if (udf->funcType == UDF_FUNC_TYPE_SCALAR) {
char processFuncName[TSDB_FUNC_NAME_LEN] = {0};
snprintf(processFuncName, sizeof(processFuncName), "%s", udfName);
uv_dlsym(&udfCtx->lib, processFuncName, (void **)(&udfCtx->scalarProcFunc));
if (uv_dlsym(&udfCtx->lib, processFuncName, (void **)(&udfCtx->scalarProcFunc)) != 0) {
fnError("can not load library function %s. error: %s", processFuncName, uv_strerror(err));
err = TSDB_CODE_UDF_LOAD_UDF_FAILURE;
goto _exit;
}
} else if (udf->funcType == UDF_FUNC_TYPE_AGG) {
udfdCPluginUdfInitLoadAggFuncs(udfCtx, udfName);
err = udfdCPluginUdfInitLoadAggFuncs(udfCtx, udfName);
if (err != 0) {
fnError("can not load aggregation functions. error: %d", err);
err = TSDB_CODE_UDF_LOAD_UDF_FAILURE;
goto _exit;
}
}
int32_t code = 0;
if (udfCtx->initFunc) {
code = (udfCtx->initFunc)();
if (code != 0) {
uv_dlclose(&udfCtx->lib);
taosMemoryFree(udfCtx);
return code;
err = (udfCtx->initFunc)();
if (err != 0) {
fnError("udf init function failed. error: %d", err);
goto _exit;
}
}
*pUdfCtx = udfCtx;
return 0;
_exit:
uv_dlclose(&udfCtx->lib);
taosMemoryFree(udfCtx);
return err;
}
int32_t udfdCPluginUdfDestroy(void *udfCtx) {
@ -303,7 +320,7 @@ static int32_t udfdConnectToMnode();
static bool udfdRpcRfp(int32_t code, tmsg_t msgType);
static int initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet);
static int32_t udfdOpenClientRpc();
static int32_t udfdCloseClientRpc();
static void udfdCloseClientRpc();
static void udfdProcessSetupRequest(SUvUdfWork *uvUdf, SUdfRequest *request);
static void udfdProcessCallRequest(SUvUdfWork *uvUdf, SUdfRequest *request);
@ -320,7 +337,7 @@ static void udfdPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf
static void udfdOnNewConnection(uv_stream_t *server, int status);
static void udfdIntrSignalHandler(uv_signal_t *handle, int signum);
static int32_t removeListeningPipe();
static void removeListeningPipe();
static void udfdPrintVersion();
static int32_t udfdParseArgs(int32_t argc, char *argv[]);
@ -330,13 +347,13 @@ static void udfdCtrlAllocBufCb(uv_handle_t *handle, size_t suggested_size, uv
static void udfdCtrlReadCb(uv_stream_t *q, ssize_t nread, const uv_buf_t *buf);
static int32_t udfdUvInit();
static void udfdCloseWalkCb(uv_handle_t *handle, void *arg);
static int32_t udfdRun();
static void udfdRun();
static void udfdConnectMnodeThreadFunc(void *args);
SUdf *udfdNewUdf(const char *udfName);
int32_t udfdNewUdf(SUdf **pUdf, const char *udfName);
void udfdGetFuncBodyPath(const SUdf *udf, char *path);
void udfdInitializeCPlugin(SUdfScriptPlugin *plugin) {
int32_t udfdInitializeCPlugin(SUdfScriptPlugin *plugin) {
plugin->scriptType = TSDB_FUNC_SCRIPT_BIN_LIB;
plugin->openFunc = udfdCPluginOpen;
plugin->closeFunc = udfdCPluginClose;
@ -349,8 +366,9 @@ void udfdInitializeCPlugin(SUdfScriptPlugin *plugin) {
plugin->udfAggFinishFunc = udfdCPluginUdfAggFinish;
SScriptUdfEnvItem items[1] = {{"LD_LIBRARY_PATH", tsUdfdLdLibPath}};
plugin->openFunc(items, 1);
return;
int32_t err = plugin->openFunc(items, 1);
if (err != 0) return err;
return 0;
}
int32_t udfdLoadSharedLib(char *libPath, uv_lib_t *pLib, const char *funcName[], void **func[], int numOfFuncs) {
@ -412,7 +430,9 @@ int32_t udfdInitializePythonPlugin(SUdfScriptPlugin *plugin) {
void udfdDeinitCPlugin(SUdfScriptPlugin *plugin) {
if (plugin->closeFunc) {
plugin->closeFunc();
if (plugin->closeFunc() != 0) {
fnError("udf script c plugin close func failed.line:%d", __LINE__);
}
}
plugin->openFunc = NULL;
plugin->closeFunc = NULL;
@ -428,7 +448,9 @@ void udfdDeinitCPlugin(SUdfScriptPlugin *plugin) {
void udfdDeinitPythonPlugin(SUdfScriptPlugin *plugin) {
if (plugin->closeFunc) {
plugin->closeFunc();
if(plugin->closeFunc() != 0) {
fnError("udf script python plugin close func failed.line:%d", __LINE__);
}
}
uv_dlclose(&plugin->lib);
if (plugin->libLoaded) {
@ -447,14 +469,23 @@ void udfdDeinitPythonPlugin(SUdfScriptPlugin *plugin) {
int32_t udfdInitScriptPlugin(int8_t scriptType) {
SUdfScriptPlugin *plugin = taosMemoryCalloc(1, sizeof(SUdfScriptPlugin));
if (plugin == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
int32_t err = 0;
switch (scriptType) {
case TSDB_FUNC_SCRIPT_BIN_LIB:
udfdInitializeCPlugin(plugin);
err = udfdInitializeCPlugin(plugin);
if (err != 0) {
fnError("udf script c plugin init failed. error: %d", err);
taosMemoryFree(plugin);
return err;
}
break;
case TSDB_FUNC_SCRIPT_PYTHON: {
int32_t err = udfdInitializePythonPlugin(plugin);
err = udfdInitializePythonPlugin(plugin);
if (err != 0) {
fnError("udf script python plugin init failed. error: %d", err);
taosMemoryFree(plugin);
return err;
}
@ -489,7 +520,7 @@ void udfdDeinitScriptPlugins() {
void udfdProcessRequest(uv_work_t *req) {
SUvUdfWork *uvUdf = (SUvUdfWork *)(req->data);
SUdfRequest request = {0};
decodeUdfRequest(uvUdf->input.base, &request);
if(decodeUdfRequest(uvUdf->input.base, &request) == NULL) return;
switch (request.type) {
case UDF_TASK_SETUP: {
@ -544,6 +575,7 @@ int32_t udfdInitUdf(char *udfName, SUdf *udf) {
if (scriptPlugin == NULL) {
err = udfdInitScriptPlugin(udf->scriptType);
if (err != 0) {
fnError("udf name %s init script plugin failed. error %d", udfName, err);
uv_mutex_unlock(&global.scriptPluginsMutex);
return err;
}
@ -563,15 +595,15 @@ int32_t udfdInitUdf(char *udfName, SUdf *udf) {
return 0;
}
SUdf *udfdNewUdf(const char *udfName) {
int32_t udfdNewUdf(SUdf **pUdf, const char *udfName) {
SUdf *udfNew = taosMemoryCalloc(1, sizeof(SUdf));
udfNew->refCount = 1;
udfNew->lastFetchTime = taosGetTimestampMs();
strncpy(udfNew->name, udfName, TSDB_FUNC_NAME_LEN);
udfNew->state = UDF_STATE_INIT;
uv_mutex_init(&udfNew->lock);
uv_cond_init(&udfNew->condReady);
if (uv_mutex_init(&udfNew->lock) != 0) return TSDB_CODE_UDF_UV_EXEC_FAILURE;
if (uv_cond_init(&udfNew->condReady) != 0) return TSDB_CODE_UDF_UV_EXEC_FAILURE;
udfNew->resident = false;
udfNew->expired = false;
@ -582,10 +614,28 @@ SUdf *udfdNewUdf(const char *udfName) {
break;
}
}
return udfNew;
*pUdf = udfNew;
return 0;
}
SUdf *udfdGetOrCreateUdf(const char *udfName) {
void udfdFreeUdf(void *pData) {
SUdf *pSudf = (SUdf *)pData;
if (pSudf == NULL) {
return;
}
if (pSudf->scriptPlugin != NULL) {
if(pSudf->scriptPlugin->udfDestroyFunc(pSudf->scriptUdfCtx) != 0) {
fnError("udfdFreeUdf: udfd destroy udf %s failed", pSudf->name);
}
}
uv_mutex_destroy(&pSudf->lock);
uv_cond_destroy(&pSudf->condReady);
taosMemoryFree(pSudf);
}
int32_t udfdGetOrCreateUdf(SUdf **ppUdf, const char *udfName) {
uv_mutex_lock(&global.udfsMutex);
SUdf **pUdfHash = taosHashGet(global.udfsHash, udfName, strlen(udfName));
int64_t currTime = taosGetTimestampMs();
@ -594,26 +644,34 @@ SUdf *udfdGetOrCreateUdf(const char *udfName) {
expired = currTime - (*pUdfHash)->lastFetchTime > 10 * 1000; // 10s
if (!expired) {
++(*pUdfHash)->refCount;
SUdf *udf = *pUdfHash;
*ppUdf = *pUdfHash;
uv_mutex_unlock(&global.udfsMutex);
fnInfo("udfd reuse existing udf. udf %s udf version %d, udf created time %" PRIx64, udf->name, udf->version,
udf->createdTime);
return udf;
fnInfo("udfd reuse existing udf. udf %s udf version %d, udf created time %" PRIx64, (*ppUdf)->name, (*ppUdf)->version,
(*ppUdf)->createdTime);
return 0;
} else {
(*pUdfHash)->expired = true;
fnInfo("udfd expired, check for new version. existing udf %s udf version %d, udf created time %" PRIx64,
(*pUdfHash)->name, (*pUdfHash)->version, (*pUdfHash)->createdTime);
taosHashRemove(global.udfsHash, udfName, strlen(udfName));
if(taosHashRemove(global.udfsHash, udfName, strlen(udfName)) != 0) {
fnError("udfdGetOrCreateUdf: udfd remove udf %s failed", udfName);
}
}
}
SUdf *udf = udfdNewUdf(udfName);
int32_t code = udfdNewUdf(ppUdf, udfName);
if(code != 0) {
uv_mutex_unlock(&global.udfsMutex);
return code;
}
SUdf **pUdf = &udf;
taosHashPut(global.udfsHash, udfName, strlen(udfName), pUdf, POINTER_BYTES);
if ((code = taosHashPut(global.udfsHash, udfName, strlen(udfName), ppUdf, POINTER_BYTES)) != 0) {
uv_mutex_unlock(&global.udfsMutex);
return code;
}
uv_mutex_unlock(&global.udfsMutex);
return udf;
return 0;
}
void udfdProcessSetupRequest(SUvUdfWork *uvUdf, SUdfRequest *request) {
@ -622,10 +680,13 @@ void udfdProcessSetupRequest(SUvUdfWork *uvUdf, SUdfRequest *request) {
SUdfSetupRequest *setup = &request->setup;
int32_t code = TSDB_CODE_SUCCESS;
SUdf *udf = NULL;
udf = udfdGetOrCreateUdf(setup->udfName);
SUdf *udf = NULL;
code = udfdGetOrCreateUdf(&udf, setup->udfName);
if(code != 0) {
fnError("udfdGetOrCreateUdf failed. udf name %s", setup->udfName);
goto _send;
}
uv_mutex_lock(&udf->lock);
if (udf->state == UDF_STATE_INIT) {
udf->state = UDF_STATE_LOADING;
@ -646,6 +707,8 @@ void udfdProcessSetupRequest(SUvUdfWork *uvUdf, SUdfRequest *request) {
SUdfcFuncHandle *handle = taosMemoryMalloc(sizeof(SUdfcFuncHandle));
handle->udf = udf;
_send:
;
SUdfResponse rsp;
rsp.seqNum = request->seqNum;
rsp.type = request->type;
@ -656,11 +719,23 @@ void udfdProcessSetupRequest(SUvUdfWork *uvUdf, SUdfRequest *request) {
rsp.setupRsp.bufSize = udf->bufSize;
int32_t len = encodeUdfResponse(NULL, &rsp);
if(len < 0) {
fnError("udfdProcessSetupRequest: encode udf response failed. len %d", len);
return;
}
rsp.msgLen = len;
void *bufBegin = taosMemoryMalloc(len);
if(bufBegin == NULL) {
fnError("udfdProcessSetupRequest: malloc failed. len %d", len);
return;
}
void *buf = bufBegin;
encodeUdfResponse(&buf, &rsp);
if(encodeUdfResponse(&buf, &rsp) < 0) {
fnError("udfdProcessSetupRequest: encode udf response failed. len %d", len);
taosMemoryFree(bufBegin);
return;
}
uvUdf->output = uv_buf_init(bufBegin, len);
taosMemoryFree(uvUdf->input.base);
@ -685,30 +760,35 @@ void udfdProcessCallRequest(SUvUdfWork *uvUdf, SUdfRequest *request) {
output.colMeta.type = udf->outputType;
output.colMeta.precision = 0;
output.colMeta.scale = 0;
udfColEnsureCapacity(&output, call->block.info.rows);
SUdfDataBlock input = {0};
convertDataBlockToUdfDataBlock(&call->block, &input);
code = udf->scriptPlugin->udfScalarProcFunc(&input, &output, udf->scriptUdfCtx);
freeUdfDataDataBlock(&input);
if(code == 0) convertUdfColumnToDataBlock(&output, &response.callRsp.resultData);
if (udfColEnsureCapacity(&output, call->block.info.rows) == TSDB_CODE_SUCCESS) {
SUdfDataBlock input = {0};
code = convertDataBlockToUdfDataBlock(&call->block, &input);
if (code == TSDB_CODE_SUCCESS) code = udf->scriptPlugin->udfScalarProcFunc(&input, &output, udf->scriptUdfCtx);
freeUdfDataDataBlock(&input);
if (code == TSDB_CODE_SUCCESS) code = convertUdfColumnToDataBlock(&output, &response.callRsp.resultData);
}
freeUdfColumn(&output);
break;
}
case TSDB_UDF_CALL_AGG_INIT: {
SUdfInterBuf outBuf = {.buf = taosMemoryMalloc(udf->bufSize), .bufLen = udf->bufSize, .numOfResult = 0};
code = udf->scriptPlugin->udfAggStartFunc(&outBuf, udf->scriptUdfCtx);
if (outBuf.buf != NULL) {
code = udf->scriptPlugin->udfAggStartFunc(&outBuf, udf->scriptUdfCtx);
} else {
code = TSDB_CODE_OUT_OF_MEMORY;
}
subRsp->resultBuf = outBuf;
break;
}
case TSDB_UDF_CALL_AGG_PROC: {
SUdfDataBlock input = {0};
convertDataBlockToUdfDataBlock(&call->block, &input);
SUdfInterBuf outBuf = {.buf = taosMemoryMalloc(udf->bufSize), .bufLen = udf->bufSize, .numOfResult = 0};
code = udf->scriptPlugin->udfAggProcFunc(&input, &call->interBuf, &outBuf, udf->scriptUdfCtx);
freeUdfInterBuf(&call->interBuf);
if (convertDataBlockToUdfDataBlock(&call->block, &input) == TSDB_CODE_SUCCESS) {
SUdfInterBuf outBuf = {.buf = taosMemoryMalloc(udf->bufSize), .bufLen = udf->bufSize, .numOfResult = 0};
code = udf->scriptPlugin->udfAggProcFunc(&input, &call->interBuf, &outBuf, udf->scriptUdfCtx);
freeUdfInterBuf(&call->interBuf);
subRsp->resultBuf = outBuf;
}
freeUdfDataDataBlock(&input);
subRsp->resultBuf = outBuf;
break;
}
@ -738,10 +818,19 @@ void udfdProcessCallRequest(SUvUdfWork *uvUdf, SUdfRequest *request) {
subRsp->callType = call->callType;
int32_t len = encodeUdfResponse(NULL, rsp);
if(len < 0) {
fnError("udfdProcessCallRequest: encode udf response failed. len %d", len);
return;
}
rsp->msgLen = len;
void *bufBegin = taosMemoryMalloc(len);
void *buf = bufBegin;
encodeUdfResponse(&buf, rsp);
if(encodeUdfResponse(&buf, rsp) < 0) {
fnError("udfdProcessCallRequest: encode udf response failed. len %d", len);
taosMemoryFree(bufBegin);
return;
}
uvUdf->output = uv_buf_init(bufBegin, len);
switch (call->callType) {
@ -787,7 +876,11 @@ void udfdProcessTeardownRequest(SUvUdfWork *uvUdf, SUdfRequest *request) {
udf->refCount--;
if (udf->refCount == 0 && (!udf->resident || udf->expired)) {
unloadUdf = true;
taosHashRemove(global.udfsHash, udf->name, strlen(udf->name));
code = taosHashRemove(global.udfsHash, udf->name, strlen(udf->name));
if (code != 0) {
fnError("udf name %s remove from hash failed", udf->name);
goto _send;
}
}
uv_mutex_unlock(&global.udfsMutex);
if (unloadUdf) {
@ -798,18 +891,27 @@ void udfdProcessTeardownRequest(SUvUdfWork *uvUdf, SUdfRequest *request) {
fnDebug("udfd destroy function returns %d", code);
taosMemoryFree(udf);
}
taosMemoryFree(handle);
_send:
taosMemoryFree(handle);
SUdfResponse response = {0};
SUdfResponse *rsp = &response;
rsp->seqNum = request->seqNum;
rsp->type = request->type;
rsp->code = code;
int32_t len = encodeUdfResponse(NULL, rsp);
if (len < 0) {
fnError("udfdProcessTeardownRequest: encode udf response failed. len %d", len);
return;
}
rsp->msgLen = len;
void *bufBegin = taosMemoryMalloc(len);
void *buf = bufBegin;
encodeUdfResponse(&buf, rsp);
if (encodeUdfResponse(&buf, rsp) < 0) {
fnError("udfdProcessTeardownRequest: encode udf response failed. len %d", len);
taosMemoryFree(bufBegin);
return;
}
uvUdf->output = uv_buf_init(bufBegin, len);
taosMemoryFree(uvUdf->input.base);
@ -865,7 +967,10 @@ int32_t udfdSaveFuncBodyToFile(SFuncInfo *pFuncInfo, SUdf *udf) {
fnError("udfd write udf shared library failed");
return TSDB_CODE_FILE_CORRUPTED;
}
taosCloseFile(&file);
if(taosCloseFile(&file) != 0) {
fnError("udfdSaveFuncBodyToFile, udfd close file failed");
return TSDB_CODE_FILE_CORRUPTED;
}
strncpy(udf->path, path, PATH_MAX);
return TSDB_CODE_SUCCESS;
@ -888,7 +993,10 @@ void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) {
if (msgInfo->rpcType == UDFD_RPC_MNODE_CONNECT) {
SConnectRsp connectRsp = {0};
tDeserializeSConnectRsp(pMsg->pCont, pMsg->contLen, &connectRsp);
if(tDeserializeSConnectRsp(pMsg->pCont, pMsg->contLen, &connectRsp) < 0){
fnError("udfd deserialize connect response failed");
goto _return;
}
int32_t now = taosGetTimestampSec();
int32_t delta = abs(now - connectRsp.svrTimestamp);
@ -908,7 +1016,10 @@ void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) {
msgInfo->code = 0;
} else if (msgInfo->rpcType == UDFD_RPC_RETRIVE_FUNC) {
SRetrieveFuncRsp retrieveRsp = {0};
tDeserializeSRetrieveFuncRsp(pMsg->pCont, pMsg->contLen, &retrieveRsp);
if(tDeserializeSRetrieveFuncRsp(pMsg->pCont, pMsg->contLen, &retrieveRsp) < 0){
fnError("udfd deserialize retrieve func response failed");
goto _return;
}
SFuncInfo *pFuncInfo = (SFuncInfo *)taosArrayGet(retrieveRsp.pFuncInfos, 0);
SUdf *udf = msgInfo->param;
@ -940,28 +1051,43 @@ int32_t udfdFillUdfInfoFromMNode(void *clientRpc, char *udfName, SUdf *udf) {
SRetrieveFuncReq retrieveReq = {0};
retrieveReq.numOfFuncs = 1;
retrieveReq.pFuncNames = taosArrayInit(1, TSDB_FUNC_NAME_LEN);
taosArrayPush(retrieveReq.pFuncNames, udfName);
if(taosArrayPush(retrieveReq.pFuncNames, udfName) == NULL) {
taosArrayDestroy(retrieveReq.pFuncNames);
return terrno;
}
int32_t contLen = tSerializeSRetrieveFuncReq(NULL, 0, &retrieveReq);
if(contLen < 0) {
taosArrayDestroy(retrieveReq.pFuncNames);
return terrno;
}
void *pReq = rpcMallocCont(contLen);
tSerializeSRetrieveFuncReq(pReq, contLen, &retrieveReq);
if(tSerializeSRetrieveFuncReq(pReq, contLen, &retrieveReq) < 0) {
taosArrayDestroy(retrieveReq.pFuncNames);
rpcFreeCont(pReq);
return terrno;
}
taosArrayDestroy(retrieveReq.pFuncNames);
SUdfdRpcSendRecvInfo *msgInfo = taosMemoryCalloc(1, sizeof(SUdfdRpcSendRecvInfo));
msgInfo->rpcType = UDFD_RPC_RETRIVE_FUNC;
msgInfo->param = udf;
uv_sem_init(&msgInfo->resultSem, 0);
if(uv_sem_init(&msgInfo->resultSem, 0) != 0) {
taosMemoryFree(msgInfo);
return TSDB_CODE_UDF_UV_EXEC_FAILURE;
}
SRpcMsg rpcMsg = {0};
rpcMsg.pCont = pReq;
rpcMsg.contLen = contLen;
rpcMsg.msgType = TDMT_MND_RETRIEVE_FUNC;
rpcMsg.info.ahandle = msgInfo;
rpcSendRequest(clientRpc, &global.mgmtEp.epSet, &rpcMsg, NULL);
uv_sem_wait(&msgInfo->resultSem);
uv_sem_destroy(&msgInfo->resultSem);
int32_t code = msgInfo->code;
int32_t code = rpcSendRequest(clientRpc, &global.mgmtEp.epSet, &rpcMsg, NULL);
if (code == 0) {
uv_sem_wait(&msgInfo->resultSem);
uv_sem_destroy(&msgInfo->resultSem);
code = msgInfo->code;
}
taosMemoryFree(msgInfo);
return code;
}
@ -1009,8 +1135,12 @@ int initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSe
return -1;
}
taosGetFqdnPortFromEp(secondEp, &mgmtEpSet->eps[mgmtEpSet->numOfEps]);
mgmtEpSet->numOfEps++;
int32_t code = taosGetFqdnPortFromEp(secondEp, &mgmtEpSet->eps[mgmtEpSet->numOfEps]);
if (code != TSDB_CODE_SUCCESS) {
fnError("invalid ep %s", secondEp);
} else {
mgmtEpSet->numOfEps++;
}
}
if (mgmtEpSet->numOfEps == 0) {
@ -1039,7 +1169,7 @@ int32_t udfdOpenClientRpc() {
connLimitNum = TMIN(connLimitNum, 500);
rpcInit.connLimitNum = connLimitNum;
rpcInit.timeToGetConn = tsTimeToGetAvailableConn;
taosVersionStrToInt(version, &(rpcInit.compatibilityVer));
TAOS_CHECK_RETURN(taosVersionStrToInt(version, &(rpcInit.compatibilityVer)));
global.clientRpc = rpcOpen(&rpcInit);
if (global.clientRpc == NULL) {
fnError("failed to init dnode rpc client");
@ -1048,11 +1178,10 @@ int32_t udfdOpenClientRpc() {
return 0;
}
int32_t udfdCloseClientRpc() {
void udfdCloseClientRpc() {
fnInfo("udfd begin closing rpc");
rpcClose(global.clientRpc);
fnInfo("udfd finish closing rpc");
return 0;
}
void udfdOnWrite(uv_write_t *req, int status) {
@ -1082,7 +1211,11 @@ void udfdSendResponse(uv_work_t *work, int status) {
if (udfWork->conn != NULL) {
uv_write_t *write_req = taosMemoryMalloc(sizeof(uv_write_t));
write_req->data = udfWork;
uv_write(write_req, udfWork->conn->client, &udfWork->output, 1, udfdOnWrite);
int32_t code = uv_write(write_req, udfWork->conn->client, &udfWork->output, 1, udfdOnWrite);
if (code != 0) {
fnError("udfd send response error %s", uv_strerror(code));
taosMemoryFree(write_req);
}
}
taosMemoryFree(work);
}
@ -1146,7 +1279,12 @@ void udfdHandleRequest(SUdfdUvConn *conn) {
conn->inputCap = 0;
conn->inputTotal = -1;
work->data = udfWork;
uv_queue_work(global.loop, work, udfdProcessRequest, udfdSendResponse);
if(uv_queue_work(global.loop, work, udfdProcessRequest, udfdSendResponse) != 0)
{
fnError("udfd queue work failed");
taosMemoryFree(work);
taosMemoryFree(udfWork);
}
}
void udfdPipeCloseCb(uv_handle_t *pipe) {
@ -1193,9 +1331,15 @@ void udfdOnNewConnection(uv_stream_t *server, int status) {
fnError("udfd new connection error. code: %s", uv_strerror(status));
return;
}
int32_t code = 0;
uv_pipe_t *client = (uv_pipe_t *)taosMemoryMalloc(sizeof(uv_pipe_t));
uv_pipe_init(global.loop, client, 0);
code = uv_pipe_init(global.loop, client, 0);
if (code) {
fnError("udfd pipe init error %s", uv_strerror(code));
taosMemoryFree(client);
return;
}
if (uv_accept(server, (uv_stream_t *)client) == 0) {
SUdfdUvConn *ctx = taosMemoryMalloc(sizeof(SUdfdUvConn));
ctx->pWorkList = NULL;
@ -1205,7 +1349,13 @@ void udfdOnNewConnection(uv_stream_t *server, int status) {
ctx->inputCap = 0;
client->data = ctx;
ctx->client = (uv_stream_t *)client;
uv_read_start((uv_stream_t *)client, udfdAllocBuffer, udfdPipeRead);
code = uv_read_start((uv_stream_t *)client, udfdAllocBuffer, udfdPipeRead);
if (code) {
fnError("udfd read start error %s", uv_strerror(code));
udfdUvHandleError(ctx);
taosMemoryFree(ctx);
taosMemoryFree(client);
}
} else {
uv_close((uv_handle_t *)client, NULL);
}
@ -1214,8 +1364,14 @@ void udfdOnNewConnection(uv_stream_t *server, int status) {
void udfdIntrSignalHandler(uv_signal_t *handle, int signum) {
fnInfo("udfd signal received: %d\n", signum);
uv_fs_t req;
uv_fs_unlink(global.loop, &req, global.listenPipeName, NULL);
uv_signal_stop(handle);
int32_t code = uv_fs_unlink(global.loop, &req, global.listenPipeName, NULL);
if(code) {
fnError("remove listening pipe %s failed, reason:%s, lino:%d", global.listenPipeName, uv_strerror(code), __LINE__);
}
code = uv_signal_stop(handle);
if(code) {
fnError("stop signal handler failed, reason:%s", uv_strerror(code));
}
uv_stop(global.loop);
}
@ -1224,12 +1380,12 @@ static int32_t udfdParseArgs(int32_t argc, char *argv[]) {
if (strcmp(argv[i], "-c") == 0) {
if (i < argc - 1) {
if (strlen(argv[++i]) >= PATH_MAX) {
printf("config file path overflow");
(void)printf("config file path overflow");
return -1;
}
tstrncpy(configDir, argv[i], PATH_MAX);
} else {
printf("'-c' requires a parameter, default is %s\n", configDir);
(void)printf("'-c' requires a parameter, default is %s\n", configDir);
return -1;
}
} else if (strcmp(argv[i], "-V") == 0) {
@ -1242,9 +1398,9 @@ static int32_t udfdParseArgs(int32_t argc, char *argv[]) {
}
static void udfdPrintVersion() {
printf("udfd version: %s compatible_version: %s\n", version, compatible_version);
printf("git: %s\n", gitinfo);
printf("build: %s\n", buildinfo);
(void)printf("udfd version: %s compatible_version: %s\n", version, compatible_version);
(void)printf("git: %s\n", gitinfo);
(void)printf("build: %s\n", buildinfo);
}
static int32_t udfdInitLog() {
@ -1270,35 +1426,31 @@ void udfdCtrlReadCb(uv_stream_t *q, ssize_t nread, const uv_buf_t *buf) {
taosMemoryFree(buf->base);
}
static int32_t removeListeningPipe() {
static void removeListeningPipe() {
uv_fs_t req;
int err = uv_fs_unlink(global.loop, &req, global.listenPipeName, NULL);
uv_fs_req_cleanup(&req);
return err;
if(err) {
fnError("remove listening pipe %s failed, reason:%s, lino:%d", global.listenPipeName, uv_strerror(err), __LINE__);
}
}
static int32_t udfdUvInit() {
uv_loop_t *loop = taosMemoryMalloc(sizeof(uv_loop_t));
if (loop) {
uv_loop_init(loop);
} else {
return -1;
}
global.loop = loop;
TAOS_CHECK_RETURN(uv_loop_init(global.loop));
if (tsStartUdfd) { // udfd is started by taosd, which shall exit when taosd exit
uv_pipe_init(global.loop, &global.ctrlPipe, 1);
uv_pipe_open(&global.ctrlPipe, 0);
uv_read_start((uv_stream_t *)&global.ctrlPipe, udfdCtrlAllocBufCb, udfdCtrlReadCb);
TAOS_CHECK_RETURN(uv_pipe_init(global.loop, &global.ctrlPipe, 1));
TAOS_CHECK_RETURN(uv_pipe_open(&global.ctrlPipe, 0));
TAOS_CHECK_RETURN(uv_read_start((uv_stream_t *)&global.ctrlPipe, udfdCtrlAllocBufCb, udfdCtrlReadCb));
}
getUdfdPipeName(global.listenPipeName, sizeof(global.listenPipeName));
removeListeningPipe();
uv_pipe_init(global.loop, &global.listeningPipe, 0);
TAOS_CHECK_RETURN(uv_pipe_init(global.loop, &global.listeningPipe, 0));
uv_signal_init(global.loop, &global.intrSignal);
uv_signal_start(&global.intrSignal, udfdIntrSignalHandler, SIGINT);
TAOS_CHECK_RETURN(uv_signal_init(global.loop, &global.intrSignal));
TAOS_CHECK_RETURN(uv_signal_start(&global.intrSignal, udfdIntrSignalHandler, SIGINT));
int r;
fnInfo("bind to pipe %s", global.listenPipeName);
@ -1321,25 +1473,59 @@ static void udfdCloseWalkCb(uv_handle_t *handle, void *arg) {
}
}
static int32_t udfdRun() {
uv_mutex_init(&global.scriptPluginsMutex);
static int32_t udfdGlobalDataInit() {
uv_loop_t *loop = taosMemoryMalloc(sizeof(uv_loop_t));
if (loop == NULL) {
fnError("udfd init uv loop failed, mem overflow");
return -1;
}
global.loop = loop;
if (uv_mutex_init(&global.scriptPluginsMutex) != 0) {
fnError("udfd init script plugins mutex failed");
return -1;
}
global.udfsHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
uv_mutex_init(&global.udfsMutex);
if (global.udfsHash == NULL) {
return terrno;
}
// taosHashSetFreeFp(global.udfsHash, udfdFreeUdf);
fnInfo("start udfd event loop");
uv_run(global.loop, UV_RUN_DEFAULT);
fnInfo("udfd event loop stopped.");
uv_loop_close(global.loop);
uv_walk(global.loop, udfdCloseWalkCb, NULL);
uv_run(global.loop, UV_RUN_DEFAULT);
uv_loop_close(global.loop);
if (uv_mutex_init(&global.udfsMutex) != 0) {
fnError("udfd init udfs mutex failed");
return -2;
}
return 0;
}
static void udfdGlobalDataDeinit() {
taosHashCleanup(global.udfsHash);
uv_mutex_destroy(&global.udfsMutex);
uv_mutex_destroy(&global.scriptPluginsMutex);
taosMemoryFree(global.loop);
fnInfo("udfd global data deinit");
}
static void udfdRun() {
fnInfo("start udfd event loop");
int32_t code = uv_run(global.loop, UV_RUN_DEFAULT);
if(code != 0) {
fnError("udfd event loop still has active handles or requests.");
}
fnInfo("udfd event loop stopped.");
(void)uv_loop_close(global.loop);
uv_walk(global.loop, udfdCloseWalkCb, NULL);
code = uv_run(global.loop, UV_RUN_DEFAULT);
if(code != 0) {
fnError("udfd event loop still has active handles or requests.");
}
(void)uv_loop_close(global.loop);
}
int32_t udfdInitResidentFuncs() {
if (strlen(tsUdfdResFuncs) == 0) {
return TSDB_CODE_SUCCESS;
@ -1352,13 +1538,17 @@ int32_t udfdInitResidentFuncs() {
char func[TSDB_FUNC_NAME_LEN + 1] = {0};
strncpy(func, token, TSDB_FUNC_NAME_LEN);
fnInfo("udfd add resident function %s", func);
taosArrayPush(global.residentFuncs, func);
if(taosArrayPush(global.residentFuncs, func) == NULL)
{
taosArrayDestroy(global.residentFuncs);
return TSDB_CODE_OUT_OF_MEMORY;
}
}
return TSDB_CODE_SUCCESS;
}
int32_t udfdDeinitResidentFuncs() {
void udfdDeinitResidentFuncs() {
for (int32_t i = 0; i < taosArrayGetSize(global.residentFuncs); ++i) {
char *funcName = taosArrayGet(global.residentFuncs, i);
SUdf **udfInHash = taosHashGet(global.udfsHash, funcName, strlen(funcName));
@ -1366,18 +1556,15 @@ int32_t udfdDeinitResidentFuncs() {
SUdf *udf = *udfInHash;
int32_t code = udf->scriptPlugin->udfDestroyFunc(udf->scriptUdfCtx);
fnDebug("udfd destroy function returns %d", code);
taosHashRemove(global.udfsHash, funcName, strlen(funcName));
if(taosHashRemove(global.udfsHash, funcName, strlen(funcName)) != 0)
{
fnError("udfd remove resident function %s failed", funcName);
}
taosMemoryFree(udf);
}
}
taosArrayDestroy(global.residentFuncs);
return TSDB_CODE_SUCCESS;
}
int32_t udfdCleanup() {
uv_mutex_destroy(&global.udfsMutex);
taosHashCleanup(global.udfsHash);
return 0;
fnInfo("udfd resident functions are deinit");
}
int32_t udfdCreateUdfSourceDir() {
@ -1392,20 +1579,27 @@ int32_t udfdCreateUdfSourceDir() {
return code;
}
int32_t udfdDestroyUdfSourceDir() {
void udfdDestroyUdfSourceDir() {
fnInfo("destory udf source directory %s", global.udfDataDir);
taosRemoveDir(global.udfDataDir);
return 0;
}
int main(int argc, char *argv[]) {
int code = 0;
bool logInitialized = false;
bool cfgInitialized = false;
bool openClientRpcFinished = false;
bool residentFuncsInited = false;
bool udfSourceDirInited = false;
bool globalDataInited = false;
if (!taosCheckSystemIsLittleEnd()) {
printf("failed to start since on non-little-end machines\n");
(void)printf("failed to start since on non-little-end machines\n");
return -1;
}
if (udfdParseArgs(argc, argv) != 0) {
printf("failed to start since parse args error\n");
(void)printf("failed to start since parse args error\n");
return -1;
}
@ -1416,47 +1610,89 @@ int main(int argc, char *argv[]) {
if (udfdInitLog() != 0) {
// ignore create log failed, because this error no matter
printf("failed to start since init log error\n");
(void)printf("failed to init udfd log.");
} else {
logInitialized = true; // log is initialized
}
if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 0) != 0) {
fnError("failed to start since read config error");
taosCloseLog();
return -2;
code = -2;
goto _exit;
}
cfgInitialized = true; // cfg is initialized
fnInfo("udfd start with config file %s", configDir);
initEpSetFromCfg(tsFirst, tsSecond, &global.mgmtEp);
if (initEpSetFromCfg(tsFirst, tsSecond, &global.mgmtEp) != 0) {
fnError("init ep set from cfg failed");
code = -3;
goto _exit;
}
fnInfo("udfd start with mnode ep %s", global.mgmtEp.epSet.eps[0].fqdn);
if (udfdOpenClientRpc() != 0) {
fnError("open rpc connection to mnode failed");
taosCloseLog();
return -3;
code = -4;
goto _exit;
}
fnInfo("udfd rpc client is opened");
openClientRpcFinished = true; // rpc is opened
if (udfdCreateUdfSourceDir() != 0) {
fnError("create udf source directory failed");
taosCloseLog();
return -4;
code = -5;
goto _exit;
}
udfSourceDirInited = true; // udf source dir is created
fnInfo("udfd udf source directory is created");
if (udfdGlobalDataInit() != 0) {
fnError("init global data failed");
code = -6;
goto _exit;
}
globalDataInited = true; // global data is inited
fnInfo("udfd global data is inited");
if (udfdUvInit() != 0) {
fnError("uv init failure");
taosCloseLog();
return -5;
code = -7;
goto _exit;
}
fnInfo("udfd uv is inited");
udfdInitResidentFuncs();
if (udfdInitResidentFuncs() != 0) {
fnError("init resident functions failed");
code = -8;
goto _exit;
}
residentFuncsInited = true; // resident functions are inited
fnInfo("udfd resident functions are inited");
udfdRun();
fnInfo("udfd exit normally");
removeListeningPipe();
udfdDestroyUdfSourceDir();
udfdCloseClientRpc();
udfdDeinitResidentFuncs();
udfdDeinitScriptPlugins();
taosCloseLog();
udfdCleanup();
return 0;
_exit:
if (globalDataInited) {
udfdGlobalDataDeinit();
}
if (residentFuncsInited) {
udfdDeinitResidentFuncs();
}
if (udfSourceDirInited) {
udfdDestroyUdfSourceDir();
}
if (openClientRpcFinished) {
udfdCloseClientRpc();
}
if (cfgInitialized) {
taosCleanupCfg();
}
if (logInitialized) {
taosCloseLog();
}
return code;
}

136
source/libs/function/test/runUdf.c
View File

@ -9,17 +9,20 @@
#include "tglobal.h"
#include "tudf.h"
#define TAOSFPRINTF(stream, format, ...) ((void)fprintf(stream, format, ##__VA_ARGS__))
#define TAOSPRINTF(format, ...) ((void)printf(format, ##__VA_ARGS__))
static int32_t parseArgs(int32_t argc, char *argv[]) {
for (int32_t i = 1; i < argc; ++i) {
if (strcmp(argv[i], "-c") == 0) {
if (i < argc - 1) {
if (strlen(argv[++i]) >= PATH_MAX) {
printf("config file path overflow");
TAOSPRINTF("config file path overflow");
return -1;
}
tstrncpy(configDir, argv[i], PATH_MAX);
} else {
printf("'-c' requires a parameter, default is %s\n", configDir);
TAOSPRINTF("'-c' requires a parameter, default is %s\n", configDir);
return -1;
}
}
@ -35,6 +38,7 @@ static int32_t initLog() {
}
int scalarFuncTest() {
int32_t ret = 0;
UdfcFuncHandle handle;
if (doSetupUdf("udf1", &handle) != 0) {
@ -47,10 +51,18 @@ int scalarFuncTest() {
SSDataBlock *pBlock = &block;
for (int32_t i = 0; i < 1; ++i) {
SColumnInfoData colInfo = createColumnInfoData(TSDB_DATA_TYPE_INT, sizeof(int32_t), 1);
blockDataAppendColInfo(pBlock, &colInfo);
ret = blockDataAppendColInfo(pBlock, &colInfo);
if (ret != 0) {
fnError("failed to append column info");
return -1;
}
}
blockDataEnsureCapacity(pBlock, 1024);
ret = blockDataEnsureCapacity(pBlock, 1024);
if (ret != 0) {
fnError("failed to ensure capacity");
return -1;
}
pBlock->info.rows = 1024;
SColumnInfoData *pCol = taosArrayGet(pBlock->pDataBlock, 0);
@ -63,41 +75,69 @@ int scalarFuncTest() {
input.columnData = taosArrayGet(pBlock->pDataBlock, 0);
SScalarParam output = {0};
doCallUdfScalarFunc(handle, &input, 1, &output);
ret = doCallUdfScalarFunc(handle, &input, 1, &output);
if (ret != 0) {
fnError("failed to call udf scalar function");
return -1;
}
taosArrayDestroy(pBlock->pDataBlock);
SColumnInfoData *col = output.columnData;
for (int32_t i = 0; i < output.numOfRows; ++i) {
if (i % 100 == 0) fprintf(stderr, "%d\t%d\n", i, *(int32_t *)(col->pData + i * sizeof(int32_t)));
if (i % 100 == 0) TAOSFPRINTF(stderr, "%d\t%d\n", i, *(int32_t *)(col->pData + i * sizeof(int32_t)));
}
colDataDestroy(output.columnData);
taosMemoryFree(output.columnData);
}
int64_t end = taosGetTimestampUs();
fprintf(stderr, "time: %f\n", (end - beg) / 1000.0);
doTeardownUdf(handle);
TAOSFPRINTF(stderr, "time: %f\n", (end - beg) / 1000.0);
ret = doTeardownUdf(handle);
if (ret != 0) {
fnError("failed to teardown udf");
return -1;
}
return 0;
}
int aggregateFuncTest() {
int32_t ret = 0;
UdfcFuncHandle handle;
if (doSetupUdf("udf2", &handle) != 0) {
fnError("setup udf failure");
ret = doSetupUdf("udf2", &handle);
if (ret != 0) {
fnError("setup udf failure, code:%d", ret);
return -1;
}
SSDataBlock *pBlock = createDataBlock();
for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) {
SColumnInfoData colInfo = createColumnInfoData(TSDB_DATA_TYPE_INT, sizeof(int32_t), 1);
blockDataAppendColInfo(pBlock, &colInfo);
SSDataBlock *pBlock = NULL;
int32_t code = createDataBlock(&pBlock);
if (code) {
return code;
}
blockDataEnsureCapacity(pBlock, 1024);
for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) {
SColumnInfoData colInfo = createColumnInfoData(TSDB_DATA_TYPE_INT, sizeof(int32_t), 1);
ret = blockDataAppendColInfo(pBlock, &colInfo);
if(ret != 0) {
fnError( "failed to append column info. code:%d", ret);
return -1;
}
}
ret = blockDataEnsureCapacity(pBlock, 1024);
if (ret != 0) {
fnError( "failed to ensure capacity. code:%d", ret);
return -1;
}
pBlock->info.rows = 1024;
SColumnInfoData *pColInfo = bdGetColumnInfoData(pBlock, 0);
SColumnInfoData *pColInfo = NULL;
code = bdGetColumnInfoData(pBlock, 0, &pColInfo);
if (code) {
return code;
}
for (int32_t j = 0; j < pBlock->info.rows; ++j) {
colDataSetInt32(pColInfo, j, &j);
}
@ -105,37 +145,77 @@ int aggregateFuncTest() {
SUdfInterBuf buf = {0};
SUdfInterBuf newBuf = {0};
SUdfInterBuf resultBuf = {0};
doCallUdfAggInit(handle, &buf);
doCallUdfAggProcess(handle, pBlock, &buf, &newBuf);
ret = doCallUdfAggInit(handle, &buf);
if (ret != 0) {
fnError("failed to init udf. code:%d", ret);
return -1;
}
ret = doCallUdfAggProcess(handle, pBlock, &buf, &newBuf);
if (ret != 0) {
fnError("failed to process udf. code:%d", ret);
return -1;
}
taosArrayDestroy(pBlock->pDataBlock);
doCallUdfAggFinalize(handle, &newBuf, &resultBuf);
ret = doCallUdfAggFinalize(handle, &newBuf, &resultBuf);
if (ret != 0) {
TAOSFPRINTF(stderr,"failed to finalize udf. code:%d", ret);
return -1;
}
if (resultBuf.buf != NULL) {
fprintf(stderr, "agg result: %f\n", *(double *)resultBuf.buf);
TAOSFPRINTF(stderr, "agg result: %f\n", *(double *)resultBuf.buf);
} else {
fprintf(stderr, "result buffer is null");
fnError("result buffer is null");
}
freeUdfInterBuf(&buf);
freeUdfInterBuf(&newBuf);
freeUdfInterBuf(&resultBuf);
doTeardownUdf(handle);
ret = doTeardownUdf(handle);
if (ret != 0) {
fnError("failed to teardown udf. code:%d", ret);
return -1;
}
blockDataDestroy(pBlock);
return 0;
}
int main(int argc, char *argv[]) {
parseArgs(argc, argv);
initLog();
int32_t ret = 0;
ret = parseArgs(argc, argv);
if (ret != 0) {
fnError("failed to parse args");
return -1;
}
ret = initLog();
if (ret != 0) {
fnError("failed to init log");
return -1;
}
if (taosInitCfg(configDir, NULL, NULL, NULL, NULL, 0) != 0) {
fnError("failed to start since read config error");
return -1;
}
udfcOpen();
if (udfcOpen() != 0) {
fnError("failed to open udfc");
return -1;
}
uv_sleep(1000);
scalarFuncTest();
aggregateFuncTest();
udfcClose();
ret = scalarFuncTest();
if (ret != 0) {
fnError("failed to run scalar function test");
return -1;
}
ret = aggregateFuncTest();
if (ret != 0) {
fnError("failed to run aggregate function test");
return -1;
}
ret = udfcClose();
if (ret != 0) {
fnError("failed to close udfc");
return -1;
}
}

11
source/libs/function/test/udf1.c
View File

@ -14,18 +14,25 @@ DLL_EXPORT int32_t udf1_init() { return 0; }
DLL_EXPORT int32_t udf1_destroy() { return 0; }
DLL_EXPORT int32_t udf1(SUdfDataBlock *block, SUdfColumn *resultCol) {
int32_t code = 0;
SUdfColumnData *resultData = &resultCol->colData;
for (int32_t i = 0; i < block->numOfRows; ++i) {
int j = 0;
for (; j < block->numOfCols; ++j) {
if (udfColDataIsNull(block->udfCols[j], i)) {
udfColDataSetNull(resultCol, i);
code = udfColDataSetNull(resultCol, i);
if (code != 0) {
return code;
}
break;
}
}
if (j == block->numOfCols) {
int32_t luckyNum = 1;
udfColDataSet(resultCol, i, (char *)&luckyNum, false);
code = udfColDataSet(resultCol, i, (char *)&luckyNum, false);
if (code != 0) {
return code;
}
}
}
// to simulate actual processing delay by udf

11
source/libs/function/test/udf1_dup.c
View File

@ -15,18 +15,25 @@ DLL_EXPORT int32_t udf1_dup_init() { return 0; }
DLL_EXPORT int32_t udf1_dup_destroy() { return 0; }
DLL_EXPORT int32_t udf1_dup(SUdfDataBlock *block, SUdfColumn *resultCol) {
int32_t code = 0;
SUdfColumnData *resultData = &resultCol->colData;
for (int32_t i = 0; i < block->numOfRows; ++i) {
int j = 0;
for (; j < block->numOfCols; ++j) {
if (udfColDataIsNull(block->udfCols[j], i)) {
udfColDataSetNull(resultCol, i);
code = udfColDataSetNull(resultCol, i);
if (code != 0) {
return code;
}
break;
}
}
if (j == block->numOfCols) {
int32_t luckyNum = 2;
udfColDataSet(resultCol, i, (char *)&luckyNum, false);
code = udfColDataSet(resultCol, i, (char *)&luckyNum, false);
if (code != 0) {
return code;
}
}
}
// to simulate actual processing delay by udf

36
source/libs/monitor/src/monMain.c
View File

@ -28,7 +28,7 @@ char *tsMonSlowLogUri = "/slow-sql-detail-batch";
char *tsMonFwBasicUri = "/taosd-cluster-basic";
void monRecordLog(int64_t ts, ELogLevel level, const char *content) {
taosThreadMutexLock(&tsMonitor.lock);
(void)taosThreadMutexLock(&tsMonitor.lock);
int32_t size = taosArrayGetSize(tsMonitor.logs);
if (size < tsMonitor.cfg.maxLogs) {
SMonLogItem item = {.ts = ts, .level = level};
@ -37,11 +37,11 @@ void monRecordLog(int64_t ts, ELogLevel level, const char *content) {
tstrncpy(pItem->content, content, MON_LOG_LEN);
}
}
taosThreadMutexUnlock(&tsMonitor.lock);
(void)taosThreadMutexUnlock(&tsMonitor.lock);
}
int32_t monGetLogs(SMonLogs *logs) {
taosThreadMutexLock(&tsMonitor.lock);
(void)taosThreadMutexLock(&tsMonitor.lock);
logs->logs = taosArrayDup(tsMonitor.logs, NULL);
logs->numOfInfoLogs = tsNumOfInfoLogs;
logs->numOfErrorLogs = tsNumOfErrorLogs;
@ -52,7 +52,7 @@ int32_t monGetLogs(SMonLogs *logs) {
tsNumOfDebugLogs = 0;
tsNumOfTraceLogs = 0;
taosArrayClear(tsMonitor.logs);
taosThreadMutexUnlock(&tsMonitor.lock);
(void)taosThreadMutexUnlock(&tsMonitor.lock);
if (logs->logs == NULL) {
TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY);
}
@ -60,44 +60,44 @@ int32_t monGetLogs(SMonLogs *logs) {
}
void monSetDmInfo(SMonDmInfo *pInfo) {
taosThreadMutexLock(&tsMonitor.lock);
(void)taosThreadMutexLock(&tsMonitor.lock);
memcpy(&tsMonitor.dmInfo, pInfo, sizeof(SMonDmInfo));
taosThreadMutexUnlock(&tsMonitor.lock);
(void)taosThreadMutexUnlock(&tsMonitor.lock);
memset(pInfo, 0, sizeof(SMonDmInfo));
}
void monSetMmInfo(SMonMmInfo *pInfo) {
taosThreadMutexLock(&tsMonitor.lock);
(void)taosThreadMutexLock(&tsMonitor.lock);
memcpy(&tsMonitor.mmInfo, pInfo, sizeof(SMonMmInfo));
taosThreadMutexUnlock(&tsMonitor.lock);
(void)taosThreadMutexUnlock(&tsMonitor.lock);
memset(pInfo, 0, sizeof(SMonMmInfo));
}
void monSetVmInfo(SMonVmInfo *pInfo) {
taosThreadMutexLock(&tsMonitor.lock);
(void)taosThreadMutexLock(&tsMonitor.lock);
memcpy(&tsMonitor.vmInfo, pInfo, sizeof(SMonVmInfo));
taosThreadMutexUnlock(&tsMonitor.lock);
(void)taosThreadMutexUnlock(&tsMonitor.lock);
memset(pInfo, 0, sizeof(SMonVmInfo));
}
void monSetQmInfo(SMonQmInfo *pInfo) {
taosThreadMutexLock(&tsMonitor.lock);
(void)taosThreadMutexLock(&tsMonitor.lock);
memcpy(&tsMonitor.qmInfo, pInfo, sizeof(SMonQmInfo));
taosThreadMutexUnlock(&tsMonitor.lock);
(void)taosThreadMutexUnlock(&tsMonitor.lock);
memset(pInfo, 0, sizeof(SMonQmInfo));
}
void monSetSmInfo(SMonSmInfo *pInfo) {
taosThreadMutexLock(&tsMonitor.lock);
(void)taosThreadMutexLock(&tsMonitor.lock);
memcpy(&tsMonitor.smInfo, pInfo, sizeof(SMonSmInfo));
taosThreadMutexUnlock(&tsMonitor.lock);
(void)taosThreadMutexUnlock(&tsMonitor.lock);
memset(pInfo, 0, sizeof(SMonSmInfo));
}
void monSetBmInfo(SMonBmInfo *pInfo) {
taosThreadMutexLock(&tsMonitor.lock);
(void)taosThreadMutexLock(&tsMonitor.lock);
memcpy(&tsMonitor.bmInfo, pInfo, sizeof(SMonBmInfo));
taosThreadMutexUnlock(&tsMonitor.lock);
(void)taosThreadMutexUnlock(&tsMonitor.lock);
memset(pInfo, 0, sizeof(SMonBmInfo));
}
@ -153,7 +153,7 @@ static SMonInfo *monCreateMonitorInfo() {
monGetLogs(&pMonitor->log);
taosThreadMutexLock(&tsMonitor.lock);
(void)taosThreadMutexLock(&tsMonitor.lock);
memcpy(&pMonitor->dmInfo, &tsMonitor.dmInfo, sizeof(SMonDmInfo));
memcpy(&pMonitor->mmInfo, &tsMonitor.mmInfo, sizeof(SMonMmInfo));
memcpy(&pMonitor->vmInfo, &tsMonitor.vmInfo, sizeof(SMonVmInfo));
@ -166,7 +166,7 @@ static SMonInfo *monCreateMonitorInfo() {
memset(&tsMonitor.smInfo, 0, sizeof(SMonSmInfo));
memset(&tsMonitor.qmInfo, 0, sizeof(SMonQmInfo));
memset(&tsMonitor.bmInfo, 0, sizeof(SMonBmInfo));
taosThreadMutexUnlock(&tsMonitor.lock);
(void)taosThreadMutexUnlock(&tsMonitor.lock);
pMonitor->pJson = tjsonCreateObject();
if (pMonitor->pJson == NULL || pMonitor->log.logs == NULL) {

5
source/libs/scalar/test/filter/filterTests.cpp
View File

@ -112,7 +112,10 @@ int32_t flttMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType,
}
if (NULL == *block) {
SSDataBlock *res = createDataBlock();
SSDataBlock *res = NULL;
int32_t code = createDataBlock(&res);
ASSERT(code == 0);
for (int32_t i = 0; i < 2; ++i) {
SColumnInfoData idata = createColumnInfoData(TSDB_DATA_TYPE_NULL, 10, 1 + i);
FLT_ERR_RET(blockDataAppendColInfo(res, &idata));

14
source/libs/scalar/test/scalar/scalarTests.cpp
View File

@ -91,14 +91,16 @@ void scltInitLogFile() {
int32_t scltAppendReservedSlot(SArray *pBlockList, int16_t *dataBlockId, int16_t *slotId, bool newBlock, int32_t rows,
SColumnInfo *colInfo) {
if (newBlock) {
SSDataBlock *res = createDataBlock();
if (NULL == res || NULL == res->pDataBlock) {
SSDataBlock *res = NULL;
int32_t code = createDataBlock(&res);
if (code != 0 || NULL == res->pDataBlock) {
SCL_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
SColumnInfoData idata = {0};
idata.info = *colInfo;
int32_t code = colInfoDataEnsureCapacity(&idata, rows, true);
code = colInfoDataEnsureCapacity(&idata, rows, true);
if (code != TSDB_CODE_SUCCESS) {
taosMemoryFree(&idata);
SCL_ERR_RET(code);
@ -185,7 +187,11 @@ int32_t scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType,
}
if (NULL == *block) {
SSDataBlock *res = createDataBlock();
SSDataBlock *res = NULL;
int32_t code = createDataBlock(&res);
ASSERT(code == 0);
for (int32_t i = 0; i < 2; ++i) {
SColumnInfoData idata = createColumnInfoData(TSDB_DATA_TYPE_INT, 10, i + 1);
code = colInfoDataEnsureCapacity(&idata, rowNum, true);

48
source/libs/stream/src/streamData.c
View File

@ -37,19 +37,30 @@ int32_t createStreamBlockFromDispatchMsg(const SStreamDispatchReq* pReq, int32_t
for (int32_t i = 0; i < blockNum; i++) {
SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(pReq->data, i);
SSDataBlock* pDataBlock = taosArrayGet(pArray, i);
if (pDataBlock == NULL) {
return terrno;
}
int32_t compLen = *(int32_t*)pRetrieve->data;
int32_t fullLen = *(int32_t*)(pRetrieve->data + sizeof(int32_t));
char* pInput = pRetrieve->data + PAYLOAD_PREFIX_LEN;
if (pRetrieve->compressed && compLen < fullLen) {
char* p = taosMemoryMalloc(fullLen);
char* p = taosMemoryMalloc(fullLen);
if (p == NULL) {
return terrno;
}
int32_t len = tsDecompressString(pInput, compLen, 1, p, fullLen, ONE_STAGE_COMP, NULL, 0);
ASSERT(len == fullLen);
pInput = p;
}
(void) blockDecode(pDataBlock, pInput);
const char* pDummy = NULL;
code = blockDecode(pDataBlock, pInput, &pDummy);
if (code) {
return code;
}
if (pRetrieve->compressed && compLen < fullLen) {
taosMemoryFree(pInput);
@ -109,18 +120,31 @@ void destroyStreamDataBlock(SStreamDataBlock* pBlock) {
}
int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock* pData, const char* id) {
SArray* pArray = taosArrayInit(1, sizeof(SSDataBlock));
const char* pDummy = NULL;
SRetrieveTableRsp* pRetrieve = pReq->pRetrieve;
SArray* pArray = taosArrayInit(1, sizeof(SSDataBlock));
if (pArray == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
stError("failed to prepare retrieve block, %s", id);
return terrno;
}
(void) taosArrayPush(pArray, &(SSDataBlock){0});
SRetrieveTableRsp* pRetrieve = pReq->pRetrieve;
SSDataBlock* pDataBlock = taosArrayGet(pArray, 0);
void* px = taosArrayPush(pArray, &(SSDataBlock){0});
if (px == NULL) {
taosArrayDestroy(pArray);
return terrno;
}
(void) blockDecode(pDataBlock, pRetrieve->data + PAYLOAD_PREFIX_LEN);
SSDataBlock* pDataBlock = taosArrayGet(pArray, 0);
if (pDataBlock == NULL) {
taosArrayDestroy(pArray);
return terrno;
}
int32_t code = blockDecode(pDataBlock, pRetrieve->data + PAYLOAD_PREFIX_LEN, &pDummy);
if (code) {
taosArrayDestroy(pArray);
return code;
}
// TODO: refactor
pDataBlock->info.window.skey = be64toh(pRetrieve->skey);
@ -132,7 +156,7 @@ int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock
pData->reqId = pReq->reqId;
pData->blocks = pArray;
return TSDB_CODE_SUCCESS;
return code;
}
int32_t streamDataSubmitNew(SPackedData* pData, int32_t type, SStreamDataSubmit** pSubmit) {
@ -178,7 +202,7 @@ int32_t streamMergedSubmitNew(SStreamMergedSubmit** pSubmit) {
int32_t streamMergeSubmit(SStreamMergedSubmit* pMerged, SStreamDataSubmit* pSubmit) {
void* p = taosArrayPush(pMerged->submits, &pSubmit->submit);
if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
return terrno;
}
if (pSubmit->ver > pMerged->ver) {
@ -260,8 +284,12 @@ void streamFreeQitem(SStreamQueueItem* data) {
int32_t sz = taosArrayGetSize(pMerge->submits);
for (int32_t i = 0; i < sz; i++) {
SPackedData* pSubmit = (SPackedData*)taosArrayGet(pMerge->submits, i);
if (pSubmit == NULL) {
continue;
}
taosMemoryFree(pSubmit->msgStr);
}
taosArrayDestroy(pMerge->submits);
taosFreeQitem(pMerge);
} else if (type == STREAM_INPUT__REF_DATA_BLOCK) {

82
source/libs/stream/test/backendTest.cpp
View File

@ -69,7 +69,8 @@ void *backendOpen() {
key.ts = ts;
const char *val = "value data";
int32_t vlen = strlen(val);
streamStatePut_rocksdb(p, &key, (char *)val, vlen);
int32_t code = streamStatePut_rocksdb(p, &key, (char *)val, vlen);
ASSERT(code == 0);
tsArray.push_back(ts);
}
@ -82,7 +83,9 @@ void *backendOpen() {
const char *val = "value data";
int32_t len = 0;
char *newVal = NULL;
streamStateGet_rocksdb(p, &key, (void **)&newVal, &len);
int32_t code = streamStateGet_rocksdb(p, &key, (void **)&newVal, &len);
ASSERT(code == 0);
ASSERT(len == strlen(val));
}
int64_t ts = tsArray[0];
@ -90,9 +93,11 @@ void *backendOpen() {
key.groupId = (uint64_t)(0);
key.ts = ts;
streamStateDel_rocksdb(p, &key);
int32_t code = streamStateDel_rocksdb(p, &key);
ASSERT(code == 0);
streamStateClear_rocksdb(p);
code = streamStateClear_rocksdb(p);
ASSERT(code == 0);
for (int i = 0; i < size; i++) {
int64_t ts = tsArray[i];
@ -118,11 +123,12 @@ void *backendOpen() {
const char *val = "value data";
int32_t vlen = strlen(val);
streamStatePut_rocksdb(p, &key, (char *)val, vlen);
code = streamStatePut_rocksdb(p, &key, (char *)val, vlen);
ASSERT(code == 0);
}
SWinKey winkey;
int32_t code = streamStateGetFirst_rocksdb(p, &key);
code = streamStateGetFirst_rocksdb(p, &key);
ASSERT(code == 0);
ASSERT(key.ts == tsArray[0]);
@ -151,7 +157,8 @@ void *backendOpen() {
const char *val = "Value";
int32_t len = strlen(val);
streamStateFuncPut_rocksdb(p, &key, val, len);
code = streamStateFuncPut_rocksdb(p, &key, val, len);
ASSERT(code == 0);
}
for (int i = 0; i < size; i++) {
STupleKey key = {0}; //{.groupId = (uint64_t)(0), .ts = tsArray[i], .exprIdx = i};
@ -161,7 +168,9 @@ void *backendOpen() {
char *val = NULL;
int32_t len = 0;
streamStateFuncGet_rocksdb(p, &key, (void **)&val, &len);
int32_t code = streamStateFuncGet_rocksdb(p, &key, (void **)&val, &len);
ASSERT(code == 0);
ASSERT(len == strlen("Value"));
}
for (int i = 0; i < size; i++) {
@ -172,7 +181,8 @@ void *backendOpen() {
char *val = NULL;
int32_t len = 0;
streamStateFuncDel_rocksdb(p, &key);
int32_t code = streamStateFuncDel_rocksdb(p, &key);
ASSERT(code == 0);
}
// session put
@ -187,7 +197,8 @@ void *backendOpen() {
const char *val = "Value";
int32_t len = strlen(val);
streamStateSessionPut_rocksdb(p, &key, val, len);
code = streamStateSessionPut_rocksdb(p, &key, val, len);
ASSERT(code == 0);
char *pval = NULL;
ASSERT(0 == streamStateSessionGet_rocksdb(p, &key, (void **)&pval, &len));
@ -346,7 +357,9 @@ void *backendOpen() {
ASSERT(code == 0);
}
SArray *result = taosArrayInit(8, sizeof(void *));
streamDefaultIterGet_rocksdb(p, "tbname", "tbname_99", result);
code = streamDefaultIterGet_rocksdb(p, "tbname", "tbname_99", result);
ASSERT(code == 0);
ASSERT(taosArrayGetSize(result) >= 0);
return p;
@ -363,10 +376,14 @@ TEST_F(BackendEnv, checkOpen) {
sprintf(key, "key_%d", i);
char val[128] = {0};
sprintf(val, "val_%d", i);
streamStatePutBatch(p, "default", (rocksdb_writebatch_t *)pBatch, (void *)key, (void *)val,
(int32_t)(strlen(val)), tsStart + 100000);
int32_t code = streamStatePutBatch(p, "default", (rocksdb_writebatch_t *)pBatch, (void *)key, (void *)val,
(int32_t)(strlen(val)), tsStart + 100000);
ASSERT(code == 0);
}
streamStatePutBatch_rocksdb(p, pBatch);
int32_t code = streamStatePutBatch_rocksdb(p, pBatch);
ASSERT(code == 0);
streamStateDestroyBatch(pBatch);
}
{
@ -378,14 +395,18 @@ TEST_F(BackendEnv, checkOpen) {
sprintf(key, "key_%d", i);
char val[128] = {0};
sprintf(val, "val_%d", i);
streamStatePutBatchOptimize(p, 0, (rocksdb_writebatch_t *)pBatch, (void *)key, (void *)val,
(int32_t)(strlen(val)), tsStart + 100000, (void *)valBuf);
int32_t code = streamStatePutBatchOptimize(p, 0, (rocksdb_writebatch_t *)pBatch, (void *)key, (void *)val,
(int32_t)(strlen(val)), tsStart + 100000, (void *)valBuf);
ASSERT(code == 0);
}
streamStatePutBatch_rocksdb(p, pBatch);
int32_t code = streamStatePutBatch_rocksdb(p, pBatch);
ASSERT(code == 0);
streamStateDestroyBatch(pBatch);
}
// do checkpoint 2
taskDbDoCheckpoint(p->pTdbState->pOwner->pBackend, 2, 0);
int32_t code = taskDbDoCheckpoint(p->pTdbState->pOwner->pBackend, 2, 0);
ASSERT(code == 0);
{
void *pBatch = streamStateCreateBatch();
int32_t size = 0;
@ -395,14 +416,18 @@ TEST_F(BackendEnv, checkOpen) {
sprintf(key, "key_%d", i);
char val[128] = {0};
sprintf(val, "val_%d", i);
streamStatePutBatchOptimize(p, 0, (rocksdb_writebatch_t *)pBatch, (void *)key, (void *)val,
(int32_t)(strlen(val)), tsStart + 100000, (void *)valBuf);
int32_t code = streamStatePutBatchOptimize(p, 0, (rocksdb_writebatch_t *)pBatch, (void *)key, (void *)val,
(int32_t)(strlen(val)), tsStart + 100000, (void *)valBuf);
ASSERT(code == 0);
}
streamStatePutBatch_rocksdb(p, pBatch);
code = streamStatePutBatch_rocksdb(p, pBatch);
ASSERT(code == 0);
streamStateDestroyBatch(pBatch);
}
taskDbDoCheckpoint(p->pTdbState->pOwner->pBackend, 3, 0);
code = taskDbDoCheckpoint(p->pTdbState->pOwner->pBackend, 3, 0);
ASSERT(code == 0);
const char *path = "/tmp/backend/stream";
const char *dump = "/tmp/backend/stream/dump";
@ -410,14 +435,17 @@ TEST_F(BackendEnv, checkOpen) {
taosMulMkDir(dump);
SBkdMgt *mgt = NULL;
int32_t code = bkdMgtCreate((char *)path, &mgt);
code = bkdMgtCreate((char *)path, &mgt);
SArray *result = taosArrayInit(4, sizeof(void *));
bkdMgtGetDelta(mgt, p->pTdbState->idstr, 3, result, (char *)dump);
taskDbDoCheckpoint(p->pTdbState->pOwner->pBackend, 4, 0);
code = taskDbDoCheckpoint(p->pTdbState->pOwner->pBackend, 4, 0);
ASSERT(code == 0);
taosArrayClear(result);
bkdMgtGetDelta(mgt, p->pTdbState->idstr, 4, result, (char *)dump);
code = bkdMgtGetDelta(mgt, p->pTdbState->idstr, 4, result, (char *)dump);
ASSERT(code == 0);
bkdMgtDestroy(mgt);
streamStateClose((SStreamState *)p, true);
// {
@ -446,7 +474,9 @@ TEST_F(BackendEnv, backendUtil) {
}
TEST_F(BackendEnv, oldBackendInit) {
const char *path = "/tmp/backend1";
taosMulMkDir(path);
int32_t code = taosMulMkDir(path);
ASSERT(code == 0);
{
SBackendWrapper *p = (SBackendWrapper *)streamBackendInit(path, 10, 10);
streamBackendCleanup((void *)p);

4
source/libs/sync/src/syncRespMgr.c
View File

@ -142,7 +142,9 @@ static int32_t syncRespCleanByTTL(SSyncRespMgr *pObj, int64_t ttl, bool rsp) {
int64_t nowMS = taosGetTimestampMs();
if (nowMS - pStub->createTime > ttl || -1 == ttl) {
taosArrayPush(delIndexArray, pSeqNum);
if (taosArrayPush(delIndexArray, pSeqNum) == NULL) {
return terrno;
}
cnt++;
SFsmCbMeta cbMeta = {

15
source/libs/tdb/src/db/tdbBtree.c
View File

@ -1366,11 +1366,6 @@ static int tdbBtreeDecodePayload(SPage *pPage, const SCell *pCell, int nHeader,
if (ret < 0) {
return ret;
}
/*
if (pDecoder->ofps) {
taosArrayPush(pDecoder->ofps, &ofp);
}
*/
ofpCell = tdbPageGetCell(ofp, 0);
if (nLeft <= ofp->maxLocal - sizeof(SPgno)) {
@ -1411,11 +1406,6 @@ static int tdbBtreeDecodePayload(SPage *pPage, const SCell *pCell, int nHeader,
if (ret < 0) {
return ret;
}
/*
if (pDecoder->ofps) {
taosArrayPush(pDecoder->ofps, &ofp);
}
*/
ofpCell = tdbPageGetCell(ofp, 0);
int lastKeyPage = 0;
@ -1642,7 +1632,10 @@ static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell, int dropOfp, TXN *
SArray *ofps = pPage->pPager->ofps;
if (ofps) {
taosArrayPush(ofps, &ofp);
if (taosArrayPush(ofps, &ofp) == NULL) {
ASSERT(0);
return terrno;
}
}
tdbPagerReturnPage(pPage->pPager, ofp, pTxn);

46
source/libs/transport/src/thttp.c
View File

@ -76,9 +76,9 @@ static void httpHandleReq(SHttpMsg* msg);
static void httpHandleQuit(SHttpMsg* msg);
static int32_t httpSendQuit(SHttpModule* http, int64_t chanId);
static int32_t httpCreateMsg(const char* server, const char* uri, uint16_t port, char* pCont, int32_t contLen,
EHttpCompFlag flag, int64_t chanId, SHttpMsg** httpMsg);
static void httpDestroyMsg(SHttpMsg* msg);
static int32_t httpCreateMsg(const char* server, const char* uri, uint16_t port, char* pCont, int32_t contLen,
EHttpCompFlag flag, int64_t chanId, SHttpMsg** httpMsg);
static void httpDestroyMsg(SHttpMsg* msg);
static bool httpFailFastShoudIgnoreMsg(SHashObj* pTable, char* server, int16_t port);
static void httpFailFastMayUpdate(SHashObj* pTable, char* server, int16_t port, int8_t succ);
@ -91,27 +91,27 @@ static int32_t taosSendHttpReportImplByChan(const char* server, const char* uri,
static int32_t taosBuildHttpHeader(const char* server, const char* uri, int32_t contLen, char* pHead, int32_t headLen,
EHttpCompFlag flag) {
int32_t code = 0;
EHttpCompFlag flag) {
int32_t code = 0;
int32_t len = 0;
if (flag == HTTP_FLAT) {
len = snprintf(pHead, headLen,
"POST %s HTTP/1.1\n"
"Host: %s\n"
"Content-Type: application/json\n"
"Content-Length: %d\n\n",
uri, server, contLen);
"POST %s HTTP/1.1\n"
"Host: %s\n"
"Content-Type: application/json\n"
"Content-Length: %d\n\n",
uri, server, contLen);
if (len < 0 || len >= headLen) {
code = TSDB_CODE_OUT_OF_RANGE;
}
} else if (flag == HTTP_GZIP) {
len = snprintf(pHead, headLen,
"POST %s HTTP/1.1\n"
"Host: %s\n"
"Content-Type: application/json\n"
"Content-Encoding: gzip\n"
"Content-Length: %d\n\n",
uri, server, contLen);
"POST %s HTTP/1.1\n"
"Host: %s\n"
"Content-Type: application/json\n"
"Content-Encoding: gzip\n"
"Content-Length: %d\n\n",
uri, server, contLen);
if (len < 0 || len >= headLen) {
code = TSDB_CODE_OUT_OF_RANGE;
}
@ -127,7 +127,7 @@ static int32_t taosCompressHttpRport(char* pSrc, int32_t srcLen) {
void* pDest = taosMemoryMalloc(destLen);
if (pDest == NULL) {
code= TSDB_CODE_OUT_OF_MEMORY;
code = TSDB_CODE_OUT_OF_MEMORY;
goto _OVER;
}
@ -184,14 +184,14 @@ _OVER:
if (code == 0) {
memcpy(pSrc, pDest, gzipStream.total_out);
code = gzipStream.total_out;
}
}
taosMemoryFree(pDest);
return code;
}
static FORCE_INLINE int32_t taosBuildDstAddr(const char* server, uint16_t port, struct sockaddr_in* dest) {
uint32_t ip = 0;
int32_t code = taosGetIpv4FromFqdn(server, &ip);
int32_t code = taosGetIpv4FromFqdn(server, &ip);
if (code) {
tError("http-report failed to resolving domain names: %s", server);
return TSDB_CODE_RPC_FQDN_ERROR;
@ -293,7 +293,7 @@ static void httpAsyncCb(uv_async_t* handle) {
static int32_t BATCH_SIZE = 5;
int32_t count = 0;
taosThreadMutexLock(&item->mtx);
(void)taosThreadMutexLock(&item->mtx);
httpMayDiscardMsg(http, item);
while (!QUEUE_IS_EMPTY(&item->qmsg) && count++ < BATCH_SIZE) {
@ -301,7 +301,7 @@ static void httpAsyncCb(uv_async_t* handle) {
QUEUE_REMOVE(h);
QUEUE_PUSH(&wq, h);
}
taosThreadMutexUnlock(&item->mtx);
(void)taosThreadMutexUnlock(&item->mtx);
while (!QUEUE_IS_EMPTY(&wq)) {
queue* h = QUEUE_HEAD(&wq);
@ -636,8 +636,8 @@ void httpModuleDestroy2(SHttpModule* http) {
static int32_t taosSendHttpReportImplByChan(const char* server, const char* uri, uint16_t port, char* pCont,
int32_t contLen, EHttpCompFlag flag, int64_t chanId) {
SHttpModule* load = NULL;
SHttpMsg *msg = NULL;
int32_t code = httpCreateMsg(server, uri, port, pCont, contLen, flag, chanId,&msg);
SHttpMsg* msg = NULL;
int32_t code = httpCreateMsg(server, uri, port, pCont, contLen, flag, chanId, &msg);
if (code != 0) {
goto _ERROR;
}

8
source/os/src/osSysinfo.c
View File

@ -582,22 +582,22 @@ _end:
#endif
}
int32_t taosGetCpuCores(float *numOfCores, bool physical) {
void taosGetCpuCores(float *numOfCores, bool physical) {
#ifdef WINDOWS
SYSTEM_INFO info;
GetSystemInfo(&info);
*numOfCores = info.dwNumberOfProcessors;
return 0;
return;
#elif defined(_TD_DARWIN_64)
*numOfCores = sysconf(_SC_NPROCESSORS_ONLN);
return 0;
return;
#else
if (physical) {
*numOfCores = sysconf(_SC_NPROCESSORS_ONLN);
} else {
(void)taosCntrGetCpuCores(numOfCores);
}
return 0;
return;
#endif
}

2
source/util/src/tarray.c
View File

@ -34,14 +34,12 @@ SArray* taosArrayInit(size_t size, size_t elemSize) {
SArray* pArray = taosMemoryMalloc(sizeof(SArray));
if (pArray == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
pArray->size = 0;
pArray->pData = taosMemoryCalloc(size, elemSize);
if (pArray->pData == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
taosMemoryFree(pArray);
return NULL;
}

Some files were not shown because too many files have changed in this diff Show More