other: merge 3.0

This commit is contained in:
Haojun Liao 2024-08-05 13:38:05 +08:00
commit e9ac0d6cd4
71 changed files with 1521 additions and 363 deletions

View File

@ -4,7 +4,6 @@ import jenkins.model.CauseOfInterruption
docs_only=0 docs_only=0
node { node {
} }
def abortPreviousBuilds() { def abortPreviousBuilds() {
def currentJobName = env.JOB_NAME def currentJobName = env.JOB_NAME
def currentBuildNumber = env.BUILD_NUMBER.toInteger() def currentBuildNumber = env.BUILD_NUMBER.toInteger()
@ -71,6 +70,7 @@ def check_docs() {
} else { } else {
echo file_changed echo file_changed
} }
env.FILE_CHANGED = file_changed
} }
} }
def pre_test(){ def pre_test(){
@ -137,7 +137,7 @@ def pre_test(){
''' '''
} else { } else {
sh ''' sh '''
echo "unmatched reposiotry ${CHANGE_URL}" echo "unmatched repository ${CHANGE_URL}"
''' '''
} }
sh ''' sh '''
@ -247,7 +247,7 @@ def pre_test_win(){
''' '''
} else { } else {
bat ''' bat '''
echo "unmatched reposiotry %CHANGE_URL%" echo "unmatched repository %CHANGE_URL%"
''' '''
} }
} }
@ -350,7 +350,6 @@ pipeline {
when { when {
allOf { allOf {
not { expression { env.CHANGE_BRANCH =~ /docs\// }} not { expression { env.CHANGE_BRANCH =~ /docs\// }}
not { expression { env.CHANGE_URL =~ /\/TDinternal\// }}
} }
} }
parallel { parallel {
@ -419,6 +418,10 @@ pipeline {
timeout(time: 200, unit: 'MINUTES'){ timeout(time: 200, unit: 'MINUTES'){
pre_test() pre_test()
script { script {
sh '''
mkdir -p ${WKDIR}/tmp/${BRANCH_NAME}_${BUILD_ID}
echo "''' + env.FILE_CHANGED + '''" > ${WKDIR}/tmp/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt
'''
sh ''' sh '''
date date
rm -rf ${WKC}/debug rm -rf ${WKC}/debug
@ -450,6 +453,10 @@ pipeline {
} }
} }
} }
sh '''
cd ${WKC}/tests/parallel_test
./run_scan_container.sh -d ${WKDIR} -b ${BRANCH_NAME}_${BUILD_ID} -f ${WKDIR}/tmp/${BRANCH_NAME}_${BUILD_ID}/docs_changed.txt ''' + extra_param + '''
'''
sh ''' sh '''
cd ${WKC}/tests/parallel_test cd ${WKC}/tests/parallel_test
export DEFAULT_RETRY_TIME=2 export DEFAULT_RETRY_TIME=2

View File

@ -16,6 +16,12 @@ if (NOT DEFINED TD_GRANT)
SET(TD_GRANT FALSE) SET(TD_GRANT FALSE)
endif() endif()
IF (NOT DEFINED BUILD_WITH_RAND_ERR)
SET(BUILD_WITH_RAND_ERR FALSE)
ELSE ()
SET(BUILD_WITH_RAND_ERR TRUE)
endif()
IF ("${WEBSOCKET}" MATCHES "true") IF ("${WEBSOCKET}" MATCHES "true")
SET(TD_WEBSOCKET TRUE) SET(TD_WEBSOCKET TRUE)
MESSAGE("Enable websocket") MESSAGE("Enable websocket")

View File

@ -146,6 +146,13 @@ option(
ENDIF () ENDIF ()
IF(NOT TD_ENTERPRISE)
MESSAGE("switch s3 off with community version")
set(BUILD_S3 OFF)
set(BUILD_WITH_S3 OFF)
set(BUILD_WITH_COS OFF)
ENDIF ()
IF(${BUILD_S3}) IF(${BUILD_S3})
IF(${BUILD_WITH_S3}) IF(${BUILD_WITH_S3})

View File

@ -102,6 +102,7 @@ The detailed beaviors of `NULL`, `NULL_F`, `VALUE`, and VALUE_F are described be
1. A huge volume of interpolation output may be returned using `FILL`, so it's recommended to specify the time range when using `FILL`. The maximum number of interpolation values that can be returned in a single query is 10,000,000. 1. A huge volume of interpolation output may be returned using `FILL`, so it's recommended to specify the time range when using `FILL`. The maximum number of interpolation values that can be returned in a single query is 10,000,000.
2. The result set is in ascending order of timestamp when you aggregate by time window. 2. The result set is in ascending order of timestamp when you aggregate by time window.
3. If aggregate by window is used on STable, the aggregate function is performed on all the rows matching the filter conditions. If `PARTITION BY` is not used in the query, the result set will be returned in strict ascending order of timestamp; otherwise the result set will be returned in the order of ascending timestamp in each group. 3. If aggregate by window is used on STable, the aggregate function is performed on all the rows matching the filter conditions. If `PARTITION BY` is not used in the query, the result set will be returned in strict ascending order of timestamp; otherwise the result set will be returned in the order of ascending timestamp in each group.
4. The output windows of Fill are related with time range of WHERE Clause. For asc fill, the first output window is the first window that conains the start time of WHERE clause. The last output window is the last window that contains the end time of WHERE clause.
::: :::

View File

@ -97,6 +97,7 @@ NULL, NULL_F, VALUE, VALUE_F 这几种填充模式针对不同场景区别如下
1. 使用 FILL 语句的时候可能生成大量的填充输出,务必指定查询的时间区间。针对每次查询,系统可返回不超过 1 千万条具有插值的结果。 1. 使用 FILL 语句的时候可能生成大量的填充输出,务必指定查询的时间区间。针对每次查询,系统可返回不超过 1 千万条具有插值的结果。
2. 在时间维度聚合中,返回的结果中时间序列严格单调递增。 2. 在时间维度聚合中,返回的结果中时间序列严格单调递增。
3. 如果查询对象是超级表,则聚合函数会作用于该超级表下满足值过滤条件的所有表的数据。如果查询中没有使用 PARTITION BY 语句,则返回的结果按照时间序列严格单调递增;如果查询中使用了 PARTITION BY 语句分组,则返回结果中每个 PARTITION 内按照时间序列严格单调递增。 3. 如果查询对象是超级表,则聚合函数会作用于该超级表下满足值过滤条件的所有表的数据。如果查询中没有使用 PARTITION BY 语句,则返回的结果按照时间序列严格单调递增;如果查询中使用了 PARTITION BY 语句分组,则返回结果中每个 PARTITION 内按照时间序列严格单调递增。
4. Fill输出的起始和结束窗口与WHERE条件的时间范围有关, 如增序Fill时, 第一个窗口是包含WHERE条件开始时间的第一个窗口, 最后一个窗口是包含WHERE条件结束时间的最后一个窗口。
::: :::

View File

@ -1847,6 +1847,7 @@ typedef struct SMStreamDropOrphanMsg {
int32_t tSerializeDropOrphanTaskMsg(void* buf, int32_t bufLen, SMStreamDropOrphanMsg* pMsg); int32_t tSerializeDropOrphanTaskMsg(void* buf, int32_t bufLen, SMStreamDropOrphanMsg* pMsg);
int32_t tDeserializeDropOrphanTaskMsg(void* buf, int32_t bufLen, SMStreamDropOrphanMsg* pMsg); int32_t tDeserializeDropOrphanTaskMsg(void* buf, int32_t bufLen, SMStreamDropOrphanMsg* pMsg);
void tDestroyDropOrphanTaskMsg(SMStreamDropOrphanMsg* pMsg);
typedef struct { typedef struct {
int32_t id; int32_t id;

View File

@ -534,7 +534,7 @@ void tFreeStreamTask(SStreamTask* pTask);
int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask); int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask);
int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask); int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask);
int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, int64_t ver); int32_t streamTaskInit(SStreamTask* pTask, SStreamMeta* pMeta, SMsgCb* pMsgCb, int64_t ver);
void streamFreeTaskState(SStreamTask* pTask, ETaskStatus status); void streamFreeTaskState(SStreamTask* pTask, int8_t remove);
int32_t tDecodeStreamTaskChkInfo(SDecoder* pDecoder, SCheckpointInfo* pChkpInfo); int32_t tDecodeStreamTaskChkInfo(SDecoder* pDecoder, SCheckpointInfo* pChkpInfo);
int32_t tDecodeStreamTaskId(SDecoder* pDecoder, STaskId* pTaskId); int32_t tDecodeStreamTaskId(SDecoder* pDecoder, STaskId* pTaskId);
@ -647,6 +647,7 @@ const char* streamTaskGetStatusStr(ETaskStatus status);
void streamTaskResetStatus(SStreamTask* pTask); void streamTaskResetStatus(SStreamTask* pTask);
void streamTaskSetStatusReady(SStreamTask* pTask); void streamTaskSetStatusReady(SStreamTask* pTask);
ETaskStatus streamTaskGetPrevStatus(const SStreamTask* pTask); ETaskStatus streamTaskGetPrevStatus(const SStreamTask* pTask);
const char* streamTaskGetExecType(int32_t type);
bool streamTaskUpdateEpsetInfo(SStreamTask* pTask, SArray* pNodeList); bool streamTaskUpdateEpsetInfo(SStreamTask* pTask, SArray* pNodeList);
void streamTaskResetUpstreamStageInfo(SStreamTask* pTask); void streamTaskResetUpstreamStageInfo(SStreamTask* pTask);

View File

@ -124,6 +124,9 @@ extern "C" {
#include "taoserror.h" #include "taoserror.h"
#include "tlog.h" #include "tlog.h"
extern int32_t tsRandErrChance;
extern threadlocal bool tsEnableRandErr;
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

View File

@ -188,9 +188,10 @@ void syslog(int unused, const char *format, ...);
#define ALIGN8(n) ALIGN_NUM(n, 8) #define ALIGN8(n) ALIGN_NUM(n, 8)
#undef threadlocal #undef threadlocal
#ifdef _ISOC11_SOURCE //#ifdef _ISOC11_SOURCE
#define threadlocal _Thread_local //#define threadlocal _Thread_local
#elif defined(__APPLE__) //#elif defined(__APPLE__)
#if defined(__APPLE__)
#define threadlocal __thread #define threadlocal __thread
#elif defined(__GNUC__) && !defined(threadlocal) #elif defined(__GNUC__) && !defined(threadlocal)
#define threadlocal __thread #define threadlocal __thread

View File

@ -488,6 +488,7 @@ int32_t taosGetErrSize();
//mnode-compact //mnode-compact
#define TSDB_CODE_MND_INVALID_COMPACT_ID TAOS_DEF_ERROR_CODE(0, 0x04B1) #define TSDB_CODE_MND_INVALID_COMPACT_ID TAOS_DEF_ERROR_CODE(0, 0x04B1)
#define TSDB_CODE_MND_COMPACT_DETAIL_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x04B2) #define TSDB_CODE_MND_COMPACT_DETAIL_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x04B2)
#define TSDB_CODE_MND_COMPACT_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x04B3)
// vnode // vnode
// #define TSDB_CODE_VND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0500) // 2.x // #define TSDB_CODE_VND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0500) // 2.x

View File

@ -373,6 +373,7 @@ int taos_options_imp(TSDB_OPTION option, const char* str);
int32_t openTransporter(const char* user, const char* auth, int32_t numOfThreads, void **pDnodeConn); int32_t openTransporter(const char* user, const char* auth, int32_t numOfThreads, void **pDnodeConn);
void tscStopCrashReport(); void tscStopCrashReport();
void cleanupAppInfo();
typedef struct AsyncArg { typedef struct AsyncArg {
SRpcMsg msg; SRpcMsg msg;

View File

@ -688,7 +688,6 @@ void doDestroyRequest(void *p) {
taosArrayDestroy(pRequest->tableList); taosArrayDestroy(pRequest->tableList);
taosArrayDestroy(pRequest->targetTableList); taosArrayDestroy(pRequest->targetTableList);
destroyQueryExecRes(&pRequest->body.resInfo.execRes); destroyQueryExecRes(&pRequest->body.resInfo.execRes);
if (pRequest->self) { if (pRequest->self) {
@ -702,12 +701,7 @@ void doDestroyRequest(void *p) {
} }
taosMemoryFree(pRequest->body.interParam); taosMemoryFree(pRequest->body.interParam);
if (TSDB_CODE_SUCCESS == nodesSimAcquireAllocator(pRequest->allocatorRefId)) {
qDestroyQuery(pRequest->pQuery); qDestroyQuery(pRequest->pQuery);
if (TSDB_CODE_SUCCESS != nodesSimReleaseAllocator(pRequest->allocatorRefId)) {
tscError("failed to release allocator");
}
}
nodesDestroyAllocator(pRequest->allocatorRefId); nodesDestroyAllocator(pRequest->allocatorRefId);
taosMemoryFreeClear(pRequest->effectiveUser); taosMemoryFreeClear(pRequest->effectiveUser);

View File

@ -68,6 +68,10 @@ bool chkRequestKilled(void* param) {
return killed; return killed;
} }
void cleanupAppInfo() {
taosHashCleanup(appInfo.pInstMap);
}
static int32_t taosConnectImpl(const char* user, const char* auth, const char* db, __taos_async_fn_t fp, void* param, static int32_t taosConnectImpl(const char* user, const char* auth, const char* db, __taos_async_fn_t fp, void* param,
SAppInstInfo* pAppInfo, int connType, STscObj** pTscObj); SAppInstInfo* pAppInfo, int connType, STscObj** pTscObj);

View File

@ -67,7 +67,6 @@ void taos_cleanup(void) {
fmFuncMgtDestroy(); fmFuncMgtDestroy();
qCleanupKeywordsTable(); qCleanupKeywordsTable();
nodesDestroyAllocatorSet();
if (TSDB_CODE_SUCCESS != cleanupTaskQueue()) { if (TSDB_CODE_SUCCESS != cleanupTaskQueue()) {
tscWarn("failed to cleanup task queue"); tscWarn("failed to cleanup task queue");
@ -85,6 +84,8 @@ void taos_cleanup(void) {
tscWarn("failed to close clientReqRefPool"); tscWarn("failed to close clientReqRefPool");
} }
nodesDestroyAllocatorSet();
cleanupAppInfo();
rpcCleanup(); rpcCleanup();
tscDebug("rpc cleanup"); tscDebug("rpc cleanup");

View File

@ -644,6 +644,10 @@ int32_t blockDataUpdatePkRange(SSDataBlock* pDataBlock, int32_t pkColumnIndex, b
SDataBlockInfo* pInfo = &pDataBlock->info; SDataBlockInfo* pInfo = &pDataBlock->info;
SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, pkColumnIndex); SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, pkColumnIndex);
if (pColInfoData == NULL) {
return terrno;
}
if (!IS_NUMERIC_TYPE(pColInfoData->info.type) && (pColInfoData->info.type != TSDB_DATA_TYPE_VARCHAR)) { if (!IS_NUMERIC_TYPE(pColInfoData->info.type) && (pColInfoData->info.type != TSDB_DATA_TYPE_VARCHAR)) {
return 0; return 0;
} }
@ -685,6 +689,9 @@ int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc) {
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pCol2 = taosArrayGet(pDest->pDataBlock, i); SColumnInfoData* pCol2 = taosArrayGet(pDest->pDataBlock, i);
SColumnInfoData* pCol1 = taosArrayGet(pSrc->pDataBlock, i); SColumnInfoData* pCol1 = taosArrayGet(pSrc->pDataBlock, i);
if (pCol1 == NULL || pCol2 == NULL) {
return terrno;
}
capacity = pDest->info.capacity; capacity = pDest->info.capacity;
int32_t ret = colDataMergeCol(pCol2, pDest->info.rows, &capacity, pCol1, pSrc->info.rows); int32_t ret = colDataMergeCol(pCol2, pDest->info.rows, &capacity, pCol1, pSrc->info.rows);
@ -709,6 +716,9 @@ int32_t blockDataMergeNRows(SSDataBlock* pDest, const SSDataBlock* pSrc, int32_t
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pCol2 = taosArrayGet(pDest->pDataBlock, i); SColumnInfoData* pCol2 = taosArrayGet(pDest->pDataBlock, i);
SColumnInfoData* pCol1 = taosArrayGet(pSrc->pDataBlock, i); SColumnInfoData* pCol1 = taosArrayGet(pSrc->pDataBlock, i);
if (pCol2 == NULL || pCol1 == NULL) {
return terrno;
}
code = colDataAssignNRows(pCol2, pDest->info.rows, pCol1, srcIdx, numOfRows); code = colDataAssignNRows(pCol2, pDest->info.rows, pCol1, srcIdx, numOfRows);
if (code) { if (code) {
@ -729,6 +739,10 @@ void blockDataShrinkNRows(SSDataBlock* pBlock, int32_t numOfRows) {
size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i);
if (pCol == NULL) {
continue;
}
if (IS_VAR_DATA_TYPE(pCol->info.type)) { if (IS_VAR_DATA_TYPE(pCol->info.type)) {
pCol->varmeta.length = pCol->varmeta.offset[pBlock->info.rows - numOfRows]; pCol->varmeta.length = pCol->varmeta.offset[pBlock->info.rows - numOfRows];
memset(pCol->varmeta.offset + pBlock->info.rows - numOfRows, 0, sizeof(*pCol->varmeta.offset) * numOfRows); memset(pCol->varmeta.offset + pBlock->info.rows - numOfRows, 0, sizeof(*pCol->varmeta.offset) * numOfRows);
@ -760,6 +774,10 @@ size_t blockDataGetSize(const SSDataBlock* pBlock) {
size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
if (pColInfoData == NULL) {
continue;
}
total += colDataGetFullLength(pColInfoData, pBlock->info.rows); total += colDataGetFullLength(pColInfoData, pBlock->info.rows);
} }
@ -861,6 +879,10 @@ int32_t blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int32_t r
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i);
SColumnInfoData* pDstCol = taosArrayGet(pDst->pDataBlock, i); SColumnInfoData* pDstCol = taosArrayGet(pDst->pDataBlock, i);
if (pColData == NULL || pDstCol == NULL) {
continue;
}
for (int32_t j = startIndex; j < (startIndex + rowCount); ++j) { for (int32_t j = startIndex; j < (startIndex + rowCount); ++j) {
bool isNull = false; bool isNull = false;
if (pBlock->pBlockAgg == NULL) { if (pBlock->pBlockAgg == NULL) {
@ -908,6 +930,10 @@ int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock) {
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i);
if (pCol == NULL) {
continue;
}
if (IS_VAR_DATA_TYPE(pCol->info.type)) { if (IS_VAR_DATA_TYPE(pCol->info.type)) {
memcpy(pStart, pCol->varmeta.offset, numOfRows * sizeof(int32_t)); memcpy(pStart, pCol->varmeta.offset, numOfRows * sizeof(int32_t));
pStart += numOfRows * sizeof(int32_t); pStart += numOfRows * sizeof(int32_t);
@ -958,6 +984,9 @@ int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) {
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i);
if (pCol == NULL) {
continue;
}
if (IS_VAR_DATA_TYPE(pCol->info.type)) { if (IS_VAR_DATA_TYPE(pCol->info.type)) {
size_t metaSize = pBlock->info.rows * sizeof(int32_t); size_t metaSize = pBlock->info.rows * sizeof(int32_t);
@ -965,6 +994,7 @@ int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) {
if (tmp == NULL) { if (tmp == NULL) {
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
pCol->varmeta.offset = (int32_t*)tmp; pCol->varmeta.offset = (int32_t*)tmp;
memcpy(pCol->varmeta.offset, pStart, metaSize); memcpy(pCol->varmeta.offset, pStart, metaSize);
pStart += metaSize; pStart += metaSize;
@ -1039,6 +1069,10 @@ int32_t blockDataFromBuf1(SSDataBlock* pBlock, const char* buf, size_t capacity)
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i);
if (pCol == NULL) {
continue;
}
pCol->hasNull = true; pCol->hasNull = true;
if (IS_VAR_DATA_TYPE(pCol->info.type)) { if (IS_VAR_DATA_TYPE(pCol->info.type)) {
@ -1087,6 +1121,10 @@ size_t blockDataGetRowSize(SSDataBlock* pBlock) {
size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);
if (pColInfo == NULL) {
continue;
}
rowSize += pColInfo->info.bytes; rowSize += pColInfo->info.bytes;
} }
@ -1114,8 +1152,11 @@ double blockDataGetSerialRowSize(const SSDataBlock* pBlock) {
size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);
rowSize += pColInfo->info.bytes; if (pColInfo == NULL) {
continue;
}
rowSize += pColInfo->info.bytes;
if (IS_VAR_DATA_TYPE(pColInfo->info.type)) { if (IS_VAR_DATA_TYPE(pColInfo->info.type)) {
rowSize += sizeof(int32_t); rowSize += sizeof(int32_t);
} else { } else {
@ -1193,6 +1234,9 @@ static int32_t blockDataAssign(SColumnInfoData* pCols, const SSDataBlock* pDataB
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pDst = &pCols[i]; SColumnInfoData* pDst = &pCols[i];
SColumnInfoData* pSrc = taosArrayGet(pDataBlock->pDataBlock, i); SColumnInfoData* pSrc = taosArrayGet(pDataBlock->pDataBlock, i);
if (pSrc == NULL) {
continue;
}
if (IS_VAR_DATA_TYPE(pSrc->info.type)) { if (IS_VAR_DATA_TYPE(pSrc->info.type)) {
if (pSrc->varmeta.length != 0) { if (pSrc->varmeta.length != 0) {
@ -1228,8 +1272,11 @@ static SColumnInfoData* createHelpColInfoData(const SSDataBlock* pDataBlock) {
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, i); SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, i);
pCols[i].info = pColInfoData->info; if (pColInfoData == NULL) {
continue;
}
pCols[i].info = pColInfoData->info;
if (IS_VAR_DATA_TYPE(pCols[i].info.type)) { if (IS_VAR_DATA_TYPE(pCols[i].info.type)) {
pCols[i].varmeta.offset = taosMemoryCalloc(rows, sizeof(int32_t)); pCols[i].varmeta.offset = taosMemoryCalloc(rows, sizeof(int32_t));
pCols[i].pData = taosMemoryCalloc(1, pColInfoData->varmeta.length); pCols[i].pData = taosMemoryCalloc(1, pColInfoData->varmeta.length);
@ -1256,8 +1303,11 @@ static void copyBackToBlock(SSDataBlock* pDataBlock, SColumnInfoData* pCols) {
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, i); SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, i);
pColInfoData->info = pCols[i].info; if (pColInfoData == NULL) {
continue;
}
pColInfoData->info = pCols[i].info;
if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) {
taosMemoryFreeClear(pColInfoData->varmeta.offset); taosMemoryFreeClear(pColInfoData->varmeta.offset);
pColInfoData->varmeta = pCols[i].varmeta; pColInfoData->varmeta = pCols[i].varmeta;
@ -1301,8 +1351,15 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) {
for (int32_t i = 0; i < taosArrayGetSize(pOrderInfo); ++i) { for (int32_t i = 0; i < taosArrayGetSize(pOrderInfo); ++i) {
SBlockOrderInfo* pInfo = taosArrayGet(pOrderInfo, i); SBlockOrderInfo* pInfo = taosArrayGet(pOrderInfo, i);
if (pInfo == NULL) {
continue;
}
SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, pInfo->slotId); SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, pInfo->slotId);
if (pColInfoData == NULL) {
continue;
}
if (pColInfoData->hasNull) { if (pColInfoData->hasNull) {
sortColumnHasNull = true; sortColumnHasNull = true;
} }
@ -1319,6 +1376,9 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) {
if (!varTypeSort) { if (!varTypeSort) {
SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, 0); SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, 0);
SBlockOrderInfo* pOrder = taosArrayGet(pOrderInfo, 0); SBlockOrderInfo* pOrder = taosArrayGet(pOrderInfo, 0);
if (pColInfoData == NULL || pOrder == NULL) {
return errno;
}
int64_t p0 = taosGetTimestampUs(); int64_t p0 = taosGetTimestampUs();
@ -1346,7 +1406,14 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) {
SSDataBlockSortHelper helper = {.pDataBlock = pDataBlock, .orderInfo = pOrderInfo}; SSDataBlockSortHelper helper = {.pDataBlock = pDataBlock, .orderInfo = pOrderInfo};
for (int32_t i = 0; i < taosArrayGetSize(helper.orderInfo); ++i) { for (int32_t i = 0; i < taosArrayGetSize(helper.orderInfo); ++i) {
struct SBlockOrderInfo* pInfo = taosArrayGet(helper.orderInfo, i); struct SBlockOrderInfo* pInfo = taosArrayGet(helper.orderInfo, i);
if (pInfo == NULL) {
continue;
}
pInfo->pColData = taosArrayGet(pDataBlock->pDataBlock, pInfo->slotId); pInfo->pColData = taosArrayGet(pDataBlock->pDataBlock, pInfo->slotId);
if (pInfo->pColData == NULL) {
continue;
}
pInfo->compFn = getKeyComparFunc(pInfo->pColData->info.type, pInfo->order); pInfo->compFn = getKeyComparFunc(pInfo->pColData->info.type, pInfo->order);
} }
@ -1399,6 +1466,10 @@ void blockDataEmpty(SSDataBlock* pDataBlock) {
size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i);
if (p == NULL) {
continue;
}
colInfoDataCleanup(p, pInfo->capacity); colInfoDataCleanup(p, pInfo->capacity);
} }
@ -1417,6 +1488,10 @@ void blockDataReset(SSDataBlock* pDataBlock) {
size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i);
if (p == NULL) {
continue;
}
p->hasNull = false; p->hasNull = false;
p->reassigned = false; p->reassigned = false;
if (IS_VAR_DATA_TYPE(p->info.type)) { if (IS_VAR_DATA_TYPE(p->info.type)) {
@ -1527,6 +1602,10 @@ int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) {
size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i);
if (p == NULL) {
return terrno;
}
code = doEnsureCapacity(p, &pDataBlock->info, numOfRows, false); code = doEnsureCapacity(p, &pDataBlock->info, numOfRows, false);
if (code) { if (code) {
return code; return code;
@ -1544,6 +1623,10 @@ void blockDataFreeRes(SSDataBlock* pBlock) {
int32_t numOfOutput = taosArrayGetSize(pBlock->pDataBlock); int32_t numOfOutput = taosArrayGetSize(pBlock->pDataBlock);
for (int32_t i = 0; i < numOfOutput; ++i) { for (int32_t i = 0; i < numOfOutput; ++i) {
SColumnInfoData* pColInfoData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pColInfoData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, i);
if (pColInfoData == NULL) {
continue;
}
colDataDestroy(pColInfoData); colDataDestroy(pColInfoData);
} }
@ -1579,6 +1662,10 @@ int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src) {
size_t numOfCols = taosArrayGetSize(src->pDataBlock); size_t numOfCols = taosArrayGetSize(src->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* p = taosArrayGet(src->pDataBlock, i); SColumnInfoData* p = taosArrayGet(src->pDataBlock, i);
if (p == NULL) {
return terrno;
}
SColumnInfoData colInfo = {.hasNull = true, .info = p->info}; SColumnInfoData colInfo = {.hasNull = true, .info = p->info};
code = blockDataAppendColInfo(dst, &colInfo); code = blockDataAppendColInfo(dst, &colInfo);
if (code) { if (code) {
@ -1594,7 +1681,7 @@ int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src) {
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pDst = taosArrayGet(dst->pDataBlock, i); SColumnInfoData* pDst = taosArrayGet(dst->pDataBlock, i);
SColumnInfoData* pSrc = taosArrayGet(src->pDataBlock, i); SColumnInfoData* pSrc = taosArrayGet(src->pDataBlock, i);
if (pSrc->pData == NULL && (!IS_VAR_DATA_TYPE(pSrc->info.type))) { if (pSrc == NULL || pDst == NULL || (pSrc->pData == NULL && (!IS_VAR_DATA_TYPE(pSrc->info.type)))) {
continue; continue;
} }
@ -1622,6 +1709,10 @@ int32_t copyDataBlock(SSDataBlock* pDst, const SSDataBlock* pSrc) {
for (int32_t i = 0; i < numOfCols; ++i) { for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pDstCol = taosArrayGet(pDst->pDataBlock, i); SColumnInfoData* pDstCol = taosArrayGet(pDst->pDataBlock, i);
SColumnInfoData* pSrcCol = taosArrayGet(pSrc->pDataBlock, i); SColumnInfoData* pSrcCol = taosArrayGet(pSrc->pDataBlock, i);
if (pDstCol == NULL || pSrcCol == NULL) {
continue;
}
int32_t ret = colDataAssign(pDstCol, pSrcCol, pSrc->info.rows, &pSrc->info); int32_t ret = colDataAssign(pDstCol, pSrcCol, pSrc->info.rows, &pSrc->info);
if (ret < 0) { if (ret < 0) {
code = ret; code = ret;
@ -3149,15 +3240,26 @@ int32_t blockDataGetSortedRows(SSDataBlock* pDataBlock, SArray* pOrderInfo) {
if (!pDataBlock || !pOrderInfo) return 0; if (!pDataBlock || !pOrderInfo) return 0;
for (int32_t i = 0; i < taosArrayGetSize(pOrderInfo); ++i) { for (int32_t i = 0; i < taosArrayGetSize(pOrderInfo); ++i) {
SBlockOrderInfo* pOrder = taosArrayGet(pOrderInfo, i); SBlockOrderInfo* pOrder = taosArrayGet(pOrderInfo, i);
if (pOrder == NULL) {
continue;
}
pOrder->pColData = taosArrayGet(pDataBlock->pDataBlock, pOrder->slotId); pOrder->pColData = taosArrayGet(pDataBlock->pDataBlock, pOrder->slotId);
if (pOrder->pColData == NULL) {
continue;
}
pOrder->compFn = getKeyComparFunc(pOrder->pColData->info.type, pOrder->order); pOrder->compFn = getKeyComparFunc(pOrder->pColData->info.type, pOrder->order);
} }
SSDataBlockSortHelper sortHelper = {.orderInfo = pOrderInfo, .pDataBlock = pDataBlock}; SSDataBlockSortHelper sortHelper = {.orderInfo = pOrderInfo, .pDataBlock = pDataBlock};
int32_t rowIdx = 0, nextRowIdx = 1; int32_t rowIdx = 0, nextRowIdx = 1;
for (; rowIdx < pDataBlock->info.rows && nextRowIdx < pDataBlock->info.rows; ++rowIdx, ++nextRowIdx) { for (; rowIdx < pDataBlock->info.rows && nextRowIdx < pDataBlock->info.rows; ++rowIdx, ++nextRowIdx) {
if (dataBlockCompar(&nextRowIdx, &rowIdx, &sortHelper) < 0) { if (dataBlockCompar(&nextRowIdx, &rowIdx, &sortHelper) < 0) {
break; break;
} }
} }
return nextRowIdx; return nextRowIdx;
} }

View File

@ -319,6 +319,7 @@ int32_t tsMaxTsmaNum = 3;
int32_t tsMaxTsmaCalcDelay = 600; int32_t tsMaxTsmaCalcDelay = 600;
int64_t tsmaDataDeleteMark = 1000 * 60 * 60 * 24; // in ms, default to 1d int64_t tsmaDataDeleteMark = 1000 * 60 * 60 * 24; // in ms, default to 1d
#define TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, pName) \ #define TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, pName) \
if ((pItem = cfgGetItem(pCfg, pName)) == NULL) { \ if ((pItem = cfgGetItem(pCfg, pName)) == NULL) { \
TAOS_RETURN(TSDB_CODE_CFG_NOT_FOUND); \ TAOS_RETURN(TSDB_CODE_CFG_NOT_FOUND); \
@ -732,6 +733,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "transPullupInterval", tsTransPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "transPullupInterval", tsTransPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "compactPullupInterval", tsCompactPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "compactPullupInterval", tsCompactPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "mqRebalanceInterval", tsMqRebalanceInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "mqRebalanceInterval", tsMqRebalanceInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "randErrorChance", tsRandErrChance, 0, 10000, CFG_SCOPE_BOTH, CFG_DYN_NONE));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "ttlUnit", tsTtlUnit, 1, 86400 * 365, CFG_SCOPE_SERVER, CFG_DYN_NONE)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "ttlUnit", tsTtlUnit, 1, 86400 * 365, CFG_SCOPE_SERVER, CFG_DYN_NONE));
TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "ttlPushInterval", tsTtlPushIntervalSec, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER)); TAOS_CHECK_RETURN(cfgAddInt32(pCfg, "ttlPushInterval", tsTtlPushIntervalSec, 1, 100000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER));
@ -1212,7 +1214,17 @@ static int32_t taosSetSystemCfg(SConfig *pCfg) {
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "charset"); TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "charset");
const char *charset = pItem->str; const char *charset = pItem->str;
(void)taosSetSystemLocale(locale, charset); // ignore this error temporarily int32_t code = taosSetSystemLocale(locale, charset);
if (TSDB_CODE_SUCCESS != code) {
uInfo("failed to set locale %s, since: %s", locale, tstrerror(code));
char curLocale[TD_LOCALE_LEN] = {0};
char curCharset[TD_CHARSET_LEN] = {0};
taosGetSystemLocale(curLocale, curCharset);
if (0 != strlen(curLocale) && 0 != strlen(curCharset)) {
uInfo("current locale: %s, charset: %s", curLocale, curCharset);
}
}
osSetSystemLocale(locale, charset); osSetSystemLocale(locale, charset);
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "enableCoreFile"); TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "enableCoreFile");
@ -1408,6 +1420,9 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "mqRebalanceInterval"); TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "mqRebalanceInterval");
tsMqRebalanceInterval = pItem->i32; tsMqRebalanceInterval = pItem->i32;
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "randErrorChance");
tsRandErrChance = pItem->i32;
TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "ttlUnit"); TAOS_CHECK_GET_CFG_ITEM(pCfg, pItem, "ttlUnit");
tsTtlUnit = pItem->i32; tsTtlUnit = pItem->i32;
@ -1669,25 +1684,28 @@ static int32_t cfgInitWrapper(SConfig **pCfg) {
} }
TAOS_RETURN(TSDB_CODE_SUCCESS); TAOS_RETURN(TSDB_CODE_SUCCESS);
} }
int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs, int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs,
bool tsc) { bool tsc) {
if (tsCfg != NULL) TAOS_RETURN(TSDB_CODE_SUCCESS); if (tsCfg != NULL) TAOS_RETURN(TSDB_CODE_SUCCESS);
TAOS_CHECK_RETURN(cfgInitWrapper(&tsCfg)); int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = -1;
TAOS_CHECK_GOTO(cfgInitWrapper(&tsCfg), &lino, _exit);
if (tsc) { if (tsc) {
TAOS_CHECK_RETURN(taosAddClientCfg(tsCfg)); TAOS_CHECK_GOTO(taosAddClientCfg(tsCfg), &lino, _exit);
TAOS_CHECK_RETURN(taosAddClientLogCfg(tsCfg)); TAOS_CHECK_GOTO(taosAddClientLogCfg(tsCfg), &lino, _exit);
} else { } else {
TAOS_CHECK_RETURN(taosAddClientCfg(tsCfg)); TAOS_CHECK_GOTO(taosAddClientCfg(tsCfg), &lino, _exit);
TAOS_CHECK_RETURN(taosAddServerCfg(tsCfg)); TAOS_CHECK_GOTO(taosAddServerCfg(tsCfg), &lino, _exit);
TAOS_CHECK_RETURN(taosAddClientLogCfg(tsCfg)); TAOS_CHECK_GOTO(taosAddClientLogCfg(tsCfg), &lino, _exit);
TAOS_CHECK_RETURN(taosAddServerLogCfg(tsCfg)); TAOS_CHECK_GOTO(taosAddServerLogCfg(tsCfg), &lino, _exit);
} }
TAOS_CHECK_RETURN(taosAddSystemCfg(tsCfg)); TAOS_CHECK_GOTO(taosAddSystemCfg(tsCfg), &lino, _exit);
int32_t code = TSDB_CODE_SUCCESS;
if ((code = taosLoadCfg(tsCfg, envCmd, cfgDir, envFile, apolloUrl)) != 0) { if ((code = taosLoadCfg(tsCfg, envCmd, cfgDir, envFile, apolloUrl)) != 0) {
uError("failed to load cfg since %s", tstrerror(code)); uError("failed to load cfg since %s", tstrerror(code));
cfgCleanup(tsCfg); cfgCleanup(tsCfg);
@ -1703,31 +1721,38 @@ int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile
} }
if (tsc) { if (tsc) {
TAOS_CHECK_RETURN(taosSetClientCfg(tsCfg)); TAOS_CHECK_GOTO(taosSetClientCfg(tsCfg), &lino, _exit);
} else { } else {
TAOS_CHECK_RETURN(taosSetClientCfg(tsCfg)); TAOS_CHECK_GOTO(taosSetClientCfg(tsCfg), &lino, _exit);
TAOS_CHECK_RETURN(taosUpdateServerCfg(tsCfg)); TAOS_CHECK_GOTO(taosUpdateServerCfg(tsCfg), &lino, _exit);
TAOS_CHECK_RETURN(taosSetServerCfg(tsCfg)); TAOS_CHECK_GOTO(taosSetServerCfg(tsCfg), &lino, _exit);
TAOS_CHECK_RETURN(taosSetReleaseCfg(tsCfg)); TAOS_CHECK_GOTO(taosSetReleaseCfg(tsCfg), &lino, _exit);
TAOS_CHECK_RETURN(taosSetTfsCfg(tsCfg)); TAOS_CHECK_GOTO(taosSetTfsCfg(tsCfg), &lino, _exit);
TAOS_CHECK_RETURN(taosSetS3Cfg(tsCfg)); TAOS_CHECK_GOTO(taosSetS3Cfg(tsCfg), &lino, _exit);
} }
TAOS_CHECK_RETURN(taosSetSystemCfg(tsCfg)); TAOS_CHECK_GOTO(taosSetSystemCfg(tsCfg), &lino, _exit);
TAOS_CHECK_RETURN(taosSetFileHandlesLimit()); TAOS_CHECK_GOTO(taosSetFileHandlesLimit(), &lino, _exit);
SConfigItem *pItem = cfgGetItem(tsCfg, "debugFlag"); SConfigItem *pItem = cfgGetItem(tsCfg, "debugFlag");
if (NULL == pItem) { if (NULL == pItem) {
uError("debugFlag not found in cfg"); uError("debugFlag not found in cfg");
TAOS_RETURN(TSDB_CODE_CFG_NOT_FOUND); TAOS_RETURN(TSDB_CODE_CFG_NOT_FOUND);
} }
TAOS_CHECK_RETURN(taosSetAllDebugFlag(tsCfg, pItem->i32)); TAOS_CHECK_GOTO(taosSetAllDebugFlag(tsCfg, pItem->i32), &lino, _exit);
cfgDumpCfg(tsCfg, tsc, false); cfgDumpCfg(tsCfg, tsc, false);
TAOS_CHECK_RETURN(taosCheckGlobalCfg()); TAOS_CHECK_GOTO(taosCheckGlobalCfg(), &lino, _exit);
TAOS_RETURN(TSDB_CODE_SUCCESS); _exit:
if (TSDB_CODE_SUCCESS != code) {
cfgCleanup(tsCfg);
tsCfg = NULL;
uError("failed to init cfg at %d since %s", lino, tstrerror(code));
}
TAOS_RETURN(code);
} }
void taosCleanupCfg() { void taosCleanupCfg() {

View File

@ -5360,6 +5360,14 @@ int32_t tDeserializeDropOrphanTaskMsg(void* buf, int32_t bufLen, SMStreamDropOrp
return 0; return 0;
} }
void tDestroyDropOrphanTaskMsg(SMStreamDropOrphanMsg *pMsg) {
if (pMsg == NULL) {
return;
}
taosArrayDestroy(pMsg->pList);
}
int32_t tEncodeSReplica(SEncoder *pEncoder, SReplica *pReplica) { int32_t tEncodeSReplica(SEncoder *pEncoder, SReplica *pReplica) {
if (tEncodeI32(pEncoder, pReplica->id) < 0) return -1; if (tEncodeI32(pEncoder, pReplica->id) < 0) return -1;
if (tEncodeU16(pEncoder, pReplica->port) < 0) return -1; if (tEncodeU16(pEncoder, pReplica->port) < 0) return -1;

View File

@ -523,6 +523,10 @@ static int32_t mndUpdateCompactProgress(SMnode *pMnode, SRpcMsg *pReq, int32_t c
int32_t mndProcessQueryCompactRsp(SRpcMsg *pReq) { int32_t mndProcessQueryCompactRsp(SRpcMsg *pReq) {
int32_t code = 0; int32_t code = 0;
SQueryCompactProgressRsp req = {0}; SQueryCompactProgressRsp req = {0};
if (pReq->code != 0) {
mError("received wrong compact response, req code is %s", tstrerror(pReq->code));
TAOS_RETURN(pReq->code);
}
code = tDeserializeSQueryCompactProgressRsp(pReq->pCont, pReq->contLen, &req); code = tDeserializeSQueryCompactProgressRsp(pReq->pCont, pReq->contLen, &req);
if (code != 0) { if (code != 0) {
mError("failed to deserialize vnode-query-compact-progress-rsp, ret:%d, pCont:%p, len:%d", code, pReq->pCont, mError("failed to deserialize vnode-query-compact-progress-rsp, ret:%d, pCont:%p, len:%d", code, pReq->pCont,

View File

@ -293,10 +293,13 @@ static int32_t createSchemaByFields(const SArray *pFields, SSchemaWrapper *pWrap
return terrno; return terrno;
} }
SNode *pNode;
int32_t index = 0; int32_t index = 0;
for (int32_t i = 0; i < pWrapper->nCols; i++) { for (int32_t i = 0; i < pWrapper->nCols; i++) {
SField *pField = (SField *)taosArrayGet(pFields, i); SField *pField = (SField *)taosArrayGet(pFields, i);
if (pField == NULL) {
return terrno;
}
if (TSDB_DATA_TYPE_NULL == pField->type) { if (TSDB_DATA_TYPE_NULL == pField->type) {
pWrapper->pSchema[index].type = TSDB_DATA_TYPE_VARCHAR; pWrapper->pSchema[index].type = TSDB_DATA_TYPE_VARCHAR;
pWrapper->pSchema[index].bytes = VARSTR_HEADER_SIZE; pWrapper->pSchema[index].bytes = VARSTR_HEADER_SIZE;
@ -609,6 +612,8 @@ static int32_t mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStre
// build fields // build fields
for (int32_t i = 0; i < createReq.numOfColumns; i++) { for (int32_t i = 0; i < createReq.numOfColumns; i++) {
SFieldWithOptions *pField = taosArrayGet(createReq.pColumns, i); SFieldWithOptions *pField = taosArrayGet(createReq.pColumns, i);
TSDB_CHECK_NULL(pField, code, lino, _OVER, terrno);
tstrncpy(pField->name, pStream->outputSchema.pSchema[i].name, TSDB_COL_NAME_LEN); tstrncpy(pField->name, pStream->outputSchema.pSchema[i].name, TSDB_COL_NAME_LEN);
pField->flags = pStream->outputSchema.pSchema[i].flags; pField->flags = pStream->outputSchema.pSchema[i].flags;
pField->type = pStream->outputSchema.pSchema[i].type; pField->type = pStream->outputSchema.pSchema[i].type;
@ -623,6 +628,8 @@ static int32_t mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStre
// build tags // build tags
SField *pField = taosArrayGet(createReq.pTags, 0); SField *pField = taosArrayGet(createReq.pTags, 0);
TSDB_CHECK_NULL(pField, code, lino, _OVER, terrno);
strcpy(pField->name, "group_id"); strcpy(pField->name, "group_id");
pField->type = TSDB_DATA_TYPE_UBIGINT; pField->type = TSDB_DATA_TYPE_UBIGINT;
pField->flags = 0; pField->flags = 0;
@ -634,6 +641,10 @@ static int32_t mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStre
for (int32_t i = 0; i < createReq.numOfTags; i++) { for (int32_t i = 0; i < createReq.numOfTags; i++) {
SField *pField = taosArrayGet(createReq.pTags, i); SField *pField = taosArrayGet(createReq.pTags, i);
if (pField == NULL) {
continue;
}
pField->bytes = pStream->tagSchema.pSchema[i].bytes; pField->bytes = pStream->tagSchema.pSchema[i].bytes;
pField->flags = pStream->tagSchema.pSchema[i].flags; pField->flags = pStream->tagSchema.pSchema[i].flags;
pField->type = pStream->tagSchema.pSchema[i].type; pField->type = pStream->tagSchema.pSchema[i].type;
@ -738,6 +749,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
int32_t sqlLen = 0; int32_t sqlLen = 0;
const char *pMsg = "create stream tasks on dnodes"; const char *pMsg = "create stream tasks on dnodes";
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
int32_t lino = 0;
terrno = TSDB_CODE_SUCCESS; terrno = TSDB_CODE_SUCCESS;
SCMCreateStreamReq createReq = {0}; SCMCreateStreamReq createReq = {0};
@ -777,11 +789,7 @@ static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq) {
if (createReq.sql != NULL) { if (createReq.sql != NULL) {
sqlLen = strlen(createReq.sql); sqlLen = strlen(createReq.sql);
sql = taosMemoryMalloc(sqlLen + 1); sql = taosMemoryMalloc(sqlLen + 1);
if (sql == NULL) { TSDB_CHECK_NULL(sql, code, lino, _OVER, terrno);
code = terrno;
goto _OVER;
}
memset(sql, 0, sqlLen + 1); memset(sql, 0, sqlLen + 1);
memcpy(sql, createReq.sql, sqlLen); memcpy(sql, createReq.sql, sqlLen);
} }
@ -921,9 +929,8 @@ int64_t mndStreamGenChkptId(SMnode *pMnode, bool lock) {
for (int32_t i = 0; i < taosArrayGetSize(execInfo.pTaskList); ++i) { for (int32_t i = 0; i < taosArrayGetSize(execInfo.pTaskList); ++i) {
STaskId *p = taosArrayGet(execInfo.pTaskList, i); STaskId *p = taosArrayGet(execInfo.pTaskList, i);
STaskStatusEntry *pEntry = taosHashGet(execInfo.pTaskMap, p, sizeof(*p)); STaskStatusEntry *pEntry = taosHashGet(execInfo.pTaskMap, p, sizeof(*p));
if (pEntry == NULL) { if (p == NULL || pEntry == NULL) {
continue; continue;
} }
@ -967,8 +974,7 @@ static int32_t mndBuildStreamCheckpointSourceReq(void **pBuf, int32_t *pLen, int
tEncodeSize(tEncodeStreamCheckpointSourceReq, &req, blen, code); tEncodeSize(tEncodeStreamCheckpointSourceReq, &req, blen, code);
if (code < 0) { if (code < 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY; TAOS_RETURN(TSDB_CODE_OUT_OF_MEMORY);
return -1;
} }
int32_t tlen = sizeof(SMsgHead) + blen; int32_t tlen = sizeof(SMsgHead) + blen;
@ -1002,18 +1008,20 @@ static int32_t doSetCheckpointAction(SMnode *pMnode, STrans *pTrans, SStreamTask
int8_t mndTrigger) { int8_t mndTrigger) {
void *buf; void *buf;
int32_t tlen; int32_t tlen;
if (mndBuildStreamCheckpointSourceReq(&buf, &tlen, pTask->info.nodeId, checkpointId, pTask->id.streamId, int32_t code = 0;
pTask->id.taskId, pTrans->id, mndTrigger) < 0) {
taosMemoryFree(buf);
return -1;
}
SEpSet epset = {0}; SEpSet epset = {0};
bool hasEpset = false; bool hasEpset = false;
int32_t code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId);
if ((code = mndBuildStreamCheckpointSourceReq(&buf, &tlen, pTask->info.nodeId, checkpointId, pTask->id.streamId,
pTask->id.taskId, pTrans->id, mndTrigger)) < 0) {
taosMemoryFree(buf);
return code;
}
code = extractNodeEpset(pMnode, &epset, &hasEpset, pTask->id.taskId, pTask->info.nodeId);
if (code != TSDB_CODE_SUCCESS || !hasEpset) { if (code != TSDB_CODE_SUCCESS || !hasEpset) {
taosMemoryFree(buf); taosMemoryFree(buf);
return -1; return code;
} }
code = setTransAction(pTrans, buf, tlen, TDMT_VND_STREAM_CHECK_POINT_SOURCE, &epset, TSDB_CODE_SYN_PROPOSE_NOT_READY, code = setTransAction(pTrans, buf, tlen, TDMT_VND_STREAM_CHECK_POINT_SOURCE, &epset, TSDB_CODE_SYN_PROPOSE_NOT_READY,
@ -1130,6 +1138,10 @@ static bool taskNodeIsUpdated(SMnode *pMnode) {
for (int32_t i = 0; i < numOfNodes; ++i) { for (int32_t i = 0; i < numOfNodes; ++i) {
SNodeEntry *pNodeEntry = taosArrayGet(execInfo.pNodeList, i); SNodeEntry *pNodeEntry = taosArrayGet(execInfo.pNodeList, i);
if (pNodeEntry == NULL) {
continue;
}
if (pNodeEntry->stageUpdated) { if (pNodeEntry->stageUpdated) {
mDebug("stream task not ready due to node update detected, checkpoint not issued"); mDebug("stream task not ready due to node update detected, checkpoint not issued");
streamMutexUnlock(&execInfo.lock); streamMutexUnlock(&execInfo.lock);
@ -1170,7 +1182,7 @@ static bool taskNodeIsUpdated(SMnode *pMnode) {
static int32_t mndCheckTaskAndNodeStatus(SMnode *pMnode) { static int32_t mndCheckTaskAndNodeStatus(SMnode *pMnode) {
bool ready = true; bool ready = true;
if (taskNodeIsUpdated(pMnode)) { if (taskNodeIsUpdated(pMnode)) {
return -1; TAOS_RETURN(TSDB_CODE_STREAM_TASK_IVLD_STATUS);
} }
streamMutexLock(&execInfo.lock); streamMutexLock(&execInfo.lock);
@ -1240,7 +1252,7 @@ int64_t getStreamTaskLastReadyState(SArray *pTaskList, int64_t streamId) {
for (int32_t i = 0; i < taosArrayGetSize(pTaskList); ++i) { for (int32_t i = 0; i < taosArrayGetSize(pTaskList); ++i) {
STaskId *p = taosArrayGet(pTaskList, i); STaskId *p = taosArrayGet(pTaskList, i);
STaskStatusEntry *pEntry = taosHashGet(execInfo.pTaskMap, p, sizeof(*p)); STaskStatusEntry *pEntry = taosHashGet(execInfo.pTaskMap, p, sizeof(*p));
if (pEntry == NULL || pEntry->id.streamId != streamId) { if (p == NULL || pEntry == NULL || pEntry->id.streamId != streamId) {
continue; continue;
} }
@ -1278,13 +1290,12 @@ static int32_t mndProcessStreamCheckpoint(SRpcMsg *pReq) {
int32_t numOfCheckpointTrans = 0; int32_t numOfCheckpointTrans = 0;
if ((code = mndCheckTaskAndNodeStatus(pMnode)) != 0) { if ((code = mndCheckTaskAndNodeStatus(pMnode)) != 0) {
terrno = TSDB_CODE_STREAM_TASK_IVLD_STATUS; TAOS_RETURN(TSDB_CODE_STREAM_TASK_IVLD_STATUS);
return -1;
} }
SArray *pList = taosArrayInit(4, sizeof(SCheckpointInterval)); SArray *pList = taosArrayInit(4, sizeof(SCheckpointInterval));
if (pList == NULL) { if (pList == NULL) {
return -1; return terrno;
} }
int64_t now = taosGetTimestampMs(); int64_t now = taosGetTimestampMs();
@ -1353,6 +1364,9 @@ static int32_t mndProcessStreamCheckpoint(SRpcMsg *pReq) {
for (int32_t i = 0; i < numOfQual; ++i) { for (int32_t i = 0; i < numOfQual; ++i) {
SCheckpointInterval *pCheckpointInfo = taosArrayGet(pList, i); SCheckpointInterval *pCheckpointInfo = taosArrayGet(pList, i);
if (pCheckpointInfo == NULL) {
continue;
}
SStreamObj *p = NULL; SStreamObj *p = NULL;
code = mndGetStreamObj(pMnode, pCheckpointInfo->streamId, &p); code = mndGetStreamObj(pMnode, pCheckpointInfo->streamId, &p);
@ -1523,6 +1537,7 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
void *pIter = NULL; void *pIter = NULL;
int32_t code = 0;
while (1) { while (1) {
SStreamObj *pStream = NULL; SStreamObj *pStream = NULL;
@ -1535,18 +1550,8 @@ int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
sdbCancelFetch(pSdb, pIter); sdbCancelFetch(pSdb, pIter);
mError("db:%s, failed to drop stream:%s since sourceDbUid:%" PRId64 " not match with targetDbUid:%" PRId64, mError("db:%s, failed to drop stream:%s since sourceDbUid:%" PRId64 " not match with targetDbUid:%" PRId64,
pDb->name, pStream->name, pStream->sourceDbUid, pStream->targetDbUid); pDb->name, pStream->name, pStream->sourceDbUid, pStream->targetDbUid);
terrno = TSDB_CODE_MND_STREAM_MUST_BE_DELETED; TAOS_RETURN(TSDB_CODE_MND_STREAM_MUST_BE_DELETED);
return -1;
} else { } else {
#if 0
if (mndStreamSetDropAction(pMnode, pTrans, pStream) < 0) {
mError("stream:%s, failed to drop task since %s", pStream->name, terrstr());
sdbRelease(pMnode->pSdb, pStream);
sdbCancelFetch(pSdb, pIter);
return -1;
}
#endif
// kill the related checkpoint trans // kill the related checkpoint trans
int32_t transId = mndStreamGetRelTrans(pMnode, pStream->uid); int32_t transId = mndStreamGetRelTrans(pMnode, pStream->uid);
if (transId != 0) { if (transId != 0) {
@ -1557,10 +1562,11 @@ int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) {
// drop the stream obj in execInfo // drop the stream obj in execInfo
removeStreamTasksInBuf(pStream, &execInfo); removeStreamTasksInBuf(pStream, &execInfo);
if (mndPersistTransLog(pStream, pTrans, SDB_STATUS_DROPPED) < 0) { code = mndPersistTransLog(pStream, pTrans, SDB_STATUS_DROPPED);
if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_ACTION_IN_PROGRESS) {
sdbRelease(pSdb, pStream); sdbRelease(pSdb, pStream);
sdbCancelFetch(pSdb, pIter); sdbCancelFetch(pSdb, pIter);
return -1; return code;
} }
} }
} }
@ -1575,8 +1581,7 @@ int32_t mndGetNumOfStreams(SMnode *pMnode, char *dbName, int32_t *pNumOfStreams)
SSdb *pSdb = pMnode->pSdb; SSdb *pSdb = pMnode->pSdb;
SDbObj *pDb = mndAcquireDb(pMnode, dbName); SDbObj *pDb = mndAcquireDb(pMnode, dbName);
if (pDb == NULL) { if (pDb == NULL) {
terrno = TSDB_CODE_MND_DB_NOT_SELECTED; TAOS_RETURN(TSDB_CODE_MND_DB_NOT_SELECTED);
return -1;
} }
int32_t numOfStreams = 0; int32_t numOfStreams = 0;
@ -1704,8 +1709,7 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) {
SMPauseStreamReq pauseReq = {0}; SMPauseStreamReq pauseReq = {0};
if (tDeserializeSMPauseStreamReq(pReq->pCont, pReq->contLen, &pauseReq) < 0) { if (tDeserializeSMPauseStreamReq(pReq->pCont, pReq->contLen, &pauseReq) < 0) {
terrno = TSDB_CODE_INVALID_MSG; TAOS_RETURN(TSDB_CODE_INVALID_MSG);
return -1;
} }
code = mndAcquireStream(pMnode, pauseReq.name, &pStream); code = mndAcquireStream(pMnode, pauseReq.name, &pStream);
@ -1715,8 +1719,7 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) {
return 0; return 0;
} else { } else {
mError("stream:%s not exist, failed to pause stream", pauseReq.name); mError("stream:%s not exist, failed to pause stream", pauseReq.name);
terrno = TSDB_CODE_MND_STREAM_NOT_EXIST; TAOS_RETURN(TSDB_CODE_MND_STREAM_NOT_EXIST);
return -1;
} }
} }
@ -1736,14 +1739,14 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) {
bool conflict = mndStreamTransConflictCheck(pMnode, pStream->uid, MND_STREAM_PAUSE_NAME, true); bool conflict = mndStreamTransConflictCheck(pMnode, pStream->uid, MND_STREAM_PAUSE_NAME, true);
if (conflict) { if (conflict) {
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
return -1; TAOS_RETURN(TSDB_CODE_MND_TRANS_CONFLICT);
} }
bool updated = taskNodeIsUpdated(pMnode); bool updated = taskNodeIsUpdated(pMnode);
if (updated) { if (updated) {
mError("tasks are not ready for pause, node update detected"); mError("tasks are not ready for pause, node update detected");
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
return -1; TAOS_RETURN(TSDB_CODE_STREAM_TASK_IVLD_STATUS);
} }
{ // check for tasks, if tasks are not ready, not allowed to pause { // check for tasks, if tasks are not ready, not allowed to pause
@ -1753,6 +1756,9 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) {
for (int32_t i = 0; i < taosArrayGetSize(execInfo.pTaskList); ++i) { for (int32_t i = 0; i < taosArrayGetSize(execInfo.pTaskList); ++i) {
STaskId *p = taosArrayGet(execInfo.pTaskList, i); STaskId *p = taosArrayGet(execInfo.pTaskList, i);
if (p == NULL) {
continue;
}
STaskStatusEntry *pEntry = taosHashGet(execInfo.pTaskMap, p, sizeof(*p)); STaskStatusEntry *pEntry = taosHashGet(execInfo.pTaskMap, p, sizeof(*p));
if (pEntry == NULL) { if (pEntry == NULL) {
@ -1776,13 +1782,13 @@ static int32_t mndProcessPauseStreamReq(SRpcMsg *pReq) {
if (!found) { if (!found) {
mError("stream:%s task not report status yet, not ready for pause", pauseReq.name); mError("stream:%s task not report status yet, not ready for pause", pauseReq.name);
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
return -1; TAOS_RETURN(TSDB_CODE_STREAM_TASK_IVLD_STATUS);
} }
if (!readyToPause) { if (!readyToPause) {
mError("stream:%s task not ready for pause yet", pauseReq.name); mError("stream:%s task not ready for pause yet", pauseReq.name);
sdbRelease(pMnode->pSdb, pStream); sdbRelease(pMnode->pSdb, pStream);
return -1; TAOS_RETURN(TSDB_CODE_STREAM_TASK_IVLD_STATUS);
} }
} }
@ -1843,13 +1849,12 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) {
int32_t code = 0; int32_t code = 0;
if ((terrno = grantCheckExpire(TSDB_GRANT_STREAMS)) < 0) { if ((terrno = grantCheckExpire(TSDB_GRANT_STREAMS)) < 0) {
return -1; return terrno;
} }
SMResumeStreamReq resumeReq = {0}; SMResumeStreamReq resumeReq = {0};
if (tDeserializeSMResumeStreamReq(pReq->pCont, pReq->contLen, &resumeReq) < 0) { if (tDeserializeSMResumeStreamReq(pReq->pCont, pReq->contLen, &resumeReq) < 0) {
terrno = TSDB_CODE_INVALID_MSG; TAOS_RETURN(TSDB_CODE_INVALID_MSG);
return -1;
} }
code = mndAcquireStream(pMnode, resumeReq.name, &pStream); code = mndAcquireStream(pMnode, resumeReq.name, &pStream);
@ -1860,8 +1865,7 @@ static int32_t mndProcessResumeStreamReq(SRpcMsg *pReq) {
return 0; return 0;
} else { } else {
mError("stream:%s not exist, failed to resume stream", resumeReq.name); mError("stream:%s not exist, failed to resume stream", resumeReq.name);
terrno = TSDB_CODE_MND_STREAM_NOT_EXIST; TAOS_RETURN(TSDB_CODE_MND_STREAM_NOT_EXIST);
return -1;
} }
} }
@ -1956,10 +1960,16 @@ static SVgroupChangeInfo mndFindChangedNodeInfo(SMnode *pMnode, const SArray *pP
int32_t numOfNodes = taosArrayGetSize(pPrevNodeList); int32_t numOfNodes = taosArrayGetSize(pPrevNodeList);
for (int32_t i = 0; i < numOfNodes; ++i) { for (int32_t i = 0; i < numOfNodes; ++i) {
SNodeEntry *pPrevEntry = taosArrayGet(pPrevNodeList, i); SNodeEntry *pPrevEntry = taosArrayGet(pPrevNodeList, i);
if (pPrevEntry == NULL) {
continue;
}
int32_t num = taosArrayGetSize(pNodeList); int32_t num = taosArrayGetSize(pNodeList);
for (int32_t j = 0; j < num; ++j) { for (int32_t j = 0; j < num; ++j) {
SNodeEntry *pCurrent = taosArrayGet(pNodeList, j); SNodeEntry *pCurrent = taosArrayGet(pNodeList, j);
if(pCurrent == NULL) {
continue;
}
if (pCurrent->nodeId == pPrevEntry->nodeId) { if (pCurrent->nodeId == pPrevEntry->nodeId) {
if (pPrevEntry->stageUpdated || isNodeEpsetChanged(&pPrevEntry->epset, &pCurrent->epset)) { if (pPrevEntry->stageUpdated || isNodeEpsetChanged(&pPrevEntry->epset, &pCurrent->epset)) {
@ -2302,7 +2312,7 @@ void saveTaskAndNodeInfoIntoBuf(SStreamObj *pStream, SStreamExecInfo *pExecNode)
bool exist = false; bool exist = false;
for (int32_t j = 0; j < taosArrayGetSize(pExecNode->pNodeList); ++j) { for (int32_t j = 0; j < taosArrayGetSize(pExecNode->pNodeList); ++j) {
SNodeEntry *pEntry = taosArrayGet(pExecNode->pNodeList, j); SNodeEntry *pEntry = taosArrayGet(pExecNode->pNodeList, j);
if (pEntry->nodeId == pTask->info.nodeId) { if ((pEntry != NULL) && (pEntry->nodeId == pTask->info.nodeId)) {
exist = true; exist = true;
break; break;
} }
@ -2329,14 +2339,17 @@ static void doAddTaskId(SArray *pList, int32_t taskId, int64_t uid, int32_t numO
int32_t num = taosArrayGetSize(pList); int32_t num = taosArrayGetSize(pList);
for (int32_t i = 0; i < num; ++i) { for (int32_t i = 0; i < num; ++i) {
int32_t *pId = taosArrayGet(pList, i); int32_t *pId = taosArrayGet(pList, i);
if (pId == NULL) {
continue;
}
if (taskId == *pId) { if (taskId == *pId) {
return; return;
} }
} }
void* p = taosArrayPush(pList, &taskId);
int32_t numOfTasks = taosArrayGetSize(pList); int32_t numOfTasks = taosArrayGetSize(pList);
void *p = taosArrayPush(pList, &taskId);
if (p) { if (p) {
mDebug("stream:0x%" PRIx64 " receive %d reqs for checkpoint, remain:%d", uid, numOfTasks, numOfTotal - numOfTasks); mDebug("stream:0x%" PRIx64 " receive %d reqs for checkpoint, remain:%d", uid, numOfTasks, numOfTotal - numOfTasks);
} else { } else {
@ -2445,6 +2458,10 @@ static void doAddReportStreamTask(SArray* pList, const SCheckpointReport* pRepor
bool existed = false; bool existed = false;
for (int32_t i = 0; i < taosArrayGetSize(pList); ++i) { for (int32_t i = 0; i < taosArrayGetSize(pList); ++i) {
STaskChkptInfo *p = taosArrayGet(pList, i); STaskChkptInfo *p = taosArrayGet(pList, i);
if (p == NULL) {
continue;
}
if (p->taskId == pReport->taskId) { if (p->taskId == pReport->taskId) {
existed = true; existed = true;
break; break;
@ -2554,6 +2571,10 @@ static int64_t getConsensusId(int64_t streamId, int32_t numOfTasks, int32_t* pEx
for(int32_t i = 0; i < taosArrayGetSize(execInfo.pTaskList); ++i) { for(int32_t i = 0; i < taosArrayGetSize(execInfo.pTaskList); ++i) {
STaskId* p = taosArrayGet(execInfo.pTaskList, i); STaskId* p = taosArrayGet(execInfo.pTaskList, i);
if (p == NULL) {
continue;
}
if (p->streamId != streamId) { if (p->streamId != streamId) {
continue; continue;
} }
@ -2634,6 +2655,10 @@ int32_t mndProcessConsensusInTmr(SRpcMsg *pMsg) {
for (int32_t j = 0; j < num; ++j) { for (int32_t j = 0; j < num; ++j) {
SCheckpointConsensusEntry *pe = taosArrayGet(pInfo->pTaskList, j); SCheckpointConsensusEntry *pe = taosArrayGet(pInfo->pTaskList, j);
if (pe == NULL) {
continue;
}
streamId = pe->req.streamId; streamId = pe->req.streamId;
int32_t existed = 0; int32_t existed = 0;
@ -2670,9 +2695,13 @@ int32_t mndProcessConsensusInTmr(SRpcMsg *pMsg) {
if (taosArrayGetSize(pList) > 0) { if (taosArrayGetSize(pList) > 0) {
for (int32_t i = 0; i < taosArrayGetSize(pList); ++i) { for (int32_t i = 0; i < taosArrayGetSize(pList); ++i) {
int32_t *taskId = taosArrayGet(pList, i); int32_t *taskId = taosArrayGet(pList, i);
if (taskId == NULL) {
continue;
}
for (int32_t k = 0; k < taosArrayGetSize(pInfo->pTaskList); ++k) { for (int32_t k = 0; k < taosArrayGetSize(pInfo->pTaskList); ++k) {
SCheckpointConsensusEntry *pe = taosArrayGet(pInfo->pTaskList, k); SCheckpointConsensusEntry *pe = taosArrayGet(pInfo->pTaskList, k);
if (pe->req.taskId == *taskId) { if ((pe != NULL) && (pe->req.taskId == *taskId)) {
taosArrayRemove(pInfo->pTaskList, k); taosArrayRemove(pInfo->pTaskList, k);
break; break;
} }
@ -2694,6 +2723,10 @@ int32_t mndProcessConsensusInTmr(SRpcMsg *pMsg) {
for (int32_t i = 0; i < taosArrayGetSize(pStreamList); ++i) { for (int32_t i = 0; i < taosArrayGetSize(pStreamList); ++i) {
int64_t *pStreamId = (int64_t *)taosArrayGet(pStreamList, i); int64_t *pStreamId = (int64_t *)taosArrayGet(pStreamList, i);
if (pStreamId == NULL) {
continue;
}
code = mndClearConsensusCheckpointId(execInfo.pStreamConsensus, *pStreamId); code = mndClearConsensusCheckpointId(execInfo.pStreamConsensus, *pStreamId);
} }
@ -2800,6 +2833,8 @@ static int32_t mndProcessDropOrphanTaskReq(SRpcMsg *pReq) {
int32_t code = 0; int32_t code = 0;
SOrphanTask *pTask = NULL; SOrphanTask *pTask = NULL;
int32_t i = 0; int32_t i = 0;
STrans *pTrans = NULL;
int32_t numOfTasks = 0;
SMStreamDropOrphanMsg msg = {0}; SMStreamDropOrphanMsg msg = {0};
code = tDeserializeDropOrphanTaskMsg(pReq->pCont, pReq->contLen, &msg); code = tDeserializeDropOrphanTaskMsg(pReq->pCont, pReq->contLen, &msg);
@ -2807,10 +2842,10 @@ static int32_t mndProcessDropOrphanTaskReq(SRpcMsg *pReq) {
return code; return code;
} }
int32_t numOfTasks = taosArrayGetSize(msg.pList); numOfTasks = taosArrayGetSize(msg.pList);
if (numOfTasks == 0) { if (numOfTasks == 0) {
mDebug("no orphan tasks to drop, no need to create trans"); mDebug("no orphan tasks to drop, no need to create trans");
return code; goto _err;
} }
mDebug("create trans to drop %d orphan tasks", numOfTasks); mDebug("create trans to drop %d orphan tasks", numOfTasks);
@ -2822,52 +2857,52 @@ static int32_t mndProcessDropOrphanTaskReq(SRpcMsg *pReq) {
if (pTask == NULL) { if (pTask == NULL) {
mError("failed to extract entry in drop orphan task list, not create trans to drop orphan-task"); mError("failed to extract entry in drop orphan task list, not create trans to drop orphan-task");
return TSDB_CODE_SUCCESS; goto _err;
} }
// check if it is conflict with other trans in both sourceDb and targetDb. // check if it is conflict with other trans in both sourceDb and targetDb.
bool conflict = mndStreamTransConflictCheck(pMnode, pTask->streamId, MND_STREAM_DROP_NAME, false); bool conflict = mndStreamTransConflictCheck(pMnode, pTask->streamId, MND_STREAM_DROP_NAME, false);
if (conflict) { if (conflict) {
return -1; TAOS_RETURN(TSDB_CODE_MND_TRANS_CONFLICT);
} }
SStreamObj dummyObj = {.uid = pTask->streamId, .sourceDb = "", .targetSTbName = ""}; SStreamObj dummyObj = {.uid = pTask->streamId, .sourceDb = "", .targetSTbName = ""};
STrans *pTrans = NULL;
code = doCreateTrans(pMnode, &dummyObj, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_DROP_NAME, "drop stream", &pTrans); code = doCreateTrans(pMnode, &dummyObj, NULL, TRN_CONFLICT_NOTHING, MND_STREAM_DROP_NAME, "drop stream", &pTrans);
if (pTrans == NULL || code != 0) { if (pTrans == NULL || code != 0) {
mError("failed to create trans to drop orphan tasks since %s", terrstr()); mError("failed to create trans to drop orphan tasks since %s", terrstr());
return code; goto _err;
} }
code = mndStreamRegisterTrans(pTrans, MND_STREAM_DROP_NAME, pTask->streamId); code = mndStreamRegisterTrans(pTrans, MND_STREAM_DROP_NAME, pTask->streamId);
if (code) { if (code) {
mndTransDrop(pTrans);
return code; return code;
} }
// drop all tasks // drop all tasks
if ((code = mndStreamSetDropActionFromList(pMnode, pTrans, msg.pList)) < 0) { if ((code = mndStreamSetDropActionFromList(pMnode, pTrans, msg.pList)) < 0) {
mError("failed to create trans to drop orphan tasks since %s", terrstr()); mError("failed to create trans to drop orphan tasks since %s", terrstr());
mndTransDrop(pTrans); goto _err;
return code;
} }
// drop stream // drop stream
if ((code = mndPersistTransLog(&dummyObj, pTrans, SDB_STATUS_DROPPED)) < 0) { if ((code = mndPersistTransLog(&dummyObj, pTrans, SDB_STATUS_DROPPED)) < 0) {
mndTransDrop(pTrans); goto _err;
return code;
} }
code = mndTransPrepare(pMnode, pTrans); code = mndTransPrepare(pMnode, pTrans);
if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_ACTION_IN_PROGRESS) { if (code != TSDB_CODE_SUCCESS && code != TSDB_CODE_ACTION_IN_PROGRESS) {
mError("trans:%d, failed to prepare drop stream trans since %s", pTrans->id, terrstr()); mError("trans:%d, failed to prepare drop stream trans since %s", pTrans->id, terrstr());
mndTransDrop(pTrans); goto _err;
return code;
} }
_err:
tDestroyDropOrphanTaskMsg(&msg);
mndTransDrop(pTrans);
if (code == TSDB_CODE_SUCCESS) { if (code == TSDB_CODE_SUCCESS) {
mDebug("create drop %d orphan tasks trans succ", numOfTasks); mDebug("create drop %d orphan tasks trans succ", numOfTasks);
} }
mndTransDrop(pTrans);
return code; return code;
} }

View File

@ -33,6 +33,10 @@ int32_t mndStreamClearFinishedTrans(SMnode *pMnode, int32_t *pNumOfActiveChkpt)
SArray *pList = taosArrayInit(4, sizeof(SKeyInfo)); SArray *pList = taosArrayInit(4, sizeof(SKeyInfo));
int32_t num = 0; int32_t num = 0;
if (pList == NULL) {
return terrno;
}
while ((pIter = taosHashIterate(execInfo.transMgmt.pDBTrans, pIter)) != NULL) { while ((pIter = taosHashIterate(execInfo.transMgmt.pDBTrans, pIter)) != NULL) {
SStreamTransInfo *pEntry = (SStreamTransInfo *)pIter; SStreamTransInfo *pEntry = (SStreamTransInfo *)pIter;
@ -59,6 +63,10 @@ int32_t mndStreamClearFinishedTrans(SMnode *pMnode, int32_t *pNumOfActiveChkpt)
int32_t size = taosArrayGetSize(pList); int32_t size = taosArrayGetSize(pList);
for (int32_t i = 0; i < size; ++i) { for (int32_t i = 0; i < size; ++i) {
SKeyInfo *pKey = taosArrayGet(pList, i); SKeyInfo *pKey = taosArrayGet(pList, i);
if (pKey == NULL) {
continue;
}
int32_t code = taosHashRemove(execInfo.transMgmt.pDBTrans, pKey->pKey, pKey->keyLen); int32_t code = taosHashRemove(execInfo.transMgmt.pDBTrans, pKey->pKey, pKey->keyLen);
if (code != 0) { if (code != 0) {
taosArrayDestroy(pList); taosArrayDestroy(pList);
@ -171,25 +179,24 @@ int32_t doCreateTrans(SMnode *pMnode, SStreamObj *pStream, SRpcMsg *pReq, ETrnCo
*pTrans1 = NULL; *pTrans1 = NULL;
terrno = 0; terrno = 0;
int32_t code = 0;
STrans *p = mndTransCreate(pMnode, TRN_POLICY_RETRY, conflict, pReq, name); STrans *p = mndTransCreate(pMnode, TRN_POLICY_RETRY, conflict, pReq, name);
if (p == NULL) { if (p == NULL) {
mError("failed to build trans:%s, reason: %s", name, tstrerror(TSDB_CODE_OUT_OF_MEMORY)); mError("failed to build trans:%s, reason: %s", name, tstrerror(terrno));
terrno = TSDB_CODE_OUT_OF_MEMORY;
return terrno; return terrno;
} }
mInfo("stream:0x%" PRIx64 " start to build trans %s, transId:%d", pStream->uid, pMsg, p->id); mInfo("stream:0x%" PRIx64 " start to build trans %s, transId:%d", pStream->uid, pMsg, p->id);
mndTransSetDbName(p, pStream->sourceDb, pStream->targetSTbName); mndTransSetDbName(p, pStream->sourceDb, pStream->targetSTbName);
if (mndTransCheckConflict(pMnode, p) != 0) { if ((code = mndTransCheckConflict(pMnode, p)) != 0) {
terrno = TSDB_CODE_MND_TRANS_CONFLICT;
mError("failed to build trans:%s for stream:0x%" PRIx64 " code:%s", name, pStream->uid, tstrerror(terrno)); mError("failed to build trans:%s for stream:0x%" PRIx64 " code:%s", name, pStream->uid, tstrerror(terrno));
mndTransDrop(p); mndTransDrop(p);
return terrno; return code;
} }
*pTrans1 = p; *pTrans1 = p;
return 0; return code;
} }
SSdbRaw *mndStreamActionEncode(SStreamObj *pStream) { SSdbRaw *mndStreamActionEncode(SStreamObj *pStream) {

View File

@ -106,10 +106,14 @@ int32_t tqBuildDeleteReq(STQ* pTq, const char* stbFullName, const SSDataBlock* p
strncpy(req.tbname, name, TSDB_TABLE_NAME_LEN - 1); strncpy(req.tbname, name, TSDB_TABLE_NAME_LEN - 1);
void* p = taosArrayPush(deleteReq->deleteReqs, &req); void* p = taosArrayPush(deleteReq->deleteReqs, &req);
if (p == NULL) { if (p == NULL) {
return TSDB_CODE_OUT_OF_MEMORY; return terrno;
} }
} }
if (originName) name = originName;
if (originName) {
name = originName;
}
taosMemoryFreeClear(name); taosMemoryFreeClear(name);
} }
@ -190,6 +194,7 @@ int32_t initCreateTableMsg(SVCreateTbReq* pCreateTableReq, uint64_t suid, const
pCreateTableReq->ctb.stbName = taosStrdup((char*)tNameGetTableName(&name)); pCreateTableReq->ctb.stbName = taosStrdup((char*)tNameGetTableName(&name));
if (pCreateTableReq->ctb.stbName == NULL) { // ignore this error code if (pCreateTableReq->ctb.stbName == NULL) { // ignore this error code
tqError("failed to duplicate the stb name:%s, failed to init create-table msg and create req table", stbFullName); tqError("failed to duplicate the stb name:%s, failed to init create-table msg and create req table", stbFullName);
code = TSDB_CODE_OUT_OF_MEMORY;
} }
} }
@ -202,14 +207,14 @@ int32_t createDefaultTagColName(SArray** pColNameList) {
SArray* pTagColNameList = taosArrayInit(1, TSDB_COL_NAME_LEN); SArray* pTagColNameList = taosArrayInit(1, TSDB_COL_NAME_LEN);
if (pTagColNameList == NULL) { if (pTagColNameList == NULL) {
return TSDB_CODE_OUT_OF_MEMORY; return terrno;
} }
char tagNameStr[TSDB_COL_NAME_LEN] = "group_id"; char tagNameStr[TSDB_COL_NAME_LEN] = "group_id";
void* p = taosArrayPush(pTagColNameList, tagNameStr); void* p = taosArrayPush(pTagColNameList, tagNameStr);
if (p == NULL) { if (p == NULL) {
taosArrayDestroy(pTagColNameList); taosArrayDestroy(pTagColNameList);
return TSDB_CODE_OUT_OF_MEMORY; return terrno;
} }
*pColNameList = pTagColNameList; *pColNameList = pTagColNameList;
@ -252,15 +257,14 @@ static int32_t doBuildAndSendCreateTableMsg(SVnode* pVnode, char* stbFullName, S
SArray* tagArray = taosArrayInit(4, sizeof(STagVal)); SArray* tagArray = taosArrayInit(4, sizeof(STagVal));
const char* id = pTask->id.idStr; const char* id = pTask->id.idStr;
int32_t vgId = pTask->pMeta->vgId; int32_t vgId = pTask->pMeta->vgId;
tqDebug("s-task:%s build create %d table(s) msg", id, rows);
int32_t code = 0; int32_t code = 0;
tqDebug("s-task:%s build create %d table(s) msg", id, rows);
SVCreateTbBatchReq reqs = {0}; SVCreateTbBatchReq reqs = {0};
SArray* crTblArray = reqs.pArray = taosArrayInit(1, sizeof(SVCreateTbReq)); SArray* crTblArray = reqs.pArray = taosArrayInit(1, sizeof(SVCreateTbReq));
if (NULL == reqs.pArray) { if ((NULL == reqs.pArray) || (tagArray == NULL)) {
tqError("s-task:%s failed to init create table msg, code:%s", id, tstrerror(terrno)); tqError("s-task:%s failed to init create table msg, code:%s", id, tstrerror(terrno));
code = terrno;
goto _end; goto _end;
} }
@ -418,8 +422,8 @@ int32_t doMergeExistedRows(SSubmitTbData* pExisted, const SSubmitTbData* pNew, c
int32_t j = 0, k = 0; int32_t j = 0, k = 0;
SArray* pFinal = taosArrayInit(oldLen + newLen, POINTER_BYTES); SArray* pFinal = taosArrayInit(oldLen + newLen, POINTER_BYTES);
if (pFinal == NULL) { if (pFinal == NULL) {
tqError("s-task:%s failed to prepare merge result datablock, code:%s", id, tstrerror(TSDB_CODE_OUT_OF_MEMORY)); tqError("s-task:%s failed to prepare merge result datablock, code:%s", id, tstrerror(terrno));
return TSDB_CODE_OUT_OF_MEMORY; return terrno;
} }
while (j < newLen && k < oldLen) { while (j < newLen && k < oldLen) {
@ -872,6 +876,9 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat
tqDebug("s-task:%s stream write into table:%s, table auto created", id, dstTableName); tqDebug("s-task:%s stream write into table:%s, table auto created", id, dstTableName);
SArray* pTagArray = taosArrayInit(pTSchema->numOfCols + 1, sizeof(STagVal)); SArray* pTagArray = taosArrayInit(pTSchema->numOfCols + 1, sizeof(STagVal));
if (pTagArray == NULL) {
return terrno;
}
pTableData->flags = SUBMIT_REQ_AUTO_CREATE_TABLE; pTableData->flags = SUBMIT_REQ_AUTO_CREATE_TABLE;
code = code =
@ -1167,6 +1174,9 @@ int32_t doRemoveFromCache(SSHashObj* pSinkTableMap, uint64_t groupId, const char
int32_t doBuildAndSendDeleteMsg(SVnode* pVnode, char* stbFullName, SSDataBlock* pDataBlock, SStreamTask* pTask, int32_t doBuildAndSendDeleteMsg(SVnode* pVnode, char* stbFullName, SSDataBlock* pDataBlock, SStreamTask* pTask,
int64_t suid) { int64_t suid) {
SBatchDeleteReq deleteReq = {.suid = suid, .deleteReqs = taosArrayInit(0, sizeof(SSingleDeleteReq))}; SBatchDeleteReq deleteReq = {.suid = suid, .deleteReqs = taosArrayInit(0, sizeof(SSingleDeleteReq))};
if (deleteReq.deleteReqs == NULL) {
return terrno;
}
int32_t code = tqBuildDeleteReq(pVnode->pTq, stbFullName, pDataBlock, &deleteReq, pTask->id.idStr, int32_t code = tqBuildDeleteReq(pVnode->pTq, stbFullName, pDataBlock, &deleteReq, pTask->id.idStr,
pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && pTask->subtableWithoutMd5 != 1); pTask->ver >= SSTREAM_TASK_SUBTABLE_CHANGED_VER && pTask->subtableWithoutMd5 != 1);

View File

@ -82,7 +82,7 @@ static void doStartScanWal(void* param, void* tmrId) {
taosMemoryFree(pParam); taosMemoryFree(pParam);
if (code) { if (code) {
tqError("vgId:%d failed sched task to scan wal", vgId); tqError("vgId:%d failed sched task to scan wal, code:%s", vgId, tstrerror(code));
} }
} }

View File

@ -553,8 +553,15 @@ int32_t tqStreamTaskProcessCheckpointReadyMsg(SStreamMeta* pMeta, SRpcMsg* pMsg)
return code; return code;
} }
if (pTask->info.taskLevel == TASK_LEVEL__SINK) {
tqDebug("vgId:%d s-task:%s recv invalid the checkpoint-ready msg from task:0x%x (vgId:%d), discard", vgId,
pTask->id.idStr, req.downstreamTaskId, req.downstreamNodeId);
streamMetaReleaseTask(pMeta, pTask);
return TSDB_CODE_INVALID_MSG;
} else {
tqDebug("vgId:%d s-task:%s received the checkpoint-ready msg from task:0x%x (vgId:%d), handle it", vgId, tqDebug("vgId:%d s-task:%s received the checkpoint-ready msg from task:0x%x (vgId:%d), handle it", vgId,
pTask->id.idStr, req.downstreamTaskId, req.downstreamNodeId); pTask->id.idStr, req.downstreamTaskId, req.downstreamNodeId);
}
code = streamProcessCheckpointReadyMsg(pTask, req.checkpointId, req.downstreamTaskId, req.downstreamNodeId); code = streamProcessCheckpointReadyMsg(pTask, req.checkpointId, req.downstreamTaskId, req.downstreamNodeId);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);

View File

@ -176,7 +176,7 @@ int32_t extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNod
int32_t createExprFromOneNode(SExprInfo* pExp, SNode* pNode, int16_t slotId); int32_t createExprFromOneNode(SExprInfo* pExp, SNode* pNode, int16_t slotId);
int32_t createExprFromTargetNode(SExprInfo* pExp, STargetNode* pTargetNode); int32_t createExprFromTargetNode(SExprInfo* pExp, STargetNode* pTargetNode);
SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* numOfExprs); int32_t createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, SExprInfo** pExprInfo, int32_t* numOfExprs);
SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowEntryInfoOffset, SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowEntryInfoOffset,
SFunctionStateStore* pStore); SFunctionStateStore* pStore);
@ -197,9 +197,6 @@ char* getStreamOpName(uint16_t opType);
void printDataBlock(SSDataBlock* pBlock, const char* flag, const char* taskIdStr); void printDataBlock(SSDataBlock* pBlock, const char* flag, const char* taskIdStr);
void printSpecDataBlock(SSDataBlock* pBlock, const char* flag, const char* opStr, const char* taskIdStr); void printSpecDataBlock(SSDataBlock* pBlock, const char* flag, const char* opStr, const char* taskIdStr);
void getNextTimeWindow(const SInterval* pInterval, STimeWindow* tw, int32_t order);
void getInitialStartTimeWindow(SInterval* pInterval, TSKEY ts, STimeWindow* w, bool ascQuery);
TSKEY getStartTsKey(STimeWindow* win, const TSKEY* tsCols); TSKEY getStartTsKey(STimeWindow* win, const TSKEY* tsCols);
void updateTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pWin, int64_t delta); void updateTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pWin, int64_t delta);

View File

@ -73,7 +73,13 @@ int32_t createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* pA
SOperatorInfo** pOptrInfo) { SOperatorInfo** pOptrInfo) {
QRY_OPTR_CHECK(pOptrInfo); QRY_OPTR_CHECK(pOptrInfo);
int32_t lino = 0;
int32_t code = 0; int32_t code = 0;
int32_t num = 0;
SExprInfo* pExprInfo = NULL;
int32_t numOfScalarExpr = 0;
SExprInfo* pScalarExprInfo = NULL;
SAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SAggOperatorInfo)); SAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SAggOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) { if (pInfo == NULL || pOperator == NULL) {
@ -89,29 +95,23 @@ int32_t createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* pA
size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
initResultSizeInfo(&pOperator->resultInfo, 4096); initResultSizeInfo(&pOperator->resultInfo, 4096);
int32_t num = 0; code = createExprInfo(pAggNode->pAggFuncs, pAggNode->pGroupKeys, &pExprInfo, &num);
SExprInfo* pExprInfo = createExprInfo(pAggNode->pAggFuncs, pAggNode->pGroupKeys, &num); TSDB_CHECK_CODE(code, lino, _error);
code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str, code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str,
pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore); pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) { TSDB_CHECK_CODE(code, lino, _error);
goto _error;
}
int32_t numOfScalarExpr = 0;
SExprInfo* pScalarExprInfo = NULL;
if (pAggNode->pExprs != NULL) { if (pAggNode->pExprs != NULL) {
pScalarExprInfo = createExprInfo(pAggNode->pExprs, NULL, &numOfScalarExpr); code = createExprInfo(pAggNode->pExprs, NULL, &pScalarExprInfo, &numOfScalarExpr);
TSDB_CHECK_CODE(code, lino, _error);
} }
code = initExprSupp(&pInfo->scalarExprSup, pScalarExprInfo, numOfScalarExpr, &pTaskInfo->storageAPI.functionStore); code = initExprSupp(&pInfo->scalarExprSup, pScalarExprInfo, numOfScalarExpr, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) { TSDB_CHECK_CODE(code, lino, _error);
goto _error;
}
code = filterInitFromNode((SNode*)pAggNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); code = filterInitFromNode((SNode*)pAggNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
if (code != TSDB_CODE_SUCCESS) { TSDB_CHECK_CODE(code, lino, _error);
goto _error;
}
pInfo->binfo.mergeResultBlock = pAggNode->mergeDataBlock; pInfo->binfo.mergeResultBlock = pAggNode->mergeDataBlock;
pInfo->groupKeyOptimized = pAggNode->groupKeyOptimized; pInfo->groupKeyOptimized = pAggNode->groupKeyOptimized;

View File

@ -200,7 +200,9 @@ int32_t createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SReadHandl
if (pScanNode->scan.pScanPseudoCols != NULL) { if (pScanNode->scan.pScanPseudoCols != NULL) {
SExprSupp* p = &pInfo->pseudoExprSup; SExprSupp* p = &pInfo->pseudoExprSup;
p->pExprInfo = createExprInfo(pScanNode->scan.pScanPseudoCols, NULL, &p->numOfExprs); code = createExprInfo(pScanNode->scan.pScanPseudoCols, NULL, &p->pExprInfo, &p->numOfExprs);
TSDB_CHECK_CODE(code, lino, _error);
p->pCtx = p->pCtx =
createSqlFunctionCtx(p->pExprInfo, p->numOfExprs, &p->rowEntryInfoOffset, &pTaskInfo->storageAPI.functionStore); createSqlFunctionCtx(p->pExprInfo, p->numOfExprs, &p->rowEntryInfoOffset, &pTaskInfo->storageAPI.functionStore);
} }

View File

@ -256,14 +256,19 @@ int32_t createCountwindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* phy
if (pCountWindowNode->window.pExprs != NULL) { if (pCountWindowNode->window.pExprs != NULL) {
int32_t numOfScalarExpr = 0; int32_t numOfScalarExpr = 0;
SExprInfo* pScalarExprInfo = createExprInfo(pCountWindowNode->window.pExprs, NULL, &numOfScalarExpr); SExprInfo* pScalarExprInfo = NULL;
code = createExprInfo(pCountWindowNode->window.pExprs, NULL, &pScalarExprInfo, &numOfScalarExpr);
QUERY_CHECK_CODE(code, lino, _error);
code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr, &pTaskInfo->storageAPI.functionStore); code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr, &pTaskInfo->storageAPI.functionStore);
QUERY_CHECK_CODE(code, lino, _error); QUERY_CHECK_CODE(code, lino, _error);
} }
size_t keyBufSize = 0; size_t keyBufSize = 0;
int32_t num = 0; int32_t num = 0;
SExprInfo* pExprInfo = createExprInfo(pCountWindowNode->window.pFuncs, NULL, &num); SExprInfo* pExprInfo = NULL;
code = createExprInfo(pCountWindowNode->window.pFuncs, NULL, &pExprInfo, &num);
QUERY_CHECK_CODE(code, lino, _error);
initResultSizeInfo(&pOperator->resultInfo, 4096); initResultSizeInfo(&pOperator->resultInfo, 4096);
code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str, code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str,
@ -286,6 +291,7 @@ int32_t createCountwindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* phy
if (pInfo->windowCount != pInfo->windowSliding) { if (pInfo->windowCount != pInfo->windowSliding) {
numOfItem = pInfo->windowCount / pInfo->windowSliding + 1; numOfItem = pInfo->windowCount / pInfo->windowSliding + 1;
} }
pInfo->countSup.pWinStates = taosArrayInit_s(itemSize, numOfItem); pInfo->countSup.pWinStates = taosArrayInit_s(itemSize, numOfItem);
if (!pInfo->countSup.pWinStates) { if (!pInfo->countSup.pWinStates) {
goto _error; goto _error;

View File

@ -84,7 +84,10 @@ int32_t createEventwindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* phy
if (pEventWindowNode->window.pExprs != NULL) { if (pEventWindowNode->window.pExprs != NULL) {
int32_t numOfScalarExpr = 0; int32_t numOfScalarExpr = 0;
SExprInfo* pScalarExprInfo = createExprInfo(pEventWindowNode->window.pExprs, NULL, &numOfScalarExpr); SExprInfo* pScalarExprInfo = NULL;
code = createExprInfo(pEventWindowNode->window.pExprs, NULL, &pScalarExprInfo, &numOfScalarExpr);
QUERY_CHECK_CODE(code, lino, _error);
code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr, &pTaskInfo->storageAPI.functionStore); code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr, &pTaskInfo->storageAPI.functionStore);
QUERY_CHECK_CODE(code, lino, _error); QUERY_CHECK_CODE(code, lino, _error);
} }
@ -95,7 +98,10 @@ int32_t createEventwindowOperatorInfo(SOperatorInfo* downstream, SPhysiNode* phy
size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
int32_t num = 0; int32_t num = 0;
SExprInfo* pExprInfo = createExprInfo(pEventWindowNode->window.pFuncs, NULL, &num); SExprInfo* pExprInfo = NULL;
code = createExprInfo(pEventWindowNode->window.pFuncs, NULL, &pExprInfo, &num);
QUERY_CHECK_CODE(code, lino, _error);
initResultSizeInfo(&pOperator->resultInfo, 4096); initResultSizeInfo(&pOperator->resultInfo, 4096);
code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str, code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str,

View File

@ -1822,7 +1822,10 @@ SExprInfo* createExpr(SNodeList* pNodeList, int32_t* numOfExprs) {
return pExprs; return pExprs;
} }
SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* numOfExprs) { int32_t createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, SExprInfo** pExprInfo, int32_t* numOfExprs) {
QRY_OPTR_CHECK(pExprInfo);
int32_t code = 0;
int32_t numOfFuncs = LIST_LENGTH(pNodeList); int32_t numOfFuncs = LIST_LENGTH(pNodeList);
int32_t numOfGroupKeys = 0; int32_t numOfGroupKeys = 0;
if (pGroupKeys != NULL) { if (pGroupKeys != NULL) {
@ -1831,10 +1834,13 @@ SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t*
*numOfExprs = numOfFuncs + numOfGroupKeys; *numOfExprs = numOfFuncs + numOfGroupKeys;
if (*numOfExprs == 0) { if (*numOfExprs == 0) {
return NULL; return code;
} }
SExprInfo* pExprs = taosMemoryCalloc(*numOfExprs, sizeof(SExprInfo)); SExprInfo* pExprs = taosMemoryCalloc(*numOfExprs, sizeof(SExprInfo));
if (pExprs == NULL) {
return terrno;
}
for (int32_t i = 0; i < (*numOfExprs); ++i) { for (int32_t i = 0; i < (*numOfExprs); ++i) {
STargetNode* pTargetNode = NULL; STargetNode* pTargetNode = NULL;
@ -1845,15 +1851,16 @@ SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t*
} }
SExprInfo* pExp = &pExprs[i]; SExprInfo* pExp = &pExprs[i];
int32_t code = createExprFromTargetNode(pExp, pTargetNode); code = createExprFromTargetNode(pExp, pTargetNode);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
taosMemoryFreeClear(pExprs); taosMemoryFreeClear(pExprs);
qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code)); qError("%s failed at line %d since %s", __func__, __LINE__, tstrerror(code));
return NULL; return code;
} }
} }
return pExprs; *pExprInfo = pExprs;
return code;
} }
// set the output buffer for the selectivity + tag query // set the output buffer for the selectivity + tag query
@ -2001,6 +2008,7 @@ _end:
taosMemoryFree(pFuncCtx[i].input.pData); taosMemoryFree(pFuncCtx[i].input.pData);
taosMemoryFree(pFuncCtx[i].input.pColumnDataAgg); taosMemoryFree(pFuncCtx[i].input.pColumnDataAgg);
} }
taosMemoryFreeClear(*rowEntryInfoOffset);
taosMemoryFreeClear(pFuncCtx); taosMemoryFreeClear(pFuncCtx);
return NULL; return NULL;
} }

View File

@ -59,6 +59,7 @@ static void revisedFillStartKey(SFillOperatorInfo* pInfo, SSDataBlock* pBlock, i
static void destroyFillOperatorInfo(void* param); static void destroyFillOperatorInfo(void* param);
static void doApplyScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_t order, int32_t scanFlag); static void doApplyScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_t order, int32_t scanFlag);
static void fillResetPrevForNewGroup(SFillInfo* pFillInfo); static void fillResetPrevForNewGroup(SFillInfo* pFillInfo);
static void reviseFillStartAndEndKey(SFillOperatorInfo* pInfo, int32_t order);
static void doHandleRemainBlockForNewGroupImpl(SOperatorInfo* pOperator, SFillOperatorInfo* pInfo, static void doHandleRemainBlockForNewGroupImpl(SOperatorInfo* pOperator, SFillOperatorInfo* pInfo,
SResultInfo* pResultInfo, int32_t order) { SResultInfo* pResultInfo, int32_t order) {
@ -74,7 +75,7 @@ static void doHandleRemainBlockForNewGroupImpl(SOperatorInfo* pOperator, SFillOp
blockDataCleanup(pInfo->pRes); blockDataCleanup(pInfo->pRes);
doApplyScalarCalculation(pOperator, pInfo->existNewGroupBlock, order, scanFlag); doApplyScalarCalculation(pOperator, pInfo->existNewGroupBlock, order, scanFlag);
revisedFillStartKey(pInfo, pInfo->existNewGroupBlock, order); reviseFillStartAndEndKey(pOperator->info, order);
int64_t ts = (order == TSDB_ORDER_ASC) ? pInfo->existNewGroupBlock->info.window.ekey int64_t ts = (order == TSDB_ORDER_ASC) ? pInfo->existNewGroupBlock->info.window.ekey
: pInfo->existNewGroupBlock->info.window.skey; : pInfo->existNewGroupBlock->info.window.skey;
@ -258,7 +259,7 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) {
if (pInfo->curGroupId == 0 || (pInfo->curGroupId == pInfo->pRes->info.id.groupId)) { if (pInfo->curGroupId == 0 || (pInfo->curGroupId == pInfo->pRes->info.id.groupId)) {
if (pInfo->curGroupId == 0 && taosFillNotStarted(pInfo->pFillInfo)) { if (pInfo->curGroupId == 0 && taosFillNotStarted(pInfo->pFillInfo)) {
revisedFillStartKey(pInfo, pBlock, order); reviseFillStartAndEndKey(pInfo, order);
} }
pInfo->curGroupId = pInfo->pRes->info.id.groupId; // the first data block pInfo->curGroupId = pInfo->pRes->info.id.groupId; // the first data block
@ -454,6 +455,7 @@ int32_t createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* pPhyFi
SExecTaskInfo* pTaskInfo, SOperatorInfo** pOptrInfo) { SExecTaskInfo* pTaskInfo, SOperatorInfo** pOptrInfo) {
QRY_OPTR_CHECK(pOptrInfo); QRY_OPTR_CHECK(pOptrInfo);
int32_t code = 0; int32_t code = 0;
int32_t lino = 0;
SFillOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SFillOperatorInfo)); SFillOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SFillOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
@ -463,21 +465,23 @@ int32_t createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* pPhyFi
} }
pInfo->pRes = createDataBlockFromDescNode(pPhyFillNode->node.pOutputDataBlockDesc); pInfo->pRes = createDataBlockFromDescNode(pPhyFillNode->node.pOutputDataBlockDesc);
SExprInfo* pExprInfo = createExprInfo(pPhyFillNode->pFillExprs, NULL, &pInfo->numOfExpr); SExprInfo* pExprInfo = NULL;
code = createExprInfo(pPhyFillNode->pFillExprs, NULL, &pExprInfo, &pInfo->numOfExpr);
QUERY_CHECK_CODE(code, lino, _error);
pOperator->exprSupp.pExprInfo = pExprInfo; pOperator->exprSupp.pExprInfo = pExprInfo;
SExprSupp* pNoFillSupp = &pInfo->noFillExprSupp; SExprSupp* pNoFillSupp = &pInfo->noFillExprSupp;
pNoFillSupp->pExprInfo = createExprInfo(pPhyFillNode->pNotFillExprs, NULL, &pNoFillSupp->numOfExprs); code = createExprInfo(pPhyFillNode->pNotFillExprs, NULL, &pNoFillSupp->pExprInfo, &pNoFillSupp->numOfExprs);
QUERY_CHECK_CODE(code, lino, _error);
code = createPrimaryTsExprIfNeeded(pInfo, pPhyFillNode, pNoFillSupp, pTaskInfo->id.str); code = createPrimaryTsExprIfNeeded(pInfo, pPhyFillNode, pNoFillSupp, pTaskInfo->id.str);
if (code != TSDB_CODE_SUCCESS) { QUERY_CHECK_CODE(code, lino, _error);
goto _error;
}
code = code =
initExprSupp(pNoFillSupp, pNoFillSupp->pExprInfo, pNoFillSupp->numOfExprs, &pTaskInfo->storageAPI.functionStore); initExprSupp(pNoFillSupp, pNoFillSupp->pExprInfo, pNoFillSupp->numOfExprs, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) { QUERY_CHECK_CODE(code, lino, _error);
goto _error;
}
SInterval* pInterval = SInterval* pInterval =
QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL == downstream->operatorType QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL == downstream->operatorType
@ -549,3 +553,31 @@ _error:
taosMemoryFreeClear(pOperator); taosMemoryFreeClear(pOperator);
return code; return code;
} }
static void reviseFillStartAndEndKey(SFillOperatorInfo* pInfo, int32_t order) {
int64_t skey, ekey, next;
if (order == TSDB_ORDER_ASC) {
skey = taosTimeTruncate(pInfo->win.skey, &pInfo->pFillInfo->interval);
taosFillUpdateStartTimestampInfo(pInfo->pFillInfo, skey);
ekey = taosTimeTruncate(pInfo->win.ekey, &pInfo->pFillInfo->interval);
next = ekey;
while (next < pInfo->win.ekey) {
next = taosTimeAdd(ekey, pInfo->pFillInfo->interval.sliding, pInfo->pFillInfo->interval.slidingUnit,
pInfo->pFillInfo->interval.precision);
ekey = next > pInfo->win.ekey ? ekey : next;
}
pInfo->win.ekey = ekey;
} else {
assert(order == TSDB_ORDER_DESC);
skey = taosTimeTruncate(pInfo->win.skey, &pInfo->pFillInfo->interval);
next = skey;
while (next < pInfo->win.skey) {
next = taosTimeAdd(skey, pInfo->pFillInfo->interval.sliding, pInfo->pFillInfo->interval.slidingUnit,
pInfo->pFillInfo->interval.precision);
skey = next > pInfo->win.skey ? skey : next;
}
taosFillUpdateStartTimestampInfo(pInfo->pFillInfo, skey);
pInfo->win.ekey = taosTimeTruncate(pInfo->win.ekey, &pInfo->pFillInfo->interval);
}
}

View File

@ -551,12 +551,17 @@ int32_t createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* pAggNo
pOperator->exprSupp.hasWindowOrGroup = true; pOperator->exprSupp.hasWindowOrGroup = true;
SSDataBlock* pResBlock = createDataBlockFromDescNode(pAggNode->node.pOutputDataBlockDesc); SSDataBlock* pResBlock = createDataBlockFromDescNode(pAggNode->node.pOutputDataBlockDesc);
if (pResBlock == NULL) {
code = terrno;
goto _error;
}
initBasicInfo(&pInfo->binfo, pResBlock); initBasicInfo(&pInfo->binfo, pResBlock);
int32_t numOfScalarExpr = 0; int32_t numOfScalarExpr = 0;
SExprInfo* pScalarExprInfo = NULL; SExprInfo* pScalarExprInfo = NULL;
if (pAggNode->pExprs != NULL) { if (pAggNode->pExprs != NULL) {
pScalarExprInfo = createExprInfo(pAggNode->pExprs, NULL, &numOfScalarExpr); code = createExprInfo(pAggNode->pExprs, NULL, &pScalarExprInfo, &numOfScalarExpr);
QUERY_CHECK_CODE(code, lino, _error);
} }
pInfo->pGroupCols = NULL; pInfo->pGroupCols = NULL;
@ -574,7 +579,11 @@ int32_t createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* pAggNo
QUERY_CHECK_CODE(code, lino, _error); QUERY_CHECK_CODE(code, lino, _error);
int32_t num = 0; int32_t num = 0;
SExprInfo* pExprInfo = createExprInfo(pAggNode->pAggFuncs, pAggNode->pGroupKeys, &num); SExprInfo* pExprInfo = NULL;
code = createExprInfo(pAggNode->pAggFuncs, pAggNode->pGroupKeys, &pExprInfo, &num);
QUERY_CHECK_CODE(code, lino, _error);
code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, pInfo->groupKeyLen, pTaskInfo->id.str, code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, pInfo->groupKeyLen, pTaskInfo->id.str,
pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore); pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore);
QUERY_CHECK_CODE(code, lino, _error); QUERY_CHECK_CODE(code, lino, _error);
@ -602,6 +611,7 @@ _error:
if (pInfo != NULL) { if (pInfo != NULL) {
destroyGroupOperatorInfo(pInfo); destroyGroupOperatorInfo(pInfo);
} }
destroyOperator(pOperator);
taosMemoryFreeClear(pOperator); taosMemoryFreeClear(pOperator);
return code; return code;
} }
@ -1120,42 +1130,42 @@ int32_t createPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNo
SPartitionOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SPartitionOperatorInfo)); SPartitionOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SPartitionOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) { if (pInfo == NULL || pOperator == NULL) {
pTaskInfo->code = code = TSDB_CODE_OUT_OF_MEMORY; pTaskInfo->code = code = terrno;
goto _error; goto _error;
} }
int32_t numOfCols = 0; int32_t numOfCols = 0;
SExprInfo* pExprInfo = createExprInfo(pPartNode->pTargets, NULL, &numOfCols); SExprInfo* pExprInfo = NULL;
code = createExprInfo(pPartNode->pTargets, NULL, &pExprInfo, &numOfCols);
pInfo->pGroupCols = makeColumnArrayFromList(pPartNode->pPartitionKeys); pInfo->pGroupCols = makeColumnArrayFromList(pPartNode->pPartitionKeys);
if (pPartNode->needBlockOutputTsOrder) { if (pPartNode->needBlockOutputTsOrder) {
SBlockOrderInfo order = {.order = ORDER_ASC, .pColData = NULL, .nullFirst = false, .slotId = pPartNode->tsSlotId}; SBlockOrderInfo order = {.order = ORDER_ASC, .pColData = NULL, .nullFirst = false, .slotId = pPartNode->tsSlotId};
pInfo->pOrderInfoArr = taosArrayInit(1, sizeof(SBlockOrderInfo)); pInfo->pOrderInfoArr = taosArrayInit(1, sizeof(SBlockOrderInfo));
if (!pInfo->pOrderInfoArr) { if (!pInfo->pOrderInfoArr) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
pTaskInfo->code = terrno; pTaskInfo->code = terrno;
goto _error; goto _error;
} }
void* tmp = taosArrayPush(pInfo->pOrderInfoArr, &order); void* tmp = taosArrayPush(pInfo->pOrderInfoArr, &order);
QUERY_CHECK_NULL(tmp, code, lino, _error, terrno); QUERY_CHECK_NULL(tmp, code, lino, _error, terrno);
} }
if (pPartNode->pExprs != NULL) { if (pPartNode->pExprs != NULL) {
int32_t num = 0; int32_t num = 0;
SExprInfo* pExprInfo1 = createExprInfo(pPartNode->pExprs, NULL, &num); SExprInfo* pExprInfo1 = NULL;
code = createExprInfo(pPartNode->pExprs, NULL, &pExprInfo1, &num);
QUERY_CHECK_CODE(code, lino, _error);
code = initExprSupp(&pInfo->scalarSup, pExprInfo1, num, &pTaskInfo->storageAPI.functionStore); code = initExprSupp(&pInfo->scalarSup, pExprInfo1, num, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) { QUERY_CHECK_CODE(code, lino, _error);
terrno = code;
pTaskInfo->code = terrno;
goto _error;
}
} }
_hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
pInfo->pGroupSet = taosHashInit(100, hashFn, false, HASH_NO_LOCK); pInfo->pGroupSet = taosHashInit(100, hashFn, false, HASH_NO_LOCK);
if (pInfo->pGroupSet == NULL) { if (pInfo->pGroupSet == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
pTaskInfo->code = terrno;
goto _error; goto _error;
} }
@ -1165,22 +1175,17 @@ int32_t createPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNo
pInfo->binfo.pRes = createDataBlockFromDescNode(pPartNode->node.pOutputDataBlockDesc); pInfo->binfo.pRes = createDataBlockFromDescNode(pPartNode->node.pOutputDataBlockDesc);
code = getBufferPgSize(pInfo->binfo.pRes->info.rowSize, &defaultPgsz, &defaultBufsz); code = getBufferPgSize(pInfo->binfo.pRes->info.rowSize, &defaultPgsz, &defaultBufsz);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
terrno = code;
pTaskInfo->code = code;
goto _error; goto _error;
} }
if (!osTempSpaceAvailable()) { if (!osTempSpaceAvailable()) {
terrno = TSDB_CODE_NO_DISKSPACE; terrno = TSDB_CODE_NO_DISKSPACE;
pTaskInfo->code = terrno;
qError("Create partition operator info failed since %s, tempDir:%s", terrstr(), tsTempDir); qError("Create partition operator info failed since %s, tempDir:%s", terrstr(), tsTempDir);
goto _error; goto _error;
} }
code = createDiskbasedBuf(&pInfo->pBuf, defaultPgsz, defaultBufsz, pTaskInfo->id.str, tsTempDir); code = createDiskbasedBuf(&pInfo->pBuf, defaultPgsz, defaultBufsz, pTaskInfo->id.str, tsTempDir);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
terrno = code;
pTaskInfo->code = code;
goto _error; goto _error;
} }
@ -1190,8 +1195,6 @@ int32_t createPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNo
pInfo->columnOffset = setupColumnOffset(pInfo->binfo.pRes, pInfo->rowCapacity); pInfo->columnOffset = setupColumnOffset(pInfo->binfo.pRes, pInfo->rowCapacity);
code = initGroupOptrInfo(&pInfo->pGroupColVals, &pInfo->groupKeyLen, &pInfo->keyBuf, pInfo->pGroupCols); code = initGroupOptrInfo(&pInfo->pGroupColVals, &pInfo->groupKeyLen, &pInfo->keyBuf, pInfo->pGroupCols);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
terrno = code;
pTaskInfo->code = code;
goto _error; goto _error;
} }
@ -1205,8 +1208,6 @@ int32_t createPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNo
code = appendDownstream(pOperator, &downstream, 1); code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
terrno = code;
pTaskInfo->code = code;
goto _error; goto _error;
} }
@ -1219,7 +1220,7 @@ _error:
} }
pTaskInfo->code = code; pTaskInfo->code = code;
taosMemoryFreeClear(pOperator); taosMemoryFreeClear(pOperator);
return code; TAOS_RETURN(code);
} }
int32_t setGroupResultOutputBuf(SOperatorInfo* pOperator, SOptrBasicInfo* binfo, int32_t numOfCols, char* pData, int32_t setGroupResultOutputBuf(SOperatorInfo* pOperator, SOptrBasicInfo* binfo, int32_t numOfCols, char* pData,
@ -1658,7 +1659,10 @@ int32_t createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStreamPart
if (pPartNode->part.pExprs != NULL) { if (pPartNode->part.pExprs != NULL) {
int32_t num = 0; int32_t num = 0;
SExprInfo* pCalExprInfo = createExprInfo(pPartNode->part.pExprs, NULL, &num); SExprInfo* pCalExprInfo = NULL;
code = createExprInfo(pPartNode->part.pExprs, NULL, &pCalExprInfo, &num);
QUERY_CHECK_CODE(code, lino, _error);
code = initExprSupp(&pInfo->scalarSup, pCalExprInfo, num, &pTaskInfo->storageAPI.functionStore); code = initExprSupp(&pInfo->scalarSup, pCalExprInfo, num, &pTaskInfo->storageAPI.functionStore);
QUERY_CHECK_CODE(code, lino, _error); QUERY_CHECK_CODE(code, lino, _error);
} }
@ -1719,7 +1723,9 @@ int32_t createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStreamPart
QUERY_CHECK_CODE(code, lino, _error); QUERY_CHECK_CODE(code, lino, _error);
int32_t numOfCols = 0; int32_t numOfCols = 0;
SExprInfo* pExprInfo = createExprInfo(pPartNode->part.pTargets, NULL, &numOfCols); SExprInfo* pExprInfo = NULL;
code = createExprInfo(pPartNode->part.pTargets, NULL, &pExprInfo, &numOfCols);
QUERY_CHECK_CODE(code, lino, _error);
setOperatorInfo(pOperator, "StreamPartitionOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION, false, OP_NOT_OPENED, setOperatorInfo(pOperator, "StreamPartitionOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION, false, OP_NOT_OPENED,
pInfo, pTaskInfo); pInfo, pTaskInfo);

View File

@ -479,6 +479,9 @@ int32_t createOperator(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHand
SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, i); SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, i);
code = createOperator(pChildNode, pTaskInfo, pHandle, pTagCond, pTagIndexCond, pUser, dbname, &ops[i]); code = createOperator(pChildNode, pTaskInfo, pHandle, pTagCond, pTagIndexCond, pUser, dbname, &ops[i]);
if (ops[i] == NULL || code != 0) { if (ops[i] == NULL || code != 0) {
for (int32_t j = 0; j < i; ++j) {
destroyOperator(ops[j]);
}
taosMemoryFree(ops); taosMemoryFree(ops);
return code; return code;
} }

View File

@ -108,9 +108,13 @@ int32_t createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhysiNode*
int32_t lino = 0; int32_t lino = 0;
int32_t numOfCols = 0; int32_t numOfCols = 0;
SExprInfo* pExprInfo = createExprInfo(pProjPhyNode->pProjections, NULL, &numOfCols); SExprInfo* pExprInfo = NULL;
code = createExprInfo(pProjPhyNode->pProjections, NULL, &pExprInfo, &numOfCols);
TSDB_CHECK_CODE(code, lino, _error);
SSDataBlock* pResBlock = createDataBlockFromDescNode(pProjPhyNode->node.pOutputDataBlockDesc); SSDataBlock* pResBlock = createDataBlockFromDescNode(pProjPhyNode->node.pOutputDataBlockDesc);
TSDB_CHECK_NULL(pResBlock, code, lino, _error, terrno);
initLimitInfo(pProjPhyNode->node.pLimit, pProjPhyNode->node.pSlimit, &pInfo->limitInfo); initLimitInfo(pProjPhyNode->node.pLimit, pProjPhyNode->node.pSlimit, &pInfo->limitInfo);
pInfo->binfo.pRes = pResBlock; pInfo->binfo.pRes = pResBlock;
@ -258,7 +262,6 @@ int32_t doProjectOperation(SOperatorInfo* pOperator, SSDataBlock** pResBlock) {
SProjectOperatorInfo* pProjectInfo = pOperator->info; SProjectOperatorInfo* pProjectInfo = pOperator->info;
SOptrBasicInfo* pInfo = &pProjectInfo->binfo; SOptrBasicInfo* pInfo = &pProjectInfo->binfo;
SExprSupp* pSup = &pOperator->exprSupp; SExprSupp* pSup = &pOperator->exprSupp;
SSDataBlock* pRes = pInfo->pRes; SSDataBlock* pRes = pInfo->pRes;
SSDataBlock* pFinalRes = pProjectInfo->pFinalRes; SSDataBlock* pFinalRes = pProjectInfo->pFinalRes;
@ -465,11 +468,16 @@ int32_t createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
SIndefRowsFuncPhysiNode* pPhyNode = (SIndefRowsFuncPhysiNode*)pNode; SIndefRowsFuncPhysiNode* pPhyNode = (SIndefRowsFuncPhysiNode*)pNode;
int32_t numOfExpr = 0; int32_t numOfExpr = 0;
SExprInfo* pExprInfo = createExprInfo(pPhyNode->pFuncs, NULL, &numOfExpr); SExprInfo* pExprInfo = NULL;
code = createExprInfo(pPhyNode->pFuncs, NULL, &pExprInfo, &numOfExpr);
TSDB_CHECK_CODE(code, lino, _error);
if (pPhyNode->pExprs != NULL) { if (pPhyNode->pExprs != NULL) {
int32_t num = 0; int32_t num = 0;
SExprInfo* pSExpr = createExprInfo(pPhyNode->pExprs, NULL, &num); SExprInfo* pSExpr = NULL;
code = createExprInfo(pPhyNode->pExprs, NULL, &pSExpr, &num);
QUERY_CHECK_CODE(code, lino, _error);
code = initExprSupp(&pInfo->scalarSup, pSExpr, num, &pTaskInfo->storageAPI.functionStore); code = initExprSupp(&pInfo->scalarSup, pSExpr, num, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto _error; goto _error;

View File

@ -1338,7 +1338,10 @@ int32_t createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHa
if (pScanNode->pScanPseudoCols != NULL) { if (pScanNode->pScanPseudoCols != NULL) {
SExprSupp* pSup = &pInfo->base.pseudoSup; SExprSupp* pSup = &pInfo->base.pseudoSup;
pSup->pExprInfo = createExprInfo(pScanNode->pScanPseudoCols, NULL, &pSup->numOfExprs); pSup->pExprInfo = NULL;
code = createExprInfo(pScanNode->pScanPseudoCols, NULL, &pSup->pExprInfo, &pSup->numOfExprs);
QUERY_CHECK_CODE(code, lino, _error);
pSup->pCtx = createSqlFunctionCtx(pSup->pExprInfo, pSup->numOfExprs, &pSup->rowEntryInfoOffset, pSup->pCtx = createSqlFunctionCtx(pSup->pExprInfo, pSup->numOfExprs, &pSup->rowEntryInfoOffset,
&pTaskInfo->storageAPI.functionStore); &pTaskInfo->storageAPI.functionStore);
} }
@ -3981,13 +3984,12 @@ int32_t createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode*
// create the pseduo columns info // create the pseduo columns info
if (pTableScanNode->scan.pScanPseudoCols != NULL) { if (pTableScanNode->scan.pScanPseudoCols != NULL) {
pInfo->pPseudoExpr = createExprInfo(pTableScanNode->scan.pScanPseudoCols, NULL, &pInfo->numOfPseudoExpr); code = createExprInfo(pTableScanNode->scan.pScanPseudoCols, NULL, &pInfo->pPseudoExpr, &pInfo->numOfPseudoExpr);
QUERY_CHECK_CODE(code, lino, _error);
} }
code = filterInitFromNode((SNode*)pScanPhyNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0); code = filterInitFromNode((SNode*)pScanPhyNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
if (code != TSDB_CODE_SUCCESS) { QUERY_CHECK_CODE(code, lino, _error);
goto _error;
}
pInfo->pRes = createDataBlockFromDescNode(pDescNode); pInfo->pRes = createDataBlockFromDescNode(pDescNode);
code = createSpecialDataBlock(STREAM_CLEAR, &pInfo->pUpdateRes); code = createSpecialDataBlock(STREAM_CLEAR, &pInfo->pUpdateRes);
@ -4539,7 +4541,11 @@ int32_t createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* p
SDataBlockDescNode* pDescNode = pPhyNode->node.pOutputDataBlockDesc; SDataBlockDescNode* pDescNode = pPhyNode->node.pOutputDataBlockDesc;
int32_t numOfExprs = 0; int32_t numOfExprs = 0;
SExprInfo* pExprInfo = createExprInfo(pPhyNode->pScanPseudoCols, NULL, &numOfExprs); SExprInfo* pExprInfo = NULL;
code = createExprInfo(pPhyNode->pScanPseudoCols, NULL, &pExprInfo, &numOfExprs);
QUERY_CHECK_CODE(code, lino, _error);
code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfExprs, &pTaskInfo->storageAPI.functionStore); code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfExprs, &pTaskInfo->storageAPI.functionStore);
QUERY_CHECK_CODE(code, lino, _error); QUERY_CHECK_CODE(code, lino, _error);
@ -5694,7 +5700,9 @@ int32_t createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SR
if (pTableScanNode->scan.pScanPseudoCols != NULL) { if (pTableScanNode->scan.pScanPseudoCols != NULL) {
SExprSupp* pSup = &pInfo->base.pseudoSup; SExprSupp* pSup = &pInfo->base.pseudoSup;
pSup->pExprInfo = createExprInfo(pTableScanNode->scan.pScanPseudoCols, NULL, &pSup->numOfExprs); code = createExprInfo(pTableScanNode->scan.pScanPseudoCols, NULL, &pSup->pExprInfo, &pSup->numOfExprs);
QUERY_CHECK_CODE(code, lino, _error);
pSup->pCtx = createSqlFunctionCtx(pSup->pExprInfo, pSup->numOfExprs, &pSup->rowEntryInfoOffset, pSup->pCtx = createSqlFunctionCtx(pSup->pExprInfo, pSup->numOfExprs, &pSup->rowEntryInfoOffset,
&pTaskInfo->storageAPI.functionStore); &pTaskInfo->storageAPI.functionStore);
} }

View File

@ -60,6 +60,8 @@ int32_t createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* pSortN
QRY_OPTR_CHECK(pOptrInfo); QRY_OPTR_CHECK(pOptrInfo);
int32_t code = 0; int32_t code = 0;
int32_t lino = 0;
SSortOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSortOperatorInfo)); SSortOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSortOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) { if (pInfo == NULL || pOperator == NULL) {
@ -71,7 +73,9 @@ int32_t createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* pSortN
SDataBlockDescNode* pDescNode = pSortNode->node.pOutputDataBlockDesc; SDataBlockDescNode* pDescNode = pSortNode->node.pOutputDataBlockDesc;
int32_t numOfCols = 0; int32_t numOfCols = 0;
pOperator->exprSupp.pExprInfo = createExprInfo(pSortNode->pExprs, NULL, &numOfCols); code = createExprInfo(pSortNode->pExprs, NULL, &pOperator->exprSupp.pExprInfo, &numOfCols);
QUERY_CHECK_CODE(code, lino, _error);
pOperator->exprSupp.numOfExprs = numOfCols; pOperator->exprSupp.numOfExprs = numOfCols;
int32_t numOfOutputCols = 0; int32_t numOfOutputCols = 0;
code = code =
@ -770,7 +774,9 @@ int32_t createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSortPhysiNo
SDataBlockDescNode* pDescNode = pSortPhyNode->node.pOutputDataBlockDesc; SDataBlockDescNode* pDescNode = pSortPhyNode->node.pOutputDataBlockDesc;
int32_t numOfCols = 0; int32_t numOfCols = 0;
SExprInfo* pExprInfo = createExprInfo(pSortPhyNode->pExprs, NULL, &numOfCols); SExprInfo* pExprInfo = NULL;
code = createExprInfo(pSortPhyNode->pExprs, NULL, &pExprInfo, &numOfCols);
QUERY_CHECK_CODE(code, lino, _error);
pSup->pExprInfo = pExprInfo; pSup->pExprInfo = pExprInfo;
pSup->numOfExprs = numOfCols; pSup->numOfExprs = numOfCols;

View File

@ -823,13 +823,19 @@ int32_t createStreamCountAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
initResultSizeInfo(&pOperator->resultInfo, 4096); initResultSizeInfo(&pOperator->resultInfo, 4096);
if (pCountNode->window.pExprs != NULL) { if (pCountNode->window.pExprs != NULL) {
int32_t numOfScalar = 0; int32_t numOfScalar = 0;
SExprInfo* pScalarExprInfo = createExprInfo(pCountNode->window.pExprs, NULL, &numOfScalar); SExprInfo* pScalarExprInfo = NULL;
code = createExprInfo(pCountNode->window.pExprs, NULL, &pScalarExprInfo, &numOfScalar);
QUERY_CHECK_CODE(code, lino, _error);
code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore); code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore);
QUERY_CHECK_CODE(code, lino, _error); QUERY_CHECK_CODE(code, lino, _error);
} }
SExprSupp* pExpSup = &pOperator->exprSupp; SExprSupp* pExpSup = &pOperator->exprSupp;
SExprInfo* pExprInfo = createExprInfo(pCountNode->window.pFuncs, NULL, &numOfCols); SExprInfo* pExprInfo = NULL;
code = createExprInfo(pCountNode->window.pFuncs, NULL, &pExprInfo, &numOfCols);
QUERY_CHECK_CODE(code, lino, _error);
SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc); SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc);
code = initBasicInfoEx(&pInfo->binfo, pExpSup, pExprInfo, numOfCols, pResBlock, &pTaskInfo->storageAPI.functionStore); code = initBasicInfoEx(&pInfo->binfo, pExpSup, pExprInfo, numOfCols, pResBlock, &pTaskInfo->storageAPI.functionStore);
QUERY_CHECK_CODE(code, lino, _error); QUERY_CHECK_CODE(code, lino, _error);

View File

@ -864,7 +864,10 @@ int32_t createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
initResultSizeInfo(&pOperator->resultInfo, 4096); initResultSizeInfo(&pOperator->resultInfo, 4096);
if (pEventNode->window.pExprs != NULL) { if (pEventNode->window.pExprs != NULL) {
int32_t numOfScalar = 0; int32_t numOfScalar = 0;
SExprInfo* pScalarExprInfo = createExprInfo(pEventNode->window.pExprs, NULL, &numOfScalar); SExprInfo* pScalarExprInfo = NULL;
code = createExprInfo(pEventNode->window.pExprs, NULL, &pScalarExprInfo, &numOfScalar);
QUERY_CHECK_CODE(code, lino, _error);
code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore); code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto _error; goto _error;
@ -884,7 +887,10 @@ int32_t createStreamEventAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
SExprSupp* pExpSup = &pOperator->exprSupp; SExprSupp* pExpSup = &pOperator->exprSupp;
int32_t numOfCols = 0; int32_t numOfCols = 0;
SExprInfo* pExprInfo = createExprInfo(pEventNode->window.pFuncs, NULL, &numOfCols); SExprInfo* pExprInfo = NULL;
code = createExprInfo(pEventNode->window.pFuncs, NULL, &pExprInfo, &numOfCols);
QUERY_CHECK_CODE(code, lino, _error);
SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc); SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc);
code = initBasicInfoEx(&pInfo->binfo, pExpSup, pExprInfo, numOfCols, pResBlock, &pTaskInfo->storageAPI.functionStore); code = initBasicInfoEx(&pInfo->binfo, pExpSup, pExprInfo, numOfCols, pResBlock, &pTaskInfo->storageAPI.functionStore);
QUERY_CHECK_CODE(code, lino, _error); QUERY_CHECK_CODE(code, lino, _error);

View File

@ -470,6 +470,7 @@ static int32_t checkResult(SStreamFillSupporter* pFillSup, TSKEY ts, uint64_t gr
SWinKey key = {.groupId = groupId, .ts = ts}; SWinKey key = {.groupId = groupId, .ts = ts};
if (tSimpleHashGet(pFillSup->pResMap, &key, sizeof(SWinKey)) != NULL) { if (tSimpleHashGet(pFillSup->pResMap, &key, sizeof(SWinKey)) != NULL) {
(*pRes) = false; (*pRes) = false;
goto _end;
} }
code = tSimpleHashPut(pFillSup->pResMap, &key, sizeof(SWinKey), NULL, 0); code = tSimpleHashPut(pFillSup->pResMap, &key, sizeof(SWinKey), NULL, 0);
QUERY_CHECK_CODE(code, lino, _end); QUERY_CHECK_CODE(code, lino, _end);
@ -1189,7 +1190,11 @@ static SStreamFillSupporter* initStreamFillSup(SStreamFillPhysiNode* pPhyFillNod
} }
pFillSup->numOfFillCols = numOfFillCols; pFillSup->numOfFillCols = numOfFillCols;
int32_t numOfNotFillCols = 0; int32_t numOfNotFillCols = 0;
SExprInfo* noFillExprInfo = createExprInfo(pPhyFillNode->pNotFillExprs, NULL, &numOfNotFillCols); SExprInfo* noFillExprInfo = NULL;
code = createExprInfo(pPhyFillNode->pNotFillExprs, NULL, &noFillExprInfo, &numOfNotFillCols);
QUERY_CHECK_CODE(code, lino, _end);
pFillSup->pAllColInfo = createFillColInfo(pFillExprInfo, pFillSup->numOfFillCols, noFillExprInfo, numOfNotFillCols, pFillSup->pAllColInfo = createFillColInfo(pFillExprInfo, pFillSup->numOfFillCols, noFillExprInfo, numOfNotFillCols,
(const SNodeListNode*)(pPhyFillNode->pValues)); (const SNodeListNode*)(pPhyFillNode->pValues));
pFillSup->type = convertFillType(pPhyFillNode->mode); pFillSup->type = convertFillType(pPhyFillNode->mode);
@ -1200,7 +1205,10 @@ static SStreamFillSupporter* initStreamFillSup(SStreamFillPhysiNode* pPhyFillNod
code = initResultBuf(pFillSup); code = initResultBuf(pFillSup);
QUERY_CHECK_CODE(code, lino, _end); QUERY_CHECK_CODE(code, lino, _end);
SExprInfo* noFillExpr = createExprInfo(pPhyFillNode->pNotFillExprs, NULL, &numOfNotFillCols); SExprInfo* noFillExpr = NULL;
code = createExprInfo(pPhyFillNode->pNotFillExprs, NULL, &noFillExpr, &numOfNotFillCols);
QUERY_CHECK_CODE(code, lino, _end);
code = initExprSupp(&pFillSup->notFillExprSup, noFillExpr, numOfNotFillCols, &pAPI->functionStore); code = initExprSupp(&pFillSup->notFillExprSup, noFillExpr, numOfNotFillCols, &pAPI->functionStore);
QUERY_CHECK_CODE(code, lino, _end); QUERY_CHECK_CODE(code, lino, _end);
@ -1342,7 +1350,11 @@ int32_t createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFillPhysi
SInterval* pInterval = &((SStreamIntervalOperatorInfo*)downstream->info)->interval; SInterval* pInterval = &((SStreamIntervalOperatorInfo*)downstream->info)->interval;
int32_t numOfFillCols = 0; int32_t numOfFillCols = 0;
SExprInfo* pFillExprInfo = createExprInfo(pPhyFillNode->pFillExprs, NULL, &numOfFillCols); SExprInfo* pFillExprInfo = NULL;
code = createExprInfo(pPhyFillNode->pFillExprs, NULL, &pFillExprInfo, &numOfFillCols);
QUERY_CHECK_CODE(code, lino, _error);
pInfo->pFillSup = initStreamFillSup(pPhyFillNode, pInterval, pFillExprInfo, numOfFillCols, &pTaskInfo->storageAPI); pInfo->pFillSup = initStreamFillSup(pPhyFillNode, pInterval, pFillExprInfo, numOfFillCols, &pTaskInfo->storageAPI);
if (!pInfo->pFillSup) { if (!pInfo->pFillSup) {
code = TSDB_CODE_FAILED; code = TSDB_CODE_FAILED;

View File

@ -1880,13 +1880,20 @@ int32_t createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiN
initResultSizeInfo(&pOperator->resultInfo, 4096); initResultSizeInfo(&pOperator->resultInfo, 4096);
if (pIntervalPhyNode->window.pExprs != NULL) { if (pIntervalPhyNode->window.pExprs != NULL) {
int32_t numOfScalar = 0; int32_t numOfScalar = 0;
SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar); SExprInfo* pScalarExprInfo = NULL;
code = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &pScalarExprInfo, &numOfScalar);
QUERY_CHECK_CODE(code, lino, _error);
code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore); code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore);
QUERY_CHECK_CODE(code, lino, _error); QUERY_CHECK_CODE(code, lino, _error);
} }
int32_t numOfCols = 0; int32_t numOfCols = 0;
SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &numOfCols); SExprInfo* pExprInfo = NULL;
code = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &pExprInfo, &numOfCols);
QUERY_CHECK_CODE(code, lino, _error);
SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc); SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc);
initBasicInfo(&pInfo->binfo, pResBlock); initBasicInfo(&pInfo->binfo, pResBlock);
@ -3690,7 +3697,10 @@ int32_t createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode
initResultSizeInfo(&pOperator->resultInfo, 4096); initResultSizeInfo(&pOperator->resultInfo, 4096);
if (pSessionNode->window.pExprs != NULL) { if (pSessionNode->window.pExprs != NULL) {
int32_t numOfScalar = 0; int32_t numOfScalar = 0;
SExprInfo* pScalarExprInfo = createExprInfo(pSessionNode->window.pExprs, NULL, &numOfScalar); SExprInfo* pScalarExprInfo = NULL;
code = createExprInfo(pSessionNode->window.pExprs, NULL, &pScalarExprInfo, &numOfScalar);
QUERY_CHECK_CODE(code, lino, _error);
code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore); code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto _error; goto _error;
@ -3698,7 +3708,10 @@ int32_t createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode
} }
SExprSupp* pExpSup = &pOperator->exprSupp; SExprSupp* pExpSup = &pOperator->exprSupp;
SExprInfo* pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &numOfCols); SExprInfo* pExprInfo = NULL;
code = createExprInfo(pSessionNode->window.pFuncs, NULL, &pExprInfo, &numOfCols);
QUERY_CHECK_CODE(code, lino, _error);
SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc); SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc);
code = initBasicInfoEx(&pInfo->binfo, pExpSup, pExprInfo, numOfCols, pResBlock, &pTaskInfo->storageAPI.functionStore); code = initBasicInfoEx(&pInfo->binfo, pExpSup, pExprInfo, numOfCols, pResBlock, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
@ -4831,7 +4844,10 @@ int32_t createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
initResultSizeInfo(&pOperator->resultInfo, 4096); initResultSizeInfo(&pOperator->resultInfo, 4096);
if (pStateNode->window.pExprs != NULL) { if (pStateNode->window.pExprs != NULL) {
int32_t numOfScalar = 0; int32_t numOfScalar = 0;
SExprInfo* pScalarExprInfo = createExprInfo(pStateNode->window.pExprs, NULL, &numOfScalar); SExprInfo* pScalarExprInfo = NULL;
code = createExprInfo(pStateNode->window.pExprs, NULL, &pScalarExprInfo, &numOfScalar);
QUERY_CHECK_CODE(code, lino, _error);
code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore); code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore);
QUERY_CHECK_CODE(code, lino, _error); QUERY_CHECK_CODE(code, lino, _error);
} }
@ -4849,7 +4865,10 @@ int32_t createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
SExprSupp* pExpSup = &pOperator->exprSupp; SExprSupp* pExpSup = &pOperator->exprSupp;
int32_t numOfCols = 0; int32_t numOfCols = 0;
SExprInfo* pExprInfo = createExprInfo(pStateNode->window.pFuncs, NULL, &numOfCols); SExprInfo* pExprInfo = NULL;
code = createExprInfo(pStateNode->window.pFuncs, NULL, &pExprInfo, &numOfCols);
QUERY_CHECK_CODE(code, lino, _error);
SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc); SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc);
code = initBasicInfoEx(&pInfo->binfo, pExpSup, pExprInfo, numOfCols, pResBlock, &pTaskInfo->storageAPI.functionStore); code = initBasicInfoEx(&pInfo->binfo, pExpSup, pExprInfo, numOfCols, pResBlock, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
@ -5126,7 +5145,10 @@ int32_t createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
} }
SStreamIntervalPhysiNode* pIntervalPhyNode = (SStreamIntervalPhysiNode*)pPhyNode; SStreamIntervalPhysiNode* pIntervalPhyNode = (SStreamIntervalPhysiNode*)pPhyNode;
SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &numOfCols);
SExprInfo* pExprInfo = NULL;
code = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &pExprInfo, &numOfCols);
QUERY_CHECK_CODE(code, lino, _error);
SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc); SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc);
pInfo->interval = (SInterval){ pInfo->interval = (SInterval){
@ -5174,7 +5196,11 @@ int32_t createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode*
if (pIntervalPhyNode->window.pExprs != NULL) { if (pIntervalPhyNode->window.pExprs != NULL) {
int32_t numOfScalar = 0; int32_t numOfScalar = 0;
SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar); SExprInfo* pScalarExprInfo = NULL;
code = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &pScalarExprInfo, &numOfScalar);
QUERY_CHECK_CODE(code, lino, _error);
code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore); code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore);
QUERY_CHECK_CODE(code, lino, _error); QUERY_CHECK_CODE(code, lino, _error);
} }

View File

@ -2716,7 +2716,10 @@ int32_t createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDistScanP
pInfo->uid = (pBlockScanNode->suid != 0) ? pBlockScanNode->suid : pBlockScanNode->uid; pInfo->uid = (pBlockScanNode->suid != 0) ? pBlockScanNode->suid : pBlockScanNode->uid;
int32_t numOfCols = 0; int32_t numOfCols = 0;
SExprInfo* pExprInfo = createExprInfo(pBlockScanNode->pScanPseudoCols, NULL, &numOfCols); SExprInfo* pExprInfo = NULL;
code = createExprInfo(pBlockScanNode->pScanPseudoCols, NULL, &pExprInfo, &numOfCols);
QUERY_CHECK_CODE(code, lino, _error);
code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfCols, &pTaskInfo->storageAPI.functionStore); code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfCols, &pTaskInfo->storageAPI.functionStore);
QUERY_CHECK_CODE(code, lino, _error); QUERY_CHECK_CODE(code, lino, _error);

View File

@ -1126,13 +1126,19 @@ int32_t createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyN
SExprSupp* pSup = &pOperator->exprSupp; SExprSupp* pSup = &pOperator->exprSupp;
int32_t numOfExprs = 0; int32_t numOfExprs = 0;
SExprInfo* pExprInfo = createExprInfo(pInterpPhyNode->pFuncs, NULL, &numOfExprs); SExprInfo* pExprInfo = NULL;
code = createExprInfo(pInterpPhyNode->pFuncs, NULL, &pExprInfo, &numOfExprs);
QUERY_CHECK_CODE(code, lino, _error);
code = initExprSupp(pSup, pExprInfo, numOfExprs, &pTaskInfo->storageAPI.functionStore); code = initExprSupp(pSup, pExprInfo, numOfExprs, &pTaskInfo->storageAPI.functionStore);
QUERY_CHECK_CODE(code, lino, _error); QUERY_CHECK_CODE(code, lino, _error);
if (pInterpPhyNode->pExprs != NULL) { if (pInterpPhyNode->pExprs != NULL) {
int32_t num = 0; int32_t num = 0;
SExprInfo* pScalarExprInfo = createExprInfo(pInterpPhyNode->pExprs, NULL, &num); SExprInfo* pScalarExprInfo = NULL;
code = createExprInfo(pInterpPhyNode->pExprs, NULL, &pScalarExprInfo, &num);
QUERY_CHECK_CODE(code, lino, _error);
code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, num, &pTaskInfo->storageAPI.functionStore); code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, num, &pTaskInfo->storageAPI.functionStore);
QUERY_CHECK_CODE(code, lino, _error); QUERY_CHECK_CODE(code, lino, _error);
} }

View File

@ -1298,7 +1298,10 @@ int32_t createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPhysiNode
QUERY_CHECK_CODE(code, lino, _error); QUERY_CHECK_CODE(code, lino, _error);
int32_t num = 0; int32_t num = 0;
SExprInfo* pExprInfo = createExprInfo(pPhyNode->window.pFuncs, NULL, &num); SExprInfo* pExprInfo = NULL;
code = createExprInfo(pPhyNode->window.pFuncs, NULL, &pExprInfo, &num);
QUERY_CHECK_CODE(code, lino, _error);
code = initAggSup(pSup, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str, pTaskInfo->streamInfo.pState, code = initAggSup(pSup, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str, pTaskInfo->streamInfo.pState,
&pTaskInfo->storageAPI.functionStore); &pTaskInfo->storageAPI.functionStore);
QUERY_CHECK_CODE(code, lino, _error); QUERY_CHECK_CODE(code, lino, _error);
@ -1336,7 +1339,10 @@ int32_t createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPhysiNode
if (pPhyNode->window.pExprs != NULL) { if (pPhyNode->window.pExprs != NULL) {
int32_t numOfScalar = 0; int32_t numOfScalar = 0;
SExprInfo* pScalarExprInfo = createExprInfo(pPhyNode->window.pExprs, NULL, &numOfScalar); SExprInfo* pScalarExprInfo = NULL;
code = createExprInfo(pPhyNode->window.pExprs, NULL, &pScalarExprInfo, &numOfScalar);
QUERY_CHECK_CODE(code, lino, _error);
code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore); code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto _error; goto _error;
@ -1578,7 +1584,10 @@ int32_t createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWinodwPhy
if (pStateNode->window.pExprs != NULL) { if (pStateNode->window.pExprs != NULL) {
int32_t numOfScalarExpr = 0; int32_t numOfScalarExpr = 0;
SExprInfo* pScalarExprInfo = createExprInfo(pStateNode->window.pExprs, NULL, &numOfScalarExpr); SExprInfo* pScalarExprInfo = NULL;
code = createExprInfo(pStateNode->window.pExprs, NULL, &pScalarExprInfo, &numOfScalarExpr);
QUERY_CHECK_CODE(code, lino, _error);
code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr, &pTaskInfo->storageAPI.functionStore); code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto _error; goto _error;
@ -1603,7 +1612,10 @@ int32_t createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWinodwPhy
size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
int32_t num = 0; int32_t num = 0;
SExprInfo* pExprInfo = createExprInfo(pStateNode->window.pFuncs, NULL, &num); SExprInfo* pExprInfo = NULL;
code = createExprInfo(pStateNode->window.pFuncs, NULL, &pExprInfo, &num);
QUERY_CHECK_CODE(code, lino, _error);
initResultSizeInfo(&pOperator->resultInfo, 4096); initResultSizeInfo(&pOperator->resultInfo, 4096);
code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str, code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str,
@ -1682,7 +1694,10 @@ int32_t createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionWinodwPh
initResultSizeInfo(&pOperator->resultInfo, 4096); initResultSizeInfo(&pOperator->resultInfo, 4096);
int32_t numOfCols = 0; int32_t numOfCols = 0;
SExprInfo* pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &numOfCols); SExprInfo* pExprInfo = NULL;
code = createExprInfo(pSessionNode->window.pFuncs, NULL, &pExprInfo, &numOfCols);
QUERY_CHECK_CODE(code, lino, _error);
SSDataBlock* pResBlock = createDataBlockFromDescNode(pSessionNode->window.node.pOutputDataBlockDesc); SSDataBlock* pResBlock = createDataBlockFromDescNode(pSessionNode->window.node.pOutputDataBlockDesc);
initBasicInfo(&pInfo->binfo, pResBlock); initBasicInfo(&pInfo->binfo, pResBlock);
@ -1709,7 +1724,10 @@ int32_t createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionWinodwPh
if (pSessionNode->window.pExprs != NULL) { if (pSessionNode->window.pExprs != NULL) {
int32_t numOfScalar = 0; int32_t numOfScalar = 0;
SExprInfo* pScalarExprInfo = createExprInfo(pSessionNode->window.pExprs, NULL, &numOfScalar); SExprInfo* pScalarExprInfo = NULL;
code = createExprInfo(pSessionNode->window.pExprs, NULL, &pScalarExprInfo, &numOfScalar);
QUERY_CHECK_CODE(code, lino, _error);
code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore); code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar, &pTaskInfo->storageAPI.functionStore);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto _error; goto _error;
@ -2012,7 +2030,9 @@ int32_t createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, SMerge
initResultSizeInfo(&pOperator->resultInfo, 512); initResultSizeInfo(&pOperator->resultInfo, 512);
int32_t num = 0; int32_t num = 0;
SExprInfo* pExprInfo = createExprInfo(pNode->window.pFuncs, NULL, &num); SExprInfo* pExprInfo = NULL;
code = createExprInfo(pNode->window.pFuncs, NULL, &pExprInfo, &num);
QUERY_CHECK_CODE(code, lino, _error);
code = initAggSup(&pOperator->exprSupp, &iaInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str, code = initAggSup(&pOperator->exprSupp, &iaInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str,
pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore); pTaskInfo->streamInfo.pState, &pTaskInfo->storageAPI.functionStore);
@ -2312,7 +2332,9 @@ int32_t createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMergeInterva
} }
int32_t num = 0; int32_t num = 0;
SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &num); SExprInfo* pExprInfo = NULL;
code = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &pExprInfo, &num);
QUERY_CHECK_CODE(code, lino, _error);
SInterval interval = {.interval = pIntervalPhyNode->interval, SInterval interval = {.interval = pIntervalPhyNode->interval,
.sliding = pIntervalPhyNode->sliding, .sliding = pIntervalPhyNode->sliding,

View File

@ -1699,6 +1699,7 @@ static int32_t initRowIdSort(SSortHandle* pHandle) {
taosArrayDestroy(pHandle->pSortInfo); taosArrayDestroy(pHandle->pSortInfo);
pHandle->pSortInfo = pOrderInfoList; pHandle->pSortInfo = pOrderInfoList;
pHandle->cmpParam.pPkOrder = (pHandle->bSortPk) ? taosArrayGet(pHandle->pSortInfo, 1) : NULL;
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }

View File

@ -5401,7 +5401,7 @@ static int32_t translateFill(STranslateContext* pCxt, SSelectStmt* pSelect, SInt
return checkFill(pCxt, (SFillNode*)pInterval->pFill, (SValueNode*)pInterval->pInterval, false); return checkFill(pCxt, (SFillNode*)pInterval->pFill, (SValueNode*)pInterval->pInterval, false);
} }
static int32_t getMonthsFromTimeVal(int64_t val, int32_t fromPrecision, char unit, int64_t* pMonth) { static int32_t getMonthsFromTimeVal(int64_t val, int32_t fromPrecision, char unit, double* pMonth) {
int64_t days = -1; int64_t days = -1;
int32_t code = convertTimeFromPrecisionToUnit(val, fromPrecision, 'd', &days); int32_t code = convertTimeFromPrecisionToUnit(val, fromPrecision, 'd', &days);
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
@ -5416,7 +5416,7 @@ static int32_t getMonthsFromTimeVal(int64_t val, int32_t fromPrecision, char uni
case 'h': case 'h':
case 'd': case 'd':
case 'w': case 'w':
*pMonth = days / 28; *pMonth = days / 28.0;
return code; return code;
case 'n': case 'n':
*pMonth = val; *pMonth = val;
@ -5499,7 +5499,7 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode*
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_OFFSET_TOO_BIG); return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_OFFSET_TOO_BIG);
} }
if (!fixed) { if (!fixed) {
int64_t offsetMonth = 0, intervalMonth = 0; double offsetMonth = 0, intervalMonth = 0;
int32_t code = getMonthsFromTimeVal(pOffset->datum.i, precision, pOffset->unit, &offsetMonth); int32_t code = getMonthsFromTimeVal(pOffset->datum.i, precision, pOffset->unit, &offsetMonth);
if (TSDB_CODE_SUCCESS != code) { if (TSDB_CODE_SUCCESS != code) {
return code; return code;
@ -5530,7 +5530,21 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode*
(pInter->datum.i / pSliding->datum.i > INTERVAL_SLIDING_FACTOR)) { (pInter->datum.i / pSliding->datum.i > INTERVAL_SLIDING_FACTOR)) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_SLIDING_TOO_SMALL); return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_SLIDING_TOO_SMALL);
} }
if (pSliding->datum.i > pInter->datum.i) { if (valInter) {
double slidingMonth = 0, intervalMonth = 0;
int32_t code = getMonthsFromTimeVal(pSliding->datum.i, precision, pSliding->unit, &slidingMonth);
if (TSDB_CODE_SUCCESS != code) {
return code;
}
code = getMonthsFromTimeVal(pInter->datum.i, precision, pInter->unit, &intervalMonth);
if (TSDB_CODE_SUCCESS != code) {
return code;
}
if (slidingMonth > intervalMonth) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_SLIDING_TOO_BIG);
}
}
if (!valInter && pSliding->datum.i > pInter->datum.i) {
return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_SLIDING_TOO_BIG); return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_SLIDING_TOO_BIG);
} }
} }

View File

@ -276,7 +276,9 @@ void qwFreeTaskHandle(qTaskInfo_t *taskHandle) {
// Note: free/kill may in RC // Note: free/kill may in RC
qTaskInfo_t otaskHandle = atomic_load_ptr(taskHandle); qTaskInfo_t otaskHandle = atomic_load_ptr(taskHandle);
if (otaskHandle && atomic_val_compare_exchange_ptr(taskHandle, otaskHandle, NULL)) { if (otaskHandle && atomic_val_compare_exchange_ptr(taskHandle, otaskHandle, NULL)) {
tsEnableRandErr = true;
qDestroyTask(otaskHandle); qDestroyTask(otaskHandle);
tsEnableRandErr = false;
qDebug("task handle destroyed"); qDebug("task handle destroyed");
} }
} }

View File

@ -161,7 +161,9 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) {
if (taskHandle) { if (taskHandle) {
qwDbgSimulateSleep(); qwDbgSimulateSleep();
tsEnableRandErr = true;
code = qExecTaskOpt(taskHandle, pResList, &useconds, &hasMore, &localFetch); code = qExecTaskOpt(taskHandle, pResList, &useconds, &hasMore, &localFetch);
tsEnableRandErr = false;
if (code) { if (code) {
if (code != TSDB_CODE_OPS_NOT_SUPPORT) { if (code != TSDB_CODE_OPS_NOT_SUPPORT) {
QW_TASK_ELOG("qExecTask failed, code:%x - %s", code, tstrerror(code)); QW_TASK_ELOG("qExecTask failed, code:%x - %s", code, tstrerror(code));
@ -768,8 +770,11 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, char *sql) {
QW_ERR_JRET(code); QW_ERR_JRET(code);
} }
tsEnableRandErr = true;
code = qCreateExecTask(qwMsg->node, mgmt->nodeId, tId, plan, &pTaskInfo, &sinkHandle, qwMsg->msgInfo.compressMsg, sql, code = qCreateExecTask(qwMsg->node, mgmt->nodeId, tId, plan, &pTaskInfo, &sinkHandle, qwMsg->msgInfo.compressMsg, sql,
OPTR_EXEC_MODEL_BATCH); OPTR_EXEC_MODEL_BATCH);
tsEnableRandErr = false;
sql = NULL; sql = NULL;
if (code) { if (code) {
QW_TASK_ELOG("qCreateExecTask failed, code:%x - %s", code, tstrerror(code)); QW_TASK_ELOG("qCreateExecTask failed, code:%x - %s", code, tstrerror(code));
@ -1266,7 +1271,10 @@ int32_t qwProcessDelete(QW_FPARAMS_DEF, SQWMsg *qwMsg, SDeleteRes *pRes) {
QW_ERR_JRET(code); QW_ERR_JRET(code);
} }
tsEnableRandErr = true;
code = qCreateExecTask(qwMsg->node, mgmt->nodeId, tId, plan, &pTaskInfo, &sinkHandle, 0, NULL, OPTR_EXEC_MODEL_BATCH); code = qCreateExecTask(qwMsg->node, mgmt->nodeId, tId, plan, &pTaskInfo, &sinkHandle, 0, NULL, OPTR_EXEC_MODEL_BATCH);
tsEnableRandErr = false;
if (code) { if (code) {
QW_TASK_ELOG("qCreateExecTask failed, code:%x - %s", code, tstrerror(code)); QW_TASK_ELOG("qCreateExecTask failed, code:%x - %s", code, tstrerror(code));
QW_ERR_JRET(code); QW_ERR_JRET(code);

View File

@ -81,6 +81,7 @@ typedef struct {
int64_t dataWritten; int64_t dataWritten;
void* pMeta; void* pMeta;
int8_t removeAllFiles;
} STaskDbWrapper; } STaskDbWrapper;
@ -152,6 +153,8 @@ void taskDbUpdateChkpId(void* pTaskDb, int64_t chkpId);
void* taskDbAddRef(void* pTaskDb); void* taskDbAddRef(void* pTaskDb);
void taskDbRemoveRef(void* pTaskDb); void taskDbRemoveRef(void* pTaskDb);
void taskDbSetClearFileFlag(void* pTaskDb);
int streamStateOpenBackend(void* backend, SStreamState* pState); int streamStateOpenBackend(void* backend, SStreamState* pState);
void streamStateCloseBackend(SStreamState* pState, bool remove); void streamStateCloseBackend(SStreamState* pState, bool remove);
void streamStateDestroyCompar(void* arg); void streamStateDestroyCompar(void* arg);

View File

@ -2331,6 +2331,15 @@ void taskDbRemoveRef(void* pTaskDb) {
(void)taosReleaseRef(taskDbWrapperId, pBackend->refId); (void)taosReleaseRef(taskDbWrapperId, pBackend->refId);
} }
void taskDbSetClearFileFlag(void* pTaskDb) {
if (pTaskDb == NULL) {
return;
}
STaskDbWrapper* pBackend = pTaskDb;
atomic_store_8(&pBackend->removeAllFiles, 1);
}
void taskDbInitOpt(STaskDbWrapper* pTaskDb) { void taskDbInitOpt(STaskDbWrapper* pTaskDb) {
rocksdb_env_t* env = rocksdb_create_default_env(); rocksdb_env_t* env = rocksdb_create_default_env();
@ -2573,8 +2582,7 @@ void taskDbDestroy(void* pDb, bool flush) {
stDebug("succ to destroy stream backend:%p", wrapper); stDebug("succ to destroy stream backend:%p", wrapper);
int8_t nCf = tListLen(ginitDict); int8_t nCf = tListLen(ginitDict);
if (flush && wrapper->removeAllFiles == 0) {
if (flush) {
if (wrapper->db && wrapper->pCf) { if (wrapper->db && wrapper->pCf) {
rocksdb_flushoptions_t* flushOpt = rocksdb_flushoptions_create(); rocksdb_flushoptions_t* flushOpt = rocksdb_flushoptions_create();
rocksdb_flushoptions_set_wait(flushOpt, 1); rocksdb_flushoptions_set_wait(flushOpt, 1);
@ -2636,6 +2644,11 @@ void taskDbDestroy(void* pDb, bool flush) {
taskDbDestroyChkpOpt(wrapper); taskDbDestroyChkpOpt(wrapper);
taosMemoryFree(wrapper->idstr); taosMemoryFree(wrapper->idstr);
if (wrapper->removeAllFiles) {
char* err = NULL;
taosRemoveDir(wrapper->path);
}
taosMemoryFree(wrapper->path); taosMemoryFree(wrapper->path);
taosMemoryFree(wrapper); taosMemoryFree(wrapper);

View File

@ -94,12 +94,17 @@ int32_t appendCheckpointIntoInputQ(SStreamTask* pTask, int32_t checkpointType, i
} }
int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSourceReq* pReq) { int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSourceReq* pReq) {
ASSERT(pTask->info.taskLevel == TASK_LEVEL__SOURCE); if (pTask->info.taskLevel != TASK_LEVEL__SOURCE) {
return TSDB_CODE_INVALID_MSG;
}
// todo this status may not be set here. // todo this status may not be set here.
// 1. set task status to be prepared for check point, no data are allowed to put into inputQ. // 1. set task status to be prepared for check point, no data are allowed to put into inputQ.
int32_t code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_GEN_CHECKPOINT); int32_t code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_GEN_CHECKPOINT);
ASSERT(code == TSDB_CODE_SUCCESS); if (code != TSDB_CODE_SUCCESS) {
stError("s-task:%s failed to handle gen-checkpoint event, failed to start checkpoint procedure", pTask->id.idStr);
return code;
}
pTask->chkInfo.pActiveInfo->transId = pReq->transId; pTask->chkInfo.pActiveInfo->transId = pReq->transId;
pTask->chkInfo.pActiveInfo->activeId = pReq->checkpointId; pTask->chkInfo.pActiveInfo->activeId = pReq->checkpointId;
@ -112,7 +117,10 @@ int32_t streamProcessCheckpointSourceReq(SStreamTask* pTask, SStreamCheckpointSo
} }
int32_t streamTaskProcessCheckpointTriggerRsp(SStreamTask* pTask, SCheckpointTriggerRsp* pRsp) { int32_t streamTaskProcessCheckpointTriggerRsp(SStreamTask* pTask, SCheckpointTriggerRsp* pRsp) {
ASSERT(pTask->info.taskLevel != TASK_LEVEL__SOURCE); if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
stError("s-task:%s invalid msg recv, checkpoint-trigger rsp not handled", pTask->id.idStr);
return TSDB_CODE_INVALID_MSG;
}
if (pRsp->rspCode != TSDB_CODE_SUCCESS) { if (pRsp->rspCode != TSDB_CODE_SUCCESS) {
stDebug("s-task:%s retrieve checkpoint-trgger rsp from upstream:0x%x invalid, code:%s", pTask->id.idStr, stDebug("s-task:%s retrieve checkpoint-trgger rsp from upstream:0x%x invalid, code:%s", pTask->id.idStr,
@ -258,7 +266,6 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
} }
if (p->upstreamTaskId == pBlock->srcTaskId) { if (p->upstreamTaskId == pBlock->srcTaskId) {
ASSERT(p->checkpointId == checkpointId);
stWarn("s-task:%s repeatly recv checkpoint-source msg from task:0x%x vgId:%d, checkpointId:%" PRId64 stWarn("s-task:%s repeatly recv checkpoint-source msg from task:0x%x vgId:%d, checkpointId:%" PRId64
", prev recvTs:%" PRId64 " discard", ", prev recvTs:%" PRId64 " discard",
pTask->id.idStr, p->upstreamTaskId, p->upstreamNodeId, p->checkpointId, p->recvTs); pTask->id.idStr, p->upstreamTaskId, p->upstreamNodeId, p->checkpointId, p->recvTs);
@ -332,7 +339,6 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
streamFreeQitem((SStreamQueueItem*)pBlock); streamFreeQitem((SStreamQueueItem*)pBlock);
} }
} else if (taskLevel == TASK_LEVEL__SINK || taskLevel == TASK_LEVEL__AGG) { } else if (taskLevel == TASK_LEVEL__SINK || taskLevel == TASK_LEVEL__AGG) {
ASSERT(taosArrayGetSize(pTask->upstreamInfo.pList) > 0);
if (pTask->chkInfo.startTs == 0) { if (pTask->chkInfo.startTs == 0) {
pTask->chkInfo.startTs = taosGetTimestampMs(); pTask->chkInfo.startTs = taosGetTimestampMs();
pTask->execInfo.checkpoint += 1; pTask->execInfo.checkpoint += 1;
@ -367,19 +373,60 @@ int32_t streamProcessCheckpointTriggerBlock(SStreamTask* pTask, SStreamDataBlock
return code; return code;
} }
// only when all downstream tasks are send checkpoint rsp, we can start the checkpoint procedure for the agg task
static int32_t processCheckpointReadyHelp(SActiveCheckpointInfo* pInfo, int32_t numOfDownstream,
int32_t downstreamNodeId, int64_t streamId, int32_t downstreamTaskId,
const char* id, int32_t* pNotReady, int32_t* pTransId) {
bool received = false;
int32_t size = taosArrayGetSize(pInfo->pCheckpointReadyRecvList);
for (int32_t i = 0; i < size; ++i) {
STaskDownstreamReadyInfo* p = taosArrayGet(pInfo->pCheckpointReadyRecvList, i);
if (p == NULL) {
return TSDB_CODE_INVALID_PARA;
}
if (p->downstreamTaskId == downstreamTaskId) {
received = true;
break;
}
}
if (received) {
stDebug("s-task:%s already recv checkpoint-ready msg from downstream:0x%x, ignore. %d/%d downstream not ready", id,
downstreamTaskId, (int32_t)(numOfDownstream - taosArrayGetSize(pInfo->pCheckpointReadyRecvList)),
numOfDownstream);
} else {
STaskDownstreamReadyInfo info = {.recvTs = taosGetTimestampMs(),
.downstreamTaskId = downstreamTaskId,
.checkpointId = pInfo->activeId,
.transId = pInfo->transId,
.streamId = streamId,
.downstreamNodeId = downstreamNodeId};
void* p = taosArrayPush(pInfo->pCheckpointReadyRecvList, &info);
if (p == NULL) {
stError("s-task:%s failed to set checkpoint ready recv msg, code:%s", id, tstrerror(terrno));
return terrno;
}
}
*pNotReady = numOfDownstream - taosArrayGetSize(pInfo->pCheckpointReadyRecvList);
*pTransId = pInfo->transId;
return 0;
}
/** /**
* All down stream tasks have successfully completed the check point task. * All down stream tasks have successfully completed the check point task.
* Current stream task is allowed to start to do checkpoint things in ASYNC model. * Current stream task is allowed to start to do checkpoint things in ASYNC model.
*/ */
int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask, int64_t checkpointId, int32_t downstreamNodeId, int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask, int64_t checkpointId, int32_t downstreamNodeId,
int32_t downstreamTaskId) { int32_t downstreamTaskId) {
ASSERT(pTask->info.taskLevel == TASK_LEVEL__SOURCE || pTask->info.taskLevel == TASK_LEVEL__AGG);
SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo; SActiveCheckpointInfo* pInfo = pTask->chkInfo.pActiveInfo;
const char* id = pTask->id.idStr; const char* id = pTask->id.idStr;
bool received = false;
int32_t total = streamTaskGetNumOfDownstream(pTask); int32_t total = streamTaskGetNumOfDownstream(pTask);
ASSERT(total > 0); int32_t code = 0;
int32_t notReady = 0;
int32_t transId = 0;
// 1. not in checkpoint status now // 1. not in checkpoint status now
SStreamTaskState pStat = streamTaskGetStatus(pTask); SStreamTaskState pStat = streamTaskGetStatus(pTask);
@ -393,49 +440,20 @@ int32_t streamProcessCheckpointReadyMsg(SStreamTask* pTask, int64_t checkpointId
stError("s-task:%s status:%s checkpointId:%" PRId64 " new arrival checkpoint-ready msg (checkpointId:%" PRId64 stError("s-task:%s status:%s checkpointId:%" PRId64 " new arrival checkpoint-ready msg (checkpointId:%" PRId64
") from task:0x%x, expired and discard", ") from task:0x%x, expired and discard",
id, pStat.name, pTask->chkInfo.checkpointId, checkpointId, downstreamTaskId); id, pStat.name, pTask->chkInfo.checkpointId, checkpointId, downstreamTaskId);
return -1; return TSDB_CODE_INVALID_MSG;
} }
streamMutexLock(&pInfo->lock); streamMutexLock(&pInfo->lock);
code = processCheckpointReadyHelp(pInfo, total, downstreamNodeId, pTask->id.streamId, downstreamTaskId, id, &notReady,
// only when all downstream tasks are send checkpoint rsp, we can start the checkpoint procedure for the agg task &transId);
int32_t size = taosArrayGetSize(pInfo->pCheckpointReadyRecvList);
for (int32_t i = 0; i < size; ++i) {
STaskDownstreamReadyInfo* p = taosArrayGet(pInfo->pCheckpointReadyRecvList, i);
if (p == NULL) {
streamMutexUnlock(&pInfo->lock);
return TSDB_CODE_INVALID_PARA;
}
if (p->downstreamTaskId == downstreamTaskId) {
received = true;
break;
}
}
if (received) {
stDebug("s-task:%s already recv checkpoint-ready msg from downstream:0x%x, ignore. %d/%d downstream not ready", id,
downstreamTaskId, (int32_t)(total - taosArrayGetSize(pInfo->pCheckpointReadyRecvList)), total);
} else {
STaskDownstreamReadyInfo info = {.recvTs = taosGetTimestampMs(),
.downstreamTaskId = downstreamTaskId,
.checkpointId = pInfo->activeId,
.transId = pInfo->transId,
.streamId = pTask->id.streamId,
.downstreamNodeId = downstreamNodeId};
(void)taosArrayPush(pInfo->pCheckpointReadyRecvList, &info);
}
int32_t notReady = total - taosArrayGetSize(pInfo->pCheckpointReadyRecvList);
int32_t transId = pInfo->transId;
streamMutexUnlock(&pInfo->lock); streamMutexUnlock(&pInfo->lock);
if (notReady == 0) { if ((notReady == 0) && (code == 0)) {
stDebug("s-task:%s all downstream tasks have completed build checkpoint, do checkpoint for current task", id); stDebug("s-task:%s all downstream tasks have completed build checkpoint, do checkpoint for current task", id);
(void)appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT, checkpointId, transId, -1); (void)appendCheckpointIntoInputQ(pTask, STREAM_INPUT__CHECKPOINT, checkpointId, transId, -1);
} }
return 0; return code;
} }
int32_t streamTaskProcessCheckpointReadyRsp(SStreamTask* pTask, int32_t upstreamTaskId, int64_t checkpointId) { int32_t streamTaskProcessCheckpointReadyRsp(SStreamTask* pTask, int32_t upstreamTaskId, int64_t checkpointId) {
@ -798,6 +816,13 @@ void checkpointTriggerMonitorFn(void* param, void* tmrId) {
SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo; SActiveCheckpointInfo* pActiveInfo = pTask->chkInfo.pActiveInfo;
SStreamTmrInfo* pTmrInfo = &pActiveInfo->chkptTriggerMsgTmr; SStreamTmrInfo* pTmrInfo = &pActiveInfo->chkptTriggerMsgTmr;
if (pTask->info.taskLevel == TASK_LEVEL__SOURCE) {
int32_t ref = atomic_sub_fetch_32(&pTask->status.timerActive, 1);
stError("s-task:%s source task should not start the checkpoint-trigger monitor fn, ref:%d quit", id, ref);
streamMetaReleaseTask(pTask->pMeta, pTask);
return;
}
// check the status every 100ms // check the status every 100ms
if (streamTaskShouldStop(pTask)) { if (streamTaskShouldStop(pTask)) {
int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask); int32_t ref = streamCleanBeforeQuitTmr(pTmrInfo, pTask);
@ -842,8 +867,6 @@ void checkpointTriggerMonitorFn(void* param, void* tmrId) {
// send msg to retrieve checkpoint trigger msg // send msg to retrieve checkpoint trigger msg
SArray* pList = pTask->upstreamInfo.pList; SArray* pList = pTask->upstreamInfo.pList;
ASSERT(pTask->info.taskLevel > TASK_LEVEL__SOURCE);
SArray* pNotSendList = taosArrayInit(4, sizeof(SStreamUpstreamEpInfo)); SArray* pNotSendList = taosArrayInit(4, sizeof(SStreamUpstreamEpInfo));
if (pNotSendList == NULL) { if (pNotSendList == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -1070,8 +1093,7 @@ int32_t streamTaskGetNumOfConfirmed(SActiveCheckpointInfo* pInfo) {
for (int32_t i = 0; i < taosArrayGetSize(pInfo->pDispatchTriggerList); ++i) { for (int32_t i = 0; i < taosArrayGetSize(pInfo->pDispatchTriggerList); ++i) {
STaskTriggerSendInfo* p = taosArrayGet(pInfo->pDispatchTriggerList, i); STaskTriggerSendInfo* p = taosArrayGet(pInfo->pDispatchTriggerList, i);
if (p == NULL) { if (p == NULL) {
streamMutexUnlock(&pInfo->lock); continue;
return num;
} }
if (p->recved) { if (p->recved) {
@ -1107,10 +1129,12 @@ void streamTaskSetTriggerDispatchConfirmed(SStreamTask* pTask, int32_t vgId) {
streamMutexUnlock(&pInfo->lock); streamMutexUnlock(&pInfo->lock);
int32_t total = streamTaskGetNumOfDownstream(pTask); int32_t total = streamTaskGetNumOfDownstream(pTask);
if (taskId == 0) {
stError("s-task:%s recv invalid trigger-dispatch confirm, vgId:%d", pTask->id.idStr, vgId);
} else {
stDebug("s-task:%s set downstream:0x%x(vgId:%d) checkpoint-trigger dispatch confirmed, total confirmed:%d/%d", stDebug("s-task:%s set downstream:0x%x(vgId:%d) checkpoint-trigger dispatch confirmed, total confirmed:%d/%d",
pTask->id.idStr, taskId, vgId, numOfConfirmed, total); pTask->id.idStr, taskId, vgId, numOfConfirmed, total);
}
ASSERT(taskId != 0);
} }
static int32_t uploadCheckpointToS3(const char* id, const char* path) { static int32_t uploadCheckpointToS3(const char* id, const char* path) {

View File

@ -175,10 +175,11 @@ int32_t streamDataSubmitNew(SPackedData* pData, int32_t type, SStreamDataSubmit*
} }
void streamDataSubmitDestroy(SStreamDataSubmit* pDataSubmit) { void streamDataSubmitDestroy(SStreamDataSubmit* pDataSubmit) {
ASSERT(pDataSubmit->type == STREAM_INPUT__DATA_SUBMIT); if (pDataSubmit != NULL && pDataSubmit->type == STREAM_INPUT__DATA_SUBMIT) {
taosMemoryFree(pDataSubmit->submit.msgStr); taosMemoryFree(pDataSubmit->submit.msgStr);
taosFreeQitem(pDataSubmit); taosFreeQitem(pDataSubmit);
} }
}
int32_t streamMergedSubmitNew(SStreamMergedSubmit** pSubmit) { int32_t streamMergedSubmitNew(SStreamMergedSubmit** pSubmit) {
*pSubmit = NULL; *pSubmit = NULL;

View File

@ -96,8 +96,6 @@ int32_t streamTaskBroadcastRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* r
int32_t code = 0; int32_t code = 0;
void* buf = NULL; void* buf = NULL;
int32_t sz = taosArrayGetSize(pTask->upstreamInfo.pList); int32_t sz = taosArrayGetSize(pTask->upstreamInfo.pList);
ASSERT(sz > 0);
for (int32_t i = 0; i < sz; i++) { for (int32_t i = 0; i < sz; i++) {
req->reqId = tGenIdPI64(); req->reqId = tGenIdPI64();
SStreamUpstreamEpInfo* pEpInfo = taosArrayGetP(pTask->upstreamInfo.pList, i); SStreamUpstreamEpInfo* pEpInfo = taosArrayGetP(pTask->upstreamInfo.pList, i);
@ -107,7 +105,6 @@ int32_t streamTaskBroadcastRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* r
tEncodeSize(tEncodeStreamRetrieveReq, req, len, code); tEncodeSize(tEncodeStreamRetrieveReq, req, len, code);
if (code != 0) { if (code != 0) {
ASSERT(0);
return code; return code;
} }
@ -968,8 +965,6 @@ int32_t streamTaskSendCheckpointSourceRsp(SStreamTask* pTask) {
SArray* pList = pTask->chkInfo.pActiveInfo->pReadyMsgList; SArray* pList = pTask->chkInfo.pActiveInfo->pReadyMsgList;
streamMutexLock(&pTask->chkInfo.pActiveInfo->lock); streamMutexLock(&pTask->chkInfo.pActiveInfo->lock);
ASSERT(pTask->info.taskLevel == TASK_LEVEL__SOURCE);
if (taosArrayGetSize(pList) == 1) { if (taosArrayGetSize(pList) == 1) {
STaskCheckpointReadyInfo* pInfo = taosArrayGet(pList, 0); STaskCheckpointReadyInfo* pInfo = taosArrayGet(pList, 0);
tmsgSendRsp(&pInfo->msg); tmsgSendRsp(&pInfo->msg);
@ -1144,8 +1139,6 @@ int32_t streamAddCheckpointSourceRspMsg(SStreamCheckpointSourceReq* pReq, SRpcHa
void initCheckpointReadyInfo(STaskCheckpointReadyInfo* pReadyInfo, int32_t upstreamNodeId, int32_t upstreamTaskId, void initCheckpointReadyInfo(STaskCheckpointReadyInfo* pReadyInfo, int32_t upstreamNodeId, int32_t upstreamTaskId,
int32_t childId, SEpSet* pEpset, int64_t checkpointId) { int32_t childId, SEpSet* pEpset, int64_t checkpointId) {
ASSERT(upstreamTaskId != 0);
pReadyInfo->upstreamTaskId = upstreamTaskId; pReadyInfo->upstreamTaskId = upstreamTaskId;
pReadyInfo->upstreamNodeEpset = *pEpset; pReadyInfo->upstreamNodeEpset = *pEpset;
pReadyInfo->upstreamNodeId = upstreamNodeId; pReadyInfo->upstreamNodeId = upstreamNodeId;

View File

@ -1391,8 +1391,11 @@ int32_t streamMetaStartAllTasks(SStreamMeta* pMeta) {
if (ret != TSDB_CODE_SUCCESS) { if (ret != TSDB_CODE_SUCCESS) {
stError("vgId:%d failed to handle event:%d", pMeta->vgId, TASK_EVENT_INIT); stError("vgId:%d failed to handle event:%d", pMeta->vgId, TASK_EVENT_INIT);
code = ret; code = ret;
if (code != TSDB_CODE_STREAM_INVALID_STATETRANS) {
streamMetaAddFailedTaskSelf(pTask, pInfo->readyTs); streamMetaAddFailedTaskSelf(pTask, pInfo->readyTs);
} }
}
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
continue; continue;
@ -1477,12 +1480,14 @@ bool streamMetaAllTasksReady(const SStreamMeta* pMeta) {
int32_t streamMetaStartOneTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId) { int32_t streamMetaStartOneTask(SStreamMeta* pMeta, int64_t streamId, int32_t taskId) {
int32_t code = 0; int32_t code = 0;
int32_t vgId = pMeta->vgId; int32_t vgId = pMeta->vgId;
SStreamTask* pTask = NULL;
bool continueExec = true;
stInfo("vgId:%d start task:0x%x by checking it's downstream status", vgId, taskId); stInfo("vgId:%d start task:0x%x by checking it's downstream status", vgId, taskId);
SStreamTask* pTask = NULL;
code = streamMetaAcquireTask(pMeta, streamId, taskId, &pTask); code = streamMetaAcquireTask(pMeta, streamId, taskId, &pTask);
if (pTask == NULL) { if (pTask == NULL) {
stError("vgId:%d failed to acquire task:0x%x when starting task", pMeta->vgId, taskId); stError("vgId:%d failed to acquire task:0x%x when starting task", vgId, taskId);
(void)streamMetaAddFailedTask(pMeta, streamId, taskId); (void)streamMetaAddFailedTask(pMeta, streamId, taskId);
return TSDB_CODE_STREAM_TASK_IVLD_STATUS; return TSDB_CODE_STREAM_TASK_IVLD_STATUS;
} }
@ -1490,10 +1495,28 @@ int32_t streamMetaStartOneTask(SStreamMeta* pMeta, int64_t streamId, int32_t tas
// fill-history task can only be launched by related stream tasks. // fill-history task can only be launched by related stream tasks.
STaskExecStatisInfo* pInfo = &pTask->execInfo; STaskExecStatisInfo* pInfo = &pTask->execInfo;
if (pTask->info.fillHistory == 1) { if (pTask->info.fillHistory == 1) {
stError("s-task:0x%x vgId:%d fill-histroy task, not start here", taskId, vgId);
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
// the start all tasks procedure may happen to start the newly deployed stream task, and results in the
// concurrently start this task by two threads.
streamMutexLock(&pTask->lock);
SStreamTaskState status = streamTaskGetStatus(pTask);
if (status.state != TASK_STATUS__UNINIT) {
stError("s-task:0x%x vgId:%d status:%s not uninit status, not start stream task", taskId, vgId, status.name);
continueExec = false;
} else {
continueExec = true;
}
streamMutexUnlock(&pTask->lock);
if (!continueExec) {
streamMetaReleaseTask(pMeta, pTask);
return TSDB_CODE_STREAM_TASK_IVLD_STATUS;
}
ASSERT(pTask->status.downstreamReady == 0); ASSERT(pTask->status.downstreamReady == 0);
// avoid initialization and destroy running concurrently. // avoid initialization and destroy running concurrently.
@ -1509,13 +1532,19 @@ int32_t streamMetaStartOneTask(SStreamMeta* pMeta, int64_t streamId, int32_t tas
streamMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
} }
// concurrently start task may cause the later started task be failed, and also failed to added into meta result.
if (code == TSDB_CODE_SUCCESS) { if (code == TSDB_CODE_SUCCESS) {
code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_INIT); code = streamTaskHandleEvent(pTask->status.pSM, TASK_EVENT_INIT);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
stError("s-task:%s vgId:%d failed to handle event:%d", pTask->id.idStr, pMeta->vgId, TASK_EVENT_INIT); stError("s-task:%s vgId:%d failed to handle event:%d, code:%s", pTask->id.idStr, pMeta->vgId, TASK_EVENT_INIT,
tstrerror(code));
// do no added into result hashmap if it is failed due to concurrently starting of this stream task.
if (code != TSDB_CODE_STREAM_INVALID_STATETRANS) {
streamMetaAddFailedTaskSelf(pTask, pInfo->readyTs); streamMetaAddFailedTaskSelf(pTask, pInfo->readyTs);
} }
} }
}
streamMetaReleaseTask(pMeta, pTask); streamMetaReleaseTask(pMeta, pTask);
return code; return code;
@ -1547,11 +1576,12 @@ int32_t streamMetaAddTaskLaunchResult(SStreamMeta* pMeta, int64_t streamId, int3
int64_t endTs, bool ready) { int64_t endTs, bool ready) {
STaskStartInfo* pStartInfo = &pMeta->startInfo; STaskStartInfo* pStartInfo = &pMeta->startInfo;
STaskId id = {.streamId = streamId, .taskId = taskId}; STaskId id = {.streamId = streamId, .taskId = taskId};
int32_t vgId = pMeta->vgId;
streamMetaWLock(pMeta); streamMetaWLock(pMeta);
SStreamTask** p = taosHashGet(pMeta->pTasksMap, &id, sizeof(id)); SStreamTask** p = taosHashGet(pMeta->pTasksMap, &id, sizeof(id));
if (p == NULL) { // task does not exists in current vnode, not record the complete info if (p == NULL) { // task does not exists in current vnode, not record the complete info
stError("vgId:%d s-task:0x%x not exists discard the check downstream info", pMeta->vgId, taskId); stError("vgId:%d s-task:0x%x not exists discard the check downstream info", vgId, taskId);
streamMetaWUnLock(pMeta); streamMetaWUnLock(pMeta);
return 0; return 0;
} }
@ -1566,7 +1596,7 @@ int32_t streamMetaAddTaskLaunchResult(SStreamMeta* pMeta, int64_t streamId, int3
stDebug( stDebug(
"vgId:%d not in start all task(s) process, not record launch result status, s-task:0x%x launch succ:%d elapsed " "vgId:%d not in start all task(s) process, not record launch result status, s-task:0x%x launch succ:%d elapsed "
"time:%" PRId64 "ms", "time:%" PRId64 "ms",
pMeta->vgId, taskId, ready, el); vgId, taskId, ready, el);
streamMetaWUnLock(pMeta); streamMetaWUnLock(pMeta);
return 0; return 0;
} }
@ -1576,6 +1606,15 @@ int32_t streamMetaAddTaskLaunchResult(SStreamMeta* pMeta, int64_t streamId, int3
STaskInitTs initTs = {.start = startTs, .end = endTs, .success = ready}; STaskInitTs initTs = {.start = startTs, .end = endTs, .success = ready};
int32_t code = taosHashPut(pDst, &id, sizeof(id), &initTs, sizeof(STaskInitTs)); int32_t code = taosHashPut(pDst, &id, sizeof(id), &initTs, sizeof(STaskInitTs));
if (code) { if (code) {
if (code == TSDB_CODE_DUP_KEY) {
stError("vgId:%d record start task result failed, s-task:0x%" PRIx64
" already exist start results in meta start task result hashmap",
vgId, id.taskId);
} else {
stError("vgId:%d failed to record start task:0x%" PRIx64 " results, start all tasks failed", vgId, id.taskId);
}
streamMetaWUnLock(pMeta);
return code;
} }
int32_t numOfTotal = streamMetaGetNumOfTasks(pMeta); int32_t numOfTotal = streamMetaGetNumOfTasks(pMeta);
@ -1587,20 +1626,20 @@ int32_t streamMetaAddTaskLaunchResult(SStreamMeta* pMeta, int64_t streamId, int3
stDebug("vgId:%d all %d task(s) check downstream completed, last completed task:0x%x (succ:%d) startTs:%" PRId64 stDebug("vgId:%d all %d task(s) check downstream completed, last completed task:0x%x (succ:%d) startTs:%" PRId64
", readyTs:%" PRId64 " total elapsed time:%.2fs", ", readyTs:%" PRId64 " total elapsed time:%.2fs",
pMeta->vgId, numOfTotal, taskId, ready, pStartInfo->startTs, pStartInfo->readyTs, vgId, numOfTotal, taskId, ready, pStartInfo->startTs, pStartInfo->readyTs,
pStartInfo->elapsedTime / 1000.0); pStartInfo->elapsedTime / 1000.0);
// print the initialization elapsed time and info // print the initialization elapsed time and info
displayStatusInfo(pMeta, pStartInfo->pReadyTaskSet, true); displayStatusInfo(pMeta, pStartInfo->pReadyTaskSet, true);
displayStatusInfo(pMeta, pStartInfo->pFailedTaskSet, false); displayStatusInfo(pMeta, pStartInfo->pFailedTaskSet, false);
streamMetaResetStartInfo(pStartInfo, pMeta->vgId); streamMetaResetStartInfo(pStartInfo, vgId);
streamMetaWUnLock(pMeta); streamMetaWUnLock(pMeta);
code = pStartInfo->completeFn(pMeta); code = pStartInfo->completeFn(pMeta);
} else { } else {
streamMetaWUnLock(pMeta); streamMetaWUnLock(pMeta);
stDebug("vgId:%d recv check downstream results, s-task:0x%x succ:%d, received:%d, total:%d", pMeta->vgId, taskId, stDebug("vgId:%d recv check downstream results, s-task:0x%x succ:%d, received:%d, total:%d", vgId, taskId, ready,
ready, numOfRecv, numOfTotal); numOfRecv, numOfTotal);
} }
return code; return code;

View File

@ -48,14 +48,15 @@ int32_t streamTaskSchedTask(SMsgCb* pMsgCb, int32_t vgId, int64_t streamId, int3
SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq)); SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq));
if (pRunReq == NULL) { if (pRunReq == NULL) {
stError("vgId:%d failed to create msg to start stream task:0x%x exec, type:%d, code:%s", vgId, taskId, execType, stError("vgId:%d failed to create msg to start stream task:0x%x exec, type:%d, code:%s", vgId, taskId, execType,
terrstr()); terrstr(terrno));
return TSDB_CODE_OUT_OF_MEMORY; return terrno;
} }
if (streamId != 0) { if (streamId != 0) {
stDebug("vgId:%d create msg to start stream task:0x%x, exec type:%d", vgId, taskId, execType); stDebug("vgId:%d create msg to for task:0x%x, exec type:%d, %s", vgId, taskId, execType,
streamTaskGetExecType(execType));
} else { } else {
stDebug("vgId:%d create msg to exec, type:%d", vgId, execType); stDebug("vgId:%d create msg to exec, type:%d, %s", vgId, execType, streamTaskGetExecType(execType));
} }
pRunReq->head.vgId = vgId; pRunReq->head.vgId = vgId;

View File

@ -275,7 +275,7 @@ void tFreeStreamTask(SStreamTask* pTask) {
} }
streamTaskCleanupCheckInfo(&pTask->taskCheckInfo); streamTaskCleanupCheckInfo(&pTask->taskCheckInfo);
streamFreeTaskState(pTask, status1); streamFreeTaskState(pTask, pTask->status.removeBackendFiles ? 1 : 0);
if (pTask->pNameMap) { if (pTask->pNameMap) {
tSimpleHashCleanup(pTask->pNameMap); tSimpleHashCleanup(pTask->pNameMap);
@ -296,14 +296,14 @@ void tFreeStreamTask(SStreamTask* pTask) {
taosArrayDestroy(pTask->outputInfo.pNodeEpsetUpdateList); taosArrayDestroy(pTask->outputInfo.pNodeEpsetUpdateList);
pTask->outputInfo.pNodeEpsetUpdateList = NULL; pTask->outputInfo.pNodeEpsetUpdateList = NULL;
if ((pTask->status.removeBackendFiles) && (pTask->pMeta != NULL)) { // if ((pTask->status.removeBackendFiles) && (pTask->pMeta != NULL)) {
char* path = taosMemoryCalloc(1, strlen(pTask->pMeta->path) + 128); // char* path = taosMemoryCalloc(1, strlen(pTask->pMeta->path) + 128);
sprintf(path, "%s%s%s", pTask->pMeta->path, TD_DIRSEP, pTask->id.idStr); // sprintf(path, "%s%s%s", pTask->pMeta->path, TD_DIRSEP, pTask->id.idStr);
taosRemoveDir(path); // taosRemoveDir(path);
stInfo("s-task:0x%x vgId:%d remove all backend files:%s", taskId, pTask->pMeta->vgId, path); // stInfo("s-task:0x%x vgId:%d remove all backend files:%s", taskId, pTask->pMeta->vgId, path);
taosMemoryFree(path); // taosMemoryFree(path);
} // }
if (pTask->id.idStr != NULL) { if (pTask->id.idStr != NULL) {
taosMemoryFree((void*)pTask->id.idStr); taosMemoryFree((void*)pTask->id.idStr);
@ -316,10 +316,12 @@ void tFreeStreamTask(SStreamTask* pTask) {
stDebug("s-task:0x%x free task completed", taskId); stDebug("s-task:0x%x free task completed", taskId);
} }
void streamFreeTaskState(SStreamTask* pTask, ETaskStatus status) { void streamFreeTaskState(SStreamTask* pTask, int8_t remove) {
if (pTask->pState != NULL) { if (pTask->pState != NULL) {
stDebug("s-task:0x%x start to free task state", pTask->id.taskId); stDebug("s-task:0x%x start to free task state", pTask->id.taskId);
streamStateClose(pTask->pState, status == TASK_STATUS__DROPPING); streamStateClose(pTask->pState, remove);
taskDbSetClearFileFlag(pTask->pBackend);
taskDbRemoveRef(pTask->pBackend); taskDbRemoveRef(pTask->pBackend);
pTask->pBackend = NULL; pTask->pBackend = NULL;
pTask->pState = NULL; pTask->pState = NULL;
@ -1152,3 +1154,24 @@ void streamTaskClearActiveInfo(SActiveCheckpointInfo* pInfo) {
taosArrayClear(pInfo->pDispatchTriggerList); taosArrayClear(pInfo->pDispatchTriggerList);
taosArrayClear(pInfo->pCheckpointReadyRecvList); taosArrayClear(pInfo->pCheckpointReadyRecvList);
} }
const char* streamTaskGetExecType(int32_t type) {
switch (type) {
case STREAM_EXEC_T_EXTRACT_WAL_DATA:
return "scan-wal-file";
case STREAM_EXEC_T_START_ALL_TASKS:
return "start-all-tasks";
case STREAM_EXEC_T_START_ONE_TASK:
return "start-one-task";
case STREAM_EXEC_T_RESTART_ALL_TASKS:
return "restart-all-tasks";
case STREAM_EXEC_T_STOP_ALL_TASKS:
return "stop-all-tasks";
case STREAM_EXEC_T_RESUME_TASK:
return "resume-task-from-idle";
case STREAM_EXEC_T_ADD_FAILED_TASK:
return "record-start-failed-task";
default:
return "invalid-exec-type";
}
}

View File

@ -316,12 +316,11 @@ static int32_t doHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event, STaskSt
if (pTrans->attachEvent.event != 0) { if (pTrans->attachEvent.event != 0) {
code = attachWaitedEvent(pTask, &pTrans->attachEvent); code = attachWaitedEvent(pTask, &pTrans->attachEvent);
streamMutexUnlock(&pTask->lock);
if (code) { if (code) {
return code; return code;
} }
streamMutexUnlock(&pTask->lock);
while (1) { while (1) {
// wait for the task to be here // wait for the task to be here
streamMutexLock(&pTask->lock); streamMutexLock(&pTask->lock);
@ -400,7 +399,7 @@ int32_t streamTaskHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event) {
EStreamTaskEvent evt = pSM->pActiveTrans->event; EStreamTaskEvent evt = pSM->pActiveTrans->event;
streamMutexUnlock(&pTask->lock); streamMutexUnlock(&pTask->lock);
stDebug("s-task:%s status:%s handling event:%s by some other thread, wait for 100ms and check if completed", stDebug("s-task:%s status:%s handling event:%s by another thread, wait for 100ms and check if completed",
pTask->id.idStr, pSM->current.name, GET_EVT_NAME(evt)); pTask->id.idStr, pSM->current.name, GET_EVT_NAME(evt));
taosMsleep(100); taosMsleep(100);
} else { } else {
@ -413,6 +412,13 @@ int32_t streamTaskHandleEvent(SStreamTaskSM* pSM, EStreamTaskEvent event) {
} }
if (pSM->pActiveTrans != NULL) { if (pSM->pActiveTrans != NULL) {
// not allowed concurrently initialization
if (event == TASK_EVENT_INIT && pSM->pActiveTrans->event == TASK_EVENT_INIT) {
streamMutexUnlock(&pTask->lock);
stError("s-task:%s already in handling init procedure, handle this init event failed", pTask->id.idStr);
return TSDB_CODE_STREAM_INVALID_STATETRANS;
}
// currently in some state transfer procedure, not auto invoke transfer, abort it // currently in some state transfer procedure, not auto invoke transfer, abort it
stDebug("s-task:%s event:%s handle procedure quit, status %s -> %s failed, handle event %s now", stDebug("s-task:%s event:%s handle procedure quit, status %s -> %s failed, handle event %s now",
pTask->id.idStr, GET_EVT_NAME(pSM->pActiveTrans->event), pSM->current.name, pTask->id.idStr, GET_EVT_NAME(pSM->pActiveTrans->event), pSM->current.name,
@ -551,6 +557,11 @@ ETaskStatus streamTaskGetPrevStatus(const SStreamTask* pTask) {
} }
const char* streamTaskGetStatusStr(ETaskStatus status) { const char* streamTaskGetStatusStr(ETaskStatus status) {
int32_t index = status;
if (index < 0 || index > tListLen(StreamTaskStatusList)) {
return "";
}
return StreamTaskStatusList[status].name; return StreamTaskStatusList[status].name;
} }

View File

@ -21,6 +21,9 @@ endif ()
if(USE_TD_MEMORY) if(USE_TD_MEMORY)
add_definitions(-DUSE_TD_MEMORY) add_definitions(-DUSE_TD_MEMORY)
endif () endif ()
if(BUILD_WITH_RAND_ERR)
add_definitions(-DBUILD_WITH_RAND_ERR)
endif ()
if(BUILD_ADDR2LINE) if(BUILD_ADDR2LINE)
if(NOT TD_WINDOWS) if(NOT TD_WINDOWS)
target_include_directories( target_include_directories(

View File

@ -75,7 +75,7 @@ char *taosCharsetReplace(char *charsetstr) {
* *
* In case that the setLocale failed to be executed, the right charset needs to be set. * In case that the setLocale failed to be executed, the right charset needs to be set.
*/ */
int32_t taosSetSystemLocale(const char *inLocale, const char *inCharSet) {\ int32_t taosSetSystemLocale(const char *inLocale, const char *inCharSet) {
if (!taosValidateEncodec(inCharSet)) { if (!taosValidateEncodec(inCharSet)) {
return terrno; return terrno;
} }

View File

@ -21,6 +21,10 @@
#endif #endif
#include "os.h" #include "os.h"
int32_t tsRandErrChance = 1;
threadlocal bool tsEnableRandErr = 0;
#if defined(USE_TD_MEMORY) || defined(USE_ADDR2LINE) #if defined(USE_TD_MEMORY) || defined(USE_ADDR2LINE)
#define TD_MEMORY_SYMBOL ('T' << 24 | 'A' << 16 | 'O' << 8 | 'S') #define TD_MEMORY_SYMBOL ('T' << 24 | 'A' << 16 | 'O' << 8 | 'S')
@ -266,6 +270,16 @@ void *taosMemoryMalloc(int64_t size) {
return (char *)tmp + sizeof(TdMemoryInfo); return (char *)tmp + sizeof(TdMemoryInfo);
#else #else
#ifdef BUILD_WITH_RAND_ERR
if (tsEnableRandErr) {
uint32_t r = taosRand() % 10001;
if ((r + 1) <= tsRandErrChance) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
}
#endif
void *p = malloc(size); void *p = malloc(size);
if (NULL == p) { if (NULL == p) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -287,6 +301,16 @@ void *taosMemoryCalloc(int64_t num, int64_t size) {
return (char *)tmp + sizeof(TdMemoryInfo); return (char *)tmp + sizeof(TdMemoryInfo);
#else #else
#ifdef BUILD_WITH_RAND_ERR
if (tsEnableRandErr) {
uint32_t r = taosRand() % 10001;
if ((r + 1) <= tsRandErrChance) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
}
#endif
void *p = calloc(num, size); void *p = calloc(num, size);
if (NULL == p) { if (NULL == p) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -317,6 +341,16 @@ void *taosMemoryRealloc(void *ptr, int64_t size) {
return (char *)tmp + sizeof(TdMemoryInfo); return (char *)tmp + sizeof(TdMemoryInfo);
#else #else
#ifdef BUILD_WITH_RAND_ERR
if (tsEnableRandErr) {
uint32_t r = taosRand() % 10001;
if ((r + 1) <= tsRandErrChance) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
}
#endif
void *p = realloc(ptr, size); void *p = realloc(ptr, size);
if (size > 0 && NULL == p) { if (size > 0 && NULL == p) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -342,6 +376,16 @@ char *taosStrdup(const char *ptr) {
return (char *)tmp + sizeof(TdMemoryInfo); return (char *)tmp + sizeof(TdMemoryInfo);
#else #else
#ifdef BUILD_WITH_RAND_ERR
if (tsEnableRandErr) {
uint32_t r = taosRand() % 10001;
if ((r + 1) <= tsRandErrChance) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
}
#endif
return tstrdup(ptr); return tstrdup(ptr);
#endif #endif
} }
@ -398,6 +442,16 @@ void *taosMemoryMallocAlign(uint32_t alignment, int64_t size) {
ASSERT(0); ASSERT(0);
#else #else
#if defined(LINUX) #if defined(LINUX)
#ifdef BUILD_WITH_RAND_ERR
if (tsEnableRandErr) {
uint32_t r = taosRand() % 10001;
if ((r + 1) <= tsRandErrChance) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
}
#endif
void *p = memalign(alignment, size); void *p = memalign(alignment, size);
if (NULL == p) { if (NULL == p) {
if (ENOMEM == errno) { if (ENOMEM == errno) {

View File

@ -357,6 +357,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_VIEW_NOT_EXIST, "view not exists in db
//mnode-compact //mnode-compact
TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_COMPACT_ID, "Invalid compact id") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_COMPACT_ID, "Invalid compact id")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_COMPACT_DETAIL_NOT_EXIST, "compact detail doesn't exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_COMPACT_DETAIL_NOT_EXIST, "compact detail doesn't exist")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_COMPACT_ALREADY_EXIST, "compact already exist")
// dnode // dnode
TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_OFFLINE, "Dnode is offline") TAOS_DEFINE_ERROR(TSDB_CODE_DNODE_OFFLINE, "Dnode is offline")

View File

@ -0,0 +1,24 @@
match callExpr(
hasParent(anyOf(
compoundStmt(),
doStmt(hasCondition(expr().bind("cond"))))
),
unless(hasType(voidType())),
unless(callee(functionDecl(hasName("memcpy")))),
unless(callee(functionDecl(hasName("strcpy")))),
unless(callee(functionDecl(hasName("strcat")))),
unless(callee(functionDecl(hasName("strncpy")))),
unless(callee(functionDecl(hasName("memset")))),
unless(callee(functionDecl(hasName("memmove")))),
unless(callee(functionDecl(hasName("sprintf")))),
unless(callee(functionDecl(hasName("snprintf")))),
unless(callee(functionDecl(hasName("scanf")))),
unless(callee(functionDecl(hasName("sncanf")))),
unless(callee(functionDecl(hasName("printf")))),
unless(callee(functionDecl(hasName("printRow")))),
unless(callee(functionDecl(hasName("puts")))),
unless(callee(functionDecl(hasName("sleep")))),
unless(callee(functionDecl(hasName("printResult")))),
unless(callee(functionDecl(hasName("getchar")))),
unless(callee(functionDecl(hasName("taos_print_row")))),
unless(callee(functionDecl(hasName("fprintf")))))

106
tests/ci/scan.py Normal file
View File

@ -0,0 +1,106 @@
import os
import subprocess
import csv
from datetime import datetime
from loguru import logger
# log file path
log_file_path = "/root/charles/scan.log"
logger.add(log_file_path, rotation="10MB", retention="7 days", level="DEBUG")
# scan result base path
scan_result_base_path = "/root/charles/clang_scan_result/"
# the base source code file path
source_path = "/root/charles/TDinternal/"
# the compile commands json file path
compile_commands_path = "/root/charles/TDinternal/debug/compile_commands.json"
# the ast parser rule for c file
clang_scan_rules_path = "/root/charles/clang_scan_rules"
# all the c files path will be checked
all_file_path = []
class CommandExecutor:
def __init__(self):
self._process = None
def execute(self, command, timeout=None):
try:
self._process = subprocess.Popen(command,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = self._process.communicate(timeout=timeout)
return stdout.decode('utf-8'), stderr.decode('utf-8')
except subprocess.TimeoutExpired:
self._process.kill()
self._process.communicate()
raise Exception("Command execution timeout")
except Exception as e:
raise Exception("Command execution failed: %s" % e)
def scan_files_path(source_file_path):
# scan_dir_list = ["source", "include", "docs/examples", "tests/script/api", "src/plugins"]
scan_dir_list = ["source", "include", "docs/examples", "src/plugins"]
scan_skip_file_list = ["/root/charles/TDinternal/community/tools/taosws-rs/target/release/build/openssl-sys-7811e597b848e397/out/openssl-build/install/include/openssl",
"/test/", "contrib", "debug", "deps", "/root/charles/TDinternal/community/source/libs/parser/src/sql.c", "/root/charles/TDinternal/community/source/client/jni/windows/win32/bridge/AccessBridgeCalls.c"]
for root, dirs, files in os.walk(source_file_path):
for file in files:
if any(item in root for item in scan_dir_list):
file_path = os.path.join(root, file)
if (file_path.endswith(".c") or file_path.endswith(".h") or file_path.endswith(".cpp")) and all(item not in file_path for item in scan_skip_file_list):
all_file_path.append(file_path)
logger.info("Found %s files" % len(all_file_path))
def save_scan_res(res_base_path, file_path, out, err):
file_res_path = os.path.join(res_base_path, file_path.replace("/root/charles/", "").split(".")[0] + ".res")
if not os.path.exists(os.path.dirname(file_res_path)):
os.makedirs(os.path.dirname(file_res_path))
logger.info("Save scan result to: %s" % file_res_path)
# save scan result
with open(file_res_path, "w") as f:
f.write(out)
f.write(err)
def write_csv(file_path, data):
try:
with open(file_path, 'w') as f:
writer = csv.writer(f)
writer.writerows(data)
except Exception as ex:
raise Exception("Failed to write the csv file: {} with msg: {}".format(file_path, repr(ex)))
if __name__ == "__main__":
command_executor = CommandExecutor()
# get all the c files path
scan_files_path(source_path)
res = []
# create dir
current_time = datetime.now().strftime("%Y%m%d%H%M%S")
scan_result_path = os.path.join(scan_result_base_path, current_time)
if not os.path.exists(scan_result_path):
os.makedirs(scan_result_path)
for file in all_file_path:
cmd = "clang-query -p %s %s -f %s" % (compile_commands_path, file, clang_scan_rules_path)
try:
stdout, stderr = command_executor.execute(cmd)
lines = stdout.split("\n")
if lines[-2].endswith("matches.") or lines[-2].endswith("match."):
match_num = int(lines[-2].split(" ")[0])
logger.info("The match lines of file %s: %s" % (file, match_num))
if match_num > 0:
save_scan_res(scan_result_path, file, stdout, stderr)
res.append([file, match_num, 'Pass' if match_num == 0 else 'Fail'])
else:
logger.warning("The result of scan is invalid for: %s" % file)
except Exception as e:
logger.error("Execute command failed: %s" % e)
# data = ""
# for item in res:
# data += item[0] + "," + str(item[1]) + "\n"
# logger.info("Csv data: %s" % data)
write_csv(os.path.join(scan_result_path, "scan_res.csv"), res)
logger.info("The result of scan: \n")
logger.info("Total files: %s" % len(res))
logger.info("Total match lines: %s" % sum([item[1] for item in res]))
logger.info("Pass files: %s" % len([item for item in res if item[2] == 'Pass']))
logger.info("Fail files: %s" % len([item for item in res if item[2] == 'Fail']))

229
tests/ci/scan_file_path.py Normal file
View File

@ -0,0 +1,229 @@
import os
import sys
import subprocess
import csv
from datetime import datetime
from loguru import logger
import getopt
opts, args = getopt.gnu_getopt(sys.argv[1:], 'b:f:w:', [
'branch_name='])
for key, value in opts:
if key in ['-h', '--help']:
print(
'Usage: python3 scan.py -b <branch_name> -f <file_list>')
print('-b branch name or PR ID to scan')
print('-f change files list')
print('-w web server')
sys.exit(0)
if key in ['-b', '--branchName']:
branch_name = value
if key in ['-f', '--filesName']:
change_file_list = value
if key in ['-w', '--webServer']:
web_server = value
# the base source code file path
self_path = os.path.dirname(os.path.realpath(__file__))
# if ("community" in self_path):
# TD_project_path = self_path[:self_path.find("community")]
# work_path = TD_project_path[:TD_project_path.find("TDinternal")]
# else:
# TD_project_path = self_path[:self_path.find("tests")]
# work_path = TD_project_path[:TD_project_path.find("TDengine")]
# Check if "community" or "tests" is in self_path
index_community = self_path.find("community")
if index_community != -1:
TD_project_path = self_path[:index_community]
index_TDinternal = TD_project_path.find("TDinternal")
# Check if index_TDinternal is valid and set work_path accordingly
if index_TDinternal != -1:
work_path = TD_project_path[:index_TDinternal]
else:
index_tests = self_path.find("tests")
if index_tests != -1:
TD_project_path = self_path[:index_tests]
# Check if index_TDengine is valid and set work_path accordingly
index_TDengine = TD_project_path.find("TDengine")
if index_TDengine != -1:
work_path = TD_project_path[:index_TDengine]
# log file path
current_time = datetime.now().strftime("%Y%m%d-%H%M%S")
log_file_path = f"{work_path}/scan_log/scan_{branch_name}_{current_time}/"
os.makedirs(log_file_path, exist_ok=True)
scan_log_file = f"{log_file_path}/scan_log.txt"
logger.add(scan_log_file, rotation="10MB", retention="7 days", level="DEBUG")
#if error happens, open this to debug
# print(self_path,work_path,TD_project_path,log_file_path,change_file_list)
# scan result base path
scan_result_base_path = f"{log_file_path}/clang_scan_result/"
# the compile commands json file path
# compile_commands_path = f"{work_path}/debugNoSan/compile_commands.json"
compile_commands_path = f"{TD_project_path}/debug/compile_commands.json"
#if error happens, open this to debug
# print(f"compile_commands_path:{compile_commands_path}")
# # replace the docerk worf path with real work path in compile_commands.json
# docker_work_path = "home"
# replace_path= work_path[1:-1]
# replace_path = replace_path.replace("/", "\/")
# sed_command = f"sed -i 's/{docker_work_path}/{replace_path}/g' {compile_commands_path}"
# print(sed_command)
# result = subprocess.run(sed_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
# logger.debug(f"STDOUT: {result.stdout} STDERR: {result.stderr}")
# the ast parser rule for c file
clang_scan_rules_path = f"{self_path}/filter_for_return_values"
#
# all the c files path will be checked
all_file_path = []
class CommandExecutor:
def __init__(self):
self._process = None
def execute(self, command, timeout=None):
try:
self._process = subprocess.Popen(command,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = self._process.communicate(timeout=timeout)
return stdout.decode('utf-8'), stderr.decode('utf-8')
except subprocess.TimeoutExpired:
self._process.kill()
self._process.communicate()
raise Exception("Command execution timeout")
except Exception as e:
raise Exception("Command execution failed: %s" % e)
def scan_files_path(source_file_path):
# scan_dir_list = ["source", "include", "docs/examples", "tests/script/api", "src/plugins"]
scan_dir_list = ["source", "include", "docs/examples", "src/plugins"]
scan_skip_file_list = ["/root/charles/TDinternal/community/tools/taosws-rs/target/release/build/openssl-sys-7811e597b848e397/out/openssl-build/install/include/openssl",
"/test/", "contrib", "debug", "deps", "/root/charles/TDinternal/community/source/libs/parser/src/sql.c", "/root/charles/TDinternal/community/source/client/jni/windows/win32/bridge/AccessBridgeCalls.c"]
for root, dirs, files in os.walk(source_file_path):
for file in files:
if any(item in root for item in scan_dir_list):
file_path = os.path.join(root, file)
if (file_path.endswith(".c") or file_path.endswith(".h") or file_path.endswith(".cpp")) and all(item not in file_path for item in scan_skip_file_list):
all_file_path.append(file_path)
logger.info("Found %s files" % len(all_file_path))
def input_files(change_files):
# scan_dir_list = ["source", "include", "docs/examples", "tests/script/api", "src/plugins"]
scan_dir_list = ["source", "include", "docs/examples", "src/plugins"]
scan_skip_file_list = [f"{TD_project_path}/TDinternal/community/tools/taosws-rs/target/release/build/openssl-sys-7811e597b848e397/out/openssl-build/install/include/openssl", "/test/", "contrib", "debug", "deps", f"{TD_project_path}/TDinternal/community/source/libs/parser/src/sql.c", f"{TD_project_path}/TDinternal/community/source/client/jni/windows/win32/bridge/AccessBridgeCalls.c"]
with open(change_files, 'r') as file:
for line in file:
file_name = line.strip()
if any(dir_name in file_name for dir_name in scan_dir_list):
if (file_name.endswith(".c") or file_name.endswith(".h") or line.endswith(".cpp")) and all(dir_name not in file_name for dir_name in scan_skip_file_list):
if "enterprise" in file_name:
file_name = os.path.join(TD_project_path, file_name)
else:
tdc_file_path = os.path.join(TD_project_path, "community/")
file_name = os.path.join(tdc_file_path, file_name)
all_file_path.append(file_name)
# print(f"all_file_path:{all_file_path}")
logger.info("Found %s files" % len(all_file_path))
file_res_path = ""
def save_scan_res(res_base_path, file_path, out, err):
global file_res_path
file_res_path = os.path.join(res_base_path, file_path.replace(f"{work_path}", "").split(".")[0] + ".txt")
# print(f"file_res_path:{file_res_path},res_base_path:{res_base_path},file_path:{file_path}")
if not os.path.exists(os.path.dirname(file_res_path)):
os.makedirs(os.path.dirname(file_res_path))
logger.info("Save scan result to: %s" % file_res_path)
# save scan result
with open(file_res_path, "w") as f:
f.write(err)
f.write(out)
logger.debug(f"file_res_file: {file_res_path}")
def write_csv(file_path, data):
try:
with open(file_path, 'w') as f:
writer = csv.writer(f)
writer.writerows(data)
except Exception as ex:
raise Exception("Failed to write the csv file: {} with msg: {}".format(file_path, repr(ex)))
if __name__ == "__main__":
command_executor = CommandExecutor()
# get all the c files path
# scan_files_path(TD_project_path)
input_files(change_file_list)
# print(f"all_file_path:{all_file_path}")
res = []
web_path = []
res.append(["scan_source_file", "scan_result_file", "match_num", "check_result"])
# create dir
# current_time = datetime.now().strftime("%Y%m%d%H%M%S")
# scan_result_path = os.path.join(scan_result_base_path, current_time)
# scan_result_path = scan_result_base_path
# if not os.path.exists(scan_result_path):
# os.makedirs(scan_result_path)
for file in all_file_path:
cmd = f"clang-query-10 -p {compile_commands_path} {file} -f {clang_scan_rules_path}"
logger.debug(f"cmd:{cmd}")
try:
stdout, stderr = command_executor.execute(cmd)
#if "error" in stderr:
# print(stderr)
lines = stdout.split("\n")
if lines[-2].endswith("matches.") or lines[-2].endswith("match."):
match_num = int(lines[-2].split(" ")[0])
logger.info("The match lines of file %s: %s" % (file, match_num))
if match_num > 0:
logger.info(f"log_file_path: {log_file_path} ,file:{file}")
save_scan_res(log_file_path, file, stdout, stderr)
index_tests = file_res_path.find("scan_log")
if index_tests != -1:
web_path_file = file_res_path[index_tests:]
web_path_file = os.path.join(web_server, web_path_file)
web_path.append(web_path_file)
res.append([file, file_res_path, match_num, 'Pass' if match_num == 0 else 'Fail'])
else:
logger.warning("The result of scan is invalid for: %s" % file)
except Exception as e:
logger.error("Execute command failed: %s" % e)
# data = ""
# for item in res:
# data += item[0] + "," + str(item[1]) + "\n"
# logger.info("Csv data: %s" % data)
write_csv(os.path.join(log_file_path, "scan_res.txt"), res)
scan_result_log = f"{log_file_path}/scan_res.txt"
# delete the first element of res
res= res[1:]
logger.info("The result of scan: \n")
logger.info("Total scan files: %s" % len(res))
logger.info("Total match lines: %s" % sum([item[2] for item in res]))
logger.info(f"scan log file : {scan_result_log}")
logger.info("Pass files: %s" % len([item for item in res if item[3] == 'Pass']))
logger.info("Fail files: %s" % len([item for item in res if item[3] == 'Fail']))
if len([item for item in res if item[3] == 'Fail']) > 0:
logger.error(f"Scan failed,please check the log file:{scan_result_log}")
for index, failed_result_file in enumerate(web_path):
logger.error(f"failed number: {index}, failed_result_file: {failed_result_file}")
exit(1)

View File

@ -83,7 +83,7 @@ docker run \
-v ${REP_REAL_PATH}/community/contrib/xml2/:${REP_DIR}/community/contrib/xml2 \ -v ${REP_REAL_PATH}/community/contrib/xml2/:${REP_DIR}/community/contrib/xml2 \
-v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \ -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \
-v ${REP_REAL_PATH}/community/contrib/zstd/:${REP_DIR}/community/contrib/zstd \ -v ${REP_REAL_PATH}/community/contrib/zstd/:${REP_DIR}/community/contrib/zstd \
--rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0;make -j 10|| exit 1" --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0 -DCMAKE_EXPORT_COMPILE_COMMANDS=1 ;make -j 10|| exit 1"
# -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \ # -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \
if [[ -d ${WORKDIR}/debugNoSan ]] ;then if [[ -d ${WORKDIR}/debugNoSan ]] ;then

View File

@ -0,0 +1,98 @@
#!/bin/bash
function usage() {
echo "$0"
echo -e "\t -d work dir"
echo -e "\t -b pr and id"
echo -e "\t -w web server "
echo -e "\t -f scan file "
echo -e "\t -h help"
}
while getopts "d:b:w:f:h" opt; do
case $opt in
d)
WORKDIR=$OPTARG
;;
b)
branch_name_id=$OPTARG
;;
f)
scan_file_name=$OPTARG
;;
w)
web_server=$OPTARG
;;
h)
usage
exit 0
;;
\?)
echo "Invalid option: -$OPTARG"
usage
exit 0
;;
esac
done
if [ -z "$branch_name_id" ]; then
usage
exit 1
fi
if [ -z "$scan_file_name" ]; then
usage
exit 1
fi
if [ -z "$WORKDIR" ]; then
usage
exit 1
fi
if [ -z "$web_server" ]; then
usage
exit 1
fi
# enterprise edition
INTERNAL_REPDIR=$WORKDIR/TDinternal
REPDIR_DEBUG=$WORKDIR/debugNoSan/
REP_MOUNT_DEBUG="${REPDIR_DEBUG}:/home/TDinternal/debug/"
REP_MOUNT_PARAM="$INTERNAL_REPDIR:/home/TDinternal"
CONTAINER_TESTDIR=/home/TDinternal/community
#scan change file path
scan_changefile_temp_path="$WORKDIR/tmp/${branch_name_id}/"
docker_can_changefile_temp_path="/home/tmp/${branch_name_id}/"
mkdir -p $scan_changefile_temp_path
scan_file_name="$docker_can_changefile_temp_path/docs_changed.txt"
#scan log file path
scan_log_temp_path="$WORKDIR/log/scan_log/"
docker_scan_log_temp_path="/home/scan_log/"
mkdir -p $scan_log_temp_path
scan_scripts="$CONTAINER_TESTDIR/tests/ci/scan_file_path.py"
ulimit -c unlimited
cat << EOF
docker run \
-v $REP_MOUNT_PARAM \
-v $REP_MOUNT_DEBUG \
-v $scan_changefile_temp_path:$docker_can_changefile_temp_path \
-v $scan_log_temp_path:$docker_scan_log_temp_path \
--rm --ulimit core=-1 taos_test:v1.0 python3 $scan_scripts -b "${branch_name_id}" -f "${scan_file_name}" -w ${web_server}
EOF
docker run \
-v $REP_MOUNT_PARAM \
-v $REP_MOUNT_DEBUG \
-v $scan_changefile_temp_path:$docker_can_changefile_temp_path \
-v $scan_log_temp_path:$docker_scan_log_temp_path \
--rm --ulimit core=-1 taos_test:v1.0 python3 $scan_scripts -b "${branch_name_id}" -f "${scan_file_name}" -w ${web_server}
ret=$?
exit $ret

View File

@ -36,7 +36,7 @@ if $data(3)[4] != ready then
goto step1 goto step1
endi endi
print =============== step2: create mnode 2 print =============== step2: create mnode 2 3
sql create mnode on dnode 2 sql create mnode on dnode 2
sql create mnode on dnode 3 sql create mnode on dnode 3
sql_error create mnode on dnode 4 sql_error create mnode on dnode 4
@ -115,7 +115,7 @@ if $data(3)[4] != ready then
goto step41 goto step41
endi endi
print =============== step5: stop dnode1 print =============== step5: stop dnode2
system sh/exec.sh -n dnode1 -s start system sh/exec.sh -n dnode1 -s start
system sh/exec.sh -n dnode2 -s stop system sh/exec.sh -n dnode2 -s stop
@ -154,7 +154,7 @@ if $data(3)[4] != ready then
goto step51 goto step51
endi endi
print =============== step6: stop dnode1 print =============== step6: stop dnode3
system sh/exec.sh -n dnode2 -s start system sh/exec.sh -n dnode2 -s start
system sh/exec.sh -n dnode3 -s stop system sh/exec.sh -n dnode3 -s stop

View File

@ -200,8 +200,9 @@ if $data02 != 2678400000 then
return -1 return -1
endi endi
sql_error select _wstart, count(tbcol), _wduration, _wstart, count(*) from ct3 interval(1n, 1w) sliding(2w) sql select _wstart, count(tbcol), _wduration, _wstart, count(*) from ct3 interval(1n, 1w) sliding(2w)
sql_error select _wstart, count(tbcol), _wduration, _wstart, count(*) from ct3 interval(1n, 1w) sliding(4w) sql select _wstart, count(tbcol), _wduration, _wstart, count(*) from ct3 interval(1n, 1w) sliding(4w)
sql_error select _wstart, count(tbcol), _wduration, _wstart, count(*) from ct3 interval(1n, 1w) sliding(5w)
sql select _wstart, count(tbcol), _wduration, _wstart, count(*) from ct4 interval(1y, 6n) sql select _wstart, count(tbcol), _wduration, _wstart, count(*) from ct4 interval(1y, 6n)
print ===> select _wstart, count(tbcol), _wduration, _wstart, count(*) from ct4 interval(1y, 6n) print ===> select _wstart, count(tbcol), _wduration, _wstart, count(*) from ct4 interval(1y, 6n)

View File

@ -276,6 +276,10 @@ sql insert into t6 values ("2024-03-01 14:34:07.051", 66);
sleep 300 sleep 300
sql select _wstart, count(*) from stb interval(1n) sliding(1d);
sql select _wstart, count(*) from stb interval(1n) sliding(28d);
sql_error select _wstart, count(*) from stb interval(1n) sliding(29d);
sql select _wstart, count(*) from (select * from stb partition by tbname) interval(2s); sql select _wstart, count(*) from (select * from stb partition by tbname) interval(2s);
print $data00,$data01 print $data00,$data01

View File

@ -1,6 +1,11 @@
import queue
import random
from fabric2.runners import threading
from pandas._libs import interval
import taos import taos
import sys import sys
from util.common import TDCom
from util.log import * from util.log import *
from util.sql import * from util.sql import *
from util.cases import * from util.cases import *
@ -8,6 +13,7 @@ from util.cases import *
class TDTestCase: class TDTestCase:
updatecfgDict = {'asynclog': 0, 'ttlUnit': 1, 'ttlPushInterval': 5, 'ratioOfVnodeStreamThrea': 4, 'numOfVnodeQueryThreads': 80}
def init(self, conn, logSql, replicaVar=1): def init(self, conn, logSql, replicaVar=1):
self.replicaVar = int(replicaVar) self.replicaVar = int(replicaVar)
@ -15,7 +21,115 @@ class TDTestCase:
#tdSql.init(conn.cursor()) #tdSql.init(conn.cursor())
tdSql.init(conn.cursor(), logSql) # output sql.txt file tdSql.init(conn.cursor(), logSql) # output sql.txt file
def generate_fill_range(self, data_start: int, data_end: int, interval: int, step: int):
ret = []
begin = data_start - 10 * interval
end = data_end + 10 * interval
for i in range(begin, end, step):
for j in range(begin, end, step):
ret.append((i,j))
return ret
def check_fill_range(self, where_start, where_end, res_asc, res_desc, sql: str, interval):
if len(res_asc) != len(res_desc):
tdLog.exit(f"err, asc desc with different rows, asc: {len(res_asc)}, desc: {len(res_desc)} sql: {sql}")
if len(res_asc) == 0:
tdLog.info(f'from {where_start} to {where_end} no rows returned')
return
asc_first = res_asc[0]
asc_last = res_asc[-1]
desc_first = res_desc[0]
desc_last = res_desc[-1]
if asc_first[0] != desc_last[0] or asc_last[0] != desc_first[0]:
tdLog.exit(f'fill sql different row data {sql}: asc<{asc_first[0].timestamp()}, {asc_last[0].timestamp()}>, desc<{desc_last[0].timestamp()}, {desc_first[0].timestamp()}>')
else:
tdLog.info(f'from {where_start} to {where_end} same time returned asc<{asc_first[0].timestamp()}, {asc_last[0].timestamp()}>, desc<{desc_last[0].timestamp()}, {desc_first[0].timestamp()}> interval: {interval}')
def generate_partition_by(self):
val = random.random()
if val < 0.6:
return ""
elif val < 0.8:
return "partition by location"
else:
return "partition by tbname"
def generate_fill_interval(self):
ret = []
#intervals = [60, 90, 120, 300, 3600]
intervals = [120, 300, 3600]
for i in range(0, len(intervals)):
for j in range(0, i+1):
ret.append((intervals[i], intervals[j]))
return ret
def generate_fill_sql(self, where_start, where_end, fill_interval: tuple):
partition_by = self.generate_partition_by()
where = f'where ts >= {where_start} and ts < {where_end}'
sql = f'select first(_wstart), last(_wstart) from (select _wstart, _wend, count(*) from test.meters {where} {partition_by} interval({fill_interval[0]}s) sliding({fill_interval[1]}s) fill(NULL)'
sql_asc = sql + " order by _wstart asc) t"
sql_desc = sql + " order by _wstart desc) t"
return sql_asc, sql_desc
def fill_test_thread_routine(self, cli: TDSql, interval, data_start, data_end, step):
ranges = self.generate_fill_range(data_start, data_end, interval[0], step)
for range in ranges:
sql_asc, sql_desc = self.generate_fill_sql(range[0], range[1], interval)
cli.query(sql_asc, queryTimes=1)
asc_res = cli.queryResult
cli.query(sql_desc, queryTimes=1)
desc_res = cli.queryResult
self.check_fill_range(range[0], range[1], asc_res,desc_res , sql_asc, interval)
def fill_test_task_routine(self, tdCom: TDCom, queue: queue.Queue):
cli = tdCom.newTdSql()
while True:
m: list = queue.get()
if len(m) == 0:
break
interval = m[0]
range = m[1]
sql_asc, sql_desc = self.generate_fill_sql(range[0], range[1], interval)
cli.query(sql_asc, queryTimes=1)
asc_res = cli.queryResult
cli.query(sql_desc, queryTimes=1)
desc_res = cli.queryResult
self.check_fill_range(range[0], range[1], asc_res,desc_res , sql_asc, interval)
cli.close()
def schedule_fill_test_tasks(self):
num: int = 20
threads = []
tdCom = TDCom()
q: queue.Queue = queue.Queue()
for _ in range(num):
t = threading.Thread(target=self.fill_test_task_routine, args=(tdCom, q))
t.start()
threads.append(t)
data_start = 1500000000000
data_end = 1500319968000
step = 30000000
fill_intervals: list[tuple] = self.generate_fill_interval()
for interval in fill_intervals:
ranges = self.generate_fill_range(data_start, data_end, interval[0], step)
for r in ranges:
q.put([interval, r])
for _ in range(num):
q.put([])
for t in threads:
t.join()
def test_fill_range(self):
os.system('taosBenchmark -t 10 -n 10000 -v 8 -S 32000 -y')
self.schedule_fill_test_tasks()
tdSql.execute('drop database test')
def run(self): def run(self):
self.test_fill_range()
dbname = "db" dbname = "db"
tbname = "tb" tbname = "tb"

View File

@ -176,7 +176,7 @@ class TDTestCase:
def test_query_with_window(self): def test_query_with_window(self):
# time window # time window
tdSql.query("select sum(c_int_empty) from st where ts > '2024-01-01 00:00:00.000' and ts <= '2024-01-01 14:00:00.000' interval(5m) sliding(1m) fill(value, 10);") tdSql.query("select sum(c_int_empty) from st where ts > '2024-01-01 00:00:00.000' and ts <= '2024-01-01 14:00:00.000' interval(5m) sliding(1m) fill(value, 10);")
tdSql.checkRows(841) tdSql.checkRows(845)
tdSql.checkData(0, 0, 10) tdSql.checkData(0, 0, 10)
tdSql.query("select _wstart, _wend, sum(c_int) from st where ts > '2024-01-01 00:00:00.000' and ts <= '2024-01-01 14:00:00.000' interval(5m) sliding(1m);") tdSql.query("select _wstart, _wend, sum(c_int) from st where ts > '2024-01-01 00:00:00.000' and ts <= '2024-01-01 14:00:00.000' interval(5m) sliding(1m);")

View File

@ -1504,9 +1504,9 @@ class TDTestCase:
# max number of list is 4093: 4096 - 3 - 2(原始表tag个数) - 1(tbname) # max number of list is 4093: 4096 - 3 - 2(原始表tag个数) - 1(tbname)
tdSql.execute('use db4096') tdSql.execute('use db4096')
self.create_tsma('tsma_4050', 'db4096', 'stb0', self.generate_tsma_function_list_columns(4050), '5m',check_tsma_calculation=False) self.create_tsma('tsma_4050', 'db4096', 'stb0', self.generate_tsma_function_list_columns(4050), '5m',check_tsma_calculation=True)
self.create_tsma('tsma_4090', 'db4096', 'stb0', self.generate_tsma_function_list_columns(4090), '6m',check_tsma_calculation=False) self.create_tsma('tsma_4090', 'db4096', 'stb0', self.generate_tsma_function_list_columns(4090), '6m',check_tsma_calculation=True)
self.create_error_tsma('tsma_4091', 'db4096', 'stb0', self.generate_tsma_function_list_columns(4091), '5m', -2147473856) #Too many columns self.create_error_tsma('tsma_4091', 'db4096', 'stb0', self.generate_tsma_function_list_columns(4091), '5m', -2147473856) #Too many columns

View File

@ -1170,6 +1170,7 @@ bool shellGetGrantInfo(char* buf) {
code != TSDB_CODE_PAR_PERMISSION_DENIED) { code != TSDB_CODE_PAR_PERMISSION_DENIED) {
fprintf(stderr, "Failed to check Server Edition, Reason:0x%04x:%s\r\n\r\n", code, taos_errstr(tres)); fprintf(stderr, "Failed to check Server Edition, Reason:0x%04x:%s\r\n\r\n", code, taos_errstr(tres));
} }
taos_free_result(tres);
return community; return community;
} }