xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' into fix/TS-2149

This commit is contained in:
Ganlin Zhao 2022-11-28 23:42:16 +08:00
parent 94c8ba64d2 1f77b78de7
commit 682b3b60ef
52 changed files with 2913 additions and 3790 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/zh/12-taos-sql/10-function.md
View File

@ -1249,4 +1249,4 @@ SELECT SERVER_VERSION();
SELECT SERVER_STATUS();
```
**说明**：返回服务端当前的状态。
**说明**：检测服务端是否所有 dnode 都在线，如果是则返回成功，否则返回无法建立连接的错误。

27
include/common/tcommon.h
View File

@ -174,15 +174,28 @@ typedef struct SColumnDataAgg {
} SColumnDataAgg;
#pragma pack(pop)
typedef struct SBlockID {
// The uid of table, from which current data block comes. And it is always 0, if current block is the
// result of calculation.
uint64_t uid;
// Block id, acquired and assigned from executor, which created according to the hysical planner. Block id is used
// to mark the stage of exec task.
uint64_t blockId;
// Generated by group/partition by [value|tags]. Created and assigned by table-scan operator, group-by operator,
// and partition by operator.
uint64_t groupId;
} SBlockID;
typedef struct SDataBlockInfo {
STimeWindow window;
int32_t rows; // todo hide this attribute
int32_t rowSize;
uint64_t uid; // the uid of table, from which current data block comes
uint16_t blockId; // block id, generated by physical planner
uint64_t groupId;
int16_t hasVarCol;
int32_t rows; // todo hide this attribute
uint32_t capacity;
SBlockID id;
int16_t hasVarCol;
// TODO: optimize and remove following
int64_t version; // used for stream, and need serialization
int32_t childId; // used for stream, do not serialize
@ -190,8 +203,8 @@ typedef struct SDataBlockInfo {
STimeWindow calWin; // used for stream, do not serialize
TSKEY watermark; // used for stream
char parTbName[TSDB_TABLE_NAME_LEN]; // used for stream partition
STag* pTag; // used for stream partition
char parTbName[TSDB_TABLE_NAME_LEN]; // used for stream partition
STag* pTag; // used for stream partition
} SDataBlockInfo;
typedef struct SSDataBlock {

40
packaging/MPtestJenkinsfile
View File

@ -120,20 +120,20 @@ pipeline {
for verModeSin in ${verModeList}
do
cd ${TDENGINE_ROOT_DIR}/packaging
bash testpackage.sh -f server -m ${verModeSin} -f server -l false -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar
bash testpackage.sh -m ${verModeSin} -f server -l false -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar
python3 checkPackageRuning.py
done
'''
sh '''
cd ${TDENGINE_ROOT_DIR}/packaging
bash testpackage.sh -f server -m community -f server -l true -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar
bash testpackage.sh -m community -f server -l true -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar
python3 checkPackageRuning.py
'''
sh '''
cd ${TDENGINE_ROOT_DIR}/packaging
bash testpackage.sh -f server -m community -f server -l false -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t deb
bash testpackage.sh -m community -f server -l false -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t deb
python3 checkPackageRuning.py
'''
}
@ -152,20 +152,20 @@ pipeline {
for verModeSin in ${verModeList}
do
cd ${TDENGINE_ROOT_DIR}/packaging
bash testpackage.sh -f server -m ${verModeSin} -f server -l false -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar
bash testpackage.sh -m ${verModeSin} -f server -l false -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar
python3 checkPackageRuning.py
done
'''
sh '''
cd ${TDENGINE_ROOT_DIR}/packaging
bash testpackage.sh -f server -m community -f server -l true -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar
bash testpackage.sh -m community -f server -l true -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar
python3 checkPackageRuning.py
'''
sh '''
cd ${TDENGINE_ROOT_DIR}/packaging
bash testpackage.sh -f server -m community -f server -l false -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t deb
bash testpackage.sh -m community -f server -l false -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t deb
python3 checkPackageRuning.py
dpkg -r tdengine
'''
@ -185,20 +185,20 @@ pipeline {
for verModeSin in ${verModeList}
do
cd ${TDENGINE_ROOT_DIR}/packaging
bash testpackage.sh -f server -m community -f server -l false -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar
bash testpackage.sh -m ${verModeSin} -f server -l false -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar
python3 checkPackageRuning.py
done
'''
sh '''
cd ${TDENGINE_ROOT_DIR}/packaging
bash testpackage.sh -f server -m community -f server -l true -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar
bash testpackage.sh -m community -f server -l true -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar
python3 checkPackageRuning.py
'''
sh '''
cd ${TDENGINE_ROOT_DIR}/packaging
bash testpackage.sh -f server -m community -f server -l false -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t rpm
bash testpackage.sh -m community -f server -l false -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t rpm
python3 checkPackageRuning.py
'''
}
@ -217,20 +217,20 @@ pipeline {
for verModeSin in ${verModeList}
do
cd ${TDENGINE_ROOT_DIR}/packaging
bash testpackage.sh -f server -m ${verModeSin} -f server -l false -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar
bash testpackage.sh -m ${verModeSin} -f server -l false -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar
python3 checkPackageRuning.py
done
'''
sh '''
cd ${TDENGINE_ROOT_DIR}/packaging
bash testpackage.sh -f server -m community -f server -l true -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar
bash testpackage.sh -m community -f server -l true -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar
python3 checkPackageRuning.py
'''
sh '''
cd ${TDENGINE_ROOT_DIR}/packaging
bash testpackage.sh -f server -m community -f server -l false -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t rpm
bash testpackage.sh -m community -f server -l false -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t rpm
python3 checkPackageRuning.py
sudo rpm -e tdengine
'''
@ -250,7 +250,7 @@ pipeline {
for verModeSin in ${verModeList}
do
cd ${TDENGINE_ROOT_DIR}/packaging
bash testpackage.sh -f server -m ${verModeSin} -f server -l false -c arm64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar
bash testpackage.sh -m ${verModeSin} -f server -l false -c arm64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar
python3 checkPackageRuning.py
done
'''
@ -273,10 +273,9 @@ pipeline {
for verModeSin in ${verModeList}
do
cd ${TDENGINE_ROOT_DIR}/packaging
bash testpackage.sh -f server -m ${verModeSin} -f client -l false -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar
python3 checkPackageRuning.py
bash testpackage.sh -m ${verModeSin} -f client -l false -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar
python3 checkPackageRuning.py 192.168.0.21
done
python3 checkPackageRuning.py 192.168.0.21
'''
}
}
@ -286,10 +285,8 @@ pipeline {
steps {
timeout(time: 30, unit: 'MINUTES'){
sh '''
verModeList=community
cd ${TDENGINE_ROOT_DIR}/packaging
bash testpackage.sh -f server -m ${verModeSin} -f client -l true -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar
python3 checkPackageRuning.py
bash testpackage.sh -m community -f client -l true -c x64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar
python3 checkPackageRuning.py 192.168.0.24
'''
}
@ -309,10 +306,9 @@ pipeline {
for verModeSin in ${verModeList}
do
cd ${TDENGINE_ROOT_DIR}/packaging
bash testpackage.sh -f server -m ${verModeSin} -f client -l false -c arm64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar
python3 checkPackageRuning.py
bash testpackage.sh -m ${verModeSin} -f client -l false -c arm64 -v ${version} -o ${baseVersion} -s ${sourcePath} -t tar
python3 checkPackageRuning.py 192.168.0.21
done
python3 checkPackageRuning.py 192.168.0.21
'''
}
}

6
packaging/testpackage.sh
View File

@ -206,6 +206,10 @@ else
fi
if [[ ${packgeName} =~ "server" ]] ;then
echoColor BD " pkill -9 taosd "
pkill -9 taosd
fi
echoColor G "===== new workroom path ====="
@ -334,7 +338,7 @@ elif ([[ ${packgeName} =~ "arm64" ]] && [[ ${packgeName} =~ "client" ]]);then
echoColor G "===== install taos-tools arm when package is arm64-client ====="
cd ${installPath}
if [ ! -f "taosTools-2.1.3-Linux-x64.tar.gz " ];then
wgetFile taosTools-2.1.3-Linux-x64.tar.gz v2.1.3 web
wgetFile taosTools-2.1.3-Linux-arm64.tar.gz v2.1.3 web
tar xf taosTools-2.1.3-Linux-arm64.tar.gz
fi

25
source/common/src/tdatablock.c
View File

@ -621,7 +621,7 @@ int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) {
// todo remove this
int32_t blockDataFromBuf1(SSDataBlock* pBlock, const char* buf, size_t capacity) {
pBlock->info.rows = *(int32_t*)buf;
pBlock->info.groupId = *(uint64_t*)(buf + sizeof(int32_t));
pBlock->info.id.groupId = *(uint64_t*)(buf + sizeof(int32_t));
size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
@ -1140,7 +1140,8 @@ void blockDataCleanup(SSDataBlock* pDataBlock) {
SDataBlockInfo* pInfo = &pDataBlock->info;
pInfo->rows = 0;
pInfo->groupId = 0;
pInfo->id.uid = 0;
pInfo->id.groupId = 0;
pInfo->window.ekey = 0;
pInfo->window.skey = 0;
@ -1334,7 +1335,7 @@ int32_t copyDataBlock(SSDataBlock* dst, const SSDataBlock* src) {
SSDataBlock* createSpecialDataBlock(EStreamType type) {
SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock));
pBlock->info.hasVarCol = false;
pBlock->info.groupId = 0;
pBlock->info.id.groupId = 0;
pBlock->info.rows = 0;
pBlock->info.type = type;
pBlock->info.rowSize = sizeof(TSKEY) + sizeof(TSKEY) + sizeof(uint64_t) + sizeof(uint64_t) + sizeof(TSKEY) +
@ -1675,7 +1676,7 @@ int32_t blockDataKeepFirstNRows(SSDataBlock* pBlock, size_t n) {
}
int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) {
int64_t tbUid = pBlock->info.uid;
int64_t tbUid = pBlock->info.id.uid;
int16_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
int16_t hasVarCol = pBlock->info.hasVarCol;
int32_t rows = pBlock->info.rows;
@ -1713,7 +1714,7 @@ void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock) {
int16_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
buf = taosDecodeFixedU64(buf, &pBlock->info.uid);
buf = taosDecodeFixedU64(buf, &pBlock->info.id.uid);
buf = taosDecodeFixedI16(buf, &numOfCols);
buf = taosDecodeFixedI16(buf, &pBlock->info.hasVarCol);
buf = taosDecodeFixedI32(buf, &pBlock->info.rows);
@ -1834,7 +1835,7 @@ void blockDebugShowDataBlocks(const SArray* dataBlocks, const char* flag) {
int32_t rows = pDataBlock->info.rows;
printf("%s |block ver %" PRIi64 " |block type %d |child id %d|group id %" PRIu64 "\n", flag,
pDataBlock->info.version, (int32_t)pDataBlock->info.type, pDataBlock->info.childId,
pDataBlock->info.groupId);
pDataBlock->info.id.groupId);
for (int32_t j = 0; j < rows; j++) {
printf("%s |", flag);
for (int32_t k = 0; k < numOfCols; k++) {
@ -1905,8 +1906,8 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf)
len += snprintf(dumpBuf + len, size - len,
"===stream===%s|block type %d|child id %d|group id:%" PRIu64 "|uid:%" PRId64
"|rows:%d|version:%" PRIu64 "\n",
flag, (int32_t)pDataBlock->info.type, pDataBlock->info.childId, pDataBlock->info.groupId,
pDataBlock->info.uid, pDataBlock->info.rows, pDataBlock->info.version);
flag, (int32_t)pDataBlock->info.type, pDataBlock->info.childId, pDataBlock->info.id.groupId,
pDataBlock->info.id.uid, pDataBlock->info.rows, pDataBlock->info.version);
if (len >= size - 1) return dumpBuf;
for (int32_t j = 0; j < rows; j++) {
@ -2035,8 +2036,6 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SSDataBlock* pDataB
for (int32_t i = 0; i < sz; ++i) {
int32_t colNum = taosArrayGetSize(pDataBlock->pDataBlock);
int32_t rows = pDataBlock->info.rows;
// int32_t rowSize = pDataBlock->info.rowSize;
// int64_t groupId = pDataBlock->info.groupId;
if (colNum <= 1) {
// invalid if only with TS col
@ -2049,7 +2048,7 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SSDataBlock* pDataB
SSubmitBlk* pSubmitBlk = POINTER_SHIFT(pDataBuf, msgLen);
pSubmitBlk->suid = suid;
pSubmitBlk->uid = pDataBlock->info.groupId;
pSubmitBlk->uid = pDataBlock->info.id.groupId;
pSubmitBlk->numOfRows = rows;
pSubmitBlk->sversion = pTSchema->version;
@ -2292,7 +2291,7 @@ int32_t blockEncode(const SSDataBlock* pBlock, char* data, int32_t numOfCols) {
}
*actualLen = dataLen;
*groupId = pBlock->info.groupId;
*groupId = pBlock->info.id.groupId;
ASSERT(dataLen > 0);
uDebug("build data block, actualLen:%d, rows:%d, cols:%d", dataLen, *rows, *cols);
@ -2325,7 +2324,7 @@ const char* blockDecode(SSDataBlock* pBlock, const char* pData) {
pStart += sizeof(int32_t);
// group id sizeof(uint64_t)
pBlock->info.groupId = *(uint64_t*)pStart;
pBlock->info.id.groupId = *(uint64_t*)pStart;
pStart += sizeof(uint64_t);
if (pBlock->pDataBlock == NULL) {

1
source/dnode/mnode/impl/src/mndStream.c
View File

@ -525,6 +525,7 @@ static int32_t mndCreateStbForStream(SMnode *pMnode, STrans *pTrans, const SStre
tFreeSMCreateStbReq(&createReq);
mndFreeStb(&stbObj);
mndReleaseStb(pMnode, pStb);
mndReleaseDb(pMnode, pDb);
return 0;

1
source/dnode/mnode/impl/src/mndTopic.c
View File

@ -434,6 +434,7 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq *
return -1;
}
topicObj.stbUid = pStb->uid;
mndReleaseStb(pMnode, pStb);
}
/*} else if (pCreate->subType == TOPIC_SUB_TYPE__DB) {*/
/*topicObj.ast = NULL;*/

8
source/dnode/vnode/src/sma/smaRollup.c
View File

@ -708,7 +708,7 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma
#endif
for (int32_t i = 0; i < taosArrayGetSize(pResList); ++i) {
SSDataBlock *output = taosArrayGetP(pResList, i);
smaDebug("result block, uid:%" PRIu64 ", groupid:%" PRIu64 ", rows:%d", output->info.uid, output->info.groupId,
smaDebug("result block, uid:%" PRIu64 ", groupid:%" PRIu64 ", rows:%d", output->info.id.uid, output->info.id.groupId,
output->info.rows);
STsdb *sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb[0] : pSma->pRSmaTsdb[1]);
@ -718,7 +718,7 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma
if (buildSubmitReqFromDataBlock(&pReq, output, pTSchema, SMA_VID(pSma), suid) < 0) {
smaError("vgId:%d, build submit req for rsma table suid:%" PRIu64 ", uid:%" PRIu64 ", level %" PRIi8
" failed since %s",
SMA_VID(pSma), suid, output->info.groupId, pItem->level, terrstr());
SMA_VID(pSma), suid, output->info.id.groupId, pItem->level, terrstr());
goto _err;
}
@ -726,13 +726,13 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma
taosMemoryFreeClear(pReq);
smaError("vgId:%d, process submit req for rsma suid:%" PRIu64 ", uid:%" PRIu64 " level %" PRIi8
" failed since %s",
SMA_VID(pSma), suid, output->info.groupId, pItem->level, terrstr());
SMA_VID(pSma), suid, output->info.id.groupId, pItem->level, terrstr());
goto _err;
}
smaDebug("vgId:%d, process submit req for rsma suid:%" PRIu64 ",uid:%" PRIu64 ", level %" PRIi8 " ver %" PRIi64
" len %" PRIu32,
SMA_VID(pSma), suid, output->info.groupId, pItem->level, output->info.version,
SMA_VID(pSma), suid, output->info.id.groupId, pItem->level, output->info.version,
htonl(pReq->header.contLen));
taosMemoryFreeClear(pReq);

2
source/dnode/vnode/src/tq/tqExec.c
View File

@ -299,7 +299,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp
taosArrayDestroyP(pSchemas, (FDelete)tDeleteSSchemaWrapper);
pBlocks = taosArrayInit(0, sizeof(SSDataBlock));
pSchemas = taosArrayInit(0, sizeof(void*));
return -1;
continue;
}
}
if (pHandle->fetchMeta) {

1
source/dnode/vnode/src/tq/tqOffset.c
View File

@ -61,6 +61,7 @@ int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname) {
ASSERT(0);
// TODO
}
taosMemoryFree(memBuf);
}
taosCloseFile(&pFile);

4
source/dnode/vnode/src/tq/tqRead.c
View File

@ -530,7 +530,7 @@ int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReader* pReader) {
tInitSubmitBlkIter(&pReader->msgIter, pReader->pBlock, &pReader->blkIter);
pBlock->info.uid = pReader->msgIter.uid;
pBlock->info.id.uid = pReader->msgIter.uid;
pBlock->info.rows = pReader->msgIter.numOfRows;
pBlock->info.version = pReader->pMsg->version;
@ -649,7 +649,7 @@ int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas
}
SSDataBlock* pBlock = taosArrayGetLast(blocks);
pBlock->info.uid = pReader->msgIter.uid;
pBlock->info.id.uid = pReader->msgIter.uid;
pBlock->info.rows = 0;
pBlock->info.version = pReader->pMsg->version;

10
source/dnode/vnode/src/tq/tqSink.c
View File

@ -103,7 +103,7 @@ SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchem
// STagVal tagVal = {
// .cid = pTagSchemaWrapper->pSchema[j].colId,
// .type = pTagSchemaWrapper->pSchema[j].type,
// .i64 = (int64_t)pDataBlock->info.groupId,
// .i64 = (int64_t)pDataBlock->info.id.groupId,
// };
// taosArrayPush(tagArray, &tagVal);
// taosArrayPush(tagName, pTagSchemaWrapper->pSchema[j].name);
@ -134,7 +134,7 @@ SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchem
STagVal tagVal = {
.cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1,
.type = TSDB_DATA_TYPE_UBIGINT,
.i64 = (int64_t)pDataBlock->info.groupId,
.i64 = (int64_t)pDataBlock->info.id.groupId,
};
taosArrayPush(tagArray, &tagVal);
createTbReq.ctb.tagNum = taosArrayGetSize(tagArray);
@ -161,7 +161,7 @@ SSubmitReq* tqBlockToSubmit(SVnode* pVnode, const SArray* pBlocks, const STSchem
if (pDataBlock->info.parTbName[0]) {
createTbReq.name = strdup(pDataBlock->info.parTbName);
} else {
createTbReq.name = buildCtbNameByGroupId(stbFullName, pDataBlock->info.groupId);
createTbReq.name = buildCtbNameByGroupId(stbFullName, pDataBlock->info.id.groupId);
}
// save schema len
@ -358,7 +358,7 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d
if (pDataBlock->info.parTbName[0]) {
ctbName = strdup(pDataBlock->info.parTbName);
} else {
ctbName = buildCtbNameByGroupId(stbFullName, pDataBlock->info.groupId);
ctbName = buildCtbNameByGroupId(stbFullName, pDataBlock->info.id.groupId);
}
int32_t schemaLen = 0;
@ -390,7 +390,7 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d
STagVal tagVal = {
.cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1,
.type = TSDB_DATA_TYPE_UBIGINT,
.i64 = (int64_t)pDataBlock->info.groupId,
.i64 = (int64_t)pDataBlock->info.id.groupId,
};
taosArrayPush(tagArray, &tagVal);
createTbReq.ctb.tagNum = taosArrayGetSize(tagArray);

10
source/dnode/vnode/src/tsdb/tsdbRead.c
View File

@ -1624,7 +1624,7 @@ static int32_t buildDataBlockFromBuf(STsdbReader* pReader, STableBlockScanInfo*
int32_t code = buildDataBlockFromBufImpl(pBlockScanInfo, endKey, pReader->capacity, pReader);
blockDataUpdateTsWindow(pBlock, 0);
pBlock->info.uid = pBlockScanInfo->uid;
pBlock->info.id.uid = pBlockScanInfo->uid;
setComposedBlockFlag(pReader, true);
@ -2494,7 +2494,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) {
}
_end:
pResBlock->info.uid = (pBlockScanInfo != NULL) ? pBlockScanInfo->uid : 0;
pResBlock->info.id.uid = (pBlockScanInfo != NULL) ? pBlockScanInfo->uid : 0;
blockDataUpdateTsWindow(pResBlock, 0);
setComposedBlockFlag(pReader, true);
@ -2506,7 +2506,7 @@ _end:
if (pResBlock->info.rows > 0) {
tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64
" rows:%d, elapsed time:%.2f ms %s",
pReader, pResBlock->info.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey,
pReader, pResBlock->info.id.uid, pResBlock->info.window.skey, pResBlock->info.window.ekey,
pResBlock->info.rows, el, pReader->idStr);
}
@ -2830,7 +2830,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) {
} else { // whole block is required, return it directly
SDataBlockInfo* pInfo = &pReader->pResBlock->info;
pInfo->rows = pBlock->nRow;
pInfo->uid = pScanInfo->uid;
pInfo->id.uid = pScanInfo->uid;
pInfo->window = (STimeWindow){.skey = pBlock->minKey.ts, .ekey = pBlock->maxKey.ts};
setComposedBlockFlag(pReader, false);
setBlockAllDumped(&pStatus->fBlockDumpInfo, pBlock->maxKey.ts, pReader->order);
@ -4020,7 +4020,7 @@ bool tsdbTableNextDataBlock(STsdbReader* pReader, uint64_t uid) {
static void setBlockInfo(const STsdbReader* pReader, int32_t* rows, uint64_t* uid, STimeWindow* pWindow) {
ASSERT(pReader != NULL);
*rows = pReader->pResBlock->info.rows;
*uid = pReader->pResBlock->info.uid;
*uid = pReader->pResBlock->info.id.uid;
*pWindow = pReader->pResBlock->info.window;
}

2
source/dnode/vnode/test/tsdbSmaTest.cpp
View File

@ -436,7 +436,7 @@ TEST(testCase, tSma_Data_Insert_Query_Test) {
pDataBlock->pBlockAgg = NULL;
taosArrayGetSize(pDataBlock->pDataBlock) = tSmaNumOfCols;
pDataBlock->info.rows = tSmaNumOfRows;
pDataBlock->info.groupId = tSmaGroupId + g;
pDataBlock->info.id.groupId = tSmaGroupId + g;
pDataBlock->pDataBlock = taosArrayInit(tSmaNumOfCols, sizeof(SColumnInfoData *));
EXPECT_NE(pDataBlock->pDataBlock, nullptr);

24
source/libs/command/src/command.c
View File

@ -270,10 +270,10 @@ static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char* dbFName, S
"CREATE DATABASE `%s` BUFFER %d CACHESIZE %d CACHEMODEL '%s' COMP %d DURATION %dm "
"WAL_FSYNC_PERIOD %d MAXROWS %d MINROWS %d KEEP %dm,%dm,%dm PAGES %d PAGESIZE %d PRECISION '%s' REPLICA %d "
"STRICT '%s' WAL_LEVEL %d VGROUPS %d SINGLE_STABLE %d",
dbFName, pCfg->buffer, pCfg->cacheSize, cacheModelStr(pCfg->cacheLast), pCfg->compression, pCfg->daysPerFile, pCfg->walFsyncPeriod,
pCfg->maxRows, pCfg->minRows, pCfg->daysToKeep0, pCfg->daysToKeep1, pCfg->daysToKeep2, pCfg->pages,
pCfg->pageSize, prec, pCfg->replications, strictStr(pCfg->strict), pCfg->walLevel, pCfg->numOfVgroups,
1 == pCfg->numOfStables);
dbFName, pCfg->buffer, pCfg->cacheSize, cacheModelStr(pCfg->cacheLast), pCfg->compression, pCfg->daysPerFile,
pCfg->walFsyncPeriod, pCfg->maxRows, pCfg->minRows, pCfg->daysToKeep0, pCfg->daysToKeep1, pCfg->daysToKeep2,
pCfg->pages, pCfg->pageSize, prec, pCfg->replications, strictStr(pCfg->strict), pCfg->walLevel,
pCfg->numOfVgroups, 1 == pCfg->numOfStables);
if (retentions) {
len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, " RETENTIONS %s", retentions);
@ -500,8 +500,8 @@ static int32_t setCreateTBResultIntoDataBlock(SSDataBlock* pBlock, SDbCfgInfo* p
terrno = TSDB_CODE_TSC_OUT_OF_MEMORY;
return terrno;
}
int32_t len = 0;
int32_t len = 0;
if (TSDB_SUPER_TABLE == pCfg->tableType) {
len += sprintf(buf2 + VARSTR_HEADER_SIZE, "CREATE STABLE `%s` (", tbName);
@ -533,7 +533,7 @@ static int32_t setCreateTBResultIntoDataBlock(SSDataBlock* pBlock, SDbCfgInfo* p
colDataAppend(pCol2, 0, buf2, false);
taosMemoryFree(buf2);
return TSDB_CODE_SUCCESS;
}
@ -691,9 +691,15 @@ static int32_t createSelectResultDataBlock(SNodeList* pProjects, SSDataBlock** p
SNode* pProj = NULL;
FOREACH(pProj, pProjects) {
SExprNode* pExpr = (SExprNode*)pProj;
SColumnInfoData infoData = {0};
infoData.info.type = ((SExprNode*)pProj)->resType.type;
infoData.info.bytes = ((SExprNode*)pProj)->resType.bytes;
if (TSDB_DATA_TYPE_NULL == pExpr->resType.type) {
infoData.info.type = TSDB_DATA_TYPE_VARCHAR;
infoData.info.bytes = 0;
} else {
infoData.info.type = pExpr->resType.type;
infoData.info.bytes = pExpr->resType.bytes;
}
blockDataAppendColInfo(pBlock, &infoData);
}
*pOutput = pBlock;

32
source/libs/executor/inc/executil.h
View File

@ -38,16 +38,7 @@
memcpy((_k) + sizeof(uint64_t), (_ori), (_len)); \
} while (0)
#define SET_RES_EXT_WINDOW_KEY(_k, _ori, _len, _uid, _buf) \
do { \
assert(sizeof(_uid) == sizeof(uint64_t)); \
*(void**)(_k) = (_buf); \
*(uint64_t*)((_k) + POINTER_BYTES) = (_uid); \
memcpy((_k) + POINTER_BYTES + sizeof(uint64_t), (_ori), (_len)); \
} while (0)
#define GET_RES_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t))
#define GET_RES_EXT_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t) + POINTER_BYTES)
#define GET_TASKID(_t) (((SExecTaskInfo*)(_t))->id.str)
@ -104,16 +95,17 @@ int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags
STableListInfo* pTableListInfo, SNode* pTagCond, SNode* pTagIndexCond, SExecTaskInfo* pTaskInfo);
STableListInfo* tableListCreate();
void* tableListDestroy(STableListInfo* pTableListInfo);
void tableListClear(STableListInfo* pTableListInfo);
int32_t tableListGetOutputGroups(const STableListInfo* pTableList);
bool oneTableForEachGroup(const STableListInfo* pTableList);
uint64_t getTableGroupId(const STableListInfo* pTableList, uint64_t tableUid);
int32_t tableListAddTableInfo(STableListInfo* pTableList, uint64_t uid, uint64_t gid);
int32_t tableListGetGroupList(const STableListInfo* pTableList, int32_t ordinalIndex, STableKeyInfo** pKeyInfo, int32_t* num);
uint64_t tableListGetSize(const STableListInfo* pTableList);
uint64_t tableListGetSuid(const STableListInfo* pTableList);
STableKeyInfo* tableListGetInfo(const STableListInfo* pTableList, int32_t index);
void* tableListDestroy(STableListInfo* pTableListInfo);
void tableListClear(STableListInfo* pTableListInfo);
int32_t tableListGetOutputGroups(const STableListInfo* pTableList);
bool oneTableForEachGroup(const STableListInfo* pTableList);
uint64_t getTableGroupId(const STableListInfo* pTableList, uint64_t tableUid);
int32_t tableListAddTableInfo(STableListInfo* pTableList, uint64_t uid, uint64_t gid);
int32_t tableListGetGroupList(const STableListInfo* pTableList, int32_t ordinalIndex, STableKeyInfo** pKeyInfo,
int32_t* num);
uint64_t tableListGetSize(const STableListInfo* pTableList);
uint64_t tableListGetSuid(const STableListInfo* pTableList);
STableKeyInfo* tableListGetInfo(const STableListInfo* pTableList, int32_t index);
size_t getResultRowSize(struct SqlFunctionCtx* pCtx, int32_t numOfOutput);
void initResultRowInfo(SResultRowInfo* pResultRowInfo);
@ -140,7 +132,7 @@ bool hasRemainResults(SGroupResInfo* pGroupResInfo);
int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo);
SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode);
SSDataBlock* createDataBlockFromDescNode(SDataBlockDescNode* pNode);
EDealRes doTranslateTagExpr(SNode** pNode, void* pContext);
int32_t getGroupIdFromTagsVal(void* pMeta, uint64_t uid, SNodeList* pGroupNode, char* keyBuf, uint64_t* pGroupId);

220
source/libs/executor/inc/executorimpl.h
View File

@ -537,15 +537,6 @@ typedef struct SStreamIntervalOperatorInfo {
SWinKey delKey;
} SStreamIntervalOperatorInfo;
typedef struct SAggOperatorInfo {
SOptrBasicInfo binfo;
SAggSupporter aggSup;
STableQueryInfo* current;
uint64_t groupId;
SGroupResInfo groupResInfo;
SExprSupp scalarExprSup;
} SAggOperatorInfo;
typedef struct SFillOperatorInfo {
struct SFillInfo* pFillInfo;
SSDataBlock* pRes;
@ -577,18 +568,6 @@ typedef struct SWindowRowsSup {
uint64_t groupId;
} SWindowRowsSup;
typedef struct SSessionAggOperatorInfo {
SOptrBasicInfo binfo;
SAggSupporter aggSup;
SGroupResInfo groupResInfo;
SWindowRowsSup winSup;
bool reptScan; // next round scan
int64_t gap; // session window gap
int32_t tsSlotId; // primary timestamp slot id
STimeWindowAggSupp twAggSup;
} SSessionAggOperatorInfo;
typedef struct SResultWindowInfo {
void* pOutputBuf;
SSessionKey sessionWin;
@ -681,53 +660,30 @@ typedef struct SStreamFillOperatorInfo {
SStreamFillInfo* pFillInfo;
} SStreamFillOperatorInfo;
typedef struct STimeSliceOperatorInfo {
SSDataBlock* pRes;
STimeWindow win;
SInterval interval;
int64_t current;
SArray* pPrevRow; // SArray<SGroupValue>
SArray* pNextRow; // SArray<SGroupValue>
SArray* pLinearInfo; // SArray<SFillLinearInfo>
bool isPrevRowSet;
bool isNextRowSet;
int32_t fillType; // fill type
SColumn tsCol; // primary timestamp column
SExprSupp scalarSup; // scalar calculation
struct SFillColInfo* pFillColInfo; // fill column info
} STimeSliceOperatorInfo;
typedef struct SStateWindowOperatorInfo {
// SOptrBasicInfo should be first, SAggSupporter should be second for stream encode
SOptrBasicInfo binfo;
SAggSupporter aggSup;
SExprSupp scalarSup;
SGroupResInfo groupResInfo;
SWindowRowsSup winSup;
SColumn stateCol; // start row index
bool hasKey;
SStateKeys stateKey;
int32_t tsSlotId; // primary timestamp column slot id
STimeWindowAggSupp twAggSup;
} SStateWindowOperatorInfo;
#define OPTR_IS_OPENED(_optr) (((_optr)->status & OP_OPENED) == OP_OPENED)
#define OPTR_SET_OPENED(_optr) ((_optr)->status |= OP_OPENED)
SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t cleanup,
__optr_close_fn_t closeFn, __optr_explain_fn_t explain);
int32_t operatorDummyOpenFn(SOperatorInfo* pOperator);
int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t num);
int32_t operatorDummyOpenFn(SOperatorInfo* pOperator);
int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t num);
void setOperatorCompleted(SOperatorInfo* pOperator);
void setOperatorInfo(SOperatorInfo* pOperator, const char* name, int32_t type, bool blocking, int32_t status,
void* pInfo, SExecTaskInfo* pTaskInfo);
void destroyOperatorInfo(SOperatorInfo* pOperator);
void initBasicInfo(SOptrBasicInfo* pInfo, SSDataBlock* pBlock);
void cleanupBasicInfo(SOptrBasicInfo* pInfo);
int32_t initExprSupp(SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfExpr);
void cleanupExprSupp(SExprSupp* pSup);
void destroyExprInfo(SExprInfo* pExpr, int32_t numOfExprs);
int32_t initAggInfo(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize,
const char* pkey);
int32_t initAggSup(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize,
const char* pkey);
void cleanupAggSup(SAggSupporter* pAggSup);
void initResultSizeInfo(SResultInfo* pResultInfo, int32_t numOfRows);
void doBuildStreamResBlock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo,
@ -735,12 +691,12 @@ void doBuildStreamResBlock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGr
void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo,
SDiskbasedBuf* pBuf);
bool hasLimitOffsetInfo(SLimitInfo* pLimitInfo);
void initLimitInfo(const SNode* pLimit, const SNode* pSLimit, SLimitInfo* pLimitInfo);
bool hasLimitOffsetInfo(SLimitInfo* pLimitInfo);
void initLimitInfo(const SNode* pLimit, const SNode* pSLimit, SLimitInfo* pLimitInfo);
void applyLimitOffset(SLimitInfo* pLimitInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo, SOperatorInfo* pOperator);
void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, SColumnInfoData* pTimeWindowData, int32_t offset,
int32_t forwardStep, int32_t numOfTotal, int32_t numOfOutput);
void applyAggFunctionOnPartialTuples(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, SColumnInfoData* pTimeWindowData,
int32_t offset, int32_t forwardStep, int32_t numOfTotal, int32_t numOfOutput);
int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, SArray* pColList, char** pNextStart);
void updateLoadRemoteInfo(SLoadRemoteDataInfo* pInfo, int32_t numOfRows, int32_t dataLen, int64_t startTs,
@ -751,130 +707,108 @@ STimeWindow getFirstQualifiedTimeWindow(int64_t ts, STimeWindow* pWindow, SInter
int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scanFlag);
int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaultBufsz);
void doDestroyExchangeOperatorInfo(void* param);
extern void doDestroyExchangeOperatorInfo(void* param);
void setOperatorCompleted(SOperatorInfo* pOperator);
void setOperatorInfo(SOperatorInfo* pOperator, const char* name, int32_t type, bool blocking, int32_t status,
void* pInfo, SExecTaskInfo* pTaskInfo);
void doFilter(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SColMatchInfo* pColMatchInfo);
int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int32_t numOfExpr, SSDataBlock* pBlock,
int32_t rows, const char* idStr, STableMetaCacheInfo* pCache);
void cleanupAggSup(SAggSupporter* pAggSup);
void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle);
void setTbNameColData(const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId, const char* name);
SSDataBlock* loadNextDataBlock(void* param);
void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset);
SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pResultRowInfo, char* pData,
int16_t bytes, bool masterscan, uint64_t groupId, SExecTaskInfo* pTaskInfo,
bool isIntervalQuery, SAggSupporter* pSup);
// operator creater functions
// clang-format off
SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode* pExNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle,
SExecTaskInfo* pTaskInfo);
SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode,
STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode* pScanPhyNode,
const char* pUser, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode* pScanPhyNode, const char* pUser, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* pNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pNode,
SExecTaskInfo* pTaskInfo);
SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhysiNode* pProjPhyNode,
SExecTaskInfo* pTaskInfo);
SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhysiNode* pProjPhyNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* pSortNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** dowStreams, size_t numStreams,
SMergePhysiNode* pMergePhysiNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pTableScanNode, SReadHandle* readHandle,
SExecTaskInfo* pTaskInfo);
SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPhysiNode* pPhyNode,
SExecTaskInfo* pTaskInfo, bool isStream);
SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMergeIntervalPhysiNode* pIntervalPhyNode,
SExecTaskInfo* pTaskInfo);
SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, SMergeAlignedIntervalPhysiNode* pNode,
SExecTaskInfo* pTaskInfo);
SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
SExecTaskInfo* pTaskInfo, int32_t numOfChild);
SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionWinodwPhysiNode* pSessionNode,
SExecTaskInfo* pTaskInfo);
SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** dowStreams, size_t numStreams, SMergePhysiNode* pMergePhysiNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pTableScanNode, SReadHandle* readHandle, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SIntervalPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, bool isStream);
SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMergeIntervalPhysiNode* pIntervalPhyNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, SMergeAlignedIntervalPhysiNode* pNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild);
SOperatorInfo* createSessionAggOperatorInfo(SOperatorInfo* downstream, SSessionWinodwPhysiNode* pSessionNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode* pAggNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDistScanPhysiNode* pBlockScanNode,
SExecTaskInfo* pTaskInfo);
SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, SNode* pTagCond,
SExecTaskInfo* pTaskInfo);
SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDistScanPhysiNode* pBlockScanNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhysiNode* pTableScanNode, SNode* pTagCond, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createRawScanOperatorInfo(SReadHandle* pHandle, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* pPhyFillNode,
SExecTaskInfo* pTaskInfo);
SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWinodwPhysiNode* pStateNode,
SExecTaskInfo* pTaskInfo);
SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNode* pPartNode,
SExecTaskInfo* pTaskInfo);
SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* pPhyFillNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStreamPartitionPhysiNode* pPartNode,
SExecTaskInfo* pTaskInfo);
SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWinodwPhysiNode* pStateNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNode* pPartNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStreamPartitionPhysiNode* pPartNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream,
SSortMergeJoinPhysiNode* pJoinNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
SExecTaskInfo* pTaskInfo);
SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
SExecTaskInfo* pTaskInfo, int32_t numOfChild);
SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
SExecTaskInfo* pTaskInfo);
SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SSortMergeJoinPhysiNode* pJoinNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
SExecTaskInfo* pTaskInfo);
SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFillPhysiNode* pPhyFillNode,
SExecTaskInfo* pTaskInfo);
SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild);
SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFillPhysiNode* pPhyFillNode, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSortPhysiNode* pSortPhyNode, SExecTaskInfo* pTaskInfo);
// clang-format on
int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* pSrcBlock, SqlFunctionCtx* pCtx,
int32_t numOfOutput, SArray* pPseudoList);
void setInputDataBlock(SExprSupp* pExprSupp, SSDataBlock* pBlock, int32_t order, int32_t scanFlag, bool createDummyCol);
bool isTaskKilled(SExecTaskInfo* pTaskInfo);
int32_t checkForQueryBuf(size_t numOfTables);
bool isTaskKilled(SExecTaskInfo* pTaskInfo);
void setTaskKilled(SExecTaskInfo* pTaskInfo);
void queryCostStatis(SExecTaskInfo* pTaskInfo);
void doDestroyTask(SExecTaskInfo* pTaskInfo);
void destroyOperatorInfo(SOperatorInfo* pOperator);
int32_t getMaximumIdleDurationSec();
/*
* ops: root operator
* data: *data save the result of encode, need to be freed by caller
* length: *length save the length of *data
* nOptrWithVal: *nOptrWithVal save the number of optr with value
* return: result code, 0 means success
*/
int32_t encodeOperator(SOperatorInfo* ops, char** data, int32_t* length, int32_t* nOptrWithVal);
/*
* ops: root operator, created by caller
* data: save the result of decode
* length: the length of data
* return: result code, 0 means success
*/
int32_t decodeOperator(SOperatorInfo* ops, const char* data, int32_t length);
void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status);
int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId,
char* sql, EOPTR_EXEC_MODEL model);
int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, qTaskInfo_t* pTaskInfo, SReadHandle* readHandle);
int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SArray* pExecInfoList);
void printTaskExecCostInLog(SExecTaskInfo* pTaskInfo);
int32_t getMaximumIdleDurationSec();
STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, SInterval* pInterval,
int32_t order);
int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos, TSKEY ekey,
@ -897,15 +831,7 @@ void calBlockTbName(SStreamScanInfo* pInfo, SSDataBlock* pBlock);
int32_t finalizeResultRows(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPosition, SExprSupp* pSup,
SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo);
SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSortPhysiNode* pSortPhyNode,
SExecTaskInfo* pTaskInfo);
SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle,
SExecTaskInfo* pTaskInfo);
void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex);
bool groupbyTbname(SNodeList* pGroupList);
void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput);
int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, SStreamState* pState, SSDataBlock* pBlock, SExprSupp* pSup,
SGroupResInfo* pGroupResInfo);
int32_t saveSessionDiscBuf(SStreamState* pState, SSessionKey* key, void* buf, int32_t size);

10
source/libs/executor/src/cachescanoperator.c
View File

@ -59,7 +59,7 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe
pInfo->readHandle = *readHandle;
SDataBlockDescNode* pDescNode = pScanNode->scan.node.pOutputDataBlockDesc;
pInfo->pRes = createResDataBlock(pDescNode);
pInfo->pRes = createDataBlockFromDescNode(pDescNode);
int32_t numOfCols = 0;
code =
@ -185,7 +185,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
}
}
pRes->info.uid = *(tb_uid_t*)taosArrayGet(pInfo->pUidList, pInfo->indexOfBufferedRes);
pRes->info.id.uid = *(tb_uid_t*)taosArrayGet(pInfo->pUidList, pInfo->indexOfBufferedRes);
pRes->info.rows = 1;
SExprSupp* pSup = &pInfo->pseudoExprSup;
@ -196,7 +196,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
return NULL;
}
pRes->info.groupId = getTableGroupId(pTableList, pRes->info.uid);
pRes->info.id.groupId = getTableGroupId(pTableList, pRes->info.id.uid);
pInfo->indexOfBufferedRes += 1;
return pRes;
} else {
@ -232,12 +232,12 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) {
SExprSupp* pSup = &pInfo->pseudoExprSup;
STableKeyInfo* pKeyInfo = &((STableKeyInfo*)pList)[0];
pInfo->pRes->info.groupId = pKeyInfo->groupId;
pInfo->pRes->info.id.groupId = pKeyInfo->groupId;
if (taosArrayGetSize(pInfo->pUidList) > 0) {
ASSERT((pInfo->retrieveType & CACHESCAN_RETRIEVE_LAST_ROW) == CACHESCAN_RETRIEVE_LAST_ROW);
pInfo->pRes->info.uid = *(tb_uid_t*)taosArrayGet(pInfo->pUidList, 0);
pInfo->pRes->info.id.uid = *(tb_uid_t*)taosArrayGet(pInfo->pUidList, 0);
code = addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pInfo->pRes, pInfo->pRes->info.rows,
GET_TASKID(pTaskInfo), NULL);
if (code != TSDB_CODE_SUCCESS) {

12
source/libs/executor/src/exchangeoperator.c
View File

@ -303,7 +303,7 @@ SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode
}
tsem_init(&pInfo->ready, 0, 0);
pInfo->pDummyBlock = createResDataBlock(pExNode->node.pOutputDataBlockDesc);
pInfo->pDummyBlock = createDataBlockFromDescNode(pExNode->node.pOutputDataBlockDesc);
pInfo->pResultBlockList = taosArrayInit(64, POINTER_BYTES);
pInfo->pRecycledBlocks = taosArrayInit(64, POINTER_BYTES);
@ -717,10 +717,10 @@ int32_t prepareLoadRemoteData(SOperatorInfo* pOperator) {
int32_t handleLimitOffset(SOperatorInfo* pOperator, SLimitInfo* pLimitInfo, SSDataBlock* pBlock, bool holdDataInBuf) {
if (pLimitInfo->remainGroupOffset > 0) {
if (pLimitInfo->currentGroupId == 0) { // it is the first group
pLimitInfo->currentGroupId = pBlock->info.groupId;
pLimitInfo->currentGroupId = pBlock->info.id.groupId;
blockDataCleanup(pBlock);
return PROJECT_RETRIEVE_CONTINUE;
} else if (pLimitInfo->currentGroupId != pBlock->info.groupId) {
} else if (pLimitInfo->currentGroupId != pBlock->info.id.groupId) {
// now it is the data from a new group
pLimitInfo->remainGroupOffset -= 1;
@ -732,11 +732,11 @@ int32_t handleLimitOffset(SOperatorInfo* pOperator, SLimitInfo* pLimitInfo, SSDa
}
// set current group id of the project operator
pLimitInfo->currentGroupId = pBlock->info.groupId;
pLimitInfo->currentGroupId = pBlock->info.id.groupId;
}
// here check for a new group data, we need to handle the data of the previous group.
if (pLimitInfo->currentGroupId != 0 && pLimitInfo->currentGroupId != pBlock->info.groupId) {
if (pLimitInfo->currentGroupId != 0 && pLimitInfo->currentGroupId != pBlock->info.id.groupId) {
pLimitInfo->numOfOutputGroups += 1;
if ((pLimitInfo->slimit.limit > 0) && (pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups)) {
pOperator->status = OP_EXEC_DONE;
@ -758,7 +758,7 @@ int32_t handleLimitOffset(SOperatorInfo* pOperator, SLimitInfo* pLimitInfo, SSDa
// here we reach the start position, according to the limit/offset requirements.
// set current group id
pLimitInfo->currentGroupId = pBlock->info.groupId;
pLimitInfo->currentGroupId = pBlock->info.id.groupId;
if (pLimitInfo->remainOffset >= pBlock->info.rows) {
pLimitInfo->remainOffset -= pBlock->info.rows;

4
source/libs/executor/src/executil.c
View File

@ -208,12 +208,12 @@ SArray* createSortInfo(SNodeList* pNodeList) {
return pList;
}
SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode) {
SSDataBlock* createDataBlockFromDescNode(SDataBlockDescNode* pNode) {
int32_t numOfCols = LIST_LENGTH(pNode->pSlots);
SSDataBlock* pBlock = createDataBlock();
pBlock->info.blockId = pNode->dataBlockId;
pBlock->info.id.blockId = pNode->dataBlockId;
pBlock->info.type = STREAM_INVALID;
pBlock->info.calWin = (STimeWindow){.skey = INT64_MIN, .ekey = INT64_MAX};
pBlock->info.watermark = INT64_MIN;

17
source/libs/executor/src/executor.c
View File

@ -712,7 +712,7 @@ void qDestroyTask(qTaskInfo_t qTaskHandle) {
qDebug("%s execTask completed, numOfRows:%" PRId64, GET_TASKID(pTaskInfo), pTaskInfo->pRoot->resultInfo.totalRows);
queryCostStatis(pTaskInfo); // print the query cost summary
printTaskExecCostInLog(pTaskInfo); // print the query cost summary
doDestroyTask(pTaskInfo);
}
@ -728,12 +728,12 @@ int32_t qSerializeTaskStatus(qTaskInfo_t tinfo, char** pOutput, int32_t* len) {
}
int32_t nOptrWithVal = 0;
int32_t code = encodeOperator(pTaskInfo->pRoot, pOutput, len, &nOptrWithVal);
if ((code == TSDB_CODE_SUCCESS) && (nOptrWithVal == 0)) {
taosMemoryFreeClear(*pOutput);
*len = 0;
}
return code;
// int32_t code = encodeOperator(pTaskInfo->pRoot, pOutput, len, &nOptrWithVal);
// if ((code == TSDB_CODE_SUCCESS) && (nOptrWithVal == 0)) {
// taosMemoryFreeClear(*pOutput);
// *len = 0;
// }
return 0;
}
int32_t qDeserializeTaskStatus(qTaskInfo_t tinfo, const char* pInput, int32_t len) {
@ -743,7 +743,8 @@ int32_t qDeserializeTaskStatus(qTaskInfo_t tinfo, const char* pInput, int32_t le
return TSDB_CODE_INVALID_PARA;
}
return decodeOperator(pTaskInfo->pRoot, pInput, len);
return 0;
// return decodeOperator(pTaskInfo->pRoot, pInput, len);
}
int32_t qExtractStreamScanner(qTaskInfo_t tinfo, void** scanner) {

362
source/libs/executor/src/executorimpl.c
View File

@ -76,6 +76,15 @@ static UNUSED_FUNC void* u_realloc(void* p, size_t __size) {
#define CLEAR_QUERY_STATUS(q, st) ((q)->status &= (~(st)))
#define QUERY_IS_INTERVAL_QUERY(_q) ((_q)->interval.interval > 0)
typedef struct SAggOperatorInfo {
SOptrBasicInfo binfo;
SAggSupporter aggSup;
STableQueryInfo* current;
uint64_t groupId;
SGroupResInfo groupResInfo;
SExprSupp scalarExprSup;
} SAggOperatorInfo;
int32_t getMaximumIdleDurationSec() { return tsShellActivityTimer * 2; }
static void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExpr, SSDataBlock* pBlock);
@ -316,8 +325,8 @@ static void functionCtxRestore(SqlFunctionCtx* pCtx, SFunctionCtxStatus* pStatus
pCtx->input.startRowIndex = pStatus->startOffset;
}
void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, SColumnInfoData* pTimeWindowData, int32_t offset,
int32_t forwardStep, int32_t numOfTotal, int32_t numOfOutput) {
void applyAggFunctionOnPartialTuples(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, SColumnInfoData* pTimeWindowData,
int32_t offset, int32_t forwardStep, int32_t numOfTotal, int32_t numOfOutput) {
for (int32_t k = 0; k < numOfOutput; ++k) {
// keep it temporarily
SFunctionCtxStatus status = {0};
@ -438,7 +447,7 @@ static int32_t doSetInputDataBlock(SExprSupp* pExprSup, SSDataBlock* pBlock, int
pCtx[i].scanFlag = scanFlag;
SInputColumnInfoData* pInput = &pCtx[i].input;
pInput->uid = pBlock->info.uid;
pInput->uid = pBlock->info.id.uid;
pInput->colDataSMAIsSet = false;
SExprInfo* pOneExpr = &pExprSup->pExprInfo[i];
@ -497,184 +506,6 @@ static int32_t doAggregateImpl(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx) {
return TSDB_CODE_SUCCESS;
}
static void setPseudoOutputColInfo(SSDataBlock* pResult, SqlFunctionCtx* pCtx, SArray* pPseudoList) {
size_t num = (pPseudoList != NULL) ? taosArrayGetSize(pPseudoList) : 0;
for (int32_t i = 0; i < num; ++i) {
pCtx[i].pOutput = taosArrayGet(pResult->pDataBlock, i);
}
}
int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* pSrcBlock, SqlFunctionCtx* pCtx,
int32_t numOfOutput, SArray* pPseudoList) {
setPseudoOutputColInfo(pResult, pCtx, pPseudoList);
if (pSrcBlock == NULL) {
for (int32_t k = 0; k < numOfOutput; ++k) {
int32_t outputSlotId = pExpr[k].base.resSchema.slotId;
ASSERT(pExpr[k].pExpr->nodeType == QUERY_NODE_VALUE);
SColumnInfoData* pColInfoData = taosArrayGet(pResult->pDataBlock, outputSlotId);
int32_t type = pExpr[k].base.pParam[0].param.nType;
if (TSDB_DATA_TYPE_NULL == type) {
colDataAppendNNULL(pColInfoData, 0, 1);
} else {
colDataAppend(pColInfoData, 0, taosVariantGet(&pExpr[k].base.pParam[0].param, type), false);
}
}
pResult->info.rows = 1;
return TSDB_CODE_SUCCESS;
}
if (pResult != pSrcBlock) {
pResult->info.groupId = pSrcBlock->info.groupId;
memcpy(pResult->info.parTbName, pSrcBlock->info.parTbName, TSDB_TABLE_NAME_LEN);
}
// if the source equals to the destination, it is to create a new column as the result of scalar
// function or some operators.
bool createNewColModel = (pResult == pSrcBlock);
if (createNewColModel) {
blockDataEnsureCapacity(pResult, pResult->info.rows);
}
int32_t numOfRows = 0;
for (int32_t k = 0; k < numOfOutput; ++k) {
int32_t outputSlotId = pExpr[k].base.resSchema.slotId;
SqlFunctionCtx* pfCtx = &pCtx[k];
SInputColumnInfoData* pInputData = &pfCtx->input;
if (pExpr[k].pExpr->nodeType == QUERY_NODE_COLUMN) { // it is a project query
SColumnInfoData* pColInfoData = taosArrayGet(pResult->pDataBlock, outputSlotId);
if (pResult->info.rows > 0 && !createNewColModel) {
colDataMergeCol(pColInfoData, pResult->info.rows, (int32_t*)&pResult->info.capacity, pInputData->pData[0],
pInputData->numOfRows);
} else {
colDataAssign(pColInfoData, pInputData->pData[0], pInputData->numOfRows, &pResult->info);
}
numOfRows = pInputData->numOfRows;
} else if (pExpr[k].pExpr->nodeType == QUERY_NODE_VALUE) {
SColumnInfoData* pColInfoData = taosArrayGet(pResult->pDataBlock, outputSlotId);
int32_t offset = createNewColModel ? 0 : pResult->info.rows;
int32_t type = pExpr[k].base.pParam[0].param.nType;
if (TSDB_DATA_TYPE_NULL == type) {
colDataAppendNNULL(pColInfoData, offset, pSrcBlock->info.rows);
} else {
for (int32_t i = 0; i < pSrcBlock->info.rows; ++i) {
colDataAppend(pColInfoData, i + offset, taosVariantGet(&pExpr[k].base.pParam[0].param, type), false);
}
}
numOfRows = pSrcBlock->info.rows;
} else if (pExpr[k].pExpr->nodeType == QUERY_NODE_OPERATOR) {
SArray* pBlockList = taosArrayInit(4, POINTER_BYTES);
taosArrayPush(pBlockList, &pSrcBlock);
SColumnInfoData* pResColData = taosArrayGet(pResult->pDataBlock, outputSlotId);
SColumnInfoData idata = {.info = pResColData->info, .hasNull = true};
SScalarParam dest = {.columnData = &idata};
int32_t code = scalarCalculate(pExpr[k].pExpr->_optrRoot.pRootNode, pBlockList, &dest);
if (code != TSDB_CODE_SUCCESS) {
taosArrayDestroy(pBlockList);
return code;
}
int32_t startOffset = createNewColModel ? 0 : pResult->info.rows;
ASSERT(pResult->info.capacity > 0);
colDataMergeCol(pResColData, startOffset, (int32_t*)&pResult->info.capacity, &idata, dest.numOfRows);
colDataDestroy(&idata);
numOfRows = dest.numOfRows;
taosArrayDestroy(pBlockList);
} else if (pExpr[k].pExpr->nodeType == QUERY_NODE_FUNCTION) {
// _rowts/_c0, not tbname column
if (fmIsPseudoColumnFunc(pfCtx->functionId) && (!fmIsScanPseudoColumnFunc(pfCtx->functionId))) {
// do nothing
} else if (fmIsIndefiniteRowsFunc(pfCtx->functionId)) {
SResultRowEntryInfo* pResInfo = GET_RES_INFO(pfCtx);
pfCtx->fpSet.init(pfCtx, pResInfo);
pfCtx->pOutput = taosArrayGet(pResult->pDataBlock, outputSlotId);
pfCtx->offset = createNewColModel ? 0 : pResult->info.rows; // set the start offset
// set the timestamp(_rowts) output buffer
if (taosArrayGetSize(pPseudoList) > 0) {
int32_t* outputColIndex = taosArrayGet(pPseudoList, 0);
pfCtx->pTsOutput = (SColumnInfoData*)pCtx[*outputColIndex].pOutput;
}
// link pDstBlock to set selectivity value
if (pfCtx->subsidiaries.num > 0) {
pfCtx->pDstBlock = pResult;
}
int32_t code = pfCtx->fpSet.process(pfCtx);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
numOfRows = pResInfo->numOfRes;
} else if (fmIsAggFunc(pfCtx->functionId)) {
// selective value output should be set during corresponding function execution
if (fmIsSelectValueFunc(pfCtx->functionId)) {
continue;
}
// _group_key function for "partition by tbname" + csum(col_name) query
SColumnInfoData* pOutput = taosArrayGet(pResult->pDataBlock, outputSlotId);
int32_t slotId = pfCtx->param[0].pCol->slotId;
// todo handle the json tag
SColumnInfoData* pInput = taosArrayGet(pSrcBlock->pDataBlock, slotId);
for (int32_t f = 0; f < pSrcBlock->info.rows; ++f) {
bool isNull = colDataIsNull_s(pInput, f);
if (isNull) {
colDataAppendNULL(pOutput, pResult->info.rows + f);
} else {
char* data = colDataGetData(pInput, f);
colDataAppend(pOutput, pResult->info.rows + f, data, isNull);
}
}
} else {
SArray* pBlockList = taosArrayInit(4, POINTER_BYTES);
taosArrayPush(pBlockList, &pSrcBlock);
SColumnInfoData* pResColData = taosArrayGet(pResult->pDataBlock, outputSlotId);
SColumnInfoData idata = {.info = pResColData->info, .hasNull = true};
SScalarParam dest = {.columnData = &idata};
int32_t code = scalarCalculate((SNode*)pExpr[k].pExpr->_function.pFunctNode, pBlockList, &dest);
if (code != TSDB_CODE_SUCCESS) {
taosArrayDestroy(pBlockList);
return code;
}
int32_t startOffset = createNewColModel ? 0 : pResult->info.rows;
ASSERT(pResult->info.capacity > 0);
colDataMergeCol(pResColData, startOffset, (int32_t*)&pResult->info.capacity, &idata, dest.numOfRows);
colDataDestroy(&idata);
numOfRows = dest.numOfRows;
taosArrayDestroy(pBlockList);
}
} else {
return TSDB_CODE_OPS_NOT_SUPPORT;
}
}
if (!createNewColModel) {
pResult->info.rows += numOfRows;
}
return TSDB_CODE_SUCCESS;
}
bool functionNeedToExecute(SqlFunctionCtx* pCtx) {
struct SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
@ -853,7 +684,7 @@ int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableSc
pTableScanInfo->pCtx, pTableScanInfo->numOfOutput,
pTableScanInfo->rowEntryInfoOffset);
} else {
if (setResultOutputBufByKey(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pBlock->info.uid, &win, masterScan, &pResult, groupId,
if (setResultOutputBufByKey(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pBlock->info.id.uid, &win, masterScan, &pResult, groupId,
pTableScanInfo->pCtx, pTableScanInfo->numOfOutput,
pTableScanInfo->rowEntryInfoOffset) != TSDB_CODE_SUCCESS) {
T_LONG_JMP(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
@ -904,7 +735,7 @@ int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableSc
TSKEY k = ascQuery? pBlock->info.window.skey : pBlock->info.window.ekey;
STimeWindow win = getActiveTimeWindow(pTableScanInfo->pResultRowInfo, k, pQueryAttr);
if (setResultOutputBufByKey(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pBlock->info.uid, &win, masterScan, &pResult, groupId,
if (setResultOutputBufByKey(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pBlock->info.id.uid, &win, masterScan, &pResult, groupId,
pTableScanInfo->pCtx, pTableScanInfo->numOfOutput,
pTableScanInfo->rowEntryInfoOffset) != TSDB_CODE_SUCCESS) {
T_LONG_JMP(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
@ -1236,11 +1067,11 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprS
continue;
}
if (pBlock->info.groupId == 0) {
pBlock->info.groupId = pPos->groupId;
if (pBlock->info.id.groupId == 0) {
pBlock->info.id.groupId = pPos->groupId;
} else {
// current value belongs to different group, it can't be packed into one datablock
if (pBlock->info.groupId != pPos->groupId) {
if (pBlock->info.id.groupId != pPos->groupId) {
releaseBufPage(pBuf, page);
break;
}
@ -1260,7 +1091,7 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprS
}
qDebug("%s result generated, rows:%d, groupId:%" PRIu64, GET_TASKID(pTaskInfo), pBlock->info.rows,
pBlock->info.groupId);
pBlock->info.id.groupId);
blockDataUpdateTsWindow(pBlock, 0);
return 0;
@ -1280,12 +1111,12 @@ void doBuildStreamResBlock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGr
}
// clear the existed group id
pBlock->info.groupId = 0;
pBlock->info.id.groupId = 0;
ASSERT(!pbInfo->mergeResultBlock);
doCopyToSDataBlock(pTaskInfo, pBlock, &pOperator->exprSupp, pBuf, pGroupResInfo);
void* tbname = NULL;
if (streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.groupId, &tbname) < 0) {
if (streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname) < 0) {
pBlock->info.parTbName[0] = 0;
} else {
memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN);
@ -1307,7 +1138,7 @@ void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SG
}
// clear the existed group id
pBlock->info.groupId = 0;
pBlock->info.id.groupId = 0;
if (!pbInfo->mergeResultBlock) {
doCopyToSDataBlock(pTaskInfo, pBlock, &pOperator->exprSupp, pBuf, pGroupResInfo);
} else {
@ -1318,15 +1149,15 @@ void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SG
}
// clearing group id to continue to merge data that belong to different groups
pBlock->info.groupId = 0;
pBlock->info.id.groupId = 0;
}
// clear the group id info in SSDataBlock, since the client does not need it
pBlock->info.groupId = 0;
pBlock->info.id.groupId = 0;
}
}
void queryCostStatis(SExecTaskInfo* pTaskInfo) {
void printTaskExecCostInLog(SExecTaskInfo* pTaskInfo) {
STaskCostInfo* pSummary = &pTaskInfo->cost;
SFileBlockLoadRecorder* pRecorder = pSummary->pRecoder;
@ -1706,7 +1537,7 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) {
}
// the pDataBlock are always the same one, no need to call this again
setExecutionContext(pOperator, pOperator->exprSupp.numOfExprs, pBlock->info.groupId);
setExecutionContext(pOperator, pOperator->exprSupp.numOfExprs, pBlock->info.id.groupId);
setInputDataBlock(pSup, pBlock, order, scanFlag, true);
code = doAggregateImpl(pOperator, pSup->pCtx);
if (code != 0) {
@ -1787,7 +1618,7 @@ static void doHandleRemainBlockForNewGroupImpl(SOperatorInfo* pOperator, SFillOp
int32_t numOfResultRows = pResultInfo->capacity - pResBlock->info.rows;
taosFillResultDataBlock(pInfo->pFillInfo, pResBlock, numOfResultRows);
pInfo->curGroupId = pInfo->existNewGroupBlock->info.groupId;
pInfo->curGroupId = pInfo->existNewGroupBlock->info.id.groupId;
pInfo->existNewGroupBlock = NULL;
}
@ -1796,7 +1627,7 @@ static void doHandleRemainBlockFromNewGroup(SOperatorInfo* pOperator, SFillOpera
if (taosFillHasMoreResults(pInfo->pFillInfo)) {
int32_t numOfResultRows = pResultInfo->capacity - pInfo->pFinalRes->info.rows;
taosFillResultDataBlock(pInfo->pFillInfo, pInfo->pFinalRes, numOfResultRows);
pInfo->pRes->info.groupId = pInfo->curGroupId;
pInfo->pRes->info.id.groupId = pInfo->curGroupId;
return;
}
@ -1818,7 +1649,7 @@ static void doApplyScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pBlo
setInputDataBlock(pNoFillSupp, pBlock, order, scanFlag, false);
projectApplyFunctions(pNoFillSupp->pExprInfo, pInfo->pRes, pBlock, pNoFillSupp->pCtx, pNoFillSupp->numOfExprs, NULL);
pInfo->pRes->info.groupId = pBlock->info.groupId;
pInfo->pRes->info.id.groupId = pBlock->info.id.groupId;
}
static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) {
@ -1836,7 +1667,7 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) {
doHandleRemainBlockFromNewGroup(pOperator, pInfo, pResultInfo, pTaskInfo);
if (pResBlock->info.rows > 0) {
pResBlock->info.groupId = pInfo->curGroupId;
pResBlock->info.id.groupId = pInfo->curGroupId;
return pResBlock;
}
@ -1858,8 +1689,8 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) {
blockDataEnsureCapacity(pInfo->pFinalRes, pBlock->info.rows);
doApplyScalarCalculation(pOperator, pBlock, order, scanFlag);
if (pInfo->curGroupId == 0 || pInfo->curGroupId == pInfo->pRes->info.groupId) {
pInfo->curGroupId = pInfo->pRes->info.groupId; // the first data block
if (pInfo->curGroupId == 0 || pInfo->curGroupId == pInfo->pRes->info.id.groupId) {
pInfo->curGroupId = pInfo->pRes->info.id.groupId; // the first data block
pInfo->totalInputRows += pInfo->pRes->info.rows;
if (order == pInfo->pFillInfo->order) {
@ -1868,7 +1699,7 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) {
taosFillSetStartInfo(pInfo->pFillInfo, pInfo->pRes->info.rows, pBlock->info.window.skey);
}
taosFillSetInputDataBlock(pInfo->pFillInfo, pInfo->pRes);
} else if (pInfo->curGroupId != pBlock->info.groupId) { // the new group data block
} else if (pInfo->curGroupId != pBlock->info.id.groupId) { // the new group data block
pInfo->existNewGroupBlock = pBlock;
// Fill the previous group data block, before handle the data block of new group.
@ -1885,13 +1716,13 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) {
// 1. The result in current group not reach the threshold of output result, continue
// 2. If multiple group results existing in one SSDataBlock is not allowed, return immediately
if (pResBlock->info.rows > pResultInfo->threshold || pBlock == NULL || pInfo->existNewGroupBlock != NULL) {
pResBlock->info.groupId = pInfo->curGroupId;
pResBlock->info.id.groupId = pInfo->curGroupId;
return pResBlock;
}
doHandleRemainBlockFromNewGroup(pOperator, pInfo, pResultInfo, pTaskInfo);
if (pResBlock->info.rows >= pOperator->resultInfo.threshold || pBlock == NULL) {
pResBlock->info.groupId = pInfo->curGroupId;
pResBlock->info.id.groupId = pInfo->curGroupId;
return pResBlock;
}
} else if (pInfo->existNewGroupBlock) { // try next group
@ -1901,7 +1732,7 @@ static SSDataBlock* doFillImpl(SOperatorInfo* pOperator) {
doHandleRemainBlockForNewGroupImpl(pOperator, pInfo, pResultInfo, pTaskInfo);
if (pResBlock->info.rows > pResultInfo->threshold) {
pResBlock->info.groupId = pInfo->curGroupId;
pResBlock->info.id.groupId = pInfo->curGroupId;
return pResBlock;
}
} else {
@ -2033,7 +1864,7 @@ void cleanupAggSup(SAggSupporter* pAggSup) {
destroyDiskbasedBuf(pAggSup->pResultBuf);
}
int32_t initAggInfo(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize,
int32_t initAggSup(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize,
const char* pkey) {
int32_t code = initExprSupp(pSup, pExprInfo, numOfCols);
if (code != TSDB_CODE_SUCCESS) {
@ -2123,7 +1954,7 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiN
goto _error;
}
SSDataBlock* pResBlock = createResDataBlock(pAggNode->node.pOutputDataBlockDesc);
SSDataBlock* pResBlock = createDataBlockFromDescNode(pAggNode->node.pOutputDataBlockDesc);
initBasicInfo(&pInfo->binfo, pResBlock);
size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
@ -2131,7 +1962,7 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiN
int32_t num = 0;
SExprInfo* pExprInfo = createExprInfo(pAggNode->pAggFuncs, pAggNode->pGroupKeys, &num);
int32_t code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str);
int32_t code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@ -2297,7 +2128,7 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode*
goto _error;
}
pInfo->pRes = createResDataBlock(pPhyFillNode->node.pOutputDataBlockDesc);
pInfo->pRes = createDataBlockFromDescNode(pPhyFillNode->node.pOutputDataBlockDesc);
SExprInfo* pExprInfo = createExprInfo(pPhyFillNode->pFillExprs, NULL, &pInfo->numOfExpr);
pOperator->exprSupp.pExprInfo = pExprInfo;
@ -2596,7 +2427,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
return NULL;
}
pOperator = createTagScanOperatorInfo(pHandle, pScanPhyNode, pTableListInfo, pTaskInfo);
pOperator = createTagScanOperatorInfo(pHandle, pScanPhyNode, pTaskInfo);
} else if (QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN == type) {
SBlockDistScanPhysiNode* pBlockNode = (SBlockDistScanPhysiNode*)pPhyNode;
@ -2809,103 +2640,6 @@ int32_t rebuildReader(SOperatorInfo* pOperator, SSubplan* plan, SReadHandle* pHa
}
#endif
int32_t encodeOperator(SOperatorInfo* ops, char** result, int32_t* length, int32_t* nOptrWithVal) {
int32_t code = TDB_CODE_SUCCESS;
char* pCurrent = NULL;
int32_t currLength = 0;
if (ops->fpSet.encodeResultRow) {
if (result == NULL || length == NULL || nOptrWithVal == NULL) {
return TSDB_CODE_TSC_INVALID_INPUT;
}
code = ops->fpSet.encodeResultRow(ops, &pCurrent, &currLength);
if (code != TDB_CODE_SUCCESS) {
if (*result != NULL) {
taosMemoryFree(*result);
*result = NULL;
}
return code;
} else if (currLength == 0) {
ASSERT(!pCurrent);
goto _downstream;
}
++(*nOptrWithVal);
ASSERT(currLength >= 0);
if (*result == NULL) {
*result = (char*)taosMemoryCalloc(1, currLength + sizeof(int32_t));
if (*result == NULL) {
taosMemoryFree(pCurrent);
return TSDB_CODE_OUT_OF_MEMORY;
}
memcpy(*result + sizeof(int32_t), pCurrent, currLength);
*(int32_t*)(*result) = currLength + sizeof(int32_t);
} else {
int32_t sizePre = *(int32_t*)(*result);
char* tmp = (char*)taosMemoryRealloc(*result, sizePre + currLength);
if (tmp == NULL) {
taosMemoryFree(pCurrent);
taosMemoryFree(*result);
*result = NULL;
return TSDB_CODE_OUT_OF_MEMORY;
}
*result = tmp;
memcpy(*result + sizePre, pCurrent, currLength);
*(int32_t*)(*result) += currLength;
}
taosMemoryFree(pCurrent);
*length = *(int32_t*)(*result);
}
_downstream:
for (int32_t i = 0; i < ops->numOfDownstream; ++i) {
code = encodeOperator(ops->pDownstream[i], result, length, nOptrWithVal);
if (code != TDB_CODE_SUCCESS) {
return code;
}
}
return TDB_CODE_SUCCESS;
}
int32_t decodeOperator(SOperatorInfo* ops, const char* result, int32_t length) {
int32_t code = TDB_CODE_SUCCESS;
if (ops->fpSet.decodeResultRow) {
if (result == NULL) {
return TSDB_CODE_TSC_INVALID_INPUT;
}
ASSERT(length == *(int32_t*)result);
const char* data = result + sizeof(int32_t);
code = ops->fpSet.decodeResultRow(ops, (char*)data);
if (code != TDB_CODE_SUCCESS) {
return code;
}
int32_t totalLength = *(int32_t*)result;
int32_t dataLength = *(int32_t*)data;
if (totalLength == dataLength + sizeof(int32_t)) { // the last data
result = NULL;
length = 0;
} else {
result += dataLength;
*(int32_t*)(result) = totalLength - dataLength;
length = totalLength - dataLength;
}
}
for (int32_t i = 0; i < ops->numOfDownstream; ++i) {
code = decodeOperator(ops->pDownstream[i], result, length);
if (code != TDB_CODE_SUCCESS) {
return code;
}
}
return TDB_CODE_SUCCESS;
}
int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, qTaskInfo_t* pTaskInfo, SReadHandle* readHandle) {
SExecTaskInfo* pTask = *(SExecTaskInfo**)pTaskInfo;
@ -3139,10 +2873,10 @@ int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, SStreamState* pStat
continue;
}
if (pBlock->info.groupId == 0) {
pBlock->info.groupId = pPos->groupId;
if (pBlock->info.id.groupId == 0) {
pBlock->info.id.groupId = pPos->groupId;
void* tbname = NULL;
if (streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.groupId, &tbname) < 0) {
if (streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname) < 0) {
pBlock->info.parTbName[0] = 0;
} else {
memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN);
@ -3150,7 +2884,7 @@ int32_t buildDataBlockFromGroupRes(SOperatorInfo* pOperator, SStreamState* pStat
tdbFree(tbname);
} else {
// current value belongs to different group, it can't be packed into one datablock
if (pBlock->info.groupId != pPos->groupId) {
if (pBlock->info.id.groupId != pPos->groupId) {
releaseOutputBuf(pState, &key, pRow);
break;
}
@ -3230,11 +2964,11 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, SStreamState* pSta
continue;
}
if (pBlock->info.groupId == 0) {
pBlock->info.groupId = pKey->groupId;
if (pBlock->info.id.groupId == 0) {
pBlock->info.id.groupId = pKey->groupId;
void* tbname = NULL;
if (streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.groupId, &tbname) < 0) {
if (streamStateGetParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname) < 0) {
pBlock->info.parTbName[0] = 0;
} else {
memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN);
@ -3242,7 +2976,7 @@ int32_t buildSessionResultDataBlock(SOperatorInfo* pOperator, SStreamState* pSta
tdbFree(tbname);
} else {
// current value belongs to different group, it can't be packed into one datablock
if (pBlock->info.groupId != pKey->groupId) {
if (pBlock->info.id.groupId != pKey->groupId) {
releaseOutputBuf(pState, NULL, pRow);
break;
}

20
source/libs/executor/src/groupoperator.c
View File

@ -308,13 +308,13 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) {
len = buildGroupKeys(pInfo->keyBuf, pInfo->pGroupColVals);
int32_t ret = setGroupResultOutputBuf(pOperator, &(pInfo->binfo), pOperator->exprSupp.numOfExprs, pInfo->keyBuf,
len, pBlock->info.groupId, pInfo->aggSup.pResultBuf, &pInfo->aggSup);
len, pBlock->info.id.groupId, pInfo->aggSup.pResultBuf, &pInfo->aggSup);
if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code
T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR);
}
int32_t rowIndex = j - num;
doApplyFunctions(pTaskInfo, pCtx, NULL, rowIndex, num, pBlock->info.rows, pOperator->exprSupp.numOfExprs);
applyAggFunctionOnPartialTuples(pTaskInfo, pCtx, NULL, rowIndex, num, pBlock->info.rows, pOperator->exprSupp.numOfExprs);
// assign the group keys or user input constant values if required
doAssignGroupKeys(pCtx, pOperator->exprSupp.numOfExprs, pBlock->info.rows, rowIndex);
@ -325,13 +325,13 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) {
if (num > 0) {
len = buildGroupKeys(pInfo->keyBuf, pInfo->pGroupColVals);
int32_t ret = setGroupResultOutputBuf(pOperator, &(pInfo->binfo), pOperator->exprSupp.numOfExprs, pInfo->keyBuf,
len, pBlock->info.groupId, pInfo->aggSup.pResultBuf, &pInfo->aggSup);
len, pBlock->info.id.groupId, pInfo->aggSup.pResultBuf, &pInfo->aggSup);
if (ret != TSDB_CODE_SUCCESS) {
T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR);
}
int32_t rowIndex = pBlock->info.rows - num;
doApplyFunctions(pTaskInfo, pCtx, NULL, rowIndex, num, pBlock->info.rows, pOperator->exprSupp.numOfExprs);
applyAggFunctionOnPartialTuples(pTaskInfo, pCtx, NULL, rowIndex, num, pBlock->info.rows, pOperator->exprSupp.numOfExprs);
doAssignGroupKeys(pCtx, pOperator->exprSupp.numOfExprs, pBlock->info.rows, rowIndex);
}
}
@ -431,7 +431,7 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode*
goto _error;
}
SSDataBlock* pResBlock = createResDataBlock(pAggNode->node.pOutputDataBlockDesc);
SSDataBlock* pResBlock = createDataBlockFromDescNode(pAggNode->node.pOutputDataBlockDesc);
initBasicInfo(&pInfo->binfo, pResBlock);
int32_t numOfScalarExpr = 0;
@ -456,7 +456,7 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SAggPhysiNode*
int32_t num = 0;
SExprInfo* pExprInfo = createExprInfo(pAggNode->pAggFuncs, pAggNode->pGroupKeys, &num);
code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, pInfo->groupKeyLen, pTaskInfo->id.str);
code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, pInfo->groupKeyLen, pTaskInfo->id.str);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@ -697,7 +697,7 @@ static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) {
releaseBufPage(pInfo->pBuf, page);
blockDataUpdateTsWindow(pInfo->binfo.pRes, 0);
pInfo->binfo.pRes->info.groupId = pGroupInfo->groupId;
pInfo->binfo.pRes->info.id.groupId = pGroupInfo->groupId;
pOperator->resultInfo.totalRows += pInfo->binfo.pRes->info.rows;
return pInfo->binfo.pRes;
@ -823,7 +823,7 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition
uint32_t defaultPgsz = 0;
uint32_t defaultBufsz = 0;
pInfo->binfo.pRes = createResDataBlock(pPartNode->node.pOutputDataBlockDesc);
pInfo->binfo.pRes = createDataBlockFromDescNode(pPartNode->node.pOutputDataBlockDesc);
getBufferPgSize(pInfo->binfo.pRes->info.rowSize, &defaultPgsz, &defaultBufsz);
if (!osTempSpaceAvailable()) {
@ -952,7 +952,7 @@ static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) {
taosArrayDestroy(pParInfo->rowIds);
pParInfo->rowIds = NULL;
blockDataUpdateTsWindow(pDest, pInfo->tsColIndex);
pDest->info.groupId = pParInfo->groupId;
pDest->info.id.groupId = pParInfo->groupId;
pOperator->resultInfo.totalRows += pDest->info.rows;
pInfo->parIte = taosHashIterate(pInfo->pPartitions, pInfo->parIte);
ASSERT(pDest->info.rows > 0);
@ -1119,7 +1119,7 @@ SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStr
}
pInfo->partitionSup.needCalc = true;
pInfo->binfo.pRes = createResDataBlock(pPartNode->part.node.pOutputDataBlockDesc);
pInfo->binfo.pRes = createDataBlockFromDescNode(pPartNode->part.node.pOutputDataBlockDesc);
if (pInfo->binfo.pRes == NULL) {
goto _error;
}

4
source/libs/executor/src/joinoperator.c
View File

@ -87,7 +87,7 @@ SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t
}
int32_t numOfCols = 0;
SSDataBlock* pResBlock = createResDataBlock(pJoinNode->node.pOutputDataBlockDesc);
SSDataBlock* pResBlock = createDataBlockFromDescNode(pJoinNode->node.pOutputDataBlockDesc);
SExprInfo* pExprInfo = createExprInfo(pJoinNode->pTargets, NULL, &numOfCols);
initResultSizeInfo(&pOperator->resultInfo, 4096);
@ -185,7 +185,7 @@ static void mergeJoinJoinLeftRight(struct SOperatorInfo* pOperator, SSDataBlock*
int32_t rowIndex = -1;
SColumnInfoData* pSrc = NULL;
if (pLeftBlock->info.blockId == blockId) {
if (pLeftBlock->info.id.blockId == blockId) {
pSrc = taosArrayGet(pLeftBlock->pDataBlock, slotId);
rowIndex = leftPos;
} else {

211
source/libs/executor/src/projectoperator.c
View File

@ -85,7 +85,7 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhys
int32_t numOfCols = 0;
SExprInfo* pExprInfo = createExprInfo(pProjPhyNode->pProjections, NULL, &numOfCols);
SSDataBlock* pResBlock = createResDataBlock(pProjPhyNode->node.pOutputDataBlockDesc);
SSDataBlock* pResBlock = createDataBlockFromDescNode(pProjPhyNode->node.pOutputDataBlockDesc);
initLimitInfo(pProjPhyNode->node.pLimit, pProjPhyNode->node.pSlimit, &pInfo->limitInfo);
pInfo->binfo.pRes = pResBlock;
@ -102,7 +102,7 @@ SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhys
}
initResultSizeInfo(&pOperator->resultInfo, numOfRows);
code = initAggInfo(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@ -138,13 +138,13 @@ _error:
static int32_t discardGroupDataBlock(SSDataBlock* pBlock, SLimitInfo* pLimitInfo) {
if (pLimitInfo->remainGroupOffset > 0) {
// it is the first group
if (pLimitInfo->currentGroupId == 0 || pLimitInfo->currentGroupId == pBlock->info.groupId) {
pLimitInfo->currentGroupId = pBlock->info.groupId;
if (pLimitInfo->currentGroupId == 0 || pLimitInfo->currentGroupId == pBlock->info.id.groupId) {
pLimitInfo->currentGroupId = pBlock->info.id.groupId;
return PROJECT_RETRIEVE_CONTINUE;
} else if (pLimitInfo->currentGroupId != pBlock->info.groupId) {
} else if (pLimitInfo->currentGroupId != pBlock->info.id.groupId) {
// now it is the data from a new group
pLimitInfo->remainGroupOffset -= 1;
pLimitInfo->currentGroupId = pBlock->info.groupId;
pLimitInfo->currentGroupId = pBlock->info.id.groupId;
// ignore data block in current group
if (pLimitInfo->remainGroupOffset > 0) {
@ -153,7 +153,7 @@ static int32_t discardGroupDataBlock(SSDataBlock* pBlock, SLimitInfo* pLimitInfo
}
// set current group id of the project operator
pLimitInfo->currentGroupId = pBlock->info.groupId;
pLimitInfo->currentGroupId = pBlock->info.id.groupId;
}
return PROJECT_RETRIEVE_DONE;
@ -164,7 +164,7 @@ static int32_t setInfoForNewGroup(SSDataBlock* pBlock, SLimitInfo* pLimitInfo, S
// here check for a new group data, we need to handle the data of the previous group.
ASSERT(pLimitInfo->remainGroupOffset == 0 || pLimitInfo->remainGroupOffset == -1);
if (pLimitInfo->currentGroupId != 0 && pLimitInfo->currentGroupId != pBlock->info.groupId) {
if (pLimitInfo->currentGroupId != 0 && pLimitInfo->currentGroupId != pBlock->info.id.groupId) {
pLimitInfo->numOfOutputGroups += 1;
if ((pLimitInfo->slimit.limit > 0) && (pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups)) {
setOperatorCompleted(pOperator);
@ -306,7 +306,7 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
T_LONG_JMP(pTaskInfo->env, code);
}
status = doIngroupLimitOffset(pLimitInfo, pBlock->info.groupId, pInfo->pRes, pOperator);
status = doIngroupLimitOffset(pLimitInfo, pBlock->info.id.groupId, pInfo->pRes, pOperator);
if (status == PROJECT_RETRIEVE_CONTINUE) {
continue;
}
@ -316,7 +316,7 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
if (pProjectInfo->mergeDataBlocks) {
if (pRes->info.rows > 0) {
pFinalRes->info.groupId = pRes->info.groupId;
pFinalRes->info.id.groupId = pRes->info.id.groupId;
pFinalRes->info.version = pRes->info.version;
// continue merge data, ignore the group id
@ -385,7 +385,7 @@ SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhy
}
}
SSDataBlock* pResBlock = createResDataBlock(pPhyNode->node.pOutputDataBlockDesc);
SSDataBlock* pResBlock = createDataBlockFromDescNode(pPhyNode->node.pOutputDataBlockDesc);
int32_t numOfRows = 4096;
size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
@ -400,7 +400,7 @@ SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhy
initResultSizeInfo(&pOperator->resultInfo, numOfRows);
blockDataEnsureCapacity(pResBlock, numOfRows);
int32_t code = initAggInfo(pSup, &pInfo->aggSup, pExprInfo, numOfExpr, keyBufSize, pTaskInfo->id.str);
int32_t code = initAggSup(pSup, &pInfo->aggSup, pExprInfo, numOfExpr, keyBufSize, pTaskInfo->id.str);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
@ -511,11 +511,11 @@ SSDataBlock* doApplyIndefinitFunction(SOperatorInfo* pOperator) {
break;
}
if (pIndefInfo->groupId == 0 && pBlock->info.groupId != 0) {
pIndefInfo->groupId = pBlock->info.groupId; // this is the initial group result
if (pIndefInfo->groupId == 0 && pBlock->info.id.groupId != 0) {
pIndefInfo->groupId = pBlock->info.id.groupId; // this is the initial group result
} else {
if (pIndefInfo->groupId != pBlock->info.groupId) { // reset output buffer and computing status
pIndefInfo->groupId = pBlock->info.groupId;
if (pIndefInfo->groupId != pBlock->info.id.groupId) { // reset output buffer and computing status
pIndefInfo->groupId = pBlock->info.id.groupId;
pIndefInfo->pNextGroupRes = pBlock;
break;
}
@ -643,3 +643,182 @@ SSDataBlock* doGenerateSourceData(SOperatorInfo* pOperator) {
return (pRes->info.rows > 0) ? pRes : NULL;
}
static void setPseudoOutputColInfo(SSDataBlock* pResult, SqlFunctionCtx* pCtx, SArray* pPseudoList) {
size_t num = (pPseudoList != NULL) ? taosArrayGetSize(pPseudoList) : 0;
for (int32_t i = 0; i < num; ++i) {
pCtx[i].pOutput = taosArrayGet(pResult->pDataBlock, i);
}
}
int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* pSrcBlock, SqlFunctionCtx* pCtx,
int32_t numOfOutput, SArray* pPseudoList) {
setPseudoOutputColInfo(pResult, pCtx, pPseudoList);
if (pSrcBlock == NULL) {
for (int32_t k = 0; k < numOfOutput; ++k) {
int32_t outputSlotId = pExpr[k].base.resSchema.slotId;
ASSERT(pExpr[k].pExpr->nodeType == QUERY_NODE_VALUE);
SColumnInfoData* pColInfoData = taosArrayGet(pResult->pDataBlock, outputSlotId);
int32_t type = pExpr[k].base.pParam[0].param.nType;
if (TSDB_DATA_TYPE_NULL == type) {
colDataAppendNNULL(pColInfoData, 0, 1);
} else {
colDataAppend(pColInfoData, 0, taosVariantGet(&pExpr[k].base.pParam[0].param, type), false);
}
}
pResult->info.rows = 1;
return TSDB_CODE_SUCCESS;
}
if (pResult != pSrcBlock) {
pResult->info.id.groupId = pSrcBlock->info.id.groupId;
memcpy(pResult->info.parTbName, pSrcBlock->info.parTbName, TSDB_TABLE_NAME_LEN);
}
// if the source equals to the destination, it is to create a new column as the result of scalar
// function or some operators.
bool createNewColModel = (pResult == pSrcBlock);
if (createNewColModel) {
blockDataEnsureCapacity(pResult, pResult->info.rows);
}
int32_t numOfRows = 0;
for (int32_t k = 0; k < numOfOutput; ++k) {
int32_t outputSlotId = pExpr[k].base.resSchema.slotId;
SqlFunctionCtx* pfCtx = &pCtx[k];
SInputColumnInfoData* pInputData = &pfCtx->input;
if (pExpr[k].pExpr->nodeType == QUERY_NODE_COLUMN) { // it is a project query
SColumnInfoData* pColInfoData = taosArrayGet(pResult->pDataBlock, outputSlotId);
if (pResult->info.rows > 0 && !createNewColModel) {
colDataMergeCol(pColInfoData, pResult->info.rows, (int32_t*)&pResult->info.capacity, pInputData->pData[0],
pInputData->numOfRows);
} else {
colDataAssign(pColInfoData, pInputData->pData[0], pInputData->numOfRows, &pResult->info);
}
numOfRows = pInputData->numOfRows;
} else if (pExpr[k].pExpr->nodeType == QUERY_NODE_VALUE) {
SColumnInfoData* pColInfoData = taosArrayGet(pResult->pDataBlock, outputSlotId);
int32_t offset = createNewColModel ? 0 : pResult->info.rows;
int32_t type = pExpr[k].base.pParam[0].param.nType;
if (TSDB_DATA_TYPE_NULL == type) {
colDataAppendNNULL(pColInfoData, offset, pSrcBlock->info.rows);
} else {
char* p = taosVariantGet(&pExpr[k].base.pParam[0].param, type);
for (int32_t i = 0; i < pSrcBlock->info.rows; ++i) {
colDataAppend(pColInfoData, i + offset, p, false);
}
}
numOfRows = pSrcBlock->info.rows;
} else if (pExpr[k].pExpr->nodeType == QUERY_NODE_OPERATOR) {
SArray* pBlockList = taosArrayInit(4, POINTER_BYTES);
taosArrayPush(pBlockList, &pSrcBlock);
SColumnInfoData* pResColData = taosArrayGet(pResult->pDataBlock, outputSlotId);
SColumnInfoData idata = {.info = pResColData->info, .hasNull = true};
SScalarParam dest = {.columnData = &idata};
int32_t code = scalarCalculate(pExpr[k].pExpr->_optrRoot.pRootNode, pBlockList, &dest);
if (code != TSDB_CODE_SUCCESS) {
taosArrayDestroy(pBlockList);
return code;
}
int32_t startOffset = createNewColModel ? 0 : pResult->info.rows;
ASSERT(pResult->info.capacity > 0);
colDataMergeCol(pResColData, startOffset, (int32_t*)&pResult->info.capacity, &idata, dest.numOfRows);
colDataDestroy(&idata);
numOfRows = dest.numOfRows;
taosArrayDestroy(pBlockList);
} else if (pExpr[k].pExpr->nodeType == QUERY_NODE_FUNCTION) {
// _rowts/_c0, not tbname column
if (fmIsPseudoColumnFunc(pfCtx->functionId) && (!fmIsScanPseudoColumnFunc(pfCtx->functionId))) {
// do nothing
} else if (fmIsIndefiniteRowsFunc(pfCtx->functionId)) {
SResultRowEntryInfo* pResInfo = GET_RES_INFO(pfCtx);
pfCtx->fpSet.init(pfCtx, pResInfo);
pfCtx->pOutput = taosArrayGet(pResult->pDataBlock, outputSlotId);
pfCtx->offset = createNewColModel ? 0 : pResult->info.rows; // set the start offset
// set the timestamp(_rowts) output buffer
if (taosArrayGetSize(pPseudoList) > 0) {
int32_t* outputColIndex = taosArrayGet(pPseudoList, 0);
pfCtx->pTsOutput = (SColumnInfoData*)pCtx[*outputColIndex].pOutput;
}
// link pDstBlock to set selectivity value
if (pfCtx->subsidiaries.num > 0) {
pfCtx->pDstBlock = pResult;
}
int32_t code = pfCtx->fpSet.process(pfCtx);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
numOfRows = pResInfo->numOfRes;
} else if (fmIsAggFunc(pfCtx->functionId)) {
// selective value output should be set during corresponding function execution
if (fmIsSelectValueFunc(pfCtx->functionId)) {
continue;
}
// _group_key function for "partition by tbname" + csum(col_name) query
SColumnInfoData* pOutput = taosArrayGet(pResult->pDataBlock, outputSlotId);
int32_t slotId = pfCtx->param[0].pCol->slotId;
// todo handle the json tag
SColumnInfoData* pInput = taosArrayGet(pSrcBlock->pDataBlock, slotId);
for (int32_t f = 0; f < pSrcBlock->info.rows; ++f) {
bool isNull = colDataIsNull_s(pInput, f);
if (isNull) {
colDataAppendNULL(pOutput, pResult->info.rows + f);
} else {
char* data = colDataGetData(pInput, f);
colDataAppend(pOutput, pResult->info.rows + f, data, isNull);
}
}
} else {
SArray* pBlockList = taosArrayInit(4, POINTER_BYTES);
taosArrayPush(pBlockList, &pSrcBlock);
SColumnInfoData* pResColData = taosArrayGet(pResult->pDataBlock, outputSlotId);
SColumnInfoData idata = {.info = pResColData->info, .hasNull = true};
SScalarParam dest = {.columnData = &idata};
int32_t code = scalarCalculate((SNode*)pExpr[k].pExpr->_function.pFunctNode, pBlockList, &dest);
if (code != TSDB_CODE_SUCCESS) {
taosArrayDestroy(pBlockList);
return code;
}
int32_t startOffset = createNewColModel ? 0 : pResult->info.rows;
ASSERT(pResult->info.capacity > 0);
colDataMergeCol(pResColData, startOffset, (int32_t*)&pResult->info.capacity, &idata, dest.numOfRows);
colDataDestroy(&idata);
numOfRows = dest.numOfRows;
taosArrayDestroy(pBlockList);
}
} else {
return TSDB_CODE_OPS_NOT_SUPPORT;
}
}
if (!createNewColModel) {
pResult->info.rows += numOfRows;
}
return TSDB_CODE_SUCCESS;
}

91
source/libs/executor/src/scanoperator.c
View File

@ -184,7 +184,7 @@ static int32_t doDynamicPruneDataBlock(SOperatorInfo* pOperator, SDataBlockInfo*
SExprSupp* pSup1 = pTableScanInfo->base.pdInfo.pExprSup;
SFilePage* pPage = NULL;
SResultRow* pRow = getTableGroupOutputBuf(pOperator, pBlockInfo->groupId, &pPage);
SResultRow* pRow = getTableGroupOutputBuf(pOperator, pBlockInfo->id.groupId, &pPage);
if (pRow == NULL) {
return TSDB_CODE_SUCCESS;
@ -484,13 +484,13 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int
// 1. check if it is existed in meta cache
if (pCache == NULL) {
metaReaderInit(&mr, pHandle->meta, 0);
code = metaGetTableEntryByUidCache(&mr, pBlock->info.uid);
code = metaGetTableEntryByUidCache(&mr, pBlock->info.id.uid);
if (code != TSDB_CODE_SUCCESS) {
if (terrno == TSDB_CODE_PAR_TABLE_NOT_EXIST) {
qWarn("failed to get table meta, table may have been dropped, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.uid,
qWarn("failed to get table meta, table may have been dropped, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.id.uid,
tstrerror(terrno), idStr);
} else {
qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.uid, tstrerror(terrno), idStr);
qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.id.uid, tstrerror(terrno), idStr);
}
metaReaderClear(&mr);
return terrno;
@ -505,16 +505,16 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int
} else {
pCache->metaFetch += 1;
h = taosLRUCacheLookup(pCache->pTableMetaEntryCache, &pBlock->info.uid, sizeof(pBlock->info.uid));
h = taosLRUCacheLookup(pCache->pTableMetaEntryCache, &pBlock->info.id.uid, sizeof(pBlock->info.id.uid));
if (h == NULL) {
metaReaderInit(&mr, pHandle->meta, 0);
code = metaGetTableEntryByUidCache(&mr, pBlock->info.uid);
code = metaGetTableEntryByUidCache(&mr, pBlock->info.id.uid);
if (code != TSDB_CODE_SUCCESS) {
if (terrno == TSDB_CODE_PAR_TABLE_NOT_EXIST) {
qWarn("failed to get table meta, table may have been dropped, uid:0x%" PRIx64 ", code:%s, %s",
pBlock->info.uid, tstrerror(terrno), idStr);
pBlock->info.id.uid, tstrerror(terrno), idStr);
} else {
qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.uid, tstrerror(terrno),
qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", pBlock->info.id.uid, tstrerror(terrno),
idStr);
}
metaReaderClear(&mr);
@ -528,7 +528,7 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int
val = *pVal;
freeReader = true;
int32_t ret = taosLRUCacheInsert(pCache->pTableMetaEntryCache, &pBlock->info.uid, sizeof(uint64_t), pVal,
int32_t ret = taosLRUCacheInsert(pCache->pTableMetaEntryCache, &pBlock->info.id.uid, sizeof(uint64_t), pVal,
sizeof(STableCachedVal), freeCachedMetaItem, NULL, TAOS_LRU_PRIORITY_LOW);
if (ret != TAOS_LRU_STATUS_OK) {
qError("failed to put meta into lru cache, code:%d, %s", ret, idStr);
@ -642,13 +642,13 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) {
SDataBlockInfo* pBInfo = &pBlock->info;
int32_t rows = 0;
tsdbRetrieveDataBlockInfo(pTableScanInfo->base.dataReader, &rows, &pBInfo->uid, &pBInfo->window);
tsdbRetrieveDataBlockInfo(pTableScanInfo->base.dataReader, &rows, &pBInfo->id.uid, &pBInfo->window);
blockDataEnsureCapacity(pBlock, rows); // todo remove it latter
pBInfo->rows = rows;
ASSERT(pBInfo->uid != 0);
pBlock->info.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBlock->info.uid);
ASSERT(pBInfo->id.uid != 0);
pBlock->info.id.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBlock->info.id.uid);
uint32_t status = 0;
int32_t code = loadDataBlock(pOperator, &pTableScanInfo->base, pBlock, &status);
@ -668,13 +668,13 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) {
pOperator->cost.totalCost = pTableScanInfo->base.readRecorder.elapsedTime;
// todo refactor
/*pTableScanInfo->lastStatus.uid = pBlock->info.uid;*/
/*pTableScanInfo->lastStatus.uid = pBlock->info.id.uid;*/
/*pTableScanInfo->lastStatus.ts = pBlock->info.window.ekey;*/
pTaskInfo->streamInfo.lastStatus.type = TMQ_OFFSET__SNAPSHOT_DATA;
pTaskInfo->streamInfo.lastStatus.uid = pBlock->info.uid;
pTaskInfo->streamInfo.lastStatus.uid = pBlock->info.id.uid;
pTaskInfo->streamInfo.lastStatus.ts = pBlock->info.window.ekey;
ASSERT(pBlock->info.uid != 0);
ASSERT(pBlock->info.id.uid != 0);
return pBlock;
}
return NULL;
@ -786,7 +786,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) {
SSDataBlock* result = doGroupedTableScan(pOperator);
if (result != NULL) {
ASSERT(result->info.uid != 0);
ASSERT(result->info.id.uid != 0);
return result;
}
@ -885,7 +885,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode,
pInfo->base.dataBlockLoadFlag = pTableScanNode->dataRequired;
initResultSizeInfo(&pOperator->resultInfo, 4096);
pInfo->pResBlock = createResDataBlock(pDescNode);
pInfo->pResBlock = createDataBlockFromDescNode(pDescNode);
blockDataEnsureCapacity(pInfo->pResBlock, pOperator->resultInfo.capacity);
code = filterInitFromNode((SNode*)pTableScanNode->scan.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
@ -1009,7 +1009,7 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU
SDataBlockInfo* pBInfo = &pBlock->info;
int32_t rows = 0;
tsdbRetrieveDataBlockInfo(pReader, &rows, &pBInfo->uid, &pBInfo->window);
tsdbRetrieveDataBlockInfo(pReader, &rows, &pBInfo->id.uid, &pBInfo->window);
SArray* pCols = tsdbRetrieveDataBlock(pReader, NULL);
blockDataEnsureCapacity(pBlock, rows);
@ -1018,7 +1018,7 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU
relocateColumnData(pBlock, pTableScanInfo->base.matchInfo.pList, pCols, true);
doSetTagColumnData(&pTableScanInfo->base, pBlock, pTaskInfo, rows);
pBlock->info.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBInfo->uid);
pBlock->info.id.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBInfo->id.uid);
}
tsdbReaderClose(pReader);
@ -1175,7 +1175,7 @@ static SSDataBlock* doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32
pResult->info.calWin = pInfo->updateWin;
return pResult;
}
} else if (pResult->info.groupId == pInfo->groupId) {
} else if (pResult->info.id.groupId == pInfo->groupId) {
pResult->info.calWin = pInfo->updateWin;
return pResult;
}
@ -1366,7 +1366,7 @@ void calBlockTbName(SStreamScanInfo* pInfo, SSDataBlock* pBlock) {
if (pBlock == NULL || pBlock->info.rows == 0) return;
void* tbname = NULL;
if (streamStateGetParName(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, pBlock->info.groupId, &tbname) < 0) {
if (streamStateGetParName(pInfo->pStreamScanOp->pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, &tbname) < 0) {
pBlock->info.parTbName[0] = 0;
} else {
memcpy(pBlock->info.parTbName, tbname, TSDB_TABLE_NAME_LEN);
@ -1399,8 +1399,8 @@ void calBlockTbName(SStreamScanInfo* pInfo, SSDataBlock* pBlock) {
pBlock->info.parTbName[0] = 0;
}
if (pBlock->info.groupId && pBlock->info.parTbName[0]) {
streamStatePutParName(pState, pBlock->info.groupId, pBlock->info.parTbName);
if (pBlock->info.id.groupId && pBlock->info.parTbName[0]) {
streamStatePutParName(pState, pBlock->info.id.groupId, pBlock->info.parTbName);
}
blockDataDestroy(pSrcBlock);
@ -1434,7 +1434,7 @@ static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock
SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex);
ASSERT(pColDataInfo->info.type == TSDB_DATA_TYPE_TIMESTAMP);
TSKEY* tsCol = (TSKEY*)pColDataInfo->pData;
bool tableInserted = updateInfoIsTableInserted(pInfo->pUpdateInfo, pBlock->info.uid);
bool tableInserted = updateInfoIsTableInserted(pInfo->pUpdateInfo, pBlock->info.id.uid);
for (int32_t rowId = 0; rowId < pBlock->info.rows; rowId++) {
SResultRowInfo dumyInfo;
dumyInfo.cur.pageId = -1;
@ -1445,18 +1445,18 @@ static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock
isClosed = isCloseWindow(&win, &pInfo->twAggSup);
}
// must check update info first.
bool update = updateInfoIsUpdated(pInfo->pUpdateInfo, pBlock->info.uid, tsCol[rowId]);
bool update = updateInfoIsUpdated(pInfo->pUpdateInfo, pBlock->info.id.uid, tsCol[rowId]);
bool closedWin = isClosed && isSignleIntervalWindow(pInfo) &&
isDeletedStreamWindow(&win, pBlock->info.groupId,
isDeletedStreamWindow(&win, pBlock->info.id.groupId,
pInfo->pTableScanOp->pTaskInfo->streamInfo.pState, &pInfo->twAggSup);
if ((update || closedWin) && out) {
qDebug("stream update check not pass, update %d, closedWin %d", update, closedWin);
uint64_t gpId = 0;
appendOneRowToStreamSpecialBlock(pInfo->pUpdateDataRes, tsCol + rowId, tsCol + rowId, &pBlock->info.uid, &gpId,
appendOneRowToStreamSpecialBlock(pInfo->pUpdateDataRes, tsCol + rowId, tsCol + rowId, &pBlock->info.id.uid, &gpId,
NULL);
if (closedWin && pInfo->partitionSup.needCalc) {
gpId = calGroupIdByData(&pInfo->partitionSup, pInfo->pPartScalarSup, pBlock, rowId);
appendOneRowToStreamSpecialBlock(pInfo->pUpdateDataRes, tsCol + rowId, tsCol + rowId, &pBlock->info.uid, &gpId,
appendOneRowToStreamSpecialBlock(pInfo->pUpdateDataRes, tsCol + rowId, tsCol + rowId, &pBlock->info.id.uid, &gpId,
NULL);
}
}
@ -1476,11 +1476,11 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock
blockDataEnsureCapacity(pInfo->pRes, pBlock->info.rows);
pInfo->pRes->info.rows = pBlock->info.rows;
pInfo->pRes->info.uid = pBlock->info.uid;
pInfo->pRes->info.id.uid = pBlock->info.id.uid;
pInfo->pRes->info.type = STREAM_NORMAL;
pInfo->pRes->info.version = pBlock->info.version;
pInfo->pRes->info.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBlock->info.uid);
pInfo->pRes->info.id.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBlock->info.id.uid);
// todo extract method
for (int32_t i = 0; i < taosArrayGetSize(pInfo->matchInfo.pList); ++i) {
@ -1807,8 +1807,8 @@ FETCH_NEXT_BLOCK:
int32_t current = pInfo->validBlockIndex++;
SSDataBlock* pBlock = taosArrayGetP(pInfo->pBlockLists, current);
if (pBlock->info.groupId && pBlock->info.parTbName[0]) {
streamStatePutParName(pTaskInfo->streamInfo.pState, pBlock->info.groupId, pBlock->info.parTbName);
if (pBlock->info.id.groupId && pBlock->info.parTbName[0]) {
streamStatePutParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, pBlock->info.parTbName);
}
// TODO move into scan
pBlock->info.calWin.skey = INT64_MIN;
@ -1960,7 +1960,7 @@ FETCH_NEXT_BLOCK:
setBlockIntoRes(pInfo, &block, false);
if (updateInfoIgnore(pInfo->pUpdateInfo, &pInfo->pRes->info.window, pInfo->pRes->info.groupId,
if (updateInfoIgnore(pInfo->pUpdateInfo, &pInfo->pRes->info.window, pInfo->pRes->info.id.groupId,
pInfo->pRes->info.version)) {
printDataBlock(pInfo->pRes, "stream scan ignore");
blockDataCleanup(pInfo->pRes);
@ -2050,7 +2050,7 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) {
}
int32_t rows = 0;
tsdbRetrieveDataBlockInfo(pInfo->dataReader, &rows, &pBlock->info.uid, &pBlock->info.window);
tsdbRetrieveDataBlockInfo(pInfo->dataReader, &rows, &pBlock->info.id.uid, &pBlock->info.window);
pBlock->info.rows = rows;
SArray* pCols = tsdbRetrieveDataBlock(pInfo->dataReader, NULL);
@ -2059,9 +2059,9 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) {
longjmp(pTaskInfo->env, terrno);
}
qDebug("tmqsnap doRawScan get data uid:%" PRId64 "", pBlock->info.uid);
qDebug("tmqsnap doRawScan get data uid:%" PRId64 "", pBlock->info.id.uid);
pTaskInfo->streamInfo.lastStatus.type = TMQ_OFFSET__SNAPSHOT_DATA;
pTaskInfo->streamInfo.lastStatus.uid = pBlock->info.uid;
pTaskInfo->streamInfo.lastStatus.uid = pBlock->info.id.uid;
pTaskInfo->streamInfo.lastStatus.ts = pBlock->info.window.ekey;
return pBlock;
}
@ -2352,7 +2352,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
goto _error;
}
pInfo->pRes = createResDataBlock(pDescNode);
pInfo->pRes = createDataBlockFromDescNode(pDescNode);
pInfo->pUpdateRes = createSpecialDataBlock(STREAM_CLEAR);
pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE;
pInfo->windowSup = (SWindowSupporter){.pStreamAggSup = NULL, .gap = -1, .parentType = QUERY_NODE_PHYSICAL_PLAN};
@ -2476,8 +2476,7 @@ static void destroyTagScanOperatorInfo(void* param) {
taosMemoryFreeClear(param);
}
SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode,
STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo) {
SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo) {
STagScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STagScanInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pInfo == NULL || pOperator == NULL) {
@ -2499,7 +2498,7 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi
goto _error;
}
pInfo->pRes = createResDataBlock(pDescNode);
pInfo->pRes = createDataBlockFromDescNode(pDescNode);
pInfo->readHandle = *pReadHandle;
pInfo->curPos = 0;
@ -2556,7 +2555,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) {
blockDataCleanup(pBlock);
int32_t rows = 0;
tsdbRetrieveDataBlockInfo(reader, &rows, &pBlock->info.uid, &pBlock->info.window);
tsdbRetrieveDataBlockInfo(reader, &rows, &pBlock->info.id.uid, &pBlock->info.window);
blockDataEnsureCapacity(pBlock, rows);
pBlock->info.rows = rows;
@ -2578,7 +2577,7 @@ static SSDataBlock* getTableDataBlockImpl(void* param) {
continue;
}
pBlock->info.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBlock->info.uid);
pBlock->info.id.groupId = getTableGroupId(pTaskInfo->pTableInfoList, pBlock->info.id.uid);
pOperator->resultInfo.totalRows += pBlock->info.rows;
pTableScanInfo->base.readRecorder.elapsedTime += (taosGetTimestampUs() - st) / 1000.0;
@ -2613,7 +2612,7 @@ SArray* generateSortByTsInfo(SArray* colMatchInfo, int32_t order) {
return pList;
}
int32_t dumpSQueryTableCond(const SQueryTableDataCond* src, SQueryTableDataCond* dst) {
int32_t dumpQueryTableCond(const SQueryTableDataCond* src, SQueryTableDataCond* dst) {
memcpy((void*)dst, (void*)src, sizeof(SQueryTableDataCond));
dst->colList = taosMemoryCalloc(src->numOfCols, sizeof(SColumnInfo));
for (int i = 0; i < src->numOfCols; i++) {
@ -2664,7 +2663,7 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) {
taosArrayPush(pInfo->sortSourceParams, &param);
SQueryTableDataCond cond;
dumpSQueryTableCond(&pInfo->base.cond, &cond);
dumpQueryTableCond(&pInfo->base.cond, &cond);
taosArrayPush(pInfo->queryConds, &cond);
}
@ -2774,7 +2773,7 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) {
pBlock = getSortedTableMergeScanBlockData(pInfo->pSortHandle, pInfo->pResBlock, pOperator->resultInfo.capacity,
pOperator);
if (pBlock != NULL) {
pBlock->info.groupId = pInfo->groupId;
pBlock->info.id.groupId = pInfo->groupId;
pOperator->resultInfo.totalRows += pBlock->info.rows;
return pBlock;
} else {
@ -2900,7 +2899,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN
}
initResultSizeInfo(&pOperator->resultInfo, 1024);
pInfo->pResBlock = createResDataBlock(pDescNode);
pInfo->pResBlock = createDataBlockFromDescNode(pDescNode);
blockDataEnsureCapacity(pInfo->pResBlock, pOperator->resultInfo.capacity);
pInfo->sortSourceParams = taosArrayInit(64, sizeof(STableMergeScanSortSourceParam));

20
source/libs/executor/src/sortoperator.c
View File

@ -47,7 +47,7 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode*
SDataBlockDescNode* pDescNode = pSortNode->node.pOutputDataBlockDesc;
int32_t numOfCols = 0;
SSDataBlock* pResBlock = createResDataBlock(pDescNode);
SSDataBlock* pResBlock = createDataBlockFromDescNode(pDescNode);
SExprInfo* pExprInfo = createExprInfo(pSortNode->pExprs, NULL, &numOfCols);
int32_t numOfOutputCols = 0;
@ -359,7 +359,7 @@ SSDataBlock* fetchNextGroupSortDataBlock(void* param) {
SOperatorInfo* childOp = source->childOpInfo;
SSDataBlock* block = childOp->fpSet.getNextFn(childOp);
if (block != NULL) {
if (block->info.groupId == grpSortOpInfo->currGroupId) {
if (block->info.id.groupId == grpSortOpInfo->currGroupId) {
grpSortOpInfo->childOpStatus = CHILD_OP_SAME_GROUP;
return block;
} else {
@ -439,7 +439,7 @@ SSDataBlock* doGroupSort(SOperatorInfo* pOperator) {
setOperatorCompleted(pOperator);
return NULL;
}
pInfo->currGroupId = pInfo->prefetchedSortInput->info.groupId;
pInfo->currGroupId = pInfo->prefetchedSortInput->info.id.groupId;
pInfo->childOpStatus = CHILD_OP_NEW_GROUP;
beginSortGroup(pOperator);
}
@ -451,13 +451,13 @@ SSDataBlock* doGroupSort(SOperatorInfo* pOperator) {
pBlock = getGroupSortedBlockData(pInfo->pCurrSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity,
pInfo->matchInfo.pList, pInfo);
if (pBlock != NULL) {
pBlock->info.groupId = pInfo->currGroupId;
pBlock->info.id.groupId = pInfo->currGroupId;
pOperator->resultInfo.totalRows += pBlock->info.rows;
return pBlock;
} else {
if (pInfo->childOpStatus == CHILD_OP_NEW_GROUP) {
finishSortGroup(pOperator);
pInfo->currGroupId = pInfo->prefetchedSortInput->info.groupId;
pInfo->currGroupId = pInfo->prefetchedSortInput->info.id.groupId;
beginSortGroup(pOperator);
} else if (pInfo->childOpStatus == CHILD_OP_FINISHED) {
finishSortGroup(pOperator);
@ -509,7 +509,7 @@ SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSort
initResultSizeInfo(&pOperator->resultInfo, 1024);
pOperator->exprSupp.pCtx = createSqlFunctionCtx(pExprInfo, numOfCols, &pOperator->exprSupp.rowEntryInfoOffset);
pInfo->binfo.pRes = createResDataBlock(pDescNode);
pInfo->binfo.pRes = createDataBlockFromDescNode(pDescNode);
blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
int32_t numOfOutputCols = 0;
@ -691,10 +691,10 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData
pInfo->limitInfo.numOfOutputRows += p->info.rows;
pDataBlock->info.rows = p->info.rows;
pDataBlock->info.groupId = pInfo->groupId;
pDataBlock->info.id.groupId = pInfo->groupId;
}
qDebug("%s get sorted block, groupId:0x%" PRIx64 " rows:%d", GET_TASKID(pTaskInfo), pDataBlock->info.groupId,
qDebug("%s get sorted block, groupId:0x%" PRIx64 " rows:%d", GET_TASKID(pTaskInfo), pDataBlock->info.id.groupId,
pDataBlock->info.rows);
return (pDataBlock->info.rows > 0) ? pDataBlock : NULL;
@ -766,7 +766,7 @@ SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size
}
initLimitInfo(pMergePhyNode->node.pLimit, pMergePhyNode->node.pSlimit, &pInfo->limitInfo);
pInfo->binfo.pRes = createResDataBlock(pDescNode);
pInfo->binfo.pRes = createDataBlockFromDescNode(pDescNode);
int32_t rowSize = pInfo->binfo.pRes->info.rowSize;
ASSERT(rowSize < 100 * 1024 * 1024);
@ -779,7 +779,7 @@ SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** downStreams, size
}
SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, 0);
SSDataBlock* pInputBlock = createResDataBlock(pChildNode->pOutputDataBlockDesc);
SSDataBlock* pInputBlock = createDataBlockFromDescNode(pChildNode->pOutputDataBlockDesc);
initResultSizeInfo(&pOperator->resultInfo, 4096);
blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);

54
source/libs/executor/src/sysscanoperator.c
View File

@ -1411,7 +1411,7 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan
pInfo->pUser = taosMemoryStrDup((void*)pUser);
pInfo->sysInfo = pScanPhyNode->sysInfo;
pInfo->showRewrite = pScanPhyNode->showRewrite;
pInfo->pRes = createResDataBlock(pDescNode);
pInfo->pRes = createDataBlockFromDescNode(pDescNode);
pInfo->pCondition = pScanNode->node.pConditions;
code = filterInitFromNode(pScanNode->node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
@ -1831,39 +1831,39 @@ static int32_t doGetTableRowSize(void* pMeta, uint64_t uid, int32_t* rowLen, con
}
static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) {
if (pOperator->status == OP_EXEC_DONE) {
return NULL;
}
if (pOperator->status == OP_EXEC_DONE) {
return NULL;
}
SBlockDistInfo* pBlockScanInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
SBlockDistInfo* pBlockScanInfo = pOperator->info;
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
STableBlockDistInfo blockDistInfo = {.minRows = INT_MAX, .maxRows = INT_MIN};
int32_t code = doGetTableRowSize(pBlockScanInfo->readHandle.meta, pBlockScanInfo->uid,
(int32_t*)&blockDistInfo.rowSize, GET_TASKID(pTaskInfo));
if (code != TSDB_CODE_SUCCESS) {
T_LONG_JMP(pTaskInfo->env, code);
}
STableBlockDistInfo blockDistInfo = {.minRows = INT_MAX, .maxRows = INT_MIN};
int32_t code = doGetTableRowSize(pBlockScanInfo->readHandle.meta, pBlockScanInfo->uid,
(int32_t*)&blockDistInfo.rowSize, GET_TASKID(pTaskInfo));
if (code != TSDB_CODE_SUCCESS) {
T_LONG_JMP(pTaskInfo->env, code);
}
tsdbGetFileBlocksDistInfo(pBlockScanInfo->pHandle, &blockDistInfo);
blockDistInfo.numOfInmemRows = (int32_t)tsdbGetNumOfRowsInMemTable(pBlockScanInfo->pHandle);
tsdbGetFileBlocksDistInfo(pBlockScanInfo->pHandle, &blockDistInfo);
blockDistInfo.numOfInmemRows = (int32_t)tsdbGetNumOfRowsInMemTable(pBlockScanInfo->pHandle);
SSDataBlock* pBlock = pBlockScanInfo->pResBlock;
SSDataBlock* pBlock = pBlockScanInfo->pResBlock;
int32_t slotId = pOperator->exprSupp.pExprInfo->base.resSchema.slotId;
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, slotId);
int32_t slotId = pOperator->exprSupp.pExprInfo->base.resSchema.slotId;
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, slotId);
int32_t len = tSerializeBlockDistInfo(NULL, 0, &blockDistInfo);
char* p = taosMemoryCalloc(1, len + VARSTR_HEADER_SIZE);
tSerializeBlockDistInfo(varDataVal(p), len, &blockDistInfo);
varDataSetLen(p, len);
int32_t len = tSerializeBlockDistInfo(NULL, 0, &blockDistInfo);
char* p = taosMemoryCalloc(1, len + VARSTR_HEADER_SIZE);
tSerializeBlockDistInfo(varDataVal(p), len, &blockDistInfo);
varDataSetLen(p, len);
colDataAppend(pColInfo, 0, p, false);
taosMemoryFree(p);
colDataAppend(pColInfo, 0, p, false);
taosMemoryFree(p);
pBlock->info.rows = 1;
pOperator->status = OP_EXEC_DONE;
return pBlock;
pBlock->info.rows = 1;
pOperator->status = OP_EXEC_DONE;
return pBlock;
}
static void destroyBlockDistScanOperatorInfo(void* param) {
@ -1928,7 +1928,7 @@ SOperatorInfo* createDataBlockInfoScanOperator(SReadHandle* readHandle, SBlockDi
pInfo->readHandle = *readHandle;
pInfo->uid = pBlockScanNode->suid;
pInfo->pResBlock = createResDataBlock(pBlockScanNode->node.pOutputDataBlockDesc);
pInfo->pResBlock = createDataBlockFromDescNode(pBlockScanNode->node.pOutputDataBlockDesc);
blockDataEnsureCapacity(pInfo->pResBlock, 1);
int32_t numOfCols = 0;

61
source/libs/executor/src/tfill.c
View File

@ -1092,7 +1092,7 @@ static bool checkResult(SStreamFillSupporter* pFillSup, TSKEY ts, uint64_t group
}
static void buildFillResult(SResultRowData* pResRow, SStreamFillSupporter* pFillSup, TSKEY ts, SSDataBlock* pBlock) {
uint64_t groupId = pBlock->info.groupId;
uint64_t groupId = pBlock->info.id.groupId;
if (pFillSup->hasDelete && !checkResult(pFillSup, ts, groupId)) {
return;
}
@ -1131,7 +1131,7 @@ static void doStreamFillNormal(SStreamFillSupporter* pFillSup, SStreamFillInfo*
static void doStreamFillLinear(SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo, SSDataBlock* pBlock) {
while (hasRemainCalc(pFillInfo) && pBlock->info.rows < pBlock->info.capacity) {
uint64_t groupId = pBlock->info.groupId;
uint64_t groupId = pBlock->info.id.groupId;
SWinKey key = {.groupId = groupId, .ts = pFillInfo->current};
if (pFillSup->hasDelete && !checkResult(pFillSup, pFillInfo->current, groupId)) {
pFillInfo->current = taosTimeAdd(pFillInfo->current, pFillSup->interval.sliding, pFillSup->interval.slidingUnit,
@ -1230,7 +1230,7 @@ void keepBlockRowInDiscBuf(SOperatorInfo* pOperator, SStreamFillInfo* pFillInfo,
static void doFillResults(SOperatorInfo* pOperator, SStreamFillSupporter* pFillSup, SStreamFillInfo* pFillInfo,
SSDataBlock* pBlock, TSKEY* tsCol, int32_t rowId, SSDataBlock* pRes) {
uint64_t groupId = pBlock->info.groupId;
uint64_t groupId = pBlock->info.id.groupId;
getWindowFromDiscBuf(pOperator, tsCol[rowId], groupId, pFillSup);
if (pFillSup->prev.key == pFillInfo->preRowKey) {
resetFillWindow(&pFillSup->prev);
@ -1245,9 +1245,9 @@ static void doStreamFillImpl(SOperatorInfo* pOperator) {
SStreamFillSupporter* pFillSup = pInfo->pFillSup;
SStreamFillInfo* pFillInfo = pInfo->pFillInfo;
SSDataBlock* pBlock = pInfo->pSrcBlock;
uint64_t groupId = pBlock->info.groupId;
uint64_t groupId = pBlock->info.id.groupId;
SSDataBlock* pRes = pInfo->pRes;
pRes->info.groupId = groupId;
pRes->info.id.groupId = groupId;
if (hasRemainCalc(pFillInfo)) {
doStreamFillRange(pFillInfo, pFillSup, pRes);
}
@ -1282,7 +1282,9 @@ static void doStreamFillImpl(SOperatorInfo* pOperator) {
blockDataCleanup(pInfo->pSrcBlock);
}
static void buildDeleteRange(TSKEY start, TSKEY end, uint64_t groupId, SSDataBlock* delRes) {
static void buildDeleteRange(SOperatorInfo* pOp, TSKEY start, TSKEY end, uint64_t groupId, SSDataBlock* delRes) {
SStreamState* pState = pOp->pTaskInfo->streamInfo.pState;
SSDataBlock* pBlock = delRes;
SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX);
SColumnInfoData* pEndCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX);
@ -1290,25 +1292,42 @@ static void buildDeleteRange(TSKEY start, TSKEY end, uint64_t groupId, SSDataBlo
SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX);
SColumnInfoData* pCalStartCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX);
SColumnInfoData* pCalEndCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX);
SColumnInfoData* pTbNameCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX);
colDataAppend(pStartCol, pBlock->info.rows, (const char*)&start, false);
colDataAppend(pEndCol, pBlock->info.rows, (const char*)&end, false);
colDataAppendNULL(pUidCol, pBlock->info.rows);
colDataAppend(pGroupCol, pBlock->info.rows, (const char*)&groupId, false);
colDataAppendNULL(pCalStartCol, pBlock->info.rows);
colDataAppendNULL(pCalEndCol, pBlock->info.rows);
SColumnInfoData* pTableCol = taosArrayGet(pBlock->pDataBlock, TABLE_NAME_COLUMN_INDEX);
void* tbname = NULL;
streamStateGetParName(pOp->pTaskInfo->streamInfo.pState, groupId, &tbname);
if (tbname == NULL) {
colDataAppendNULL(pTableCol, pBlock->info.rows);
} else {
char parTbName[VARSTR_HEADER_SIZE + TSDB_TABLE_NAME_LEN];
STR_WITH_MAXSIZE_TO_VARSTR(parTbName, tbname, sizeof(parTbName));
colDataAppend(pTableCol, pBlock->info.rows, (const char*)parTbName, false);
}
tdbFree(tbname);
pBlock->info.rows++;
}
static void buildDeleteResult(SStreamFillSupporter* pFillSup, TSKEY startTs, TSKEY endTs, uint64_t groupId,
static void buildDeleteResult(SOperatorInfo* pOperator, TSKEY startTs, TSKEY endTs, uint64_t groupId,
SSDataBlock* delRes) {
SStreamFillOperatorInfo* pInfo = pOperator->info;
SStreamFillSupporter* pFillSup = pInfo->pFillSup;
if (hasPrevWindow(pFillSup)) {
TSKEY start = getNextWindowTs(pFillSup->prev.key, &pFillSup->interval);
buildDeleteRange(start, endTs, groupId, delRes);
buildDeleteRange(pOperator, start, endTs, groupId, delRes);
} else if (hasNextWindow(pFillSup)) {
TSKEY end = getPrevWindowTs(pFillSup->next.key, &pFillSup->interval);
buildDeleteRange(startTs, end, groupId, delRes);
buildDeleteRange(pOperator, startTs, end, groupId, delRes);
} else {
buildDeleteRange(startTs, endTs, groupId, delRes);
buildDeleteRange(pOperator, startTs, endTs, groupId, delRes);
}
}
@ -1319,7 +1338,7 @@ static void doDeleteFillResultImpl(SOperatorInfo* pOperator, TSKEY startTs, TSKE
SWinKey key = {.ts = startTs, .groupId = groupId};
if (!pInfo->pFillInfo->needFill) {
streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &key);
buildDeleteResult(pInfo->pFillSup, startTs, endTs, groupId, pInfo->pDelRes);
buildDeleteResult(pOperator, startTs, endTs, groupId, pInfo->pDelRes);
} else {
STimeRange tw = {
.skey = startTs,
@ -1342,14 +1361,14 @@ static void doDeleteFillFinalize(SOperatorInfo* pOperator) {
tSimpleHashClear(pInfo->pFillSup->pResMap);
for (; pFillInfo->delIndex < size; pFillInfo->delIndex++) {
STimeRange* range = taosArrayGet(pFillInfo->delRanges, pFillInfo->delIndex);
if (pInfo->pRes->info.groupId != 0 && pInfo->pRes->info.groupId != range->groupId) {
if (pInfo->pRes->info.id.groupId != 0 && pInfo->pRes->info.id.groupId != range->groupId) {
return;
}
getWindowFromDiscBuf(pOperator, range->skey, range->groupId, pInfo->pFillSup);
setDeleteFillValueInfo(range->skey, range->ekey, pInfo->pFillSup, pInfo->pFillInfo);
if (pInfo->pFillInfo->needFill) {
doStreamFillRange(pInfo->pFillInfo, pInfo->pFillSup, pInfo->pRes);
pInfo->pRes->info.groupId = range->groupId;
pInfo->pRes->info.id.groupId = range->groupId;
}
SWinKey key = {.ts = range->skey, .groupId = range->groupId};
streamStateFillDel(pOperator->pTaskInfo->streamInfo.pState, &key);
@ -1435,7 +1454,7 @@ static void doApplyStreamScalarCalculation(SOperatorInfo* pOperator, SSDataBlock
pSup = &pInfo->pFillSup->notFillExprSup;
setInputDataBlock(pSup, pSrcBlock, TSDB_ORDER_ASC, MAIN_SCAN, false);
projectApplyFunctions(pSup->pExprInfo, pDstBlock, pSrcBlock, pSup->pCtx, pSup->numOfExprs, NULL);
pDstBlock->info.groupId = pSrcBlock->info.groupId;
pDstBlock->info.id.groupId = pSrcBlock->info.id.groupId;
blockDataUpdateTsWindow(pDstBlock, pInfo->primaryTsCol);
}
@ -1578,7 +1597,7 @@ static SStreamFillSupporter* initStreamFillSup(SStreamFillPhysiNode* pPhyFillNod
destroyStreamFillSupporter(pFillSup);
return NULL;
}
SExprInfo* noFillExpr = createExprInfo(pPhyFillNode->pNotFillExprs, NULL, &numOfNotFillCols);
code = initExprSupp(&pFillSup->notFillExprSup, noFillExpr, numOfNotFillCols);
if (code != TSDB_CODE_SUCCESS) {
@ -1651,9 +1670,9 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi
}
initResultSizeInfo(&pOperator->resultInfo, 4096);
pInfo->pRes = createResDataBlock(pPhyFillNode->node.pOutputDataBlockDesc);
pInfo->pSrcBlock = createResDataBlock(pPhyFillNode->node.pOutputDataBlockDesc);
pInfo->pPrevSrcBlock = createResDataBlock(pPhyFillNode->node.pOutputDataBlockDesc);
pInfo->pRes = createDataBlockFromDescNode(pPhyFillNode->node.pOutputDataBlockDesc);
pInfo->pSrcBlock = createDataBlockFromDescNode(pPhyFillNode->node.pOutputDataBlockDesc);
pInfo->pPrevSrcBlock = createDataBlockFromDescNode(pPhyFillNode->node.pOutputDataBlockDesc);
blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
blockDataEnsureCapacity(pInfo->pSrcBlock, pOperator->resultInfo.capacity);
blockDataEnsureCapacity(pInfo->pPrevSrcBlock, pOperator->resultInfo.capacity);
@ -1715,9 +1734,9 @@ SOperatorInfo* createStreamFillOperatorInfo(SOperatorInfo* downstream, SStreamFi
}
pInfo->srcRowIndex = 0;
setOperatorInfo(pOperator, "StreamFillOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL, false, OP_NOT_OPENED, pInfo, pTaskInfo);
pOperator->fpSet =
createOperatorFpSet(operatorDummyOpenFn, doStreamFill, NULL, destroyStreamFillOperatorInfo, NULL);
setOperatorInfo(pOperator, "StreamFillOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL, false, OP_NOT_OPENED, pInfo,
pTaskInfo);
pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamFill, NULL, destroyStreamFillOperatorInfo, NULL);
code = appendDownstream(pOperator, &downstream, 1);
if (code != TSDB_CODE_SUCCESS) {

587
source/libs/executor/src/timesliceoperator.c Normal file
View File

@ -0,0 +1,587 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "executorimpl.h"
#include "filter.h"
#include "function.h"
#include "functionMgt.h"
#include "tcommon.h"
#include "tcompare.h"
#include "tdatablock.h"
#include "tfill.h"
#include "ttime.h"
typedef struct STimeSliceOperatorInfo {
SSDataBlock* pRes;
STimeWindow win;
SInterval interval;
int64_t current;
SArray* pPrevRow; // SArray<SGroupValue>
SArray* pNextRow; // SArray<SGroupValue>
SArray* pLinearInfo; // SArray<SFillLinearInfo>
bool isPrevRowSet;
bool isNextRowSet;
int32_t fillType; // fill type
SColumn tsCol; // primary timestamp column
SExprSupp scalarSup; // scalar calculation
struct SFillColInfo* pFillColInfo; // fill column info
} STimeSliceOperatorInfo;
static void destroyTimeSliceOperatorInfo(void* param);
static void doKeepPrevRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock* pBlock, int32_t rowIndex) {
int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
SGroupKeys* pkey = taosArrayGet(pSliceInfo->pPrevRow, i);
if (!colDataIsNull_s(pColInfoData, rowIndex)) {
pkey->isNull = false;
char* val = colDataGetData(pColInfoData, rowIndex);
if (!IS_VAR_DATA_TYPE(pkey->type)) {
memcpy(pkey->pData, val, pkey->bytes);
} else {
memcpy(pkey->pData, val, varDataLen(val));
}
} else {
pkey->isNull = true;
}
}
pSliceInfo->isPrevRowSet = true;
}
static void doKeepNextRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock* pBlock, int32_t rowIndex) {
int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
SGroupKeys* pkey = taosArrayGet(pSliceInfo->pNextRow, i);
if (!colDataIsNull_s(pColInfoData, rowIndex)) {
pkey->isNull = false;
char* val = colDataGetData(pColInfoData, rowIndex);
if (!IS_VAR_DATA_TYPE(pkey->type)) {
memcpy(pkey->pData, val, pkey->bytes);
} else {
memcpy(pkey->pData, val, varDataLen(val));
}
} else {
pkey->isNull = true;
}
}
pSliceInfo->isNextRowSet = true;
}
static void doKeepLinearInfo(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock* pBlock, int32_t rowIndex) {
int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i);
SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, pSliceInfo->tsCol.slotId);
SFillLinearInfo* pLinearInfo = taosArrayGet(pSliceInfo->pLinearInfo, i);
// null value is represented by using key = INT64_MIN for now.
// TODO: optimize to ignore null values for linear interpolation.
if (!pLinearInfo->isStartSet) {
if (!colDataIsNull_s(pColInfoData, rowIndex)) {
pLinearInfo->start.key = *(int64_t*)colDataGetData(pTsCol, rowIndex);
memcpy(pLinearInfo->start.val, colDataGetData(pColInfoData, rowIndex), pLinearInfo->bytes);
}
pLinearInfo->isStartSet = true;
} else if (!pLinearInfo->isEndSet) {
if (!colDataIsNull_s(pColInfoData, rowIndex)) {
pLinearInfo->end.key = *(int64_t*)colDataGetData(pTsCol, rowIndex);
memcpy(pLinearInfo->end.val, colDataGetData(pColInfoData, rowIndex), pLinearInfo->bytes);
}
pLinearInfo->isEndSet = true;
} else {
pLinearInfo->start.key = pLinearInfo->end.key;
memcpy(pLinearInfo->start.val, pLinearInfo->end.val, pLinearInfo->bytes);
if (!colDataIsNull_s(pColInfoData, rowIndex)) {
pLinearInfo->end.key = *(int64_t*)colDataGetData(pTsCol, rowIndex);
memcpy(pLinearInfo->end.val, colDataGetData(pColInfoData, rowIndex), pLinearInfo->bytes);
} else {
pLinearInfo->end.key = INT64_MIN;
}
}
}
}
static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pResBlock, bool beforeTs) {
int32_t rows = pResBlock->info.rows;
blockDataEnsureCapacity(pResBlock, rows + 1);
// todo set the correct primary timestamp column
// output the result
bool hasInterp = true;
for (int32_t j = 0; j < pExprSup->numOfExprs; ++j) {
SExprInfo* pExprInfo = &pExprSup->pExprInfo[j];
int32_t dstSlot = pExprInfo->base.resSchema.slotId;
SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot);
if (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type)) {
colDataAppend(pDst, rows, (char*)&pSliceInfo->current, false);
continue;
}
int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId;
switch (pSliceInfo->fillType) {
case TSDB_FILL_NULL: {
colDataAppendNULL(pDst, rows);
break;
}
case TSDB_FILL_SET_VALUE: {
SVariant* pVar = &pSliceInfo->pFillColInfo[j].fillVal;
if (pDst->info.type == TSDB_DATA_TYPE_FLOAT) {
float v = 0;
GET_TYPED_DATA(v, float, pVar->nType, &pVar->i);
colDataAppend(pDst, rows, (char*)&v, false);
} else if (pDst->info.type == TSDB_DATA_TYPE_DOUBLE) {
double v = 0;
GET_TYPED_DATA(v, double, pVar->nType, &pVar->i);
colDataAppend(pDst, rows, (char*)&v, false);
} else if (IS_SIGNED_NUMERIC_TYPE(pDst->info.type)) {
int64_t v = 0;
GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->i);
colDataAppend(pDst, rows, (char*)&v, false);
}
break;
}
case TSDB_FILL_LINEAR: {
SFillLinearInfo* pLinearInfo = taosArrayGet(pSliceInfo->pLinearInfo, srcSlot);
SPoint start = pLinearInfo->start;
SPoint end = pLinearInfo->end;
SPoint current = {.key = pSliceInfo->current};
// do not interpolate before ts range, only increate pSliceInfo->current
if (beforeTs && !pLinearInfo->isEndSet) {
return true;
}
if (!pLinearInfo->isStartSet || !pLinearInfo->isEndSet) {
hasInterp = false;
break;
}
if (start.key == INT64_MIN || end.key == INT64_MIN) {
colDataAppendNULL(pDst, rows);
break;
}
current.val = taosMemoryCalloc(pLinearInfo->bytes, 1);
taosGetLinearInterpolationVal(&current, pLinearInfo->type, &start, &end, pLinearInfo->type);
colDataAppend(pDst, rows, (char*)current.val, false);
taosMemoryFree(current.val);
break;
}
case TSDB_FILL_PREV: {
if (!pSliceInfo->isPrevRowSet) {
hasInterp = false;
break;
}
SGroupKeys* pkey = taosArrayGet(pSliceInfo->pPrevRow, srcSlot);
if (pkey->isNull == false) {
colDataAppend(pDst, rows, pkey->pData, false);
} else {
colDataAppendNULL(pDst, rows);
}
break;
}
case TSDB_FILL_NEXT: {
if (!pSliceInfo->isNextRowSet) {
hasInterp = false;
break;
}
SGroupKeys* pkey = taosArrayGet(pSliceInfo->pNextRow, srcSlot);
if (pkey->isNull == false) {
colDataAppend(pDst, rows, pkey->pData, false);
} else {
colDataAppendNULL(pDst, rows);
}
break;
}
case TSDB_FILL_NONE:
default:
break;
}
}
if (hasInterp) {
pResBlock->info.rows += 1;
}
return hasInterp;
}
static void addCurrentRowToResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pResBlock,
SSDataBlock* pSrcBlock, int32_t index) {
blockDataEnsureCapacity(pResBlock, pResBlock->info.rows + 1);
for (int32_t j = 0; j < pExprSup->numOfExprs; ++j) {
SExprInfo* pExprInfo = &pExprSup->pExprInfo[j];
int32_t dstSlot = pExprInfo->base.resSchema.slotId;
SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot);
if (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type)) {
colDataAppend(pDst, pResBlock->info.rows, (char*)&pSliceInfo->current, false);
} else {
int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId;
SColumnInfoData* pSrc = taosArrayGet(pSrcBlock->pDataBlock, srcSlot);
if (colDataIsNull_s(pSrc, index)) {
colDataAppendNULL(pDst, pResBlock->info.rows);
continue;
}
char* v = colDataGetData(pSrc, index);
colDataAppend(pDst, pResBlock->info.rows, v, false);
}
}
pResBlock->info.rows += 1;
return;
}
static int32_t initPrevRowsKeeper(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) {
if (pInfo->pPrevRow != NULL) {
return TSDB_CODE_SUCCESS;
}
pInfo->pPrevRow = taosArrayInit(4, sizeof(SGroupKeys));
if (pInfo->pPrevRow == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);
SGroupKeys key = {0};
key.bytes = pColInfo->info.bytes;
key.type = pColInfo->info.type;
key.isNull = false;
key.pData = taosMemoryCalloc(1, pColInfo->info.bytes);
taosArrayPush(pInfo->pPrevRow, &key);
}
pInfo->isPrevRowSet = false;
return TSDB_CODE_SUCCESS;
}
static int32_t initNextRowsKeeper(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) {
if (pInfo->pNextRow != NULL) {
return TSDB_CODE_SUCCESS;
}
pInfo->pNextRow = taosArrayInit(4, sizeof(SGroupKeys));
if (pInfo->pNextRow == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);
SGroupKeys key = {0};
key.bytes = pColInfo->info.bytes;
key.type = pColInfo->info.type;
key.isNull = false;
key.pData = taosMemoryCalloc(1, pColInfo->info.bytes);
taosArrayPush(pInfo->pNextRow, &key);
}
pInfo->isNextRowSet = false;
return TSDB_CODE_SUCCESS;
}
static int32_t initFillLinearInfo(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) {
if (pInfo->pLinearInfo != NULL) {
return TSDB_CODE_SUCCESS;
}
pInfo->pLinearInfo = taosArrayInit(4, sizeof(SFillLinearInfo));
if (pInfo->pLinearInfo == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i);
SFillLinearInfo linearInfo = {0};
linearInfo.start.key = INT64_MIN;
linearInfo.end.key = INT64_MIN;
linearInfo.start.val = taosMemoryCalloc(1, pColInfo->info.bytes);
linearInfo.end.val = taosMemoryCalloc(1, pColInfo->info.bytes);
linearInfo.isStartSet = false;
linearInfo.isEndSet = false;
linearInfo.type = pColInfo->info.type;
linearInfo.bytes = pColInfo->info.bytes;
taosArrayPush(pInfo->pLinearInfo, &linearInfo);
}
return TSDB_CODE_SUCCESS;
}
static int32_t initKeeperInfo(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) {
int32_t code;
code = initPrevRowsKeeper(pInfo, pBlock);
if (code != TSDB_CODE_SUCCESS) {
return TSDB_CODE_FAILED;
}
code = initNextRowsKeeper(pInfo, pBlock);
if (code != TSDB_CODE_SUCCESS) {
return TSDB_CODE_FAILED;
}
code = initFillLinearInfo(pInfo, pBlock);
if (code != TSDB_CODE_SUCCESS) {
return TSDB_CODE_FAILED;
}
return TSDB_CODE_SUCCESS;
}
static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) {
if (pOperator->status == OP_EXEC_DONE) {
return NULL;
}
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
STimeSliceOperatorInfo* pSliceInfo = pOperator->info;
SSDataBlock* pResBlock = pSliceInfo->pRes;
SExprSupp* pSup = &pOperator->exprSupp;
int32_t order = TSDB_ORDER_ASC;
SInterval* pInterval = &pSliceInfo->interval;
SOperatorInfo* downstream = pOperator->pDownstream[0];
blockDataCleanup(pResBlock);
while (1) {
SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
if (pBlock == NULL) {
break;
}
int32_t code = initKeeperInfo(pSliceInfo, pBlock);
if (code != TSDB_CODE_SUCCESS) {
T_LONG_JMP(pTaskInfo->env, code);
}
// the pDataBlock are always the same one, no need to call this again
setInputDataBlock(pSup, pBlock, order, MAIN_SCAN, true);
SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, pSliceInfo->tsCol.slotId);
for (int32_t i = 0; i < pBlock->info.rows; ++i) {
int64_t ts = *(int64_t*)colDataGetData(pTsCol, i);
if (pSliceInfo->current > pSliceInfo->win.ekey) {
setOperatorCompleted(pOperator);
break;
}
if (ts == pSliceInfo->current) {
addCurrentRowToResult(pSliceInfo, &pOperator->exprSupp, pResBlock, pBlock, i);
doKeepPrevRows(pSliceInfo, pBlock, i);
doKeepLinearInfo(pSliceInfo, pBlock, i);
pSliceInfo->current =
taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
if (pSliceInfo->current > pSliceInfo->win.ekey) {
setOperatorCompleted(pOperator);
break;
}
} else if (ts < pSliceInfo->current) {
// in case of interpolation window starts and ends between two datapoints, fill(prev) need to interpolate
doKeepPrevRows(pSliceInfo, pBlock, i);
doKeepLinearInfo(pSliceInfo, pBlock, i);
if (i < pBlock->info.rows - 1) {
// in case of interpolation window starts and ends between two datapoints, fill(next) need to interpolate
doKeepNextRows(pSliceInfo, pBlock, i + 1);
int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1);
if (nextTs > pSliceInfo->current) {
while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) {
if (!genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, false) && pSliceInfo->fillType == TSDB_FILL_LINEAR) {
break;
} else {
pSliceInfo->current =
taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
}
}
if (pSliceInfo->current > pSliceInfo->win.ekey) {
setOperatorCompleted(pOperator);
break;
}
} else {
// ignore current row, and do nothing
}
} else { // it is the last row of current block
doKeepPrevRows(pSliceInfo, pBlock, i);
}
} else { // ts > pSliceInfo->current
// in case of interpolation window starts and ends between two datapoints, fill(next) need to interpolate
doKeepNextRows(pSliceInfo, pBlock, i);
doKeepLinearInfo(pSliceInfo, pBlock, i);
while (pSliceInfo->current < ts && pSliceInfo->current <= pSliceInfo->win.ekey) {
if (!genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, true) && pSliceInfo->fillType == TSDB_FILL_LINEAR) {
break;
} else {
pSliceInfo->current =
taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
}
}
// add current row if timestamp match
if (ts == pSliceInfo->current && pSliceInfo->current <= pSliceInfo->win.ekey) {
addCurrentRowToResult(pSliceInfo, &pOperator->exprSupp, pResBlock, pBlock, i);
doKeepPrevRows(pSliceInfo, pBlock, i);
pSliceInfo->current =
taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
}
if (pSliceInfo->current > pSliceInfo->win.ekey) {
setOperatorCompleted(pOperator);
break;
}
}
}
}
// check if need to interpolate after last datablock
// except for fill(next), fill(linear)
while (pSliceInfo->current <= pSliceInfo->win.ekey && pSliceInfo->fillType != TSDB_FILL_NEXT &&
pSliceInfo->fillType != TSDB_FILL_LINEAR) {
genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pResBlock, false);
pSliceInfo->current =
taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision);
}
// restore the value
setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED);
if (pResBlock->info.rows == 0) {
pOperator->status = OP_EXEC_DONE;
}
return pResBlock->info.rows == 0 ? NULL : pResBlock;
}
SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo) {
STimeSliceOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STimeSliceOperatorInfo));
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
if (pOperator == NULL || pInfo == NULL) {
goto _error;
}
SInterpFuncPhysiNode* pInterpPhyNode = (SInterpFuncPhysiNode*)pPhyNode;
SExprSupp* pSup = &pOperator->exprSupp;
int32_t numOfExprs = 0;
SExprInfo* pExprInfo = createExprInfo(pInterpPhyNode->pFuncs, NULL, &numOfExprs);
int32_t code = initExprSupp(pSup, pExprInfo, numOfExprs);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
if (pInterpPhyNode->pExprs != NULL) {
int32_t num = 0;
SExprInfo* pScalarExprInfo = createExprInfo(pInterpPhyNode->pExprs, NULL, &num);
code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, num);
if (code != TSDB_CODE_SUCCESS) {
goto _error;
}
}
pInfo->tsCol = extractColumnFromColumnNode((SColumnNode*)pInterpPhyNode->pTimeSeries);
pInfo->fillType = convertFillType(pInterpPhyNode->fillMode);
initResultSizeInfo(&pOperator->resultInfo, 4096);
pInfo->pFillColInfo = createFillColInfo(pExprInfo, numOfExprs, NULL, 0, (SNodeListNode*)pInterpPhyNode->pFillValues);
pInfo->pLinearInfo = NULL;
pInfo->pRes = createDataBlockFromDescNode(pPhyNode->pOutputDataBlockDesc);
pInfo->win = pInterpPhyNode->timeRange;
pInfo->interval.interval = pInterpPhyNode->interval;
pInfo->current = pInfo->win.skey;
if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) {
STableScanInfo* pScanInfo = (STableScanInfo*)downstream->info;
pScanInfo->base.cond.twindows = pInfo->win;
pScanInfo->base.cond.type = TIMEWINDOW_RANGE_EXTERNAL;
}
setOperatorInfo(pOperator, "TimeSliceOperator", QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC, false, OP_NOT_OPENED, pInfo,
pTaskInfo);
pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTimeslice, NULL, destroyTimeSliceOperatorInfo, NULL);
blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity);
code = appendDownstream(pOperator, &downstream, 1);
return pOperator;
_error:
taosMemoryFree(pInfo);
taosMemoryFree(pOperator);
pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
void destroyTimeSliceOperatorInfo(void* param) {
STimeSliceOperatorInfo* pInfo = (STimeSliceOperatorInfo*)param;
pInfo->pRes = blockDataDestroy(pInfo->pRes);
for (int32_t i = 0; i < taosArrayGetSize(pInfo->pPrevRow); ++i) {
SGroupKeys* pKey = taosArrayGet(pInfo->pPrevRow, i);
taosMemoryFree(pKey->pData);
}
taosArrayDestroy(pInfo->pPrevRow);
for (int32_t i = 0; i < taosArrayGetSize(pInfo->pNextRow); ++i) {
SGroupKeys* pKey = taosArrayGet(pInfo->pNextRow, i);
taosMemoryFree(pKey->pData);
}
taosArrayDestroy(pInfo->pNextRow);
for (int32_t i = 0; i < taosArrayGetSize(pInfo->pLinearInfo); ++i) {
SFillLinearInfo* pKey = taosArrayGet(pInfo->pLinearInfo, i);
taosMemoryFree(pKey->start.val);
taosMemoryFree(pKey->end.val);
}
taosArrayDestroy(pInfo->pLinearInfo);
taosMemoryFree(pInfo->pFillColInfo);
taosMemoryFreeClear(param);
}

701
source/libs/executor/src/timewindowoperator.c
View File

File diff suppressed because it is too large Load Diff

6
source/libs/executor/src/tsort.c
View File

@ -417,8 +417,8 @@ int32_t msortComparFn(const void* pLeft, const void* pRight, void* param) {
SSDataBlock* pRightBlock = pRightSource->src.pBlock;
if (pParam->cmpGroupId) {
if (pLeftBlock->info.groupId != pRightBlock->info.groupId) {
return pLeftBlock->info.groupId < pRightBlock->info.groupId ? -1 : 1;
if (pLeftBlock->info.id.groupId != pRightBlock->info.id.groupId) {
return pLeftBlock->info.id.groupId < pRightBlock->info.id.groupId ? -1 : 1;
}
}
@ -826,7 +826,7 @@ void* tsortGetValue(STupleHandle* pVHandle, int32_t colIndex) {
}
}
uint64_t tsortGetGroupId(STupleHandle* pVHandle) { return pVHandle->pBlock->info.groupId; }
uint64_t tsortGetGroupId(STupleHandle* pVHandle) { return pVHandle->pBlock->info.id.groupId; }
SSortExecInfo tsortGetSortExecInfo(SSortHandle* pHandle) {
SSortExecInfo info = {0};

893
source/libs/function/src/builtinsimpl.c
View File

@ -729,375 +729,6 @@ bool getSumFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
return true;
}
//int32_t getAvgInfoSize() { return (int32_t)sizeof(SAvgRes); }
//
//bool getAvgFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
// pEnv->calcMemSize = sizeof(SAvgRes);
// return true;
//}
//
//bool avgFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) {
// if (!functionSetup(pCtx, pResultInfo)) {
// return false;
// }
//
// SAvgRes* pRes = GET_ROWCELL_INTERBUF(pResultInfo);
// memset(pRes, 0, sizeof(SAvgRes));
// return true;
//}
//int32_t avgFunction(SqlFunctionCtx* pCtx) {
// int32_t numOfElem = 0;
//
// SInputColumnInfoData* pInput = &pCtx->input;
// SColumnDataAgg* pAgg = pInput->pColumnDataAgg[0];
// int32_t type = pInput->pData[0]->info.type;
//
// SAvgRes* pAvgRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
// pAvgRes->type = type;
//
// // computing based on the true data block
// SColumnInfoData* pCol = pInput->pData[0];
//
// int32_t start = pInput->startRowIndex;
// int32_t numOfRows = pInput->numOfRows;
//
// if (IS_NULL_TYPE(type)) {
// numOfElem = 0;
// goto _avg_over;
// }
//
// if (pInput->colDataSMAIsSet) {
// numOfElem = numOfRows - pAgg->numOfNull;
// ASSERT(numOfElem >= 0);
//
// pAvgRes->count += numOfElem;
// if (IS_SIGNED_NUMERIC_TYPE(type)) {
// pAvgRes->sum.isum += pAgg->sum;
// } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) {
// pAvgRes->sum.usum += pAgg->sum;
// } else if (IS_FLOAT_TYPE(type)) {
// pAvgRes->sum.dsum += GET_DOUBLE_VAL((const char*)&(pAgg->sum));
// }
// } else { // computing based on the true data block
// switch (type) {
// case TSDB_DATA_TYPE_TINYINT: {
// int8_t* plist = (int8_t*)pCol->pData;
// for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
// if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
// continue;
// }
//
// numOfElem += 1;
// pAvgRes->count += 1;
// pAvgRes->sum.isum += plist[i];
// }
//
// break;
// }
//
// case TSDB_DATA_TYPE_SMALLINT: {
// int16_t* plist = (int16_t*)pCol->pData;
// for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
// if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
// continue;
// }
//
// numOfElem += 1;
// pAvgRes->count += 1;
// pAvgRes->sum.isum += plist[i];
// }
// break;
// }
//
// case TSDB_DATA_TYPE_INT: {
// int32_t* plist = (int32_t*)pCol->pData;
// for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
// if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
// continue;
// }
//
// numOfElem += 1;
// pAvgRes->count += 1;
// pAvgRes->sum.isum += plist[i];
// }
//
// break;
// }
//
// case TSDB_DATA_TYPE_BIGINT: {
// int64_t* plist = (int64_t*)pCol->pData;
// for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
// if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
// continue;
// }
//
// numOfElem += 1;
// pAvgRes->count += 1;
// pAvgRes->sum.isum += plist[i];
// }
// break;
// }
//
// case TSDB_DATA_TYPE_UTINYINT: {
// uint8_t* plist = (uint8_t*)pCol->pData;
// for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
// if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
// continue;
// }
//
// numOfElem += 1;
// pAvgRes->count += 1;
// pAvgRes->sum.usum += plist[i];
// }
//
// break;
// }
//
// case TSDB_DATA_TYPE_USMALLINT: {
// uint16_t* plist = (uint16_t*)pCol->pData;
// for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
// if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
// continue;
// }
//
// numOfElem += 1;
// pAvgRes->count += 1;
// pAvgRes->sum.usum += plist[i];
// }
// break;
// }
//
// case TSDB_DATA_TYPE_UINT: {
// uint32_t* plist = (uint32_t*)pCol->pData;
// for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
// if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
// continue;
// }
//
// numOfElem += 1;
// pAvgRes->count += 1;
// pAvgRes->sum.usum += plist[i];
// }
//
// break;
// }
//
// case TSDB_DATA_TYPE_UBIGINT: {
// uint64_t* plist = (uint64_t*)pCol->pData;
// for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
// if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
// continue;
// }
//
// numOfElem += 1;
// pAvgRes->count += 1;
// pAvgRes->sum.usum += plist[i];
// }
// break;
// }
//
// case TSDB_DATA_TYPE_FLOAT: {
// float* plist = (float*)pCol->pData;
//// float val = 0;
// for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
// if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
// continue;
// }
//
// numOfElem += 1;
// pAvgRes->count += 1;
// pAvgRes->sum.dsum += plist[i];
// }
//// pAvgRes->sum.dsum = val;
// break;
// }
//
// case TSDB_DATA_TYPE_DOUBLE: {
// double* plist = (double*)pCol->pData;
// for (int32_t i = start; i < numOfRows + pInput->startRowIndex; ++i) {
// if (pCol->hasNull && colDataIsNull_f(pCol->nullbitmap, i)) {
// continue;
// }
//
// numOfElem += 1;
// pAvgRes->count += 1;
// pAvgRes->sum.dsum += plist[i];
// }
// break;
// }
//
// default:
// break;
// }
// }
//
//_avg_over:
// // data in the check operation are all null, not output
// SET_VAL(GET_RES_INFO(pCtx), numOfElem, 1);
// return TSDB_CODE_SUCCESS;
//}
//static void avgTransferInfo(SAvgRes* pInput, SAvgRes* pOutput) {
// pOutput->type = pInput->type;
// if (IS_SIGNED_NUMERIC_TYPE(pOutput->type)) {
// pOutput->sum.isum += pInput->sum.isum;
// } else if (IS_UNSIGNED_NUMERIC_TYPE(pOutput->type)) {
// pOutput->sum.usum += pInput->sum.usum;
// } else {
// pOutput->sum.dsum += pInput->sum.dsum;
// }
//
// pOutput->count += pInput->count;
//
// return;
//}
//
//int32_t avgFunctionMerge(SqlFunctionCtx* pCtx) {
// SInputColumnInfoData* pInput = &pCtx->input;
// SColumnInfoData* pCol = pInput->pData[0];
// ASSERT(pCol->info.type == TSDB_DATA_TYPE_BINARY);
//
// SAvgRes* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
//
// int32_t start = pInput->startRowIndex;
//
// for (int32_t i = start; i < start + pInput->numOfRows; ++i) {
// char* data = colDataGetData(pCol, i);
// SAvgRes* pInputInfo = (SAvgRes*)varDataVal(data);
// avgTransferInfo(pInputInfo, pInfo);
// }
//
// SET_VAL(GET_RES_INFO(pCtx), 1, 1);
//
// return TSDB_CODE_SUCCESS;
//}
//
//int32_t avgInvertFunction(SqlFunctionCtx* pCtx) {
// int32_t numOfElem = 0;
//
// // Only the pre-computing information loaded and actual data does not loaded
// SInputColumnInfoData* pInput = &pCtx->input;
// int32_t type = pInput->pData[0]->info.type;
//
// SAvgRes* pAvgRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
//
// // computing based on the true data block
// SColumnInfoData* pCol = pInput->pData[0];
//
// int32_t start = pInput->startRowIndex;
// int32_t numOfRows = pInput->numOfRows;
//
// switch (type) {
// case TSDB_DATA_TYPE_TINYINT: {
// LIST_AVG_N(pAvgRes->sum.isum, int8_t);
// break;
// }
// case TSDB_DATA_TYPE_SMALLINT: {
// LIST_AVG_N(pAvgRes->sum.isum, int16_t);
// break;
// }
// case TSDB_DATA_TYPE_INT: {
// LIST_AVG_N(pAvgRes->sum.isum, int32_t);
// break;
// }
// case TSDB_DATA_TYPE_BIGINT: {
// LIST_AVG_N(pAvgRes->sum.isum, int64_t);
// break;
// }
// case TSDB_DATA_TYPE_UTINYINT: {
// LIST_AVG_N(pAvgRes->sum.usum, uint8_t);
// break;
// }
// case TSDB_DATA_TYPE_USMALLINT: {
// LIST_AVG_N(pAvgRes->sum.usum, uint16_t);
// break;
// }
// case TSDB_DATA_TYPE_UINT: {
// LIST_AVG_N(pAvgRes->sum.usum, uint32_t);
// break;
// }
// case TSDB_DATA_TYPE_UBIGINT: {
// LIST_AVG_N(pAvgRes->sum.usum, uint64_t);
// break;
// }
// case TSDB_DATA_TYPE_FLOAT: {
// LIST_AVG_N(pAvgRes->sum.dsum, float);
// break;
// }
// case TSDB_DATA_TYPE_DOUBLE: {
// LIST_AVG_N(pAvgRes->sum.dsum, double);
// break;
// }
// default:
// break;
// }
//
// // data in the check operation are all null, not output
// SET_VAL(GET_RES_INFO(pCtx), numOfElem, 1);
// return TSDB_CODE_SUCCESS;
//}
//
//int32_t avgCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) {
// SResultRowEntryInfo* pDResInfo = GET_RES_INFO(pDestCtx);
// SAvgRes* pDBuf = GET_ROWCELL_INTERBUF(pDResInfo);
//
// SResultRowEntryInfo* pSResInfo = GET_RES_INFO(pSourceCtx);
// SAvgRes* pSBuf = GET_ROWCELL_INTERBUF(pSResInfo);
// int16_t type = pDBuf->type == TSDB_DATA_TYPE_NULL ? pSBuf->type : pDBuf->type;
//
// if (IS_SIGNED_NUMERIC_TYPE(type)) {
// pDBuf->sum.isum += pSBuf->sum.isum;
// } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) {
// pDBuf->sum.usum += pSBuf->sum.usum;
// } else {
// pDBuf->sum.dsum += pSBuf->sum.dsum;
// }
// pDBuf->count += pSBuf->count;
//
// return TSDB_CODE_SUCCESS;
//}
//
//int32_t avgFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
// SInputColumnInfoData* pInput = &pCtx->input;
//
// SAvgRes* pAvgRes = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
// int32_t type = pAvgRes->type;
//
// if (IS_SIGNED_NUMERIC_TYPE(type)) {
// pAvgRes->result = pAvgRes->sum.isum / ((double)pAvgRes->count);
// } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) {
// pAvgRes->result = pAvgRes->sum.usum / ((double)pAvgRes->count);
// } else {
// pAvgRes->result = pAvgRes->sum.dsum / ((double)pAvgRes->count);
// }
//
// // check for overflow
// if (isinf(pAvgRes->result) || isnan(pAvgRes->result)) {
// GET_RES_INFO(pCtx)->numOfRes = 0;
// }
//
// return functionFinalize(pCtx, pBlock);
//}
//
//int32_t avgPartialFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) {
// SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
// SAvgRes* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
// int32_t resultBytes = getAvgInfoSize();
// char* res = taosMemoryCalloc(resultBytes + VARSTR_HEADER_SIZE, sizeof(char));
//
// memcpy(varDataVal(res), pInfo, resultBytes);
// varDataSetLen(res, resultBytes);
//
// int32_t slotId = pCtx->pExpr->base.resSchema.slotId;
// SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId);
//
// colDataAppend(pCol, pBlock->info.rows, res, false);
//
// taosMemoryFree(res);
// return pResInfo->numOfRes;
//}
EFuncDataRequired statisDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWindow) {
return FUNC_DATA_REQUIRED_SMA_LOAD;
}
@ -1121,530 +752,6 @@ bool getMinmaxFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) {
return true;
}
//static STuplePos saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock,
// const STupleKey* pKey);
//static int32_t updateTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos);
//static const char* loadTupleData(SqlFunctionCtx* pCtx, const STuplePos* pPos);
//static int32_t findRowIndex(int32_t start, int32_t num, SColumnInfoData* pCol, const char* tval) {
// // the data is loaded, not only the block SMA value
// for (int32_t i = start; i < num + start; ++i) {
// char* p = colDataGetData(pCol, i);
// if (memcmp((void*)tval, p, pCol->info.bytes) == 0) {
// return i;
// }
// }
//
// // if reach here means real data of block SMA is not set in pCtx->input.
// return -1;
//}
//int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) {
// int32_t numOfElems = 0;
//
// SInputColumnInfoData* pInput = &pCtx->input;
// SColumnDataAgg* pAgg = pInput->pColumnDataAgg[0];
//
// SColumnInfoData* pCol = pInput->pData[0];
// int32_t type = pCol->info.type;
//
// SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx);
// SMinmaxResInfo* pBuf = GET_ROWCELL_INTERBUF(pResInfo);
// pBuf->type = type;
//
// if (IS_NULL_TYPE(type)) {
// numOfElems = 0;
// goto _min_max_over;
// }
//
// // data in current data block are qualified to the query
// if (pInput->colDataSMAIsSet) {
// numOfElems = pInput->numOfRows - pAgg->numOfNull;
// ASSERT(pInput->numOfRows == pInput->totalRows && numOfElems >= 0);
// if (numOfElems == 0) {
// return numOfElems;
// }
//
// void* tval = NULL;
// int16_t index = 0;
//
// if (isMinFunc) {
// tval = &pInput->pColumnDataAgg[0]->min;
// } else {
// tval = &pInput->pColumnDataAgg[0]->max;
// }
//
// if (!pBuf->assign) {
// pBuf->v = *(int64_t*)tval;
// if (pCtx->subsidiaries.num > 0) {
// index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval);
// if (index >= 0) {
// pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL);
// }
// }
// } else {
// if (IS_SIGNED_NUMERIC_TYPE(type)) {
// int64_t prev = 0;
// GET_TYPED_DATA(prev, int64_t, type, &pBuf->v);
//
// int64_t val = GET_INT64_VAL(tval);
// if ((prev < val) ^ isMinFunc) {
// *(int64_t*)&pBuf->v = val;
// if (pCtx->subsidiaries.num > 0) {
// index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval);
// if (index >= 0) {
// pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL);
// }
// }
// }
// } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) {
// uint64_t prev = 0;
// GET_TYPED_DATA(prev, uint64_t, type, &pBuf->v);
//
// uint64_t val = GET_UINT64_VAL(tval);
// if ((prev < val) ^ isMinFunc) {
// *(uint64_t*)&pBuf->v = val;
// if (pCtx->subsidiaries.num > 0) {
// index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval);
// if (index >= 0) {
// pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL);
// }
// }
// }
// } else if (type == TSDB_DATA_TYPE_DOUBLE) {
// double prev = 0;
// GET_TYPED_DATA(prev, double, type, &pBuf->v);
//
// double val = GET_DOUBLE_VAL(tval);
// if ((prev < val) ^ isMinFunc) {
// *(double*)&pBuf->v = val;
// if (pCtx->subsidiaries.num > 0) {
// index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval);
// if (index >= 0) {
// pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL);
// }
// }
// }
// } else if (type == TSDB_DATA_TYPE_FLOAT) {
// float prev = 0;
// GET_TYPED_DATA(prev, float, type, &pBuf->v);
//
// float val = GET_DOUBLE_VAL(tval);
// if ((prev < val) ^ isMinFunc) {
// *(float*)&pBuf->v = val;
// }
//
// if (pCtx->subsidiaries.num > 0) {
// index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval);
// if (index >= 0) {
// pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL);
// }
// }
// }
// }
//
// pBuf->assign = true;
// return numOfElems;
// }
//
// int32_t start = pInput->startRowIndex;
// int32_t numOfRows = pInput->numOfRows;
//
// if (IS_SIGNED_NUMERIC_TYPE(type) || type == TSDB_DATA_TYPE_BOOL) {
// if (type == TSDB_DATA_TYPE_TINYINT || type == TSDB_DATA_TYPE_BOOL) {
// int8_t* pData = (int8_t*)pCol->pData;
// int8_t* val = (int8_t*)&pBuf->v;
//
// for (int32_t i = start; i < start + numOfRows; ++i) {
// if ((pCol->hasNull) && colDataIsNull_f(pCol->nullbitmap, i)) {
// continue;
// }
//
// if (!pBuf->assign) {
// *val = pData[i];
// if (pCtx->subsidiaries.num > 0) {
// pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock, NULL);
// }
// pBuf->assign = true;
// } else {
// // ignore the equivalent data value
// // NOTE: An faster version to avoid one additional comparison with FPU.
// if (isMinFunc) { // min
// if (*val > pData[i]) {
// *val = pData[i];
// if (pCtx->subsidiaries.num > 0) {
// updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos);
// }
// }
// } else { // max
// if (*val < pData[i]) {
// *val = pData[i];
// if (pCtx->subsidiaries.num > 0) {
// updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos);
// }
// }
// }
// }
//
// numOfElems += 1;
// }
// } else if (type == TSDB_DATA_TYPE_SMALLINT) {
// int16_t* pData = (int16_t*)pCol->pData;
// int16_t* val = (int16_t*)&pBuf->v;
//
// for (int32_t i = start; i < start + numOfRows; ++i) {
// if ((pCol->hasNull) && colDataIsNull_f(pCol->nullbitmap, i)) {
// continue;
// }
//
// if (!pBuf->assign) {
// *val = pData[i];
// if (pCtx->subsidiaries.num > 0) {
// pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock, NULL);
// }
// pBuf->assign = true;
// } else {
// // ignore the equivalent data value
// // NOTE: An faster version to avoid one additional comparison with FPU.
// if (isMinFunc) { // min
// if (*val > pData[i]) {
// *val = pData[i];
// if (pCtx->subsidiaries.num > 0) {
// updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos);
// }
// }
// } else { // max
// if (*val < pData[i]) {
// *val = pData[i];
// if (pCtx->subsidiaries.num > 0) {
// updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos);
// }
// }
// }
// }
//
// numOfElems += 1;
// }
// } else if (type == TSDB_DATA_TYPE_INT) {
// int32_t* pData = (int32_t*)pCol->pData;
// int32_t* val = (int32_t*)&pBuf->v;
//
// for (int32_t i = start; i < start + numOfRows; ++i) {
// if ((pCol->hasNull) && colDataIsNull_f(pCol->nullbitmap, i)) {
// continue;
// }
//
// if (!pBuf->assign) {
// *val = pData[i];
// if (pCtx->subsidiaries.num > 0) {
// pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock, NULL);
// }
// pBuf->assign = true;
// } else {
// // ignore the equivalent data value
// // NOTE: An faster version to avoid one additional comparison with FPU.
// if (isMinFunc) { // min
// if (*val > pData[i]) {
// *val = pData[i];
// if (pCtx->subsidiaries.num > 0) {
// updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos);
// }
// }
// } else { // max
// if (*val < pData[i]) {
// *val = pData[i];
// if (pCtx->subsidiaries.num > 0) {
// updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos);
// }
// }
// }
// }
//
// numOfElems += 1;
// }
// } else if (type == TSDB_DATA_TYPE_BIGINT) {
// int64_t* pData = (int64_t*)pCol->pData;
// int64_t* val = (int64_t*)&pBuf->v;
//
// for (int32_t i = start; i < start + numOfRows; ++i) {
// if ((pCol->hasNull) && colDataIsNull_f(pCol->nullbitmap, i)) {
// continue;
// }
//
// if (!pBuf->assign) {
// *val = pData[i];
// if (pCtx->subsidiaries.num > 0) {
// pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock, NULL);
// }
// pBuf->assign = true;
// } else {
// // ignore the equivalent data value
// // NOTE: An faster version to avoid one additional comparison with FPU.
// if (isMinFunc) { // min
// if (*val > pData[i]) {
// *val = pData[i];
// if (pCtx->subsidiaries.num > 0) {
// updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos);
// }
// }
// } else { // max
// if (*val < pData[i]) {
// *val = pData[i];
// if (pCtx->subsidiaries.num > 0) {
// updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos);
// }
// }
// }
// }
//
// numOfElems += 1;
// }
// }
// } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) {
// if (type == TSDB_DATA_TYPE_UTINYINT) {
// uint8_t* pData = (uint8_t*)pCol->pData;
// uint8_t* val = (uint8_t*)&pBuf->v;
//
// for (int32_t i = start; i < start + numOfRows; ++i) {
// if ((pCol->hasNull) && colDataIsNull_f(pCol->nullbitmap, i)) {
// continue;
// }
//
// if (!pBuf->assign) {
// *val = pData[i];
// if (pCtx->subsidiaries.num > 0) {
// pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock, NULL);
// }
// pBuf->assign = true;
// } else {
// // ignore the equivalent data value
// // NOTE: An faster version to avoid one additional comparison with FPU.
// if (isMinFunc) { // min
// if (*val > pData[i]) {
// *val = pData[i];
// if (pCtx->subsidiaries.num > 0) {
// updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos);
// }
// }
// } else { // max
// if (*val < pData[i]) {
// *val = pData[i];
// if (pCtx->subsidiaries.num > 0) {
// updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos);
// }
// }
// }
// }
//
// numOfElems += 1;
// }
// } else if (type == TSDB_DATA_TYPE_USMALLINT) {
// uint16_t* pData = (uint16_t*)pCol->pData;
// uint16_t* val = (uint16_t*)&pBuf->v;
//
// for (int32_t i = start; i < start + numOfRows; ++i) {
// if ((pCol->hasNull) && colDataIsNull_f(pCol->nullbitmap, i)) {
// continue;
// }
//
// if (!pBuf->assign) {
// *val = pData[i];
// if (pCtx->subsidiaries.num > 0) {
// pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock, NULL);
// }
// pBuf->assign = true;
// } else {
// // ignore the equivalent data value
// // NOTE: An faster version to avoid one additional comparison with FPU.
// if (isMinFunc) { // min
// if (*val > pData[i]) {
// *val = pData[i];
// if (pCtx->subsidiaries.num > 0) {
// updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos);
// }
// }
// } else { // max
// if (*val < pData[i]) {
// *val = pData[i];
// if (pCtx->subsidiaries.num > 0) {
// updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos);
// }
// }
// }
// }
//
// numOfElems += 1;
// }
// } else if (type == TSDB_DATA_TYPE_UINT) {
// uint32_t* pData = (uint32_t*)pCol->pData;
// uint32_t* val = (uint32_t*)&pBuf->v;
//
// for (int32_t i = start; i < start + numOfRows; ++i) {
// if ((pCol->hasNull) && colDataIsNull_f(pCol->nullbitmap, i)) {
// continue;
// }
//
// if (!pBuf->assign) {
// *val = pData[i];
// if (pCtx->subsidiaries.num > 0) {
// pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock, NULL);
// }
// pBuf->assign = true;
// } else {
// // ignore the equivalent data value
// // NOTE: An faster version to avoid one additional comparison with FPU.
// if (isMinFunc) { // min
// if (*val > pData[i]) {
// *val = pData[i];
// if (pCtx->subsidiaries.num > 0) {
// updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos);
// }
// }
// } else { // max
// if (*val < pData[i]) {
// *val = pData[i];
// if (pCtx->subsidiaries.num > 0) {
// updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos);
// }
// }
// }
// }
//
// numOfElems += 1;
// }
// } else if (type == TSDB_DATA_TYPE_UBIGINT) {
// uint64_t* pData = (uint64_t*)pCol->pData;
// uint64_t* val = (uint64_t*)&pBuf->v;
//
// for (int32_t i = start; i < start + numOfRows; ++i) {
// if ((pCol->hasNull) && colDataIsNull_f(pCol->nullbitmap, i)) {
// continue;
// }
//
// if (!pBuf->assign) {
// *val = pData[i];
// if (pCtx->subsidiaries.num > 0) {
// pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock, NULL);
// }
// pBuf->assign = true;
// } else {
// // ignore the equivalent data value
// // NOTE: An faster version to avoid one additional comparison with FPU.
// if (isMinFunc) { // min
// if (*val > pData[i]) {
// *val = pData[i];
// if (pCtx->subsidiaries.num > 0) {
// updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos);
// }
// }
// } else { // max
// if (*val < pData[i]) {
// *val = pData[i];
// if (pCtx->subsidiaries.num > 0) {
// updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos);
// }
// }
// }
// }
//
// numOfElems += 1;
// }
// }
// } else if (type == TSDB_DATA_TYPE_DOUBLE) {
// double* pData = (double*)pCol->pData;
// double* val = (double*)&pBuf->v;
//
// for (int32_t i = start; i < start + numOfRows; ++i) {
// if ((pCol->hasNull) && colDataIsNull_f(pCol->nullbitmap, i)) {
// continue;
// }
//
// if (!pBuf->assign) {
// *val = pData[i];
// if (pCtx->subsidiaries.num > 0) {
// pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock, NULL);
// }
// pBuf->assign = true;
// } else {
// // ignore the equivalent data value
// // NOTE: An faster version to avoid one additional comparison with FPU.
// if (isMinFunc) { // min
// if (*val > pData[i]) {
// *val = pData[i];
// if (pCtx->subsidiaries.num > 0) {
// updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos);
// }
// }
// } else { // max
// if (*val < pData[i]) {
// *val = pData[i];
// if (pCtx->subsidiaries.num > 0) {
// updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos);
// }
// }
// }
// }
//
// numOfElems += 1;
// }
// } else if (type == TSDB_DATA_TYPE_FLOAT) {
// float* pData = (float*)pCol->pData;
// float* val = (float*)&pBuf->v;
//
// for (int32_t i = start; i < start + numOfRows; ++i) {
// if ((pCol->hasNull) && colDataIsNull_f(pCol->nullbitmap, i)) {
// continue;
// }
//
// if (!pBuf->assign) {
// *val = pData[i];
// if (pCtx->subsidiaries.num > 0) {
// pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock, NULL);
// }
// pBuf->assign = true;
// } else {
//#if 0
// if ((*val) == pData[i]) {
// continue;
// }
//
// if ((*val < pData[i]) ^ isMinFunc) {
// *val = pData[i];
// if (pCtx->subsidiaries.num > 0) {
// updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos);
// }
// }
//#endif
// // NOTE: An faster version to avoid one additional comparison with FPU.
// if (isMinFunc) { // min
// if (*val > pData[i]) {
// *val = pData[i];
// if (pCtx->subsidiaries.num > 0) {
// updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos);
// }
// }
// } else { // max
// if (*val < pData[i]) {
// *val = pData[i];
// if (pCtx->subsidiaries.num > 0) {
// updateTupleData(pCtx, i, pCtx->pSrcBlock, &pBuf->tuplePos);
// }
// }
// }
// }
//
// numOfElems += 1;
// }
// }
//
//_min_max_over:
// if (numOfElems == 0 && pCtx->subsidiaries.num > 0 && !pBuf->nullTupleSaved) {
// pBuf->nullTuplePos = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock, NULL);
// pBuf->nullTupleSaved = true;
// }
// return numOfElems;
//}
int32_t minFunction(SqlFunctionCtx* pCtx) {
int32_t numOfElems = doMinMaxHelper(pCtx, 1);
SET_VAL(GET_RES_INFO(pCtx), numOfElems, 1);

2
source/libs/function/src/tudf.c
View File

@ -1096,7 +1096,7 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) {
SSDataBlock *pTempBlock = createDataBlock();
pTempBlock->info.rows = pInput->totalRows;
pTempBlock->info.uid = pInput->uid;
pTempBlock->info.id.uid = pInput->uid;
for (int32_t i = 0; i < numOfCols; ++i) {
blockDataAppendColInfo(pTempBlock, pInput->pData[i]);
}

2
source/libs/index/test/index_executor_tests.cpp
View File

@ -78,7 +78,7 @@ void sifAppendReservedSlot(SArray *pBlockList, int16_t *dataBlockId, int16_t *sl
blockDataEnsureCapacity(res, rows);
*dataBlockId = taosArrayGetSize(pBlockList) - 1;
res->info.blockId = *dataBlockId;
res->info.id.blockId = *dataBlockId;
*slotId = 0;
} else {
SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(pBlockList);

2
source/libs/parser/inc/sql.y
View File

@ -654,7 +654,7 @@ cgroup_name(A) ::= NK_ID(B).
/************************************************ expression **********************************************************/
expr_or_subquery(A) ::= expression(B). { A = B; }
expr_or_subquery(A) ::= subquery(B). { A = B; }
//expr_or_subquery(A) ::= subquery(B). { A = createTempTableNode(pCxt, releaseRawExprNode(pCxt, B), NULL); }
expression(A) ::= literal(B). { A = B; }
expression(A) ::= pseudo_column(B). { A = B; }

32
source/libs/parser/src/parCalcConst.c
View File

@ -372,14 +372,42 @@ static bool isEmptyResultQuery(SNode* pStmt) {
return isEmptyResult;
}
static void resetProjectNullTypeImpl(SNodeList* pProjects) {
SNode* pProj = NULL;
FOREACH(pProj, pProjects) {
SExprNode* pExpr = (SExprNode*)pProj;
if (TSDB_DATA_TYPE_NULL == pExpr->resType.type) {
pExpr->resType.type = TSDB_DATA_TYPE_VARCHAR;
pExpr->resType.bytes = 0;
}
}
}
static void resetProjectNullType(SNode* pStmt) {
switch (nodeType(pStmt)) {
case QUERY_NODE_SELECT_STMT:
resetProjectNullTypeImpl(((SSelectStmt*)pStmt)->pProjectionList);
break;
case QUERY_NODE_SET_OPERATOR: {
resetProjectNullTypeImpl(((SSetOperator*)pStmt)->pProjectionList);
break;
}
default:
break;
}
}
int32_t calculateConstant(SParseContext* pParseCxt, SQuery* pQuery) {
SCalcConstContext cxt = {.pParseCxt = pParseCxt,
.msgBuf.buf = pParseCxt->pMsg,
.msgBuf.len = pParseCxt->msgLen,
.code = TSDB_CODE_SUCCESS};
int32_t code = calcConstQuery(&cxt, pQuery->pRoot, false);
if (TSDB_CODE_SUCCESS == code && isEmptyResultQuery(pQuery->pRoot)) {
pQuery->execMode = QUERY_EXEC_MODE_EMPTY_RESULT;
if (TSDB_CODE_SUCCESS == code) {
resetProjectNullType(pQuery->pRoot);
if (isEmptyResultQuery(pQuery->pRoot)) {
pQuery->execMode = QUERY_EXEC_MODE_EMPTY_RESULT;
}
}
return code;
}

2743
source/libs/parser/src/sql.c
View File

File diff suppressed because it is too large Load Diff

4
source/libs/scalar/src/scalar.c
View File

@ -378,7 +378,7 @@ int32_t sclInitParam(SNode *node, SScalarParam *param, SScalarCtx *ctx, int32_t
int32_t index = -1;
for (int32_t i = 0; i < taosArrayGetSize(ctx->pBlockList); ++i) {
SSDataBlock *pb = taosArrayGetP(ctx->pBlockList, i);
if (pb->info.blockId == ref->dataBlockId) {
if (pb->info.id.blockId == ref->dataBlockId) {
index = i;
break;
}
@ -1384,7 +1384,7 @@ EDealRes sclWalkTarget(SNode *pNode, SScalarCtx *ctx) {
int32_t index = -1;
for (int32_t i = 0; i < taosArrayGetSize(ctx->pBlockList); ++i) {
SSDataBlock *pb = taosArrayGetP(ctx->pBlockList, i);
if (pb->info.blockId == target->dataBlockId) {
if (pb->info.id.blockId == target->dataBlockId) {
index = i;
break;
}

2
source/libs/scalar/test/scalar/scalarTests.cpp
View File

@ -98,7 +98,7 @@ void scltAppendReservedSlot(SArray *pBlockList, int16_t *dataBlockId, int16_t *s
taosArrayPush(pBlockList, &res);
*dataBlockId = taosArrayGetSize(pBlockList) - 1;
res->info.blockId = *dataBlockId;
res->info.id.blockId = *dataBlockId;
*slotId = 0;
} else {
SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(pBlockList);

2
source/libs/stream/src/streamDispatch.c
View File

@ -463,7 +463,7 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat
continue;
}
if (streamSearchAndAddBlock(pTask, pReqs, pDataBlock, vgSz, pDataBlock->info.groupId) < 0) {
if (streamSearchAndAddBlock(pTask, pReqs, pDataBlock, vgSz, pDataBlock->info.id.groupId) < 0) {
goto FAIL_SHUFFLE_DISPATCH;
}
}

2
source/libs/stream/src/streamUpdate.c
View File

@ -166,7 +166,7 @@ bool updateInfoIsTableInserted(SUpdateInfo *pInfo, int64_t tbUid) {
TSKEY updateInfoFillBlockData(SUpdateInfo *pInfo, SSDataBlock *pBlock, int32_t primaryTsCol) {
if (pBlock == NULL || pBlock->info.rows == 0) return INT64_MIN;
TSKEY maxTs = INT64_MIN;
int64_t tbUid = pBlock->info.uid;
int64_t tbUid = pBlock->info.id.uid;
SColumnInfoData *pColDataInfo = taosArrayGet(pBlock->pDataBlock, primaryTsCol);

45
source/libs/tdb/src/db/tdbPager.c
View File

@ -210,6 +210,7 @@ int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) {
ret = tdbGetFileSize(pPager->fd, pPager->pageSize, &(pPager->dbOrigSize));
pPager->dbFileSize = pPager->dbOrigSize;
tdbTrace("pager/open reset dirty tree: %p", &pPager->rbt);
tRBTreeCreate(&pPager->rbt, pageCmpFn);
*ppPager = pPager;
@ -296,7 +297,7 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) {
// ref page one more time so the page will not be release
tdbRefPage(pPage);
tdbDebug("pcache/mdirty page %p/%d/%d", pPage, TDB_PAGE_PGNO(pPage), pPage->id);
tdbDebug("pager/mdirty page %p/%d/%d", pPage, TDB_PAGE_PGNO(pPage), pPage->id);
// Set page as dirty
pPage->isDirty = 1;
@ -316,6 +317,7 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) {
pPage->pDirtyNext = *ppPage;
*ppPage = pPage;
*/
tdbTrace("put page: %p %d to dirty tree: %p", pPage, TDB_PAGE_PGNO(pPage), &pPager->rbt);
tRBTreePut(&pPager->rbt, (SRBTreeNode *)pPage);
// Write page to journal if neccessary
@ -373,6 +375,7 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) {
SRBTreeNode *pNode = NULL;
while ((pNode = tRBTreeIterNext(&iter)) != NULL) {
pPage = (SPage *)pNode;
ASSERT(pPage->nOverflow == 0);
ret = tdbPagerWritePageToDB(pPager, pPage);
if (ret < 0) {
@ -398,6 +401,7 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) {
tdbPCacheRelease(pPager->pCache, pPage, pTxn);
}
tdbTrace("pager/commit reset dirty tree: %p", &pPager->rbt);
tRBTreeCreate(&pPager->rbt, pageCmpFn);
// sync the db file
@ -471,6 +475,7 @@ int tdbPagerPrepareAsyncCommit(SPager *pPager, TXN *pTxn) {
tdbPCacheRelease(pPager->pCache, pPage, pTxn);
}
/*
tdbTrace("reset dirty tree: %p", &pPager->rbt);
tRBTreeCreate(&pPager->rbt, pageCmpFn);
// sync the db file
@ -566,6 +571,7 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) {
tdbPCacheRelease(pPager->pCache, pPage, pTxn);
}
tdbTrace("reset dirty tree: %p", &pPager->rbt);
tRBTreeCreate(&pPager->rbt, pageCmpFn);
// 4, remove the journal file
@ -580,6 +586,8 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) {
int tdbPagerFlushPage(SPager *pPager, TXN *pTxn) {
SPage *pPage;
i32 nRef;
SPgno maxPgno = pPager->dbOrigSize;
int ret;
// loop to write the dirty pages to file
@ -587,29 +595,52 @@ int tdbPagerFlushPage(SPager *pPager, TXN *pTxn) {
SRBTreeNode *pNode = NULL;
while ((pNode = tRBTreeIterNext(&iter)) != NULL) {
pPage = (SPage *)pNode;
nRef = tdbGetPageRef(pPage);
if (nRef > 1) {
continue;
}
SPgno pgno = TDB_PAGE_PGNO(pPage);
if (pgno > maxPgno) {
maxPgno = pgno;
}
ret = tdbPagerWritePageToDB(pPager, pPage);
if (ret < 0) {
tdbError("failed to write page to db since %s", tstrerror(terrno));
return -1;
}
}
tdbTrace("tdbttl commit:%p, %d/%d", pPager, pPager->dbOrigSize, pPager->dbFileSize);
pPager->dbOrigSize = pPager->dbFileSize;
tdbTrace("tdb/flush:%p, %d/%d/%d", pPager, pPager->dbOrigSize, pPager->dbFileSize, maxPgno);
pPager->dbOrigSize = maxPgno;
pPage->isDirty = 0;
tdbTrace("pager/flush drop page: %p %d from dirty tree: %p", pPage, TDB_PAGE_PGNO(pPage), &pPager->rbt);
tRBTreeDrop(&pPager->rbt, (SRBTreeNode *)pPage);
tdbPCacheRelease(pPager->pCache, pPage, pTxn);
break;
}
/*
tdbTrace("tdb/flush:%p, %d/%d/%d", pPager, pPager->dbOrigSize, pPager->dbFileSize, maxPgno);
pPager->dbOrigSize = maxPgno;
// release the page
iter = tRBTreeIterCreate(&pPager->rbt, 1);
while ((pNode = tRBTreeIterNext(&iter)) != NULL) {
pPage = (SPage *)pNode;
nRef = tdbGetPageRef(pPage);
if (nRef > 1) {
continue;
}
pPage->isDirty = 0;
tdbTrace("pager/flush drop page: %p %d from dirty tree: %p", pPage, TDB_PAGE_PGNO(pPage), &pPager->rbt);
tRBTreeDrop(&pPager->rbt, (SRBTreeNode *)pPage);
tdbPCacheRelease(pPager->pCache, pPage, pTxn);
}
tRBTreeCreate(&pPager->rbt, pageCmpFn);
*/
return 0;
}

287
source/util/src/trbtree.c
View File

@ -104,61 +104,6 @@ static void tRBTreeTransplant(SRBTree *pTree, SRBTreeNode *u, SRBTreeNode *v) {
v->parent = u->parent;
}
static void tRBTreeDropFix(SRBTree *pTree, SRBTreeNode *x) {
while (x != pTree->root && x->color == BLACK) {
if (x == x->parent->left) {
SRBTreeNode *w = x->parent->right;
if (w->color == RED) {
w->color = BLACK;
x->parent->color = RED;
tRBTreeRotateLeft(pTree, x->parent);
w = x->parent->right;
}
if (w->left->color == BLACK && w->right->color == BLACK) {
w->color = RED;
x = x->parent;
} else {
if (w->right->color == BLACK) {
w->left->color = BLACK;
w->color = RED;
tRBTreeRotateRight(pTree, w);
w = x->parent->right;
}
w->color = x->parent->color;
x->parent->color = BLACK;
w->right->color = BLACK;
tRBTreeRotateLeft(pTree, x->parent);
x = pTree->root;
}
} else {
SRBTreeNode *w = x->parent->left;
if (w->color == RED) {
w->color = BLACK;
x->parent->color = RED;
tRBTreeRotateRight(pTree, x->parent);
w = x->parent->left;
}
if (w->right->color == BLACK && w->left->color == BLACK) {
w->color = RED;
x = x->parent;
} else {
if (w->left->color == BLACK) {
w->right->color = BLACK;
w->color = RED;
tRBTreeRotateLeft(pTree, w);
w = x->parent->left;
}
w->color = x->parent->color;
x->parent->color = BLACK;
w->left->color = BLACK;
tRBTreeRotateRight(pTree, x->parent);
x = pTree->root;
}
}
}
x->color = BLACK;
}
static SRBTreeNode *tRBTreeSuccessor(SRBTree *pTree, SRBTreeNode *pNode) {
if (pNode->right != pTree->NIL) {
pNode = pNode->right;
@ -255,11 +200,205 @@ SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *z) {
return z;
}
void tRBTreeDrop(SRBTree *pTree, SRBTreeNode *z) {
SRBTreeNode *y = z;
SRBTreeNode *x;
ECOLOR y_orignal_color = y->color;
#define RBTREE_NULL rbtree->NIL
#define rbtree_t SRBTree
#define rbnode_t SRBTreeNode
#define rbtree_rotate_left tRBTreeRotateLeft
#define rbtree_rotate_right tRBTreeRotateRight
static void rbtree_delete_fixup(rbtree_t *rbtree, rbnode_t *child, rbnode_t *child_parent) {
rbnode_t *sibling;
int go_up = 1;
/* determine sibling to the node that is one-black short */
if (child_parent->right == child)
sibling = child_parent->left;
else
sibling = child_parent->right;
while (go_up) {
if (child_parent == RBTREE_NULL) {
/* removed parent==black from root, every path, so ok */
return;
}
if (sibling->color == RED) { /* rotate to get a black sibling */
child_parent->color = RED;
sibling->color = BLACK;
if (child_parent->right == child)
rbtree_rotate_right(rbtree, child_parent);
else
rbtree_rotate_left(rbtree, child_parent);
/* new sibling after rotation */
if (child_parent->right == child)
sibling = child_parent->left;
else
sibling = child_parent->right;
}
if (child_parent->color == BLACK && sibling->color == BLACK && sibling->left->color == BLACK &&
sibling->right->color == BLACK) { /* fixup local with recolor of sibling */
if (sibling != RBTREE_NULL) sibling->color = RED;
child = child_parent;
child_parent = child_parent->parent;
/* prepare to go up, new sibling */
if (child_parent->right == child)
sibling = child_parent->left;
else
sibling = child_parent->right;
} else
go_up = 0;
}
if (child_parent->color == RED && sibling->color == BLACK && sibling->left->color == BLACK &&
sibling->right->color == BLACK) {
/* move red to sibling to rebalance */
if (sibling != RBTREE_NULL) sibling->color = RED;
child_parent->color = BLACK;
return;
}
assert(sibling != RBTREE_NULL);
/* get a new sibling, by rotating at sibling. See which child
of sibling is red */
if (child_parent->right == child && sibling->color == BLACK && sibling->right->color == RED &&
sibling->left->color == BLACK) {
sibling->color = RED;
sibling->right->color = BLACK;
rbtree_rotate_left(rbtree, sibling);
/* new sibling after rotation */
if (child_parent->right == child)
sibling = child_parent->left;
else
sibling = child_parent->right;
} else if (child_parent->left == child && sibling->color == BLACK && sibling->left->color == RED &&
sibling->right->color == BLACK) {
sibling->color = RED;
sibling->left->color = BLACK;
rbtree_rotate_right(rbtree, sibling);
/* new sibling after rotation */
if (child_parent->right == child)
sibling = child_parent->left;
else
sibling = child_parent->right;
}
/* now we have a black sibling with a red child. rotate and exchange colors. */
sibling->color = child_parent->color;
child_parent->color = BLACK;
if (child_parent->right == child) {
assert(sibling->left->color == RED);
sibling->left->color = BLACK;
rbtree_rotate_right(rbtree, child_parent);
} else {
assert(sibling->right->color == RED);
sibling->right->color = BLACK;
rbtree_rotate_left(rbtree, child_parent);
}
}
/** helpers for delete: swap node colours */
static void swap_int8(ECOLOR *x, ECOLOR *y) {
ECOLOR t = *x;
*x = *y;
*y = t;
}
/** helpers for delete: swap node pointers */
static void swap_np(rbnode_t **x, rbnode_t **y) {
rbnode_t *t = *x;
*x = *y;
*y = t;
}
/** Update parent pointers of child trees of 'parent' */
static void change_parent_ptr(rbtree_t *rbtree, rbnode_t *parent, rbnode_t *old, rbnode_t *new) {
if (parent == RBTREE_NULL) {
assert(rbtree->root == old);
if (rbtree->root == old) rbtree->root = new;
return;
}
assert(parent->left == old || parent->right == old || parent->left == new || parent->right == new);
if (parent->left == old) parent->left = new;
if (parent->right == old) parent->right = new;
}
/** Update parent pointer of a node 'child' */
static void change_child_ptr(rbtree_t *rbtree, rbnode_t *child, rbnode_t *old, rbnode_t *new) {
if (child == RBTREE_NULL) return;
assert(child->parent == old || child->parent == new);
if (child->parent == old) child->parent = new;
}
rbnode_t *rbtree_delete(rbtree_t *rbtree, void *key) {
rbnode_t *to_delete = key;
rbnode_t *child;
/* make sure we have at most one non-leaf child */
if (to_delete->left != RBTREE_NULL && to_delete->right != RBTREE_NULL) {
/* swap with smallest from right subtree (or largest from left) */
rbnode_t *smright = to_delete->right;
while (smright->left != RBTREE_NULL) smright = smright->left;
/* swap the smright and to_delete elements in the tree,
* but the rbnode_t is first part of user data struct
* so cannot just swap the keys and data pointers. Instead
* readjust the pointers left,right,parent */
/* swap colors - colors are tied to the position in the tree */
swap_int8(&to_delete->color, &smright->color);
/* swap child pointers in parents of smright/to_delete */
change_parent_ptr(rbtree, to_delete->parent, to_delete, smright);
if (to_delete->right != smright) change_parent_ptr(rbtree, smright->parent, smright, to_delete);
/* swap parent pointers in children of smright/to_delete */
change_child_ptr(rbtree, smright->left, smright, to_delete);
change_child_ptr(rbtree, smright->left, smright, to_delete);
change_child_ptr(rbtree, smright->right, smright, to_delete);
change_child_ptr(rbtree, smright->right, smright, to_delete);
change_child_ptr(rbtree, to_delete->left, to_delete, smright);
if (to_delete->right != smright) change_child_ptr(rbtree, to_delete->right, to_delete, smright);
if (to_delete->right == smright) {
/* set up so after swap they work */
to_delete->right = to_delete;
smright->parent = smright;
}
/* swap pointers in to_delete/smright nodes */
swap_np(&to_delete->parent, &smright->parent);
swap_np(&to_delete->left, &smright->left);
swap_np(&to_delete->right, &smright->right);
/* now delete to_delete (which is at the location where the smright previously was) */
}
assert(to_delete->left == RBTREE_NULL || to_delete->right == RBTREE_NULL);
if (to_delete->left != RBTREE_NULL)
child = to_delete->left;
else
child = to_delete->right;
/* unlink to_delete from the tree, replace to_delete with child */
change_parent_ptr(rbtree, to_delete->parent, to_delete, child);
change_child_ptr(rbtree, child, to_delete, to_delete->parent);
if (to_delete->color == RED) {
/* if node is red then the child (black) can be swapped in */
} else if (child->color == RED) {
/* change child to BLACK, removing a RED node is no problem */
if (child != RBTREE_NULL) child->color = BLACK;
} else
rbtree_delete_fixup(rbtree, child, to_delete->parent);
/* unlink completely */
to_delete->parent = RBTREE_NULL;
to_delete->left = RBTREE_NULL;
to_delete->right = RBTREE_NULL;
to_delete->color = BLACK;
return to_delete;
}
void tRBTreeDrop(SRBTree *pTree, SRBTreeNode *z) {
// update min/max node
if (pTree->min == z) {
pTree->min = tRBTreeSuccessor(pTree, pTree->min);
@ -268,34 +407,8 @@ void tRBTreeDrop(SRBTree *pTree, SRBTreeNode *z) {
pTree->max = tRBTreePredecessor(pTree, pTree->max);
}
// drop impl
if (z->left == pTree->NIL) {
x = z->right;
tRBTreeTransplant(pTree, z, z->right);
} else if (z->right == pTree->NIL) {
x = z->left;
tRBTreeTransplant(pTree, z, z->left);
} else {
y = tRBTreeSuccessor(pTree, z);
y_orignal_color = y->color;
x = y->right;
if (y->parent == z) {
x->parent = z;
} else {
tRBTreeTransplant(pTree, y, y->right);
y->right = z->right;
y->right->parent = y;
}
tRBTreeTransplant(pTree, z, y);
y->left = z->left;
y->left->parent = y;
y->color = z->color;
}
rbtree_delete(pTree, z);
// fix
if (y_orignal_color == BLACK) {
tRBTreeDropFix(pTree, x);
}
pTree->n--;
}
@ -343,4 +456,4 @@ SRBTreeNode *tRBTreeIterNext(SRBTreeIter *pIter) {
_exit:
return (pNode == pIter->pTree->NIL) ? NULL : pNode;
}
}

90
tests/parallel_test/cases.task
View File

@ -90,7 +90,7 @@
,,y,script,./test.sh -f tsim/parser/auto_create_tb.sim
,,y,script,./test.sh -f tsim/parser/between_and.sim
,,y,script,./test.sh -f tsim/parser/binary_escapeCharacter.sim
,,y,script,./test.sh -f tsim/parser/col_arithmetic_operation.sim
,,y,script,./test.sh -f tsim/parser/col_arithmetic_operation.sim
,,y,script,./test.sh -f tsim/parser/columnValue_bigint.sim
,,y,script,./test.sh -f tsim/parser/columnValue_bool.sim
,,y,script,./test.sh -f tsim/parser/columnValue_double.sim
@ -409,7 +409,7 @@
,,n,system-test,python3 ./test.py -f 0-others/taosdShell.py -N 5 -M 3 -Q 3
,,n,system-test,python3 ./test.py -f 0-others/udfTest.py
,,n,system-test,python3 ./test.py -f 0-others/udf_create.py
,,n,system-test,python3 ./test.py -f 0-others/udf_restart_taosd.py
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/udf_restart_taosd.py
,,n,system-test,python3 ./test.py -f 0-others/udf_cfg1.py
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/udf_cfg2.py
,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/cachemodel.py
@ -425,7 +425,7 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/test_stmt_set_tbname_tag.py
,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_stable.py
,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/alter_table.py
,,n,system-test,python3 ./test.py -f 1-insert/boundary.py
,,,system-test,python3 ./test.py -f 1-insert/boundary.py
,,n,system-test,python3 ./test.py -f 1-insert/insertWithMoreVgroup.py
,,y,system-test,./pytest.sh python3 ./test.py -f 1-insert/table_comment.py
,,n,system-test,python3 ./test.py -f 1-insert/time_range_wise.py
@ -633,41 +633,41 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/csum.py
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/function_diff.py
,,n,system-test,python3 ./test.py -f 2-query/queryQnode.py
,,,system-test,python3 ./test.py -f 6-cluster/5dnode1mnode.py
,,,system-test,python3 ./test.py -f 6-cluster/5dnode2mnode.py -N 5
,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3
,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3 -i False
,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -N 5 -M 3
,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -N 5 -M 3 -i False
,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeStopLoop.py -N 5 -M 3
,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 6 -M 3
,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 6 -M 3 -n 3
,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 6 -M 3
,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 6 -M 3 -n 3
,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 6 -M 3
,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 6 -M 3 -n 3
,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py -N 6 -M 3
,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py -N 6 -M 3 -n 3
,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py -N 6 -M 3
,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py -N 6 -M 3 -n 3
,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py -N 6 -M 3
,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py -N 6 -M 3 -n 3
,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertData.py -N 6 -M 3
,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertData.py -N 6 -M 3 -n 3
,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 6 -M 3
,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 6 -M 3 -n 3
,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6
,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6 -n 3
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode1mnode.py
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode2mnode.py -N 5
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 -M 3 -i False
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -N 5 -M 3
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -N 5 -M 3 -i False
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeStopLoop.py -N 5 -M 3
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 6 -M 3
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 6 -M 3 -n 3
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 6 -M 3
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -N 6 -M 3 -n 3
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 6 -M 3
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -N 6 -M 3 -n 3
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py -N 6 -M 3
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py -N 6 -M 3 -n 3
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py -N 6 -M 3
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py -N 6 -M 3 -n 3
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py -N 6 -M 3
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py -N 6 -M 3 -n 3
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertData.py -N 6 -M 3
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertData.py -N 6 -M 3 -n 3
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 6 -M 3
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 6 -M 3 -n 3
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6 -n 3
,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5
,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeRecreateMnode.py -N 5 -M 3
,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeStopFollowerLeader.py -N 5 -M 3
,,,system-test,python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -N 5 -M 3
,,,system-test,python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_createDb_replica1.py -N 4 -M 1
,,,system-test,python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas.py -N 4 -M 1
,,,system-test,python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas_querys.py -N 4 -M 1
,,,system-test,python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas.py -N 4 -M 1
,,,system-test,python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys.py -N 4 -M 1
,,,system-test,python3 test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py -N 4 -M 1
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeRecreateMnode.py -N 5 -M 3
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeStopFollowerLeader.py -N 5 -M 3
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -N 5 -M 3
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/vnode/4dnode1mnode_basic_createDb_replica1.py -N 4 -M 1
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas.py -N 4 -M 1
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas_querys.py -N 4 -M 1
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas.py -N 4 -M 1
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys.py -N 4 -M 1
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py -N 4 -M 1
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/create_wrong_topic.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dropDbR3ConflictTransaction.py -N 3
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/basic5.py
@ -702,19 +702,19 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py
,,,system-test,python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqAutoCreateTbl.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDnodeRestart.py
,,,system-test,python3 ./test.py -f 7-tmq/tmqDnodeRestart1.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDnodeRestart1.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUpdate-1ctb.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUpdateWithConsume.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot0.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot1.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDelete-1ctb.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDelete-multiCtb.py
,,,system-test,python3 ./test.py -f 7-tmq/tmqDropStb.py
,,,system-test,python3 ./test.py -f 7-tmq/tmqDropStbCtb.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropStb.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropStbCtb.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot0.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot1.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUdf.py
@ -723,13 +723,13 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/stbTagFilter-1ctb.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dataFromTsdbNWal.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py
,,,system-test,python3 ./test.py -f 7-tmq/tmq_taosx.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq_taosx.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py
,,y,system-test,./pytest.sh python3 ./test.py -f 99-TDcase/TD-19201.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqSubscribeStb-r3.py -N 5
,,,system-test,python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3
,,,system-test,python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -n 3
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/between.py -Q 2
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -N 6 -M 3 -n 3
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/between.py -Q 2
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/distinct.py -Q 2
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/varchar.py -Q 2
,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/ltrim.py -Q 2

3
tests/pytest/util/dnodes.py
View File

@ -823,5 +823,8 @@ class TDDnodes:
def addSimExtraCfg(self, option, value):
self.sim.addExtraCfg(option, value)
def getAsan(self):
return self.asan
tdDnodes = TDDnodes()

2
tests/system-test/0-others/udf_cluster.py
View File

@ -259,6 +259,8 @@ class TDTestCase:
self.TDDnodes.init("")
self.TDDnodes.setTestCluster(testCluster)
self.TDDnodes.setValgrind(valgrind)
self.TDDnodes.setAsan(tdDnodes.getAsan())
self.TDDnodes.stopAll()
for dnode in self.TDDnodes.dnodes:
self.TDDnodes.deploy(dnode.index,{})

3
tests/system-test/6-cluster/5dnode1mnode.py
View File

@ -7,6 +7,7 @@ import os
from util.log import *
from util.sql import *
from util.cases import *
from util.dnodes import *
from util.dnodes import TDDnodes
from util.dnodes import TDDnode
import time
@ -68,6 +69,8 @@ class TDTestCase:
self.TDDnodes.init("")
self.TDDnodes.setTestCluster(testCluster)
self.TDDnodes.setValgrind(valgrind)
self.TDDnodes.setAsan(tdDnodes.getAsan())
self.TDDnodes.stopAll()
for dnode in self.TDDnodes.dnodes:
self.TDDnodes.deploy(dnode.index,{})

6
tests/system-test/6-cluster/5dnode3mnodeDrop.py
View File

@ -8,6 +8,7 @@ import os
from util.log import *
from util.sql import *
from util.cases import *
from util.dnodes import *
from util.dnodes import TDDnodes
from util.dnodes import TDDnode
import time
@ -94,6 +95,8 @@ class TDTestCase:
self.TDDnodes.init("")
self.TDDnodes.setTestCluster(testCluster)
self.TDDnodes.setValgrind(valgrind)
self.TDDnodes.setAsan(tdDnodes.getAsan())
self.TDDnodes.stopAll()
for dnode in self.TDDnodes.dnodes:
self.TDDnodes.deploy(dnode.index,{})
@ -249,6 +252,9 @@ class TDTestCase:
tdSql.checkData(0,1,'%s:6030'%self.host)
tdSql.checkData(4,1,'%s:6430'%self.host)
tdSql.checkData(0,4,'ready')
tdSql.checkData(1,4,'ready')
tdSql.checkData(2,4,'ready')
tdSql.checkData(3,4,'ready')
tdSql.checkData(4,4,'ready')
tdSql.query("select * from information_schema.ins_mnodes;")
tdSql.checkRows(1)

2
tests/system-test/6-cluster/5dnode3mnodeDropInsert.py
View File

@ -159,6 +159,8 @@ class TDTestCase:
self.TDDnodes.init("")
self.TDDnodes.setTestCluster(testCluster)
self.TDDnodes.setValgrind(valgrind)
self.TDDnodes.setAsan(tdDnodes.getAsan())
self.TDDnodes.stopAll()
for dnode in self.TDDnodes.dnodes:
self.TDDnodes.deploy(dnode.index,{})

2
tests/system-test/6-cluster/5dnode3mnodeStopInsert.py
View File

@ -117,6 +117,8 @@ class TDTestCase:
self.TDDnodes.init("")
self.TDDnodes.setTestCluster(testCluster)
self.TDDnodes.setValgrind(valgrind)
self.TDDnodes.setAsan(tdDnodes.getAsan())
self.TDDnodes.stopAll()
for dnode in self.TDDnodes.dnodes:
self.TDDnodes.deploy(dnode.index,{})

4
tests/system-test/test-all.bat
View File

@ -90,7 +90,7 @@ goto :eof
:CheckSkipCase
set skipCase=false
if "%*" == "python3 ./test.py -f 1-insert/insertWithMoreVgroup.py" ( set skipCase=true )
if "%*" == "python3 ./test.py -f 2-query/queryQnode.py" ( set skipCase=true )
if "%*" == "python3 ./test.py -f 1-insert/insertWithMoreVgroup.py" ( set skipCase=false )
if "%*" == "python3 ./test.py -f 2-query/queryQnode.py" ( set skipCase=false )
echo %* | grep "\-R" && set skipCase=true
:goto eof