enh(query): add event window.
This commit is contained in:
Haojun Liao
parent
892be91001
commit
5dd9322cbf
|
|
@ -0,0 +1,295 @@
|
||||||
|
/*
|
||||||
|
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
|
||||||
|
*
|
||||||
|
* This program is free software: you can use, redistribute, and/or modify
|
||||||
|
* it under the terms of the GNU Affero General Public License, version 3
|
||||||
|
* or later ("AGPL"), as published by the Free Software Foundation.
|
||||||
|
*
|
||||||
|
* This program is distributed in the hope that it will be useful, but WITHOUT
|
||||||
|
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
||||||
|
* FITNESS FOR A PARTICULAR PURPOSE.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU Affero General Public License
|
||||||
|
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
*/
|
||||||
|
|
||||||
|
#include "executorimpl.h"
|
||||||
|
#include "filter.h"
|
||||||
|
#include "function.h"
|
||||||
|
#include "functionMgt.h"
|
||||||
|
#include "tcommon.h"
|
||||||
|
#include "tcompare.h"
|
||||||
|
#include "tdatablock.h"
|
||||||
|
#include "tfill.h"
|
||||||
|
#include "ttime.h"
|
||||||
|
|
||||||
|
typedef struct SEventWindowOperatorInfo {
|
||||||
|
SOptrBasicInfo binfo;
|
||||||
|
SAggSupporter aggSup;
|
||||||
|
SExprSupp scalarSup;
|
||||||
|
SGroupResInfo groupResInfo;
|
||||||
|
SWindowRowsSup winSup;
|
||||||
|
SColumn stateCol; // start row index
|
||||||
|
bool hasKey;
|
||||||
|
SStateKeys stateKey;
|
||||||
|
int32_t tsSlotId; // primary timestamp column slot id
|
||||||
|
STimeWindowAggSupp twAggSup;
|
||||||
|
} SEventWindowOperatorInfo;
|
||||||
|
|
||||||
|
static int32_t openEventWindowAggOptr(SOperatorInfo* pOperator);
|
||||||
|
static void destroyEWindowOperatorInfo(void* param);
|
||||||
|
static void doEventWindowAggImpl(SOperatorInfo* pOperator, SEventWindowOperatorInfo* pInfo, SSDataBlock* pBlock);
|
||||||
|
static SSDataBlock* doEventWindowAgg(SOperatorInfo* pOperator);
|
||||||
|
|
||||||
|
SOperatorInfo* createEventwindowOperatorInfo(SOperatorInfo* downstream, SEventWinodwPhysiNode* pStateNode,
|
||||||
|
SExecTaskInfo* pTaskInfo) {
|
||||||
|
SEventWindowOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SEventWindowOperatorInfo));
|
||||||
|
SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
|
||||||
|
if (pInfo == NULL || pOperator == NULL) {
|
||||||
|
goto _error;
|
||||||
|
}
|
||||||
|
|
||||||
|
int32_t tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId;
|
||||||
|
SColumnNode* pColNode = (SColumnNode*)((STargetNode*)pStateNode->pStartCond)->pExpr;
|
||||||
|
|
||||||
|
if (pStateNode->window.pExprs != NULL) {
|
||||||
|
int32_t numOfScalarExpr = 0;
|
||||||
|
SExprInfo* pScalarExprInfo = createExprInfo(pStateNode->window.pExprs, NULL, &numOfScalarExpr);
|
||||||
|
int32_t code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr);
|
||||||
|
if (code != TSDB_CODE_SUCCESS) {
|
||||||
|
goto _error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pInfo->stateCol = extractColumnFromColumnNode(pColNode);
|
||||||
|
pInfo->stateKey.type = pInfo->stateCol.type;
|
||||||
|
pInfo->stateKey.bytes = pInfo->stateCol.bytes;
|
||||||
|
pInfo->stateKey.pData = taosMemoryCalloc(1, pInfo->stateCol.bytes);
|
||||||
|
if (pInfo->stateKey.pData == NULL) {
|
||||||
|
goto _error;
|
||||||
|
}
|
||||||
|
|
||||||
|
int32_t code = filterInitFromNode((SNode*)pStateNode->window.node.pConditions, &pOperator->exprSupp.pFilterInfo, 0);
|
||||||
|
if (code != TSDB_CODE_SUCCESS) {
|
||||||
|
goto _error;
|
||||||
|
}
|
||||||
|
|
||||||
|
size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
|
||||||
|
|
||||||
|
int32_t num = 0;
|
||||||
|
SExprInfo* pExprInfo = createExprInfo(pStateNode->window.pFuncs, NULL, &num);
|
||||||
|
initResultSizeInfo(&pOperator->resultInfo, 4096);
|
||||||
|
|
||||||
|
code = initAggSup(&pOperator->exprSupp, &pInfo->aggSup, pExprInfo, num, keyBufSize, pTaskInfo->id.str);
|
||||||
|
if (code != TSDB_CODE_SUCCESS) {
|
||||||
|
goto _error;
|
||||||
|
}
|
||||||
|
|
||||||
|
SSDataBlock* pResBlock = createDataBlockFromDescNode(pStateNode->window.node.pOutputDataBlockDesc);
|
||||||
|
initBasicInfo(&pInfo->binfo, pResBlock);
|
||||||
|
initResultRowInfo(&pInfo->binfo.resultRowInfo);
|
||||||
|
|
||||||
|
pInfo->twAggSup =
|
||||||
|
(STimeWindowAggSupp){.waterMark = pStateNode->window.watermark, .calTrigger = pStateNode->window.triggerType};
|
||||||
|
|
||||||
|
initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
|
||||||
|
|
||||||
|
pInfo->tsSlotId = tsSlotId;
|
||||||
|
|
||||||
|
setOperatorInfo(pOperator, "StateWindowOperator", QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE, true, OP_NOT_OPENED, pInfo,
|
||||||
|
pTaskInfo);
|
||||||
|
pOperator->fpSet = createOperatorFpSet(openEventWindowAggOptr, doEventWindowAgg, NULL, destroyEWindowOperatorInfo,
|
||||||
|
optrDefaultBufFn, NULL);
|
||||||
|
|
||||||
|
code = appendDownstream(pOperator, &downstream, 1);
|
||||||
|
if (code != TSDB_CODE_SUCCESS) {
|
||||||
|
goto _error;
|
||||||
|
}
|
||||||
|
|
||||||
|
return pOperator;
|
||||||
|
|
||||||
|
_error:
|
||||||
|
if (pInfo != NULL) {
|
||||||
|
destroyEWindowOperatorInfo(pInfo);
|
||||||
|
}
|
||||||
|
|
||||||
|
taosMemoryFreeClear(pOperator);
|
||||||
|
pTaskInfo->code = code;
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
void destroyEWindowOperatorInfo(void* param) {
|
||||||
|
SEventWindowOperatorInfo* pInfo = (SEventWindowOperatorInfo*)param;
|
||||||
|
if (pInfo == NULL) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
cleanupBasicInfo(&pInfo->binfo);
|
||||||
|
colDataDestroy(&pInfo->twAggSup.timeWindowData);
|
||||||
|
|
||||||
|
cleanupAggSup(&pInfo->aggSup);
|
||||||
|
cleanupGroupResInfo(&pInfo->groupResInfo);
|
||||||
|
taosMemoryFreeClear(param);
|
||||||
|
}
|
||||||
|
|
||||||
|
static int32_t openEventWindowAggOptr(SOperatorInfo* pOperator) {
|
||||||
|
if (OPTR_IS_OPENED(pOperator)) {
|
||||||
|
return TSDB_CODE_SUCCESS;
|
||||||
|
}
|
||||||
|
|
||||||
|
SEventWindowOperatorInfo* pInfo = pOperator->info;
|
||||||
|
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
|
||||||
|
|
||||||
|
SExprSupp* pSup = &pOperator->exprSupp;
|
||||||
|
int32_t order = TSDB_ORDER_ASC;
|
||||||
|
int64_t st = taosGetTimestampUs();
|
||||||
|
|
||||||
|
SOperatorInfo* downstream = pOperator->pDownstream[0];
|
||||||
|
while (1) {
|
||||||
|
SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
|
||||||
|
if (pBlock == NULL) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
setInputDataBlock(pSup, pBlock, order, MAIN_SCAN, true);
|
||||||
|
blockDataUpdateTsWindow(pBlock, pInfo->tsSlotId);
|
||||||
|
|
||||||
|
// there is an scalar expression that needs to be calculated right before apply the group aggregation.
|
||||||
|
if (pInfo->scalarSup.pExprInfo != NULL) {
|
||||||
|
pTaskInfo->code = projectApplyFunctions(pInfo->scalarSup.pExprInfo, pBlock, pBlock, pInfo->scalarSup.pCtx,
|
||||||
|
pInfo->scalarSup.numOfExprs, NULL);
|
||||||
|
if (pTaskInfo->code != TSDB_CODE_SUCCESS) {
|
||||||
|
T_LONG_JMP(pTaskInfo->env, pTaskInfo->code);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
doEventWindowAggImpl(pOperator, pInfo, pBlock);
|
||||||
|
}
|
||||||
|
|
||||||
|
pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0;
|
||||||
|
initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, TSDB_ORDER_ASC);
|
||||||
|
pOperator->status = OP_RES_TO_RETURN;
|
||||||
|
|
||||||
|
return TSDB_CODE_SUCCESS;
|
||||||
|
}
|
||||||
|
|
||||||
|
void doEventWindowAggImpl(SOperatorInfo* pOperator, SEventWindowOperatorInfo* pInfo, SSDataBlock* pBlock) {
|
||||||
|
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
|
||||||
|
SExprSupp* pSup = &pOperator->exprSupp;
|
||||||
|
|
||||||
|
SColumnInfoData* pStateColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->stateCol.slotId);
|
||||||
|
int64_t gid = pBlock->info.id.groupId;
|
||||||
|
|
||||||
|
bool masterScan = true;
|
||||||
|
int32_t numOfOutput = pOperator->exprSupp.numOfExprs;
|
||||||
|
int16_t bytes = pStateColInfoData->info.bytes;
|
||||||
|
|
||||||
|
SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->tsSlotId);
|
||||||
|
TSKEY* tsList = (TSKEY*)pColInfoData->pData;
|
||||||
|
|
||||||
|
SWindowRowsSup* pRowSup = &pInfo->winSup;
|
||||||
|
pRowSup->numOfRows = 0;
|
||||||
|
|
||||||
|
struct SColumnDataAgg* pAgg = NULL;
|
||||||
|
for (int32_t j = 0; j < pBlock->info.rows; ++j) {
|
||||||
|
pAgg = (pBlock->pBlockAgg != NULL) ? pBlock->pBlockAgg[pInfo->stateCol.slotId] : NULL;
|
||||||
|
if (colDataIsNull(pStateColInfoData, pBlock->info.rows, j, pAgg)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
char* val = colDataGetData(pStateColInfoData, j);
|
||||||
|
|
||||||
|
if (gid != pRowSup->groupId || !pInfo->hasKey) {
|
||||||
|
// todo extract method
|
||||||
|
if (IS_VAR_DATA_TYPE(pInfo->stateKey.type)) {
|
||||||
|
varDataCopy(pInfo->stateKey.pData, val);
|
||||||
|
} else {
|
||||||
|
memcpy(pInfo->stateKey.pData, val, bytes);
|
||||||
|
}
|
||||||
|
|
||||||
|
pInfo->hasKey = true;
|
||||||
|
|
||||||
|
doKeepNewWindowStartInfo(pRowSup, tsList, j, gid);
|
||||||
|
doKeepTuple(pRowSup, tsList[j], gid);
|
||||||
|
} else if (compareVal(val, &pInfo->stateKey)) {
|
||||||
|
doKeepTuple(pRowSup, tsList[j], gid);
|
||||||
|
if (j == 0 && pRowSup->startRowIndex != 0) {
|
||||||
|
pRowSup->startRowIndex = 0;
|
||||||
|
}
|
||||||
|
} else { // a new state window started
|
||||||
|
SResultRow* pResult = NULL;
|
||||||
|
|
||||||
|
// keep the time window for the closed time window.
|
||||||
|
STimeWindow window = pRowSup->win;
|
||||||
|
|
||||||
|
pRowSup->win.ekey = pRowSup->win.skey;
|
||||||
|
int32_t ret = setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &window, masterScan, &pResult, gid, pSup->pCtx,
|
||||||
|
numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
|
||||||
|
if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code
|
||||||
|
T_LONG_JMP(pTaskInfo->env, TSDB_CODE_APP_ERROR);
|
||||||
|
}
|
||||||
|
|
||||||
|
updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &window, false);
|
||||||
|
applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex,
|
||||||
|
pRowSup->numOfRows, pBlock->info.rows, numOfOutput);
|
||||||
|
|
||||||
|
// here we start a new session window
|
||||||
|
doKeepNewWindowStartInfo(pRowSup, tsList, j, gid);
|
||||||
|
doKeepTuple(pRowSup, tsList[j], gid);
|
||||||
|
|
||||||
|
// todo extract method
|
||||||
|
if (IS_VAR_DATA_TYPE(pInfo->stateKey.type)) {
|
||||||
|
varDataCopy(pInfo->stateKey.pData, val);
|
||||||
|
} else {
|
||||||
|
memcpy(pInfo->stateKey.pData, val, bytes);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
SResultRow* pResult = NULL;
|
||||||
|
pRowSup->win.ekey = tsList[pBlock->info.rows - 1];
|
||||||
|
int32_t ret = setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &pRowSup->win, masterScan, &pResult, gid,
|
||||||
|
pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
|
||||||
|
if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code
|
||||||
|
T_LONG_JMP(pTaskInfo->env, TSDB_CODE_APP_ERROR);
|
||||||
|
}
|
||||||
|
|
||||||
|
updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pRowSup->win, false);
|
||||||
|
applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, pRowSup->startRowIndex, pRowSup->numOfRows,
|
||||||
|
pBlock->info.rows, numOfOutput);
|
||||||
|
}
|
||||||
|
|
||||||
|
SSDataBlock* doEventWindowAgg(SOperatorInfo* pOperator) {
|
||||||
|
if (pOperator->status == OP_EXEC_DONE) {
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
SStateWindowOperatorInfo* pInfo = pOperator->info;
|
||||||
|
SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo;
|
||||||
|
SOptrBasicInfo* pBInfo = &pInfo->binfo;
|
||||||
|
|
||||||
|
pTaskInfo->code = pOperator->fpSet._openFn(pOperator);
|
||||||
|
if (pTaskInfo->code != TSDB_CODE_SUCCESS) {
|
||||||
|
setOperatorCompleted(pOperator);
|
||||||
|
return NULL;
|
||||||
|
}
|
||||||
|
|
||||||
|
blockDataEnsureCapacity(pBInfo->pRes, pOperator->resultInfo.capacity);
|
||||||
|
while (1) {
|
||||||
|
doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
|
||||||
|
doFilter(pBInfo->pRes, pOperator->exprSupp.pFilterInfo, NULL);
|
||||||
|
|
||||||
|
bool hasRemain = hasRemainResults(&pInfo->groupResInfo);
|
||||||
|
if (!hasRemain) {
|
||||||
|
setOperatorCompleted(pOperator);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (pBInfo->pRes->info.rows > 0) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pOperator->resultInfo.totalRows += pBInfo->pRes->info.rows;
|
||||||
|
return (pBInfo->pRes->info.rows == 0) ? NULL : pBInfo->pRes;
|
||||||
|
}
|
||||||
|
|
@ -1964,6 +1964,7 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) {
|
||||||
return (pBInfo->pRes->info.rows == 0) ? NULL : pBInfo->pRes;
|
return (pBInfo->pRes->info.rows == 0) ? NULL : pBInfo->pRes;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// todo make this as an non-blocking operator
|
||||||
SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWinodwPhysiNode* pStateNode,
|
SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWinodwPhysiNode* pStateNode,
|
||||||
SExecTaskInfo* pTaskInfo) {
|
SExecTaskInfo* pTaskInfo) {
|
||||||
SStateWindowOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStateWindowOperatorInfo));
|
SStateWindowOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStateWindowOperatorInfo));
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue