From 7bbe197e47331f6ee7346d8ff98c1181d6619d35 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 20 Sep 2024 08:49:02 +0800 Subject: [PATCH 01/29] enh: add query plan test --- source/libs/executor/test/CMakeLists.txt | 12 + source/libs/executor/test/queryPlanTests.cpp | 3780 ++++++++++++++++++ 2 files changed, 3792 insertions(+) create mode 100755 source/libs/executor/test/queryPlanTests.cpp diff --git a/source/libs/executor/test/CMakeLists.txt b/source/libs/executor/test/CMakeLists.txt index ebf7131aa5..a3c22cdd57 100644 --- a/source/libs/executor/test/CMakeLists.txt +++ b/source/libs/executor/test/CMakeLists.txt @@ -33,3 +33,15 @@ TARGET_INCLUDE_DIRECTORIES( PUBLIC "${TD_SOURCE_DIR}/include/common" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) + +ADD_EXECUTABLE(queryPlanTests queryPlanTests.cpp) +TARGET_LINK_LIBRARIES( + queryPlanTests + PRIVATE os util common executor gtest_main qcom function planner scalar nodes vnode +) + +TARGET_INCLUDE_DIRECTORIES( + queryPlanTests + PUBLIC "${TD_SOURCE_DIR}/include/common" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) diff --git a/source/libs/executor/test/queryPlanTests.cpp b/source/libs/executor/test/queryPlanTests.cpp new file mode 100755 index 0000000000..6dd42c3a93 --- /dev/null +++ b/source/libs/executor/test/queryPlanTests.cpp @@ -0,0 +1,3780 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wwrite-strings" +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" +#pragma GCC diagnostic ignored "-Wformat" +#include + + +#ifdef WINDOWS +#define TD_USE_WINSOCK +#endif + +#include "os.h" + +#include "executor.h" +#include "executorInt.h" +#include "function.h" +#include "operator.h" +#include "taos.h" +#include "tdatablock.h" +#include "tdef.h" +#include "tvariant.h" +#include "stub.h" +#include "querytask.h" + + +namespace { + +typedef struct { + bool succeed; + int32_t blkNum; + int32_t rowNum; + int32_t addRowNum; + int32_t subRowNum; + int32_t matchNum; +} SJoinTestResInfo; + +typedef struct { + int32_t maxResRows; + int32_t maxResBlkRows; + int64_t totalResRows; + int64_t useMSecs; + SArray* pHistory; +} SJoinTestStat; + + +enum { + TEST_NO_COND = 1, + TEST_EQ_COND, + TEST_ON_COND, + TEST_FULL_COND +}; + +#define JT_PRINTF (void)printf + +#define COL_DISPLAY_WIDTH 18 +#define JT_MAX_LOOP 1000 + +#define LEFT_BLK_ID 0 +#define RIGHT_BLK_ID 1 +#define RES_BLK_ID 2 +#define MAX_SLOT_NUM 4 + +#define LEFT_TABLE_COLS 0x1 +#define RIGHT_TABLE_COLS 0x2 +#define ALL_TABLE_COLS (LEFT_TABLE_COLS | RIGHT_TABLE_COLS) + +#define JT_MAX_JLIMIT 20 +#define JT_MAX_WINDOW_OFFSET 5 +#define JT_KEY_SOLT_ID (MAX_SLOT_NUM - 1) +#define JT_PRIM_TS_SLOT_ID 0 +int32_t jtInputColType[MAX_SLOT_NUM] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_BIGINT}; + +char* jtColCondStr[] = {"", "NO COND", "EQ COND", "ON COND", "FULL COND"}; +char* jtJoinTypeStr[] = {"INNER", "LEFT", "RIGHT", "FULL"}; +char* jtSubTypeStr[] = {"", "OUTER", "SEMI", "ANTI", "ASOF", "WINDOW"}; + +int64_t TIMESTAMP_FILTER_VALUE = 10000000000; +int32_t INT_FILTER_VALUE = 200000000; +int64_t BIGINT_FILTER_VALUE = 3000000000000000; + +int64_t jtFilterValue[] = {TIMESTAMP_FILTER_VALUE, INT_FILTER_VALUE, INT_FILTER_VALUE, BIGINT_FILTER_VALUE}; + +bool jtErrorRerun = false; +bool jtInRerun = false; + +typedef struct { + bool printTestInfo; + bool printInputRow; + bool printResRow; + bool logHistory; + bool noKeepResRows; +} SJoinTestCtrl; + + +typedef struct { + bool filter; + bool asc; + bool grpJoin; + int32_t leftMaxRows; + int32_t leftMaxGrpRows; + int32_t rightMaxRows; + int32_t rightMaxGrpRows; + int32_t blkRows; + int32_t colCond; + int32_t joinType; + int32_t subType; + int32_t asofOpType; + int64_t jLimit; + int64_t winStartOffset; + int64_t winEndOffset; + int64_t inGrpId; + + int32_t leftTotalRows; + int32_t rightTotalRows; + int32_t blkRowSize; + int32_t inputStat; + + int32_t colEqNum; + int32_t colEqList[MAX_SLOT_NUM]; + + int32_t colOnNum; + int32_t colOnList[MAX_SLOT_NUM]; + + int32_t leftFilterNum; + int32_t leftFilterColList[MAX_SLOT_NUM]; + + int32_t rightFilterNum; + int32_t rightFilterColList[MAX_SLOT_NUM]; + + int32_t keyInSlotIdx; + int32_t keyOutSlotIdx; + int32_t keyColOffset; + + int32_t resColNum; + int32_t resColInSlot[MAX_SLOT_NUM * 2]; + int32_t resColList[MAX_SLOT_NUM * 2]; + int32_t resColOffset[MAX_SLOT_NUM * 2]; + int32_t resColSize; + char* resColBuf; + + int32_t colRowDataBufSize; + char* colRowDataBuf; + int32_t colRowOffset[MAX_SLOT_NUM]; + + int64_t curTs; + int64_t curKeyOffset; + int32_t grpOffset[MAX_SLOT_NUM]; + + int32_t leftBlkReadIdx; + SArray* leftBlkList; + int32_t rightBlkReadIdx; + SArray* rightBlkList; + + int64_t resRows; + bool leftColOnly; + bool rightColOnly; + SSHashObj* jtResRows; + + SOperatorInfo* pJoinOp; + + int32_t loopIdx; + + int32_t rightFinMatchNum; + bool* rightFinMatch; + + int32_t inColOffset[MAX_SLOT_NUM]; + int32_t inColSize; + char* inColBuf; + SArray* leftRowsList; + SArray* rightRowsList; + SArray* rightFilterOut; + + int64_t startTsUs; +} SJoinTestCtx; + +typedef struct { + SJoinTestResInfo res; + SJoinTestCtx ctx; +} SJoinTestHistory; + +typedef struct { + EJoinType joinType; + EJoinSubType subType; + int32_t asofOp; + int64_t jLimit; + int32_t cond; + bool filter; + bool asc; + bool grpJoin; + bool timetruncate; + SExecTaskInfo* pTask; +} SJoinTestParam; + + +SJoinTestCtx jtCtx = {0}; +SJoinTestCtrl jtCtrl = {0, 0, 0, 0, 0}; +SJoinTestStat jtStat = {0}; +SJoinTestResInfo jtRes = {0}; + + + +void printResRow(char* value, int32_t type) { + if (!jtCtrl.printResRow) { + return; + } + + JT_PRINTF(" "); + for (int32_t i = 0; i < jtCtx.resColNum; ++i) { + int32_t slot = jtCtx.resColInSlot[i]; + if (0 == type && ((jtCtx.leftColOnly && slot >= MAX_SLOT_NUM) || + (jtCtx.rightColOnly && slot < MAX_SLOT_NUM))) { + ("%18s", " "); + continue; + } + + if (*(bool*)(value + slot)) { + JT_PRINTF("%18s", " NULL"); + continue; + } + + switch (jtInputColType[slot % MAX_SLOT_NUM]) { + case TSDB_DATA_TYPE_TIMESTAMP: + JT_PRINTF("%18" PRId64 , *(int64_t*)(value + jtCtx.resColOffset[slot])); + break; + case TSDB_DATA_TYPE_INT: + JT_PRINTF("%18d", *(int32_t*)(value + jtCtx.resColOffset[slot])); + break; + case TSDB_DATA_TYPE_BIGINT: + JT_PRINTF("%18" PRId64, *(int64_t*)(value + jtCtx.resColOffset[slot])); + break; + } + } + JT_PRINTF("\t %s\n", 0 == type ? "-" : (1 == type ? "+" : "")); +} + +void pushResRow(char* buf, int32_t size) { + jtCtx.resRows++; + + if (!jtCtrl.noKeepResRows) { + int32_t* rows = (int32_t*)tSimpleHashGet(jtCtx.jtResRows, buf, size); + if (rows) { + (*rows)++; + } else { + int32_t n = 1; + assert(0 == tSimpleHashPut(jtCtx.jtResRows, buf, size, &n, sizeof(n))); + } + } +} + +void rmResRow() { + int32_t* rows = (int32_t*)tSimpleHashGet(jtCtx.jtResRows, jtCtx.resColBuf, jtCtx.resColSize); + if (rows) { + (*rows)--; + if ((*rows) == 0) { + (void)tSimpleHashRemove(jtCtx.jtResRows, jtCtx.resColBuf, jtCtx.resColSize); + } + } else { + assert(0); + } +} + +static int32_t jtMergeEqCond(SNode** ppDst, SNode** ppSrc) { + if (NULL == *ppSrc) { + return TSDB_CODE_SUCCESS; + } + if (NULL == *ppDst) { + *ppDst = *ppSrc; + *ppSrc = NULL; + return TSDB_CODE_SUCCESS; + } + if (QUERY_NODE_LOGIC_CONDITION == nodeType(*ppSrc)) { + TSWAP(*ppDst, *ppSrc); + } + if (QUERY_NODE_LOGIC_CONDITION == nodeType(*ppDst)) { + SLogicConditionNode* pLogic = (SLogicConditionNode*)*ppDst; + if (QUERY_NODE_LOGIC_CONDITION == nodeType(*ppSrc)) { + assert(0 == nodesListStrictAppendList(pLogic->pParameterList, ((SLogicConditionNode*)(*ppSrc))->pParameterList)); + ((SLogicConditionNode*)(*ppSrc))->pParameterList = NULL; + } else { + assert(0 == nodesListStrictAppend(pLogic->pParameterList, *ppSrc)); + *ppSrc = NULL; + } + nodesDestroyNode(*ppSrc); + *ppSrc = NULL; + return TSDB_CODE_SUCCESS; + } + + SLogicConditionNode* pLogicCond = NULL; + int32_t code = nodesMakeNode(QUERY_NODE_LOGIC_CONDITION, (SNode**)&pLogicCond); + if (NULL == pLogicCond) { + return code; + } + pLogicCond->node.resType.type = TSDB_DATA_TYPE_BOOL; + pLogicCond->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; + pLogicCond->condType = LOGIC_COND_TYPE_AND; + pLogicCond->pParameterList = NULL; + code = nodesMakeList(&pLogicCond->pParameterList); + assert(0 == nodesListStrictAppend(pLogicCond->pParameterList, *ppSrc)); + assert(0 == nodesListStrictAppend(pLogicCond->pParameterList, *ppDst)); + + *ppDst = (SNode*)pLogicCond; + *ppSrc = NULL; + + return TSDB_CODE_SUCCESS; +} + + +void createDummyDownstreamOperators(int32_t num, SOperatorInfo** ppRes) { + for (int32_t i = 0; i < num; ++i) { + SOperatorInfo* p = (SOperatorInfo*)taosMemoryCalloc(1, sizeof(SOperatorInfo)); + assert(NULL != p); + p->resultDataBlockId = i; + ppRes[i] = p; + } +} + +void createTargetSlotList(SSortMergeJoinPhysiNode* p) { + jtCtx.resColNum = 0; + TAOS_MEMSET(jtCtx.resColList, 0, sizeof(jtCtx.resColList)); + jtCtx.resColSize = MAX_SLOT_NUM * 2 * sizeof(bool); + jtCtx.keyInSlotIdx = -1; + + for (int32_t i = 0; i < MAX_SLOT_NUM; ++i) { + if (jtCtx.colOnList[i] || jtCtx.colEqList[i] || jtCtx.leftFilterColList[i]) { + jtCtx.resColList[i] = 1; + } + } + + for (int32_t i = 0; i < MAX_SLOT_NUM; ++i) { + if (jtCtx.colOnList[i] || jtCtx.colEqList[i] || jtCtx.rightFilterColList[i]) { + jtCtx.resColList[MAX_SLOT_NUM + i] = 1; + } + } + + for (int32_t i = 0; i < MAX_SLOT_NUM; ++i) { + if (0 == jtCtx.resColList[i]) { + jtCtx.resColList[i]= taosRand() % 2; + } + + if ((jtCtx.joinType == JOIN_TYPE_LEFT || jtCtx.joinType == JOIN_TYPE_FULL) && (i == JT_KEY_SOLT_ID)) { + jtCtx.resColList[i] = 1; + } + + if (jtCtx.resColList[i] && i == JT_KEY_SOLT_ID && (jtCtx.joinType == JOIN_TYPE_LEFT || jtCtx.joinType == JOIN_TYPE_FULL)) { + jtCtx.keyInSlotIdx = JT_KEY_SOLT_ID; + } + } + + if (jtCtx.keyInSlotIdx < 0 || ((jtCtx.joinType == JOIN_TYPE_RIGHT || jtCtx.joinType == JOIN_TYPE_FULL))) { + jtCtx.resColList[MAX_SLOT_NUM + JT_KEY_SOLT_ID]= 1; + jtCtx.keyInSlotIdx = JT_KEY_SOLT_ID + MAX_SLOT_NUM; + } + + for (int32_t i = 0; i < MAX_SLOT_NUM; ++i) { + if (0 == jtCtx.resColList[MAX_SLOT_NUM + i]) { + jtCtx.resColList[MAX_SLOT_NUM + i]= taosRand() % 2; + } + } + + int32_t idx = 0; + int32_t dstIdx = 0; + int32_t dstOffset = jtCtx.resColSize; + + for (int32_t i = 0; i < MAX_SLOT_NUM; ++i) { + if (jtCtx.resColList[i]) { + jtCtx.resColOffset[i] = dstOffset; + jtCtx.resColInSlot[dstIdx] = i; + if (jtCtx.keyInSlotIdx == i) { + jtCtx.keyColOffset = dstOffset; + } + + STargetNode* pTarget = NULL; + int32_t code = nodesMakeNode(QUERY_NODE_TARGET, (SNode**)&pTarget); + SColumnNode* pCol = NULL; + code = nodesMakeNode(QUERY_NODE_COLUMN, (SNode**)&pCol); + assert(NULL != pTarget && NULL != pCol); + pCol->dataBlockId = LEFT_BLK_ID; + pCol->slotId = i; + pTarget->dataBlockId = RES_BLK_ID; + pTarget->slotId = dstIdx++; + pTarget->pExpr = (SNode*)pCol; + dstOffset += tDataTypes[jtInputColType[i]].bytes; + jtCtx.resColSize += tDataTypes[jtInputColType[i]].bytes; + + assert(0 == nodesListMakeStrictAppend(&p->pTargets, (SNode*)pTarget)); + + jtCtx.resColNum++; + } + } + + for (int32_t i = 0; i < MAX_SLOT_NUM; ++i) { + if (jtCtx.resColList[MAX_SLOT_NUM + i]) { + jtCtx.resColOffset[MAX_SLOT_NUM + i] = dstOffset; + jtCtx.resColInSlot[dstIdx] = i + MAX_SLOT_NUM; + if (jtCtx.keyInSlotIdx == (i + MAX_SLOT_NUM)) { + jtCtx.keyColOffset = dstOffset; + } + + STargetNode* pTarget = NULL; + int32_t code = nodesMakeNode(QUERY_NODE_TARGET, (SNode**)&pTarget); + SColumnNode* pCol = NULL; + code = nodesMakeNode(QUERY_NODE_COLUMN, (SNode**)&pCol); + assert(NULL != pTarget && NULL != pCol); + pCol->dataBlockId = RIGHT_BLK_ID; + pCol->slotId = i; + pTarget->dataBlockId = RES_BLK_ID; + pTarget->slotId = dstIdx++; + pTarget->pExpr = (SNode*)pCol; + dstOffset += tDataTypes[jtInputColType[i]].bytes; + jtCtx.resColSize += tDataTypes[jtInputColType[i]].bytes; + + assert(0 == nodesListMakeStrictAppend(&p->pTargets, (SNode*)pTarget)); + jtCtx.resColNum++; + } + } + + jtCtx.resColBuf = (char*)taosMemoryRealloc(jtCtx.resColBuf, jtCtx.resColSize); + assert(NULL != jtCtx.resColBuf); +} + +void createColEqCondStart(SSortMergeJoinPhysiNode* p) { + jtCtx.colEqNum = 0; + do { + jtCtx.colEqNum = taosRand() % MAX_SLOT_NUM; // except TIMESTAMP + } while (0 == jtCtx.colEqNum); + + int32_t idx = 0; + TAOS_MEMSET(jtCtx.colEqList, 0, sizeof(jtCtx.colEqList)); + for (int32_t i = 0; i < jtCtx.colEqNum; ) { + idx = taosRand() % MAX_SLOT_NUM; + if (jtCtx.colEqList[idx]) { + continue; + } + if (TSDB_DATA_TYPE_TIMESTAMP == jtInputColType[idx]) { + continue; + } + jtCtx.colEqList[idx] = 1; + ++i; + } + + for (int32_t i = 0; i < MAX_SLOT_NUM; ++i) { + if (jtCtx.colEqList[i]) { + SColumnNode* pCol1 = NULL; + int32_t code = nodesMakeNode(QUERY_NODE_COLUMN, (SNode**)&pCol1); + assert(pCol1); + pCol1->dataBlockId = LEFT_BLK_ID; + pCol1->slotId = i; + pCol1->node.resType.type = jtInputColType[i]; + pCol1->node.resType.bytes = tDataTypes[jtInputColType[i]].bytes; + + assert(0 == nodesListMakeStrictAppend(&p->pEqLeft, (SNode*)pCol1)); + + SColumnNode* pCol2 = NULL; + code = nodesMakeNode(QUERY_NODE_COLUMN, (SNode**)&pCol2); + pCol2->dataBlockId = RIGHT_BLK_ID; + pCol2->slotId = i; + pCol2->node.resType.type = jtInputColType[i]; + pCol2->node.resType.bytes = tDataTypes[jtInputColType[i]].bytes; + + assert(0 == nodesListMakeStrictAppend(&p->pEqRight, (SNode*)pCol2)); + } + } +} + +void createColOnCondStart(SSortMergeJoinPhysiNode* p) { + jtCtx.colOnNum = 0; + do { + jtCtx.colOnNum = taosRand() % (MAX_SLOT_NUM + 1); + } while (0 == jtCtx.colOnNum || (jtCtx.colOnNum + jtCtx.colEqNum) > MAX_SLOT_NUM); + + int32_t idx = 0; + TAOS_MEMSET(jtCtx.colOnList, 0, sizeof(jtCtx.colOnList)); + for (int32_t i = 0; i < jtCtx.colOnNum; ) { + idx = taosRand() % MAX_SLOT_NUM; + if (jtCtx.colOnList[idx] || jtCtx.colEqList[idx]) { + continue; + } + jtCtx.colOnList[idx] = 1; + ++i; + } +} + +int32_t getDstSlotId(int32_t srcIdx) { + for (int32_t i = 0; i < jtCtx.resColNum; ++i) { + if (jtCtx.resColInSlot[i] == srcIdx) { + return i; + } + } + + return -1; +} + + +void createColEqCondEnd(SSortMergeJoinPhysiNode* p) { + if (jtCtx.colEqNum <= 0) { + return; + } + + SLogicConditionNode* pLogic = NULL; + if (jtCtx.colEqNum > 1) { + int32_t code = nodesMakeNode(QUERY_NODE_LOGIC_CONDITION, (SNode**)&pLogic); + assert(pLogic); + pLogic->node.resType.type = TSDB_DATA_TYPE_BOOL; + pLogic->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; + pLogic->condType = LOGIC_COND_TYPE_AND; + } + + for (int32_t i = 0; i < MAX_SLOT_NUM; ++i) { + if (jtCtx.colEqList[i]) { + SColumnNode* pCol1 = NULL; + int32_t code = nodesMakeNode(QUERY_NODE_COLUMN, (SNode**)&pCol1); + assert(pCol1); + pCol1->dataBlockId = RES_BLK_ID; + pCol1->slotId = getDstSlotId(i); + pCol1->node.resType.type = jtInputColType[i]; + pCol1->node.resType.bytes = tDataTypes[jtInputColType[i]].bytes; + + SColumnNode* pCol2 = NULL; + code = nodesMakeNode(QUERY_NODE_COLUMN, (SNode**)&pCol2); + assert(pCol2); + pCol2->dataBlockId = RES_BLK_ID; + pCol2->slotId = getDstSlotId(MAX_SLOT_NUM + i); + pCol2->node.resType.type = jtInputColType[i]; + pCol2->node.resType.bytes = tDataTypes[jtInputColType[i]].bytes; + + SOperatorNode* pOp = NULL; + code = nodesMakeNode(QUERY_NODE_OPERATOR, (SNode**)&pOp); + assert(pOp); + pOp->opType = OP_TYPE_EQUAL; + pOp->node.resType.type = TSDB_DATA_TYPE_BOOL; + pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; + pOp->pLeft = (SNode*)pCol1; + pOp->pRight = (SNode*)pCol2; + + if (jtCtx.colEqNum > 1) { + assert(0 == nodesListMakeStrictAppend(&pLogic->pParameterList, (SNode*)pOp)); + } else { + p->pFullOnCond = (SNode*)pOp; + break; + } + } + } + + if (jtCtx.colEqNum > 1) { + p->pFullOnCond = (SNode*)pLogic; + } +} + +void createColOnCondEnd(SSortMergeJoinPhysiNode* p) { + if (jtCtx.colOnNum <= 0) { + return; + } + + SLogicConditionNode* pLogic = NULL; + if (jtCtx.colOnNum > 1) { + int32_t code = nodesMakeNode(QUERY_NODE_LOGIC_CONDITION, (SNode**)&pLogic); + assert(pLogic); + pLogic->node.resType.type = TSDB_DATA_TYPE_BOOL; + pLogic->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; + pLogic->condType = LOGIC_COND_TYPE_AND; + } + + for (int32_t i = 0; i < MAX_SLOT_NUM; ++i) { + if (jtCtx.colOnList[i]) { + SColumnNode* pCol1 = NULL; + int32_t code = nodesMakeNode(QUERY_NODE_COLUMN, (SNode**)&pCol1); + assert(pCol1); + pCol1->dataBlockId = RES_BLK_ID; + pCol1->slotId = getDstSlotId(i); + pCol1->node.resType.type = jtInputColType[i]; + pCol1->node.resType.bytes = tDataTypes[jtInputColType[i]].bytes; + + SColumnNode* pCol2 = NULL; + code = nodesMakeNode(QUERY_NODE_COLUMN, (SNode**)&pCol2); + assert(pCol2); + pCol2->dataBlockId = RES_BLK_ID; + pCol2->slotId = getDstSlotId(MAX_SLOT_NUM + i); + pCol2->node.resType.type = jtInputColType[i]; + pCol2->node.resType.bytes = tDataTypes[jtInputColType[i]].bytes; + + SOperatorNode* pOp = NULL; + code = nodesMakeNode(QUERY_NODE_OPERATOR, (SNode**)&pOp); + assert(pOp); + pOp->opType = OP_TYPE_GREATER_THAN; + pOp->node.resType.type = TSDB_DATA_TYPE_BOOL; + pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; + pOp->pLeft = (SNode*)pCol1; + pOp->pRight = (SNode*)pCol2; + + if (jtCtx.colOnNum > 1) { + assert(0 == nodesListMakeStrictAppend(&pLogic->pParameterList, (SNode*)pOp)); + } else { + p->pColOnCond = (SNode*)pOp; + break; + } + } + } + + if (jtCtx.colOnNum > 1) { + p->pColOnCond = (SNode*)pLogic; + } + + SNode* pTmp = NULL; + int32_t code = nodesCloneNode(p->pColOnCond, &pTmp); + assert(pTmp); + assert(0 == jtMergeEqCond(&p->pFullOnCond, &pTmp)); +} + + + +void createColCond(SSortMergeJoinPhysiNode* p, int32_t cond) { + jtCtx.colCond = cond; + switch (cond) { + case TEST_NO_COND: + jtCtx.colEqNum = 0; + jtCtx.colOnNum = 0; + TAOS_MEMSET(jtCtx.colEqList, 0, sizeof(jtCtx.colEqList)); + TAOS_MEMSET(jtCtx.colOnList, 0, sizeof(jtCtx.colOnList)); + break; + case TEST_EQ_COND: + createColEqCondStart(p); + jtCtx.colOnNum = 0; + TAOS_MEMSET(jtCtx.colOnList, 0, sizeof(jtCtx.colOnList)); + break; + case TEST_ON_COND: + createColOnCondStart(p); + jtCtx.colEqNum = 0; + TAOS_MEMSET(jtCtx.colEqList, 0, sizeof(jtCtx.colEqList)); + break; + case TEST_FULL_COND: + createColEqCondStart(p); + createColOnCondStart(p); + break; + default: + break; + } +} + +void* getFilterValue(int32_t type) { + switch (type) { + case TSDB_DATA_TYPE_TIMESTAMP: + return &TIMESTAMP_FILTER_VALUE; + case TSDB_DATA_TYPE_INT: + return &INT_FILTER_VALUE; + case TSDB_DATA_TYPE_BIGINT: + return &BIGINT_FILTER_VALUE; + default: + return NULL; + } +} + +void createFilterStart(SSortMergeJoinPhysiNode* p, bool filter) { + jtCtx.filter = filter; + if (!filter) { + jtCtx.leftFilterNum = 0; + jtCtx.rightFilterNum = 0; + TAOS_MEMSET(jtCtx.leftFilterColList, 0, sizeof(jtCtx.leftFilterColList)); + TAOS_MEMSET(jtCtx.rightFilterColList, 0, sizeof(jtCtx.rightFilterColList)); + return; + } + + if ((JOIN_STYPE_SEMI == jtCtx.subType || JOIN_STYPE_ANTI == jtCtx.subType) && JOIN_TYPE_LEFT == jtCtx.joinType) { + jtCtx.rightFilterNum = 0; + jtCtx.leftFilterNum = taosRand() % (MAX_SLOT_NUM + 1); + if (0 == jtCtx.leftFilterNum) { + do { + jtCtx.leftFilterNum = taosRand() % (MAX_SLOT_NUM + 1); + } while (0 == jtCtx.leftFilterNum); + } + } else if ((JOIN_STYPE_SEMI == jtCtx.subType || JOIN_STYPE_ANTI == jtCtx.subType) && JOIN_TYPE_RIGHT == jtCtx.joinType) { + jtCtx.leftFilterNum = 0; + jtCtx.rightFilterNum = taosRand() % (MAX_SLOT_NUM + 1); + if (0 == jtCtx.rightFilterNum) { + do { + jtCtx.rightFilterNum = taosRand() % (MAX_SLOT_NUM + 1); + } while (0 == jtCtx.rightFilterNum); + } + } else { + jtCtx.leftFilterNum = taosRand() % (MAX_SLOT_NUM + 1); + if (0 == jtCtx.leftFilterNum) { + do { + jtCtx.rightFilterNum = taosRand() % (MAX_SLOT_NUM + 1); + } while (0 == jtCtx.rightFilterNum); + } else { + jtCtx.rightFilterNum = taosRand() % (MAX_SLOT_NUM + 1); + } + } + + int32_t idx = 0; + TAOS_MEMSET(jtCtx.leftFilterColList, 0, sizeof(jtCtx.leftFilterColList)); + TAOS_MEMSET(jtCtx.rightFilterColList, 0, sizeof(jtCtx.rightFilterColList)); + for (int32_t i = 0; i < jtCtx.leftFilterNum; ) { + idx = taosRand() % MAX_SLOT_NUM; + if (jtCtx.leftFilterColList[idx]) { + continue; + } + jtCtx.leftFilterColList[idx] = 1; + ++i; + } + + for (int32_t i = 0; i < jtCtx.rightFilterNum; ) { + idx = taosRand() % MAX_SLOT_NUM; + if (jtCtx.rightFilterColList[idx]) { + continue; + } + jtCtx.rightFilterColList[idx] = 1; + ++i; + } +} + +void createFilterEnd(SSortMergeJoinPhysiNode* p, bool filter) { + if (!filter || (jtCtx.leftFilterNum <= 0 && jtCtx.rightFilterNum <= 0)) { + return; + } + + SLogicConditionNode* pLogic = NULL; + if ((jtCtx.leftFilterNum + jtCtx.rightFilterNum) > 1) { + int32_t code = nodesMakeNode(QUERY_NODE_LOGIC_CONDITION, (SNode**)&pLogic); + assert(pLogic); + pLogic->node.resType.type = TSDB_DATA_TYPE_BOOL; + pLogic->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; + pLogic->condType = LOGIC_COND_TYPE_AND; + } + + for (int32_t i = 0; i < MAX_SLOT_NUM; ++i) { + if (jtCtx.leftFilterColList[i]) { + SColumnNode* pCol = NULL; + int32_t code = nodesMakeNode(QUERY_NODE_COLUMN,(SNode**)&pCol); + assert(pCol); + pCol->dataBlockId = RES_BLK_ID; + pCol->slotId = getDstSlotId(i); + pCol->node.resType.type = jtInputColType[i]; + pCol->node.resType.bytes = tDataTypes[jtInputColType[i]].bytes; + (void)sprintf(pCol->colName, "l%d", i); + + SValueNode* pVal = NULL; + code = nodesMakeNode(QUERY_NODE_VALUE, (SNode**)&pVal); + assert(pVal); + pVal->node.resType.type = jtInputColType[i]; + pVal->node.resType.bytes = tDataTypes[jtInputColType[i]].bytes; + assert(0 == nodesSetValueNodeValue(pVal, getFilterValue(jtInputColType[i]))); + + SOperatorNode* pOp = NULL; + code = nodesMakeNode(QUERY_NODE_OPERATOR, (SNode**)&pOp); + assert(pOp); + pOp->opType = OP_TYPE_GREATER_THAN; + pOp->node.resType.type = TSDB_DATA_TYPE_BOOL; + pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; + pOp->pLeft = (SNode*)pCol; + pOp->pRight = (SNode*)pVal; + + if ((jtCtx.leftFilterNum + jtCtx.rightFilterNum) > 1) { + assert(0 == nodesListMakeStrictAppend(&pLogic->pParameterList, (SNode*)pOp)); + } else { + p->node.pConditions = (SNode*)pOp; + break; + } + } + } + + for (int32_t i = 0; i < MAX_SLOT_NUM; ++i) { + if (jtCtx.rightFilterColList[i]) { + SColumnNode* pCol = NULL; + int32_t code = nodesMakeNode(QUERY_NODE_COLUMN, (SNode**)&pCol); + assert(pCol); + pCol->dataBlockId = RES_BLK_ID; + pCol->slotId = getDstSlotId(MAX_SLOT_NUM + i); + pCol->node.resType.type = jtInputColType[i]; + pCol->node.resType.bytes = tDataTypes[jtInputColType[i]].bytes; + (void)sprintf(pCol->colName, "r%d", i); + + SValueNode* pVal = NULL; + code = nodesMakeNode(QUERY_NODE_VALUE, (SNode**)&pVal); + assert(pVal); + pVal->node.resType.type = jtInputColType[i]; + pVal->node.resType.bytes = tDataTypes[jtInputColType[i]].bytes; + assert(0 == nodesSetValueNodeValue(pVal, getFilterValue(jtInputColType[i]))); + + SOperatorNode* pOp = NULL; + code = nodesMakeNode(QUERY_NODE_OPERATOR, (SNode**)&pOp); + assert(pOp); + pOp->opType = OP_TYPE_GREATER_THAN; + pOp->node.resType.type = TSDB_DATA_TYPE_BOOL; + pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; + pOp->pLeft = (SNode*)pCol; + pOp->pRight = (SNode*)pVal; + + if ((jtCtx.leftFilterNum + jtCtx.rightFilterNum) > 1) { + assert(0 == nodesListMakeStrictAppend(&pLogic->pParameterList, (SNode*)pOp)); + } else { + p->node.pConditions = (SNode*)pOp; + break; + } + } + } + + if ((jtCtx.leftFilterNum + jtCtx.rightFilterNum) > 1) { + p->node.pConditions = (SNode*)pLogic; + } +} + + +void updateColRowInfo() { + jtCtx.blkRowSize = MAX_SLOT_NUM * sizeof(bool); + + for (int32_t i = 0; i < MAX_SLOT_NUM; ++i) { + jtCtx.colRowOffset[i] = jtCtx.blkRowSize; + jtCtx.blkRowSize += tDataTypes[jtInputColType[i]].bytes; + } +} + +void createBlockDescNode(SDataBlockDescNode** ppNode) { + SDataBlockDescNode* pDesc = NULL; + int32_t code = nodesMakeNode(QUERY_NODE_DATABLOCK_DESC, (SNode**)&pDesc); + assert(pDesc); + pDesc->dataBlockId = RES_BLK_ID; + pDesc->totalRowSize = jtCtx.resColSize - MAX_SLOT_NUM * 2 * sizeof(bool); + pDesc->outputRowSize = pDesc->totalRowSize; + for (int32_t i = 0; i < jtCtx.resColNum; ++i) { + SSlotDescNode* pSlot = NULL; + int32_t code = nodesMakeNode(QUERY_NODE_SLOT_DESC, (SNode**)&pSlot); + assert(pSlot); + pSlot->slotId = i; + int32_t slotIdx = jtCtx.resColInSlot[i] >= MAX_SLOT_NUM ? jtCtx.resColInSlot[i] - MAX_SLOT_NUM : jtCtx.resColInSlot[i]; + pSlot->dataType.type = jtInputColType[slotIdx]; + pSlot->dataType.bytes = tDataTypes[pSlot->dataType.type].bytes; + + assert(0 == nodesListMakeStrictAppend(&pDesc->pSlots, (SNode *)pSlot)); + } + + *ppNode = pDesc; +} + +SSortMergeJoinPhysiNode* createDummySortMergeJoinPhysiNode(SJoinTestParam* param) { + SSortMergeJoinPhysiNode* p = NULL; + int32_t code = nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN, (SNode**)&p); + assert(p); + p->joinType = param->joinType; + p->subType = param->subType; + p->asofOpType = param->asofOp; + p->grpJoin = param->grpJoin; + if (p->subType == JOIN_STYPE_WIN || param->jLimit > 1 || taosRand() % 2) { + SLimitNode* limitNode = NULL; + code = nodesMakeNode(QUERY_NODE_LIMIT, (SNode**)&limitNode); + assert(limitNode); + limitNode->limit = param->jLimit; + p->pJLimit = (SNode*)limitNode; + } + + p->leftPrimSlotId = JT_PRIM_TS_SLOT_ID; + p->rightPrimSlotId = JT_PRIM_TS_SLOT_ID; + p->node.inputTsOrder = param->asc ? ORDER_ASC : ORDER_DESC; + if (JOIN_STYPE_WIN == p->subType) { + SWindowOffsetNode* pOffset = NULL; + code = nodesMakeNode(QUERY_NODE_WINDOW_OFFSET, (SNode**)&pOffset); + assert(pOffset); + SValueNode* pStart = NULL; + code = nodesMakeNode(QUERY_NODE_VALUE, (SNode**)&pStart); + assert(pStart); + SValueNode* pEnd = NULL; + code = nodesMakeNode(QUERY_NODE_VALUE, (SNode**)&pEnd); + assert(pEnd); + pStart->node.resType.type = TSDB_DATA_TYPE_BIGINT; + pStart->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; + pStart->datum.i = (taosRand() % 2) ? (((int32_t)-1) * (int64_t)(taosRand() % JT_MAX_WINDOW_OFFSET)) : (taosRand() % JT_MAX_WINDOW_OFFSET); + pEnd->node.resType.type = TSDB_DATA_TYPE_BIGINT; + pEnd->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; + pEnd->datum.i = (taosRand() % 2) ? (((int32_t)-1) * (int64_t)(taosRand() % JT_MAX_WINDOW_OFFSET)) : (taosRand() % JT_MAX_WINDOW_OFFSET); + if (pStart->datum.i > pEnd->datum.i) { + TSWAP(pStart->datum.i, pEnd->datum.i); + } + pOffset->pStartOffset = (SNode*)pStart; + pOffset->pEndOffset = (SNode*)pEnd; + p->pWindowOffset = (SNode*)pOffset; + + jtCtx.winStartOffset = pStart->datum.i; + jtCtx.winEndOffset = pEnd->datum.i; + } + + jtCtx.grpJoin = param->grpJoin; + jtCtx.joinType = param->joinType; + jtCtx.subType = param->subType; + jtCtx.asc = param->asc; + jtCtx.jLimit = param->jLimit; + jtCtx.asofOpType = param->asofOp; + jtCtx.leftColOnly = (JOIN_TYPE_LEFT == param->joinType && JOIN_STYPE_SEMI == param->subType); + jtCtx.rightColOnly = (JOIN_TYPE_RIGHT == param->joinType && JOIN_STYPE_SEMI == param->subType); + jtCtx.inGrpId = 1; + + createColCond(p, param->cond); + createFilterStart(p, param->filter); + createTargetSlotList(p); + createColEqCondEnd(p); + createColOnCondEnd(p); + createFilterEnd(p, param->filter); + updateColRowInfo(); + createBlockDescNode(&p->node.pOutputDataBlockDesc); + + return p; +} + +SExecTaskInfo* createDummyTaskInfo(char* taskId) { + SExecTaskInfo* p = (SExecTaskInfo*)taosMemoryCalloc(1, sizeof(SExecTaskInfo)); + assert(p); + p->id.str = taskId; + + return p; +} + +SSDataBlock* createDummyBlock(int32_t blkId) { + SSDataBlock* p = NULL; + int32_t code = createDataBlock(&p); + assert(code == 0); + + p->info.id.blockId = blkId; + p->info.type = STREAM_INVALID; + p->info.calWin.skey = INT64_MIN; + p->info.calWin.ekey = INT64_MAX; + p->info.watermark = INT64_MIN; + + for (int32_t i = 0; i < MAX_SLOT_NUM; ++i) { + SColumnInfoData idata = + createColumnInfoData(jtInputColType[i], tDataTypes[jtInputColType[i]].bytes, i); + + assert(0 == blockDataAppendColInfo(p, &idata)); + } + + return p; +} + +void appendAsofLeftEachResGrps(char* leftInRow, int32_t rightOffset, int32_t rightRows) { + TAOS_MEMSET(jtCtx.resColBuf, 0, jtCtx.resColSize); + for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { + if (!jtCtx.resColList[c]) { + continue; + } + + if (*((bool*)leftInRow + c)) { + *(char*)(jtCtx.resColBuf + c) = true; + } else { + TAOS_MEMCPY(jtCtx.resColBuf + jtCtx.resColOffset[c], leftInRow + jtCtx.inColOffset[c], tDataTypes[jtInputColType[c]].bytes); + } + } + + int32_t endIdx = TMIN(rightRows, taosArrayGetSize(jtCtx.rightRowsList) - rightOffset) + rightOffset; + for (int32_t r = rightOffset; r < endIdx; ++r) { + bool* rightFilterOut = (bool*)taosArrayGet(jtCtx.rightFilterOut, r); + if (*rightFilterOut) { + continue; + } + + char* rightResRows = (char*)taosArrayGet(jtCtx.rightRowsList, r); + for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { + if (jtCtx.resColList[MAX_SLOT_NUM + c]) { + if (*(bool*)(rightResRows + c)) { + *(bool*)(jtCtx.resColBuf + MAX_SLOT_NUM + c) = true; + TAOS_MEMSET(jtCtx.resColBuf + jtCtx.resColOffset[MAX_SLOT_NUM + c], 0, tDataTypes[jtInputColType[c]].bytes); + } else { + *(bool*)(jtCtx.resColBuf + MAX_SLOT_NUM + c) = false; + TAOS_MEMCPY(jtCtx.resColBuf + jtCtx.resColOffset[MAX_SLOT_NUM + c], rightResRows + jtCtx.inColOffset[c], tDataTypes[jtInputColType[c]].bytes); + } + } + } + + pushResRow(jtCtx.resColBuf, jtCtx.resColSize); + } +} + +void appendLeftNonMatchGrp(char* leftInRow) { + if (!jtCtrl.noKeepResRows) { + TAOS_MEMSET(jtCtx.resColBuf, 0, jtCtx.resColSize); + for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { + if (!jtCtx.resColList[c]) { + continue; + } + + if (*((bool*)leftInRow + c)) { + *(char*)(jtCtx.resColBuf + c) = true; + } else { + TAOS_MEMCPY(jtCtx.resColBuf + jtCtx.resColOffset[c], leftInRow + jtCtx.inColOffset[c], tDataTypes[jtInputColType[c]].bytes); + } + } + + for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { + if (jtCtx.resColList[MAX_SLOT_NUM + c]) { + *(char*)(jtCtx.resColBuf + MAX_SLOT_NUM + c) = true; + } + } + } + + pushResRow(jtCtx.resColBuf, jtCtx.resColSize); +} + +void appendAllAsofResRows() { + int32_t leftRows = taosArrayGetSize(jtCtx.leftRowsList); + int32_t rightRows = taosArrayGetSize(jtCtx.rightRowsList); + if (rightRows <= 0) { + if (0 == jtCtx.rightFilterNum) { + for (int32_t i = 0; i < leftRows; ++i) { + char* leftInRow = (char*)taosArrayGet(jtCtx.leftRowsList, i); + assert(leftInRow); + appendLeftNonMatchGrp(leftInRow); + } + } + } else { + assert(rightRows <= jtCtx.jLimit); + for (int32_t i = 0; i < leftRows; ++i) { + char* leftInRow = (char*)taosArrayGet(jtCtx.leftRowsList, i); + assert(leftInRow); + appendAsofLeftEachResGrps(leftInRow, 0, rightRows); + } + } + taosArrayClear(jtCtx.leftRowsList); +} + +void chkAppendAsofForwardGrpResRows(bool forceOut) { + int32_t rightRows = taosArrayGetSize(jtCtx.rightRowsList); + if (rightRows < jtCtx.jLimit && !forceOut) { + return; + } + + int32_t rightRemains = rightRows; + int32_t rightOffset = 0; + int32_t leftRows = taosArrayGetSize(jtCtx.leftRowsList); + int32_t i = 0; + for (; i < leftRows; ++i) { + char* leftRow = (char*)taosArrayGet(jtCtx.leftRowsList, i); + assert(leftRow); + int64_t* leftTs = (int64_t*)(leftRow + jtCtx.inColOffset[JT_PRIM_TS_SLOT_ID]); + bool append = false; + for (int32_t r = rightOffset; r < rightRows; ++r) { + char* rightRow = (char*)taosArrayGet(jtCtx.rightRowsList, r); + assert(rightRow); + int64_t* rightTs = (int64_t*)(rightRow + jtCtx.inColOffset[JT_PRIM_TS_SLOT_ID]); + if (((jtCtx.asc && *leftTs > *rightTs) || (!jtCtx.asc && *leftTs < *rightTs)) || (*leftTs == *rightTs && (OP_TYPE_LOWER_THAN == jtCtx.asofOpType || OP_TYPE_GREATER_THAN == jtCtx.asofOpType))) { + rightOffset++; + rightRemains--; + if (rightRemains < jtCtx.jLimit && !forceOut) { + taosArrayPopFrontBatch(jtCtx.rightRowsList, rightOffset); + taosArrayPopFrontBatch(jtCtx.rightFilterOut, rightOffset); + taosArrayPopFrontBatch(jtCtx.leftRowsList, i); + return; + } + + continue; + } + + appendAsofLeftEachResGrps(leftRow, rightOffset, jtCtx.jLimit); + append = true; + break; + } + + if (!append) { + if (!forceOut) { + break; + } + + if (0 == jtCtx.rightFilterNum) { + appendLeftNonMatchGrp(leftRow); + } + } + } + + taosArrayPopFrontBatch(jtCtx.rightRowsList, rightOffset); + taosArrayPopFrontBatch(jtCtx.rightFilterOut, rightOffset); + taosArrayPopFrontBatch(jtCtx.leftRowsList, i); +} + + +void appendWinEachResGrps(char* leftInRow, int32_t rightOffset, int32_t rightRows) { + if (rightOffset < 0) { + if (0 == jtCtx.rightFilterNum) { + appendLeftNonMatchGrp(leftInRow); + } + return; + } + + TAOS_MEMSET(jtCtx.resColBuf, 0, jtCtx.resColSize); + for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { + if (!jtCtx.resColList[c]) { + continue; + } + + if (*((bool*)leftInRow + c)) { + *(char*)(jtCtx.resColBuf + c) = true; + } else { + TAOS_MEMCPY(jtCtx.resColBuf + jtCtx.resColOffset[c], leftInRow + jtCtx.inColOffset[c], tDataTypes[jtInputColType[c]].bytes); + } + } + + int32_t endIdx = rightRows + rightOffset; + int32_t beginIdx = (!jtCtx.asc && rightRows > jtCtx.jLimit) ? (endIdx - jtCtx.jLimit) : rightOffset; + for (int32_t r = beginIdx; r < endIdx; ++r) { + bool* rightFilterOut = (bool*)taosArrayGet(jtCtx.rightFilterOut, r); + if (*rightFilterOut) { + continue; + } + + char* rightResRows = (char*)taosArrayGet(jtCtx.rightRowsList, r); + assert(rightResRows); + for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { + if (jtCtx.resColList[MAX_SLOT_NUM + c]) { + if (*(bool*)(rightResRows + c)) { + *(bool*)(jtCtx.resColBuf + MAX_SLOT_NUM + c) = true; + TAOS_MEMSET(jtCtx.resColBuf + jtCtx.resColOffset[MAX_SLOT_NUM + c], 0, tDataTypes[jtInputColType[c]].bytes); + } else { + *(bool*)(jtCtx.resColBuf + MAX_SLOT_NUM + c) = false; + TAOS_MEMCPY(jtCtx.resColBuf + jtCtx.resColOffset[MAX_SLOT_NUM + c], rightResRows + jtCtx.inColOffset[c], tDataTypes[jtInputColType[c]].bytes); + } + } + } + + pushResRow(jtCtx.resColBuf, jtCtx.resColSize); + } +} + +void chkAppendWinResRows(bool forceOut) { + int32_t rightRows = taosArrayGetSize(jtCtx.rightRowsList); + if (rightRows < jtCtx.jLimit && !forceOut) { + return; + } + + int32_t rightRemains = rightRows; + int32_t rightOffset = 0; + int32_t leftRows = taosArrayGetSize(jtCtx.leftRowsList); + int32_t i = 0; + for (; i < leftRows; ++i) { + char* leftRow = (char*)taosArrayGet(jtCtx.leftRowsList, i); + assert(leftRow); + int64_t* leftTs = (int64_t*)(leftRow + jtCtx.inColOffset[JT_PRIM_TS_SLOT_ID]); + int64_t winStart = *leftTs + jtCtx.winStartOffset; + int64_t winEnd = *leftTs + jtCtx.winEndOffset; + int32_t winBeginIdx = -1; + bool append = false; + bool winClosed = false; + for (int32_t r = rightOffset; r < rightRows; ++r) { + char* rightRow = (char*)taosArrayGet(jtCtx.rightRowsList, r); + assert(rightRow); + int64_t* rightTs = (int64_t*)(rightRow + jtCtx.inColOffset[JT_PRIM_TS_SLOT_ID]); + if ((jtCtx.asc && *rightTs < winStart) || (!jtCtx.asc && *rightTs > winEnd)) { + rightOffset++; + rightRemains--; + if (rightRemains < jtCtx.jLimit && !forceOut) { + taosArrayPopFrontBatch(jtCtx.rightRowsList, rightOffset); + taosArrayPopFrontBatch(jtCtx.rightFilterOut, rightOffset); + taosArrayPopFrontBatch(jtCtx.leftRowsList, i); + return; + } + + continue; + } else if ((jtCtx.asc && *rightTs > winEnd) || (!jtCtx.asc && *rightTs < winStart)) { + winClosed = true; + appendWinEachResGrps(leftRow, winBeginIdx, r - winBeginIdx); + append = true; + break; + } + + if (-1 == winBeginIdx) { + winBeginIdx = r; + } + + if (jtCtx.asc && (r - winBeginIdx + 1) >= jtCtx.jLimit) { + appendWinEachResGrps(leftRow, winBeginIdx, jtCtx.jLimit); + append = true; + break; + } + } + + if (!append) { + if (!forceOut) { + break; + } + + if (winBeginIdx >= 0) { + appendWinEachResGrps(leftRow, winBeginIdx, rightRows - winBeginIdx); + } else if (0 == jtCtx.rightFilterNum) { + appendLeftNonMatchGrp(leftRow); + } + } + } + + taosArrayPopFrontBatch(jtCtx.rightRowsList, rightOffset); + taosArrayPopFrontBatch(jtCtx.rightFilterOut, rightOffset); + taosArrayPopFrontBatch(jtCtx.leftRowsList, i); +} + + +void trimForAsofJlimit() { + int32_t rowNum = taosArrayGetSize(jtCtx.rightRowsList); + if (rowNum <= jtCtx.jLimit) { + return; + } + + taosArrayPopFrontBatch(jtCtx.rightRowsList, rowNum - jtCtx.jLimit); + taosArrayPopFrontBatch(jtCtx.rightFilterOut, rowNum - jtCtx.jLimit); +} + +void createGrpRows(SSDataBlock** ppBlk, int32_t blkId, int32_t grpRows) { + if (grpRows <= 0) { + return; + } + + if (NULL == *ppBlk) { + *ppBlk = createDummyBlock((blkId == LEFT_BLK_ID) ? LEFT_BLK_ID : RIGHT_BLK_ID); + assert(*ppBlk); + assert(0 == blockDataEnsureCapacity(*ppBlk, jtCtx.blkRows)); + assert(NULL != taosArrayPush((blkId == LEFT_BLK_ID) ? jtCtx.leftBlkList : jtCtx.rightBlkList, ppBlk)); + } + + if (jtCtx.grpJoin) { + (*ppBlk)->info.id.groupId = jtCtx.inGrpId; + } + + jtCtx.inputStat |= (1 << blkId); + + SArray* pTableRows = NULL; + int32_t tableOffset = 0; + int32_t peerOffset = 0; + bool keepRes = false; + bool keepInput = false; + if (blkId == LEFT_BLK_ID) { + if ((jtCtx.joinType == JOIN_TYPE_LEFT || jtCtx.joinType == JOIN_TYPE_FULL) && (jtCtx.subType != JOIN_STYPE_SEMI && jtCtx.subType != JOIN_STYPE_ASOF && jtCtx.subType != JOIN_STYPE_WIN)) { + keepRes = true; + } + peerOffset = MAX_SLOT_NUM; + } else { + if ((jtCtx.joinType == JOIN_TYPE_RIGHT || jtCtx.joinType == JOIN_TYPE_FULL) && (jtCtx.subType != JOIN_STYPE_SEMI && jtCtx.subType != JOIN_STYPE_ASOF && jtCtx.subType != JOIN_STYPE_WIN)) { + keepRes = true; + } + tableOffset = MAX_SLOT_NUM; + } + + if (JOIN_STYPE_ASOF == jtCtx.subType) { + keepInput = jtCtx.asofOpType != OP_TYPE_EQUAL ? true : (blkId == LEFT_BLK_ID); + pTableRows = (blkId == LEFT_BLK_ID) ? jtCtx.leftRowsList : jtCtx.rightRowsList; + } else if (JOIN_STYPE_WIN == jtCtx.subType) { + keepInput = true; + pTableRows = (blkId == LEFT_BLK_ID) ? jtCtx.leftRowsList : jtCtx.rightRowsList; + } + + int32_t filterNum = (blkId == LEFT_BLK_ID) ? jtCtx.leftFilterNum : jtCtx.rightFilterNum; + int32_t peerFilterNum = (blkId == LEFT_BLK_ID) ? jtCtx.rightFilterNum : jtCtx.leftFilterNum; + int32_t* filterCol = (blkId == LEFT_BLK_ID) ? jtCtx.leftFilterColList : jtCtx.rightFilterColList; + + char* pData = NULL; + int32_t tmpInt = 0; + int64_t tmpBigint = 0; + bool isNull = false; + bool filterOut = false; + bool addToRowList = false; + int32_t vRange = TMAX(grpRows / 3, 3); + for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { + jtCtx.grpOffset[c] = c * TMAX(100, grpRows); + } + + for (int32_t i = 0; i < grpRows; ++i) { + if ((*ppBlk)->info.rows >= (*ppBlk)->info.capacity) { + *ppBlk = createDummyBlock((blkId == LEFT_BLK_ID) ? LEFT_BLK_ID : RIGHT_BLK_ID); + assert(*ppBlk); + assert(0 == blockDataEnsureCapacity(*ppBlk, jtCtx.blkRows)); + assert(NULL != taosArrayPush((blkId == LEFT_BLK_ID) ? jtCtx.leftBlkList : jtCtx.rightBlkList, ppBlk)); + if (jtCtx.grpJoin) { + (*ppBlk)->info.id.groupId = jtCtx.inGrpId; + } + } + + filterOut = (peerFilterNum > 0 && (jtCtx.subType != JOIN_STYPE_ASOF && jtCtx.subType != JOIN_STYPE_WIN)) ? true : false; + if (!filterOut) { + TAOS_MEMSET(jtCtx.resColBuf, 0, jtCtx.resColSize); + if (keepInput) { + TAOS_MEMSET(jtCtx.inColBuf, 0, jtCtx.inColSize); + } + } + + addToRowList = true; + + for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { + switch (jtInputColType[c]) { + case TSDB_DATA_TYPE_TIMESTAMP: + jtCtx.asc ? ++jtCtx.curTs : --jtCtx.curTs; + pData = (char*)&jtCtx.curTs; + isNull = false; + if (!filterOut && filterNum && filterCol[c] && jtCtx.curTs <= TIMESTAMP_FILTER_VALUE) { + filterOut = true; + } + break; + case TSDB_DATA_TYPE_INT: + if (taosRand() % 10) { + tmpInt = (taosRand() % 2) ? INT_FILTER_VALUE + jtCtx.grpOffset[c] + taosRand() % vRange : INT_FILTER_VALUE - jtCtx.grpOffset[c] - taosRand() % vRange; + pData = (char*)&tmpInt; + isNull = false; + if (!filterOut && filterNum && filterCol[c] && tmpInt <= INT_FILTER_VALUE) { + filterOut = true; + } + } else { + isNull = true; + if (filterNum && filterCol[c]) { + filterOut = true; + } + } + break; + case TSDB_DATA_TYPE_BIGINT: + tmpBigint = (taosRand() % 2) ? BIGINT_FILTER_VALUE + jtCtx.curKeyOffset++ : BIGINT_FILTER_VALUE - jtCtx.curKeyOffset++; + pData = (char*)&tmpBigint; + isNull = false; + if (!filterOut && filterNum && filterCol[c] && tmpBigint <= BIGINT_FILTER_VALUE) { + filterOut = true; + } + break; + default: + break; + } + + SColumnInfoData* pCol = (SColumnInfoData*)taosArrayGet((*ppBlk)->pDataBlock, c); + assert(pCol); + assert(0 == colDataSetVal(pCol, (*ppBlk)->info.rows, pData, isNull)); + + if (keepInput) { + if (!filterOut || (blkId != LEFT_BLK_ID)) { + if (isNull) { + *(char*)(jtCtx.inColBuf + c) = true; + } else { + TAOS_MEMCPY(jtCtx.inColBuf + jtCtx.inColOffset[c], pData, tDataTypes[jtInputColType[c]].bytes); + } + } else { + addToRowList = false; + } + } else if (keepRes && !filterOut && jtCtx.resColList[tableOffset + c]) { + if (isNull) { + *(char*)(jtCtx.resColBuf + tableOffset + c) = true; + } else { + TAOS_MEMCPY(jtCtx.resColBuf + jtCtx.resColOffset[tableOffset + c], pData, tDataTypes[jtInputColType[c]].bytes); + } + } + } + + if (keepInput && addToRowList) { + assert(NULL != taosArrayPush(pTableRows, jtCtx.inColBuf)); + if (blkId == RIGHT_BLK_ID) { + bool fout = filterOut ? true : false; + assert(NULL != taosArrayPush(jtCtx.rightFilterOut, &fout)); + } + } + + if (keepRes && !filterOut) { + for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { + if (jtCtx.resColList[peerOffset + c]) { + *(char*)(jtCtx.resColBuf + peerOffset + c) = true; + } + } + + pushResRow(jtCtx.resColBuf, jtCtx.resColSize); + } + + (*ppBlk)->info.rows++; + } + + if (keepInput) { + if (JOIN_STYPE_ASOF == jtCtx.subType) { + if (((jtCtx.asc && (jtCtx.asofOpType == OP_TYPE_GREATER_EQUAL || jtCtx.asofOpType == OP_TYPE_GREATER_THAN)) || (!jtCtx.asc && (jtCtx.asofOpType == OP_TYPE_LOWER_EQUAL || jtCtx.asofOpType == OP_TYPE_LOWER_THAN)) ) || jtCtx.asofOpType == OP_TYPE_EQUAL) { + if (blkId == LEFT_BLK_ID) { + appendAllAsofResRows(); + } else { + trimForAsofJlimit(); + } + } else { + chkAppendAsofForwardGrpResRows(false); + } + } else { + chkAppendWinResRows(false); + } + } + +} + +void createRowData(SSDataBlock* pBlk, int64_t tbOffset, int32_t rowIdx, int32_t vRange) { + int32_t tmpInt = 0; + int64_t tmpBig = 0; + + for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { + SColumnInfoData* pCol = (SColumnInfoData*)taosArrayGet(pBlk->pDataBlock, c); + assert(pCol); + + int32_t rv = taosRand() % 10; + switch (jtInputColType[c]) { + case TSDB_DATA_TYPE_TIMESTAMP: + *(int64_t*)(jtCtx.colRowDataBuf + tbOffset + rowIdx * jtCtx.blkRowSize + jtCtx.colRowOffset[c]) = jtCtx.curTs; + assert(0 == colDataSetVal(pCol, pBlk->info.rows, (char*)&jtCtx.curTs, false)); + break; + case TSDB_DATA_TYPE_INT: + if (rv) { + tmpInt = (taosRand() % 2) ? INT_FILTER_VALUE + jtCtx.grpOffset[c] + taosRand() % vRange : INT_FILTER_VALUE - taosRand() % vRange; + *(int32_t*)(jtCtx.colRowDataBuf + tbOffset + rowIdx * jtCtx.blkRowSize + jtCtx.colRowOffset[c]) = tmpInt; + assert(0 == colDataSetVal(pCol, pBlk->info.rows, (char*)&tmpInt, false)); + } else { + *(bool*)(jtCtx.colRowDataBuf + tbOffset + rowIdx * jtCtx.blkRowSize + c) = true; + assert(0 == colDataSetVal(pCol, pBlk->info.rows, NULL, true)); + } + break; + case TSDB_DATA_TYPE_BIGINT: + tmpBig = (taosRand() % 2) ? BIGINT_FILTER_VALUE + jtCtx.curKeyOffset++ : BIGINT_FILTER_VALUE - jtCtx.curKeyOffset++; + *(int64_t*)(jtCtx.colRowDataBuf + tbOffset + rowIdx * jtCtx.blkRowSize + jtCtx.colRowOffset[c]) = tmpBig; + assert(0 == colDataSetVal(pCol, pBlk->info.rows, (char*)&tmpBig, false)); + break; + default: + break; + } + } + + pBlk->info.rows++; +} + +void makeAppendBlkData(SSDataBlock** ppLeft, SSDataBlock** ppRight, int32_t leftGrpRows, int32_t rightGrpRows) { + int64_t totalSize = (leftGrpRows + rightGrpRows) * jtCtx.blkRowSize; + int64_t rightOffset = leftGrpRows * jtCtx.blkRowSize; + + if (jtCtx.colRowDataBufSize < totalSize) { + jtCtx.colRowDataBuf = (char*)taosMemoryRealloc(jtCtx.colRowDataBuf, totalSize); + assert(jtCtx.colRowDataBuf); + } + + TAOS_MEMSET(jtCtx.colRowDataBuf, 0, totalSize); + + for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { + jtCtx.grpOffset[c] = c * TMAX(leftGrpRows, rightGrpRows); + } + + int32_t vRange = TMAX(leftGrpRows / 100, 3); + for (int32_t i = 0; i < leftGrpRows; ++i) { + if ((*ppLeft)->info.rows >= (*ppLeft)->info.capacity) { + *ppLeft = createDummyBlock(LEFT_BLK_ID); + assert(*ppLeft); + assert(0 == blockDataEnsureCapacity(*ppLeft, jtCtx.blkRows)); + assert(NULL != taosArrayPush(jtCtx.leftBlkList, ppLeft)); + if (jtCtx.grpJoin) { + (*ppLeft)->info.id.groupId = jtCtx.inGrpId; + } + } + + createRowData(*ppLeft, 0, i, vRange); + } + + vRange = TMAX(rightGrpRows / 100, 3); + for (int32_t i = 0; i < rightGrpRows; ++i) { + if ((*ppRight)->info.rows >= (*ppRight)->info.capacity) { + *ppRight = createDummyBlock(RIGHT_BLK_ID); + assert(*ppRight); + assert(0 == blockDataEnsureCapacity(*ppRight, jtCtx.blkRows)); + assert(NULL != taosArrayPush(jtCtx.rightBlkList, ppRight)); + if (jtCtx.grpJoin) { + (*ppRight)->info.id.groupId = jtCtx.inGrpId; + } + } + + createRowData(*ppRight, rightOffset, i, vRange); + } + +} + +void putNMatchRowToRes(char* lrow, int32_t tableOffset, int32_t peerOffset) { + if (!jtCtrl.noKeepResRows) { + TAOS_MEMSET(jtCtx.resColBuf, 0, jtCtx.resColSize); + + for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { + if (jtCtx.resColList[tableOffset + c]) { + if (*(bool*)(lrow + c)) { + *(bool*)(jtCtx.resColBuf + tableOffset + c) = true; + } else { + TAOS_MEMCPY(jtCtx.resColBuf + jtCtx.resColOffset[tableOffset + c], lrow + jtCtx.colRowOffset[c], tDataTypes[jtInputColType[c]].bytes); + } + } + } + + for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { + if (jtCtx.resColList[peerOffset + c]) { + *(bool*)(jtCtx.resColBuf + peerOffset + c) = true; + } + } + } + + pushResRow(jtCtx.resColBuf, jtCtx.resColSize); +} + +void putMatchRowToRes(char* lrow, char* rrow, int32_t cols) { + if (!jtCtrl.noKeepResRows) { + TAOS_MEMSET(jtCtx.resColBuf, 0, jtCtx.resColSize); + + if (cols & LEFT_TABLE_COLS) { + for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { + if (jtCtx.resColList[c]) { + if (*(bool*)(lrow + c)) { + *(bool*)(jtCtx.resColBuf + c) = true; + } else { + TAOS_MEMCPY(jtCtx.resColBuf + jtCtx.resColOffset[c], lrow + jtCtx.colRowOffset[c], tDataTypes[jtInputColType[c]].bytes); + } + } + } + } + + if (cols & RIGHT_TABLE_COLS) { + for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { + if (jtCtx.resColList[MAX_SLOT_NUM + c]) { + if (*(bool*)(rrow + c)) { + *(bool*)(jtCtx.resColBuf + MAX_SLOT_NUM + c) = true; + } else { + TAOS_MEMCPY(jtCtx.resColBuf + jtCtx.resColOffset[MAX_SLOT_NUM + c], rrow + jtCtx.colRowOffset[c], tDataTypes[jtInputColType[c]].bytes); + } + } + } + } + } + + pushResRow(jtCtx.resColBuf, jtCtx.resColSize); +} + + + +void innerJoinAppendEqGrpRes(int32_t leftGrpRows, int32_t rightGrpRows) { + bool leftMatch = false, rightMatch = false, filterOut = false; + void* lValue = NULL, *rValue = NULL, *filterValue = NULL; + int64_t lBig = 0, rBig = 0, fbig = 0; + int64_t rightTbOffset = jtCtx.blkRowSize * leftGrpRows; + + for (int32_t l = 0; l < leftGrpRows; ++l) { + char* lrow = jtCtx.colRowDataBuf + jtCtx.blkRowSize * l; + + filterOut = false; + leftMatch = true; + + for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { + lValue = lrow + jtCtx.colRowOffset[c]; + switch (jtInputColType[c]) { + case TSDB_DATA_TYPE_TIMESTAMP: + fbig = TIMESTAMP_FILTER_VALUE; + lBig = *(int64_t*)lValue; + break; + case TSDB_DATA_TYPE_INT: + fbig = INT_FILTER_VALUE; + lBig = *(int32_t*)lValue; + break; + case TSDB_DATA_TYPE_BIGINT: + fbig = BIGINT_FILTER_VALUE; + lBig = *(int64_t*)lValue; + break; + default: + break; + } + + if (jtCtx.leftFilterNum && jtCtx.leftFilterColList[c] && ((*(bool*)(lrow + c)) || lBig <= fbig)) { + filterOut = true; + break; + } + + if (jtCtx.colEqNum && jtCtx.colEqList[c] && (*(bool*)(lrow + c))) { + leftMatch = false; + break; + } + + if (jtCtx.colOnNum && jtCtx.colOnList[c] && (*(bool*)(lrow + c))) { + leftMatch = false; + break; + } + } + + if (filterOut || !leftMatch) { + continue; + } + + for (int32_t r = 0; r < rightGrpRows; ++r) { + char* rrow = jtCtx.colRowDataBuf + rightTbOffset + jtCtx.blkRowSize * r; + rightMatch = true; + filterOut = false; + + for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { + lValue = lrow + jtCtx.colRowOffset[c]; + + if (!*(bool*)(rrow + c)) { + rValue = rrow + jtCtx.colRowOffset[c]; + } + + switch (jtInputColType[c]) { + case TSDB_DATA_TYPE_TIMESTAMP: + fbig = TIMESTAMP_FILTER_VALUE; + lBig = *(int64_t*)lValue; + rBig = *(int64_t*)rValue; + break; + case TSDB_DATA_TYPE_INT: + fbig = INT_FILTER_VALUE; + lBig = *(int32_t*)lValue; + rBig = *(int32_t*)rValue; + break; + case TSDB_DATA_TYPE_BIGINT: + fbig = BIGINT_FILTER_VALUE; + lBig = *(int64_t*)lValue; + rBig = *(int64_t*)rValue; + break; + default: + break; + } + + if (jtCtx.colEqNum && jtCtx.colEqList[c] && ((*(bool*)(rrow + c)) || lBig != rBig)) { + rightMatch = false; + break; + } + + if (jtCtx.colOnNum && jtCtx.colOnList[c] && ((*(bool*)(rrow + c)) || lBig <= rBig)) { + rightMatch = false; + break; + } + + if (jtCtx.rightFilterNum && jtCtx.rightFilterColList[c] && ((*(bool*)(rrow + c)) || rBig <= fbig)) { + filterOut = true; + break; + } + } + + if (filterOut || !rightMatch) { + continue; + } + + putMatchRowToRes(lrow, rrow, ALL_TABLE_COLS); + } + } + + +} + + + +void leftJoinAppendEqGrpRes(int32_t leftGrpRows, int32_t rightGrpRows) { + bool leftMatch = false, rightMatch = false, filterOut = false; + void* lValue = NULL, *rValue = NULL, *filterValue = NULL; + int64_t lBig = 0, rBig = 0, fbig = 0; + int64_t rightTbOffset = jtCtx.blkRowSize * leftGrpRows; + + for (int32_t l = 0; l < leftGrpRows; ++l) { + char* lrow = jtCtx.colRowDataBuf + jtCtx.blkRowSize * l; + + filterOut = false; + leftMatch = true; + + for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { + lValue = lrow + jtCtx.colRowOffset[c]; + switch (jtInputColType[c]) { + case TSDB_DATA_TYPE_TIMESTAMP: + fbig = TIMESTAMP_FILTER_VALUE; + lBig = *(int64_t*)lValue; + break; + case TSDB_DATA_TYPE_INT: + fbig = INT_FILTER_VALUE; + lBig = *(int32_t*)lValue; + break; + case TSDB_DATA_TYPE_BIGINT: + fbig = BIGINT_FILTER_VALUE; + lBig = *(int64_t*)lValue; + break; + default: + break; + } + + if (jtCtx.leftFilterNum && jtCtx.leftFilterColList[c] && ((*(bool*)(lrow + c)) || lBig <= fbig)) { + filterOut = true; + break; + } + + if (jtCtx.colEqNum && jtCtx.colEqList[c] && (*(bool*)(lrow + c))) { + leftMatch = false; + } + + if (jtCtx.colOnNum && jtCtx.colOnList[c] && (*(bool*)(lrow + c))) { + leftMatch = false; + } + } + + if (filterOut) { + continue; + } + + if (false == leftMatch) { + if (0 == jtCtx.rightFilterNum) { + putNMatchRowToRes(lrow, 0, MAX_SLOT_NUM); + } + continue; + } + + leftMatch = false; + for (int32_t r = 0; r < rightGrpRows; ++r) { + char* rrow = jtCtx.colRowDataBuf + rightTbOffset + jtCtx.blkRowSize * r; + rightMatch = true; + filterOut = false; + + for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { + lValue = lrow + jtCtx.colRowOffset[c]; + + if (!*(bool*)(rrow + c)) { + rValue = rrow + jtCtx.colRowOffset[c]; + } + + switch (jtInputColType[c]) { + case TSDB_DATA_TYPE_TIMESTAMP: + fbig = TIMESTAMP_FILTER_VALUE; + lBig = *(int64_t*)lValue; + rBig = *(int64_t*)rValue; + break; + case TSDB_DATA_TYPE_INT: + fbig = INT_FILTER_VALUE; + lBig = *(int32_t*)lValue; + rBig = *(int32_t*)rValue; + break; + case TSDB_DATA_TYPE_BIGINT: + fbig = BIGINT_FILTER_VALUE; + lBig = *(int64_t*)lValue; + rBig = *(int64_t*)rValue; + break; + default: + break; + } + + if (jtCtx.colEqNum && jtCtx.colEqList[c] && ((*(bool*)(rrow + c)) || lBig != rBig)) { + rightMatch = false; + break; + } + + if (jtCtx.colOnNum && jtCtx.colOnList[c] && ((*(bool*)(rrow + c)) || lBig <= rBig)) { + rightMatch = false; + break; + } + + if (jtCtx.rightFilterNum && jtCtx.rightFilterColList[c] && ((*(bool*)(rrow + c)) || rBig <= fbig)) { + filterOut = true; + } + } + + if (rightMatch) { + leftMatch = true; + } + + if (filterOut) { + continue; + } + + if (rightMatch) { + putMatchRowToRes(lrow, rrow, ALL_TABLE_COLS); + } + } + + if (!leftMatch && 0 == jtCtx.rightFilterNum) { + putNMatchRowToRes(lrow, 0, MAX_SLOT_NUM); + } + } + + +} + + + +void semiJoinAppendEqGrpRes(int32_t leftGrpRows, int32_t rightGrpRows) { + bool leftMatch = false, rightMatch = false, filterOut = false; + void* lValue = NULL, *rValue = NULL, *filterValue = NULL; + int64_t lBig = 0, rBig = 0, fbig = 0; + int64_t leftTbOffset = 0; + int64_t rightTbOffset = jtCtx.blkRowSize * leftGrpRows; + char* rrow = NULL; + + for (int32_t l = 0; l < leftGrpRows; ++l) { + char* lrow = jtCtx.colRowDataBuf + leftTbOffset + jtCtx.blkRowSize * l; + + filterOut = false; + leftMatch = true; + + for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { + lValue = lrow + jtCtx.colRowOffset[c]; + switch (jtInputColType[c]) { + case TSDB_DATA_TYPE_TIMESTAMP: + fbig = TIMESTAMP_FILTER_VALUE; + lBig = *(int64_t*)lValue; + break; + case TSDB_DATA_TYPE_INT: + fbig = INT_FILTER_VALUE; + lBig = *(int32_t*)lValue; + break; + case TSDB_DATA_TYPE_BIGINT: + fbig = BIGINT_FILTER_VALUE; + lBig = *(int64_t*)lValue; + break; + default: + break; + } + + if (jtCtx.leftFilterNum && jtCtx.leftFilterColList[c] && ((*(bool*)(lrow + c)) || lBig <= fbig)) { + filterOut = true; + break; + } + + if (jtCtx.colEqNum && jtCtx.colEqList[c] && (*(bool*)(lrow + c))) { + leftMatch = false; + break; + } + + if (jtCtx.colOnNum && jtCtx.colOnList[c] && (*(bool*)(lrow + c))) { + leftMatch = false; + break; + } + } + + if (filterOut || !leftMatch) { + continue; + } + + for (int32_t r = 0; r < rightGrpRows; ++r) { + rrow = jtCtx.colRowDataBuf + rightTbOffset + jtCtx.blkRowSize * r; + rightMatch = true; + filterOut = false; + + for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { + lValue = lrow + jtCtx.colRowOffset[c]; + + if (!*(bool*)(rrow + c)) { + rValue = rrow + jtCtx.colRowOffset[c]; + } + + switch (jtInputColType[c]) { + case TSDB_DATA_TYPE_TIMESTAMP: + fbig = TIMESTAMP_FILTER_VALUE; + lBig = *(int64_t*)lValue; + rBig = *(int64_t*)rValue; + break; + case TSDB_DATA_TYPE_INT: + fbig = INT_FILTER_VALUE; + lBig = *(int32_t*)lValue; + rBig = *(int32_t*)rValue; + break; + case TSDB_DATA_TYPE_BIGINT: + fbig = BIGINT_FILTER_VALUE; + lBig = *(int64_t*)lValue; + rBig = *(int64_t*)rValue; + break; + default: + break; + } + + if (jtCtx.colEqNum && jtCtx.colEqList[c] && ((*(bool*)(rrow + c)) || lBig != rBig)) { + rightMatch = false; + break; + } + + if (jtCtx.colOnNum && jtCtx.colOnList[c] && ((*(bool*)(rrow + c)) || lBig <= rBig)) { + rightMatch = false; + break; + } + + if (jtCtx.rightFilterNum && jtCtx.rightFilterColList[c] && ((*(bool*)(rrow + c)) || rBig <= fbig)) { + filterOut = true; + break; + } + } + + if (filterOut || !rightMatch) { + continue; + } + + break; + } + + if (!filterOut && rightMatch) { + putMatchRowToRes(lrow, rrow, LEFT_TABLE_COLS); + } + } + + +} + + + + +void antiJoinAppendEqGrpRes(int32_t leftGrpRows, int32_t rightGrpRows) { + bool leftMatch = false, rightMatch = false, filterOut = false; + void* lValue = NULL, *rValue = NULL, *filterValue = NULL; + int64_t lBig = 0, rBig = 0, fbig = 0; + int64_t rightTbOffset = jtCtx.blkRowSize * leftGrpRows; + + assert(0 == jtCtx.rightFilterNum); + + for (int32_t l = 0; l < leftGrpRows; ++l) { + char* lrow = jtCtx.colRowDataBuf + jtCtx.blkRowSize * l; + + filterOut = false; + leftMatch = true; + + for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { + lValue = lrow + jtCtx.colRowOffset[c]; + switch (jtInputColType[c]) { + case TSDB_DATA_TYPE_TIMESTAMP: + fbig = TIMESTAMP_FILTER_VALUE; + lBig = *(int64_t*)lValue; + break; + case TSDB_DATA_TYPE_INT: + fbig = INT_FILTER_VALUE; + lBig = *(int32_t*)lValue; + break; + case TSDB_DATA_TYPE_BIGINT: + fbig = BIGINT_FILTER_VALUE; + lBig = *(int64_t*)lValue; + break; + default: + break; + } + + if (jtCtx.leftFilterNum && jtCtx.leftFilterColList[c] && ((*(bool*)(lrow + c)) || lBig <= fbig)) { + filterOut = true; + break; + } + + if (jtCtx.colEqNum && jtCtx.colEqList[c] && (*(bool*)(lrow + c))) { + leftMatch = false; + } + + if (jtCtx.colOnNum && jtCtx.colOnList[c] && (*(bool*)(lrow + c))) { + leftMatch = false; + } + } + + if (filterOut) { + continue; + } + + if (false == leftMatch) { + putNMatchRowToRes(lrow, 0, MAX_SLOT_NUM); + continue; + } + + leftMatch = false; + for (int32_t r = 0; r < rightGrpRows; ++r) { + char* rrow = jtCtx.colRowDataBuf + rightTbOffset + jtCtx.blkRowSize * r; + rightMatch = true; + + for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { + lValue = lrow + jtCtx.colRowOffset[c]; + + if (!*(bool*)(rrow + c)) { + rValue = rrow + jtCtx.colRowOffset[c]; + } + + switch (jtInputColType[c]) { + case TSDB_DATA_TYPE_TIMESTAMP: + fbig = TIMESTAMP_FILTER_VALUE; + lBig = *(int64_t*)lValue; + rBig = *(int64_t*)rValue; + break; + case TSDB_DATA_TYPE_INT: + fbig = INT_FILTER_VALUE; + lBig = *(int32_t*)lValue; + rBig = *(int32_t*)rValue; + break; + case TSDB_DATA_TYPE_BIGINT: + fbig = BIGINT_FILTER_VALUE; + lBig = *(int64_t*)lValue; + rBig = *(int64_t*)rValue; + break; + default: + break; + } + + if (jtCtx.colEqNum && jtCtx.colEqList[c] && ((*(bool*)(rrow + c)) || lBig != rBig)) { + rightMatch = false; + break; + } + + if (jtCtx.colOnNum && jtCtx.colOnList[c] && ((*(bool*)(rrow + c)) || lBig <= rBig)) { + rightMatch = false; + break; + } + } + + if (rightMatch) { + leftMatch = true; + break; + } + } + + if (!leftMatch) { + putNMatchRowToRes(lrow, 0, MAX_SLOT_NUM); + } + } + + +} + +void addAsofEqInRows(int32_t rowsNum, int64_t tbOffset, bool leftTable) { + bool filterOut = false; + void* cvalue = NULL; + int64_t cbig = 0, fbig = 0; + int32_t filterNum = leftTable ? jtCtx.leftFilterNum : jtCtx.rightFilterNum; + int32_t* filterCol = leftTable ? jtCtx.leftFilterColList : jtCtx.rightFilterColList; + SArray* rowList = leftTable ? jtCtx.leftRowsList : jtCtx.rightRowsList; + + if (!leftTable) { + rowsNum = TMIN(rowsNum, jtCtx.jLimit); + } + + for (int32_t l = 0; l < rowsNum; ++l) { + char* row = jtCtx.colRowDataBuf + tbOffset + jtCtx.blkRowSize * l; + + filterOut = false; + + for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { + cvalue = row + jtCtx.colRowOffset[c]; + switch (jtInputColType[c]) { + case TSDB_DATA_TYPE_TIMESTAMP: + fbig = TIMESTAMP_FILTER_VALUE; + cbig = *(int64_t*)cvalue; + break; + case TSDB_DATA_TYPE_INT: + fbig = INT_FILTER_VALUE; + cbig = *(int32_t*)cvalue; + break; + case TSDB_DATA_TYPE_BIGINT: + fbig = BIGINT_FILTER_VALUE; + cbig = *(int64_t*)cvalue; + break; + default: + break; + } + + if (filterNum && filterCol[c] && ((*(bool*)(row + c)) || cbig <= fbig)) { + filterOut = true; + break; + } + } + + if (filterOut && leftTable) { + continue; + } + + assert(NULL != taosArrayPush(rowList, row)); + if (!leftTable) { + assert(NULL != taosArrayPush(jtCtx.rightFilterOut, &filterOut)); + } + } + + if (!leftTable && ((jtCtx.asc && (jtCtx.asofOpType == OP_TYPE_GREATER_EQUAL || jtCtx.asofOpType == OP_TYPE_GREATER_THAN)) || (!jtCtx.asc && (jtCtx.asofOpType == OP_TYPE_LOWER_EQUAL || jtCtx.asofOpType == OP_TYPE_LOWER_THAN))) || jtCtx.asofOpType == OP_TYPE_EQUAL) { + trimForAsofJlimit(); + } +} + +void asofJoinAppendEqGrpRes(int32_t leftGrpRows, int32_t rightGrpRows) { + int64_t rightTbOffset = jtCtx.blkRowSize * leftGrpRows; + + if (jtCtx.asc) { + switch (jtCtx.asofOpType) { + case OP_TYPE_GREATER_THAN: + addAsofEqInRows(leftGrpRows, 0, true); + appendAllAsofResRows(); + addAsofEqInRows(rightGrpRows, rightTbOffset, false); + break; + case OP_TYPE_GREATER_EQUAL: + addAsofEqInRows(leftGrpRows, 0, true); + addAsofEqInRows(rightGrpRows, rightTbOffset, false); + appendAllAsofResRows(); + break; + case OP_TYPE_LOWER_THAN: + case OP_TYPE_LOWER_EQUAL: + addAsofEqInRows(leftGrpRows, 0, true); + addAsofEqInRows(rightGrpRows, rightTbOffset, false); + chkAppendAsofForwardGrpResRows(false); + break; + case OP_TYPE_EQUAL: + taosArrayClear(jtCtx.leftRowsList); + taosArrayClear(jtCtx.rightRowsList); + taosArrayClear(jtCtx.rightFilterOut); + addAsofEqInRows(leftGrpRows, 0, true); + addAsofEqInRows(rightGrpRows, rightTbOffset, false); + chkAppendAsofForwardGrpResRows(true); + taosArrayClear(jtCtx.leftRowsList); + taosArrayClear(jtCtx.rightRowsList); + taosArrayClear(jtCtx.rightFilterOut); + break; + default: + return; + } + + return; + } + + switch (jtCtx.asofOpType) { + case OP_TYPE_LOWER_THAN: + addAsofEqInRows(leftGrpRows, 0, true); + appendAllAsofResRows(); + addAsofEqInRows(rightGrpRows, rightTbOffset, false); + break; + case OP_TYPE_LOWER_EQUAL: + addAsofEqInRows(leftGrpRows, 0, true); + addAsofEqInRows(rightGrpRows, rightTbOffset, false); + appendAllAsofResRows(); + break; + case OP_TYPE_GREATER_THAN: + case OP_TYPE_GREATER_EQUAL: + addAsofEqInRows(leftGrpRows, 0, true); + addAsofEqInRows(rightGrpRows, rightTbOffset, false); + chkAppendAsofForwardGrpResRows(false); + break; + case OP_TYPE_EQUAL: + taosArrayClear(jtCtx.leftRowsList); + taosArrayClear(jtCtx.rightRowsList); + taosArrayClear(jtCtx.rightFilterOut); + addAsofEqInRows(leftGrpRows, 0, true); + addAsofEqInRows(rightGrpRows, rightTbOffset, false); + chkAppendAsofForwardGrpResRows(true); + taosArrayClear(jtCtx.leftRowsList); + taosArrayClear(jtCtx.rightRowsList); + taosArrayClear(jtCtx.rightFilterOut); + break; + default: + return; + } +} + + +void addWinEqInRows(int32_t rowsNum, int64_t tbOffset, bool leftTable) { + bool filterOut = false; + void* cvalue = NULL; + int64_t cbig = 0, fbig = 0; + int32_t filterNum = leftTable ? jtCtx.leftFilterNum : jtCtx.rightFilterNum; + int32_t* filterCol = leftTable ? jtCtx.leftFilterColList : jtCtx.rightFilterColList; + SArray* rowList = leftTable ? jtCtx.leftRowsList : jtCtx.rightRowsList; + + for (int32_t l = 0; l < rowsNum; ++l) { + char* row = jtCtx.colRowDataBuf + tbOffset + jtCtx.blkRowSize * l; + + filterOut = false; + + for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { + cvalue = row + jtCtx.colRowOffset[c]; + switch (jtInputColType[c]) { + case TSDB_DATA_TYPE_TIMESTAMP: + fbig = TIMESTAMP_FILTER_VALUE; + cbig = *(int64_t*)cvalue; + break; + case TSDB_DATA_TYPE_INT: + fbig = INT_FILTER_VALUE; + cbig = *(int32_t*)cvalue; + break; + case TSDB_DATA_TYPE_BIGINT: + fbig = BIGINT_FILTER_VALUE; + cbig = *(int64_t*)cvalue; + break; + default: + break; + } + + if (filterNum && filterCol[c] && ((*(bool*)(row + c)) || cbig <= fbig)) { + filterOut = true; + break; + } + } + + if (filterOut && leftTable) { + continue; + } + + assert(NULL != taosArrayPush(rowList, row)); + if (!leftTable) { + assert(NULL != taosArrayPush(jtCtx.rightFilterOut, &filterOut)); + } + } +} + + +void winJoinAppendEqGrpRes(int32_t leftGrpRows, int32_t rightGrpRows) { + int64_t rightTbOffset = jtCtx.blkRowSize * leftGrpRows; + + addWinEqInRows(leftGrpRows, 0, true); + addWinEqInRows(rightGrpRows, rightTbOffset, false); + chkAppendWinResRows(false); +} + + + +void fullJoinAppendEqGrpRes(int32_t leftGrpRows, int32_t rightGrpRows) { + bool leftMatch = false, rightMatch = false, lfilterOut = false, rfilterOut = false; + void* lValue = NULL, *rValue = NULL, *filterValue = NULL; + int64_t lBig = 0, rBig = 0, fbig = 0; + int64_t rightTbOffset = jtCtx.blkRowSize * leftGrpRows; + + TAOS_MEMSET(jtCtx.rightFinMatch, 0, rightGrpRows * sizeof(bool)); + + for (int32_t l = 0; l < leftGrpRows; ++l) { + char* lrow = jtCtx.colRowDataBuf + jtCtx.blkRowSize * l; + + lfilterOut = false; + leftMatch = false; + + for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { + lValue = lrow + jtCtx.colRowOffset[c]; + switch (jtInputColType[c]) { + case TSDB_DATA_TYPE_TIMESTAMP: + fbig = TIMESTAMP_FILTER_VALUE; + lBig = *(int64_t*)lValue; + break; + case TSDB_DATA_TYPE_INT: + fbig = INT_FILTER_VALUE; + lBig = *(int32_t*)lValue; + break; + case TSDB_DATA_TYPE_BIGINT: + fbig = BIGINT_FILTER_VALUE; + lBig = *(int64_t*)lValue; + break; + default: + break; + } + + if (jtCtx.leftFilterNum && jtCtx.leftFilterColList[c] && ((*(bool*)(lrow + c)) || lBig <= fbig)) { + lfilterOut = true; + } + } + + for (int32_t r = 0; r < rightGrpRows; ++r) { + char* rrow = jtCtx.colRowDataBuf + rightTbOffset + jtCtx.blkRowSize * r; + rightMatch = true; + rfilterOut = false; + + for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { + lValue = lrow + jtCtx.colRowOffset[c]; + + if (!*(bool*)(rrow + c)) { + rValue = rrow + jtCtx.colRowOffset[c]; + } + + switch (jtInputColType[c]) { + case TSDB_DATA_TYPE_TIMESTAMP: + fbig = TIMESTAMP_FILTER_VALUE; + lBig = *(int64_t*)lValue; + rBig = *(int64_t*)rValue; + break; + case TSDB_DATA_TYPE_INT: + fbig = INT_FILTER_VALUE; + lBig = *(int32_t*)lValue; + rBig = *(int32_t*)rValue; + break; + case TSDB_DATA_TYPE_BIGINT: + fbig = BIGINT_FILTER_VALUE; + lBig = *(int64_t*)lValue; + rBig = *(int64_t*)rValue; + break; + default: + break; + } + + if (jtCtx.colEqNum && jtCtx.colEqList[c] && ((*(bool*)(lrow + c)) || (*(bool*)(rrow + c)) || lBig != rBig)) { + rightMatch = false; + } + + if (jtCtx.colOnNum && jtCtx.colOnList[c] && ((*(bool*)(lrow + c)) || (*(bool*)(rrow + c)) || lBig <= rBig)) { + rightMatch = false; + } + + if (jtCtx.rightFilterNum && jtCtx.rightFilterColList[c] && ((*(bool*)(rrow + c)) || rBig <= fbig)) { + rfilterOut = true; + } + } + + if (rightMatch) { + jtCtx.rightFinMatch[r] = true; + } + + if (rfilterOut) { + if (!rightMatch) { + jtCtx.rightFinMatch[r] = true; + } + continue; + } + + if (!lfilterOut && rightMatch) { + putMatchRowToRes(lrow, rrow, ALL_TABLE_COLS); + leftMatch= true; + } + } + + if (!lfilterOut && !leftMatch && 0 == jtCtx.rightFilterNum) { + putNMatchRowToRes(lrow, 0, MAX_SLOT_NUM); + } + } + + if (0 == jtCtx.leftFilterNum) { + for (int32_t r = 0; r < rightGrpRows; ++r) { + if (!jtCtx.rightFinMatch[r]) { + char* rrow = jtCtx.colRowDataBuf + rightTbOffset + jtCtx.blkRowSize * r; + putNMatchRowToRes(rrow, MAX_SLOT_NUM, 0); + } + } + } +} + + +void appendEqGrpRes(int32_t leftGrpRows, int32_t rightGrpRows) { + switch (jtCtx.joinType) { + case JOIN_TYPE_INNER: + innerJoinAppendEqGrpRes(leftGrpRows, rightGrpRows); + break; + case JOIN_TYPE_LEFT: { + switch (jtCtx.subType) { + case JOIN_STYPE_OUTER: + leftJoinAppendEqGrpRes(leftGrpRows, rightGrpRows); + break; + case JOIN_STYPE_SEMI: + semiJoinAppendEqGrpRes(leftGrpRows, rightGrpRows); + break; + case JOIN_STYPE_ANTI: + antiJoinAppendEqGrpRes(leftGrpRows, rightGrpRows); + break; + case JOIN_STYPE_ASOF: + asofJoinAppendEqGrpRes(leftGrpRows, rightGrpRows); + break; + case JOIN_STYPE_WIN: + winJoinAppendEqGrpRes(leftGrpRows, rightGrpRows); + break; + default: + break; + } + break; + } + case JOIN_TYPE_FULL: + fullJoinAppendEqGrpRes(leftGrpRows, rightGrpRows); + break; + default: + break; + } +} + +void createTsEqGrpRows(SSDataBlock** ppLeft, SSDataBlock** ppRight, int32_t leftGrpRows, int32_t rightGrpRows) { + if (leftGrpRows <= 0 && rightGrpRows <= 0) { + return; + } + + if (leftGrpRows > 0 && rightGrpRows > 0) { + jtCtx.inputStat |= (1 << 2); + } + + jtCtx.asc ? ++jtCtx.curTs : --jtCtx.curTs; + + if (NULL == *ppLeft && leftGrpRows > 0) { + *ppLeft = createDummyBlock(LEFT_BLK_ID); + assert(*ppLeft); + assert(0 == blockDataEnsureCapacity(*ppLeft, jtCtx.blkRows)); + assert(NULL != taosArrayPush(jtCtx.leftBlkList, ppLeft)); + } + + if (jtCtx.grpJoin) { + (*ppLeft)->info.id.groupId = jtCtx.inGrpId; + } + + if (NULL == *ppRight && rightGrpRows > 0) { + *ppRight = createDummyBlock(RIGHT_BLK_ID); + assert(*ppRight); + assert(0 == blockDataEnsureCapacity(*ppRight, jtCtx.blkRows)); + assert(NULL != taosArrayPush(jtCtx.rightBlkList, ppRight)); + } + + if (jtCtx.grpJoin) { + (*ppRight)->info.id.groupId = jtCtx.inGrpId; + } + + + makeAppendBlkData(ppLeft, ppRight, leftGrpRows, rightGrpRows); + + appendEqGrpRes(leftGrpRows, rightGrpRows); +} + +void forceFlushResRows() { + if (JOIN_STYPE_ASOF == jtCtx.subType && taosArrayGetSize(jtCtx.leftRowsList) > 0) { + assert((jtCtx.asc && (OP_TYPE_LOWER_EQUAL == jtCtx.asofOpType || OP_TYPE_LOWER_THAN == jtCtx.asofOpType)) + || (!jtCtx.asc && (OP_TYPE_GREATER_EQUAL == jtCtx.asofOpType || OP_TYPE_GREATER_THAN == jtCtx.asofOpType))); + chkAppendAsofForwardGrpResRows(true); + } else if (JOIN_STYPE_WIN == jtCtx.subType && taosArrayGetSize(jtCtx.leftRowsList) > 0) { + chkAppendWinResRows(true); + } + + taosArrayClear(jtCtx.rightRowsList); + taosArrayClear(jtCtx.rightFilterOut); + taosArrayClear(jtCtx.leftRowsList); + +} + +void createBothBlkRowsData(void) { + SSDataBlock* pLeft = NULL; + SSDataBlock* pRight = NULL; + + jtCtx.leftTotalRows = taosRand() % jtCtx.leftMaxRows; + jtCtx.rightTotalRows = taosRand() % jtCtx.rightMaxRows; + + int32_t minTotalRows = TMIN(jtCtx.leftTotalRows, jtCtx.rightTotalRows); + int32_t maxTotalRows = TMAX(jtCtx.leftTotalRows, jtCtx.rightTotalRows); + jtCtx.curTs = jtCtx.asc ? (TIMESTAMP_FILTER_VALUE - minTotalRows / 5) : (TIMESTAMP_FILTER_VALUE + 4 * maxTotalRows / 5); + + int32_t leftTotalRows = 0, rightTotalRows = 0; + int32_t leftGrpRows = 0, rightGrpRows = 0; + int32_t grpType = 0; + while (leftTotalRows < jtCtx.leftTotalRows || rightTotalRows < jtCtx.rightTotalRows) { + if (leftTotalRows >= jtCtx.leftTotalRows) { + grpType = 1; + } else if (rightTotalRows >= jtCtx.rightTotalRows) { + grpType = 0; + } else { + grpType = taosRand() % 10; + } + + leftGrpRows = taosRand() % jtCtx.leftMaxGrpRows; + rightGrpRows = taosRand() % jtCtx.rightMaxGrpRows; + + if ((leftTotalRows + leftGrpRows) > jtCtx.leftTotalRows) { + leftGrpRows = jtCtx.leftTotalRows - leftTotalRows; + } + + if ((rightTotalRows + rightGrpRows) > jtCtx.rightTotalRows) { + rightGrpRows = jtCtx.rightTotalRows - rightTotalRows; + } + + if (0 != grpType && 1 != grpType && (leftGrpRows <= 0 || rightGrpRows <= 0)) { + if (leftGrpRows <= 0) { + grpType = 1; + } else { + grpType = 0; + } + } + + if (jtCtx.grpJoin && (0 == taosRand() % 3)) { + forceFlushResRows(); + jtCtx.inGrpId++; + pLeft = NULL; + pRight = NULL; + } + + switch (grpType) { + case 0: + createGrpRows(&pLeft, LEFT_BLK_ID, leftGrpRows); + leftTotalRows += leftGrpRows; + break; + case 1: + createGrpRows(&pRight, RIGHT_BLK_ID, rightGrpRows); + rightTotalRows += rightGrpRows; + break; + default: + createTsEqGrpRows(&pLeft, &pRight, leftGrpRows, rightGrpRows); + leftTotalRows += leftGrpRows; + rightTotalRows += rightGrpRows; + break; + } + } + + forceFlushResRows(); +} + +void createDummyBlkList(int32_t leftMaxRows, int32_t leftMaxGrpRows, int32_t rightMaxRows, int32_t rightMaxGrpRows, int32_t blkRows) { + jtCtx.leftMaxRows = leftMaxRows; + jtCtx.leftMaxGrpRows = leftMaxGrpRows; + jtCtx.rightMaxRows = rightMaxRows; + jtCtx.rightMaxGrpRows = rightMaxGrpRows; + jtCtx.blkRows = blkRows; + + int32_t maxGrpRows = TMAX(leftMaxGrpRows, rightMaxGrpRows); + if (maxGrpRows > jtCtx.rightFinMatchNum) { + jtCtx.rightFinMatchNum = maxGrpRows; + jtCtx.rightFinMatch = (bool*)taosMemoryRealloc(jtCtx.rightFinMatch, maxGrpRows * sizeof(bool)); + assert(jtCtx.rightFinMatch); + } + + taosArrayClear(jtCtx.leftRowsList); + taosArrayClear(jtCtx.rightRowsList); + taosArrayClear(jtCtx.rightFilterOut); + + createBothBlkRowsData(); +} + +void rerunBlockedHere() { + while (jtInRerun) { + taosSsleep(1); + } +} + + +SSDataBlock* getDummyInputBlock(struct SOperatorInfo* pOperator, int32_t idx) { + switch (idx) { + case LEFT_BLK_ID: + if (jtCtx.leftBlkReadIdx >= taosArrayGetSize(jtCtx.leftBlkList)) { + return NULL; + } + return (SSDataBlock*)taosArrayGetP(jtCtx.leftBlkList, jtCtx.leftBlkReadIdx++); + break; + case RIGHT_BLK_ID: + if (jtCtx.rightBlkReadIdx >= taosArrayGetSize(jtCtx.rightBlkList)) { + return NULL; + } + return (SSDataBlock*)taosArrayGetP(jtCtx.rightBlkList, jtCtx.rightBlkReadIdx++); + break; + default: + return NULL; + } +} + + +void joinTestReplaceRetrieveFp() { + static Stub stub; + stub.set(getNextBlockFromDownstreamRemain, getDummyInputBlock); + { +#ifdef WINDOWS + AddrAny any; + std::map result; + any.get_func_addr("getNextBlockFromDownstreamRemain", result); + for (const auto &f : result) { + stub.set(f.second, getDummyInputBlock); + } +#endif +#ifdef LINUX + AddrAny any("libexecutor.so"); + std::map result; + any.get_global_func_addr_dynsym("^getNextBlockFromDownstreamRemain$", result); + for (const auto &f : result) { + stub.set(f.second, getDummyInputBlock); + } +#endif + } +} + +void printColList(char* title, bool left, int32_t* colList, bool filter, char* opStr) { + bool first = true; + + JT_PRINTF("\t %s:", title); + for (int32_t i = 0; i < MAX_SLOT_NUM; ++i) { + if (colList[i]) { + if (!first) { + JT_PRINTF(" AND "); + } + first = false; + if (filter) { + JT_PRINTF("%sc%d%s%" PRId64 , left ? "l" : "r", i, opStr, jtFilterValue[i]); + } else { + JT_PRINTF("lc%d%src%d", i, opStr, i); + } + } + } + JT_PRINTF("\n"); +} + +void printInputRowData(SSDataBlock* pBlk, int32_t* rowIdx) { + if (jtCtx.grpJoin) { + JT_PRINTF("%5" PRIu64, pBlk->info.id.groupId); + } + for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { + SColumnInfoData* pCol = (SColumnInfoData*)taosArrayGet(pBlk->pDataBlock, c); + assert(pCol); + assert(pCol->info.type == jtInputColType[c]); + if (colDataIsNull_s(pCol, *rowIdx)) { + JT_PRINTF("%18s", " NULL"); + } else { + switch (jtInputColType[c]) { + case TSDB_DATA_TYPE_TIMESTAMP: + case TSDB_DATA_TYPE_BIGINT: + JT_PRINTF("%18" PRId64, *(int64_t*)colDataGetData(pCol, *rowIdx)); + break; + case TSDB_DATA_TYPE_INT: + JT_PRINTF("%18d", *(int32_t*)colDataGetData(pCol, *rowIdx)); + break; + default: + assert(0); + } + } + } + + (*rowIdx)++; +} + +void printInputData() { + int32_t leftRowIdx = 0, rightRowIdx = 0; + + JT_PRINTF("\nInput Data:\n"); + while (jtCtx.leftBlkReadIdx < taosArrayGetSize(jtCtx.leftBlkList) || jtCtx.rightBlkReadIdx < taosArrayGetSize(jtCtx.rightBlkList)) { + if (jtCtx.leftBlkReadIdx < taosArrayGetSize(jtCtx.leftBlkList)) { + while (true) { + SSDataBlock* pBlk = (SSDataBlock*)taosArrayGetP(jtCtx.leftBlkList, jtCtx.leftBlkReadIdx); + assert(pBlk); + if (leftRowIdx < pBlk->info.rows) { + printInputRowData(pBlk, &leftRowIdx); + break; + } + + JT_PRINTF("\t%*s-------------------------blk end-------------------------------", jtCtx.grpJoin ? 6 : 0, " "); + jtCtx.leftBlkReadIdx++; + leftRowIdx = 0; + break; + } + } else { + JT_PRINTF("%*s", jtCtx.grpJoin ? 77 : 72, " "); + } + + if (jtCtx.rightBlkReadIdx < taosArrayGetSize(jtCtx.rightBlkList)) { + while (true) { + SSDataBlock* pBlk = (SSDataBlock*)taosArrayGetP(jtCtx.rightBlkList, jtCtx.rightBlkReadIdx); + assert(pBlk); + if (rightRowIdx < pBlk->info.rows) { + printInputRowData(pBlk, &rightRowIdx); + break; + } + + JT_PRINTF("\t%*s--------------------------blk end----------------------------\t", jtCtx.grpJoin ? 6 : 0, " "); + jtCtx.rightBlkReadIdx++; + rightRowIdx = 0; + break; + } + } + + JT_PRINTF("\n"); + } + + jtCtx.leftBlkReadIdx = jtCtx.rightBlkReadIdx = 0; +} + +char* getInputStatStr(char* inputStat) { + if (jtCtx.inputStat & (1 << LEFT_BLK_ID)) { + TAOS_STRCAT(inputStat, "L"); + } + if (jtCtx.inputStat & (1 << RIGHT_BLK_ID)) { + TAOS_STRCAT(inputStat, "R"); + } + if (jtCtx.inputStat & (1 << 2)) { + TAOS_STRCAT(inputStat, "E"); + } + return inputStat; +} + +char* getAsofOpStr() { + switch (jtCtx.asofOpType) { + case OP_TYPE_GREATER_THAN: + return ">"; + case OP_TYPE_GREATER_EQUAL: + return ">="; + case OP_TYPE_LOWER_THAN: + return "<"; + case OP_TYPE_LOWER_EQUAL: + return "<="; + case OP_TYPE_EQUAL: + return "="; + default: + return "UNKNOWN"; + } +} + +void printBasicInfo(char* caseName) { + if (!jtCtrl.printTestInfo) { + return; + } + + char inputStat[4] = {0}; + JT_PRINTF("\n%dth TEST [%s] START\nBasic Info:\n\t asc:%d\n\t filter:%d\n\t maxRows:left-%d right-%d\n\t " + "maxGrpRows:left-%d right-%d\n\t blkRows:%d\n\t colCond:%s\n\t joinType:%s\n\t " + "subType:%s\n\t inputStat:%s\n\t groupJoin:%s\n", jtCtx.loopIdx, caseName, jtCtx.asc, jtCtx.filter, jtCtx.leftMaxRows, jtCtx.rightMaxRows, + jtCtx.leftMaxGrpRows, jtCtx.rightMaxGrpRows, jtCtx.blkRows, jtColCondStr[jtCtx.colCond], jtJoinTypeStr[jtCtx.joinType], + jtSubTypeStr[jtCtx.subType], getInputStatStr(inputStat), jtCtx.grpJoin ? "true" : "false"); + + if (JOIN_STYPE_ASOF == jtCtx.subType) { + JT_PRINTF("\t asofOp:%s\n\t JLimit:%" PRId64 "\n", getAsofOpStr(), jtCtx.jLimit); + } else if (JOIN_STYPE_WIN == jtCtx.subType) { + JT_PRINTF("\t windowOffset:[%" PRId64 ", %" PRId64 "]\n\t JLimit:%" PRId64 "\n", jtCtx.winStartOffset, jtCtx.winEndOffset, jtCtx.jLimit); + } + + JT_PRINTF("Input Info:\n\t totalBlk:left-%d right-%d\n\t totalRows:left-%d right-%d\n\t " + "blkRowSize:%d\n\t inputCols:left-%s %s %s %s right-%s %s %s %s\n", + (int32_t)taosArrayGetSize(jtCtx.leftBlkList), (int32_t)taosArrayGetSize(jtCtx.rightBlkList), + jtCtx.leftTotalRows, jtCtx.rightTotalRows, + jtCtx.blkRowSize, tDataTypes[jtInputColType[0]].name, tDataTypes[jtInputColType[1]].name, + tDataTypes[jtInputColType[2]].name, tDataTypes[jtInputColType[3]].name, tDataTypes[jtInputColType[0]].name, + tDataTypes[jtInputColType[1]].name, tDataTypes[jtInputColType[2]].name, tDataTypes[jtInputColType[3]].name); + + if (jtCtx.colEqNum) { + JT_PRINTF("\t colEqNum:%d\n", jtCtx.colEqNum); + printColList("colEqList", false, jtCtx.colEqList, false, "="); + } + + if (jtCtx.colOnNum) { + JT_PRINTF("\t colOnNum:%d\n", jtCtx.colOnNum); + printColList("colOnList", false, jtCtx.colOnList, false, ">"); + } + + if (jtCtx.leftFilterNum) { + JT_PRINTF("\t leftFilterNum:%d\n", jtCtx.leftFilterNum); + printColList("leftFilterList", true, jtCtx.leftFilterColList, true, ">"); + } + + if (jtCtx.rightFilterNum) { + JT_PRINTF("\t rightFilterNum:%d\n", jtCtx.rightFilterNum); + printColList("rightFilterList", false, jtCtx.rightFilterColList, true, ">"); + } + + JT_PRINTF("\t resColSize:%d\n\t resColNum:%d\n\t resColList:", jtCtx.resColSize, jtCtx.resColNum); + for (int32_t i = 0; i < jtCtx.resColNum; ++i) { + int32_t s = jtCtx.resColInSlot[i]; + int32_t idx = s >= MAX_SLOT_NUM ? s - MAX_SLOT_NUM : s; + JT_PRINTF("%sc%d[%s]\t", s >= MAX_SLOT_NUM ? "r" : "l", s, tDataTypes[jtInputColType[idx]].name); + } + + if (jtCtrl.printInputRow) { + printInputData(); + } +} + +void printOutputInfo() { + if (!jtCtrl.printTestInfo) { + return; + } + + JT_PRINTF("\nOutput Info:\n\t expectedRows:%d\n\t ", jtCtx.resRows); + JT_PRINTF("Actual Result:\n"); +} + +void printActualResInfo() { + if (!jtCtrl.printTestInfo) { + return; + } + + JT_PRINTF("Actual Result Summary:\n\t blkNum:%d\n\t rowNum:%d%s\n\t leftBlkRead:%d\n\t rightBlkRead:%d\n\t +rows:%d%s\n\t " + "-rows:%d%s\n\t matchRows:%d%s\n\t executionTime:%" PRId64 "us\n", + jtRes.blkNum, jtRes.rowNum, + jtRes.rowNum == jtCtx.resRows ? "" : "*", + jtCtx.leftBlkReadIdx, jtCtx.rightBlkReadIdx, + jtRes.addRowNum, jtRes.addRowNum ? "*" : "", + jtRes.subRowNum, jtRes.subRowNum ? "*" : "", + jtRes.matchNum, jtRes.matchNum == jtCtx.resRows ? "" : "*", + taosGetTimestampUs() - jtCtx.startTsUs); +} + +void printStatInfo(char* caseName) { + JT_PRINTF("\n TEST [%s] Stat:\n\t maxResRows:%d\n\t maxResBlkRows:%d\n\t totalResRows:%" PRId64 "\n\t useMSecs:%" PRId64 "\n", + caseName, jtStat.maxResRows, jtStat.maxResBlkRows, jtStat.totalResRows, jtStat.useMSecs); + +} + +void checkJoinDone(char* caseName) { + int32_t iter = 0; + void* p = NULL; + void* key = NULL; + if (!jtCtrl.noKeepResRows) { + while (NULL != (p = tSimpleHashIterate(jtCtx.jtResRows, p, &iter))) { + key = tSimpleHashGetKey(p, NULL); + jtRes.succeed = false; + jtRes.subRowNum += *(int32_t*)p; + for (int32_t i = 0; i < *(int32_t*)p; ++i) { + printResRow((char*)key, 0); + } + } + } + + printActualResInfo(); + + JT_PRINTF("\n%dth TEST [%s] Final Result: %s\n", jtCtx.loopIdx, caseName, jtRes.succeed ? "SUCCEED" : "FAILED"); +} + +void putRowToResColBuf(SSDataBlock* pBlock, int32_t r, bool ignoreTbCols) { + for (int32_t c = 0; c < jtCtx.resColNum; ++c) { + int32_t slot = jtCtx.resColInSlot[c]; + if (ignoreTbCols && ((jtCtx.leftColOnly && slot >= MAX_SLOT_NUM) || + (jtCtx.rightColOnly && slot < MAX_SLOT_NUM))) { + continue; + } + + SColumnInfoData* pCol = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, c); + assert(pCol); + switch (jtInputColType[slot % MAX_SLOT_NUM]) { + case TSDB_DATA_TYPE_TIMESTAMP: + case TSDB_DATA_TYPE_BIGINT: + if (colDataIsNull_s(pCol, r)) { + *(bool*)(jtCtx.resColBuf + slot) = true; + } else { + *(int64_t*)(jtCtx.resColBuf + jtCtx.resColOffset[slot]) = *(int64_t*)colDataGetData(pCol, r); + } + break; + case TSDB_DATA_TYPE_INT: + if (colDataIsNull_s(pCol, r)) { + *(bool*)(jtCtx.resColBuf + slot) = true; + } else { + *(int32_t*)(jtCtx.resColBuf + jtCtx.resColOffset[slot]) = *(int32_t*)colDataGetData(pCol, r); + } + break; + default: + break; + } + } +} + +void checkJoinRes(SSDataBlock* pBlock) { + jtRes.rowNum += pBlock->info.rows; + if (jtRes.rowNum > jtStat.maxResRows) { + jtStat.maxResRows = jtRes.rowNum; + } + jtRes.blkNum++; + + if (pBlock->info.rows > jtStat.maxResBlkRows) { + jtStat.maxResBlkRows = pBlock->info.rows; + } + + jtStat.totalResRows += pBlock->info.rows; + if (jtCtrl.noKeepResRows) { + jtRes.matchNum += pBlock->info.rows; + } else { + for (int32_t r = 0; r < pBlock->info.rows; ++r) { + TAOS_MEMSET(jtCtx.resColBuf, 0, jtCtx.resColSize); + + putRowToResColBuf(pBlock, r, true); + + char* value = (char*)tSimpleHashGet(jtCtx.jtResRows, jtCtx.resColBuf, jtCtx.resColSize); + if (NULL == value) { + putRowToResColBuf(pBlock, r, false); + printResRow(jtCtx.resColBuf, 1); + jtRes.succeed = false; + jtRes.addRowNum++; + continue; + } + + rmResRow(); + + putRowToResColBuf(pBlock, r, false); + printResRow(jtCtx.resColBuf, 2); + jtRes.matchNum++; + } + } +} + +void resetForJoinRerun(int32_t dsNum, SSortMergeJoinPhysiNode* pNode, SExecTaskInfo* pTask) { + jtCtx.leftBlkReadIdx = 0; + jtCtx.rightBlkReadIdx = 0; + jtCtx.curKeyOffset = 0; + + TAOS_MEMSET(&jtRes, 0, sizeof(jtRes)); + jtRes.succeed = true; + + SOperatorInfo* pDownstreams[2]; + createDummyDownstreamOperators(2, pDownstreams); + SOperatorInfo* ppDownstreams[] = {pDownstreams[0], pDownstreams[1]}; + int32_t code = createMergeJoinOperatorInfo(ppDownstreams, 2, pNode, pTask, &jtCtx.pJoinOp); + ASSERT_TRUE(NULL != jtCtx.pJoinOp); +} + +void handleJoinDone(bool* contLoop) { + destroyOperator(jtCtx.pJoinOp); + jtCtx.pJoinOp = NULL; + + if (jtRes.succeed) { + *contLoop = false; + return; + } + + if (jtErrorRerun) { + *contLoop = false; + return; + } + + jtInRerun = true; +} + + +void jtInitLogFile() { + const char *defaultLogFileNamePrefix = "jtlog"; + const int32_t maxLogFileNum = 10; + + tsAsyncLog = 0; + qDebugFlag = 159; + TAOS_STRCPY(tsLogDir, TD_LOG_DIR_PATH); + + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) { + JT_PRINTF("failed to open log file in directory:%s\n", tsLogDir); + } +} + + + +void initJoinTest() { + jtCtx.leftBlkList = taosArrayInit(10, POINTER_BYTES); + jtCtx.rightBlkList = taosArrayInit(10, POINTER_BYTES); + assert(jtCtx.leftBlkList && jtCtx.rightBlkList); + jtCtx.jtResRows = tSimpleHashInit(10000000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY)); + assert(jtCtx.jtResRows); + + joinTestReplaceRetrieveFp(); + + if (jtCtrl.logHistory) { + jtStat.pHistory = taosArrayInit(100000, sizeof(SJoinTestHistory)); + assert(jtStat.pHistory); + } + + int32_t offset = MAX_SLOT_NUM * sizeof(bool); + for (int32_t i = 0; i < MAX_SLOT_NUM; ++i) { + jtCtx.inColOffset[i] = offset; + offset += tDataTypes[jtInputColType[i]].bytes; + } + jtCtx.inColSize = offset; + jtCtx.inColBuf = (char*)taosMemoryMalloc(jtCtx.inColSize); + assert(jtCtx.inColBuf); + + jtCtx.leftRowsList = taosArrayInit(1024, jtCtx.inColSize); + jtCtx.rightRowsList = taosArrayInit(1024, jtCtx.inColSize); + jtCtx.rightFilterOut = taosArrayInit(1024, sizeof(bool)); + assert(jtCtx.leftRowsList && jtCtx.rightRowsList && jtCtx.rightFilterOut); + + jtInitLogFile(); +} + +void handleTestDone() { + if (jtCtrl.logHistory) { + SJoinTestHistory h; + TAOS_MEMCPY(&h.ctx, &jtCtx, sizeof(h.ctx)); + TAOS_MEMCPY(&h.res, &jtRes, sizeof(h.res)); + assert(NULL != taosArrayPush(jtStat.pHistory, &h)); + } + + int32_t blkNum = taosArrayGetSize(jtCtx.leftBlkList); + for (int32_t i = 0; i < blkNum; ++i) { + SSDataBlock* pBlk = (SSDataBlock*)taosArrayGetP(jtCtx.leftBlkList, i); + assert(pBlk); + (void)blockDataDestroy(pBlk); + } + taosArrayClear(jtCtx.leftBlkList); + + blkNum = taosArrayGetSize(jtCtx.rightBlkList); + for (int32_t i = 0; i < blkNum; ++i) { + SSDataBlock* pBlk = (SSDataBlock*)taosArrayGetP(jtCtx.rightBlkList, i); + assert(pBlk); + (void)blockDataDestroy(pBlk); + } + taosArrayClear(jtCtx.rightBlkList); + + tSimpleHashClear(jtCtx.jtResRows); + jtCtx.resRows = 0; + + jtCtx.inputStat = 0; +} + +void runSingleTest(char* caseName, SJoinTestParam* param) { + bool contLoop = true; + + SSortMergeJoinPhysiNode* pNode = createDummySortMergeJoinPhysiNode(param); + assert(pNode); + createDummyBlkList(1000, 1000, 1000, 1000, 100); + + while (contLoop) { + rerunBlockedHere(); + resetForJoinRerun(2, pNode, param->pTask); + printBasicInfo(caseName); + printOutputInfo(); + + jtCtx.startTsUs = taosGetTimestampUs(); + while (true) { + SSDataBlock* pBlock = NULL; + int32_t code = jtCtx.pJoinOp->fpSet.getNextFn(jtCtx.pJoinOp, &pBlock); + if (NULL == pBlock) { + checkJoinDone(caseName); + break; + } else { + checkJoinRes(pBlock); + } + } + + handleJoinDone(&contLoop); + } + + nodesDestroyNode((SNode*)pNode); + handleTestDone(); +} + +void handleCaseEnd() { + taosMemoryFreeClear(jtCtx.rightFinMatch); + jtCtx.rightFinMatchNum = 0; +} + +} // namespace + +#if 1 +#if 1 +TEST(innerJoin, noCondTest) { + SJoinTestParam param; + char* caseName = "innerJoin:noCondTest"; + SExecTaskInfo* pTask = createDummyTaskInfo(caseName); + assert(pTask); + + param.pTask = pTask; + param.joinType = JOIN_TYPE_INNER; + param.subType = JOIN_STYPE_NONE; + param.cond = TEST_NO_COND; + param.asc = true; + param.grpJoin = false; + + for (jtCtx.loopIdx = 0; jtCtx.loopIdx < JT_MAX_LOOP; ++jtCtx.loopIdx) { + param.asc = !param.asc; + param.filter = false; + runSingleTest(caseName, ¶m); + + param.filter = true; + runSingleTest(caseName, ¶m); + } + + printStatInfo(caseName); + taosMemoryFree(pTask); +} +#endif + +#if 1 +TEST(innerJoin, eqCondTest) { + SJoinTestParam param; + char* caseName = "innerJoin:eqCondTest"; + SExecTaskInfo* pTask = createDummyTaskInfo(caseName); + assert(pTask); + + param.pTask = pTask; + param.joinType = JOIN_TYPE_INNER; + param.subType = JOIN_STYPE_NONE; + param.cond = TEST_EQ_COND; + param.asc = true; + param.grpJoin = false; + + for (jtCtx.loopIdx = 0; jtCtx.loopIdx < JT_MAX_LOOP; ++jtCtx.loopIdx) { + param.asc = !param.asc; + param.filter = false; + runSingleTest(caseName, ¶m); + + param.filter = true; + runSingleTest(caseName, ¶m); + } + + printStatInfo(caseName); + taosMemoryFree(pTask); +} +#endif + +#if 1 +TEST(innerJoin, onCondTest) { + SJoinTestParam param; + char* caseName = "innerJoin:onCondTest"; + SExecTaskInfo* pTask = createDummyTaskInfo(caseName); + assert(pTask); + + param.pTask = pTask; + param.joinType = JOIN_TYPE_INNER; + param.subType = JOIN_STYPE_NONE; + param.cond = TEST_ON_COND; + param.asc = true; + param.grpJoin = false; + + for (jtCtx.loopIdx = 0; jtCtx.loopIdx < JT_MAX_LOOP; ++jtCtx.loopIdx) { + param.asc = !param.asc; + param.filter = false; + runSingleTest(caseName, ¶m); + + param.filter = true; + runSingleTest(caseName, ¶m); + } + + printStatInfo(caseName); + taosMemoryFree(pTask); +} +#endif + +#if 1 +TEST(innerJoin, fullCondTest) { + SJoinTestParam param; + char* caseName = "innerJoin:fullCondTest"; + SExecTaskInfo* pTask = createDummyTaskInfo(caseName); + assert(pTask); + + param.pTask = pTask; + param.joinType = JOIN_TYPE_INNER; + param.subType = JOIN_STYPE_NONE; + param.cond = TEST_FULL_COND; + param.asc = true; + param.grpJoin = false; + + for (jtCtx.loopIdx = 0; jtCtx.loopIdx < JT_MAX_LOOP; ++jtCtx.loopIdx) { + param.asc = !param.asc; + param.filter = false; + runSingleTest(caseName, ¶m); + + param.filter = true; + runSingleTest(caseName, ¶m); + } + + printStatInfo(caseName); + taosMemoryFree(pTask); +} +#endif +#endif + + +#if 1 +#if 1 +TEST(leftOuterJoin, noCondTest) { + SJoinTestParam param; + char* caseName = "leftOuterJoin:noCondTest"; + SExecTaskInfo* pTask = createDummyTaskInfo(caseName); + assert(pTask); + + param.pTask = pTask; + param.joinType = JOIN_TYPE_LEFT; + param.subType = JOIN_STYPE_OUTER; + param.cond = TEST_NO_COND; + param.asc = true; + + for (jtCtx.loopIdx = 0; jtCtx.loopIdx < JT_MAX_LOOP; ++jtCtx.loopIdx) { + param.asc = !param.asc; + param.grpJoin = taosRand() % 2 ? true : false; + param.filter = false; + runSingleTest(caseName, ¶m); + + param.grpJoin = taosRand() % 2 ? true : false; + param.filter = true; + runSingleTest(caseName, ¶m); + } + + printStatInfo(caseName); + taosMemoryFree(pTask); +} +#endif + +#if 1 +TEST(leftOuterJoin, eqCondTest) { + SJoinTestParam param; + char* caseName = "leftOuterJoin:eqCondTest"; + SExecTaskInfo* pTask = createDummyTaskInfo(caseName); + assert(pTask); + + param.pTask = pTask; + param.joinType = JOIN_TYPE_LEFT; + param.subType = JOIN_STYPE_OUTER; + param.cond = TEST_EQ_COND; + param.asc = true; + param.grpJoin = false; + + for (jtCtx.loopIdx = 0; jtCtx.loopIdx < JT_MAX_LOOP; ++jtCtx.loopIdx) { + param.asc = !param.asc; + param.filter = false; + runSingleTest(caseName, ¶m); + + param.filter = true; + runSingleTest(caseName, ¶m); + } + + printStatInfo(caseName); + taosMemoryFree(pTask); +} +#endif + +#if 1 +TEST(leftOuterJoin, onCondTest) { + SJoinTestParam param; + char* caseName = "leftOuterJoin:onCondTest"; + SExecTaskInfo* pTask = createDummyTaskInfo(caseName); + assert(pTask); + + param.pTask = pTask; + param.joinType = JOIN_TYPE_LEFT; + param.subType = JOIN_STYPE_OUTER; + param.cond = TEST_ON_COND; + param.asc = true; + param.grpJoin = false; + + for (jtCtx.loopIdx = 0; jtCtx.loopIdx < JT_MAX_LOOP; ++jtCtx.loopIdx) { + param.asc = !param.asc; + param.filter = false; + runSingleTest(caseName, ¶m); + + param.filter = true; + runSingleTest(caseName, ¶m); + } + + printStatInfo(caseName); + taosMemoryFree(pTask); +} +#endif + +#if 1 +TEST(leftOuterJoin, fullCondTest) { + SJoinTestParam param; + char* caseName = "leftOuterJoin:fullCondTest"; + SExecTaskInfo* pTask = createDummyTaskInfo(caseName); + assert(pTask); + + param.pTask = pTask; + param.joinType = JOIN_TYPE_LEFT; + param.subType = JOIN_STYPE_OUTER; + param.cond = TEST_FULL_COND; + param.asc = true; + param.grpJoin = false; + + for (jtCtx.loopIdx = 0; jtCtx.loopIdx < JT_MAX_LOOP; ++jtCtx.loopIdx) { + param.asc = !param.asc; + param.filter = false; + runSingleTest(caseName, ¶m); + + param.filter = true; + runSingleTest(caseName, ¶m); + } + + printStatInfo(caseName); + taosMemoryFree(pTask); +} +#endif +#endif + +#if 1 +#if 1 +TEST(fullOuterJoin, noCondTest) { + SJoinTestParam param; + char* caseName = "fullOuterJoin:noCondTest"; + SExecTaskInfo* pTask = createDummyTaskInfo(caseName); + assert(pTask); + + param.pTask = pTask; + param.joinType = JOIN_TYPE_FULL; + param.subType = JOIN_STYPE_OUTER; + param.cond = TEST_NO_COND; + param.asc = true; + param.grpJoin = false; + + for (jtCtx.loopIdx = 0; jtCtx.loopIdx < JT_MAX_LOOP; ++jtCtx.loopIdx) { + param.asc = !param.asc; + param.filter = false; + runSingleTest(caseName, ¶m); + + param.filter = true; + runSingleTest(caseName, ¶m); + } + + printStatInfo(caseName); + taosMemoryFree(pTask); +} +#endif + +#if 1 +TEST(fullOuterJoin, eqCondTest) { + SJoinTestParam param; + char* caseName = "fullOuterJoin:eqCondTest"; + SExecTaskInfo* pTask = createDummyTaskInfo(caseName); + assert(pTask); + + param.pTask = pTask; + param.joinType = JOIN_TYPE_FULL; + param.subType = JOIN_STYPE_OUTER; + param.cond = TEST_EQ_COND; + param.asc = true; + param.grpJoin = false; + + for (jtCtx.loopIdx = 0; jtCtx.loopIdx < JT_MAX_LOOP; ++jtCtx.loopIdx) { + param.asc = !param.asc; + param.filter = false; + runSingleTest(caseName, ¶m); + + param.filter = true; + runSingleTest(caseName, ¶m); + } + + printStatInfo(caseName); + taosMemoryFree(pTask); + handleCaseEnd(); +} +#endif + +#if 1 +TEST(fullOuterJoin, onCondTest) { + SJoinTestParam param; + char* caseName = "fullOuterJoin:onCondTest"; + SExecTaskInfo* pTask = createDummyTaskInfo(caseName); + assert(pTask); + + param.pTask = pTask; + param.joinType = JOIN_TYPE_FULL; + param.subType = JOIN_STYPE_OUTER; + param.cond = TEST_ON_COND; + param.asc = true; + param.grpJoin = false; + + for (jtCtx.loopIdx = 0; jtCtx.loopIdx < JT_MAX_LOOP; ++jtCtx.loopIdx) { + param.asc = !param.asc; + param.filter = false; + runSingleTest(caseName, ¶m); + + param.filter = true; + runSingleTest(caseName, ¶m); + } + + printStatInfo(caseName); + taosMemoryFree(pTask); +} +#endif + +#if 1 +TEST(fullOuterJoin, fullCondTest) { + SJoinTestParam param; + char* caseName = "fullOuterJoin:fullCondTest"; + SExecTaskInfo* pTask = createDummyTaskInfo(caseName); + assert(pTask); + + param.pTask = pTask; + param.joinType = JOIN_TYPE_FULL; + param.subType = JOIN_STYPE_OUTER; + param.cond = TEST_FULL_COND; + param.asc = true; + param.grpJoin = false; + + for (jtCtx.loopIdx = 0; jtCtx.loopIdx < JT_MAX_LOOP; ++jtCtx.loopIdx) { + param.asc = !param.asc; + param.filter = false; + runSingleTest(caseName, ¶m); + + param.filter = true; + runSingleTest(caseName, ¶m); + } + + printStatInfo(caseName); + taosMemoryFree(pTask); +} +#endif +#endif + +#if 1 +#if 1 +TEST(leftSemiJoin, noCondTest) { + SJoinTestParam param; + char* caseName = "leftSemiJoin:noCondTest"; + SExecTaskInfo* pTask = createDummyTaskInfo(caseName); + assert(pTask); + + param.pTask = pTask; + param.joinType = JOIN_TYPE_LEFT; + param.subType = JOIN_STYPE_SEMI; + param.cond = TEST_NO_COND; + param.asc = true; + param.grpJoin = false; + + for (jtCtx.loopIdx = 0; jtCtx.loopIdx < JT_MAX_LOOP; ++jtCtx.loopIdx) { + param.asc = !param.asc; + param.filter = false; + runSingleTest(caseName, ¶m); + + param.filter = true; + runSingleTest(caseName, ¶m); + } + + printStatInfo(caseName); + taosMemoryFree(pTask); +} +#endif + +#if 1 +TEST(leftSemiJoin, eqCondTest) { + SJoinTestParam param; + char* caseName = "leftSemiJoin:eqCondTest"; + SExecTaskInfo* pTask = createDummyTaskInfo(caseName); + assert(pTask); + + param.pTask = pTask; + param.joinType = JOIN_TYPE_LEFT; + param.subType = JOIN_STYPE_SEMI; + param.cond = TEST_EQ_COND; + param.asc = true; + param.grpJoin = false; + + for (jtCtx.loopIdx = 0; jtCtx.loopIdx < JT_MAX_LOOP; ++jtCtx.loopIdx) { + param.asc = !param.asc; + param.filter = false; + runSingleTest(caseName, ¶m); + + param.filter = true; + runSingleTest(caseName, ¶m); + } + + printStatInfo(caseName); + taosMemoryFree(pTask); + handleCaseEnd(); +} +#endif + +#if 1 +TEST(leftSemiJoin, onCondTest) { + SJoinTestParam param; + char* caseName = "leftSemiJoin:onCondTest"; + SExecTaskInfo* pTask = createDummyTaskInfo(caseName); + assert(pTask); + + param.pTask = pTask; + param.joinType = JOIN_TYPE_LEFT; + param.subType = JOIN_STYPE_SEMI; + param.cond = TEST_ON_COND; + param.asc = true; + param.grpJoin = false; + + for (jtCtx.loopIdx = 0; jtCtx.loopIdx < JT_MAX_LOOP; ++jtCtx.loopIdx) { + param.asc = !param.asc; + param.filter = false; + runSingleTest(caseName, ¶m); + + param.filter = true; + runSingleTest(caseName, ¶m); + } + + printStatInfo(caseName); + taosMemoryFree(pTask); +} +#endif + +#if 1 +TEST(leftSemiJoin, fullCondTest) { + SJoinTestParam param; + char* caseName = "leftSemiJoin:fullCondTest"; + SExecTaskInfo* pTask = createDummyTaskInfo(caseName); + assert(pTask); + + param.pTask = pTask; + param.joinType = JOIN_TYPE_LEFT; + param.subType = JOIN_STYPE_SEMI; + param.cond = TEST_FULL_COND; + param.asc = true; + param.grpJoin = false; + + for (jtCtx.loopIdx = 0; jtCtx.loopIdx < JT_MAX_LOOP; ++jtCtx.loopIdx) { + param.asc = !param.asc; + param.filter = false; + runSingleTest(caseName, ¶m); + + param.filter = true; + runSingleTest(caseName, ¶m); + } + + printStatInfo(caseName); + taosMemoryFree(pTask); +} +#endif +#endif + +#if 1 +#if 1 +TEST(leftAntiJoin, noCondTest) { + SJoinTestParam param; + char* caseName = "leftAntiJoin:noCondTest"; + SExecTaskInfo* pTask = createDummyTaskInfo(caseName); + assert(pTask); + + param.pTask = pTask; + param.joinType = JOIN_TYPE_LEFT; + param.subType = JOIN_STYPE_ANTI; + param.cond = TEST_NO_COND; + param.asc = true; + param.grpJoin = false; + + for (jtCtx.loopIdx = 0; jtCtx.loopIdx < JT_MAX_LOOP; ++jtCtx.loopIdx) { + param.asc = !param.asc; + param.filter = false; + runSingleTest(caseName, ¶m); + + param.filter = true; + runSingleTest(caseName, ¶m); + } + + printStatInfo(caseName); + taosMemoryFree(pTask); +} +#endif + +#if 1 +TEST(leftAntiJoin, eqCondTest) { + SJoinTestParam param; + char* caseName = "leftAntiJoin:eqCondTest"; + SExecTaskInfo* pTask = createDummyTaskInfo(caseName); + assert(pTask); + + param.pTask = pTask; + param.joinType = JOIN_TYPE_LEFT; + param.subType = JOIN_STYPE_ANTI; + param.cond = TEST_EQ_COND; + param.asc = true; + param.grpJoin = false; + + for (jtCtx.loopIdx = 0; jtCtx.loopIdx < JT_MAX_LOOP; ++jtCtx.loopIdx) { + param.asc = !param.asc; + param.filter = false; + runSingleTest(caseName, ¶m); + + param.filter = true; + runSingleTest(caseName, ¶m); + } + + printStatInfo(caseName); + taosMemoryFree(pTask); + handleCaseEnd(); +} +#endif + +#if 1 +TEST(leftAntiJoin, onCondTest) { + SJoinTestParam param; + char* caseName = "leftAntiJoin:onCondTest"; + SExecTaskInfo* pTask = createDummyTaskInfo(caseName); + assert(pTask); + + param.pTask = pTask; + param.joinType = JOIN_TYPE_LEFT; + param.subType = JOIN_STYPE_ANTI; + param.cond = TEST_ON_COND; + param.asc = true; + param.grpJoin = false; + + for (jtCtx.loopIdx = 0; jtCtx.loopIdx < JT_MAX_LOOP; ++jtCtx.loopIdx) { + param.asc = !param.asc; + param.filter = false; + runSingleTest(caseName, ¶m); + + param.filter = true; + runSingleTest(caseName, ¶m); + } + + printStatInfo(caseName); + taosMemoryFree(pTask); +} +#endif + +#if 1 +TEST(leftAntiJoin, fullCondTest) { + SJoinTestParam param; + char* caseName = "leftAntiJoin:fullCondTest"; + SExecTaskInfo* pTask = createDummyTaskInfo(caseName); + assert(pTask); + + param.pTask = pTask; + param.joinType = JOIN_TYPE_LEFT; + param.subType = JOIN_STYPE_ANTI; + param.cond = TEST_FULL_COND; + param.asc = true; + param.grpJoin = false; + + for (jtCtx.loopIdx = 0; jtCtx.loopIdx < JT_MAX_LOOP; ++jtCtx.loopIdx) { + param.asc = !param.asc; + param.filter = false; + runSingleTest(caseName, ¶m); + + param.filter = true; + runSingleTest(caseName, ¶m); + } + + printStatInfo(caseName); + taosMemoryFree(pTask); +} +#endif +#endif + +#if 1 +#if 1 +TEST(leftAsofJoin, noCondGreaterThanTest) { + SJoinTestParam param; + char* caseName = "leftAsofJoin:noCondGreaterThanTest"; + SExecTaskInfo* pTask = createDummyTaskInfo(caseName); + assert(pTask); + + param.pTask = pTask; + param.joinType = JOIN_TYPE_LEFT; + param.subType = JOIN_STYPE_ASOF; + param.cond = TEST_NO_COND; + param.asofOp = OP_TYPE_GREATER_THAN; + param.asc = true; + + for (jtCtx.loopIdx = 0; jtCtx.loopIdx < JT_MAX_LOOP; ++jtCtx.loopIdx) { + param.asc = !param.asc; + param.jLimit = taosRand() % 2 ? (1 + (taosRand() % JT_MAX_JLIMIT)) : 1; + + param.grpJoin = taosRand() % 2 ? true : false; + param.filter = false; + runSingleTest(caseName, ¶m); + + param.grpJoin = taosRand() % 2 ? true : false; + param.filter = true; + runSingleTest(caseName, ¶m); + } + + printStatInfo(caseName); + taosMemoryFree(pTask); +} +#endif + +#if 1 +TEST(leftAsofJoin, noCondGreaterEqTest) { + SJoinTestParam param; + char* caseName = "leftAsofJoin:noCondGreaterEqTest"; + SExecTaskInfo* pTask = createDummyTaskInfo(caseName); + assert(pTask); + + param.pTask = pTask; + param.joinType = JOIN_TYPE_LEFT; + param.subType = JOIN_STYPE_ASOF; + param.cond = TEST_NO_COND; + param.asofOp = OP_TYPE_GREATER_EQUAL; + param.asc = true; + + for (jtCtx.loopIdx = 0; jtCtx.loopIdx < JT_MAX_LOOP; ++jtCtx.loopIdx) { + param.asc = !param.asc; + param.jLimit = taosRand() % 2 ? (1 + (taosRand() % JT_MAX_JLIMIT)) : 1; + + param.grpJoin = taosRand() % 2 ? true : false; + param.filter = false; + runSingleTest(caseName, ¶m); + + param.grpJoin = taosRand() % 2 ? true : false; + param.filter = true; + runSingleTest(caseName, ¶m); + } + + printStatInfo(caseName); + taosMemoryFree(pTask); +} +#endif + +#if 1 +TEST(leftAsofJoin, noCondEqTest) { + SJoinTestParam param; + char* caseName = "leftAsofJoin:noCondEqTest"; + SExecTaskInfo* pTask = createDummyTaskInfo(caseName); + assert(pTask); + + param.pTask = pTask; + param.joinType = JOIN_TYPE_LEFT; + param.subType = JOIN_STYPE_ASOF; + param.cond = TEST_NO_COND; + param.asofOp = OP_TYPE_EQUAL; + param.asc = true; + + for (jtCtx.loopIdx = 0; jtCtx.loopIdx < JT_MAX_LOOP; ++jtCtx.loopIdx) { + param.asc = !param.asc; + param.jLimit = taosRand() % 2 ? (1 + (taosRand() % JT_MAX_JLIMIT)) : 1; + + param.grpJoin = taosRand() % 2 ? true : false; + param.filter = false; + runSingleTest(caseName, ¶m); + + param.grpJoin = taosRand() % 2 ? true : false; + param.filter = true; + runSingleTest(caseName, ¶m); + } + + printStatInfo(caseName); + taosMemoryFree(pTask); +} +#endif + +#if 1 +TEST(leftAsofJoin, noCondLowerThanTest) { + SJoinTestParam param; + char* caseName = "leftAsofJoin:noCondLowerThanTest"; + SExecTaskInfo* pTask = createDummyTaskInfo(caseName); + assert(pTask); + + param.pTask = pTask; + param.joinType = JOIN_TYPE_LEFT; + param.subType = JOIN_STYPE_ASOF; + param.cond = TEST_NO_COND; + param.asofOp = OP_TYPE_LOWER_THAN; + param.asc = true; + + for (jtCtx.loopIdx = 0; jtCtx.loopIdx < JT_MAX_LOOP; ++jtCtx.loopIdx) { + param.asc = !param.asc; + param.jLimit = taosRand() % 2 ? (1 + (taosRand() % JT_MAX_JLIMIT)) : 1; + + param.grpJoin = taosRand() % 2 ? true : false; + param.filter = false; + runSingleTest(caseName, ¶m); + + param.grpJoin = taosRand() % 2 ? true : false; + param.filter = true; + runSingleTest(caseName, ¶m); + } + + printStatInfo(caseName); + taosMemoryFree(pTask); +} +#endif + + +#if 1 +TEST(leftAsofJoin, noCondLowerEqTest) { + SJoinTestParam param; + char* caseName = "leftAsofJoin:noCondLowerEqTest"; + SExecTaskInfo* pTask = createDummyTaskInfo(caseName); + assert(pTask); + + param.pTask = pTask; + param.joinType = JOIN_TYPE_LEFT; + param.subType = JOIN_STYPE_ASOF; + param.cond = TEST_NO_COND; + param.asofOp = OP_TYPE_LOWER_EQUAL; + param.asc = true; + + for (jtCtx.loopIdx = 0; jtCtx.loopIdx < JT_MAX_LOOP; ++jtCtx.loopIdx) { + param.asc = !param.asc; + param.jLimit = taosRand() % 2 ? (1 + (taosRand() % JT_MAX_JLIMIT)) : 1; + + param.grpJoin = taosRand() % 2 ? true : false; + param.filter = false; + runSingleTest(caseName, ¶m); + + param.grpJoin = taosRand() % 2 ? true : false; + param.filter = true; + runSingleTest(caseName, ¶m); + } + + printStatInfo(caseName); + taosMemoryFree(pTask); +} +#endif + +#endif + + +#if 1 +#if 1 +TEST(leftWinJoin, noCondProjectionTest) { + SJoinTestParam param; + char* caseName = "leftWinJoin:noCondProjectionTest"; + SExecTaskInfo* pTask = createDummyTaskInfo(caseName); + assert(pTask); + + param.pTask = pTask; + param.joinType = JOIN_TYPE_LEFT; + param.subType = JOIN_STYPE_WIN; + param.cond = TEST_NO_COND; + param.asc = true; + + for (jtCtx.loopIdx = 0; jtCtx.loopIdx < JT_MAX_LOOP; ++jtCtx.loopIdx) { + param.asc = !param.asc; + param.jLimit = taosRand() % 2 ? (1 + (taosRand() % JT_MAX_JLIMIT)) : 1; + + param.grpJoin = taosRand() % 2 ? true : false; + param.filter = false; + runSingleTest(caseName, ¶m); + + param.grpJoin = taosRand() % 2 ? true : false; + param.filter = true; + runSingleTest(caseName, ¶m); + } + + printStatInfo(caseName); + taosMemoryFree(pTask); +} +#endif + + +#endif + + + +int main(int argc, char** argv) { + taosSeedRand(taosGetTimestampSec()); + initJoinTest(); + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + + + +#pragma GCC diagnosti From 8be15d5b138c0b21c85fa02cd1c236f9f156fb0d Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 26 Sep 2024 16:29:57 +0800 Subject: [PATCH 02/29] enh: add plan test cases --- include/util/tdef.h | 41 + source/libs/executor/test/queryPlanTests.cpp | 4390 ++++-------------- source/libs/nodes/src/nodesUtilFuncs.c | 5 + 3 files changed, 913 insertions(+), 3523 deletions(-) diff --git a/include/util/tdef.h b/include/util/tdef.h index 46c84ab26a..07be21e044 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -188,6 +188,47 @@ typedef enum EOperatorType { OP_TYPE_ASSIGN = 200 } EOperatorType; +static const EOperatorType OPERATOR_ARRAY[] = { + OP_TYPE_ADD, + OP_TYPE_SUB, + OP_TYPE_MULTI, + OP_TYPE_DIV, + OP_TYPE_REM, + + OP_TYPE_MINUS, + + OP_TYPE_BIT_AND, + OP_TYPE_BIT_OR, + + OP_TYPE_GREATER_THAN, + OP_TYPE_GREATER_EQUAL, + OP_TYPE_LOWER_THAN, + OP_TYPE_LOWER_EQUAL, + OP_TYPE_EQUAL, + OP_TYPE_NOT_EQUAL, + OP_TYPE_IN, + OP_TYPE_NOT_IN, + OP_TYPE_LIKE, + OP_TYPE_NOT_LIKE, + OP_TYPE_MATCH, + OP_TYPE_NMATCH, + + OP_TYPE_IS_NULL, + OP_TYPE_IS_NOT_NULL, + OP_TYPE_IS_TRUE, + OP_TYPE_IS_FALSE, + OP_TYPE_IS_UNKNOWN, + OP_TYPE_IS_NOT_TRUE, + OP_TYPE_IS_NOT_FALSE, + OP_TYPE_IS_NOT_UNKNOWN, + //OP_TYPE_COMPARE_MAX_VALUE, + + OP_TYPE_JSON_GET_VALUE, + OP_TYPE_JSON_CONTAINS, + + OP_TYPE_ASSIGN +}; + #define OP_TYPE_CALC_MAX OP_TYPE_BIT_OR typedef enum ELogicConditionType { diff --git a/source/libs/executor/test/queryPlanTests.cpp b/source/libs/executor/test/queryPlanTests.cpp index 6dd42c3a93..8354620afb 100755 --- a/source/libs/executor/test/queryPlanTests.cpp +++ b/source/libs/executor/test/queryPlanTests.cpp @@ -45,63 +45,146 @@ namespace { -typedef struct { - bool succeed; - int32_t blkNum; - int32_t rowNum; - int32_t addRowNum; - int32_t subRowNum; - int32_t matchNum; -} SJoinTestResInfo; +#define QPT_MAX_LOOP 1000 +#define QPT_MAX_SUBPLAN_LEVEL 1000 +#define QPT_MAX_WHEN_THEN_NUM 10 +#define QPT_MAX_NODE_LEVEL 5 +#define QPT_MAX_STRING_LEN 1048576 +#define QPT_MAX_FUNC_PARAM 5 +#define QPT_MAX_LOGIC_PARAM 5 +#define QPT_MAX_NODE_LIST_NUM 5 -typedef struct { - int32_t maxResRows; - int32_t maxResBlkRows; - int64_t totalResRows; - int64_t useMSecs; - SArray* pHistory; -} SJoinTestStat; - - -enum { - TEST_NO_COND = 1, - TEST_EQ_COND, - TEST_ON_COND, - TEST_FULL_COND +int32_t QPT_PHYSIC_NODE_LIST[] = { + QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN, + QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, + QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN, + QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN, + QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN, + QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN, + QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN, + QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN, + QUERY_NODE_PHYSICAL_PLAN_PROJECT, + QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN, + QUERY_NODE_PHYSICAL_PLAN_HASH_AGG, + QUERY_NODE_PHYSICAL_PLAN_EXCHANGE, + QUERY_NODE_PHYSICAL_PLAN_MERGE, + QUERY_NODE_PHYSICAL_PLAN_SORT, + QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT, + QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL, + QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL, + QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL, + QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, + QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL, + QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL, + QUERY_NODE_PHYSICAL_PLAN_FILL, + QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL, + QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION, + QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION, + QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION, + QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION, + QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE, + QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE, + QUERY_NODE_PHYSICAL_PLAN_PARTITION, + QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION, + QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC, + QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC, + QUERY_NODE_PHYSICAL_PLAN_DISPATCH, + QUERY_NODE_PHYSICAL_PLAN_INSERT, + QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT, + QUERY_NODE_PHYSICAL_PLAN_DELETE, + QUERY_NODE_PHYSICAL_SUBPLAN, + QUERY_NODE_PHYSICAL_PLAN, + QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN, + QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT, + QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT, + QUERY_NODE_PHYSICAL_PLAN_HASH_JOIN, + QUERY_NODE_PHYSICAL_PLAN_GROUP_CACHE, + QUERY_NODE_PHYSICAL_PLAN_DYN_QUERY_CTRL, + QUERY_NODE_PHYSICAL_PLAN_MERGE_COUNT, + QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT, + QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL }; -#define JT_PRINTF (void)printf +#define QPT_PHYSIC_NODE_NUM() (sizeof(QPT_PHYSIC_NODE_LIST)/sizeof(QPT_PHYSIC_NODE_LIST[0])) -#define COL_DISPLAY_WIDTH 18 -#define JT_MAX_LOOP 1000 +typedef struct { + ENodeType type; + void* param; +} SQPTNodeParam; -#define LEFT_BLK_ID 0 -#define RIGHT_BLK_ID 1 -#define RES_BLK_ID 2 -#define MAX_SLOT_NUM 4 +typedef struct { + bool singlePhysiNode; + int32_t subplanMaxLevel; + int32_t subplanType[QPT_MAX_SUBPLAN_LEVEL]; + int32_t physiNodeParamNum; + SQPTNodeParam* physicNodeParam; +} SQPTPlanParam; -#define LEFT_TABLE_COLS 0x1 -#define RIGHT_TABLE_COLS 0x2 -#define ALL_TABLE_COLS (LEFT_TABLE_COLS | RIGHT_TABLE_COLS) +typedef struct { + uint8_t precision; + char dbName[TSDB_DB_NAME_LEN]; +} SQPTDbParam; -#define JT_MAX_JLIMIT 20 -#define JT_MAX_WINDOW_OFFSET 5 -#define JT_KEY_SOLT_ID (MAX_SLOT_NUM - 1) -#define JT_PRIM_TS_SLOT_ID 0 -int32_t jtInputColType[MAX_SLOT_NUM] = {TSDB_DATA_TYPE_TIMESTAMP, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_INT, TSDB_DATA_TYPE_BIGINT}; -char* jtColCondStr[] = {"", "NO COND", "EQ COND", "ON COND", "FULL COND"}; -char* jtJoinTypeStr[] = {"INNER", "LEFT", "RIGHT", "FULL"}; -char* jtSubTypeStr[] = {"", "OUTER", "SEMI", "ANTI", "ASOF", "WINDOW"}; +typedef struct { + int32_t vnodeNum; +} SQPTVnodeParam; -int64_t TIMESTAMP_FILTER_VALUE = 10000000000; -int32_t INT_FILTER_VALUE = 200000000; -int64_t BIGINT_FILTER_VALUE = 3000000000000000; +typedef struct { + char name[TSDB_COL_NAME_LEN]; + int32_t type; + int32_t len; + int8_t inUse; + bool hasIndex; + bool isPrimTs; + bool isPk; + EColumnType colType; +} SQPTCol; -int64_t jtFilterValue[] = {TIMESTAMP_FILTER_VALUE, INT_FILTER_VALUE, INT_FILTER_VALUE, BIGINT_FILTER_VALUE}; +typedef struct { + int64_t uid; + int64_t suid; + int8_t tblType; + int32_t colNum; + int32_t tagNum; + int16_t pkNum; + char tblName[TSDB_TABLE_NAME_LEN]; + char tblAlias[TSDB_TABLE_NAME_LEN]; + SQPTCol* pCol; + SQPTCol* pTag; +} SQPTTblParam; -bool jtErrorRerun = false; -bool jtInRerun = false; + +typedef struct { + bool correctExpected; + SQPTPlanParam plan; + SQPTDbParam db; + SQPTVnodeParam vnode; + SQPTTblParam tbl; +} SQPTParam; + + +typedef struct { + SPhysiNode* pCurr; + SPhysiNode* pChild; + EOrder currTsOrder; + SPhysiPlanContext* pCxt; +} SQPTBuildCtx; + +typedef struct { + int32_t nodeLevel; + bool onlyTag; + SDataBlockDescNode* pInputDataBlockDesc; +} SQPTMakeNodeCtx; + + +typedef struct { + int32_t loopIdx; + SQPTParam param; + SQPTBuildCtx buildCtx; + SQPTMakeNodeCtx makeCtx; + int64_t startTsUs; +} SQPTCtx; typedef struct { bool printTestInfo; @@ -109,752 +192,665 @@ typedef struct { bool printResRow; bool logHistory; bool noKeepResRows; -} SJoinTestCtrl; +} SQPTCtrl; -typedef struct { - bool filter; - bool asc; - bool grpJoin; - int32_t leftMaxRows; - int32_t leftMaxGrpRows; - int32_t rightMaxRows; - int32_t rightMaxGrpRows; - int32_t blkRows; - int32_t colCond; - int32_t joinType; - int32_t subType; - int32_t asofOpType; - int64_t jLimit; - int64_t winStartOffset; - int64_t winEndOffset; - int64_t inGrpId; +SQPTCtx qptCtx = {0}; +SQPTCtrl qptCtrl = {1, 0, 0, 0, 0}; +bool qptErrorRerun = false; +bool qptInRerun = false; - int32_t leftTotalRows; - int32_t rightTotalRows; - int32_t blkRowSize; - int32_t inputStat; - - int32_t colEqNum; - int32_t colEqList[MAX_SLOT_NUM]; - - int32_t colOnNum; - int32_t colOnList[MAX_SLOT_NUM]; - - int32_t leftFilterNum; - int32_t leftFilterColList[MAX_SLOT_NUM]; - - int32_t rightFilterNum; - int32_t rightFilterColList[MAX_SLOT_NUM]; - - int32_t keyInSlotIdx; - int32_t keyOutSlotIdx; - int32_t keyColOffset; - - int32_t resColNum; - int32_t resColInSlot[MAX_SLOT_NUM * 2]; - int32_t resColList[MAX_SLOT_NUM * 2]; - int32_t resColOffset[MAX_SLOT_NUM * 2]; - int32_t resColSize; - char* resColBuf; - - int32_t colRowDataBufSize; - char* colRowDataBuf; - int32_t colRowOffset[MAX_SLOT_NUM]; - - int64_t curTs; - int64_t curKeyOffset; - int32_t grpOffset[MAX_SLOT_NUM]; - - int32_t leftBlkReadIdx; - SArray* leftBlkList; - int32_t rightBlkReadIdx; - SArray* rightBlkList; - - int64_t resRows; - bool leftColOnly; - bool rightColOnly; - SSHashObj* jtResRows; - - SOperatorInfo* pJoinOp; - - int32_t loopIdx; - - int32_t rightFinMatchNum; - bool* rightFinMatch; - - int32_t inColOffset[MAX_SLOT_NUM]; - int32_t inColSize; - char* inColBuf; - SArray* leftRowsList; - SArray* rightRowsList; - SArray* rightFilterOut; - - int64_t startTsUs; -} SJoinTestCtx; - -typedef struct { - SJoinTestResInfo res; - SJoinTestCtx ctx; -} SJoinTestHistory; - -typedef struct { - EJoinType joinType; - EJoinSubType subType; - int32_t asofOp; - int64_t jLimit; - int32_t cond; - bool filter; - bool asc; - bool grpJoin; - bool timetruncate; - SExecTaskInfo* pTask; -} SJoinTestParam; - - -SJoinTestCtx jtCtx = {0}; -SJoinTestCtrl jtCtrl = {0, 0, 0, 0, 0}; -SJoinTestStat jtStat = {0}; -SJoinTestResInfo jtRes = {0}; - - - -void printResRow(char* value, int32_t type) { - if (!jtCtrl.printResRow) { +void qptPrintBeginInfo(char* caseName) { + if (!qptCtrl.printTestInfo) { return; } + +/* + char inputStat[4] = {0}; + JT_PRINTF("\n%dth TEST [%s] START\nBasic Info:\n\t asc:%d\n\t filter:%d\n\t maxRows:left-%d right-%d\n\t " + "maxGrpRows:left-%d right-%d\n\t blkRows:%d\n\t colCond:%s\n\t joinType:%s\n\t " + "subType:%s\n\t inputStat:%s\n\t groupJoin:%s\n", jtCtx.loopIdx, caseName, jtCtx.asc, jtCtx.filter, jtCtx.leftMaxRows, jtCtx.rightMaxRows, + jtCtx.leftMaxGrpRows, jtCtx.rightMaxGrpRows, jtCtx.blkRows, jtColCondStr[jtCtx.colCond], jtJoinTypeStr[jtCtx.joinType], + jtSubTypeStr[jtCtx.subType], getInputStatStr(inputStat), jtCtx.grpJoin ? "true" : "false"); + + if (JOIN_STYPE_ASOF == jtCtx.subType) { + JT_PRINTF("\t asofOp:%s\n\t JLimit:%" PRId64 "\n", getAsofOpStr(), jtCtx.jLimit); + } else if (JOIN_STYPE_WIN == jtCtx.subType) { + JT_PRINTF("\t windowOffset:[%" PRId64 ", %" PRId64 "]\n\t JLimit:%" PRId64 "\n", jtCtx.winStartOffset, jtCtx.winEndOffset, jtCtx.jLimit); + } - JT_PRINTF(" "); + JT_PRINTF("Input Info:\n\t totalBlk:left-%d right-%d\n\t totalRows:left-%d right-%d\n\t " + "blkRowSize:%d\n\t inputCols:left-%s %s %s %s right-%s %s %s %s\n", + (int32_t)taosArrayGetSize(jtCtx.leftBlkList), (int32_t)taosArrayGetSize(jtCtx.rightBlkList), + jtCtx.leftTotalRows, jtCtx.rightTotalRows, + jtCtx.blkRowSize, tDataTypes[jtInputColType[0]].name, tDataTypes[jtInputColType[1]].name, + tDataTypes[jtInputColType[2]].name, tDataTypes[jtInputColType[3]].name, tDataTypes[jtInputColType[0]].name, + tDataTypes[jtInputColType[1]].name, tDataTypes[jtInputColType[2]].name, tDataTypes[jtInputColType[3]].name); + + if (jtCtx.colEqNum) { + JT_PRINTF("\t colEqNum:%d\n", jtCtx.colEqNum); + printColList("colEqList", false, jtCtx.colEqList, false, "="); + } + + if (jtCtx.colOnNum) { + JT_PRINTF("\t colOnNum:%d\n", jtCtx.colOnNum); + printColList("colOnList", false, jtCtx.colOnList, false, ">"); + } + + if (jtCtx.leftFilterNum) { + JT_PRINTF("\t leftFilterNum:%d\n", jtCtx.leftFilterNum); + printColList("leftFilterList", true, jtCtx.leftFilterColList, true, ">"); + } + + if (jtCtx.rightFilterNum) { + JT_PRINTF("\t rightFilterNum:%d\n", jtCtx.rightFilterNum); + printColList("rightFilterList", false, jtCtx.rightFilterColList, true, ">"); + } + + JT_PRINTF("\t resColSize:%d\n\t resColNum:%d\n\t resColList:", jtCtx.resColSize, jtCtx.resColNum); for (int32_t i = 0; i < jtCtx.resColNum; ++i) { - int32_t slot = jtCtx.resColInSlot[i]; - if (0 == type && ((jtCtx.leftColOnly && slot >= MAX_SLOT_NUM) || - (jtCtx.rightColOnly && slot < MAX_SLOT_NUM))) { - ("%18s", " "); - continue; - } - - if (*(bool*)(value + slot)) { - JT_PRINTF("%18s", " NULL"); - continue; - } - - switch (jtInputColType[slot % MAX_SLOT_NUM]) { - case TSDB_DATA_TYPE_TIMESTAMP: - JT_PRINTF("%18" PRId64 , *(int64_t*)(value + jtCtx.resColOffset[slot])); - break; - case TSDB_DATA_TYPE_INT: - JT_PRINTF("%18d", *(int32_t*)(value + jtCtx.resColOffset[slot])); - break; - case TSDB_DATA_TYPE_BIGINT: - JT_PRINTF("%18" PRId64, *(int64_t*)(value + jtCtx.resColOffset[slot])); - break; - } + int32_t s = jtCtx.resColInSlot[i]; + int32_t idx = s >= MAX_SLOT_NUM ? s - MAX_SLOT_NUM : s; + JT_PRINTF("%sc%d[%s]\t", s >= MAX_SLOT_NUM ? "r" : "l", s, tDataTypes[jtInputColType[idx]].name); } - JT_PRINTF("\t %s\n", 0 == type ? "-" : (1 == type ? "+" : "")); + + if (jtCtrl.printInputRow) { + printInputData(); + } +*/ } -void pushResRow(char* buf, int32_t size) { - jtCtx.resRows++; +void qptPrintStatInfo(char* caseName) { - if (!jtCtrl.noKeepResRows) { - int32_t* rows = (int32_t*)tSimpleHashGet(jtCtx.jtResRows, buf, size); - if (rows) { - (*rows)++; - } else { - int32_t n = 1; - assert(0 == tSimpleHashPut(jtCtx.jtResRows, buf, size, &n, sizeof(n))); - } - } } -void rmResRow() { - int32_t* rows = (int32_t*)tSimpleHashGet(jtCtx.jtResRows, jtCtx.resColBuf, jtCtx.resColSize); - if (rows) { - (*rows)--; - if ((*rows) == 0) { - (void)tSimpleHashRemove(jtCtx.jtResRows, jtCtx.resColBuf, jtCtx.resColSize); + +bool qptGetDynamicOp() { + if (!qptCtx->param.correctExpected) { + return (taosRand() % 2) : true : false; + } + + if (qptCtx->buildCtx.pChild) { + return qptCtx->buildCtx.pChild->dynamicOp; + } + + return (taosRand() % 2) : true : false; +} + +EOrder qptGetInputTsOrder() { + return qptCtx->buildCtx.currTsOrder; +} + + +SNode* qptMakeLimitNode() { + SNode* pNode = NULL; + assert(0 == nodesMakeNode(QUERY_NODE_LIMIT, &pNode)); + assert(pNode); + + SLimitNode* pLimit = (SLimitNode*)pNode; + + if (!qptCtx->param->correctExpected) { + if (taosRand() % 2) { + pLimit->limit = taosRand() * ((taosRand() % 2) : 1 : -1); + } + if (taosRand() % 2) { + pLimit->offset = taosRand() * ((taosRand() % 2) : 1 : -1); } } else { + pLimit->limit = taosRand(); + if (taosRand() % 2) { + pLimit->offset = taosRand(); + } + } + + return pLimit; +} + +SNode* qptMakeColumnNodeFromTable(int32_t colIdx, EColumnType colType, SScanPhysiNode* pScanPhysiNode) { + SColumnNode* pCol = NULL; + assert(0 == nodesMakeNode(QUERY_NODE_COLUMN, (SNode**)&pCol)); + assert(pCol); + + pCol->node.resType.type = qptCtx->param.tbl.pCol[colIdx].type; + pCol->node.resType.bytes = qptCtx->param.tbl.pCol[colIdx].len; + + pCol->tableId = qptCtx->param.tbl.uid; + pCol->tableType = qptCtx->param.tbl.tblType; + pCol->colId = colIdx; + pCol->projIdx = colIdx; + pCol->colType = qptCtx->param.tbl.pCol[colIdx].colType; + pCol->hasIndex = qptCtx->param.tbl.pCol[colIdx].hasIndex; + pCol->isPrimTs = qptCtx->param.tbl.pCol[colIdx].isPrimTs; + strcpy(pCol->dbName, qptCtx->param.db.dbName); + strcpy(pCol->tableName, qptCtx->param.tbl.tblName); + strcpy(pCol->tableAlias, qptCtx->param.tbl.tblAlias); + strcpy(pCol->colName, qptCtx->param.tbl.pCol[colIdx].name); + pCol->dataBlockId = pScanPhysiNode->node.pOutputDataBlockDesc->dataBlockId; + pCol->slotId = colIdx; + pCol->numOfPKs = qptCtx->param.tbl.pkNum; + pCol->tableHasPk = qptCtx->param.tbl.pkNum > 0; + pCol->isPk = qptCtx->param.tbl.pCol[colIdx].isPk; + pCol->projRefIdx = 0; + pCol->resIdx = 0; + + return (SNode*)pCol; +} + + +void qptMakeWhenThenNode(SNode** ppNode) { + assert(0 == nodesMakeNode(QUERY_NODE_WHEN_THEN, ppNode)); + assert(*ppNode); + SWhenThenNode* pWhenThen = (SWhenThenNode*)*ppNode; + + qptMakeExprNode(&pWhenThen->pWhen); + assert(pWhenThen->pWhen); + + qptMakeExprNode(&pWhenThen->pThen); + assert(pWhenThen->pThen); +} + + +void qptMakeCaseWhenNode(SNode** ppNode) { + assert(0 == nodesMakeNode(QUERY_NODE_CASE_WHEN, ppNode)); + assert(*ppNode); + SCaseWhenNode* pCaseWhen = (SCaseWhenNode*)*ppNode; + + qptCtx->makeCtx.nodeLevel++; + + qptMakeExprNode(&pCaseWhen->pCase); + assert(pCaseWhen->pCase); + + qptMakeExprNode(&pCaseWhen->pElse); + assert(pCaseWhen->pElse); + + int32_t whenNum = taosRand() % QPT_MAX_WHEN_THEN_NUM + 1; + for (int32_t i = 0; i < whenNum; ++i) { + SNode* pNode = NULL; + qptMakeWhenThenNode(&pNode); + assert(0 == nodesListMakeStrictAppend(&pCaseWhen->pWhenThenList, pNode)); + } +} + + +void qptMakeOperatorNode(SNode** ppNode) { + EOperatorType opType; + opType = OPERATOR_ARRAY[taosRand() % (sizeof(OPERATOR_ARRAY)/sizeof(OPERATOR_ARRAY[0]))]; + assert(0 == nodesMakeNode(QUERY_NODE_OPERATOR, ppNode)); + SOperatorNode* pOp = (SOperatorNode*)*ppNode; + pOp->opType = opType; + switch (opType) { + case OP_TYPE_ADD: + case OP_TYPE_SUB: + case OP_TYPE_MULTI: + case OP_TYPE_DIV: + case OP_TYPE_REM: + case OP_TYPE_BIT_AND: + case OP_TYPE_BIT_OR: + case OP_TYPE_GREATER_THAN: + case OP_TYPE_GREATER_EQUAL: + case OP_TYPE_LOWER_THAN: + case OP_TYPE_LOWER_EQUAL: + case OP_TYPE_EQUAL: + case OP_TYPE_NOT_EQUAL: + case OP_TYPE_LIKE: + case OP_TYPE_NOT_LIKE: + case OP_TYPE_MATCH: + case OP_TYPE_NMATCH: + case OP_TYPE_IN: + case OP_TYPE_NOT_IN: + case OP_TYPE_JSON_GET_VALUE: + case OP_TYPE_JSON_CONTAINS: + case OP_TYPE_ASSIGN: + qptMakeOperatorNode(&pOp->pLeft); + qptMakeOperatorNode(&pOp->pRight); + break; + + case OP_TYPE_IS_NULL: + case OP_TYPE_IS_NOT_NULL: + case OP_TYPE_IS_TRUE: + case OP_TYPE_IS_FALSE: + case OP_TYPE_IS_UNKNOWN: + case OP_TYPE_IS_NOT_TRUE: + case OP_TYPE_IS_NOT_FALSE: + case OP_TYPE_IS_NOT_UNKNOWN: + case OP_TYPE_MINUS: + qptMakeOperatorNode(&pOp->pLeft); + break; + default: + assert(0); + break; + } +} + +void qptMakeColumnNode(SNode** ppNode) { + SNodeList* pColList = qptCtx->makeCtx.pInputDataBlockDesc->pSlots; + int32_t colIdx = taosRand() % pColList->length; + SNode* pNode = nodesListGetNode(pColList, colIdx); + assert(nodeType(pNode) == QUERY_NODE_SLOT_DESC); + SSlotDescNode* pSlot = (SSlotDescNode*)pNode; + SColumnNode* pCol = NULL; + nodesMakeNode(QUERY_NODE_COLUMN, (SNode**)&pCol); + pCol->node.resType = pSlot->dataType; + pCol->dataBlockId = qptCtx->makeCtx.pInputDataBlockDesc->dataBlockId; + pCol->slotId = pSlot->slotId; + + *ppNode = (SNode*)pCol; +} + +void qptGetRandValue(int32_t* pType, int32_t* pLen, void** ppVal) { + *pType = taosRand() % TSDB_DATA_TYPE_MAX; + switch (*pType) { + case TSDB_DATA_TYPE_NULL: + *pLen = 0; + *ppVal = NULL; + break; + case TSDB_DATA_TYPE_BOOL: + *pLen = tDataTypes[*pType].bytes; + *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); + assert(*ppVal); + *(bool*)*ppVal = (taosRand() % 2) : true : false; + break; + case TSDB_DATA_TYPE_TINYINT: + *pLen = tDataTypes[*pType].bytes; + *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); + assert(*ppVal); + *(int8_t*)*ppVal = taosRand(); + break; + case TSDB_DATA_TYPE_SMALLINT: + *pLen = tDataTypes[*pType].bytes; + *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); + assert(*ppVal); + *(int16_t*)*ppVal = taosRand(); + break; + case TSDB_DATA_TYPE_INT: + *pLen = tDataTypes[*pType].bytes; + *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); + assert(*ppVal); + *(int32_t*)*ppVal = taosRand(); + break; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: + *pLen = tDataTypes[*pType].bytes; + *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); + assert(*ppVal); + *(int64_t*)*ppVal = taosRand(); + break; + case TSDB_DATA_TYPE_FLOAT: + *pLen = tDataTypes[*pType].bytes; + *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); + assert(*ppVal); + *(float*)*ppVal = taosRand(); + break; + case TSDB_DATA_TYPE_DOUBLE: + *pLen = tDataTypes[*pType].bytes; + *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); + assert(*ppVal); + *(double*)*ppVal = taosRand(); + break; + case TSDB_DATA_TYPE_VARCHAR: + case TSDB_DATA_TYPE_GEOMETRY: + case TSDB_DATA_TYPE_JSON: + case TSDB_DATA_TYPE_VARBINARY: + case TSDB_DATA_TYPE_DECIMAL: + case TSDB_DATA_TYPE_BLOB: + case TSDB_DATA_TYPE_MEDIUMBLOB: + *pLen = taosRand() % QPT_MAX_STRING_LEN; + *ppVal = taosMemoryCalloc(1, *pLen + VARSTR_HEADER_SIZE); + assert(*ppVal); + varDataSetLen(*ppVal, *pLen); + memset((char*)*ppVal + VARSTR_HEADER_SIZE, 'A' + taosRand() % 26, *pLen); + break; + case TSDB_DATA_TYPE_NCHAR: { + *pLen = taosRand() % QPT_MAX_STRING_LEN; + char* pTmp = taosMemoryCalloc(1, *pLen + 1); + assert(pTmp); + memset(pTmp, 'A' + taosRand() % 26, *pLen); + *ppVal = taosMemoryCalloc(1, *pLen * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); + assert(*ppVal); + assert(taosMbsToUcs4(pTmp, *pLen, (TdUcs4 *)varDataVal(*ppVal), *pLen * TSDB_NCHAR_SIZE, NULL)); + *pLen *= TSDB_NCHAR_SIZE; + varDataSetLen(*ppVal, *pLen); + taosMemoryFree(pTmp); + break; + } + case TSDB_DATA_TYPE_UTINYINT: + *pLen = tDataTypes[*pType].bytes; + *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); + assert(*ppVal); + *(uint8_t*)*ppVal = taosRand(); + break; + case TSDB_DATA_TYPE_USMALLINT: + *pLen = tDataTypes[*pType].bytes; + *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); + assert(*ppVal); + *(uint16_t*)*ppVal = taosRand(); + break; + case TSDB_DATA_TYPE_UINT: + *pLen = tDataTypes[*pType].bytes; + *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); + assert(*ppVal); + *(uint32_t*)*ppVal = taosRand(); + break; + case TSDB_DATA_TYPE_UBIGINT: + *pLen = tDataTypes[*pType].bytes; + *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); + assert(*ppVal); + *(uint64_t*)*ppVal = taosRand(); + break; + default: + assert(0); + break; + } +} + +void qptFreeRandValue(int32_t* pType, void* pVal) { + switch (*pType) { + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: + case TSDB_DATA_TYPE_TIMESTAMP: + case TSDB_DATA_TYPE_UTINYINT: + case TSDB_DATA_TYPE_USMALLINT: + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_UBIGINT: + taosMemoryFree(pVal); + break; + case TSDB_DATA_TYPE_NULL: + case TSDB_DATA_TYPE_VARCHAR: + case TSDB_DATA_TYPE_GEOMETRY: + case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_JSON: + case TSDB_DATA_TYPE_VARBINARY: + case TSDB_DATA_TYPE_DECIMAL: + case TSDB_DATA_TYPE_BLOB: + case TSDB_DATA_TYPE_MEDIUMBLOB: + break; + default: + assert(0); + break; + } +} + +void qptMakeValueNode(SNode** ppNode) { + SValueNode* pVal = NULL; + nodesMakeNode(QUERY_NODE_VALUE, (SNode**)&pVal); + + int32_t valType, valBytes; + void* pValue = NULL; + qptGetRandValue(&valType, &valBytes, &pValue); + + pVal->node.resType.type = valType; + pVal->node.resType.bytes = valBytes; + nodesSetValueNodeValue(pVal, pValue); + + *ppNode = (SNode*)pVal; +} + +void qptMakeFunctionNode(SNode** ppNode) { + SFunctionNode* pFunc = NULL; + nodesMakeNode(QUERY_NODE_FUNCTION, (SNode**)&pFunc); + + int32_t funcIdx = taosRand() % funcMgtBuiltinsNum; + strcpy(pFunc->functionName, funcMgtBuiltins[funcIdx].name); + + fmGetFuncInfo(pFunc, NULL, 0); + + qptCtx->makeCtx.nodeLevel++; + int32_t paramNum = taosRand() % QPT_MAX_FUNC_PARAM + 1; + for (int32_t i = 0; i < paramNum; ++i) { + SNode* pNode = NULL; + qptMakeExprNode(&pNode); + assert(0 == nodesListMakeStrictAppend(&pFunc->pParameterList, pNode)); + } + + *ppNode = (SNode*)pFunc; +} + +void qptMakeLogicCondNode(SNode** ppNode) { + SLogicConditionNode* pLogic = NULL; + nodesMakeNode(QUERY_NODE_LOGIC_CONDITION, (SNode**)&pLogic); + + pLogic->condType = (taosRand() % 3) ? ((taosRand() % 2) ? LOGIC_COND_TYPE_AND : LOGIC_COND_TYPE_OR) : LOGIC_COND_TYPE_NOT; + + qptCtx->makeCtx.nodeLevel++; + int32_t paramNum = taosRand() % QPT_MAX_LOGIC_PARAM + 1; + for (int32_t i = 0; i < paramNum; ++i) { + SNode* pNode = NULL; + qptMakeExprNode(&pNode); + assert(0 == nodesListMakeStrictAppend(&pLogic->pParameterList, pNode)); + } + + *ppNode = (SNode*)pLogic; +} + +void qptMakeNodeListNode(SNode** ppNode) { + SNodeListNode* pList = NULL; + nodesMakeNode(QUERY_NODE_NODE_LIST, (SNode**)&pList); + + qptCtx->makeCtx.nodeLevel++; + int32_t nodeNum = taosRand() % QPT_MAX_NODE_LIST_NUM + 1; + for (int32_t i = 0; i < nodeNum; ++i) { + SNode* pNode = NULL; + qptMakeExprNode(&pNode); + assert(0 == nodesListMakeStrictAppend(&pList->pNodeList, pNode)); + } + + *ppNode = (SNode*)pList; +} + +void qptMakeTempTableNode(SNode** ppNode) { + STempTableNode* pTemp = NULL; + assert(0 == nodesMakeNode(QUERY_NODE_TEMP_TABLE, (SNode**)&pTemp)); + + *ppNode = (SNode*)pTemp; +} + +void qptMakeJoinTableNode(SNode** ppNode) { + SJoinTableNode* pJoin = NULL; + assert(0 == nodesMakeNode(QUERY_NODE_JOIN_TABLE, (SNode**)&pJoin)); + + *ppNode = (SNode*)pJoin; +} + + +void qptMakeTableNode(SNode** ppNode) { + if (taosRand() % 2) { + qptMakeTempTableNode(ppNode); + } else { + qptMakeJoinTableNode(ppNode); + } +} + + +void qptMakeExprNode(SNode** ppNode) { + int32_t nodeTypeMaxValue = 9; + if (qptCtx->makeCtx.nodeLevel >= QPT_MAX_NODE_LEVEL) { + nodeTypeMaxValue = 2; + } + + switch (taosRand() % 10) { + case 0: + qptMakeColumnNode(ppNode); + break; + case 1: + qptMakeValueNode(ppNode); + break; + case 2: + qptMakeFunctionNode(ppNode); + break; + case 3: + qptMakeLogicCondNode(ppNode); + break; + case 4: + qptMakeNodeListNode(ppNode); + break; + case 5: + qptMakeOperatorNode(ppNode); + break; + case 6: + qptMakeTableNode(ppNode); + break; + case 7: + qptMakeCaseWhenNode(ppNode); + break; + case 8: + qptMakeWhenThenNode(ppNode); + break; + default: + assert(0); + break; + } +} + +void qptResetMakeNodeCtx(SDataBlockDescNode* pInput, bool onlyTag) { + SQPTMakeNodeCtx* pCtx = &qptCtx->makeCtx; + + pCtx->nodeLevel = 1; + pCtx->onlyTag = onlyTag; + pCtx->pInputDataBlockDesc = pInput; +} + +SNode* qptMakeConditionNode(bool onlyTag) { + SNode* pNode = NULL; + qptResetMakeNodeCtx(qptCtx->buildCtx.pCurr->pOutputDataBlockDesc, onlyTag); + qptMakeExprNode(&pNode); + return pNode; +} + + +SPhysiNode* qptCreatePhysiNode(int32_t nodeType) { + SPhysiNode* pPhysiNode = NULL; + int32_t code = nodesMakeNode(nodeType, (SNode**)&pPhysiNode); + if (NULL == pPhysiNode) { assert(0); } + + qptCtx.buildCtx.pCurr = pPhysiNode; + + pPhysiNode->pLimit = qptMakeLimitNode(); + pPhysiNode->pSlimit = qptMakeLimitNode(); + pPhysiNode->dynamicOp = qptGetDynamicOp(); + pPhysiNode->inputTsOrder = qptGetInputTsOrder(); + + assert(0 == createDataBlockDesc(qptCtx->buildCtx.pCxt, NULL, &pPhysiNode->pOutputDataBlockDesc)); + pPhysiNode->pOutputDataBlockDesc->precision = qptCtx->param.db.precision; + + return pPhysiNode; } -static int32_t jtMergeEqCond(SNode** ppDst, SNode** ppSrc) { - if (NULL == *ppSrc) { - return TSDB_CODE_SUCCESS; - } - if (NULL == *ppDst) { - *ppDst = *ppSrc; - *ppSrc = NULL; - return TSDB_CODE_SUCCESS; - } - if (QUERY_NODE_LOGIC_CONDITION == nodeType(*ppSrc)) { - TSWAP(*ppDst, *ppSrc); - } - if (QUERY_NODE_LOGIC_CONDITION == nodeType(*ppDst)) { - SLogicConditionNode* pLogic = (SLogicConditionNode*)*ppDst; - if (QUERY_NODE_LOGIC_CONDITION == nodeType(*ppSrc)) { - assert(0 == nodesListStrictAppendList(pLogic->pParameterList, ((SLogicConditionNode*)(*ppSrc))->pParameterList)); - ((SLogicConditionNode*)(*ppSrc))->pParameterList = NULL; - } else { - assert(0 == nodesListStrictAppend(pLogic->pParameterList, *ppSrc)); - *ppSrc = NULL; - } - nodesDestroyNode(*ppSrc); - *ppSrc = NULL; - return TSDB_CODE_SUCCESS; - } +void qptPostCreatePhysiNode(SPhysiNode* pPhysiNode) { + pPhysiNode->outputTsOrder = qptGetInputTsOrder(); - SLogicConditionNode* pLogicCond = NULL; - int32_t code = nodesMakeNode(QUERY_NODE_LOGIC_CONDITION, (SNode**)&pLogicCond); - if (NULL == pLogicCond) { - return code; + if (taosRand() % 2) { + pPhysiNode->pConditions = qptMakeConditionNode(false); } - pLogicCond->node.resType.type = TSDB_DATA_TYPE_BOOL; - pLogicCond->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; - pLogicCond->condType = LOGIC_COND_TYPE_AND; - pLogicCond->pParameterList = NULL; - code = nodesMakeList(&pLogicCond->pParameterList); - assert(0 == nodesListStrictAppend(pLogicCond->pParameterList, *ppSrc)); - assert(0 == nodesListStrictAppend(pLogicCond->pParameterList, *ppDst)); - - *ppDst = (SNode*)pLogicCond; - *ppSrc = NULL; - - return TSDB_CODE_SUCCESS; + } - -void createDummyDownstreamOperators(int32_t num, SOperatorInfo** ppRes) { - for (int32_t i = 0; i < num; ++i) { - SOperatorInfo* p = (SOperatorInfo*)taosMemoryCalloc(1, sizeof(SOperatorInfo)); - assert(NULL != p); - p->resultDataBlockId = i; - ppRes[i] = p; +void qptMarkTableInUseCols(int32_t colNum, int32_t totalColNum, SQPTCol* pCol) { + if (colNum == totalColNum) { + for (int32_t i = 0; i < colNum; ++i) { + pCol[i].inUse = 1; + } + return; } -} - -void createTargetSlotList(SSortMergeJoinPhysiNode* p) { - jtCtx.resColNum = 0; - TAOS_MEMSET(jtCtx.resColList, 0, sizeof(jtCtx.resColList)); - jtCtx.resColSize = MAX_SLOT_NUM * 2 * sizeof(bool); - jtCtx.keyInSlotIdx = -1; - - for (int32_t i = 0; i < MAX_SLOT_NUM; ++i) { - if (jtCtx.colOnList[i] || jtCtx.colEqList[i] || jtCtx.leftFilterColList[i]) { - jtCtx.resColList[i] = 1; - } - } - - for (int32_t i = 0; i < MAX_SLOT_NUM; ++i) { - if (jtCtx.colOnList[i] || jtCtx.colEqList[i] || jtCtx.rightFilterColList[i]) { - jtCtx.resColList[MAX_SLOT_NUM + i] = 1; - } - } - - for (int32_t i = 0; i < MAX_SLOT_NUM; ++i) { - if (0 == jtCtx.resColList[i]) { - jtCtx.resColList[i]= taosRand() % 2; - } - - if ((jtCtx.joinType == JOIN_TYPE_LEFT || jtCtx.joinType == JOIN_TYPE_FULL) && (i == JT_KEY_SOLT_ID)) { - jtCtx.resColList[i] = 1; - } - - if (jtCtx.resColList[i] && i == JT_KEY_SOLT_ID && (jtCtx.joinType == JOIN_TYPE_LEFT || jtCtx.joinType == JOIN_TYPE_FULL)) { - jtCtx.keyInSlotIdx = JT_KEY_SOLT_ID; - } - } - - if (jtCtx.keyInSlotIdx < 0 || ((jtCtx.joinType == JOIN_TYPE_RIGHT || jtCtx.joinType == JOIN_TYPE_FULL))) { - jtCtx.resColList[MAX_SLOT_NUM + JT_KEY_SOLT_ID]= 1; - jtCtx.keyInSlotIdx = JT_KEY_SOLT_ID + MAX_SLOT_NUM; - } - - for (int32_t i = 0; i < MAX_SLOT_NUM; ++i) { - if (0 == jtCtx.resColList[MAX_SLOT_NUM + i]) { - jtCtx.resColList[MAX_SLOT_NUM + i]= taosRand() % 2; - } - } - - int32_t idx = 0; - int32_t dstIdx = 0; - int32_t dstOffset = jtCtx.resColSize; - - for (int32_t i = 0; i < MAX_SLOT_NUM; ++i) { - if (jtCtx.resColList[i]) { - jtCtx.resColOffset[i] = dstOffset; - jtCtx.resColInSlot[dstIdx] = i; - if (jtCtx.keyInSlotIdx == i) { - jtCtx.keyColOffset = dstOffset; - } - - STargetNode* pTarget = NULL; - int32_t code = nodesMakeNode(QUERY_NODE_TARGET, (SNode**)&pTarget); - SColumnNode* pCol = NULL; - code = nodesMakeNode(QUERY_NODE_COLUMN, (SNode**)&pCol); - assert(NULL != pTarget && NULL != pCol); - pCol->dataBlockId = LEFT_BLK_ID; - pCol->slotId = i; - pTarget->dataBlockId = RES_BLK_ID; - pTarget->slotId = dstIdx++; - pTarget->pExpr = (SNode*)pCol; - dstOffset += tDataTypes[jtInputColType[i]].bytes; - jtCtx.resColSize += tDataTypes[jtInputColType[i]].bytes; - - assert(0 == nodesListMakeStrictAppend(&p->pTargets, (SNode*)pTarget)); - - jtCtx.resColNum++; - } - } - - for (int32_t i = 0; i < MAX_SLOT_NUM; ++i) { - if (jtCtx.resColList[MAX_SLOT_NUM + i]) { - jtCtx.resColOffset[MAX_SLOT_NUM + i] = dstOffset; - jtCtx.resColInSlot[dstIdx] = i + MAX_SLOT_NUM; - if (jtCtx.keyInSlotIdx == (i + MAX_SLOT_NUM)) { - jtCtx.keyColOffset = dstOffset; - } - - STargetNode* pTarget = NULL; - int32_t code = nodesMakeNode(QUERY_NODE_TARGET, (SNode**)&pTarget); - SColumnNode* pCol = NULL; - code = nodesMakeNode(QUERY_NODE_COLUMN, (SNode**)&pCol); - assert(NULL != pTarget && NULL != pCol); - pCol->dataBlockId = RIGHT_BLK_ID; - pCol->slotId = i; - pTarget->dataBlockId = RES_BLK_ID; - pTarget->slotId = dstIdx++; - pTarget->pExpr = (SNode*)pCol; - dstOffset += tDataTypes[jtInputColType[i]].bytes; - jtCtx.resColSize += tDataTypes[jtInputColType[i]].bytes; - - assert(0 == nodesListMakeStrictAppend(&p->pTargets, (SNode*)pTarget)); - jtCtx.resColNum++; - } - } - - jtCtx.resColBuf = (char*)taosMemoryRealloc(jtCtx.resColBuf, jtCtx.resColSize); - assert(NULL != jtCtx.resColBuf); -} - -void createColEqCondStart(SSortMergeJoinPhysiNode* p) { - jtCtx.colEqNum = 0; + + int32_t colInUse = 0; do { - jtCtx.colEqNum = taosRand() % MAX_SLOT_NUM; // except TIMESTAMP - } while (0 == jtCtx.colEqNum); - - int32_t idx = 0; - TAOS_MEMSET(jtCtx.colEqList, 0, sizeof(jtCtx.colEqList)); - for (int32_t i = 0; i < jtCtx.colEqNum; ) { - idx = taosRand() % MAX_SLOT_NUM; - if (jtCtx.colEqList[idx]) { + int32_t colIdx = taosRand() % totalColNum; + if (pCol[colIdx].inUse) { continue; } - if (TSDB_DATA_TYPE_TIMESTAMP == jtInputColType[idx]) { - continue; - } - jtCtx.colEqList[idx] = 1; - ++i; - } - for (int32_t i = 0; i < MAX_SLOT_NUM; ++i) { - if (jtCtx.colEqList[i]) { - SColumnNode* pCol1 = NULL; - int32_t code = nodesMakeNode(QUERY_NODE_COLUMN, (SNode**)&pCol1); - assert(pCol1); - pCol1->dataBlockId = LEFT_BLK_ID; - pCol1->slotId = i; - pCol1->node.resType.type = jtInputColType[i]; - pCol1->node.resType.bytes = tDataTypes[jtInputColType[i]].bytes; - - assert(0 == nodesListMakeStrictAppend(&p->pEqLeft, (SNode*)pCol1)); - - SColumnNode* pCol2 = NULL; - code = nodesMakeNode(QUERY_NODE_COLUMN, (SNode**)&pCol2); - pCol2->dataBlockId = RIGHT_BLK_ID; - pCol2->slotId = i; - pCol2->node.resType.type = jtInputColType[i]; - pCol2->node.resType.bytes = tDataTypes[jtInputColType[i]].bytes; - - assert(0 == nodesListMakeStrictAppend(&p->pEqRight, (SNode*)pCol2)); - } - } + pCol[colIdx].inUse = 1; + colInUse++; + } while (colInUse < colNum); } -void createColOnCondStart(SSortMergeJoinPhysiNode* p) { - jtCtx.colOnNum = 0; - do { - jtCtx.colOnNum = taosRand() % (MAX_SLOT_NUM + 1); - } while (0 == jtCtx.colOnNum || (jtCtx.colOnNum + jtCtx.colEqNum) > MAX_SLOT_NUM); - - int32_t idx = 0; - TAOS_MEMSET(jtCtx.colOnList, 0, sizeof(jtCtx.colOnList)); - for (int32_t i = 0; i < jtCtx.colOnNum; ) { - idx = taosRand() % MAX_SLOT_NUM; - if (jtCtx.colOnList[idx] || jtCtx.colEqList[idx]) { - continue; - } - jtCtx.colOnList[idx] = 1; - ++i; - } -} - -int32_t getDstSlotId(int32_t srcIdx) { - for (int32_t i = 0; i < jtCtx.resColNum; ++i) { - if (jtCtx.resColInSlot[i] == srcIdx) { - return i; - } - } - - return -1; -} - - -void createColEqCondEnd(SSortMergeJoinPhysiNode* p) { - if (jtCtx.colEqNum <= 0) { - return; - } - - SLogicConditionNode* pLogic = NULL; - if (jtCtx.colEqNum > 1) { - int32_t code = nodesMakeNode(QUERY_NODE_LOGIC_CONDITION, (SNode**)&pLogic); - assert(pLogic); - pLogic->node.resType.type = TSDB_DATA_TYPE_BOOL; - pLogic->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; - pLogic->condType = LOGIC_COND_TYPE_AND; - } +void qptCreateTableScanColsImpl( SScanPhysiNode* pScanPhysiNode, SNodeList** ppCols, int32_t totalColNum, SQPTCol* pCol) { + int32_t colNum = qptCtx->pCfg->correctExpected ? (taosRand() % totalColNum + 1) : (taosRand()); + int32_t colAdded = 0; - for (int32_t i = 0; i < MAX_SLOT_NUM; ++i) { - if (jtCtx.colEqList[i]) { - SColumnNode* pCol1 = NULL; - int32_t code = nodesMakeNode(QUERY_NODE_COLUMN, (SNode**)&pCol1); - assert(pCol1); - pCol1->dataBlockId = RES_BLK_ID; - pCol1->slotId = getDstSlotId(i); - pCol1->node.resType.type = jtInputColType[i]; - pCol1->node.resType.bytes = tDataTypes[jtInputColType[i]].bytes; - - SColumnNode* pCol2 = NULL; - code = nodesMakeNode(QUERY_NODE_COLUMN, (SNode**)&pCol2); - assert(pCol2); - pCol2->dataBlockId = RES_BLK_ID; - pCol2->slotId = getDstSlotId(MAX_SLOT_NUM + i); - pCol2->node.resType.type = jtInputColType[i]; - pCol2->node.resType.bytes = tDataTypes[jtInputColType[i]].bytes; - - SOperatorNode* pOp = NULL; - code = nodesMakeNode(QUERY_NODE_OPERATOR, (SNode**)&pOp); - assert(pOp); - pOp->opType = OP_TYPE_EQUAL; - pOp->node.resType.type = TSDB_DATA_TYPE_BOOL; - pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; - pOp->pLeft = (SNode*)pCol1; - pOp->pRight = (SNode*)pCol2; - - if (jtCtx.colEqNum > 1) { - assert(0 == nodesListMakeStrictAppend(&pLogic->pParameterList, (SNode*)pOp)); - } else { - p->pFullOnCond = (SNode*)pOp; - break; + if (qptCtx->pCfg->correctExpected) { + qptMarkTableInUseCols(colNum, totalColNum, pCol); + for (int32_t i = 0; i < totalColNum && colAdded < colNum; ++i) { + if (0 == pCol[i].inUse) { + continue; } - } - } - - if (jtCtx.colEqNum > 1) { - p->pFullOnCond = (SNode*)pLogic; - } -} - -void createColOnCondEnd(SSortMergeJoinPhysiNode* p) { - if (jtCtx.colOnNum <= 0) { - return; - } - - SLogicConditionNode* pLogic = NULL; - if (jtCtx.colOnNum > 1) { - int32_t code = nodesMakeNode(QUERY_NODE_LOGIC_CONDITION, (SNode**)&pLogic); - assert(pLogic); - pLogic->node.resType.type = TSDB_DATA_TYPE_BOOL; - pLogic->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; - pLogic->condType = LOGIC_COND_TYPE_AND; - } - - for (int32_t i = 0; i < MAX_SLOT_NUM; ++i) { - if (jtCtx.colOnList[i]) { - SColumnNode* pCol1 = NULL; - int32_t code = nodesMakeNode(QUERY_NODE_COLUMN, (SNode**)&pCol1); - assert(pCol1); - pCol1->dataBlockId = RES_BLK_ID; - pCol1->slotId = getDstSlotId(i); - pCol1->node.resType.type = jtInputColType[i]; - pCol1->node.resType.bytes = tDataTypes[jtInputColType[i]].bytes; - SColumnNode* pCol2 = NULL; - code = nodesMakeNode(QUERY_NODE_COLUMN, (SNode**)&pCol2); - assert(pCol2); - pCol2->dataBlockId = RES_BLK_ID; - pCol2->slotId = getDstSlotId(MAX_SLOT_NUM + i); - pCol2->node.resType.type = jtInputColType[i]; - pCol2->node.resType.bytes = tDataTypes[jtInputColType[i]].bytes; - - SOperatorNode* pOp = NULL; - code = nodesMakeNode(QUERY_NODE_OPERATOR, (SNode**)&pOp); - assert(pOp); - pOp->opType = OP_TYPE_GREATER_THAN; - pOp->node.resType.type = TSDB_DATA_TYPE_BOOL; - pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; - pOp->pLeft = (SNode*)pCol1; - pOp->pRight = (SNode*)pCol2; - - if (jtCtx.colOnNum > 1) { - assert(0 == nodesListMakeStrictAppend(&pLogic->pParameterList, (SNode*)pOp)); - } else { - p->pColOnCond = (SNode*)pOp; - break; - } + assert(0 == nodesListMakeStrictAppend(ppCols, qptMakeColumnNodeFromTable(i, pCol[i].colType, pScanPhysiNode))); } - } - if (jtCtx.colOnNum > 1) { - p->pColOnCond = (SNode*)pLogic; - } - - SNode* pTmp = NULL; - int32_t code = nodesCloneNode(p->pColOnCond, &pTmp); - assert(pTmp); - assert(0 == jtMergeEqCond(&p->pFullOnCond, &pTmp)); -} - - - -void createColCond(SSortMergeJoinPhysiNode* p, int32_t cond) { - jtCtx.colCond = cond; - switch (cond) { - case TEST_NO_COND: - jtCtx.colEqNum = 0; - jtCtx.colOnNum = 0; - TAOS_MEMSET(jtCtx.colEqList, 0, sizeof(jtCtx.colEqList)); - TAOS_MEMSET(jtCtx.colOnList, 0, sizeof(jtCtx.colOnList)); - break; - case TEST_EQ_COND: - createColEqCondStart(p); - jtCtx.colOnNum = 0; - TAOS_MEMSET(jtCtx.colOnList, 0, sizeof(jtCtx.colOnList)); - break; - case TEST_ON_COND: - createColOnCondStart(p); - jtCtx.colEqNum = 0; - TAOS_MEMSET(jtCtx.colEqList, 0, sizeof(jtCtx.colEqList)); - break; - case TEST_FULL_COND: - createColEqCondStart(p); - createColOnCondStart(p); - break; - default: - break; - } -} - -void* getFilterValue(int32_t type) { - switch (type) { - case TSDB_DATA_TYPE_TIMESTAMP: - return &TIMESTAMP_FILTER_VALUE; - case TSDB_DATA_TYPE_INT: - return &INT_FILTER_VALUE; - case TSDB_DATA_TYPE_BIGINT: - return &BIGINT_FILTER_VALUE; - default: - return NULL; - } -} - -void createFilterStart(SSortMergeJoinPhysiNode* p, bool filter) { - jtCtx.filter = filter; - if (!filter) { - jtCtx.leftFilterNum = 0; - jtCtx.rightFilterNum = 0; - TAOS_MEMSET(jtCtx.leftFilterColList, 0, sizeof(jtCtx.leftFilterColList)); - TAOS_MEMSET(jtCtx.rightFilterColList, 0, sizeof(jtCtx.rightFilterColList)); - return; - } - - if ((JOIN_STYPE_SEMI == jtCtx.subType || JOIN_STYPE_ANTI == jtCtx.subType) && JOIN_TYPE_LEFT == jtCtx.joinType) { - jtCtx.rightFilterNum = 0; - jtCtx.leftFilterNum = taosRand() % (MAX_SLOT_NUM + 1); - if (0 == jtCtx.leftFilterNum) { - do { - jtCtx.leftFilterNum = taosRand() % (MAX_SLOT_NUM + 1); - } while (0 == jtCtx.leftFilterNum); - } - } else if ((JOIN_STYPE_SEMI == jtCtx.subType || JOIN_STYPE_ANTI == jtCtx.subType) && JOIN_TYPE_RIGHT == jtCtx.joinType) { - jtCtx.leftFilterNum = 0; - jtCtx.rightFilterNum = taosRand() % (MAX_SLOT_NUM + 1); - if (0 == jtCtx.rightFilterNum) { - do { - jtCtx.rightFilterNum = taosRand() % (MAX_SLOT_NUM + 1); - } while (0 == jtCtx.rightFilterNum); - } - } else { - jtCtx.leftFilterNum = taosRand() % (MAX_SLOT_NUM + 1); - if (0 == jtCtx.leftFilterNum) { - do { - jtCtx.rightFilterNum = taosRand() % (MAX_SLOT_NUM + 1); - } while (0 == jtCtx.rightFilterNum); - } else { - jtCtx.rightFilterNum = taosRand() % (MAX_SLOT_NUM + 1); - } - } - - int32_t idx = 0; - TAOS_MEMSET(jtCtx.leftFilterColList, 0, sizeof(jtCtx.leftFilterColList)); - TAOS_MEMSET(jtCtx.rightFilterColList, 0, sizeof(jtCtx.rightFilterColList)); - for (int32_t i = 0; i < jtCtx.leftFilterNum; ) { - idx = taosRand() % MAX_SLOT_NUM; - if (jtCtx.leftFilterColList[idx]) { - continue; - } - jtCtx.leftFilterColList[idx] = 1; - ++i; - } - - for (int32_t i = 0; i < jtCtx.rightFilterNum; ) { - idx = taosRand() % MAX_SLOT_NUM; - if (jtCtx.rightFilterColList[idx]) { - continue; - } - jtCtx.rightFilterColList[idx] = 1; - ++i; - } -} - -void createFilterEnd(SSortMergeJoinPhysiNode* p, bool filter) { - if (!filter || (jtCtx.leftFilterNum <= 0 && jtCtx.rightFilterNum <= 0)) { return; } - SLogicConditionNode* pLogic = NULL; - if ((jtCtx.leftFilterNum + jtCtx.rightFilterNum) > 1) { - int32_t code = nodesMakeNode(QUERY_NODE_LOGIC_CONDITION, (SNode**)&pLogic); - assert(pLogic); - pLogic->node.resType.type = TSDB_DATA_TYPE_BOOL; - pLogic->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; - pLogic->condType = LOGIC_COND_TYPE_AND; + for (int32_t i = 0; i < colNum; ++i) { + int32_t colIdx = taosRand(); + colIdx = (colIdx >= totalColNum) ? -1 : colIdx; + + assert(0 == nodesListMakeStrictAppend(ppCols, qptMakeColumnNodeFromTable(colIdx, pCol[i].colType, pScanPhysiNode))); + } +} + + +void qptCreateTableScanCols( SScanPhysiNode* pScanPhysiNode) { + qptCreateTableScanColsImpl(pScanPhysiNode, &pScanPhysiNode->pScanCols, qptCtx->param.tbl.colNum, qptCtx.param->tbl.pCol); +} + +void qptCreateTableScanPseudoCols( SScanPhysiNode* pScanPhysiNode) { + qptCreateTableScanColsImpl(pScanPhysiNode, &pScanPhysiNode->pScanPseudoCols, qptCtx->param.tbl.tagNum, qptCtx->param.tbl.pTag); +} + + +void qptCreateScanPhysiNodeImpl( SScanPhysiNode* pScanPhysiNode) { + qptCreateTableScanCols(pScanPhysiNode); + + assert(0 == addDataBlockSlots(qptCtx->buildCtx.pCxt, pScanPhysiNode->pScanCols, pScanPhysiNode->node.pOutputDataBlockDesc)); + + if (taosRand() % 2) { + qptCreateTableScanPseudoCols(pScanPhysiNode); } - for (int32_t i = 0; i < MAX_SLOT_NUM; ++i) { - if (jtCtx.leftFilterColList[i]) { - SColumnNode* pCol = NULL; - int32_t code = nodesMakeNode(QUERY_NODE_COLUMN,(SNode**)&pCol); - assert(pCol); - pCol->dataBlockId = RES_BLK_ID; - pCol->slotId = getDstSlotId(i); - pCol->node.resType.type = jtInputColType[i]; - pCol->node.resType.bytes = tDataTypes[jtInputColType[i]].bytes; - (void)sprintf(pCol->colName, "l%d", i); + assert(0 == addDataBlockSlots(qptCtx->buildCtx.pCxt, pScanPhysiNode->pScanPseudoCols, pScanPhysiNode->node.pOutputDataBlockDesc)); + + pScanPhysiNode->uid = qptCtx->param.tbl.uid; + pScanPhysiNode->suid = qptCtx->param.tbl.suid; + pScanPhysiNode->tableType = qptCtx->param.tbl.tblType; + pScanPhysiNode->groupOrderScan = (taosRand() % 2) : true : false; - SValueNode* pVal = NULL; - code = nodesMakeNode(QUERY_NODE_VALUE, (SNode**)&pVal); - assert(pVal); - pVal->node.resType.type = jtInputColType[i]; - pVal->node.resType.bytes = tDataTypes[jtInputColType[i]].bytes; - assert(0 == nodesSetValueNodeValue(pVal, getFilterValue(jtInputColType[i]))); - - SOperatorNode* pOp = NULL; - code = nodesMakeNode(QUERY_NODE_OPERATOR, (SNode**)&pOp); - assert(pOp); - pOp->opType = OP_TYPE_GREATER_THAN; - pOp->node.resType.type = TSDB_DATA_TYPE_BOOL; - pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; - pOp->pLeft = (SNode*)pCol; - pOp->pRight = (SNode*)pVal; - - if ((jtCtx.leftFilterNum + jtCtx.rightFilterNum) > 1) { - assert(0 == nodesListMakeStrictAppend(&pLogic->pParameterList, (SNode*)pOp)); - } else { - p->node.pConditions = (SNode*)pOp; - break; - } - } - } - - for (int32_t i = 0; i < MAX_SLOT_NUM; ++i) { - if (jtCtx.rightFilterColList[i]) { - SColumnNode* pCol = NULL; - int32_t code = nodesMakeNode(QUERY_NODE_COLUMN, (SNode**)&pCol); - assert(pCol); - pCol->dataBlockId = RES_BLK_ID; - pCol->slotId = getDstSlotId(MAX_SLOT_NUM + i); - pCol->node.resType.type = jtInputColType[i]; - pCol->node.resType.bytes = tDataTypes[jtInputColType[i]].bytes; - (void)sprintf(pCol->colName, "r%d", i); - - SValueNode* pVal = NULL; - code = nodesMakeNode(QUERY_NODE_VALUE, (SNode**)&pVal); - assert(pVal); - pVal->node.resType.type = jtInputColType[i]; - pVal->node.resType.bytes = tDataTypes[jtInputColType[i]].bytes; - assert(0 == nodesSetValueNodeValue(pVal, getFilterValue(jtInputColType[i]))); - - SOperatorNode* pOp = NULL; - code = nodesMakeNode(QUERY_NODE_OPERATOR, (SNode**)&pOp); - assert(pOp); - pOp->opType = OP_TYPE_GREATER_THAN; - pOp->node.resType.type = TSDB_DATA_TYPE_BOOL; - pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; - pOp->pLeft = (SNode*)pCol; - pOp->pRight = (SNode*)pVal; - - if ((jtCtx.leftFilterNum + jtCtx.rightFilterNum) > 1) { - assert(0 == nodesListMakeStrictAppend(&pLogic->pParameterList, (SNode*)pOp)); - } else { - p->node.pConditions = (SNode*)pOp; - break; - } - } - } - - if ((jtCtx.leftFilterNum + jtCtx.rightFilterNum) > 1) { - p->node.pConditions = (SNode*)pLogic; - } + SName tblName = {0}; + toName(1, qptCtx->param.db.dbName, qptCtx->param.tbl.tblName, &tblName); + memcpy(&pScanPhysiNode->tableName, &tblName, sizeof(SName)); } -void updateColRowInfo() { - jtCtx.blkRowSize = MAX_SLOT_NUM * sizeof(bool); - for (int32_t i = 0; i < MAX_SLOT_NUM; ++i) { - jtCtx.colRowOffset[i] = jtCtx.blkRowSize; - jtCtx.blkRowSize += tDataTypes[jtInputColType[i]].bytes; - } +STagScanPhysiNode* qptCreateTagScanPhysiNode(int32_t nodeType) { + SPhysiNode* pPhysiNode = qptCreatePhysiNode(nodeType); + assert(pPhysiNode); + + STagScanPhysiNode* pTagScanNode = (STagScanPhysiNode*)pPhysiNode; + pTagScanNode->onlyMetaCtbIdx = (taosRand() % 2) : true : false; + + qptCreateScanPhysiNodeImpl(&pTagScanNode->scan); + + qptPostCreatePhysiNode(pPhysiNode); + + return pPhysiNode; } -void createBlockDescNode(SDataBlockDescNode** ppNode) { - SDataBlockDescNode* pDesc = NULL; - int32_t code = nodesMakeNode(QUERY_NODE_DATABLOCK_DESC, (SNode**)&pDesc); - assert(pDesc); - pDesc->dataBlockId = RES_BLK_ID; - pDesc->totalRowSize = jtCtx.resColSize - MAX_SLOT_NUM * 2 * sizeof(bool); - pDesc->outputRowSize = pDesc->totalRowSize; - for (int32_t i = 0; i < jtCtx.resColNum; ++i) { - SSlotDescNode* pSlot = NULL; - int32_t code = nodesMakeNode(QUERY_NODE_SLOT_DESC, (SNode**)&pSlot); - assert(pSlot); - pSlot->slotId = i; - int32_t slotIdx = jtCtx.resColInSlot[i] >= MAX_SLOT_NUM ? jtCtx.resColInSlot[i] - MAX_SLOT_NUM : jtCtx.resColInSlot[i]; - pSlot->dataType.type = jtInputColType[slotIdx]; - pSlot->dataType.bytes = tDataTypes[pSlot->dataType.type].bytes; - assert(0 == nodesListMakeStrictAppend(&pDesc->pSlots, (SNode *)pSlot)); - } +SSortMergeJoinPhysiNode* qptCreateSortMergeJoinPhysiNode(int32_t nodeType) { + SPhysiNode* pPhysiNode = qptCreatePhysiNode(nodeType); + assert(pPhysiNode); - *ppNode = pDesc; -} + SSortMergeJoinPhysiNode* p = (SSortMergeJoinPhysiNode*)pPhysiNode; -SSortMergeJoinPhysiNode* createDummySortMergeJoinPhysiNode(SJoinTestParam* param) { - SSortMergeJoinPhysiNode* p = NULL; - int32_t code = nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN, (SNode**)&p); - assert(p); +/* p->joinType = param->joinType; p->subType = param->subType; p->asofOpType = param->asofOp; @@ -915,1950 +911,86 @@ SSortMergeJoinPhysiNode* createDummySortMergeJoinPhysiNode(SJoinTestParam* param createFilterEnd(p, param->filter); updateColRowInfo(); createBlockDescNode(&p->node.pOutputDataBlockDesc); +*/ - return p; + return pPhysiNode; } -SExecTaskInfo* createDummyTaskInfo(char* taskId) { - SExecTaskInfo* p = (SExecTaskInfo*)taosMemoryCalloc(1, sizeof(SExecTaskInfo)); - assert(p); - p->id.str = taskId; - - return p; -} - -SSDataBlock* createDummyBlock(int32_t blkId) { - SSDataBlock* p = NULL; - int32_t code = createDataBlock(&p); - assert(code == 0); - - p->info.id.blockId = blkId; - p->info.type = STREAM_INVALID; - p->info.calWin.skey = INT64_MIN; - p->info.calWin.ekey = INT64_MAX; - p->info.watermark = INT64_MIN; - - for (int32_t i = 0; i < MAX_SLOT_NUM; ++i) { - SColumnInfoData idata = - createColumnInfoData(jtInputColType[i], tDataTypes[jtInputColType[i]].bytes, i); - - assert(0 == blockDataAppendColInfo(p, &idata)); - } - - return p; -} - -void appendAsofLeftEachResGrps(char* leftInRow, int32_t rightOffset, int32_t rightRows) { - TAOS_MEMSET(jtCtx.resColBuf, 0, jtCtx.resColSize); - for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { - if (!jtCtx.resColList[c]) { - continue; - } - - if (*((bool*)leftInRow + c)) { - *(char*)(jtCtx.resColBuf + c) = true; - } else { - TAOS_MEMCPY(jtCtx.resColBuf + jtCtx.resColOffset[c], leftInRow + jtCtx.inColOffset[c], tDataTypes[jtInputColType[c]].bytes); - } - } - - int32_t endIdx = TMIN(rightRows, taosArrayGetSize(jtCtx.rightRowsList) - rightOffset) + rightOffset; - for (int32_t r = rightOffset; r < endIdx; ++r) { - bool* rightFilterOut = (bool*)taosArrayGet(jtCtx.rightFilterOut, r); - if (*rightFilterOut) { - continue; - } - - char* rightResRows = (char*)taosArrayGet(jtCtx.rightRowsList, r); - for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { - if (jtCtx.resColList[MAX_SLOT_NUM + c]) { - if (*(bool*)(rightResRows + c)) { - *(bool*)(jtCtx.resColBuf + MAX_SLOT_NUM + c) = true; - TAOS_MEMSET(jtCtx.resColBuf + jtCtx.resColOffset[MAX_SLOT_NUM + c], 0, tDataTypes[jtInputColType[c]].bytes); - } else { - *(bool*)(jtCtx.resColBuf + MAX_SLOT_NUM + c) = false; - TAOS_MEMCPY(jtCtx.resColBuf + jtCtx.resColOffset[MAX_SLOT_NUM + c], rightResRows + jtCtx.inColOffset[c], tDataTypes[jtInputColType[c]].bytes); - } - } - } - - pushResRow(jtCtx.resColBuf, jtCtx.resColSize); - } -} - -void appendLeftNonMatchGrp(char* leftInRow) { - if (!jtCtrl.noKeepResRows) { - TAOS_MEMSET(jtCtx.resColBuf, 0, jtCtx.resColSize); - for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { - if (!jtCtx.resColList[c]) { - continue; - } - - if (*((bool*)leftInRow + c)) { - *(char*)(jtCtx.resColBuf + c) = true; - } else { - TAOS_MEMCPY(jtCtx.resColBuf + jtCtx.resColOffset[c], leftInRow + jtCtx.inColOffset[c], tDataTypes[jtInputColType[c]].bytes); - } - } - - for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { - if (jtCtx.resColList[MAX_SLOT_NUM + c]) { - *(char*)(jtCtx.resColBuf + MAX_SLOT_NUM + c) = true; - } - } - } - - pushResRow(jtCtx.resColBuf, jtCtx.resColSize); -} - -void appendAllAsofResRows() { - int32_t leftRows = taosArrayGetSize(jtCtx.leftRowsList); - int32_t rightRows = taosArrayGetSize(jtCtx.rightRowsList); - if (rightRows <= 0) { - if (0 == jtCtx.rightFilterNum) { - for (int32_t i = 0; i < leftRows; ++i) { - char* leftInRow = (char*)taosArrayGet(jtCtx.leftRowsList, i); - assert(leftInRow); - appendLeftNonMatchGrp(leftInRow); - } - } - } else { - assert(rightRows <= jtCtx.jLimit); - for (int32_t i = 0; i < leftRows; ++i) { - char* leftInRow = (char*)taosArrayGet(jtCtx.leftRowsList, i); - assert(leftInRow); - appendAsofLeftEachResGrps(leftInRow, 0, rightRows); - } - } - taosArrayClear(jtCtx.leftRowsList); -} - -void chkAppendAsofForwardGrpResRows(bool forceOut) { - int32_t rightRows = taosArrayGetSize(jtCtx.rightRowsList); - if (rightRows < jtCtx.jLimit && !forceOut) { - return; - } - - int32_t rightRemains = rightRows; - int32_t rightOffset = 0; - int32_t leftRows = taosArrayGetSize(jtCtx.leftRowsList); - int32_t i = 0; - for (; i < leftRows; ++i) { - char* leftRow = (char*)taosArrayGet(jtCtx.leftRowsList, i); - assert(leftRow); - int64_t* leftTs = (int64_t*)(leftRow + jtCtx.inColOffset[JT_PRIM_TS_SLOT_ID]); - bool append = false; - for (int32_t r = rightOffset; r < rightRows; ++r) { - char* rightRow = (char*)taosArrayGet(jtCtx.rightRowsList, r); - assert(rightRow); - int64_t* rightTs = (int64_t*)(rightRow + jtCtx.inColOffset[JT_PRIM_TS_SLOT_ID]); - if (((jtCtx.asc && *leftTs > *rightTs) || (!jtCtx.asc && *leftTs < *rightTs)) || (*leftTs == *rightTs && (OP_TYPE_LOWER_THAN == jtCtx.asofOpType || OP_TYPE_GREATER_THAN == jtCtx.asofOpType))) { - rightOffset++; - rightRemains--; - if (rightRemains < jtCtx.jLimit && !forceOut) { - taosArrayPopFrontBatch(jtCtx.rightRowsList, rightOffset); - taosArrayPopFrontBatch(jtCtx.rightFilterOut, rightOffset); - taosArrayPopFrontBatch(jtCtx.leftRowsList, i); - return; - } - - continue; - } - - appendAsofLeftEachResGrps(leftRow, rightOffset, jtCtx.jLimit); - append = true; - break; - } - - if (!append) { - if (!forceOut) { - break; - } - - if (0 == jtCtx.rightFilterNum) { - appendLeftNonMatchGrp(leftRow); - } - } - } - - taosArrayPopFrontBatch(jtCtx.rightRowsList, rightOffset); - taosArrayPopFrontBatch(jtCtx.rightFilterOut, rightOffset); - taosArrayPopFrontBatch(jtCtx.leftRowsList, i); -} - - -void appendWinEachResGrps(char* leftInRow, int32_t rightOffset, int32_t rightRows) { - if (rightOffset < 0) { - if (0 == jtCtx.rightFilterNum) { - appendLeftNonMatchGrp(leftInRow); - } - return; - } - - TAOS_MEMSET(jtCtx.resColBuf, 0, jtCtx.resColSize); - for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { - if (!jtCtx.resColList[c]) { - continue; - } - - if (*((bool*)leftInRow + c)) { - *(char*)(jtCtx.resColBuf + c) = true; - } else { - TAOS_MEMCPY(jtCtx.resColBuf + jtCtx.resColOffset[c], leftInRow + jtCtx.inColOffset[c], tDataTypes[jtInputColType[c]].bytes); - } - } - - int32_t endIdx = rightRows + rightOffset; - int32_t beginIdx = (!jtCtx.asc && rightRows > jtCtx.jLimit) ? (endIdx - jtCtx.jLimit) : rightOffset; - for (int32_t r = beginIdx; r < endIdx; ++r) { - bool* rightFilterOut = (bool*)taosArrayGet(jtCtx.rightFilterOut, r); - if (*rightFilterOut) { - continue; - } - - char* rightResRows = (char*)taosArrayGet(jtCtx.rightRowsList, r); - assert(rightResRows); - for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { - if (jtCtx.resColList[MAX_SLOT_NUM + c]) { - if (*(bool*)(rightResRows + c)) { - *(bool*)(jtCtx.resColBuf + MAX_SLOT_NUM + c) = true; - TAOS_MEMSET(jtCtx.resColBuf + jtCtx.resColOffset[MAX_SLOT_NUM + c], 0, tDataTypes[jtInputColType[c]].bytes); - } else { - *(bool*)(jtCtx.resColBuf + MAX_SLOT_NUM + c) = false; - TAOS_MEMCPY(jtCtx.resColBuf + jtCtx.resColOffset[MAX_SLOT_NUM + c], rightResRows + jtCtx.inColOffset[c], tDataTypes[jtInputColType[c]].bytes); - } - } - } - - pushResRow(jtCtx.resColBuf, jtCtx.resColSize); - } -} - -void chkAppendWinResRows(bool forceOut) { - int32_t rightRows = taosArrayGetSize(jtCtx.rightRowsList); - if (rightRows < jtCtx.jLimit && !forceOut) { - return; - } - - int32_t rightRemains = rightRows; - int32_t rightOffset = 0; - int32_t leftRows = taosArrayGetSize(jtCtx.leftRowsList); - int32_t i = 0; - for (; i < leftRows; ++i) { - char* leftRow = (char*)taosArrayGet(jtCtx.leftRowsList, i); - assert(leftRow); - int64_t* leftTs = (int64_t*)(leftRow + jtCtx.inColOffset[JT_PRIM_TS_SLOT_ID]); - int64_t winStart = *leftTs + jtCtx.winStartOffset; - int64_t winEnd = *leftTs + jtCtx.winEndOffset; - int32_t winBeginIdx = -1; - bool append = false; - bool winClosed = false; - for (int32_t r = rightOffset; r < rightRows; ++r) { - char* rightRow = (char*)taosArrayGet(jtCtx.rightRowsList, r); - assert(rightRow); - int64_t* rightTs = (int64_t*)(rightRow + jtCtx.inColOffset[JT_PRIM_TS_SLOT_ID]); - if ((jtCtx.asc && *rightTs < winStart) || (!jtCtx.asc && *rightTs > winEnd)) { - rightOffset++; - rightRemains--; - if (rightRemains < jtCtx.jLimit && !forceOut) { - taosArrayPopFrontBatch(jtCtx.rightRowsList, rightOffset); - taosArrayPopFrontBatch(jtCtx.rightFilterOut, rightOffset); - taosArrayPopFrontBatch(jtCtx.leftRowsList, i); - return; - } - - continue; - } else if ((jtCtx.asc && *rightTs > winEnd) || (!jtCtx.asc && *rightTs < winStart)) { - winClosed = true; - appendWinEachResGrps(leftRow, winBeginIdx, r - winBeginIdx); - append = true; - break; - } - - if (-1 == winBeginIdx) { - winBeginIdx = r; - } - - if (jtCtx.asc && (r - winBeginIdx + 1) >= jtCtx.jLimit) { - appendWinEachResGrps(leftRow, winBeginIdx, jtCtx.jLimit); - append = true; - break; - } - } - - if (!append) { - if (!forceOut) { - break; - } - - if (winBeginIdx >= 0) { - appendWinEachResGrps(leftRow, winBeginIdx, rightRows - winBeginIdx); - } else if (0 == jtCtx.rightFilterNum) { - appendLeftNonMatchGrp(leftRow); - } - } - } - - taosArrayPopFrontBatch(jtCtx.rightRowsList, rightOffset); - taosArrayPopFrontBatch(jtCtx.rightFilterOut, rightOffset); - taosArrayPopFrontBatch(jtCtx.leftRowsList, i); -} - - -void trimForAsofJlimit() { - int32_t rowNum = taosArrayGetSize(jtCtx.rightRowsList); - if (rowNum <= jtCtx.jLimit) { - return; - } - - taosArrayPopFrontBatch(jtCtx.rightRowsList, rowNum - jtCtx.jLimit); - taosArrayPopFrontBatch(jtCtx.rightFilterOut, rowNum - jtCtx.jLimit); -} - -void createGrpRows(SSDataBlock** ppBlk, int32_t blkId, int32_t grpRows) { - if (grpRows <= 0) { - return; - } - - if (NULL == *ppBlk) { - *ppBlk = createDummyBlock((blkId == LEFT_BLK_ID) ? LEFT_BLK_ID : RIGHT_BLK_ID); - assert(*ppBlk); - assert(0 == blockDataEnsureCapacity(*ppBlk, jtCtx.blkRows)); - assert(NULL != taosArrayPush((blkId == LEFT_BLK_ID) ? jtCtx.leftBlkList : jtCtx.rightBlkList, ppBlk)); - } - - if (jtCtx.grpJoin) { - (*ppBlk)->info.id.groupId = jtCtx.inGrpId; - } - - jtCtx.inputStat |= (1 << blkId); - - SArray* pTableRows = NULL; - int32_t tableOffset = 0; - int32_t peerOffset = 0; - bool keepRes = false; - bool keepInput = false; - if (blkId == LEFT_BLK_ID) { - if ((jtCtx.joinType == JOIN_TYPE_LEFT || jtCtx.joinType == JOIN_TYPE_FULL) && (jtCtx.subType != JOIN_STYPE_SEMI && jtCtx.subType != JOIN_STYPE_ASOF && jtCtx.subType != JOIN_STYPE_WIN)) { - keepRes = true; - } - peerOffset = MAX_SLOT_NUM; - } else { - if ((jtCtx.joinType == JOIN_TYPE_RIGHT || jtCtx.joinType == JOIN_TYPE_FULL) && (jtCtx.subType != JOIN_STYPE_SEMI && jtCtx.subType != JOIN_STYPE_ASOF && jtCtx.subType != JOIN_STYPE_WIN)) { - keepRes = true; - } - tableOffset = MAX_SLOT_NUM; - } - - if (JOIN_STYPE_ASOF == jtCtx.subType) { - keepInput = jtCtx.asofOpType != OP_TYPE_EQUAL ? true : (blkId == LEFT_BLK_ID); - pTableRows = (blkId == LEFT_BLK_ID) ? jtCtx.leftRowsList : jtCtx.rightRowsList; - } else if (JOIN_STYPE_WIN == jtCtx.subType) { - keepInput = true; - pTableRows = (blkId == LEFT_BLK_ID) ? jtCtx.leftRowsList : jtCtx.rightRowsList; - } - - int32_t filterNum = (blkId == LEFT_BLK_ID) ? jtCtx.leftFilterNum : jtCtx.rightFilterNum; - int32_t peerFilterNum = (blkId == LEFT_BLK_ID) ? jtCtx.rightFilterNum : jtCtx.leftFilterNum; - int32_t* filterCol = (blkId == LEFT_BLK_ID) ? jtCtx.leftFilterColList : jtCtx.rightFilterColList; - - char* pData = NULL; - int32_t tmpInt = 0; - int64_t tmpBigint = 0; - bool isNull = false; - bool filterOut = false; - bool addToRowList = false; - int32_t vRange = TMAX(grpRows / 3, 3); - for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { - jtCtx.grpOffset[c] = c * TMAX(100, grpRows); - } - - for (int32_t i = 0; i < grpRows; ++i) { - if ((*ppBlk)->info.rows >= (*ppBlk)->info.capacity) { - *ppBlk = createDummyBlock((blkId == LEFT_BLK_ID) ? LEFT_BLK_ID : RIGHT_BLK_ID); - assert(*ppBlk); - assert(0 == blockDataEnsureCapacity(*ppBlk, jtCtx.blkRows)); - assert(NULL != taosArrayPush((blkId == LEFT_BLK_ID) ? jtCtx.leftBlkList : jtCtx.rightBlkList, ppBlk)); - if (jtCtx.grpJoin) { - (*ppBlk)->info.id.groupId = jtCtx.inGrpId; - } - } - - filterOut = (peerFilterNum > 0 && (jtCtx.subType != JOIN_STYPE_ASOF && jtCtx.subType != JOIN_STYPE_WIN)) ? true : false; - if (!filterOut) { - TAOS_MEMSET(jtCtx.resColBuf, 0, jtCtx.resColSize); - if (keepInput) { - TAOS_MEMSET(jtCtx.inColBuf, 0, jtCtx.inColSize); - } - } - - addToRowList = true; - - for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { - switch (jtInputColType[c]) { - case TSDB_DATA_TYPE_TIMESTAMP: - jtCtx.asc ? ++jtCtx.curTs : --jtCtx.curTs; - pData = (char*)&jtCtx.curTs; - isNull = false; - if (!filterOut && filterNum && filterCol[c] && jtCtx.curTs <= TIMESTAMP_FILTER_VALUE) { - filterOut = true; - } - break; - case TSDB_DATA_TYPE_INT: - if (taosRand() % 10) { - tmpInt = (taosRand() % 2) ? INT_FILTER_VALUE + jtCtx.grpOffset[c] + taosRand() % vRange : INT_FILTER_VALUE - jtCtx.grpOffset[c] - taosRand() % vRange; - pData = (char*)&tmpInt; - isNull = false; - if (!filterOut && filterNum && filterCol[c] && tmpInt <= INT_FILTER_VALUE) { - filterOut = true; - } - } else { - isNull = true; - if (filterNum && filterCol[c]) { - filterOut = true; - } - } - break; - case TSDB_DATA_TYPE_BIGINT: - tmpBigint = (taosRand() % 2) ? BIGINT_FILTER_VALUE + jtCtx.curKeyOffset++ : BIGINT_FILTER_VALUE - jtCtx.curKeyOffset++; - pData = (char*)&tmpBigint; - isNull = false; - if (!filterOut && filterNum && filterCol[c] && tmpBigint <= BIGINT_FILTER_VALUE) { - filterOut = true; - } - break; - default: - break; - } - - SColumnInfoData* pCol = (SColumnInfoData*)taosArrayGet((*ppBlk)->pDataBlock, c); - assert(pCol); - assert(0 == colDataSetVal(pCol, (*ppBlk)->info.rows, pData, isNull)); - - if (keepInput) { - if (!filterOut || (blkId != LEFT_BLK_ID)) { - if (isNull) { - *(char*)(jtCtx.inColBuf + c) = true; - } else { - TAOS_MEMCPY(jtCtx.inColBuf + jtCtx.inColOffset[c], pData, tDataTypes[jtInputColType[c]].bytes); - } - } else { - addToRowList = false; - } - } else if (keepRes && !filterOut && jtCtx.resColList[tableOffset + c]) { - if (isNull) { - *(char*)(jtCtx.resColBuf + tableOffset + c) = true; - } else { - TAOS_MEMCPY(jtCtx.resColBuf + jtCtx.resColOffset[tableOffset + c], pData, tDataTypes[jtInputColType[c]].bytes); - } - } - } - - if (keepInput && addToRowList) { - assert(NULL != taosArrayPush(pTableRows, jtCtx.inColBuf)); - if (blkId == RIGHT_BLK_ID) { - bool fout = filterOut ? true : false; - assert(NULL != taosArrayPush(jtCtx.rightFilterOut, &fout)); - } - } - - if (keepRes && !filterOut) { - for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { - if (jtCtx.resColList[peerOffset + c]) { - *(char*)(jtCtx.resColBuf + peerOffset + c) = true; - } - } - - pushResRow(jtCtx.resColBuf, jtCtx.resColSize); - } - - (*ppBlk)->info.rows++; - } - - if (keepInput) { - if (JOIN_STYPE_ASOF == jtCtx.subType) { - if (((jtCtx.asc && (jtCtx.asofOpType == OP_TYPE_GREATER_EQUAL || jtCtx.asofOpType == OP_TYPE_GREATER_THAN)) || (!jtCtx.asc && (jtCtx.asofOpType == OP_TYPE_LOWER_EQUAL || jtCtx.asofOpType == OP_TYPE_LOWER_THAN)) ) || jtCtx.asofOpType == OP_TYPE_EQUAL) { - if (blkId == LEFT_BLK_ID) { - appendAllAsofResRows(); - } else { - trimForAsofJlimit(); - } - } else { - chkAppendAsofForwardGrpResRows(false); - } - } else { - chkAppendWinResRows(false); - } - } - -} - -void createRowData(SSDataBlock* pBlk, int64_t tbOffset, int32_t rowIdx, int32_t vRange) { - int32_t tmpInt = 0; - int64_t tmpBig = 0; - - for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { - SColumnInfoData* pCol = (SColumnInfoData*)taosArrayGet(pBlk->pDataBlock, c); - assert(pCol); - - int32_t rv = taosRand() % 10; - switch (jtInputColType[c]) { - case TSDB_DATA_TYPE_TIMESTAMP: - *(int64_t*)(jtCtx.colRowDataBuf + tbOffset + rowIdx * jtCtx.blkRowSize + jtCtx.colRowOffset[c]) = jtCtx.curTs; - assert(0 == colDataSetVal(pCol, pBlk->info.rows, (char*)&jtCtx.curTs, false)); - break; - case TSDB_DATA_TYPE_INT: - if (rv) { - tmpInt = (taosRand() % 2) ? INT_FILTER_VALUE + jtCtx.grpOffset[c] + taosRand() % vRange : INT_FILTER_VALUE - taosRand() % vRange; - *(int32_t*)(jtCtx.colRowDataBuf + tbOffset + rowIdx * jtCtx.blkRowSize + jtCtx.colRowOffset[c]) = tmpInt; - assert(0 == colDataSetVal(pCol, pBlk->info.rows, (char*)&tmpInt, false)); - } else { - *(bool*)(jtCtx.colRowDataBuf + tbOffset + rowIdx * jtCtx.blkRowSize + c) = true; - assert(0 == colDataSetVal(pCol, pBlk->info.rows, NULL, true)); - } - break; - case TSDB_DATA_TYPE_BIGINT: - tmpBig = (taosRand() % 2) ? BIGINT_FILTER_VALUE + jtCtx.curKeyOffset++ : BIGINT_FILTER_VALUE - jtCtx.curKeyOffset++; - *(int64_t*)(jtCtx.colRowDataBuf + tbOffset + rowIdx * jtCtx.blkRowSize + jtCtx.colRowOffset[c]) = tmpBig; - assert(0 == colDataSetVal(pCol, pBlk->info.rows, (char*)&tmpBig, false)); - break; - default: - break; - } - } - - pBlk->info.rows++; -} - -void makeAppendBlkData(SSDataBlock** ppLeft, SSDataBlock** ppRight, int32_t leftGrpRows, int32_t rightGrpRows) { - int64_t totalSize = (leftGrpRows + rightGrpRows) * jtCtx.blkRowSize; - int64_t rightOffset = leftGrpRows * jtCtx.blkRowSize; - - if (jtCtx.colRowDataBufSize < totalSize) { - jtCtx.colRowDataBuf = (char*)taosMemoryRealloc(jtCtx.colRowDataBuf, totalSize); - assert(jtCtx.colRowDataBuf); - } - - TAOS_MEMSET(jtCtx.colRowDataBuf, 0, totalSize); - - for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { - jtCtx.grpOffset[c] = c * TMAX(leftGrpRows, rightGrpRows); - } - - int32_t vRange = TMAX(leftGrpRows / 100, 3); - for (int32_t i = 0; i < leftGrpRows; ++i) { - if ((*ppLeft)->info.rows >= (*ppLeft)->info.capacity) { - *ppLeft = createDummyBlock(LEFT_BLK_ID); - assert(*ppLeft); - assert(0 == blockDataEnsureCapacity(*ppLeft, jtCtx.blkRows)); - assert(NULL != taosArrayPush(jtCtx.leftBlkList, ppLeft)); - if (jtCtx.grpJoin) { - (*ppLeft)->info.id.groupId = jtCtx.inGrpId; - } - } - - createRowData(*ppLeft, 0, i, vRange); - } - - vRange = TMAX(rightGrpRows / 100, 3); - for (int32_t i = 0; i < rightGrpRows; ++i) { - if ((*ppRight)->info.rows >= (*ppRight)->info.capacity) { - *ppRight = createDummyBlock(RIGHT_BLK_ID); - assert(*ppRight); - assert(0 == blockDataEnsureCapacity(*ppRight, jtCtx.blkRows)); - assert(NULL != taosArrayPush(jtCtx.rightBlkList, ppRight)); - if (jtCtx.grpJoin) { - (*ppRight)->info.id.groupId = jtCtx.inGrpId; - } - } - - createRowData(*ppRight, rightOffset, i, vRange); - } - -} - -void putNMatchRowToRes(char* lrow, int32_t tableOffset, int32_t peerOffset) { - if (!jtCtrl.noKeepResRows) { - TAOS_MEMSET(jtCtx.resColBuf, 0, jtCtx.resColSize); - - for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { - if (jtCtx.resColList[tableOffset + c]) { - if (*(bool*)(lrow + c)) { - *(bool*)(jtCtx.resColBuf + tableOffset + c) = true; - } else { - TAOS_MEMCPY(jtCtx.resColBuf + jtCtx.resColOffset[tableOffset + c], lrow + jtCtx.colRowOffset[c], tDataTypes[jtInputColType[c]].bytes); - } - } - } - - for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { - if (jtCtx.resColList[peerOffset + c]) { - *(bool*)(jtCtx.resColBuf + peerOffset + c) = true; - } - } - } - - pushResRow(jtCtx.resColBuf, jtCtx.resColSize); -} - -void putMatchRowToRes(char* lrow, char* rrow, int32_t cols) { - if (!jtCtrl.noKeepResRows) { - TAOS_MEMSET(jtCtx.resColBuf, 0, jtCtx.resColSize); - - if (cols & LEFT_TABLE_COLS) { - for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { - if (jtCtx.resColList[c]) { - if (*(bool*)(lrow + c)) { - *(bool*)(jtCtx.resColBuf + c) = true; - } else { - TAOS_MEMCPY(jtCtx.resColBuf + jtCtx.resColOffset[c], lrow + jtCtx.colRowOffset[c], tDataTypes[jtInputColType[c]].bytes); - } - } - } - } - - if (cols & RIGHT_TABLE_COLS) { - for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { - if (jtCtx.resColList[MAX_SLOT_NUM + c]) { - if (*(bool*)(rrow + c)) { - *(bool*)(jtCtx.resColBuf + MAX_SLOT_NUM + c) = true; - } else { - TAOS_MEMCPY(jtCtx.resColBuf + jtCtx.resColOffset[MAX_SLOT_NUM + c], rrow + jtCtx.colRowOffset[c], tDataTypes[jtInputColType[c]].bytes); - } - } - } - } - } - - pushResRow(jtCtx.resColBuf, jtCtx.resColSize); -} - - - -void innerJoinAppendEqGrpRes(int32_t leftGrpRows, int32_t rightGrpRows) { - bool leftMatch = false, rightMatch = false, filterOut = false; - void* lValue = NULL, *rValue = NULL, *filterValue = NULL; - int64_t lBig = 0, rBig = 0, fbig = 0; - int64_t rightTbOffset = jtCtx.blkRowSize * leftGrpRows; - - for (int32_t l = 0; l < leftGrpRows; ++l) { - char* lrow = jtCtx.colRowDataBuf + jtCtx.blkRowSize * l; - - filterOut = false; - leftMatch = true; - - for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { - lValue = lrow + jtCtx.colRowOffset[c]; - switch (jtInputColType[c]) { - case TSDB_DATA_TYPE_TIMESTAMP: - fbig = TIMESTAMP_FILTER_VALUE; - lBig = *(int64_t*)lValue; - break; - case TSDB_DATA_TYPE_INT: - fbig = INT_FILTER_VALUE; - lBig = *(int32_t*)lValue; - break; - case TSDB_DATA_TYPE_BIGINT: - fbig = BIGINT_FILTER_VALUE; - lBig = *(int64_t*)lValue; - break; - default: - break; - } - - if (jtCtx.leftFilterNum && jtCtx.leftFilterColList[c] && ((*(bool*)(lrow + c)) || lBig <= fbig)) { - filterOut = true; - break; - } - - if (jtCtx.colEqNum && jtCtx.colEqList[c] && (*(bool*)(lrow + c))) { - leftMatch = false; - break; - } - - if (jtCtx.colOnNum && jtCtx.colOnList[c] && (*(bool*)(lrow + c))) { - leftMatch = false; - break; - } - } - - if (filterOut || !leftMatch) { - continue; - } - - for (int32_t r = 0; r < rightGrpRows; ++r) { - char* rrow = jtCtx.colRowDataBuf + rightTbOffset + jtCtx.blkRowSize * r; - rightMatch = true; - filterOut = false; - - for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { - lValue = lrow + jtCtx.colRowOffset[c]; - - if (!*(bool*)(rrow + c)) { - rValue = rrow + jtCtx.colRowOffset[c]; - } - - switch (jtInputColType[c]) { - case TSDB_DATA_TYPE_TIMESTAMP: - fbig = TIMESTAMP_FILTER_VALUE; - lBig = *(int64_t*)lValue; - rBig = *(int64_t*)rValue; - break; - case TSDB_DATA_TYPE_INT: - fbig = INT_FILTER_VALUE; - lBig = *(int32_t*)lValue; - rBig = *(int32_t*)rValue; - break; - case TSDB_DATA_TYPE_BIGINT: - fbig = BIGINT_FILTER_VALUE; - lBig = *(int64_t*)lValue; - rBig = *(int64_t*)rValue; - break; - default: - break; - } - - if (jtCtx.colEqNum && jtCtx.colEqList[c] && ((*(bool*)(rrow + c)) || lBig != rBig)) { - rightMatch = false; - break; - } - - if (jtCtx.colOnNum && jtCtx.colOnList[c] && ((*(bool*)(rrow + c)) || lBig <= rBig)) { - rightMatch = false; - break; - } - - if (jtCtx.rightFilterNum && jtCtx.rightFilterColList[c] && ((*(bool*)(rrow + c)) || rBig <= fbig)) { - filterOut = true; - break; - } - } - - if (filterOut || !rightMatch) { - continue; - } - - putMatchRowToRes(lrow, rrow, ALL_TABLE_COLS); - } - } - - -} - - - -void leftJoinAppendEqGrpRes(int32_t leftGrpRows, int32_t rightGrpRows) { - bool leftMatch = false, rightMatch = false, filterOut = false; - void* lValue = NULL, *rValue = NULL, *filterValue = NULL; - int64_t lBig = 0, rBig = 0, fbig = 0; - int64_t rightTbOffset = jtCtx.blkRowSize * leftGrpRows; - - for (int32_t l = 0; l < leftGrpRows; ++l) { - char* lrow = jtCtx.colRowDataBuf + jtCtx.blkRowSize * l; - - filterOut = false; - leftMatch = true; - - for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { - lValue = lrow + jtCtx.colRowOffset[c]; - switch (jtInputColType[c]) { - case TSDB_DATA_TYPE_TIMESTAMP: - fbig = TIMESTAMP_FILTER_VALUE; - lBig = *(int64_t*)lValue; - break; - case TSDB_DATA_TYPE_INT: - fbig = INT_FILTER_VALUE; - lBig = *(int32_t*)lValue; - break; - case TSDB_DATA_TYPE_BIGINT: - fbig = BIGINT_FILTER_VALUE; - lBig = *(int64_t*)lValue; - break; - default: - break; - } - - if (jtCtx.leftFilterNum && jtCtx.leftFilterColList[c] && ((*(bool*)(lrow + c)) || lBig <= fbig)) { - filterOut = true; - break; - } - - if (jtCtx.colEqNum && jtCtx.colEqList[c] && (*(bool*)(lrow + c))) { - leftMatch = false; - } - - if (jtCtx.colOnNum && jtCtx.colOnList[c] && (*(bool*)(lrow + c))) { - leftMatch = false; - } - } - - if (filterOut) { - continue; - } - - if (false == leftMatch) { - if (0 == jtCtx.rightFilterNum) { - putNMatchRowToRes(lrow, 0, MAX_SLOT_NUM); - } - continue; - } - - leftMatch = false; - for (int32_t r = 0; r < rightGrpRows; ++r) { - char* rrow = jtCtx.colRowDataBuf + rightTbOffset + jtCtx.blkRowSize * r; - rightMatch = true; - filterOut = false; - - for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { - lValue = lrow + jtCtx.colRowOffset[c]; - - if (!*(bool*)(rrow + c)) { - rValue = rrow + jtCtx.colRowOffset[c]; - } - - switch (jtInputColType[c]) { - case TSDB_DATA_TYPE_TIMESTAMP: - fbig = TIMESTAMP_FILTER_VALUE; - lBig = *(int64_t*)lValue; - rBig = *(int64_t*)rValue; - break; - case TSDB_DATA_TYPE_INT: - fbig = INT_FILTER_VALUE; - lBig = *(int32_t*)lValue; - rBig = *(int32_t*)rValue; - break; - case TSDB_DATA_TYPE_BIGINT: - fbig = BIGINT_FILTER_VALUE; - lBig = *(int64_t*)lValue; - rBig = *(int64_t*)rValue; - break; - default: - break; - } - - if (jtCtx.colEqNum && jtCtx.colEqList[c] && ((*(bool*)(rrow + c)) || lBig != rBig)) { - rightMatch = false; - break; - } - - if (jtCtx.colOnNum && jtCtx.colOnList[c] && ((*(bool*)(rrow + c)) || lBig <= rBig)) { - rightMatch = false; - break; - } - - if (jtCtx.rightFilterNum && jtCtx.rightFilterColList[c] && ((*(bool*)(rrow + c)) || rBig <= fbig)) { - filterOut = true; - } - } - - if (rightMatch) { - leftMatch = true; - } - - if (filterOut) { - continue; - } - - if (rightMatch) { - putMatchRowToRes(lrow, rrow, ALL_TABLE_COLS); - } - } - - if (!leftMatch && 0 == jtCtx.rightFilterNum) { - putNMatchRowToRes(lrow, 0, MAX_SLOT_NUM); - } - } - - -} - - - -void semiJoinAppendEqGrpRes(int32_t leftGrpRows, int32_t rightGrpRows) { - bool leftMatch = false, rightMatch = false, filterOut = false; - void* lValue = NULL, *rValue = NULL, *filterValue = NULL; - int64_t lBig = 0, rBig = 0, fbig = 0; - int64_t leftTbOffset = 0; - int64_t rightTbOffset = jtCtx.blkRowSize * leftGrpRows; - char* rrow = NULL; - - for (int32_t l = 0; l < leftGrpRows; ++l) { - char* lrow = jtCtx.colRowDataBuf + leftTbOffset + jtCtx.blkRowSize * l; - - filterOut = false; - leftMatch = true; - - for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { - lValue = lrow + jtCtx.colRowOffset[c]; - switch (jtInputColType[c]) { - case TSDB_DATA_TYPE_TIMESTAMP: - fbig = TIMESTAMP_FILTER_VALUE; - lBig = *(int64_t*)lValue; - break; - case TSDB_DATA_TYPE_INT: - fbig = INT_FILTER_VALUE; - lBig = *(int32_t*)lValue; - break; - case TSDB_DATA_TYPE_BIGINT: - fbig = BIGINT_FILTER_VALUE; - lBig = *(int64_t*)lValue; - break; - default: - break; - } - - if (jtCtx.leftFilterNum && jtCtx.leftFilterColList[c] && ((*(bool*)(lrow + c)) || lBig <= fbig)) { - filterOut = true; - break; - } - - if (jtCtx.colEqNum && jtCtx.colEqList[c] && (*(bool*)(lrow + c))) { - leftMatch = false; - break; - } - - if (jtCtx.colOnNum && jtCtx.colOnList[c] && (*(bool*)(lrow + c))) { - leftMatch = false; - break; - } - } - - if (filterOut || !leftMatch) { - continue; - } - - for (int32_t r = 0; r < rightGrpRows; ++r) { - rrow = jtCtx.colRowDataBuf + rightTbOffset + jtCtx.blkRowSize * r; - rightMatch = true; - filterOut = false; - - for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { - lValue = lrow + jtCtx.colRowOffset[c]; - - if (!*(bool*)(rrow + c)) { - rValue = rrow + jtCtx.colRowOffset[c]; - } - - switch (jtInputColType[c]) { - case TSDB_DATA_TYPE_TIMESTAMP: - fbig = TIMESTAMP_FILTER_VALUE; - lBig = *(int64_t*)lValue; - rBig = *(int64_t*)rValue; - break; - case TSDB_DATA_TYPE_INT: - fbig = INT_FILTER_VALUE; - lBig = *(int32_t*)lValue; - rBig = *(int32_t*)rValue; - break; - case TSDB_DATA_TYPE_BIGINT: - fbig = BIGINT_FILTER_VALUE; - lBig = *(int64_t*)lValue; - rBig = *(int64_t*)rValue; - break; - default: - break; - } - - if (jtCtx.colEqNum && jtCtx.colEqList[c] && ((*(bool*)(rrow + c)) || lBig != rBig)) { - rightMatch = false; - break; - } - - if (jtCtx.colOnNum && jtCtx.colOnList[c] && ((*(bool*)(rrow + c)) || lBig <= rBig)) { - rightMatch = false; - break; - } - - if (jtCtx.rightFilterNum && jtCtx.rightFilterColList[c] && ((*(bool*)(rrow + c)) || rBig <= fbig)) { - filterOut = true; - break; - } - } - - if (filterOut || !rightMatch) { - continue; - } - - break; - } - - if (!filterOut && rightMatch) { - putMatchRowToRes(lrow, rrow, LEFT_TABLE_COLS); - } - } - - -} - - - - -void antiJoinAppendEqGrpRes(int32_t leftGrpRows, int32_t rightGrpRows) { - bool leftMatch = false, rightMatch = false, filterOut = false; - void* lValue = NULL, *rValue = NULL, *filterValue = NULL; - int64_t lBig = 0, rBig = 0, fbig = 0; - int64_t rightTbOffset = jtCtx.blkRowSize * leftGrpRows; - - assert(0 == jtCtx.rightFilterNum); - - for (int32_t l = 0; l < leftGrpRows; ++l) { - char* lrow = jtCtx.colRowDataBuf + jtCtx.blkRowSize * l; - - filterOut = false; - leftMatch = true; - - for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { - lValue = lrow + jtCtx.colRowOffset[c]; - switch (jtInputColType[c]) { - case TSDB_DATA_TYPE_TIMESTAMP: - fbig = TIMESTAMP_FILTER_VALUE; - lBig = *(int64_t*)lValue; - break; - case TSDB_DATA_TYPE_INT: - fbig = INT_FILTER_VALUE; - lBig = *(int32_t*)lValue; - break; - case TSDB_DATA_TYPE_BIGINT: - fbig = BIGINT_FILTER_VALUE; - lBig = *(int64_t*)lValue; - break; - default: - break; - } - - if (jtCtx.leftFilterNum && jtCtx.leftFilterColList[c] && ((*(bool*)(lrow + c)) || lBig <= fbig)) { - filterOut = true; - break; - } - - if (jtCtx.colEqNum && jtCtx.colEqList[c] && (*(bool*)(lrow + c))) { - leftMatch = false; - } - - if (jtCtx.colOnNum && jtCtx.colOnList[c] && (*(bool*)(lrow + c))) { - leftMatch = false; - } - } - - if (filterOut) { - continue; - } - - if (false == leftMatch) { - putNMatchRowToRes(lrow, 0, MAX_SLOT_NUM); - continue; - } - - leftMatch = false; - for (int32_t r = 0; r < rightGrpRows; ++r) { - char* rrow = jtCtx.colRowDataBuf + rightTbOffset + jtCtx.blkRowSize * r; - rightMatch = true; - - for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { - lValue = lrow + jtCtx.colRowOffset[c]; - - if (!*(bool*)(rrow + c)) { - rValue = rrow + jtCtx.colRowOffset[c]; - } - - switch (jtInputColType[c]) { - case TSDB_DATA_TYPE_TIMESTAMP: - fbig = TIMESTAMP_FILTER_VALUE; - lBig = *(int64_t*)lValue; - rBig = *(int64_t*)rValue; - break; - case TSDB_DATA_TYPE_INT: - fbig = INT_FILTER_VALUE; - lBig = *(int32_t*)lValue; - rBig = *(int32_t*)rValue; - break; - case TSDB_DATA_TYPE_BIGINT: - fbig = BIGINT_FILTER_VALUE; - lBig = *(int64_t*)lValue; - rBig = *(int64_t*)rValue; - break; - default: - break; - } - - if (jtCtx.colEqNum && jtCtx.colEqList[c] && ((*(bool*)(rrow + c)) || lBig != rBig)) { - rightMatch = false; - break; - } - - if (jtCtx.colOnNum && jtCtx.colOnList[c] && ((*(bool*)(rrow + c)) || lBig <= rBig)) { - rightMatch = false; - break; - } - } - - if (rightMatch) { - leftMatch = true; - break; - } - } - - if (!leftMatch) { - putNMatchRowToRes(lrow, 0, MAX_SLOT_NUM); - } - } - - -} - -void addAsofEqInRows(int32_t rowsNum, int64_t tbOffset, bool leftTable) { - bool filterOut = false; - void* cvalue = NULL; - int64_t cbig = 0, fbig = 0; - int32_t filterNum = leftTable ? jtCtx.leftFilterNum : jtCtx.rightFilterNum; - int32_t* filterCol = leftTable ? jtCtx.leftFilterColList : jtCtx.rightFilterColList; - SArray* rowList = leftTable ? jtCtx.leftRowsList : jtCtx.rightRowsList; - - if (!leftTable) { - rowsNum = TMIN(rowsNum, jtCtx.jLimit); - } - - for (int32_t l = 0; l < rowsNum; ++l) { - char* row = jtCtx.colRowDataBuf + tbOffset + jtCtx.blkRowSize * l; - - filterOut = false; - - for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { - cvalue = row + jtCtx.colRowOffset[c]; - switch (jtInputColType[c]) { - case TSDB_DATA_TYPE_TIMESTAMP: - fbig = TIMESTAMP_FILTER_VALUE; - cbig = *(int64_t*)cvalue; - break; - case TSDB_DATA_TYPE_INT: - fbig = INT_FILTER_VALUE; - cbig = *(int32_t*)cvalue; - break; - case TSDB_DATA_TYPE_BIGINT: - fbig = BIGINT_FILTER_VALUE; - cbig = *(int64_t*)cvalue; - break; - default: - break; - } - - if (filterNum && filterCol[c] && ((*(bool*)(row + c)) || cbig <= fbig)) { - filterOut = true; - break; - } - } - - if (filterOut && leftTable) { - continue; - } - - assert(NULL != taosArrayPush(rowList, row)); - if (!leftTable) { - assert(NULL != taosArrayPush(jtCtx.rightFilterOut, &filterOut)); - } - } - - if (!leftTable && ((jtCtx.asc && (jtCtx.asofOpType == OP_TYPE_GREATER_EQUAL || jtCtx.asofOpType == OP_TYPE_GREATER_THAN)) || (!jtCtx.asc && (jtCtx.asofOpType == OP_TYPE_LOWER_EQUAL || jtCtx.asofOpType == OP_TYPE_LOWER_THAN))) || jtCtx.asofOpType == OP_TYPE_EQUAL) { - trimForAsofJlimit(); - } -} - -void asofJoinAppendEqGrpRes(int32_t leftGrpRows, int32_t rightGrpRows) { - int64_t rightTbOffset = jtCtx.blkRowSize * leftGrpRows; - - if (jtCtx.asc) { - switch (jtCtx.asofOpType) { - case OP_TYPE_GREATER_THAN: - addAsofEqInRows(leftGrpRows, 0, true); - appendAllAsofResRows(); - addAsofEqInRows(rightGrpRows, rightTbOffset, false); - break; - case OP_TYPE_GREATER_EQUAL: - addAsofEqInRows(leftGrpRows, 0, true); - addAsofEqInRows(rightGrpRows, rightTbOffset, false); - appendAllAsofResRows(); - break; - case OP_TYPE_LOWER_THAN: - case OP_TYPE_LOWER_EQUAL: - addAsofEqInRows(leftGrpRows, 0, true); - addAsofEqInRows(rightGrpRows, rightTbOffset, false); - chkAppendAsofForwardGrpResRows(false); - break; - case OP_TYPE_EQUAL: - taosArrayClear(jtCtx.leftRowsList); - taosArrayClear(jtCtx.rightRowsList); - taosArrayClear(jtCtx.rightFilterOut); - addAsofEqInRows(leftGrpRows, 0, true); - addAsofEqInRows(rightGrpRows, rightTbOffset, false); - chkAppendAsofForwardGrpResRows(true); - taosArrayClear(jtCtx.leftRowsList); - taosArrayClear(jtCtx.rightRowsList); - taosArrayClear(jtCtx.rightFilterOut); - break; - default: - return; - } - - return; - } - - switch (jtCtx.asofOpType) { - case OP_TYPE_LOWER_THAN: - addAsofEqInRows(leftGrpRows, 0, true); - appendAllAsofResRows(); - addAsofEqInRows(rightGrpRows, rightTbOffset, false); - break; - case OP_TYPE_LOWER_EQUAL: - addAsofEqInRows(leftGrpRows, 0, true); - addAsofEqInRows(rightGrpRows, rightTbOffset, false); - appendAllAsofResRows(); - break; - case OP_TYPE_GREATER_THAN: - case OP_TYPE_GREATER_EQUAL: - addAsofEqInRows(leftGrpRows, 0, true); - addAsofEqInRows(rightGrpRows, rightTbOffset, false); - chkAppendAsofForwardGrpResRows(false); - break; - case OP_TYPE_EQUAL: - taosArrayClear(jtCtx.leftRowsList); - taosArrayClear(jtCtx.rightRowsList); - taosArrayClear(jtCtx.rightFilterOut); - addAsofEqInRows(leftGrpRows, 0, true); - addAsofEqInRows(rightGrpRows, rightTbOffset, false); - chkAppendAsofForwardGrpResRows(true); - taosArrayClear(jtCtx.leftRowsList); - taosArrayClear(jtCtx.rightRowsList); - taosArrayClear(jtCtx.rightFilterOut); - break; +SPhysiNode* qptCreatePhysicalPlanNode(int32_t nodeType) { + switch (nodeType) { + case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: + return qptCreateTagScanPhysiNode(nodeType); + case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_PROJECT: + case QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN: + return qptCreateSortMergeJoinPhysiNode(nodeType); + case QUERY_NODE_PHYSICAL_PLAN_HASH_AGG: + case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE: + case QUERY_NODE_PHYSICAL_PLAN_MERGE: + case QUERY_NODE_PHYSICAL_PLAN_SORT: + case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT: + case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_FILL: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL: + case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION: + case QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE: + case QUERY_NODE_PHYSICAL_PLAN_PARTITION: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION: + case QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC: + case QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC: + case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: + case QUERY_NODE_PHYSICAL_PLAN_INSERT: + case QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT: + case QUERY_NODE_PHYSICAL_PLAN_DELETE: + case QUERY_NODE_PHYSICAL_SUBPLAN: + case QUERY_NODE_PHYSICAL_PLAN: + case QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT: + case QUERY_NODE_PHYSICAL_PLAN_HASH_JOIN: + case QUERY_NODE_PHYSICAL_PLAN_GROUP_CACHE: + case QUERY_NODE_PHYSICAL_PLAN_DYN_QUERY_CTRL: + case QUERY_NODE_PHYSICAL_PLAN_MERGE_COUNT: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL: default: - return; + assert(0); } + + return 0; +} + +int32_t qptCreateQueryPlan(SNode** ppPlan) { + } -void addWinEqInRows(int32_t rowsNum, int64_t tbOffset, bool leftTable) { - bool filterOut = false; - void* cvalue = NULL; - int64_t cbig = 0, fbig = 0; - int32_t filterNum = leftTable ? jtCtx.leftFilterNum : jtCtx.rightFilterNum; - int32_t* filterCol = leftTable ? jtCtx.leftFilterColList : jtCtx.rightFilterColList; - SArray* rowList = leftTable ? jtCtx.leftRowsList : jtCtx.rightRowsList; - - for (int32_t l = 0; l < rowsNum; ++l) { - char* row = jtCtx.colRowDataBuf + tbOffset + jtCtx.blkRowSize * l; - - filterOut = false; - - for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { - cvalue = row + jtCtx.colRowOffset[c]; - switch (jtInputColType[c]) { - case TSDB_DATA_TYPE_TIMESTAMP: - fbig = TIMESTAMP_FILTER_VALUE; - cbig = *(int64_t*)cvalue; - break; - case TSDB_DATA_TYPE_INT: - fbig = INT_FILTER_VALUE; - cbig = *(int32_t*)cvalue; - break; - case TSDB_DATA_TYPE_BIGINT: - fbig = BIGINT_FILTER_VALUE; - cbig = *(int64_t*)cvalue; - break; - default: - break; - } - - if (filterNum && filterCol[c] && ((*(bool*)(row + c)) || cbig <= fbig)) { - filterOut = true; - break; - } - } - - if (filterOut && leftTable) { - continue; - } - - assert(NULL != taosArrayPush(rowList, row)); - if (!leftTable) { - assert(NULL != taosArrayPush(jtCtx.rightFilterOut, &filterOut)); - } - } -} - - -void winJoinAppendEqGrpRes(int32_t leftGrpRows, int32_t rightGrpRows) { - int64_t rightTbOffset = jtCtx.blkRowSize * leftGrpRows; - - addWinEqInRows(leftGrpRows, 0, true); - addWinEqInRows(rightGrpRows, rightTbOffset, false); - chkAppendWinResRows(false); -} - - - -void fullJoinAppendEqGrpRes(int32_t leftGrpRows, int32_t rightGrpRows) { - bool leftMatch = false, rightMatch = false, lfilterOut = false, rfilterOut = false; - void* lValue = NULL, *rValue = NULL, *filterValue = NULL; - int64_t lBig = 0, rBig = 0, fbig = 0; - int64_t rightTbOffset = jtCtx.blkRowSize * leftGrpRows; - - TAOS_MEMSET(jtCtx.rightFinMatch, 0, rightGrpRows * sizeof(bool)); - - for (int32_t l = 0; l < leftGrpRows; ++l) { - char* lrow = jtCtx.colRowDataBuf + jtCtx.blkRowSize * l; - - lfilterOut = false; - leftMatch = false; - - for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { - lValue = lrow + jtCtx.colRowOffset[c]; - switch (jtInputColType[c]) { - case TSDB_DATA_TYPE_TIMESTAMP: - fbig = TIMESTAMP_FILTER_VALUE; - lBig = *(int64_t*)lValue; - break; - case TSDB_DATA_TYPE_INT: - fbig = INT_FILTER_VALUE; - lBig = *(int32_t*)lValue; - break; - case TSDB_DATA_TYPE_BIGINT: - fbig = BIGINT_FILTER_VALUE; - lBig = *(int64_t*)lValue; - break; - default: - break; - } - - if (jtCtx.leftFilterNum && jtCtx.leftFilterColList[c] && ((*(bool*)(lrow + c)) || lBig <= fbig)) { - lfilterOut = true; - } - } - - for (int32_t r = 0; r < rightGrpRows; ++r) { - char* rrow = jtCtx.colRowDataBuf + rightTbOffset + jtCtx.blkRowSize * r; - rightMatch = true; - rfilterOut = false; - - for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { - lValue = lrow + jtCtx.colRowOffset[c]; - - if (!*(bool*)(rrow + c)) { - rValue = rrow + jtCtx.colRowOffset[c]; - } - - switch (jtInputColType[c]) { - case TSDB_DATA_TYPE_TIMESTAMP: - fbig = TIMESTAMP_FILTER_VALUE; - lBig = *(int64_t*)lValue; - rBig = *(int64_t*)rValue; - break; - case TSDB_DATA_TYPE_INT: - fbig = INT_FILTER_VALUE; - lBig = *(int32_t*)lValue; - rBig = *(int32_t*)rValue; - break; - case TSDB_DATA_TYPE_BIGINT: - fbig = BIGINT_FILTER_VALUE; - lBig = *(int64_t*)lValue; - rBig = *(int64_t*)rValue; - break; - default: - break; - } - - if (jtCtx.colEqNum && jtCtx.colEqList[c] && ((*(bool*)(lrow + c)) || (*(bool*)(rrow + c)) || lBig != rBig)) { - rightMatch = false; - } - - if (jtCtx.colOnNum && jtCtx.colOnList[c] && ((*(bool*)(lrow + c)) || (*(bool*)(rrow + c)) || lBig <= rBig)) { - rightMatch = false; - } - - if (jtCtx.rightFilterNum && jtCtx.rightFilterColList[c] && ((*(bool*)(rrow + c)) || rBig <= fbig)) { - rfilterOut = true; - } - } - - if (rightMatch) { - jtCtx.rightFinMatch[r] = true; - } - - if (rfilterOut) { - if (!rightMatch) { - jtCtx.rightFinMatch[r] = true; - } - continue; - } - - if (!lfilterOut && rightMatch) { - putMatchRowToRes(lrow, rrow, ALL_TABLE_COLS); - leftMatch= true; - } - } - - if (!lfilterOut && !leftMatch && 0 == jtCtx.rightFilterNum) { - putNMatchRowToRes(lrow, 0, MAX_SLOT_NUM); - } - } - - if (0 == jtCtx.leftFilterNum) { - for (int32_t r = 0; r < rightGrpRows; ++r) { - if (!jtCtx.rightFinMatch[r]) { - char* rrow = jtCtx.colRowDataBuf + rightTbOffset + jtCtx.blkRowSize * r; - putNMatchRowToRes(rrow, MAX_SLOT_NUM, 0); - } - } - } -} - - -void appendEqGrpRes(int32_t leftGrpRows, int32_t rightGrpRows) { - switch (jtCtx.joinType) { - case JOIN_TYPE_INNER: - innerJoinAppendEqGrpRes(leftGrpRows, rightGrpRows); - break; - case JOIN_TYPE_LEFT: { - switch (jtCtx.subType) { - case JOIN_STYPE_OUTER: - leftJoinAppendEqGrpRes(leftGrpRows, rightGrpRows); - break; - case JOIN_STYPE_SEMI: - semiJoinAppendEqGrpRes(leftGrpRows, rightGrpRows); - break; - case JOIN_STYPE_ANTI: - antiJoinAppendEqGrpRes(leftGrpRows, rightGrpRows); - break; - case JOIN_STYPE_ASOF: - asofJoinAppendEqGrpRes(leftGrpRows, rightGrpRows); - break; - case JOIN_STYPE_WIN: - winJoinAppendEqGrpRes(leftGrpRows, rightGrpRows); - break; - default: - break; - } - break; - } - case JOIN_TYPE_FULL: - fullJoinAppendEqGrpRes(leftGrpRows, rightGrpRows); - break; - default: - break; - } -} - -void createTsEqGrpRows(SSDataBlock** ppLeft, SSDataBlock** ppRight, int32_t leftGrpRows, int32_t rightGrpRows) { - if (leftGrpRows <= 0 && rightGrpRows <= 0) { - return; - } - - if (leftGrpRows > 0 && rightGrpRows > 0) { - jtCtx.inputStat |= (1 << 2); - } - - jtCtx.asc ? ++jtCtx.curTs : --jtCtx.curTs; - - if (NULL == *ppLeft && leftGrpRows > 0) { - *ppLeft = createDummyBlock(LEFT_BLK_ID); - assert(*ppLeft); - assert(0 == blockDataEnsureCapacity(*ppLeft, jtCtx.blkRows)); - assert(NULL != taosArrayPush(jtCtx.leftBlkList, ppLeft)); - } - - if (jtCtx.grpJoin) { - (*ppLeft)->info.id.groupId = jtCtx.inGrpId; - } - - if (NULL == *ppRight && rightGrpRows > 0) { - *ppRight = createDummyBlock(RIGHT_BLK_ID); - assert(*ppRight); - assert(0 == blockDataEnsureCapacity(*ppRight, jtCtx.blkRows)); - assert(NULL != taosArrayPush(jtCtx.rightBlkList, ppRight)); - } - - if (jtCtx.grpJoin) { - (*ppRight)->info.id.groupId = jtCtx.inGrpId; - } - - - makeAppendBlkData(ppLeft, ppRight, leftGrpRows, rightGrpRows); - - appendEqGrpRes(leftGrpRows, rightGrpRows); -} - -void forceFlushResRows() { - if (JOIN_STYPE_ASOF == jtCtx.subType && taosArrayGetSize(jtCtx.leftRowsList) > 0) { - assert((jtCtx.asc && (OP_TYPE_LOWER_EQUAL == jtCtx.asofOpType || OP_TYPE_LOWER_THAN == jtCtx.asofOpType)) - || (!jtCtx.asc && (OP_TYPE_GREATER_EQUAL == jtCtx.asofOpType || OP_TYPE_GREATER_THAN == jtCtx.asofOpType))); - chkAppendAsofForwardGrpResRows(true); - } else if (JOIN_STYPE_WIN == jtCtx.subType && taosArrayGetSize(jtCtx.leftRowsList) > 0) { - chkAppendWinResRows(true); - } - - taosArrayClear(jtCtx.rightRowsList); - taosArrayClear(jtCtx.rightFilterOut); - taosArrayClear(jtCtx.leftRowsList); - -} - -void createBothBlkRowsData(void) { - SSDataBlock* pLeft = NULL; - SSDataBlock* pRight = NULL; - - jtCtx.leftTotalRows = taosRand() % jtCtx.leftMaxRows; - jtCtx.rightTotalRows = taosRand() % jtCtx.rightMaxRows; - - int32_t minTotalRows = TMIN(jtCtx.leftTotalRows, jtCtx.rightTotalRows); - int32_t maxTotalRows = TMAX(jtCtx.leftTotalRows, jtCtx.rightTotalRows); - jtCtx.curTs = jtCtx.asc ? (TIMESTAMP_FILTER_VALUE - minTotalRows / 5) : (TIMESTAMP_FILTER_VALUE + 4 * maxTotalRows / 5); - - int32_t leftTotalRows = 0, rightTotalRows = 0; - int32_t leftGrpRows = 0, rightGrpRows = 0; - int32_t grpType = 0; - while (leftTotalRows < jtCtx.leftTotalRows || rightTotalRows < jtCtx.rightTotalRows) { - if (leftTotalRows >= jtCtx.leftTotalRows) { - grpType = 1; - } else if (rightTotalRows >= jtCtx.rightTotalRows) { - grpType = 0; - } else { - grpType = taosRand() % 10; - } - - leftGrpRows = taosRand() % jtCtx.leftMaxGrpRows; - rightGrpRows = taosRand() % jtCtx.rightMaxGrpRows; - - if ((leftTotalRows + leftGrpRows) > jtCtx.leftTotalRows) { - leftGrpRows = jtCtx.leftTotalRows - leftTotalRows; - } - - if ((rightTotalRows + rightGrpRows) > jtCtx.rightTotalRows) { - rightGrpRows = jtCtx.rightTotalRows - rightTotalRows; - } - - if (0 != grpType && 1 != grpType && (leftGrpRows <= 0 || rightGrpRows <= 0)) { - if (leftGrpRows <= 0) { - grpType = 1; - } else { - grpType = 0; - } - } - - if (jtCtx.grpJoin && (0 == taosRand() % 3)) { - forceFlushResRows(); - jtCtx.inGrpId++; - pLeft = NULL; - pRight = NULL; - } - - switch (grpType) { - case 0: - createGrpRows(&pLeft, LEFT_BLK_ID, leftGrpRows); - leftTotalRows += leftGrpRows; - break; - case 1: - createGrpRows(&pRight, RIGHT_BLK_ID, rightGrpRows); - rightTotalRows += rightGrpRows; - break; - default: - createTsEqGrpRows(&pLeft, &pRight, leftGrpRows, rightGrpRows); - leftTotalRows += leftGrpRows; - rightTotalRows += rightGrpRows; - break; - } - } - - forceFlushResRows(); -} - -void createDummyBlkList(int32_t leftMaxRows, int32_t leftMaxGrpRows, int32_t rightMaxRows, int32_t rightMaxGrpRows, int32_t blkRows) { - jtCtx.leftMaxRows = leftMaxRows; - jtCtx.leftMaxGrpRows = leftMaxGrpRows; - jtCtx.rightMaxRows = rightMaxRows; - jtCtx.rightMaxGrpRows = rightMaxGrpRows; - jtCtx.blkRows = blkRows; - - int32_t maxGrpRows = TMAX(leftMaxGrpRows, rightMaxGrpRows); - if (maxGrpRows > jtCtx.rightFinMatchNum) { - jtCtx.rightFinMatchNum = maxGrpRows; - jtCtx.rightFinMatch = (bool*)taosMemoryRealloc(jtCtx.rightFinMatch, maxGrpRows * sizeof(bool)); - assert(jtCtx.rightFinMatch); - } - - taosArrayClear(jtCtx.leftRowsList); - taosArrayClear(jtCtx.rightRowsList); - taosArrayClear(jtCtx.rightFilterOut); - - createBothBlkRowsData(); -} - -void rerunBlockedHere() { - while (jtInRerun) { +void qptRerunBlockedHere() { + while (qptInRerun) { taosSsleep(1); } } +void qptResetForReRun() { -SSDataBlock* getDummyInputBlock(struct SOperatorInfo* pOperator, int32_t idx) { - switch (idx) { - case LEFT_BLK_ID: - if (jtCtx.leftBlkReadIdx >= taosArrayGetSize(jtCtx.leftBlkList)) { - return NULL; - } - return (SSDataBlock*)taosArrayGetP(jtCtx.leftBlkList, jtCtx.leftBlkReadIdx++); - break; - case RIGHT_BLK_ID: - if (jtCtx.rightBlkReadIdx >= taosArrayGetSize(jtCtx.rightBlkList)) { - return NULL; - } - return (SSDataBlock*)taosArrayGetP(jtCtx.rightBlkList, jtCtx.rightBlkReadIdx++); - break; - default: - return NULL; - } } - -void joinTestReplaceRetrieveFp() { - static Stub stub; - stub.set(getNextBlockFromDownstreamRemain, getDummyInputBlock); - { -#ifdef WINDOWS - AddrAny any; - std::map result; - any.get_func_addr("getNextBlockFromDownstreamRemain", result); - for (const auto &f : result) { - stub.set(f.second, getDummyInputBlock); - } -#endif -#ifdef LINUX - AddrAny any("libexecutor.so"); - std::map result; - any.get_global_func_addr_dynsym("^getNextBlockFromDownstreamRemain$", result); - for (const auto &f : result) { - stub.set(f.second, getDummyInputBlock); - } -#endif - } -} - -void printColList(char* title, bool left, int32_t* colList, bool filter, char* opStr) { - bool first = true; - - JT_PRINTF("\t %s:", title); - for (int32_t i = 0; i < MAX_SLOT_NUM; ++i) { - if (colList[i]) { - if (!first) { - JT_PRINTF(" AND "); - } - first = false; - if (filter) { - JT_PRINTF("%sc%d%s%" PRId64 , left ? "l" : "r", i, opStr, jtFilterValue[i]); - } else { - JT_PRINTF("lc%d%src%d", i, opStr, i); - } - } - } - JT_PRINTF("\n"); -} - -void printInputRowData(SSDataBlock* pBlk, int32_t* rowIdx) { - if (jtCtx.grpJoin) { - JT_PRINTF("%5" PRIu64, pBlk->info.id.groupId); - } - for (int32_t c = 0; c < MAX_SLOT_NUM; ++c) { - SColumnInfoData* pCol = (SColumnInfoData*)taosArrayGet(pBlk->pDataBlock, c); - assert(pCol); - assert(pCol->info.type == jtInputColType[c]); - if (colDataIsNull_s(pCol, *rowIdx)) { - JT_PRINTF("%18s", " NULL"); - } else { - switch (jtInputColType[c]) { - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_BIGINT: - JT_PRINTF("%18" PRId64, *(int64_t*)colDataGetData(pCol, *rowIdx)); - break; - case TSDB_DATA_TYPE_INT: - JT_PRINTF("%18d", *(int32_t*)colDataGetData(pCol, *rowIdx)); - break; - default: - assert(0); - } - } - } - - (*rowIdx)++; -} - -void printInputData() { - int32_t leftRowIdx = 0, rightRowIdx = 0; - - JT_PRINTF("\nInput Data:\n"); - while (jtCtx.leftBlkReadIdx < taosArrayGetSize(jtCtx.leftBlkList) || jtCtx.rightBlkReadIdx < taosArrayGetSize(jtCtx.rightBlkList)) { - if (jtCtx.leftBlkReadIdx < taosArrayGetSize(jtCtx.leftBlkList)) { - while (true) { - SSDataBlock* pBlk = (SSDataBlock*)taosArrayGetP(jtCtx.leftBlkList, jtCtx.leftBlkReadIdx); - assert(pBlk); - if (leftRowIdx < pBlk->info.rows) { - printInputRowData(pBlk, &leftRowIdx); - break; - } - - JT_PRINTF("\t%*s-------------------------blk end-------------------------------", jtCtx.grpJoin ? 6 : 0, " "); - jtCtx.leftBlkReadIdx++; - leftRowIdx = 0; - break; - } - } else { - JT_PRINTF("%*s", jtCtx.grpJoin ? 77 : 72, " "); - } - - if (jtCtx.rightBlkReadIdx < taosArrayGetSize(jtCtx.rightBlkList)) { - while (true) { - SSDataBlock* pBlk = (SSDataBlock*)taosArrayGetP(jtCtx.rightBlkList, jtCtx.rightBlkReadIdx); - assert(pBlk); - if (rightRowIdx < pBlk->info.rows) { - printInputRowData(pBlk, &rightRowIdx); - break; - } - - JT_PRINTF("\t%*s--------------------------blk end----------------------------\t", jtCtx.grpJoin ? 6 : 0, " "); - jtCtx.rightBlkReadIdx++; - rightRowIdx = 0; - break; - } - } - - JT_PRINTF("\n"); - } - - jtCtx.leftBlkReadIdx = jtCtx.rightBlkReadIdx = 0; -} - -char* getInputStatStr(char* inputStat) { - if (jtCtx.inputStat & (1 << LEFT_BLK_ID)) { - TAOS_STRCAT(inputStat, "L"); - } - if (jtCtx.inputStat & (1 << RIGHT_BLK_ID)) { - TAOS_STRCAT(inputStat, "R"); - } - if (jtCtx.inputStat & (1 << 2)) { - TAOS_STRCAT(inputStat, "E"); - } - return inputStat; -} - -char* getAsofOpStr() { - switch (jtCtx.asofOpType) { - case OP_TYPE_GREATER_THAN: - return ">"; - case OP_TYPE_GREATER_EQUAL: - return ">="; - case OP_TYPE_LOWER_THAN: - return "<"; - case OP_TYPE_LOWER_EQUAL: - return "<="; - case OP_TYPE_EQUAL: - return "="; - default: - return "UNKNOWN"; - } -} - -void printBasicInfo(char* caseName) { - if (!jtCtrl.printTestInfo) { - return; - } - - char inputStat[4] = {0}; - JT_PRINTF("\n%dth TEST [%s] START\nBasic Info:\n\t asc:%d\n\t filter:%d\n\t maxRows:left-%d right-%d\n\t " - "maxGrpRows:left-%d right-%d\n\t blkRows:%d\n\t colCond:%s\n\t joinType:%s\n\t " - "subType:%s\n\t inputStat:%s\n\t groupJoin:%s\n", jtCtx.loopIdx, caseName, jtCtx.asc, jtCtx.filter, jtCtx.leftMaxRows, jtCtx.rightMaxRows, - jtCtx.leftMaxGrpRows, jtCtx.rightMaxGrpRows, jtCtx.blkRows, jtColCondStr[jtCtx.colCond], jtJoinTypeStr[jtCtx.joinType], - jtSubTypeStr[jtCtx.subType], getInputStatStr(inputStat), jtCtx.grpJoin ? "true" : "false"); - - if (JOIN_STYPE_ASOF == jtCtx.subType) { - JT_PRINTF("\t asofOp:%s\n\t JLimit:%" PRId64 "\n", getAsofOpStr(), jtCtx.jLimit); - } else if (JOIN_STYPE_WIN == jtCtx.subType) { - JT_PRINTF("\t windowOffset:[%" PRId64 ", %" PRId64 "]\n\t JLimit:%" PRId64 "\n", jtCtx.winStartOffset, jtCtx.winEndOffset, jtCtx.jLimit); - } - - JT_PRINTF("Input Info:\n\t totalBlk:left-%d right-%d\n\t totalRows:left-%d right-%d\n\t " - "blkRowSize:%d\n\t inputCols:left-%s %s %s %s right-%s %s %s %s\n", - (int32_t)taosArrayGetSize(jtCtx.leftBlkList), (int32_t)taosArrayGetSize(jtCtx.rightBlkList), - jtCtx.leftTotalRows, jtCtx.rightTotalRows, - jtCtx.blkRowSize, tDataTypes[jtInputColType[0]].name, tDataTypes[jtInputColType[1]].name, - tDataTypes[jtInputColType[2]].name, tDataTypes[jtInputColType[3]].name, tDataTypes[jtInputColType[0]].name, - tDataTypes[jtInputColType[1]].name, tDataTypes[jtInputColType[2]].name, tDataTypes[jtInputColType[3]].name); - - if (jtCtx.colEqNum) { - JT_PRINTF("\t colEqNum:%d\n", jtCtx.colEqNum); - printColList("colEqList", false, jtCtx.colEqList, false, "="); - } - - if (jtCtx.colOnNum) { - JT_PRINTF("\t colOnNum:%d\n", jtCtx.colOnNum); - printColList("colOnList", false, jtCtx.colOnList, false, ">"); - } - - if (jtCtx.leftFilterNum) { - JT_PRINTF("\t leftFilterNum:%d\n", jtCtx.leftFilterNum); - printColList("leftFilterList", true, jtCtx.leftFilterColList, true, ">"); - } - - if (jtCtx.rightFilterNum) { - JT_PRINTF("\t rightFilterNum:%d\n", jtCtx.rightFilterNum); - printColList("rightFilterList", false, jtCtx.rightFilterColList, true, ">"); - } - - JT_PRINTF("\t resColSize:%d\n\t resColNum:%d\n\t resColList:", jtCtx.resColSize, jtCtx.resColNum); - for (int32_t i = 0; i < jtCtx.resColNum; ++i) { - int32_t s = jtCtx.resColInSlot[i]; - int32_t idx = s >= MAX_SLOT_NUM ? s - MAX_SLOT_NUM : s; - JT_PRINTF("%sc%d[%s]\t", s >= MAX_SLOT_NUM ? "r" : "l", s, tDataTypes[jtInputColType[idx]].name); - } - - if (jtCtrl.printInputRow) { - printInputData(); - } -} - -void printOutputInfo() { - if (!jtCtrl.printTestInfo) { - return; - } - - JT_PRINTF("\nOutput Info:\n\t expectedRows:%d\n\t ", jtCtx.resRows); - JT_PRINTF("Actual Result:\n"); -} - -void printActualResInfo() { - if (!jtCtrl.printTestInfo) { - return; - } - - JT_PRINTF("Actual Result Summary:\n\t blkNum:%d\n\t rowNum:%d%s\n\t leftBlkRead:%d\n\t rightBlkRead:%d\n\t +rows:%d%s\n\t " - "-rows:%d%s\n\t matchRows:%d%s\n\t executionTime:%" PRId64 "us\n", - jtRes.blkNum, jtRes.rowNum, - jtRes.rowNum == jtCtx.resRows ? "" : "*", - jtCtx.leftBlkReadIdx, jtCtx.rightBlkReadIdx, - jtRes.addRowNum, jtRes.addRowNum ? "*" : "", - jtRes.subRowNum, jtRes.subRowNum ? "*" : "", - jtRes.matchNum, jtRes.matchNum == jtCtx.resRows ? "" : "*", - taosGetTimestampUs() - jtCtx.startTsUs); -} - -void printStatInfo(char* caseName) { - JT_PRINTF("\n TEST [%s] Stat:\n\t maxResRows:%d\n\t maxResBlkRows:%d\n\t totalResRows:%" PRId64 "\n\t useMSecs:%" PRId64 "\n", - caseName, jtStat.maxResRows, jtStat.maxResBlkRows, jtStat.totalResRows, jtStat.useMSecs); - -} - -void checkJoinDone(char* caseName) { - int32_t iter = 0; - void* p = NULL; - void* key = NULL; - if (!jtCtrl.noKeepResRows) { - while (NULL != (p = tSimpleHashIterate(jtCtx.jtResRows, p, &iter))) { - key = tSimpleHashGetKey(p, NULL); - jtRes.succeed = false; - jtRes.subRowNum += *(int32_t*)p; - for (int32_t i = 0; i < *(int32_t*)p; ++i) { - printResRow((char*)key, 0); - } - } - } - - printActualResInfo(); - - JT_PRINTF("\n%dth TEST [%s] Final Result: %s\n", jtCtx.loopIdx, caseName, jtRes.succeed ? "SUCCEED" : "FAILED"); -} - -void putRowToResColBuf(SSDataBlock* pBlock, int32_t r, bool ignoreTbCols) { - for (int32_t c = 0; c < jtCtx.resColNum; ++c) { - int32_t slot = jtCtx.resColInSlot[c]; - if (ignoreTbCols && ((jtCtx.leftColOnly && slot >= MAX_SLOT_NUM) || - (jtCtx.rightColOnly && slot < MAX_SLOT_NUM))) { - continue; - } - - SColumnInfoData* pCol = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, c); - assert(pCol); - switch (jtInputColType[slot % MAX_SLOT_NUM]) { - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_BIGINT: - if (colDataIsNull_s(pCol, r)) { - *(bool*)(jtCtx.resColBuf + slot) = true; - } else { - *(int64_t*)(jtCtx.resColBuf + jtCtx.resColOffset[slot]) = *(int64_t*)colDataGetData(pCol, r); - } - break; - case TSDB_DATA_TYPE_INT: - if (colDataIsNull_s(pCol, r)) { - *(bool*)(jtCtx.resColBuf + slot) = true; - } else { - *(int32_t*)(jtCtx.resColBuf + jtCtx.resColOffset[slot]) = *(int32_t*)colDataGetData(pCol, r); - } - break; - default: - break; - } - } -} - -void checkJoinRes(SSDataBlock* pBlock) { - jtRes.rowNum += pBlock->info.rows; - if (jtRes.rowNum > jtStat.maxResRows) { - jtStat.maxResRows = jtRes.rowNum; - } - jtRes.blkNum++; - - if (pBlock->info.rows > jtStat.maxResBlkRows) { - jtStat.maxResBlkRows = pBlock->info.rows; - } - - jtStat.totalResRows += pBlock->info.rows; - if (jtCtrl.noKeepResRows) { - jtRes.matchNum += pBlock->info.rows; - } else { - for (int32_t r = 0; r < pBlock->info.rows; ++r) { - TAOS_MEMSET(jtCtx.resColBuf, 0, jtCtx.resColSize); - - putRowToResColBuf(pBlock, r, true); - - char* value = (char*)tSimpleHashGet(jtCtx.jtResRows, jtCtx.resColBuf, jtCtx.resColSize); - if (NULL == value) { - putRowToResColBuf(pBlock, r, false); - printResRow(jtCtx.resColBuf, 1); - jtRes.succeed = false; - jtRes.addRowNum++; - continue; - } - - rmResRow(); - - putRowToResColBuf(pBlock, r, false); - printResRow(jtCtx.resColBuf, 2); - jtRes.matchNum++; - } - } -} - -void resetForJoinRerun(int32_t dsNum, SSortMergeJoinPhysiNode* pNode, SExecTaskInfo* pTask) { - jtCtx.leftBlkReadIdx = 0; - jtCtx.rightBlkReadIdx = 0; - jtCtx.curKeyOffset = 0; - - TAOS_MEMSET(&jtRes, 0, sizeof(jtRes)); - jtRes.succeed = true; - - SOperatorInfo* pDownstreams[2]; - createDummyDownstreamOperators(2, pDownstreams); - SOperatorInfo* ppDownstreams[] = {pDownstreams[0], pDownstreams[1]}; - int32_t code = createMergeJoinOperatorInfo(ppDownstreams, 2, pNode, pTask, &jtCtx.pJoinOp); - ASSERT_TRUE(NULL != jtCtx.pJoinOp); -} - -void handleJoinDone(bool* contLoop) { +void qptSingleTestDone(bool* contLoop) { destroyOperator(jtCtx.pJoinOp); jtCtx.pJoinOp = NULL; @@ -2872,12 +1004,12 @@ void handleJoinDone(bool* contLoop) { return; } - jtInRerun = true; + qptInRerun = true; } -void jtInitLogFile() { - const char *defaultLogFileNamePrefix = "jtlog"; +void qptInitLogFile() { + const char *defaultLogFileNamePrefix = "queryPlanTestlog"; const int32_t maxLogFileNum = 10; tsAsyncLog = 0; @@ -2885,881 +1017,94 @@ void jtInitLogFile() { TAOS_STRCPY(tsLogDir, TD_LOG_DIR_PATH); if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) { - JT_PRINTF("failed to open log file in directory:%s\n", tsLogDir); + printf("failed to open log file in directory:%s\n", tsLogDir); } } -void initJoinTest() { - jtCtx.leftBlkList = taosArrayInit(10, POINTER_BYTES); - jtCtx.rightBlkList = taosArrayInit(10, POINTER_BYTES); - assert(jtCtx.leftBlkList && jtCtx.rightBlkList); - jtCtx.jtResRows = tSimpleHashInit(10000000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY)); - assert(jtCtx.jtResRows); - - joinTestReplaceRetrieveFp(); - - if (jtCtrl.logHistory) { - jtStat.pHistory = taosArrayInit(100000, sizeof(SJoinTestHistory)); - assert(jtStat.pHistory); - } - - int32_t offset = MAX_SLOT_NUM * sizeof(bool); - for (int32_t i = 0; i < MAX_SLOT_NUM; ++i) { - jtCtx.inColOffset[i] = offset; - offset += tDataTypes[jtInputColType[i]].bytes; - } - jtCtx.inColSize = offset; - jtCtx.inColBuf = (char*)taosMemoryMalloc(jtCtx.inColSize); - assert(jtCtx.inColBuf); - - jtCtx.leftRowsList = taosArrayInit(1024, jtCtx.inColSize); - jtCtx.rightRowsList = taosArrayInit(1024, jtCtx.inColSize); - jtCtx.rightFilterOut = taosArrayInit(1024, sizeof(bool)); - assert(jtCtx.leftRowsList && jtCtx.rightRowsList && jtCtx.rightFilterOut); - - jtInitLogFile(); +void qptInitTest() { + qptInitLogFile(); } -void handleTestDone() { - if (jtCtrl.logHistory) { - SJoinTestHistory h; - TAOS_MEMCPY(&h.ctx, &jtCtx, sizeof(h.ctx)); - TAOS_MEMCPY(&h.res, &jtRes, sizeof(h.res)); - assert(NULL != taosArrayPush(jtStat.pHistory, &h)); - } - - int32_t blkNum = taosArrayGetSize(jtCtx.leftBlkList); - for (int32_t i = 0; i < blkNum; ++i) { - SSDataBlock* pBlk = (SSDataBlock*)taosArrayGetP(jtCtx.leftBlkList, i); - assert(pBlk); - (void)blockDataDestroy(pBlk); - } - taosArrayClear(jtCtx.leftBlkList); +void qptHandleTestEnd() { - blkNum = taosArrayGetSize(jtCtx.rightBlkList); - for (int32_t i = 0; i < blkNum; ++i) { - SSDataBlock* pBlk = (SSDataBlock*)taosArrayGetP(jtCtx.rightBlkList, i); - assert(pBlk); - (void)blockDataDestroy(pBlk); - } - taosArrayClear(jtCtx.rightBlkList); - - tSimpleHashClear(jtCtx.jtResRows); - jtCtx.resRows = 0; - - jtCtx.inputStat = 0; } -void runSingleTest(char* caseName, SJoinTestParam* param) { - bool contLoop = true; - - SSortMergeJoinPhysiNode* pNode = createDummySortMergeJoinPhysiNode(param); - assert(pNode); - createDummyBlkList(1000, 1000, 1000, 1000, 100); - - while (contLoop) { - rerunBlockedHere(); - resetForJoinRerun(2, pNode, param->pTask); - printBasicInfo(caseName); - printOutputInfo(); +void qptRunPlanTest(char* caseName) { + SNode* pNode = NULL; + SReadHandle readHandle = {0}; + SOperatorInfo* pOperator = NULL; - jtCtx.startTsUs = taosGetTimestampUs(); - while (true) { - SSDataBlock* pBlock = NULL; - int32_t code = jtCtx.pJoinOp->fpSet.getNextFn(jtCtx.pJoinOp, &pBlock); - if (NULL == pBlock) { - checkJoinDone(caseName); - break; - } else { - checkJoinRes(pBlock); - } - } - - handleJoinDone(&contLoop); + if (qptCtx->param.plan.singlePhysiNode) { + pNode = (SNode*)qptCreatePhysicalPlanNode(qptCtx->param.plan.subplanType[0]); } + qptPrintBeginInfo(caseName); + + qptCtx.startTsUs = taosGetTimestampUs(); + int32_t code = createTagScanOperatorInfo(&readHandle, (STagScanPhysiNode*)pNode, NULL, NULL, NULL, NULL, &pOperator); + + destroyOperator(pOperator); nodesDestroyNode((SNode*)pNode); - handleTestDone(); + + qptHandleTestEnd(); } -void handleCaseEnd() { - taosMemoryFreeClear(jtCtx.rightFinMatch); - jtCtx.rightFinMatchNum = 0; + + +SQPTNodeParam* qptInitNodeParam(int32_t nodeType) { + } +void qptInitTestCtx(bool correctExpected, bool singleNode, int32_t nodeType, int32_t paramNum, SQPTNodeParam* nodeParam) { + qptCtx->param.correctExpected = correctExpected; + qptCtx->param.plan.singlePhysiNode = singleNode; + if (singleNode) { + qptCtx->param.plan.subplanMaxLevel = 1; + qptCtx->param.plan.subplanType[0] = nodeType; + } else { + qptCtx->param.plan.subplanMaxLevel = taosRand() % QPT_MAX_SUBPLAN_LEVEL + 1; + for (int32_t i = 0; i < qptCtx->param.plan.subplanMaxLevel; ++i) { + qptCtx->param.plan.subplanType[i] = QPT_PHYSIC_NODE_LIST[taosRand() % QPT_PHYSIC_NODE_NUM()]; + } + } + + if (paramNum > 0) { + qptCtx->param.plan.physiNodeParamNum = paramNum; + qptCtx->param.plan.physicNodeParam = nodeParam; + } +} + + } // namespace #if 1 #if 1 -TEST(innerJoin, noCondTest) { - SJoinTestParam param; - char* caseName = "innerJoin:noCondTest"; - SExecTaskInfo* pTask = createDummyTaskInfo(caseName); - assert(pTask); +TEST(correctSingleNodeTest, tagScan) { + char* caseName = "correctSingleNodeTest:tagScan"; - param.pTask = pTask; - param.joinType = JOIN_TYPE_INNER; - param.subType = JOIN_STYPE_NONE; - param.cond = TEST_NO_COND; - param.asc = true; - param.grpJoin = false; + qptInitTestCtx(true, true, QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN, 0, NULL); - for (jtCtx.loopIdx = 0; jtCtx.loopIdx < JT_MAX_LOOP; ++jtCtx.loopIdx) { - param.asc = !param.asc; - param.filter = false; - runSingleTest(caseName, ¶m); + for (qptCtx.loopIdx = 0; qptCtx.loopIdx < QPT_MAX_LOOP; ++qptCtx.loopIdx) { + qptRunPlanTest(caseName); + } - param.filter = true; - runSingleTest(caseName, ¶m); + qptPrintStatInfo(caseName); +} +#endif + +#if 0 +TEST(randSingleNodeTest, tagScan) { + char* caseName = "randSingleNodeTest:tagScan"; + + qptInitTestCtx(false, true, QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN, 0, NULL); + + for (qptCtx.loopIdx = 0; qptCtx.loopIdx < QPT_MAX_LOOP; ++qptCtx.loopIdx) { + qptRunSingleTest(caseName); } printStatInfo(caseName); - taosMemoryFree(pTask); -} -#endif - -#if 1 -TEST(innerJoin, eqCondTest) { - SJoinTestParam param; - char* caseName = "innerJoin:eqCondTest"; - SExecTaskInfo* pTask = createDummyTaskInfo(caseName); - assert(pTask); - - param.pTask = pTask; - param.joinType = JOIN_TYPE_INNER; - param.subType = JOIN_STYPE_NONE; - param.cond = TEST_EQ_COND; - param.asc = true; - param.grpJoin = false; - - for (jtCtx.loopIdx = 0; jtCtx.loopIdx < JT_MAX_LOOP; ++jtCtx.loopIdx) { - param.asc = !param.asc; - param.filter = false; - runSingleTest(caseName, ¶m); - - param.filter = true; - runSingleTest(caseName, ¶m); - } - - printStatInfo(caseName); - taosMemoryFree(pTask); -} -#endif - -#if 1 -TEST(innerJoin, onCondTest) { - SJoinTestParam param; - char* caseName = "innerJoin:onCondTest"; - SExecTaskInfo* pTask = createDummyTaskInfo(caseName); - assert(pTask); - - param.pTask = pTask; - param.joinType = JOIN_TYPE_INNER; - param.subType = JOIN_STYPE_NONE; - param.cond = TEST_ON_COND; - param.asc = true; - param.grpJoin = false; - - for (jtCtx.loopIdx = 0; jtCtx.loopIdx < JT_MAX_LOOP; ++jtCtx.loopIdx) { - param.asc = !param.asc; - param.filter = false; - runSingleTest(caseName, ¶m); - - param.filter = true; - runSingleTest(caseName, ¶m); - } - - printStatInfo(caseName); - taosMemoryFree(pTask); -} -#endif - -#if 1 -TEST(innerJoin, fullCondTest) { - SJoinTestParam param; - char* caseName = "innerJoin:fullCondTest"; - SExecTaskInfo* pTask = createDummyTaskInfo(caseName); - assert(pTask); - - param.pTask = pTask; - param.joinType = JOIN_TYPE_INNER; - param.subType = JOIN_STYPE_NONE; - param.cond = TEST_FULL_COND; - param.asc = true; - param.grpJoin = false; - - for (jtCtx.loopIdx = 0; jtCtx.loopIdx < JT_MAX_LOOP; ++jtCtx.loopIdx) { - param.asc = !param.asc; - param.filter = false; - runSingleTest(caseName, ¶m); - - param.filter = true; - runSingleTest(caseName, ¶m); - } - - printStatInfo(caseName); - taosMemoryFree(pTask); -} -#endif -#endif - - -#if 1 -#if 1 -TEST(leftOuterJoin, noCondTest) { - SJoinTestParam param; - char* caseName = "leftOuterJoin:noCondTest"; - SExecTaskInfo* pTask = createDummyTaskInfo(caseName); - assert(pTask); - - param.pTask = pTask; - param.joinType = JOIN_TYPE_LEFT; - param.subType = JOIN_STYPE_OUTER; - param.cond = TEST_NO_COND; - param.asc = true; - - for (jtCtx.loopIdx = 0; jtCtx.loopIdx < JT_MAX_LOOP; ++jtCtx.loopIdx) { - param.asc = !param.asc; - param.grpJoin = taosRand() % 2 ? true : false; - param.filter = false; - runSingleTest(caseName, ¶m); - - param.grpJoin = taosRand() % 2 ? true : false; - param.filter = true; - runSingleTest(caseName, ¶m); - } - - printStatInfo(caseName); - taosMemoryFree(pTask); -} -#endif - -#if 1 -TEST(leftOuterJoin, eqCondTest) { - SJoinTestParam param; - char* caseName = "leftOuterJoin:eqCondTest"; - SExecTaskInfo* pTask = createDummyTaskInfo(caseName); - assert(pTask); - - param.pTask = pTask; - param.joinType = JOIN_TYPE_LEFT; - param.subType = JOIN_STYPE_OUTER; - param.cond = TEST_EQ_COND; - param.asc = true; - param.grpJoin = false; - - for (jtCtx.loopIdx = 0; jtCtx.loopIdx < JT_MAX_LOOP; ++jtCtx.loopIdx) { - param.asc = !param.asc; - param.filter = false; - runSingleTest(caseName, ¶m); - - param.filter = true; - runSingleTest(caseName, ¶m); - } - - printStatInfo(caseName); - taosMemoryFree(pTask); -} -#endif - -#if 1 -TEST(leftOuterJoin, onCondTest) { - SJoinTestParam param; - char* caseName = "leftOuterJoin:onCondTest"; - SExecTaskInfo* pTask = createDummyTaskInfo(caseName); - assert(pTask); - - param.pTask = pTask; - param.joinType = JOIN_TYPE_LEFT; - param.subType = JOIN_STYPE_OUTER; - param.cond = TEST_ON_COND; - param.asc = true; - param.grpJoin = false; - - for (jtCtx.loopIdx = 0; jtCtx.loopIdx < JT_MAX_LOOP; ++jtCtx.loopIdx) { - param.asc = !param.asc; - param.filter = false; - runSingleTest(caseName, ¶m); - - param.filter = true; - runSingleTest(caseName, ¶m); - } - - printStatInfo(caseName); - taosMemoryFree(pTask); -} -#endif - -#if 1 -TEST(leftOuterJoin, fullCondTest) { - SJoinTestParam param; - char* caseName = "leftOuterJoin:fullCondTest"; - SExecTaskInfo* pTask = createDummyTaskInfo(caseName); - assert(pTask); - - param.pTask = pTask; - param.joinType = JOIN_TYPE_LEFT; - param.subType = JOIN_STYPE_OUTER; - param.cond = TEST_FULL_COND; - param.asc = true; - param.grpJoin = false; - - for (jtCtx.loopIdx = 0; jtCtx.loopIdx < JT_MAX_LOOP; ++jtCtx.loopIdx) { - param.asc = !param.asc; - param.filter = false; - runSingleTest(caseName, ¶m); - - param.filter = true; - runSingleTest(caseName, ¶m); - } - - printStatInfo(caseName); - taosMemoryFree(pTask); -} -#endif -#endif - -#if 1 -#if 1 -TEST(fullOuterJoin, noCondTest) { - SJoinTestParam param; - char* caseName = "fullOuterJoin:noCondTest"; - SExecTaskInfo* pTask = createDummyTaskInfo(caseName); - assert(pTask); - - param.pTask = pTask; - param.joinType = JOIN_TYPE_FULL; - param.subType = JOIN_STYPE_OUTER; - param.cond = TEST_NO_COND; - param.asc = true; - param.grpJoin = false; - - for (jtCtx.loopIdx = 0; jtCtx.loopIdx < JT_MAX_LOOP; ++jtCtx.loopIdx) { - param.asc = !param.asc; - param.filter = false; - runSingleTest(caseName, ¶m); - - param.filter = true; - runSingleTest(caseName, ¶m); - } - - printStatInfo(caseName); - taosMemoryFree(pTask); -} -#endif - -#if 1 -TEST(fullOuterJoin, eqCondTest) { - SJoinTestParam param; - char* caseName = "fullOuterJoin:eqCondTest"; - SExecTaskInfo* pTask = createDummyTaskInfo(caseName); - assert(pTask); - - param.pTask = pTask; - param.joinType = JOIN_TYPE_FULL; - param.subType = JOIN_STYPE_OUTER; - param.cond = TEST_EQ_COND; - param.asc = true; - param.grpJoin = false; - - for (jtCtx.loopIdx = 0; jtCtx.loopIdx < JT_MAX_LOOP; ++jtCtx.loopIdx) { - param.asc = !param.asc; - param.filter = false; - runSingleTest(caseName, ¶m); - - param.filter = true; - runSingleTest(caseName, ¶m); - } - - printStatInfo(caseName); - taosMemoryFree(pTask); - handleCaseEnd(); -} -#endif - -#if 1 -TEST(fullOuterJoin, onCondTest) { - SJoinTestParam param; - char* caseName = "fullOuterJoin:onCondTest"; - SExecTaskInfo* pTask = createDummyTaskInfo(caseName); - assert(pTask); - - param.pTask = pTask; - param.joinType = JOIN_TYPE_FULL; - param.subType = JOIN_STYPE_OUTER; - param.cond = TEST_ON_COND; - param.asc = true; - param.grpJoin = false; - - for (jtCtx.loopIdx = 0; jtCtx.loopIdx < JT_MAX_LOOP; ++jtCtx.loopIdx) { - param.asc = !param.asc; - param.filter = false; - runSingleTest(caseName, ¶m); - - param.filter = true; - runSingleTest(caseName, ¶m); - } - - printStatInfo(caseName); - taosMemoryFree(pTask); -} -#endif - -#if 1 -TEST(fullOuterJoin, fullCondTest) { - SJoinTestParam param; - char* caseName = "fullOuterJoin:fullCondTest"; - SExecTaskInfo* pTask = createDummyTaskInfo(caseName); - assert(pTask); - - param.pTask = pTask; - param.joinType = JOIN_TYPE_FULL; - param.subType = JOIN_STYPE_OUTER; - param.cond = TEST_FULL_COND; - param.asc = true; - param.grpJoin = false; - - for (jtCtx.loopIdx = 0; jtCtx.loopIdx < JT_MAX_LOOP; ++jtCtx.loopIdx) { - param.asc = !param.asc; - param.filter = false; - runSingleTest(caseName, ¶m); - - param.filter = true; - runSingleTest(caseName, ¶m); - } - - printStatInfo(caseName); - taosMemoryFree(pTask); -} -#endif -#endif - -#if 1 -#if 1 -TEST(leftSemiJoin, noCondTest) { - SJoinTestParam param; - char* caseName = "leftSemiJoin:noCondTest"; - SExecTaskInfo* pTask = createDummyTaskInfo(caseName); - assert(pTask); - - param.pTask = pTask; - param.joinType = JOIN_TYPE_LEFT; - param.subType = JOIN_STYPE_SEMI; - param.cond = TEST_NO_COND; - param.asc = true; - param.grpJoin = false; - - for (jtCtx.loopIdx = 0; jtCtx.loopIdx < JT_MAX_LOOP; ++jtCtx.loopIdx) { - param.asc = !param.asc; - param.filter = false; - runSingleTest(caseName, ¶m); - - param.filter = true; - runSingleTest(caseName, ¶m); - } - - printStatInfo(caseName); - taosMemoryFree(pTask); -} -#endif - -#if 1 -TEST(leftSemiJoin, eqCondTest) { - SJoinTestParam param; - char* caseName = "leftSemiJoin:eqCondTest"; - SExecTaskInfo* pTask = createDummyTaskInfo(caseName); - assert(pTask); - - param.pTask = pTask; - param.joinType = JOIN_TYPE_LEFT; - param.subType = JOIN_STYPE_SEMI; - param.cond = TEST_EQ_COND; - param.asc = true; - param.grpJoin = false; - - for (jtCtx.loopIdx = 0; jtCtx.loopIdx < JT_MAX_LOOP; ++jtCtx.loopIdx) { - param.asc = !param.asc; - param.filter = false; - runSingleTest(caseName, ¶m); - - param.filter = true; - runSingleTest(caseName, ¶m); - } - - printStatInfo(caseName); - taosMemoryFree(pTask); - handleCaseEnd(); -} -#endif - -#if 1 -TEST(leftSemiJoin, onCondTest) { - SJoinTestParam param; - char* caseName = "leftSemiJoin:onCondTest"; - SExecTaskInfo* pTask = createDummyTaskInfo(caseName); - assert(pTask); - - param.pTask = pTask; - param.joinType = JOIN_TYPE_LEFT; - param.subType = JOIN_STYPE_SEMI; - param.cond = TEST_ON_COND; - param.asc = true; - param.grpJoin = false; - - for (jtCtx.loopIdx = 0; jtCtx.loopIdx < JT_MAX_LOOP; ++jtCtx.loopIdx) { - param.asc = !param.asc; - param.filter = false; - runSingleTest(caseName, ¶m); - - param.filter = true; - runSingleTest(caseName, ¶m); - } - - printStatInfo(caseName); - taosMemoryFree(pTask); -} -#endif - -#if 1 -TEST(leftSemiJoin, fullCondTest) { - SJoinTestParam param; - char* caseName = "leftSemiJoin:fullCondTest"; - SExecTaskInfo* pTask = createDummyTaskInfo(caseName); - assert(pTask); - - param.pTask = pTask; - param.joinType = JOIN_TYPE_LEFT; - param.subType = JOIN_STYPE_SEMI; - param.cond = TEST_FULL_COND; - param.asc = true; - param.grpJoin = false; - - for (jtCtx.loopIdx = 0; jtCtx.loopIdx < JT_MAX_LOOP; ++jtCtx.loopIdx) { - param.asc = !param.asc; - param.filter = false; - runSingleTest(caseName, ¶m); - - param.filter = true; - runSingleTest(caseName, ¶m); - } - - printStatInfo(caseName); - taosMemoryFree(pTask); -} -#endif -#endif - -#if 1 -#if 1 -TEST(leftAntiJoin, noCondTest) { - SJoinTestParam param; - char* caseName = "leftAntiJoin:noCondTest"; - SExecTaskInfo* pTask = createDummyTaskInfo(caseName); - assert(pTask); - - param.pTask = pTask; - param.joinType = JOIN_TYPE_LEFT; - param.subType = JOIN_STYPE_ANTI; - param.cond = TEST_NO_COND; - param.asc = true; - param.grpJoin = false; - - for (jtCtx.loopIdx = 0; jtCtx.loopIdx < JT_MAX_LOOP; ++jtCtx.loopIdx) { - param.asc = !param.asc; - param.filter = false; - runSingleTest(caseName, ¶m); - - param.filter = true; - runSingleTest(caseName, ¶m); - } - - printStatInfo(caseName); - taosMemoryFree(pTask); -} -#endif - -#if 1 -TEST(leftAntiJoin, eqCondTest) { - SJoinTestParam param; - char* caseName = "leftAntiJoin:eqCondTest"; - SExecTaskInfo* pTask = createDummyTaskInfo(caseName); - assert(pTask); - - param.pTask = pTask; - param.joinType = JOIN_TYPE_LEFT; - param.subType = JOIN_STYPE_ANTI; - param.cond = TEST_EQ_COND; - param.asc = true; - param.grpJoin = false; - - for (jtCtx.loopIdx = 0; jtCtx.loopIdx < JT_MAX_LOOP; ++jtCtx.loopIdx) { - param.asc = !param.asc; - param.filter = false; - runSingleTest(caseName, ¶m); - - param.filter = true; - runSingleTest(caseName, ¶m); - } - - printStatInfo(caseName); - taosMemoryFree(pTask); - handleCaseEnd(); -} -#endif - -#if 1 -TEST(leftAntiJoin, onCondTest) { - SJoinTestParam param; - char* caseName = "leftAntiJoin:onCondTest"; - SExecTaskInfo* pTask = createDummyTaskInfo(caseName); - assert(pTask); - - param.pTask = pTask; - param.joinType = JOIN_TYPE_LEFT; - param.subType = JOIN_STYPE_ANTI; - param.cond = TEST_ON_COND; - param.asc = true; - param.grpJoin = false; - - for (jtCtx.loopIdx = 0; jtCtx.loopIdx < JT_MAX_LOOP; ++jtCtx.loopIdx) { - param.asc = !param.asc; - param.filter = false; - runSingleTest(caseName, ¶m); - - param.filter = true; - runSingleTest(caseName, ¶m); - } - - printStatInfo(caseName); - taosMemoryFree(pTask); -} -#endif - -#if 1 -TEST(leftAntiJoin, fullCondTest) { - SJoinTestParam param; - char* caseName = "leftAntiJoin:fullCondTest"; - SExecTaskInfo* pTask = createDummyTaskInfo(caseName); - assert(pTask); - - param.pTask = pTask; - param.joinType = JOIN_TYPE_LEFT; - param.subType = JOIN_STYPE_ANTI; - param.cond = TEST_FULL_COND; - param.asc = true; - param.grpJoin = false; - - for (jtCtx.loopIdx = 0; jtCtx.loopIdx < JT_MAX_LOOP; ++jtCtx.loopIdx) { - param.asc = !param.asc; - param.filter = false; - runSingleTest(caseName, ¶m); - - param.filter = true; - runSingleTest(caseName, ¶m); - } - - printStatInfo(caseName); - taosMemoryFree(pTask); -} -#endif -#endif - -#if 1 -#if 1 -TEST(leftAsofJoin, noCondGreaterThanTest) { - SJoinTestParam param; - char* caseName = "leftAsofJoin:noCondGreaterThanTest"; - SExecTaskInfo* pTask = createDummyTaskInfo(caseName); - assert(pTask); - - param.pTask = pTask; - param.joinType = JOIN_TYPE_LEFT; - param.subType = JOIN_STYPE_ASOF; - param.cond = TEST_NO_COND; - param.asofOp = OP_TYPE_GREATER_THAN; - param.asc = true; - - for (jtCtx.loopIdx = 0; jtCtx.loopIdx < JT_MAX_LOOP; ++jtCtx.loopIdx) { - param.asc = !param.asc; - param.jLimit = taosRand() % 2 ? (1 + (taosRand() % JT_MAX_JLIMIT)) : 1; - - param.grpJoin = taosRand() % 2 ? true : false; - param.filter = false; - runSingleTest(caseName, ¶m); - - param.grpJoin = taosRand() % 2 ? true : false; - param.filter = true; - runSingleTest(caseName, ¶m); - } - - printStatInfo(caseName); - taosMemoryFree(pTask); -} -#endif - -#if 1 -TEST(leftAsofJoin, noCondGreaterEqTest) { - SJoinTestParam param; - char* caseName = "leftAsofJoin:noCondGreaterEqTest"; - SExecTaskInfo* pTask = createDummyTaskInfo(caseName); - assert(pTask); - - param.pTask = pTask; - param.joinType = JOIN_TYPE_LEFT; - param.subType = JOIN_STYPE_ASOF; - param.cond = TEST_NO_COND; - param.asofOp = OP_TYPE_GREATER_EQUAL; - param.asc = true; - - for (jtCtx.loopIdx = 0; jtCtx.loopIdx < JT_MAX_LOOP; ++jtCtx.loopIdx) { - param.asc = !param.asc; - param.jLimit = taosRand() % 2 ? (1 + (taosRand() % JT_MAX_JLIMIT)) : 1; - - param.grpJoin = taosRand() % 2 ? true : false; - param.filter = false; - runSingleTest(caseName, ¶m); - - param.grpJoin = taosRand() % 2 ? true : false; - param.filter = true; - runSingleTest(caseName, ¶m); - } - - printStatInfo(caseName); - taosMemoryFree(pTask); -} -#endif - -#if 1 -TEST(leftAsofJoin, noCondEqTest) { - SJoinTestParam param; - char* caseName = "leftAsofJoin:noCondEqTest"; - SExecTaskInfo* pTask = createDummyTaskInfo(caseName); - assert(pTask); - - param.pTask = pTask; - param.joinType = JOIN_TYPE_LEFT; - param.subType = JOIN_STYPE_ASOF; - param.cond = TEST_NO_COND; - param.asofOp = OP_TYPE_EQUAL; - param.asc = true; - - for (jtCtx.loopIdx = 0; jtCtx.loopIdx < JT_MAX_LOOP; ++jtCtx.loopIdx) { - param.asc = !param.asc; - param.jLimit = taosRand() % 2 ? (1 + (taosRand() % JT_MAX_JLIMIT)) : 1; - - param.grpJoin = taosRand() % 2 ? true : false; - param.filter = false; - runSingleTest(caseName, ¶m); - - param.grpJoin = taosRand() % 2 ? true : false; - param.filter = true; - runSingleTest(caseName, ¶m); - } - - printStatInfo(caseName); - taosMemoryFree(pTask); -} -#endif - -#if 1 -TEST(leftAsofJoin, noCondLowerThanTest) { - SJoinTestParam param; - char* caseName = "leftAsofJoin:noCondLowerThanTest"; - SExecTaskInfo* pTask = createDummyTaskInfo(caseName); - assert(pTask); - - param.pTask = pTask; - param.joinType = JOIN_TYPE_LEFT; - param.subType = JOIN_STYPE_ASOF; - param.cond = TEST_NO_COND; - param.asofOp = OP_TYPE_LOWER_THAN; - param.asc = true; - - for (jtCtx.loopIdx = 0; jtCtx.loopIdx < JT_MAX_LOOP; ++jtCtx.loopIdx) { - param.asc = !param.asc; - param.jLimit = taosRand() % 2 ? (1 + (taosRand() % JT_MAX_JLIMIT)) : 1; - - param.grpJoin = taosRand() % 2 ? true : false; - param.filter = false; - runSingleTest(caseName, ¶m); - - param.grpJoin = taosRand() % 2 ? true : false; - param.filter = true; - runSingleTest(caseName, ¶m); - } - - printStatInfo(caseName); - taosMemoryFree(pTask); -} -#endif - - -#if 1 -TEST(leftAsofJoin, noCondLowerEqTest) { - SJoinTestParam param; - char* caseName = "leftAsofJoin:noCondLowerEqTest"; - SExecTaskInfo* pTask = createDummyTaskInfo(caseName); - assert(pTask); - - param.pTask = pTask; - param.joinType = JOIN_TYPE_LEFT; - param.subType = JOIN_STYPE_ASOF; - param.cond = TEST_NO_COND; - param.asofOp = OP_TYPE_LOWER_EQUAL; - param.asc = true; - - for (jtCtx.loopIdx = 0; jtCtx.loopIdx < JT_MAX_LOOP; ++jtCtx.loopIdx) { - param.asc = !param.asc; - param.jLimit = taosRand() % 2 ? (1 + (taosRand() % JT_MAX_JLIMIT)) : 1; - - param.grpJoin = taosRand() % 2 ? true : false; - param.filter = false; - runSingleTest(caseName, ¶m); - - param.grpJoin = taosRand() % 2 ? true : false; - param.filter = true; - runSingleTest(caseName, ¶m); - } - - printStatInfo(caseName); - taosMemoryFree(pTask); -} -#endif - -#endif - - -#if 1 -#if 1 -TEST(leftWinJoin, noCondProjectionTest) { - SJoinTestParam param; - char* caseName = "leftWinJoin:noCondProjectionTest"; - SExecTaskInfo* pTask = createDummyTaskInfo(caseName); - assert(pTask); - - param.pTask = pTask; - param.joinType = JOIN_TYPE_LEFT; - param.subType = JOIN_STYPE_WIN; - param.cond = TEST_NO_COND; - param.asc = true; - - for (jtCtx.loopIdx = 0; jtCtx.loopIdx < JT_MAX_LOOP; ++jtCtx.loopIdx) { - param.asc = !param.asc; - param.jLimit = taosRand() % 2 ? (1 + (taosRand() % JT_MAX_JLIMIT)) : 1; - - param.grpJoin = taosRand() % 2 ? true : false; - param.filter = false; - runSingleTest(caseName, ¶m); - - param.grpJoin = taosRand() % 2 ? true : false; - param.filter = true; - runSingleTest(caseName, ¶m); - } - - printStatInfo(caseName); - taosMemoryFree(pTask); } #endif @@ -3767,10 +1112,9 @@ TEST(leftWinJoin, noCondProjectionTest) { #endif - int main(int argc, char** argv) { taosSeedRand(taosGetTimestampSec()); - initJoinTest(); + qptInitTest(); testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 9f831c05e9..fc9e971224 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -2010,6 +2010,8 @@ void* nodesGetValueFromNode(SValueNode* pNode) { int32_t nodesSetValueNodeValue(SValueNode* pNode, void* value) { switch (pNode->node.resType.type) { + case TSDB_DATA_TYPE_NULL: + break; case TSDB_DATA_TYPE_BOOL: pNode->datum.b = *(bool*)value; *(bool*)&pNode->typeData = pNode->datum.b; @@ -2061,7 +2063,10 @@ int32_t nodesSetValueNodeValue(SValueNode* pNode, void* value) { case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: + case TSDB_DATA_TYPE_DECIMAL: case TSDB_DATA_TYPE_JSON: + case TSDB_DATA_TYPE_BLOB: + case TSDB_DATA_TYPE_MEDIUMBLOB: case TSDB_DATA_TYPE_GEOMETRY: pNode->datum.p = (char*)value; break; From 65985c7aaa8f909d449d94786534968f4851a768 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 27 Sep 2024 18:08:36 +0800 Subject: [PATCH 03/29] enh: add tag scan test --- include/libs/function/functionMgt.h | 2 + source/libs/executor/test/queryPlanTests.cpp | 346 ++++++++++++++----- source/libs/function/src/builtins.c | 44 ++- source/libs/planner/inc/planInt.h | 12 + source/libs/planner/src/planPhysiCreater.c | 9 - 5 files changed, 311 insertions(+), 102 deletions(-) diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 519207377b..acd521ff74 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -26,6 +26,8 @@ extern "C" { #define FUNC_AGGREGATE_UDF_ID 5001 #define FUNC_SCALAR_UDF_ID 5002 +extern const int32_t funcMgtBuiltinsNum; + typedef enum EFunctionType { // aggregate function FUNCTION_TYPE_APERCENTILE = 1, diff --git a/source/libs/executor/test/queryPlanTests.cpp b/source/libs/executor/test/queryPlanTests.cpp index 8354620afb..71b21e8006 100755 --- a/source/libs/executor/test/queryPlanTests.cpp +++ b/source/libs/executor/test/queryPlanTests.cpp @@ -41,7 +41,7 @@ #include "tvariant.h" #include "stub.h" #include "querytask.h" - +#include "functionMgt.h" namespace { @@ -53,6 +53,7 @@ namespace { #define QPT_MAX_FUNC_PARAM 5 #define QPT_MAX_LOGIC_PARAM 5 #define QPT_MAX_NODE_LIST_NUM 5 +#define QPT_DEFAULT_VNODE_NUM 5 int32_t QPT_PHYSIC_NODE_LIST[] = { QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN, @@ -106,6 +107,7 @@ int32_t QPT_PHYSIC_NODE_LIST[] = { }; #define QPT_PHYSIC_NODE_NUM() (sizeof(QPT_PHYSIC_NODE_LIST)/sizeof(QPT_PHYSIC_NODE_LIST[0])) +#define QPT_RAND_BOOL_V ((taosRand() % 2) ? true : false) typedef struct { ENodeType type; @@ -168,21 +170,26 @@ typedef struct { SPhysiNode* pCurr; SPhysiNode* pChild; EOrder currTsOrder; - SPhysiPlanContext* pCxt; } SQPTBuildCtx; typedef struct { int32_t nodeLevel; bool onlyTag; + int16_t nextBlockId; SDataBlockDescNode* pInputDataBlockDesc; -} SQPTMakeNodeCtx; +} SQPTMakePlanCtx; + +typedef struct { + int32_t code; +} SQPTExecResult; typedef struct { int32_t loopIdx; SQPTParam param; SQPTBuildCtx buildCtx; - SQPTMakeNodeCtx makeCtx; + SQPTMakePlanCtx makeCtx; + SQPTExecResult result; int64_t startTsUs; } SQPTCtx; @@ -200,11 +207,16 @@ SQPTCtrl qptCtrl = {1, 0, 0, 0, 0}; bool qptErrorRerun = false; bool qptInRerun = false; +void qptMakeExprNode(SNode** ppNode); + + void qptPrintBeginInfo(char* caseName) { if (!qptCtrl.printTestInfo) { return; } + printf("\n%dth TEST [%s] START\n", qptCtx.loopIdx, caseName); + /* char inputStat[4] = {0}; JT_PRINTF("\n%dth TEST [%s] START\nBasic Info:\n\t asc:%d\n\t filter:%d\n\t maxRows:left-%d right-%d\n\t " @@ -260,25 +272,35 @@ void qptPrintBeginInfo(char* caseName) { */ } +void qptPrintEndInfo(char* caseName) { + if (!qptCtrl.printTestInfo) { + return; + } + + printf("\n\t%dth TEST [%s] END, result - %s%s\n", qptCtx.loopIdx, caseName, + (0 == qptCtx.result.code) ? "succeed" : "failed with error:", + (0 == qptCtx.result.code) ? "" : tstrerror(qptCtx.result.code)); +} + void qptPrintStatInfo(char* caseName) { } bool qptGetDynamicOp() { - if (!qptCtx->param.correctExpected) { - return (taosRand() % 2) : true : false; + if (!qptCtx.param.correctExpected) { + return QPT_RAND_BOOL_V; } - if (qptCtx->buildCtx.pChild) { - return qptCtx->buildCtx.pChild->dynamicOp; + if (qptCtx.buildCtx.pChild) { + return qptCtx.buildCtx.pChild->dynamicOp; } - return (taosRand() % 2) : true : false; + return QPT_RAND_BOOL_V; } EOrder qptGetInputTsOrder() { - return qptCtx->buildCtx.currTsOrder; + return qptCtx.buildCtx.currTsOrder; } @@ -289,12 +311,12 @@ SNode* qptMakeLimitNode() { SLimitNode* pLimit = (SLimitNode*)pNode; - if (!qptCtx->param->correctExpected) { + if (!qptCtx.param.correctExpected) { if (taosRand() % 2) { - pLimit->limit = taosRand() * ((taosRand() % 2) : 1 : -1); + pLimit->limit = taosRand() * ((taosRand() % 2) ? 1 : -1); } if (taosRand() % 2) { - pLimit->offset = taosRand() * ((taosRand() % 2) : 1 : -1); + pLimit->offset = taosRand() * ((taosRand() % 2) ? 1 : -1); } } else { pLimit->limit = taosRand(); @@ -303,7 +325,7 @@ SNode* qptMakeLimitNode() { } } - return pLimit; + return pNode; } SNode* qptMakeColumnNodeFromTable(int32_t colIdx, EColumnType colType, SScanPhysiNode* pScanPhysiNode) { @@ -311,25 +333,25 @@ SNode* qptMakeColumnNodeFromTable(int32_t colIdx, EColumnType colType, SScanPhys assert(0 == nodesMakeNode(QUERY_NODE_COLUMN, (SNode**)&pCol)); assert(pCol); - pCol->node.resType.type = qptCtx->param.tbl.pCol[colIdx].type; - pCol->node.resType.bytes = qptCtx->param.tbl.pCol[colIdx].len; + pCol->node.resType.type = qptCtx.param.tbl.pCol[colIdx].type; + pCol->node.resType.bytes = qptCtx.param.tbl.pCol[colIdx].len; - pCol->tableId = qptCtx->param.tbl.uid; - pCol->tableType = qptCtx->param.tbl.tblType; + pCol->tableId = qptCtx.param.tbl.uid; + pCol->tableType = qptCtx.param.tbl.tblType; pCol->colId = colIdx; pCol->projIdx = colIdx; - pCol->colType = qptCtx->param.tbl.pCol[colIdx].colType; - pCol->hasIndex = qptCtx->param.tbl.pCol[colIdx].hasIndex; - pCol->isPrimTs = qptCtx->param.tbl.pCol[colIdx].isPrimTs; - strcpy(pCol->dbName, qptCtx->param.db.dbName); - strcpy(pCol->tableName, qptCtx->param.tbl.tblName); - strcpy(pCol->tableAlias, qptCtx->param.tbl.tblAlias); - strcpy(pCol->colName, qptCtx->param.tbl.pCol[colIdx].name); + pCol->colType = qptCtx.param.tbl.pCol[colIdx].colType; + pCol->hasIndex = qptCtx.param.tbl.pCol[colIdx].hasIndex; + pCol->isPrimTs = qptCtx.param.tbl.pCol[colIdx].isPrimTs; + strcpy(pCol->dbName, qptCtx.param.db.dbName); + strcpy(pCol->tableName, qptCtx.param.tbl.tblName); + strcpy(pCol->tableAlias, qptCtx.param.tbl.tblAlias); + strcpy(pCol->colName, qptCtx.param.tbl.pCol[colIdx].name); pCol->dataBlockId = pScanPhysiNode->node.pOutputDataBlockDesc->dataBlockId; pCol->slotId = colIdx; - pCol->numOfPKs = qptCtx->param.tbl.pkNum; - pCol->tableHasPk = qptCtx->param.tbl.pkNum > 0; - pCol->isPk = qptCtx->param.tbl.pCol[colIdx].isPk; + pCol->numOfPKs = qptCtx.param.tbl.pkNum; + pCol->tableHasPk = qptCtx.param.tbl.pkNum > 0; + pCol->isPk = qptCtx.param.tbl.pCol[colIdx].isPk; pCol->projRefIdx = 0; pCol->resIdx = 0; @@ -355,7 +377,7 @@ void qptMakeCaseWhenNode(SNode** ppNode) { assert(*ppNode); SCaseWhenNode* pCaseWhen = (SCaseWhenNode*)*ppNode; - qptCtx->makeCtx.nodeLevel++; + qptCtx.makeCtx.nodeLevel++; qptMakeExprNode(&pCaseWhen->pCase); assert(pCaseWhen->pCase); @@ -378,6 +400,8 @@ void qptMakeOperatorNode(SNode** ppNode) { assert(0 == nodesMakeNode(QUERY_NODE_OPERATOR, ppNode)); SOperatorNode* pOp = (SOperatorNode*)*ppNode; pOp->opType = opType; + + qptCtx.makeCtx.nodeLevel++; switch (opType) { case OP_TYPE_ADD: case OP_TYPE_SUB: @@ -401,8 +425,8 @@ void qptMakeOperatorNode(SNode** ppNode) { case OP_TYPE_JSON_GET_VALUE: case OP_TYPE_JSON_CONTAINS: case OP_TYPE_ASSIGN: - qptMakeOperatorNode(&pOp->pLeft); - qptMakeOperatorNode(&pOp->pRight); + qptMakeExprNode(&pOp->pLeft); + qptMakeExprNode(&pOp->pRight); break; case OP_TYPE_IS_NULL: @@ -414,7 +438,7 @@ void qptMakeOperatorNode(SNode** ppNode) { case OP_TYPE_IS_NOT_FALSE: case OP_TYPE_IS_NOT_UNKNOWN: case OP_TYPE_MINUS: - qptMakeOperatorNode(&pOp->pLeft); + qptMakeExprNode(&pOp->pLeft); break; default: assert(0); @@ -423,7 +447,7 @@ void qptMakeOperatorNode(SNode** ppNode) { } void qptMakeColumnNode(SNode** ppNode) { - SNodeList* pColList = qptCtx->makeCtx.pInputDataBlockDesc->pSlots; + SNodeList* pColList = qptCtx.makeCtx.pInputDataBlockDesc->pSlots; int32_t colIdx = taosRand() % pColList->length; SNode* pNode = nodesListGetNode(pColList, colIdx); assert(nodeType(pNode) == QUERY_NODE_SLOT_DESC); @@ -431,7 +455,7 @@ void qptMakeColumnNode(SNode** ppNode) { SColumnNode* pCol = NULL; nodesMakeNode(QUERY_NODE_COLUMN, (SNode**)&pCol); pCol->node.resType = pSlot->dataType; - pCol->dataBlockId = qptCtx->makeCtx.pInputDataBlockDesc->dataBlockId; + pCol->dataBlockId = qptCtx.makeCtx.pInputDataBlockDesc->dataBlockId; pCol->slotId = pSlot->slotId; *ppNode = (SNode*)pCol; @@ -448,7 +472,7 @@ void qptGetRandValue(int32_t* pType, int32_t* pLen, void** ppVal) { *pLen = tDataTypes[*pType].bytes; *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); assert(*ppVal); - *(bool*)*ppVal = (taosRand() % 2) : true : false; + *(bool*)*ppVal = QPT_RAND_BOOL_V; break; case TSDB_DATA_TYPE_TINYINT: *pLen = tDataTypes[*pType].bytes; @@ -502,7 +526,7 @@ void qptGetRandValue(int32_t* pType, int32_t* pLen, void** ppVal) { break; case TSDB_DATA_TYPE_NCHAR: { *pLen = taosRand() % QPT_MAX_STRING_LEN; - char* pTmp = taosMemoryCalloc(1, *pLen + 1); + char* pTmp = (char*)taosMemoryCalloc(1, *pLen + 1); assert(pTmp); memset(pTmp, 'A' + taosRand() % 26, *pLen); *ppVal = taosMemoryCalloc(1, *pLen * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); @@ -595,11 +619,10 @@ void qptMakeFunctionNode(SNode** ppNode) { nodesMakeNode(QUERY_NODE_FUNCTION, (SNode**)&pFunc); int32_t funcIdx = taosRand() % funcMgtBuiltinsNum; - strcpy(pFunc->functionName, funcMgtBuiltins[funcIdx].name); - + strcpy(pFunc->functionName, fmGetFuncName(funcIdx)); fmGetFuncInfo(pFunc, NULL, 0); - qptCtx->makeCtx.nodeLevel++; + qptCtx.makeCtx.nodeLevel++; int32_t paramNum = taosRand() % QPT_MAX_FUNC_PARAM + 1; for (int32_t i = 0; i < paramNum; ++i) { SNode* pNode = NULL; @@ -616,7 +639,7 @@ void qptMakeLogicCondNode(SNode** ppNode) { pLogic->condType = (taosRand() % 3) ? ((taosRand() % 2) ? LOGIC_COND_TYPE_AND : LOGIC_COND_TYPE_OR) : LOGIC_COND_TYPE_NOT; - qptCtx->makeCtx.nodeLevel++; + qptCtx.makeCtx.nodeLevel++; int32_t paramNum = taosRand() % QPT_MAX_LOGIC_PARAM + 1; for (int32_t i = 0; i < paramNum; ++i) { SNode* pNode = NULL; @@ -631,7 +654,7 @@ void qptMakeNodeListNode(SNode** ppNode) { SNodeListNode* pList = NULL; nodesMakeNode(QUERY_NODE_NODE_LIST, (SNode**)&pList); - qptCtx->makeCtx.nodeLevel++; + qptCtx.makeCtx.nodeLevel++; int32_t nodeNum = taosRand() % QPT_MAX_NODE_LIST_NUM + 1; for (int32_t i = 0; i < nodeNum; ++i) { SNode* pNode = NULL; @@ -668,11 +691,11 @@ void qptMakeTableNode(SNode** ppNode) { void qptMakeExprNode(SNode** ppNode) { int32_t nodeTypeMaxValue = 9; - if (qptCtx->makeCtx.nodeLevel >= QPT_MAX_NODE_LEVEL) { + if (qptCtx.makeCtx.nodeLevel >= QPT_MAX_NODE_LEVEL) { nodeTypeMaxValue = 2; } - switch (taosRand() % 10) { + switch (taosRand() % nodeTypeMaxValue) { case 0: qptMakeColumnNode(ppNode); break; @@ -707,7 +730,7 @@ void qptMakeExprNode(SNode** ppNode) { } void qptResetMakeNodeCtx(SDataBlockDescNode* pInput, bool onlyTag) { - SQPTMakeNodeCtx* pCtx = &qptCtx->makeCtx; + SQPTMakePlanCtx* pCtx = &qptCtx.makeCtx; pCtx->nodeLevel = 1; pCtx->onlyTag = onlyTag; @@ -716,18 +739,24 @@ void qptResetMakeNodeCtx(SDataBlockDescNode* pInput, bool onlyTag) { SNode* qptMakeConditionNode(bool onlyTag) { SNode* pNode = NULL; - qptResetMakeNodeCtx(qptCtx->buildCtx.pCurr->pOutputDataBlockDesc, onlyTag); + qptResetMakeNodeCtx(qptCtx.buildCtx.pCurr->pOutputDataBlockDesc, onlyTag); qptMakeExprNode(&pNode); return pNode; } +SNode* qptMakeDataBlockDescNode() { + SDataBlockDescNode* pDesc = NULL; + assert(0 == nodesMakeNode(QUERY_NODE_DATABLOCK_DESC, (SNode**)&pDesc)); + pDesc->dataBlockId = qptCtx.makeCtx.nextBlockId++; + pDesc->precision = qptCtx.param.db.precision; + + return (SNode*)pDesc; +} + SPhysiNode* qptCreatePhysiNode(int32_t nodeType) { SPhysiNode* pPhysiNode = NULL; - int32_t code = nodesMakeNode(nodeType, (SNode**)&pPhysiNode); - if (NULL == pPhysiNode) { - assert(0); - } + assert(0 == nodesMakeNode((ENodeType)nodeType, (SNode**)&pPhysiNode)); qptCtx.buildCtx.pCurr = pPhysiNode; @@ -736,8 +765,8 @@ SPhysiNode* qptCreatePhysiNode(int32_t nodeType) { pPhysiNode->dynamicOp = qptGetDynamicOp(); pPhysiNode->inputTsOrder = qptGetInputTsOrder(); - assert(0 == createDataBlockDesc(qptCtx->buildCtx.pCxt, NULL, &pPhysiNode->pOutputDataBlockDesc)); - pPhysiNode->pOutputDataBlockDesc->precision = qptCtx->param.db.precision; + pPhysiNode->pOutputDataBlockDesc = (SDataBlockDescNode*)qptMakeDataBlockDescNode(); + assert(pPhysiNode->pOutputDataBlockDesc); return pPhysiNode; } @@ -772,10 +801,10 @@ void qptMarkTableInUseCols(int32_t colNum, int32_t totalColNum, SQPTCol* pCol) { } void qptCreateTableScanColsImpl( SScanPhysiNode* pScanPhysiNode, SNodeList** ppCols, int32_t totalColNum, SQPTCol* pCol) { - int32_t colNum = qptCtx->pCfg->correctExpected ? (taosRand() % totalColNum + 1) : (taosRand()); + int32_t colNum = qptCtx.param.correctExpected ? (taosRand() % totalColNum + 1) : (taosRand()); int32_t colAdded = 0; - if (qptCtx->pCfg->correctExpected) { + if (qptCtx.param.correctExpected) { qptMarkTableInUseCols(colNum, totalColNum, pCol); for (int32_t i = 0; i < totalColNum && colAdded < colNum; ++i) { if (0 == pCol[i].inUse) { @@ -798,53 +827,97 @@ void qptCreateTableScanColsImpl( SScanPhysiNode* pScanPhysiNode, SNodeList void qptCreateTableScanCols( SScanPhysiNode* pScanPhysiNode) { - qptCreateTableScanColsImpl(pScanPhysiNode, &pScanPhysiNode->pScanCols, qptCtx->param.tbl.colNum, qptCtx.param->tbl.pCol); + qptCreateTableScanColsImpl(pScanPhysiNode, &pScanPhysiNode->pScanCols, qptCtx.param.tbl.colNum, qptCtx.param.tbl.pCol); } void qptCreateTableScanPseudoCols( SScanPhysiNode* pScanPhysiNode) { - qptCreateTableScanColsImpl(pScanPhysiNode, &pScanPhysiNode->pScanPseudoCols, qptCtx->param.tbl.tagNum, qptCtx->param.tbl.pTag); + qptCreateTableScanColsImpl(pScanPhysiNode, &pScanPhysiNode->pScanPseudoCols, qptCtx.param.tbl.tagNum, qptCtx.param.tbl.pTag); +} + +SNode* qptMakeSlotDescNode(const char* pName, const SNode* pNode, int16_t slotId, bool output, bool reserve) { + SSlotDescNode* pSlot = NULL; + assert(0 == nodesMakeNode(QUERY_NODE_SLOT_DESC, (SNode**)&pSlot)); + snprintf(pSlot->name, sizeof(pSlot->name), "%s", pName); + pSlot->slotId = slotId; + pSlot->dataType = ((SExprNode*)pNode)->resType; + pSlot->reserve = reserve; + pSlot->output = output; + return (SNode*)pSlot; +} + +void qptCreateMakeNode(SNode* pNode, int16_t dataBlockId, int16_t slotId, SNode** pOutput) { + STargetNode* pTarget = NULL; + assert(0 == nodesMakeNode(QUERY_NODE_TARGET, (SNode**)&pTarget)); + + pTarget->dataBlockId = dataBlockId; + pTarget->slotId = slotId; + pTarget->pExpr = pNode; + + *pOutput = (SNode*)pTarget; +} + +void qptAddDataBlockSlots(SNodeList* pList, SDataBlockDescNode* pDataBlockDesc) { + int16_t nextSlotId = LIST_LENGTH(pDataBlockDesc->pSlots), slotId = 0; + SNode* pNode = NULL; + bool output = QPT_RAND_BOOL_V; + + FOREACH(pNode, pList) { + SNode* pExpr = QUERY_NODE_ORDER_BY_EXPR == nodeType(pNode) ? ((SOrderByExprNode*)pNode)->pExpr : pNode; + assert(0 == nodesListMakeStrictAppend(&pDataBlockDesc->pSlots, qptMakeSlotDescNode(NULL, pExpr, nextSlotId, output, QPT_RAND_BOOL_V))); + pDataBlockDesc->totalRowSize += ((SExprNode*)pExpr)->resType.bytes; + if (output) { + pDataBlockDesc->outputRowSize += ((SExprNode*)pExpr)->resType.bytes; + } + + slotId = nextSlotId; + ++nextSlotId; + + SNode* pTarget = NULL; + qptCreateMakeNode(pNode, pDataBlockDesc->dataBlockId, slotId, &pTarget); + REPLACE_NODE(pTarget); + } } void qptCreateScanPhysiNodeImpl( SScanPhysiNode* pScanPhysiNode) { qptCreateTableScanCols(pScanPhysiNode); - assert(0 == addDataBlockSlots(qptCtx->buildCtx.pCxt, pScanPhysiNode->pScanCols, pScanPhysiNode->node.pOutputDataBlockDesc)); + qptAddDataBlockSlots(pScanPhysiNode->pScanCols, pScanPhysiNode->node.pOutputDataBlockDesc); if (taosRand() % 2) { qptCreateTableScanPseudoCols(pScanPhysiNode); } - assert(0 == addDataBlockSlots(qptCtx->buildCtx.pCxt, pScanPhysiNode->pScanPseudoCols, pScanPhysiNode->node.pOutputDataBlockDesc)); + qptAddDataBlockSlots(pScanPhysiNode->pScanPseudoCols, pScanPhysiNode->node.pOutputDataBlockDesc); - pScanPhysiNode->uid = qptCtx->param.tbl.uid; - pScanPhysiNode->suid = qptCtx->param.tbl.suid; - pScanPhysiNode->tableType = qptCtx->param.tbl.tblType; - pScanPhysiNode->groupOrderScan = (taosRand() % 2) : true : false; + pScanPhysiNode->uid = qptCtx.param.tbl.uid; + pScanPhysiNode->suid = qptCtx.param.tbl.suid; + pScanPhysiNode->tableType = qptCtx.param.tbl.tblType; + pScanPhysiNode->groupOrderScan = (taosRand() % 2) ? true : false; SName tblName = {0}; - toName(1, qptCtx->param.db.dbName, qptCtx->param.tbl.tblName, &tblName); + toName(1, qptCtx.param.db.dbName, qptCtx.param.tbl.tblName, &tblName); memcpy(&pScanPhysiNode->tableName, &tblName, sizeof(SName)); } -STagScanPhysiNode* qptCreateTagScanPhysiNode(int32_t nodeType) { +SNode* qptCreateTagScanPhysiNode(int32_t nodeType) { SPhysiNode* pPhysiNode = qptCreatePhysiNode(nodeType); assert(pPhysiNode); STagScanPhysiNode* pTagScanNode = (STagScanPhysiNode*)pPhysiNode; - pTagScanNode->onlyMetaCtbIdx = (taosRand() % 2) : true : false; + pTagScanNode->onlyMetaCtbIdx = (taosRand() % 2) ? true : false; qptCreateScanPhysiNodeImpl(&pTagScanNode->scan); qptPostCreatePhysiNode(pPhysiNode); - return pPhysiNode; + return (SNode*)pPhysiNode; } -SSortMergeJoinPhysiNode* qptCreateSortMergeJoinPhysiNode(int32_t nodeType) { +SNode* qptCreateSortMergeJoinPhysiNode(int32_t nodeType) { SPhysiNode* pPhysiNode = qptCreatePhysiNode(nodeType); assert(pPhysiNode); @@ -913,13 +986,13 @@ SSortMergeJoinPhysiNode* qptCreateSortMergeJoinPhysiNode(int32_t nodeType) { createBlockDescNode(&p->node.pOutputDataBlockDesc); */ - return pPhysiNode; + return (SNode*)pPhysiNode; } -SPhysiNode* qptCreatePhysicalPlanNode(int32_t nodeType) { +SNode* qptCreatePhysicalPlanNode(int32_t nodeType) { switch (nodeType) { case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: - return qptCreateTagScanPhysiNode(nodeType); + return (SNode*)qptCreateTagScanPhysiNode(nodeType); case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN: case QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN: @@ -929,7 +1002,7 @@ SPhysiNode* qptCreatePhysicalPlanNode(int32_t nodeType) { case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN: case QUERY_NODE_PHYSICAL_PLAN_PROJECT: case QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN: - return qptCreateSortMergeJoinPhysiNode(nodeType); + return (SNode*)qptCreateSortMergeJoinPhysiNode(nodeType); case QUERY_NODE_PHYSICAL_PLAN_HASH_AGG: case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE: case QUERY_NODE_PHYSICAL_PLAN_MERGE: @@ -975,8 +1048,8 @@ SPhysiNode* qptCreatePhysicalPlanNode(int32_t nodeType) { return 0; } -int32_t qptCreateQueryPlan(SNode** ppPlan) { - +void qptCreateQueryPlan(SNode** ppPlan) { + } @@ -987,19 +1060,23 @@ void qptRerunBlockedHere() { } void qptResetForReRun() { - + for (int32_t i = 0; i < qptCtx.param.tbl.colNum; ++i) { + qptCtx.param.tbl.pCol[i].inUse = 0; + } + for (int32_t i = 0; i < qptCtx.param.tbl.tagNum; ++i) { + qptCtx.param.tbl.pTag[i].inUse = 0; + } } void qptSingleTestDone(bool* contLoop) { - destroyOperator(jtCtx.pJoinOp); - jtCtx.pJoinOp = NULL; - +/* if (jtRes.succeed) { *contLoop = false; return; } - - if (jtErrorRerun) { +*/ + + if (qptErrorRerun) { *contLoop = false; return; } @@ -1036,44 +1113,133 @@ void qptRunPlanTest(char* caseName) { SReadHandle readHandle = {0}; SOperatorInfo* pOperator = NULL; - if (qptCtx->param.plan.singlePhysiNode) { - pNode = (SNode*)qptCreatePhysicalPlanNode(qptCtx->param.plan.subplanType[0]); + if (qptCtx.loopIdx > 0) { + qptResetForReRun(); + } + + if (qptCtx.param.plan.singlePhysiNode) { + pNode = (SNode*)qptCreatePhysicalPlanNode(qptCtx.param.plan.subplanType[0]); } qptPrintBeginInfo(caseName); qptCtx.startTsUs = taosGetTimestampUs(); - int32_t code = createTagScanOperatorInfo(&readHandle, (STagScanPhysiNode*)pNode, NULL, NULL, NULL, NULL, &pOperator); + qptCtx.result.code = createTagScanOperatorInfo(&readHandle, (STagScanPhysiNode*)pNode, NULL, NULL, NULL, NULL, &pOperator); destroyOperator(pOperator); nodesDestroyNode((SNode*)pNode); + qptPrintEndInfo(caseName); + qptHandleTestEnd(); } SQPTNodeParam* qptInitNodeParam(int32_t nodeType) { + return NULL; +} +int32_t qptGetColumnRandLen(int32_t colType) { + switch (colType) { + case TSDB_DATA_TYPE_NULL: + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: + case TSDB_DATA_TYPE_UTINYINT: + case TSDB_DATA_TYPE_USMALLINT: + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_UBIGINT: + return tDataTypes[colType].bytes; + case TSDB_DATA_TYPE_VARCHAR: + case TSDB_DATA_TYPE_GEOMETRY: + case TSDB_DATA_TYPE_JSON: + case TSDB_DATA_TYPE_VARBINARY: + case TSDB_DATA_TYPE_DECIMAL: + case TSDB_DATA_TYPE_BLOB: + case TSDB_DATA_TYPE_MEDIUMBLOB: + case TSDB_DATA_TYPE_NCHAR: + return taosRand() % TSDB_MAX_BINARY_LEN; + default: + assert(0); + break; + } +} + +void qptInitTableCols(SQPTCol* pCol, int32_t colNum, EColumnType colType) { + int32_t tbnameIdx = -1; + if (QPT_RAND_BOOL_V && COLUMN_TYPE_TAG == colType) { + tbnameIdx = taosRand() % colNum; + } + + for (int32_t i = 0; i < colNum; ++i) { + if (tbnameIdx >= 0 && i == tbnameIdx) { + strcpy(pCol[i].name, "tbname"); + pCol[i].type = TSDB_DATA_TYPE_VARCHAR; + pCol[i].len = qptGetColumnRandLen(pCol[i].type); + pCol[i].inUse = 0; + pCol[i].hasIndex = QPT_RAND_BOOL_V; + pCol[i].isPrimTs = QPT_RAND_BOOL_V; + pCol[i].isPk = QPT_RAND_BOOL_V; + pCol[i].colType = COLUMN_TYPE_TBNAME; + continue; + } + + sprintf(pCol[i].name, "col%d", i); + pCol[i].type = taosRand() % TSDB_DATA_TYPE_MAX; + pCol[i].len = qptGetColumnRandLen(pCol[i].type); + pCol[i].inUse = 0; + pCol[i].hasIndex = QPT_RAND_BOOL_V; + pCol[i].isPrimTs = QPT_RAND_BOOL_V; + pCol[i].isPk = QPT_RAND_BOOL_V; + pCol[i].colType = colType; + } } void qptInitTestCtx(bool correctExpected, bool singleNode, int32_t nodeType, int32_t paramNum, SQPTNodeParam* nodeParam) { - qptCtx->param.correctExpected = correctExpected; - qptCtx->param.plan.singlePhysiNode = singleNode; + qptCtx.param.correctExpected = correctExpected; + qptCtx.param.plan.singlePhysiNode = singleNode; if (singleNode) { - qptCtx->param.plan.subplanMaxLevel = 1; - qptCtx->param.plan.subplanType[0] = nodeType; + qptCtx.param.plan.subplanMaxLevel = 1; + qptCtx.param.plan.subplanType[0] = nodeType; } else { - qptCtx->param.plan.subplanMaxLevel = taosRand() % QPT_MAX_SUBPLAN_LEVEL + 1; - for (int32_t i = 0; i < qptCtx->param.plan.subplanMaxLevel; ++i) { - qptCtx->param.plan.subplanType[i] = QPT_PHYSIC_NODE_LIST[taosRand() % QPT_PHYSIC_NODE_NUM()]; + qptCtx.param.plan.subplanMaxLevel = taosRand() % QPT_MAX_SUBPLAN_LEVEL + 1; + for (int32_t i = 0; i < qptCtx.param.plan.subplanMaxLevel; ++i) { + qptCtx.param.plan.subplanType[i] = QPT_PHYSIC_NODE_LIST[taosRand() % QPT_PHYSIC_NODE_NUM()]; } } if (paramNum > 0) { - qptCtx->param.plan.physiNodeParamNum = paramNum; - qptCtx->param.plan.physicNodeParam = nodeParam; + qptCtx.param.plan.physiNodeParamNum = paramNum; + qptCtx.param.plan.physicNodeParam = nodeParam; } + + qptCtx.param.db.precision = TSDB_TIME_PRECISION_MILLI; + strcpy(qptCtx.param.db.dbName, "qptdb1"); + + qptCtx.param.vnode.vnodeNum = QPT_DEFAULT_VNODE_NUM; + + qptCtx.param.tbl.uid = 100; + qptCtx.param.tbl.suid = 1; + qptCtx.param.tbl.tblType = taosRand() % TSDB_TABLE_MAX; + qptCtx.param.tbl.colNum = taosRand() % 4098; + qptCtx.param.tbl.tagNum = taosRand() % 130; + qptCtx.param.tbl.pkNum = taosRand() % 2; + strcpy(qptCtx.param.tbl.tblName, "qpttbl1"); + strcpy(qptCtx.param.tbl.tblName, "tbl1"); + qptCtx.param.tbl.pCol = (SQPTCol*)taosMemoryCalloc(qptCtx.param.tbl.colNum, sizeof(*qptCtx.param.tbl.pCol)); + assert(qptCtx.param.tbl.pCol); + qptInitTableCols(qptCtx.param.tbl.pCol, qptCtx.param.tbl.colNum, COLUMN_TYPE_COLUMN); + + qptCtx.param.tbl.pTag = (SQPTCol*)taosMemoryCalloc(qptCtx.param.tbl.tagNum, sizeof(*qptCtx.param.tbl.pTag)); + assert(qptCtx.param.tbl.pTag); + qptInitTableCols(qptCtx.param.tbl.pTag, qptCtx.param.tbl.tagNum, COLUMN_TYPE_TAG); + } diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 976d15e7d8..194e8caf55 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1860,6 +1860,11 @@ static int32_t translateIrate(SFunctionNode* pFunc, char* pErrBuf, int32_t len) } static int32_t translateIrateImpl(SFunctionNode* pFunc, char* pErrBuf, int32_t len, bool isPartial) { + int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList); + if (numOfParams <= 0) { + return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); + } + uint8_t colType = getSDataTypeFromNode(nodesListGetNode(pFunc->pParameterList, 0))->type; if (isPartial) { if (3 != LIST_LENGTH(pFunc->pParameterList) && 4 != LIST_LENGTH(pFunc->pParameterList)) { @@ -1903,7 +1908,7 @@ static int32_t translateInterp(SFunctionNode* pFunc, char* pErrBuf, int32_t len) int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList); uint8_t dbPrec = pFunc->node.resType.precision; - if (2 < numOfParams) { + if (2 < numOfParams || numOfParams <= 0) { return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); } @@ -1986,6 +1991,9 @@ static EFuncReturnRows interpEstReturnRows(SFunctionNode* pFunc) { static int32_t translateFirstLast(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { // forbid null as first/last input, since first(c0, null, 1) may have different number of input int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList); + if (numOfParams <= 0) { + return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); + } for (int32_t i = 0; i < numOfParams; ++i) { uint8_t nodeType = nodeType(nodesListGetNode(pFunc->pParameterList, i)); @@ -2000,6 +2008,11 @@ static int32_t translateFirstLast(SFunctionNode* pFunc, char* pErrBuf, int32_t l } static int32_t translateFirstLastImpl(SFunctionNode* pFunc, char* pErrBuf, int32_t len, bool isPartial) { + int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList); + if (numOfParams <= 0) { + return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); + } + // first(col_list) will be rewritten as first(col) SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0); uint8_t paraType = getSDataTypeFromNode(pPara)->type; @@ -2035,6 +2048,11 @@ static int32_t translateFirstLastMerge(SFunctionNode* pFunc, char* pErrBuf, int3 } static int32_t translateFirstLastState(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList); + if (numOfParams <= 0) { + return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); + } + SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0); int32_t paraBytes = getSDataTypeFromNode(pPara)->bytes; @@ -2045,6 +2063,11 @@ static int32_t translateFirstLastState(SFunctionNode* pFunc, char* pErrBuf, int3 } static int32_t translateFirstLastStateMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList); + if (numOfParams <= 0) { + return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); + } + SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0); int32_t paraBytes = getSDataTypeFromNode(pPara)->bytes; uint8_t paraType = getSDataTypeFromNode(pPara)->type; @@ -2081,11 +2104,16 @@ static int32_t translateMode(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { static int32_t translateDiff(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList); - if (numOfParams > 2) { + if (numOfParams > 2 || numOfParams <= 0) { return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); } - uint8_t colType = getSDataTypeFromNode(nodesListGetNode(pFunc->pParameterList, 0))->type; + SDataType* pDataType = getSDataTypeFromNode(nodesListGetNode(pFunc->pParameterList, 0)); + if (NULL == pDataType) { + return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, "Invalid first parameter for DIFF function."); + } + + uint8_t colType = pDataType->type; if (!IS_INTEGER_TYPE(colType) && !IS_FLOAT_TYPE(colType) && TSDB_DATA_TYPE_BOOL != colType && !IS_TIMESTAMP_TYPE(colType)) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); @@ -2432,6 +2460,11 @@ static int32_t translateRepeat(SFunctionNode* pFunc, char* pErrBuf, int32_t len) } static int32_t translateCast(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList); + if (numOfParams <= 0) { + return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); + } + // The number of parameters has been limited by the syntax definition SExprNode* pPara0 = (SExprNode*)nodesListGetNode(pFunc->pParameterList, 0); @@ -2835,6 +2868,11 @@ static int32_t translateIn2GeomOutBool(SFunctionNode* pFunc, char* pErrBuf, int3 } static int32_t translateSelectValue(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList); + if (numOfParams <= 0) { + return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); + } + pFunc->node.resType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/inc/planInt.h b/source/libs/planner/inc/planInt.h index beb277493c..59e771454c 100644 --- a/source/libs/planner/inc/planInt.h +++ b/source/libs/planner/inc/planInt.h @@ -24,6 +24,18 @@ extern "C" { #include "tsimplehash.h" #include "taoserror.h" + +typedef struct SPhysiPlanContext { + SPlanContext* pPlanCxt; + int32_t errCode; + int16_t nextDataBlockId; + SArray* pLocationHelper; + SArray* pProjIdxLocHelper; + bool hasScan; + bool hasSysScan; +} SPhysiPlanContext; + + #define planFatal(param, ...) qFatal("PLAN: " param, ##__VA_ARGS__) #define planError(param, ...) qError("PLAN: " param, ##__VA_ARGS__) #define planWarn(param, ...) qWarn("PLAN: " param, ##__VA_ARGS__) diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 2b0f449b77..a2f45c92ec 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -30,15 +30,6 @@ typedef struct SSlotIndex { SArray* pSlotIdsInfo; // duplicate name slot } SSlotIndex; -typedef struct SPhysiPlanContext { - SPlanContext* pPlanCxt; - int32_t errCode; - int16_t nextDataBlockId; - SArray* pLocationHelper; - SArray* pProjIdxLocHelper; - bool hasScan; - bool hasSysScan; -} SPhysiPlanContext; static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char** ppKey, int32_t *pLen, uint16_t extraBufLen) { int32_t code = 0; From a068f807a2a1d645fd0bb06716b33171966c142c Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Sun, 29 Sep 2024 17:04:57 +0800 Subject: [PATCH 04/29] fix: add random test case --- source/libs/executor/test/queryPlanTests.cpp | 543 ++++++++++++------- 1 file changed, 344 insertions(+), 199 deletions(-) diff --git a/source/libs/executor/test/queryPlanTests.cpp b/source/libs/executor/test/queryPlanTests.cpp index 71b21e8006..37d0a79761 100755 --- a/source/libs/executor/test/queryPlanTests.cpp +++ b/source/libs/executor/test/queryPlanTests.cpp @@ -108,6 +108,8 @@ int32_t QPT_PHYSIC_NODE_LIST[] = { #define QPT_PHYSIC_NODE_NUM() (sizeof(QPT_PHYSIC_NODE_LIST)/sizeof(QPT_PHYSIC_NODE_LIST[0])) #define QPT_RAND_BOOL_V ((taosRand() % 2) ? true : false) +#define QPT_RAND_ORDER_V (QPT_RAND_BOOL_V ? ORDER_ASC : ORDER_DESC) +#define QPT_RAND_INT_V (taosRand() * (QPT_RAND_BOOL_V ? 1 : -1)) typedef struct { ENodeType type; @@ -170,27 +172,26 @@ typedef struct { SPhysiNode* pCurr; SPhysiNode* pChild; EOrder currTsOrder; -} SQPTBuildCtx; +} SQPTBuildPlanCtx; typedef struct { int32_t nodeLevel; bool onlyTag; int16_t nextBlockId; SDataBlockDescNode* pInputDataBlockDesc; -} SQPTMakePlanCtx; +} SQPTMakeNodeCtx; typedef struct { int32_t code; } SQPTExecResult; - typedef struct { - int32_t loopIdx; - SQPTParam param; - SQPTBuildCtx buildCtx; - SQPTMakePlanCtx makeCtx; - SQPTExecResult result; - int64_t startTsUs; + int32_t loopIdx; + SQPTParam param; + SQPTBuildPlanCtx buildCtx; + SQPTMakeNodeCtx makeCtx; + SQPTExecResult result; + int64_t startTsUs; } SQPTCtx; typedef struct { @@ -299,12 +300,208 @@ bool qptGetDynamicOp() { return QPT_RAND_BOOL_V; } -EOrder qptGetInputTsOrder() { - return qptCtx.buildCtx.currTsOrder; +EOrder qptGetCurrTsOrder() { + return qptCtx.param.correctExpected ? qptCtx.buildCtx.currTsOrder : QPT_RAND_ORDER_V; +} + +void qptGetRandValue(int32_t* pType, int32_t* pLen, void** ppVal) { + int32_t typeMax = TSDB_DATA_TYPE_MAX + if (!qptCtx.param.correctExpected) { + typeMax++; + } + + *pType = taosRand() % TSDB_DATA_TYPE_MAX; + switch (*pType) { + case TSDB_DATA_TYPE_NULL: + *pLen = 0; + if (ppVal) { + *ppVal = NULL; + } + break; + case TSDB_DATA_TYPE_BOOL: + *pLen = tDataTypes[*pType].bytes; + if (ppVal) { + *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); + assert(*ppVal); + *(bool*)*ppVal = QPT_RAND_BOOL_V; + } + break; + case TSDB_DATA_TYPE_TINYINT: + *pLen = tDataTypes[*pType].bytes; + if (ppVal) { + *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); + assert(*ppVal); + *(int8_t*)*ppVal = taosRand(); + } + break; + case TSDB_DATA_TYPE_SMALLINT: + *pLen = tDataTypes[*pType].bytes; + if (ppVal) { + *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); + assert(*ppVal); + *(int16_t*)*ppVal = taosRand(); + } + break; + case TSDB_DATA_TYPE_INT: + *pLen = tDataTypes[*pType].bytes; + if (ppVal) { + *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); + assert(*ppVal); + *(int32_t*)*ppVal = taosRand(); + } + break; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: + *pLen = tDataTypes[*pType].bytes; + if (ppVal) { + *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); + assert(*ppVal); + *(int64_t*)*ppVal = taosRand(); + } + break; + case TSDB_DATA_TYPE_FLOAT: + *pLen = tDataTypes[*pType].bytes; + if (ppVal) { + *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); + assert(*ppVal); + *(float*)*ppVal = taosRand(); + } + break; + case TSDB_DATA_TYPE_DOUBLE: + *pLen = tDataTypes[*pType].bytes; + if (ppVal) { + *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); + assert(*ppVal); + *(double*)*ppVal = taosRand(); + } + break; + case TSDB_DATA_TYPE_VARCHAR: + case TSDB_DATA_TYPE_GEOMETRY: + case TSDB_DATA_TYPE_JSON: + case TSDB_DATA_TYPE_VARBINARY: + case TSDB_DATA_TYPE_DECIMAL: + case TSDB_DATA_TYPE_BLOB: + case TSDB_DATA_TYPE_MEDIUMBLOB: + *pLen = taosRand() % QPT_MAX_STRING_LEN; + if (ppVal) { + *ppVal = taosMemoryCalloc(1, *pLen + VARSTR_HEADER_SIZE); + assert(*ppVal); + varDataSetLen(*ppVal, *pLen); + memset((char*)*ppVal + VARSTR_HEADER_SIZE, 'A' + taosRand() % 26, *pLen); + } + break; + case TSDB_DATA_TYPE_NCHAR: { + *pLen = taosRand() % QPT_MAX_STRING_LEN; + if (ppVal) { + char* pTmp = (char*)taosMemoryCalloc(1, *pLen + 1); + assert(pTmp); + memset(pTmp, 'A' + taosRand() % 26, *pLen); + *ppVal = taosMemoryCalloc(1, *pLen * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); + assert(*ppVal); + assert(taosMbsToUcs4(pTmp, *pLen, (TdUcs4 *)varDataVal(*ppVal), *pLen * TSDB_NCHAR_SIZE, NULL)); + *pLen *= TSDB_NCHAR_SIZE; + varDataSetLen(*ppVal, *pLen); + taosMemoryFree(pTmp); + } + break; + } + case TSDB_DATA_TYPE_UTINYINT: + *pLen = tDataTypes[*pType].bytes; + if (ppVal) { + *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); + assert(*ppVal); + *(uint8_t*)*ppVal = taosRand(); + } + break; + case TSDB_DATA_TYPE_USMALLINT: + *pLen = tDataTypes[*pType].bytes; + if (ppVal) { + *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); + assert(*ppVal); + *(uint16_t*)*ppVal = taosRand(); + } + break; + case TSDB_DATA_TYPE_UINT: + *pLen = tDataTypes[*pType].bytes; + if (ppVal) { + *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); + assert(*ppVal); + *(uint32_t*)*ppVal = taosRand(); + } + break; + case TSDB_DATA_TYPE_UBIGINT: + *pLen = tDataTypes[*pType].bytes; + if (ppVal) { + *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); + assert(*ppVal); + *(uint64_t*)*ppVal = taosRand(); + } + break; + default: + *pLen = taosRand(); + if (ppVal) { + *ppVal = taosMemoryCalloc(1, *pLen); + assert(*ppVal); + memset((char*)*ppVal, 'a' + taosRand() % 26, *pLen); + } + break; + } +} + +void qptFreeRandValue(int32_t* pType, void* pVal) { + switch (*pType) { + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: + case TSDB_DATA_TYPE_TIMESTAMP: + case TSDB_DATA_TYPE_UTINYINT: + case TSDB_DATA_TYPE_USMALLINT: + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_UBIGINT: + taosMemoryFree(pVal); + break; + case TSDB_DATA_TYPE_NULL: + case TSDB_DATA_TYPE_VARCHAR: + case TSDB_DATA_TYPE_GEOMETRY: + case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_JSON: + case TSDB_DATA_TYPE_VARBINARY: + case TSDB_DATA_TYPE_DECIMAL: + case TSDB_DATA_TYPE_BLOB: + case TSDB_DATA_TYPE_MEDIUMBLOB: + break; + default: + assert(0); + break; + } +} + +void qptGetRandRealTableType(int8_t* tableType) { + while (true) { + int8_t tType = taosRand() % TSDB_TABLE_MAX; + switch (tType) { + case TSDB_SUPER_TABLE: + case TSDB_CHILD_TABLE: + case TSDB_NORMAL_TABLE: + case TSDB_SYSTEM_TABLE: + *tableType = tType; + return; + default: + break; + } + } } SNode* qptMakeLimitNode() { + if (QPT_RAND_BOOL_V) { + return NULL; + } + SNode* pNode = NULL; assert(0 == nodesMakeNode(QUERY_NODE_LIMIT, &pNode)); assert(pNode); @@ -329,31 +526,58 @@ SNode* qptMakeLimitNode() { } SNode* qptMakeColumnNodeFromTable(int32_t colIdx, EColumnType colType, SScanPhysiNode* pScanPhysiNode) { + if (colIdx < 0) { + return NULL; + } + SColumnNode* pCol = NULL; assert(0 == nodesMakeNode(QUERY_NODE_COLUMN, (SNode**)&pCol)); assert(pCol); - pCol->node.resType.type = qptCtx.param.tbl.pCol[colIdx].type; - pCol->node.resType.bytes = qptCtx.param.tbl.pCol[colIdx].len; + if (qptCtx.param.correctExpected) { + pCol->node.resType.type = qptCtx.param.tbl.pCol[colIdx].type; + pCol->node.resType.bytes = qptCtx.param.tbl.pCol[colIdx].len; - pCol->tableId = qptCtx.param.tbl.uid; - pCol->tableType = qptCtx.param.tbl.tblType; - pCol->colId = colIdx; - pCol->projIdx = colIdx; - pCol->colType = qptCtx.param.tbl.pCol[colIdx].colType; - pCol->hasIndex = qptCtx.param.tbl.pCol[colIdx].hasIndex; - pCol->isPrimTs = qptCtx.param.tbl.pCol[colIdx].isPrimTs; - strcpy(pCol->dbName, qptCtx.param.db.dbName); - strcpy(pCol->tableName, qptCtx.param.tbl.tblName); - strcpy(pCol->tableAlias, qptCtx.param.tbl.tblAlias); - strcpy(pCol->colName, qptCtx.param.tbl.pCol[colIdx].name); - pCol->dataBlockId = pScanPhysiNode->node.pOutputDataBlockDesc->dataBlockId; - pCol->slotId = colIdx; - pCol->numOfPKs = qptCtx.param.tbl.pkNum; - pCol->tableHasPk = qptCtx.param.tbl.pkNum > 0; - pCol->isPk = qptCtx.param.tbl.pCol[colIdx].isPk; - pCol->projRefIdx = 0; - pCol->resIdx = 0; + pCol->tableId = qptCtx.param.tbl.uid; + pCol->tableType = qptCtx.param.tbl.tblType; + pCol->colId = colIdx; + pCol->projIdx = colIdx; + pCol->colType = qptCtx.param.tbl.pCol[colIdx].colType; + pCol->hasIndex = qptCtx.param.tbl.pCol[colIdx].hasIndex; + pCol->isPrimTs = qptCtx.param.tbl.pCol[colIdx].isPrimTs; + strcpy(pCol->dbName, qptCtx.param.db.dbName); + strcpy(pCol->tableName, qptCtx.param.tbl.tblName); + strcpy(pCol->tableAlias, qptCtx.param.tbl.tblAlias); + strcpy(pCol->colName, qptCtx.param.tbl.pCol[colIdx].name); + pCol->dataBlockId = pScanPhysiNode->node.pOutputDataBlockDesc->dataBlockId; + pCol->slotId = colIdx; + pCol->numOfPKs = qptCtx.param.tbl.pkNum; + pCol->tableHasPk = qptCtx.param.tbl.pkNum > 0; + pCol->isPk = qptCtx.param.tbl.pCol[colIdx].isPk; + pCol->projRefIdx = 0; + pCol->resIdx = 0; + } else { + qptGetRandValue(&pCol->node.resType.type, &pCol->node.resType.bytes, NULL); + + pCol->tableId = taosRand(); + pCol->tableType = taosRand() % TSDB_TABLE_MAX; + pCol->colId = QPT_RAND_BOOL_V ? taosRand() : colIdx; + pCol->projIdx = taosRand(); + pCol->colType = QPT_RAND_BOOL_V ? qptCtx.param.tbl.pCol[colIdx].colType :taosRand() % (COLUMN_TYPE_GROUP_KEY + 1); + pCol->hasIndex = QPT_RAND_BOOL_V; + pCol->isPrimTs = QPT_RAND_BOOL_V; + QPT_RAND_BOOL_V ? (pCol->dbName[0] = 0) : strcpy(pCol->dbName, qptCtx.param.db.dbName); + QPT_RAND_BOOL_V ? (pCol->tableName[0] = 0) : strcpy(pCol->tableName, qptCtx.param.tbl.tblName); + QPT_RAND_BOOL_V ? (pCol->tableAlias[0] = 0) : strcpy(pCol->tableAlias, qptCtx.param.tbl.tblAlias); + QPT_RAND_BOOL_V ? (pCol->colName[0] = 0) : strcpy(pCol->colName, qptCtx.param.tbl.pCol[colIdx].name); + pCol->dataBlockId = QPT_RAND_BOOL_V ? taosRand() : pScanPhysiNode->node.pOutputDataBlockDesc->dataBlockId; + pCol->slotId = QPT_RAND_BOOL_V ? taosRand() : colIdx; + pCol->numOfPKs = QPT_RAND_BOOL_V ? taosRand() : qptCtx.param.tbl.pkNum; + pCol->tableHasPk = QPT_RAND_BOOL_V ? QPT_RAND_BOOL_V : (qptCtx.param.tbl.pkNum > 0); + pCol->isPk = QPT_RAND_BOOL_V ? QPT_RAND_BOOL_V : qptCtx.param.tbl.pCol[colIdx].isPk; + pCol->projRefIdx = taosRand(); + pCol->resIdx = taosRand(); + } return (SNode*)pCol; } @@ -461,143 +685,7 @@ void qptMakeColumnNode(SNode** ppNode) { *ppNode = (SNode*)pCol; } -void qptGetRandValue(int32_t* pType, int32_t* pLen, void** ppVal) { - *pType = taosRand() % TSDB_DATA_TYPE_MAX; - switch (*pType) { - case TSDB_DATA_TYPE_NULL: - *pLen = 0; - *ppVal = NULL; - break; - case TSDB_DATA_TYPE_BOOL: - *pLen = tDataTypes[*pType].bytes; - *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); - assert(*ppVal); - *(bool*)*ppVal = QPT_RAND_BOOL_V; - break; - case TSDB_DATA_TYPE_TINYINT: - *pLen = tDataTypes[*pType].bytes; - *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); - assert(*ppVal); - *(int8_t*)*ppVal = taosRand(); - break; - case TSDB_DATA_TYPE_SMALLINT: - *pLen = tDataTypes[*pType].bytes; - *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); - assert(*ppVal); - *(int16_t*)*ppVal = taosRand(); - break; - case TSDB_DATA_TYPE_INT: - *pLen = tDataTypes[*pType].bytes; - *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); - assert(*ppVal); - *(int32_t*)*ppVal = taosRand(); - break; - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_TIMESTAMP: - *pLen = tDataTypes[*pType].bytes; - *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); - assert(*ppVal); - *(int64_t*)*ppVal = taosRand(); - break; - case TSDB_DATA_TYPE_FLOAT: - *pLen = tDataTypes[*pType].bytes; - *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); - assert(*ppVal); - *(float*)*ppVal = taosRand(); - break; - case TSDB_DATA_TYPE_DOUBLE: - *pLen = tDataTypes[*pType].bytes; - *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); - assert(*ppVal); - *(double*)*ppVal = taosRand(); - break; - case TSDB_DATA_TYPE_VARCHAR: - case TSDB_DATA_TYPE_GEOMETRY: - case TSDB_DATA_TYPE_JSON: - case TSDB_DATA_TYPE_VARBINARY: - case TSDB_DATA_TYPE_DECIMAL: - case TSDB_DATA_TYPE_BLOB: - case TSDB_DATA_TYPE_MEDIUMBLOB: - *pLen = taosRand() % QPT_MAX_STRING_LEN; - *ppVal = taosMemoryCalloc(1, *pLen + VARSTR_HEADER_SIZE); - assert(*ppVal); - varDataSetLen(*ppVal, *pLen); - memset((char*)*ppVal + VARSTR_HEADER_SIZE, 'A' + taosRand() % 26, *pLen); - break; - case TSDB_DATA_TYPE_NCHAR: { - *pLen = taosRand() % QPT_MAX_STRING_LEN; - char* pTmp = (char*)taosMemoryCalloc(1, *pLen + 1); - assert(pTmp); - memset(pTmp, 'A' + taosRand() % 26, *pLen); - *ppVal = taosMemoryCalloc(1, *pLen * TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE); - assert(*ppVal); - assert(taosMbsToUcs4(pTmp, *pLen, (TdUcs4 *)varDataVal(*ppVal), *pLen * TSDB_NCHAR_SIZE, NULL)); - *pLen *= TSDB_NCHAR_SIZE; - varDataSetLen(*ppVal, *pLen); - taosMemoryFree(pTmp); - break; - } - case TSDB_DATA_TYPE_UTINYINT: - *pLen = tDataTypes[*pType].bytes; - *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); - assert(*ppVal); - *(uint8_t*)*ppVal = taosRand(); - break; - case TSDB_DATA_TYPE_USMALLINT: - *pLen = tDataTypes[*pType].bytes; - *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); - assert(*ppVal); - *(uint16_t*)*ppVal = taosRand(); - break; - case TSDB_DATA_TYPE_UINT: - *pLen = tDataTypes[*pType].bytes; - *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); - assert(*ppVal); - *(uint32_t*)*ppVal = taosRand(); - break; - case TSDB_DATA_TYPE_UBIGINT: - *pLen = tDataTypes[*pType].bytes; - *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); - assert(*ppVal); - *(uint64_t*)*ppVal = taosRand(); - break; - default: - assert(0); - break; - } -} -void qptFreeRandValue(int32_t* pType, void* pVal) { - switch (*pType) { - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_FLOAT: - case TSDB_DATA_TYPE_DOUBLE: - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_UTINYINT: - case TSDB_DATA_TYPE_USMALLINT: - case TSDB_DATA_TYPE_UINT: - case TSDB_DATA_TYPE_UBIGINT: - taosMemoryFree(pVal); - break; - case TSDB_DATA_TYPE_NULL: - case TSDB_DATA_TYPE_VARCHAR: - case TSDB_DATA_TYPE_GEOMETRY: - case TSDB_DATA_TYPE_NCHAR: - case TSDB_DATA_TYPE_JSON: - case TSDB_DATA_TYPE_VARBINARY: - case TSDB_DATA_TYPE_DECIMAL: - case TSDB_DATA_TYPE_BLOB: - case TSDB_DATA_TYPE_MEDIUMBLOB: - break; - default: - assert(0); - break; - } -} void qptMakeValueNode(SNode** ppNode) { SValueNode* pVal = NULL; @@ -689,11 +777,16 @@ void qptMakeTableNode(SNode** ppNode) { } -void qptMakeExprNode(SNode** ppNode) { +SNode* qptMakeExprNode(SNode** ppNode) { int32_t nodeTypeMaxValue = 9; if (qptCtx.makeCtx.nodeLevel >= QPT_MAX_NODE_LEVEL) { nodeTypeMaxValue = 2; } + + SNode* pNode = NULL; + if (NULL == ppNode) { + ppNode = &pNode; + } switch (taosRand() % nodeTypeMaxValue) { case 0: @@ -727,6 +820,8 @@ void qptMakeExprNode(SNode** ppNode) { assert(0); break; } + + return *ppNode; } void qptResetMakeNodeCtx(SDataBlockDescNode* pInput, bool onlyTag) { @@ -745,10 +840,15 @@ SNode* qptMakeConditionNode(bool onlyTag) { } SNode* qptMakeDataBlockDescNode() { + if (QPT_RAND_BOOL_V) { + return NULL; + } + SDataBlockDescNode* pDesc = NULL; assert(0 == nodesMakeNode(QUERY_NODE_DATABLOCK_DESC, (SNode**)&pDesc)); - pDesc->dataBlockId = qptCtx.makeCtx.nextBlockId++; - pDesc->precision = qptCtx.param.db.precision; + + pDesc->dataBlockId = qptCtx.param.correctExpected ? qptCtx.makeCtx.nextBlockId++ : QPT_RAND_INT_V; + pDesc->precision = qptCtx.param.correctExpected ? qptCtx.param.db.precision : QPT_RAND_INT_V; return (SNode*)pDesc; } @@ -763,16 +863,15 @@ SPhysiNode* qptCreatePhysiNode(int32_t nodeType) { pPhysiNode->pLimit = qptMakeLimitNode(); pPhysiNode->pSlimit = qptMakeLimitNode(); pPhysiNode->dynamicOp = qptGetDynamicOp(); - pPhysiNode->inputTsOrder = qptGetInputTsOrder(); + pPhysiNode->inputTsOrder = qptGetCurrTsOrder(); pPhysiNode->pOutputDataBlockDesc = (SDataBlockDescNode*)qptMakeDataBlockDescNode(); - assert(pPhysiNode->pOutputDataBlockDesc); return pPhysiNode; } void qptPostCreatePhysiNode(SPhysiNode* pPhysiNode) { - pPhysiNode->outputTsOrder = qptGetInputTsOrder(); + pPhysiNode->outputTsOrder = qptGetCurrTsOrder(); if (taosRand() % 2) { pPhysiNode->pConditions = qptMakeConditionNode(false); @@ -800,18 +899,56 @@ void qptMarkTableInUseCols(int32_t colNum, int32_t totalColNum, SQPTCol* pCol) { } while (colInUse < colNum); } +int32_t qptNodesListAppend(SNodeList* pList, SNode* pNode) { + SListCell* p = NULL; + assert(0 == nodesCalloc(1, sizeof(SListCell), (void**)&p)); + + p->pNode = pNode; + if (NULL == pList->pHead) { + pList->pHead = p; + } + if (NULL != pList->pTail) { + pList->pTail->pNext = p; + } + p->pPrev = pList->pTail; + pList->pTail = p; + ++(pList->length); + return TSDB_CODE_SUCCESS; +} + + +int32_t qptNodesListStrictAppend(SNodeList* pList, SNode* pNode) { + int32_t code = qptNodesListAppend(pList, pNode); + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyNode(pNode); + } + return code; +} + +int32_t qptNodesListMakeStrictAppend(SNodeList** pList, SNode* pNode) { + if (NULL == *pList) { + int32_t code = nodesMakeList(pList); + if (NULL == *pList) { + return code; + } + } + return qptNodesListStrictAppend(*pList, pNode); +} + + void qptCreateTableScanColsImpl( SScanPhysiNode* pScanPhysiNode, SNodeList** ppCols, int32_t totalColNum, SQPTCol* pCol) { int32_t colNum = qptCtx.param.correctExpected ? (taosRand() % totalColNum + 1) : (taosRand()); int32_t colAdded = 0; if (qptCtx.param.correctExpected) { qptMarkTableInUseCols(colNum, totalColNum, pCol); + for (int32_t i = 0; i < totalColNum && colAdded < colNum; ++i) { if (0 == pCol[i].inUse) { continue; } - assert(0 == nodesListMakeStrictAppend(ppCols, qptMakeColumnNodeFromTable(i, pCol[i].colType, pScanPhysiNode))); + assert(0 == qptNodesListMakeStrictAppend(ppCols, qptMakeColumnNodeFromTable(i, pCol[i].colType, pScanPhysiNode))); } return; @@ -821,7 +958,7 @@ void qptCreateTableScanColsImpl( SScanPhysiNode* pScanPhysiNode, SNodeList int32_t colIdx = taosRand(); colIdx = (colIdx >= totalColNum) ? -1 : colIdx; - assert(0 == nodesListMakeStrictAppend(ppCols, qptMakeColumnNodeFromTable(colIdx, pCol[i].colType, pScanPhysiNode))); + assert(0 == qptNodesListMakeStrictAppend(ppCols, qptMakeColumnNodeFromTable(colIdx, pCol[i].colType, pScanPhysiNode))); } } @@ -837,7 +974,8 @@ void qptCreateTableScanPseudoCols( SScanPhysiNode* pScanPhysiNode) { SNode* qptMakeSlotDescNode(const char* pName, const SNode* pNode, int16_t slotId, bool output, bool reserve) { SSlotDescNode* pSlot = NULL; assert(0 == nodesMakeNode(QUERY_NODE_SLOT_DESC, (SNode**)&pSlot)); - snprintf(pSlot->name, sizeof(pSlot->name), "%s", pName); + + QPT_RAND_BOOL_V ? (pSlot->name[0] = 0) : snprintf(pSlot->name, sizeof(pSlot->name), "%s", pName); pSlot->slotId = slotId; pSlot->dataType = ((SExprNode*)pNode)->resType; pSlot->reserve = reserve; @@ -845,7 +983,7 @@ SNode* qptMakeSlotDescNode(const char* pName, const SNode* pNode, int16_t slotId return (SNode*)pSlot; } -void qptCreateMakeNode(SNode* pNode, int16_t dataBlockId, int16_t slotId, SNode** pOutput) { +void qptMakeTargetNode(SNode* pNode, int16_t dataBlockId, int16_t slotId, SNode** pOutput) { STargetNode* pTarget = NULL; assert(0 == nodesMakeNode(QUERY_NODE_TARGET, (SNode**)&pTarget)); @@ -863,18 +1001,23 @@ void qptAddDataBlockSlots(SNodeList* pList, SDataBlockDescNode* pDataBlockDesc) FOREACH(pNode, pList) { SNode* pExpr = QUERY_NODE_ORDER_BY_EXPR == nodeType(pNode) ? ((SOrderByExprNode*)pNode)->pExpr : pNode; - assert(0 == nodesListMakeStrictAppend(&pDataBlockDesc->pSlots, qptMakeSlotDescNode(NULL, pExpr, nextSlotId, output, QPT_RAND_BOOL_V))); - pDataBlockDesc->totalRowSize += ((SExprNode*)pExpr)->resType.bytes; - if (output) { - pDataBlockDesc->outputRowSize += ((SExprNode*)pExpr)->resType.bytes; + if (qptCtx.param.correctExpected || QPT_RAND_BOOL_V) { + SNode* pDesc = qptCtx.param.correctExpected ? qptMakeSlotDescNode(NULL, pExpr, nextSlotId, output, QPT_RAND_BOOL_V) : qptMakeExprNode(NULL); + assert(0 == qptNodesListMakeStrictAppend(&pDataBlockDesc->pSlots, pDesc)); + pDataBlockDesc->totalRowSize += qptCtx.param.correctExpected ? ((SExprNode*)pExpr)->resType.bytes : taosRand(); + if (output && QPT_RAND_BOOL_V) { + pDataBlockDesc->outputRowSize += qptCtx.param.correctExpected ? ((SExprNode*)pExpr)->resType.bytes : taosRand(); + } } slotId = nextSlotId; ++nextSlotId; - SNode* pTarget = NULL; - qptCreateMakeNode(pNode, pDataBlockDesc->dataBlockId, slotId, &pTarget); - REPLACE_NODE(pTarget); + if (qptCtx.param.correctExpected || QPT_RAND_BOOL_V) { + SNode* pTarget = NULL; + qptMakeTargetNode(pNode, pDataBlockDesc->dataBlockId, slotId, &pTarget); + REPLACE_NODE(pTarget); + } } } @@ -898,13 +1041,14 @@ void qptCreateScanPhysiNodeImpl( SScanPhysiNode* pScanPhysiNode) { SName tblName = {0}; toName(1, qptCtx.param.db.dbName, qptCtx.param.tbl.tblName, &tblName); memcpy(&pScanPhysiNode->tableName, &tblName, sizeof(SName)); + + qptCtx.buildCtx.currTsOrder = (qptCtx.param.correctExpected) ? qptCtx.buildCtx.currTsOrder : QPT_RAND_ORDER_V; } SNode* qptCreateTagScanPhysiNode(int32_t nodeType) { SPhysiNode* pPhysiNode = qptCreatePhysiNode(nodeType); - assert(pPhysiNode); STagScanPhysiNode* pTagScanNode = (STagScanPhysiNode*)pPhysiNode; pTagScanNode->onlyMetaCtbIdx = (taosRand() % 2) ? true : false; @@ -1226,7 +1370,7 @@ void qptInitTestCtx(bool correctExpected, bool singleNode, int32_t nodeType, int qptCtx.param.tbl.uid = 100; qptCtx.param.tbl.suid = 1; - qptCtx.param.tbl.tblType = taosRand() % TSDB_TABLE_MAX; + qptGetRandRealTableType(qptCtx.param.tbl.tblType); qptCtx.param.tbl.colNum = taosRand() % 4098; qptCtx.param.tbl.tagNum = taosRand() % 130; qptCtx.param.tbl.pkNum = taosRand() % 2; @@ -1247,6 +1391,20 @@ void qptInitTestCtx(bool correctExpected, bool singleNode, int32_t nodeType, int #if 1 #if 1 +TEST(randSingleNodeTest, tagScan) { + char* caseName = "randSingleNodeTest:tagScan"; + + qptInitTestCtx(false, true, QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN, 0, NULL); + + for (qptCtx.loopIdx = 0; qptCtx.loopIdx < QPT_MAX_LOOP; ++qptCtx.loopIdx) { + qptRunPlanTest(caseName); + } + + qptPrintStatInfo(caseName); +} +#endif + +#if 0 TEST(correctSingleNodeTest, tagScan) { char* caseName = "correctSingleNodeTest:tagScan"; @@ -1260,19 +1418,6 @@ TEST(correctSingleNodeTest, tagScan) { } #endif -#if 0 -TEST(randSingleNodeTest, tagScan) { - char* caseName = "randSingleNodeTest:tagScan"; - - qptInitTestCtx(false, true, QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN, 0, NULL); - - for (qptCtx.loopIdx = 0; qptCtx.loopIdx < QPT_MAX_LOOP; ++qptCtx.loopIdx) { - qptRunSingleTest(caseName); - } - - printStatInfo(caseName); -} -#endif #endif From 1143adce33a84db60b3973fb499eb45410ac5ffd Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 30 Sep 2024 09:54:37 +0800 Subject: [PATCH 05/29] fix: add random case --- source/libs/executor/test/queryPlanTests.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/source/libs/executor/test/queryPlanTests.cpp b/source/libs/executor/test/queryPlanTests.cpp index 37d0a79761..dcd615b8c3 100755 --- a/source/libs/executor/test/queryPlanTests.cpp +++ b/source/libs/executor/test/queryPlanTests.cpp @@ -976,8 +976,17 @@ SNode* qptMakeSlotDescNode(const char* pName, const SNode* pNode, int16_t slotId assert(0 == nodesMakeNode(QUERY_NODE_SLOT_DESC, (SNode**)&pSlot)); QPT_RAND_BOOL_V ? (pSlot->name[0] = 0) : snprintf(pSlot->name, sizeof(pSlot->name), "%s", pName); +<<<<<<< Updated upstream pSlot->slotId = slotId; pSlot->dataType = ((SExprNode*)pNode)->resType; +======= + pSlot->slotId = qptCtx.param.correctExpected ? slotId : taosRand(); + if (qptCtx.param.correctExpected) { + pSlot->dataType = ((SExprNode*)pNode)->resType; + } else { + + } +>>>>>>> Stashed changes pSlot->reserve = reserve; pSlot->output = output; return (SNode*)pSlot; From 1b6bb5e4b9b3db4fa86657de0bc9ed8c6ce7ed1c Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 30 Sep 2024 16:30:47 +0800 Subject: [PATCH 06/29] fix: random test issues --- source/libs/executor/test/queryPlanTests.cpp | 491 ++++++++++++++----- 1 file changed, 362 insertions(+), 129 deletions(-) diff --git a/source/libs/executor/test/queryPlanTests.cpp b/source/libs/executor/test/queryPlanTests.cpp index dcd615b8c3..4c75ed4d7c 100755 --- a/source/libs/executor/test/queryPlanTests.cpp +++ b/source/libs/executor/test/queryPlanTests.cpp @@ -45,7 +45,7 @@ namespace { -#define QPT_MAX_LOOP 1000 +#define QPT_MAX_LOOP 100000000 #define QPT_MAX_SUBPLAN_LEVEL 1000 #define QPT_MAX_WHEN_THEN_NUM 10 #define QPT_MAX_NODE_LEVEL 5 @@ -54,6 +54,8 @@ namespace { #define QPT_MAX_LOGIC_PARAM 5 #define QPT_MAX_NODE_LIST_NUM 5 #define QPT_DEFAULT_VNODE_NUM 5 +#define QPT_MAX_COLUMN_NUM 8192 + int32_t QPT_PHYSIC_NODE_LIST[] = { QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN, @@ -110,6 +112,11 @@ int32_t QPT_PHYSIC_NODE_LIST[] = { #define QPT_RAND_BOOL_V ((taosRand() % 2) ? true : false) #define QPT_RAND_ORDER_V (QPT_RAND_BOOL_V ? ORDER_ASC : ORDER_DESC) #define QPT_RAND_INT_V (taosRand() * (QPT_RAND_BOOL_V ? 1 : -1)) +#define QPT_LOW_PROB() ((taosRand() % 11) == 0) +#define QPT_MID_PROB() ((taosRand() % 11) <= 4) +#define QPT_HIGH_PROB() ((taosRand() % 11) <= 7) + +#define QPT_CORRECT_HIGH_PROB() (qptCtx.param.correctExpected || QPT_HIGH_PROB()) typedef struct { ENodeType type; @@ -208,7 +215,7 @@ SQPTCtrl qptCtrl = {1, 0, 0, 0, 0}; bool qptErrorRerun = false; bool qptInRerun = false; -void qptMakeExprNode(SNode** ppNode); +SNode* qptMakeExprNode(SNode** ppNode); void qptPrintBeginInfo(char* caseName) { @@ -304,22 +311,22 @@ EOrder qptGetCurrTsOrder() { return qptCtx.param.correctExpected ? qptCtx.buildCtx.currTsOrder : QPT_RAND_ORDER_V; } -void qptGetRandValue(int32_t* pType, int32_t* pLen, void** ppVal) { - int32_t typeMax = TSDB_DATA_TYPE_MAX +void qptGetRandValue(uint8_t* pType, int32_t* pLen, void** ppVal) { + int32_t typeMax = TSDB_DATA_TYPE_MAX; if (!qptCtx.param.correctExpected) { typeMax++; } - *pType = taosRand() % TSDB_DATA_TYPE_MAX; + *pType = taosRand() % typeMax; switch (*pType) { case TSDB_DATA_TYPE_NULL: - *pLen = 0; + *pLen = qptCtx.param.correctExpected ? 0 : taosRand(); if (ppVal) { *ppVal = NULL; } break; case TSDB_DATA_TYPE_BOOL: - *pLen = tDataTypes[*pType].bytes; + *pLen = qptCtx.param.correctExpected ? tDataTypes[*pType].bytes : taosRand(); if (ppVal) { *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); assert(*ppVal); @@ -327,7 +334,7 @@ void qptGetRandValue(int32_t* pType, int32_t* pLen, void** ppVal) { } break; case TSDB_DATA_TYPE_TINYINT: - *pLen = tDataTypes[*pType].bytes; + *pLen = qptCtx.param.correctExpected ? tDataTypes[*pType].bytes : taosRand(); if (ppVal) { *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); assert(*ppVal); @@ -335,7 +342,7 @@ void qptGetRandValue(int32_t* pType, int32_t* pLen, void** ppVal) { } break; case TSDB_DATA_TYPE_SMALLINT: - *pLen = tDataTypes[*pType].bytes; + *pLen = qptCtx.param.correctExpected ? tDataTypes[*pType].bytes : taosRand(); if (ppVal) { *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); assert(*ppVal); @@ -343,7 +350,7 @@ void qptGetRandValue(int32_t* pType, int32_t* pLen, void** ppVal) { } break; case TSDB_DATA_TYPE_INT: - *pLen = tDataTypes[*pType].bytes; + *pLen = qptCtx.param.correctExpected ? tDataTypes[*pType].bytes : taosRand(); if (ppVal) { *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); assert(*ppVal); @@ -352,7 +359,7 @@ void qptGetRandValue(int32_t* pType, int32_t* pLen, void** ppVal) { break; case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_TIMESTAMP: - *pLen = tDataTypes[*pType].bytes; + *pLen = qptCtx.param.correctExpected ? tDataTypes[*pType].bytes : taosRand(); if (ppVal) { *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); assert(*ppVal); @@ -360,7 +367,7 @@ void qptGetRandValue(int32_t* pType, int32_t* pLen, void** ppVal) { } break; case TSDB_DATA_TYPE_FLOAT: - *pLen = tDataTypes[*pType].bytes; + *pLen = qptCtx.param.correctExpected ? tDataTypes[*pType].bytes : taosRand(); if (ppVal) { *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); assert(*ppVal); @@ -368,7 +375,7 @@ void qptGetRandValue(int32_t* pType, int32_t* pLen, void** ppVal) { } break; case TSDB_DATA_TYPE_DOUBLE: - *pLen = tDataTypes[*pType].bytes; + *pLen = qptCtx.param.correctExpected ? tDataTypes[*pType].bytes : taosRand(); if (ppVal) { *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); assert(*ppVal); @@ -406,7 +413,7 @@ void qptGetRandValue(int32_t* pType, int32_t* pLen, void** ppVal) { break; } case TSDB_DATA_TYPE_UTINYINT: - *pLen = tDataTypes[*pType].bytes; + *pLen = qptCtx.param.correctExpected ? tDataTypes[*pType].bytes : taosRand(); if (ppVal) { *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); assert(*ppVal); @@ -414,7 +421,7 @@ void qptGetRandValue(int32_t* pType, int32_t* pLen, void** ppVal) { } break; case TSDB_DATA_TYPE_USMALLINT: - *pLen = tDataTypes[*pType].bytes; + *pLen = qptCtx.param.correctExpected ? tDataTypes[*pType].bytes : taosRand(); if (ppVal) { *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); assert(*ppVal); @@ -422,7 +429,7 @@ void qptGetRandValue(int32_t* pType, int32_t* pLen, void** ppVal) { } break; case TSDB_DATA_TYPE_UINT: - *pLen = tDataTypes[*pType].bytes; + *pLen = qptCtx.param.correctExpected ? tDataTypes[*pType].bytes : taosRand(); if (ppVal) { *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); assert(*ppVal); @@ -430,7 +437,7 @@ void qptGetRandValue(int32_t* pType, int32_t* pLen, void** ppVal) { } break; case TSDB_DATA_TYPE_UBIGINT: - *pLen = tDataTypes[*pType].bytes; + *pLen = qptCtx.param.correctExpected ? tDataTypes[*pType].bytes : taosRand(); if (ppVal) { *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); assert(*ppVal); @@ -438,7 +445,7 @@ void qptGetRandValue(int32_t* pType, int32_t* pLen, void** ppVal) { } break; default: - *pLen = taosRand(); + *pLen = taosRand() % QPT_MAX_STRING_LEN; if (ppVal) { *ppVal = taosMemoryCalloc(1, *pLen); assert(*ppVal); @@ -497,6 +504,51 @@ void qptGetRandRealTableType(int8_t* tableType) { } + +void qptNodesCalloc(int32_t num, int32_t size, void** pOut) { + void* p = taosMemoryCalloc(num, size); + assert(p); + *(char*)p = 0; + *pOut = (char*)p + 1; +} + +int32_t qptNodesListAppend(SNodeList* pList, SNode* pNode) { + SListCell* p = NULL; + qptNodesCalloc(1, sizeof(SListCell), (void**)&p); + + p->pNode = pNode; + if (NULL == pList->pHead) { + pList->pHead = p; + } + if (NULL != pList->pTail) { + pList->pTail->pNext = p; + } + p->pPrev = pList->pTail; + pList->pTail = p; + ++(pList->length); + return TSDB_CODE_SUCCESS; +} + + +int32_t qptNodesListStrictAppend(SNodeList* pList, SNode* pNode) { + int32_t code = qptNodesListAppend(pList, pNode); + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyNode(pNode); + } + return code; +} + +int32_t qptNodesListMakeStrictAppend(SNodeList** pList, SNode* pNode) { + if (NULL == *pList) { + int32_t code = nodesMakeList(pList); + if (NULL == *pList) { + return code; + } + } + return qptNodesListStrictAppend(*pList, pNode); +} + + SNode* qptMakeLimitNode() { if (QPT_RAND_BOOL_V) { return NULL; @@ -534,7 +586,7 @@ SNode* qptMakeColumnNodeFromTable(int32_t colIdx, EColumnType colType, SScanPhys assert(0 == nodesMakeNode(QUERY_NODE_COLUMN, (SNode**)&pCol)); assert(pCol); - if (qptCtx.param.correctExpected) { + if (QPT_CORRECT_HIGH_PROB()) { pCol->node.resType.type = qptCtx.param.tbl.pCol[colIdx].type; pCol->node.resType.bytes = qptCtx.param.tbl.pCol[colIdx].len; @@ -549,7 +601,7 @@ SNode* qptMakeColumnNodeFromTable(int32_t colIdx, EColumnType colType, SScanPhys strcpy(pCol->tableName, qptCtx.param.tbl.tblName); strcpy(pCol->tableAlias, qptCtx.param.tbl.tblAlias); strcpy(pCol->colName, qptCtx.param.tbl.pCol[colIdx].name); - pCol->dataBlockId = pScanPhysiNode->node.pOutputDataBlockDesc->dataBlockId; + pCol->dataBlockId = pScanPhysiNode->node.pOutputDataBlockDesc ? pScanPhysiNode->node.pOutputDataBlockDesc->dataBlockId : taosRand(); pCol->slotId = colIdx; pCol->numOfPKs = qptCtx.param.tbl.pkNum; pCol->tableHasPk = qptCtx.param.tbl.pkNum > 0; @@ -563,14 +615,31 @@ SNode* qptMakeColumnNodeFromTable(int32_t colIdx, EColumnType colType, SScanPhys pCol->tableType = taosRand() % TSDB_TABLE_MAX; pCol->colId = QPT_RAND_BOOL_V ? taosRand() : colIdx; pCol->projIdx = taosRand(); - pCol->colType = QPT_RAND_BOOL_V ? qptCtx.param.tbl.pCol[colIdx].colType :taosRand() % (COLUMN_TYPE_GROUP_KEY + 1); + pCol->colType = QPT_RAND_BOOL_V ? qptCtx.param.tbl.pCol[colIdx].colType : (EColumnType)(taosRand() % (COLUMN_TYPE_GROUP_KEY + 1)); pCol->hasIndex = QPT_RAND_BOOL_V; pCol->isPrimTs = QPT_RAND_BOOL_V; - QPT_RAND_BOOL_V ? (pCol->dbName[0] = 0) : strcpy(pCol->dbName, qptCtx.param.db.dbName); - QPT_RAND_BOOL_V ? (pCol->tableName[0] = 0) : strcpy(pCol->tableName, qptCtx.param.tbl.tblName); - QPT_RAND_BOOL_V ? (pCol->tableAlias[0] = 0) : strcpy(pCol->tableAlias, qptCtx.param.tbl.tblAlias); - QPT_RAND_BOOL_V ? (pCol->colName[0] = 0) : strcpy(pCol->colName, qptCtx.param.tbl.pCol[colIdx].name); - pCol->dataBlockId = QPT_RAND_BOOL_V ? taosRand() : pScanPhysiNode->node.pOutputDataBlockDesc->dataBlockId; + if (QPT_RAND_BOOL_V) { + pCol->dbName[0] = 0; + } else { + strcpy(pCol->dbName, qptCtx.param.db.dbName); + } + if (QPT_RAND_BOOL_V) { + pCol->tableName[0] = 0; + } else { + strcpy(pCol->tableName, qptCtx.param.tbl.tblName); + } + if (QPT_RAND_BOOL_V) { + pCol->tableAlias[0] = 0; + } else { + strcpy(pCol->tableAlias, qptCtx.param.tbl.tblAlias); + } + if (QPT_RAND_BOOL_V) { + pCol->colName[0] = 0; + } else { + strcpy(pCol->colName, qptCtx.param.tbl.pCol[colIdx].name); + } + + pCol->dataBlockId = (QPT_RAND_BOOL_V && pScanPhysiNode->node.pOutputDataBlockDesc) ? pScanPhysiNode->node.pOutputDataBlockDesc->dataBlockId : taosRand(); pCol->slotId = QPT_RAND_BOOL_V ? taosRand() : colIdx; pCol->numOfPKs = QPT_RAND_BOOL_V ? taosRand() : qptCtx.param.tbl.pkNum; pCol->tableHasPk = QPT_RAND_BOOL_V ? QPT_RAND_BOOL_V : (qptCtx.param.tbl.pkNum > 0); @@ -589,43 +658,45 @@ void qptMakeWhenThenNode(SNode** ppNode) { SWhenThenNode* pWhenThen = (SWhenThenNode*)*ppNode; qptMakeExprNode(&pWhenThen->pWhen); - assert(pWhenThen->pWhen); qptMakeExprNode(&pWhenThen->pThen); - assert(pWhenThen->pThen); } void qptMakeCaseWhenNode(SNode** ppNode) { assert(0 == nodesMakeNode(QUERY_NODE_CASE_WHEN, ppNode)); assert(*ppNode); + SCaseWhenNode* pCaseWhen = (SCaseWhenNode*)*ppNode; qptCtx.makeCtx.nodeLevel++; - qptMakeExprNode(&pCaseWhen->pCase); - assert(pCaseWhen->pCase); - - qptMakeExprNode(&pCaseWhen->pElse); - assert(pCaseWhen->pElse); - - int32_t whenNum = taosRand() % QPT_MAX_WHEN_THEN_NUM + 1; + if (QPT_RAND_BOOL_V) { + qptMakeExprNode(&pCaseWhen->pCase); + } + + if (QPT_RAND_BOOL_V) { + qptMakeExprNode(&pCaseWhen->pElse); + } + + int32_t whenNum = taosRand() % QPT_MAX_WHEN_THEN_NUM + (QPT_CORRECT_HIGH_PROB() ? 1 : 0); for (int32_t i = 0; i < whenNum; ++i) { SNode* pNode = NULL; qptMakeWhenThenNode(&pNode); - assert(0 == nodesListMakeStrictAppend(&pCaseWhen->pWhenThenList, pNode)); + qptNodesListMakeStrictAppend(&pCaseWhen->pWhenThenList, pNode); } } void qptMakeOperatorNode(SNode** ppNode) { - EOperatorType opType; - opType = OPERATOR_ARRAY[taosRand() % (sizeof(OPERATOR_ARRAY)/sizeof(OPERATOR_ARRAY[0]))]; + EOperatorType opType = OPERATOR_ARRAY[taosRand() % (sizeof(OPERATOR_ARRAY)/sizeof(OPERATOR_ARRAY[0]))]; assert(0 == nodesMakeNode(QUERY_NODE_OPERATOR, ppNode)); + SOperatorNode* pOp = (SOperatorNode*)*ppNode; - pOp->opType = opType; + pOp->opType = QPT_CORRECT_HIGH_PROB() ? opType : (EOperatorType)(opType + 1); qptCtx.makeCtx.nodeLevel++; + switch (opType) { case OP_TYPE_ADD: case OP_TYPE_SUB: @@ -649,8 +720,17 @@ void qptMakeOperatorNode(SNode** ppNode) { case OP_TYPE_JSON_GET_VALUE: case OP_TYPE_JSON_CONTAINS: case OP_TYPE_ASSIGN: - qptMakeExprNode(&pOp->pLeft); - qptMakeExprNode(&pOp->pRight); + if (QPT_CORRECT_HIGH_PROB()) { + qptMakeExprNode(&pOp->pLeft); + qptMakeExprNode(&pOp->pRight); + } else { + if (QPT_RAND_BOOL_V) { + qptMakeExprNode(&pOp->pLeft); + } + if (QPT_RAND_BOOL_V) { + qptMakeExprNode(&pOp->pRight); + } + } break; case OP_TYPE_IS_NULL: @@ -662,42 +742,154 @@ void qptMakeOperatorNode(SNode** ppNode) { case OP_TYPE_IS_NOT_FALSE: case OP_TYPE_IS_NOT_UNKNOWN: case OP_TYPE_MINUS: - qptMakeExprNode(&pOp->pLeft); + if (QPT_CORRECT_HIGH_PROB()) { + qptMakeExprNode(&pOp->pLeft); + } else { + if (QPT_RAND_BOOL_V) { + qptMakeExprNode(&pOp->pLeft); + } + if (QPT_RAND_BOOL_V) { + qptMakeExprNode(&pOp->pRight); + } + } break; default: - assert(0); + if (QPT_RAND_BOOL_V) { + qptMakeExprNode(&pOp->pLeft); + } + if (QPT_RAND_BOOL_V) { + qptMakeExprNode(&pOp->pRight); + } break; } } void qptMakeColumnNode(SNode** ppNode) { - SNodeList* pColList = qptCtx.makeCtx.pInputDataBlockDesc->pSlots; - int32_t colIdx = taosRand() % pColList->length; - SNode* pNode = nodesListGetNode(pColList, colIdx); - assert(nodeType(pNode) == QUERY_NODE_SLOT_DESC); - SSlotDescNode* pSlot = (SSlotDescNode*)pNode; SColumnNode* pCol = NULL; - nodesMakeNode(QUERY_NODE_COLUMN, (SNode**)&pCol); - pCol->node.resType = pSlot->dataType; - pCol->dataBlockId = qptCtx.makeCtx.pInputDataBlockDesc->dataBlockId; - pCol->slotId = pSlot->slotId; + if (qptCtx.param.correctExpected) { + SNodeList* pColList = qptCtx.makeCtx.pInputDataBlockDesc->pSlots; + int32_t colIdx = taosRand() % pColList->length; + SNode* pNode = nodesListGetNode(pColList, colIdx); + assert(nodeType(pNode) == QUERY_NODE_SLOT_DESC); + + SSlotDescNode* pSlot = (SSlotDescNode*)pNode; + nodesMakeNode(QUERY_NODE_COLUMN, (SNode**)&pCol); + pCol->node.resType = pSlot->dataType; + pCol->dataBlockId = qptCtx.makeCtx.pInputDataBlockDesc->dataBlockId; + pCol->slotId = pSlot->slotId; + } else { + nodesMakeNode(QUERY_NODE_COLUMN, (SNode**)&pCol); + qptGetRandValue(&pCol->node.resType.type, &pCol->node.resType.bytes, NULL); + pCol->dataBlockId = taosRand(); + pCol->slotId = taosRand(); + } *ppNode = (SNode*)pCol; } +void qptNodesSetValueNodeValue(SValueNode* pNode, void* value) { + switch (pNode->node.resType.type) { + case TSDB_DATA_TYPE_NULL: + taosMemoryFree(value); + break; + case TSDB_DATA_TYPE_BOOL: + pNode->datum.b = *(bool*)value; + taosMemoryFree(value); + *(bool*)&pNode->typeData = qptCtx.param.correctExpected ? pNode->datum.b : QPT_RAND_BOOL_V; + break; + case TSDB_DATA_TYPE_TINYINT: + pNode->datum.i = *(int8_t*)value; + taosMemoryFree(value); + *(int8_t*)&pNode->typeData = qptCtx.param.correctExpected ? pNode->datum.i : QPT_RAND_INT_V; + break; + case TSDB_DATA_TYPE_SMALLINT: + pNode->datum.i = *(int16_t*)value; + taosMemoryFree(value); + *(int16_t*)&pNode->typeData = qptCtx.param.correctExpected ? pNode->datum.i : QPT_RAND_INT_V; + break; + case TSDB_DATA_TYPE_INT: + pNode->datum.i = *(int32_t*)value; + taosMemoryFree(value); + *(int32_t*)&pNode->typeData = qptCtx.param.correctExpected ? pNode->datum.i : QPT_RAND_INT_V; + break; + case TSDB_DATA_TYPE_BIGINT: + pNode->datum.i = *(int64_t*)value; + taosMemoryFree(value); + *(int64_t*)&pNode->typeData = qptCtx.param.correctExpected ? pNode->datum.i : QPT_RAND_INT_V; + break; + case TSDB_DATA_TYPE_TIMESTAMP: + pNode->datum.i = *(int64_t*)value; + taosMemoryFree(value); + *(int64_t*)&pNode->typeData = qptCtx.param.correctExpected ? pNode->datum.i : QPT_RAND_INT_V; + break; + case TSDB_DATA_TYPE_UTINYINT: + pNode->datum.u = *(int8_t*)value; + taosMemoryFree(value); + *(int8_t*)&pNode->typeData = qptCtx.param.correctExpected ? pNode->datum.u : QPT_RAND_INT_V; + break; + case TSDB_DATA_TYPE_USMALLINT: + pNode->datum.u = *(int16_t*)value; + taosMemoryFree(value); + *(int16_t*)&pNode->typeData = qptCtx.param.correctExpected ? pNode->datum.u : QPT_RAND_INT_V; + break; + case TSDB_DATA_TYPE_UINT: + pNode->datum.u = *(int32_t*)value; + taosMemoryFree(value); + *(int32_t*)&pNode->typeData = qptCtx.param.correctExpected ? pNode->datum.u : QPT_RAND_INT_V; + break; + case TSDB_DATA_TYPE_UBIGINT: + pNode->datum.u = *(uint64_t*)value; + taosMemoryFree(value); + *(uint64_t*)&pNode->typeData = qptCtx.param.correctExpected ? pNode->datum.u : QPT_RAND_INT_V; + break; + case TSDB_DATA_TYPE_FLOAT: + pNode->datum.d = *(float*)value; + taosMemoryFree(value); + *(float*)&pNode->typeData = qptCtx.param.correctExpected ? pNode->datum.d : QPT_RAND_INT_V; + break; + case TSDB_DATA_TYPE_DOUBLE: + pNode->datum.d = *(double*)value; + taosMemoryFree(value); + *(double*)&pNode->typeData = qptCtx.param.correctExpected ? pNode->datum.d : QPT_RAND_INT_V; + break; + case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_VARCHAR: + case TSDB_DATA_TYPE_VARBINARY: + case TSDB_DATA_TYPE_JSON: + case TSDB_DATA_TYPE_GEOMETRY: + if (qptCtx.param.correctExpected || QPT_MID_PROB()) { + pNode->datum.p = (char*)value; + } else { + } + taosMemoryFree(value); + pNode->datum.p = NULL; + break; + case TSDB_DATA_TYPE_DECIMAL: + case TSDB_DATA_TYPE_BLOB: + case TSDB_DATA_TYPE_MEDIUMBLOB: + taosMemoryFree(value); + pNode->datum.p = NULL; + break; + default: + taosMemoryFree(value); + break; + } +} void qptMakeValueNode(SNode** ppNode) { SValueNode* pVal = NULL; nodesMakeNode(QUERY_NODE_VALUE, (SNode**)&pVal); - int32_t valType, valBytes; + uint8_t valType; + int32_t valBytes; void* pValue = NULL; qptGetRandValue(&valType, &valBytes, &pValue); pVal->node.resType.type = valType; pVal->node.resType.bytes = valBytes; - nodesSetValueNodeValue(pVal, pValue); + + qptNodesSetValueNodeValue(pVal, pValue); *ppNode = (SNode*)pVal; } @@ -706,33 +898,58 @@ void qptMakeFunctionNode(SNode** ppNode) { SFunctionNode* pFunc = NULL; nodesMakeNode(QUERY_NODE_FUNCTION, (SNode**)&pFunc); - int32_t funcIdx = taosRand() % funcMgtBuiltinsNum; - strcpy(pFunc->functionName, fmGetFuncName(funcIdx)); - fmGetFuncInfo(pFunc, NULL, 0); - + if (qptCtx.param.correctExpected || QPT_HIGH_PROB()) { + int32_t funcIdx = taosRand() % funcMgtBuiltinsNum; + char* funcName = fmGetFuncName(funcIdx); + strcpy(pFunc->functionName, funcName); + taosMemoryFree(funcName); + fmGetFuncInfo(pFunc, NULL, 0); + } else { + int32_t funcIdx = taosRand(); + if (QPT_RAND_BOOL_V) { + strcpy(pFunc->functionName, "invalidFuncName"); + } else { + pFunc->functionName[0] = 0; + } + fmGetFuncInfo(pFunc, NULL, 0); + } + qptCtx.makeCtx.nodeLevel++; - int32_t paramNum = taosRand() % QPT_MAX_FUNC_PARAM + 1; - for (int32_t i = 0; i < paramNum; ++i) { - SNode* pNode = NULL; - qptMakeExprNode(&pNode); - assert(0 == nodesListMakeStrictAppend(&pFunc->pParameterList, pNode)); + + if (QPT_CORRECT_HIGH_PROB()) { + // TODO + } else { + int32_t paramNum = taosRand() % QPT_MAX_FUNC_PARAM; + for (int32_t i = 0; i < paramNum; ++i) { + SNode* pNode = NULL; + qptMakeExprNode(&pNode); + qptNodesListMakeStrictAppend(&pFunc->pParameterList, pNode); + } } *ppNode = (SNode*)pFunc; } + + + void qptMakeLogicCondNode(SNode** ppNode) { SLogicConditionNode* pLogic = NULL; nodesMakeNode(QUERY_NODE_LOGIC_CONDITION, (SNode**)&pLogic); - pLogic->condType = (taosRand() % 3) ? ((taosRand() % 2) ? LOGIC_COND_TYPE_AND : LOGIC_COND_TYPE_OR) : LOGIC_COND_TYPE_NOT; - + if (QPT_CORRECT_HIGH_PROB()) { + pLogic->condType = (taosRand() % 3) ? ((taosRand() % 2) ? LOGIC_COND_TYPE_AND : LOGIC_COND_TYPE_OR) : LOGIC_COND_TYPE_NOT; + } else { + pLogic->condType = (ELogicConditionType)taosRand(); + } + qptCtx.makeCtx.nodeLevel++; - int32_t paramNum = taosRand() % QPT_MAX_LOGIC_PARAM + 1; + + int32_t paramNum = QPT_CORRECT_HIGH_PROB() ? (taosRand() % QPT_MAX_LOGIC_PARAM + 1) : (taosRand() % QPT_MAX_LOGIC_PARAM); for (int32_t i = 0; i < paramNum; ++i) { SNode* pNode = NULL; qptMakeExprNode(&pNode); - assert(0 == nodesListMakeStrictAppend(&pLogic->pParameterList, pNode)); + qptNodesListMakeStrictAppend(&pLogic->pParameterList, pNode); } *ppNode = (SNode*)pLogic; @@ -743,11 +960,12 @@ void qptMakeNodeListNode(SNode** ppNode) { nodesMakeNode(QUERY_NODE_NODE_LIST, (SNode**)&pList); qptCtx.makeCtx.nodeLevel++; - int32_t nodeNum = taosRand() % QPT_MAX_NODE_LIST_NUM + 1; + + int32_t nodeNum = QPT_CORRECT_HIGH_PROB() ? (taosRand() % QPT_MAX_NODE_LIST_NUM + 1) : (taosRand() % QPT_MAX_NODE_LIST_NUM); for (int32_t i = 0; i < nodeNum; ++i) { SNode* pNode = NULL; qptMakeExprNode(&pNode); - assert(0 == nodesListMakeStrictAppend(&pList->pNodeList, pNode)); + qptNodesListMakeStrictAppend(&pList->pNodeList, pNode); } *ppNode = (SNode*)pList; @@ -757,6 +975,10 @@ void qptMakeTempTableNode(SNode** ppNode) { STempTableNode* pTemp = NULL; assert(0 == nodesMakeNode(QUERY_NODE_TEMP_TABLE, (SNode**)&pTemp)); + if (QPT_CORRECT_HIGH_PROB()) { + // TODO + } + *ppNode = (SNode*)pTemp; } @@ -764,20 +986,51 @@ void qptMakeJoinTableNode(SNode** ppNode) { SJoinTableNode* pJoin = NULL; assert(0 == nodesMakeNode(QUERY_NODE_JOIN_TABLE, (SNode**)&pJoin)); + if (QPT_CORRECT_HIGH_PROB()) { + // TODO + } + *ppNode = (SNode*)pJoin; } +void qptMakeRealTableNode(SNode** ppNode) { + SRealTableNode* pReal = NULL; + assert(0 == nodesMakeNode(QUERY_NODE_REAL_TABLE, (SNode**)&pReal)); -void qptMakeTableNode(SNode** ppNode) { - if (taosRand() % 2) { - qptMakeTempTableNode(ppNode); - } else { - qptMakeJoinTableNode(ppNode); + if (QPT_CORRECT_HIGH_PROB()) { + // TODO } + + *ppNode = (SNode*)pReal; } + +void qptMakeNonRealTableNode(SNode** ppNode) { + if (QPT_CORRECT_HIGH_PROB()) { + if (QPT_RAND_BOOL_V) { + qptMakeTempTableNode(ppNode); + } else { + qptMakeJoinTableNode(ppNode); + } + } else { + qptMakeRealTableNode(ppNode); + } +} + +void qptMakeRandNode(SNode** ppNode) { + nodesMakeNode((ENodeType)taosRand(), ppNode); +} + SNode* qptMakeExprNode(SNode** ppNode) { + if (!qptCtx.param.correctExpected && QPT_LOW_PROB()) { + if (ppNode) { + *ppNode = NULL; + } + + return NULL; + } + int32_t nodeTypeMaxValue = 9; if (qptCtx.makeCtx.nodeLevel >= QPT_MAX_NODE_LEVEL) { nodeTypeMaxValue = 2; @@ -787,6 +1040,11 @@ SNode* qptMakeExprNode(SNode** ppNode) { if (NULL == ppNode) { ppNode = &pNode; } + + if (!qptCtx.param.correctExpected && QPT_LOW_PROB()) { + qptMakeRandNode(ppNode); + return *ppNode; + } switch (taosRand() % nodeTypeMaxValue) { case 0: @@ -808,7 +1066,7 @@ SNode* qptMakeExprNode(SNode** ppNode) { qptMakeOperatorNode(ppNode); break; case 6: - qptMakeTableNode(ppNode); + qptMakeNonRealTableNode(ppNode); break; case 7: qptMakeCaseWhenNode(ppNode); @@ -825,7 +1083,7 @@ SNode* qptMakeExprNode(SNode** ppNode) { } void qptResetMakeNodeCtx(SDataBlockDescNode* pInput, bool onlyTag) { - SQPTMakePlanCtx* pCtx = &qptCtx.makeCtx; + SQPTMakeNodeCtx* pCtx = &qptCtx.makeCtx; pCtx->nodeLevel = 1; pCtx->onlyTag = onlyTag; @@ -836,11 +1094,12 @@ SNode* qptMakeConditionNode(bool onlyTag) { SNode* pNode = NULL; qptResetMakeNodeCtx(qptCtx.buildCtx.pCurr->pOutputDataBlockDesc, onlyTag); qptMakeExprNode(&pNode); + return pNode; } SNode* qptMakeDataBlockDescNode() { - if (QPT_RAND_BOOL_V) { + if (!qptCtx.param.correctExpected && QPT_LOW_PROB()) { return NULL; } @@ -899,45 +1158,9 @@ void qptMarkTableInUseCols(int32_t colNum, int32_t totalColNum, SQPTCol* pCol) { } while (colInUse < colNum); } -int32_t qptNodesListAppend(SNodeList* pList, SNode* pNode) { - SListCell* p = NULL; - assert(0 == nodesCalloc(1, sizeof(SListCell), (void**)&p)); - - p->pNode = pNode; - if (NULL == pList->pHead) { - pList->pHead = p; - } - if (NULL != pList->pTail) { - pList->pTail->pNext = p; - } - p->pPrev = pList->pTail; - pList->pTail = p; - ++(pList->length); - return TSDB_CODE_SUCCESS; -} - - -int32_t qptNodesListStrictAppend(SNodeList* pList, SNode* pNode) { - int32_t code = qptNodesListAppend(pList, pNode); - if (TSDB_CODE_SUCCESS != code) { - nodesDestroyNode(pNode); - } - return code; -} - -int32_t qptNodesListMakeStrictAppend(SNodeList** pList, SNode* pNode) { - if (NULL == *pList) { - int32_t code = nodesMakeList(pList); - if (NULL == *pList) { - return code; - } - } - return qptNodesListStrictAppend(*pList, pNode); -} - void qptCreateTableScanColsImpl( SScanPhysiNode* pScanPhysiNode, SNodeList** ppCols, int32_t totalColNum, SQPTCol* pCol) { - int32_t colNum = qptCtx.param.correctExpected ? (taosRand() % totalColNum + 1) : (taosRand()); + int32_t colNum = qptCtx.param.correctExpected ? (taosRand() % totalColNum + 1) : (taosRand() % QPT_MAX_COLUMN_NUM + 1); int32_t colAdded = 0; if (qptCtx.param.correctExpected) { @@ -948,7 +1171,7 @@ void qptCreateTableScanColsImpl( SScanPhysiNode* pScanPhysiNode, SNodeList continue; } - assert(0 == qptNodesListMakeStrictAppend(ppCols, qptMakeColumnNodeFromTable(i, pCol[i].colType, pScanPhysiNode))); + assert(0 == qptNodesListMakeStrictAppend(ppCols, qptMakeColumnNodeFromTable(i, pCol[0].colType, pScanPhysiNode))); } return; @@ -958,7 +1181,7 @@ void qptCreateTableScanColsImpl( SScanPhysiNode* pScanPhysiNode, SNodeList int32_t colIdx = taosRand(); colIdx = (colIdx >= totalColNum) ? -1 : colIdx; - assert(0 == qptNodesListMakeStrictAppend(ppCols, qptMakeColumnNodeFromTable(colIdx, pCol[i].colType, pScanPhysiNode))); + assert(0 == qptNodesListMakeStrictAppend(ppCols, qptMakeColumnNodeFromTable(colIdx, pCol[0].colType, pScanPhysiNode))); } } @@ -976,17 +1199,13 @@ SNode* qptMakeSlotDescNode(const char* pName, const SNode* pNode, int16_t slotId assert(0 == nodesMakeNode(QUERY_NODE_SLOT_DESC, (SNode**)&pSlot)); QPT_RAND_BOOL_V ? (pSlot->name[0] = 0) : snprintf(pSlot->name, sizeof(pSlot->name), "%s", pName); -<<<<<<< Updated upstream - pSlot->slotId = slotId; - pSlot->dataType = ((SExprNode*)pNode)->resType; -======= pSlot->slotId = qptCtx.param.correctExpected ? slotId : taosRand(); if (qptCtx.param.correctExpected) { pSlot->dataType = ((SExprNode*)pNode)->resType; } else { - + qptGetRandValue(&pSlot->dataType.type, &pSlot->dataType.bytes, NULL); } ->>>>>>> Stashed changes + pSlot->reserve = reserve; pSlot->output = output; return (SNode*)pSlot; @@ -1004,11 +1223,19 @@ void qptMakeTargetNode(SNode* pNode, int16_t dataBlockId, int16_t slotId, SNode* } void qptAddDataBlockSlots(SNodeList* pList, SDataBlockDescNode* pDataBlockDesc) { + if (NULL == pDataBlockDesc) { + return; + } + int16_t nextSlotId = LIST_LENGTH(pDataBlockDesc->pSlots), slotId = 0; SNode* pNode = NULL; bool output = QPT_RAND_BOOL_V; FOREACH(pNode, pList) { + if (NULL == pNode) { + continue; + } + SNode* pExpr = QUERY_NODE_ORDER_BY_EXPR == nodeType(pNode) ? ((SOrderByExprNode*)pNode)->pExpr : pNode; if (qptCtx.param.correctExpected || QPT_RAND_BOOL_V) { SNode* pDesc = qptCtx.param.correctExpected ? qptMakeSlotDescNode(NULL, pExpr, nextSlotId, output, QPT_RAND_BOOL_V) : qptMakeExprNode(NULL); @@ -1042,15 +1269,21 @@ void qptCreateScanPhysiNodeImpl( SScanPhysiNode* pScanPhysiNode) { qptAddDataBlockSlots(pScanPhysiNode->pScanPseudoCols, pScanPhysiNode->node.pOutputDataBlockDesc); - pScanPhysiNode->uid = qptCtx.param.tbl.uid; - pScanPhysiNode->suid = qptCtx.param.tbl.suid; - pScanPhysiNode->tableType = qptCtx.param.tbl.tblType; + pScanPhysiNode->uid = qptCtx.param.correctExpected ? qptCtx.param.tbl.uid : taosRand(); + pScanPhysiNode->suid = qptCtx.param.correctExpected ? qptCtx.param.tbl.suid : taosRand(); + pScanPhysiNode->tableType = qptCtx.param.correctExpected ? qptCtx.param.tbl.tblType : taosRand(); pScanPhysiNode->groupOrderScan = (taosRand() % 2) ? true : false; SName tblName = {0}; toName(1, qptCtx.param.db.dbName, qptCtx.param.tbl.tblName, &tblName); - memcpy(&pScanPhysiNode->tableName, &tblName, sizeof(SName)); - + if (qptCtx.param.correctExpected || QPT_RAND_BOOL_V) { + memcpy(&pScanPhysiNode->tableName, &tblName, sizeof(SName)); + } else { + pScanPhysiNode->tableName.acctId = 0; + pScanPhysiNode->tableName.dbname[0] = 0; + pScanPhysiNode->tableName.tname[0] = 0; + } + qptCtx.buildCtx.currTsOrder = (qptCtx.param.correctExpected) ? qptCtx.buildCtx.currTsOrder : QPT_RAND_ORDER_V; } @@ -1277,7 +1510,7 @@ void qptRunPlanTest(char* caseName) { qptPrintBeginInfo(caseName); qptCtx.startTsUs = taosGetTimestampUs(); - qptCtx.result.code = createTagScanOperatorInfo(&readHandle, (STagScanPhysiNode*)pNode, NULL, NULL, NULL, NULL, &pOperator); + //qptCtx.result.code = createTagScanOperatorInfo(&readHandle, (STagScanPhysiNode*)pNode, NULL, NULL, NULL, NULL, &pOperator); destroyOperator(pOperator); nodesDestroyNode((SNode*)pNode); @@ -1379,7 +1612,7 @@ void qptInitTestCtx(bool correctExpected, bool singleNode, int32_t nodeType, int qptCtx.param.tbl.uid = 100; qptCtx.param.tbl.suid = 1; - qptGetRandRealTableType(qptCtx.param.tbl.tblType); + qptGetRandRealTableType(&qptCtx.param.tbl.tblType); qptCtx.param.tbl.colNum = taosRand() % 4098; qptCtx.param.tbl.tagNum = taosRand() % 130; qptCtx.param.tbl.pkNum = taosRand() % 2; From bde18abb60445d03f3ca239fd1410f136a3ed4f8 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 30 Sep 2024 18:07:14 +0800 Subject: [PATCH 07/29] enh: add projection plan --- source/libs/executor/test/queryPlanTests.cpp | 54 ++++++++++++++++---- 1 file changed, 43 insertions(+), 11 deletions(-) diff --git a/source/libs/executor/test/queryPlanTests.cpp b/source/libs/executor/test/queryPlanTests.cpp index 4c75ed4d7c..3422611ea1 100755 --- a/source/libs/executor/test/queryPlanTests.cpp +++ b/source/libs/executor/test/queryPlanTests.cpp @@ -766,19 +766,22 @@ void qptMakeOperatorNode(SNode** ppNode) { void qptMakeColumnNode(SNode** ppNode) { SColumnNode* pCol = NULL; - if (qptCtx.param.correctExpected) { + nodesMakeNode(QUERY_NODE_COLUMN, (SNode**)&pCol); + SSlotDescNode* pSlot = NULL; + + if (QPT_CORRECT_HIGH_PROB() && qptCtx.makeCtx.pInputDataBlockDesc && qptCtx.makeCtx.pInputDataBlockDesc->pSlots) { SNodeList* pColList = qptCtx.makeCtx.pInputDataBlockDesc->pSlots; int32_t colIdx = taosRand() % pColList->length; SNode* pNode = nodesListGetNode(pColList, colIdx); - assert(nodeType(pNode) == QUERY_NODE_SLOT_DESC); - - SSlotDescNode* pSlot = (SSlotDescNode*)pNode; - nodesMakeNode(QUERY_NODE_COLUMN, (SNode**)&pCol); - pCol->node.resType = pSlot->dataType; - pCol->dataBlockId = qptCtx.makeCtx.pInputDataBlockDesc->dataBlockId; - pCol->slotId = pSlot->slotId; - } else { - nodesMakeNode(QUERY_NODE_COLUMN, (SNode**)&pCol); + if (pNode && nodeType(pNode) == QUERY_NODE_SLOT_DESC) { + pSlot = (SSlotDescNode*)pNode; + pCol->node.resType = pSlot->dataType; + pCol->dataBlockId = qptCtx.makeCtx.pInputDataBlockDesc->dataBlockId; + pCol->slotId = pSlot->slotId; + } + } + + if (NULL == pSlot) { qptGetRandValue(&pCol->node.resType.type, &pCol->node.resType.bytes, NULL); pCol->dataBlockId = taosRand(); pCol->slotId = taosRand(); @@ -1132,7 +1135,7 @@ SPhysiNode* qptCreatePhysiNode(int32_t nodeType) { void qptPostCreatePhysiNode(SPhysiNode* pPhysiNode) { pPhysiNode->outputTsOrder = qptGetCurrTsOrder(); - if (taosRand() % 2) { + if (QPT_RAND_BOOL_V) { pPhysiNode->pConditions = qptMakeConditionNode(false); } @@ -1302,6 +1305,34 @@ SNode* qptCreateTagScanPhysiNode(int32_t nodeType) { return (SNode*)pPhysiNode; } +void qptMakeExprList(SNodeList** ppList) { + int32_t exprNum = taosRand() % QPT_MAX_COLUMN_NUM + (QPT_CORRECT_HIGH_PROB() ? 1 : 0); + for (int32_t i = 0; i < exprNum; ++i) { + SNode* pNode = NULL; + qptResetMakeNodeCtx(qptCtx.buildCtx.pChild ? qptCtx.buildCtx.pChild->pOutputDataBlockDesc : NULL, false); + qptMakeExprNode(&pNode); + qptNodesListMakeStrictAppend(ppList, pNode); + } +} + +SNode* qptCreateProjectPhysiNode(int32_t nodeType) { + SPhysiNode* pPhysiNode = qptCreatePhysiNode(nodeType); + + SProjectPhysiNode* pProject = (SProjectPhysiNode*)pPhysiNode; + + pProject->mergeDataBlock = QPT_RAND_BOOL_V; + pProject->ignoreGroupId = QPT_RAND_BOOL_V; + pProject->inputIgnoreGroup = QPT_RAND_BOOL_V; + + qptMakeExprList(&pProject->pProjections); + + qptAddDataBlockSlots(pProject->pProjections, pProject->node.pOutputDataBlockDesc); + + qptPostCreatePhysiNode(pPhysiNode); + + return (SNode*)pPhysiNode; +} + SNode* qptCreateSortMergeJoinPhysiNode(int32_t nodeType) { SPhysiNode* pPhysiNode = qptCreatePhysiNode(nodeType); @@ -1387,6 +1418,7 @@ SNode* qptCreatePhysicalPlanNode(int32_t nodeType) { case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN: case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN: case QUERY_NODE_PHYSICAL_PLAN_PROJECT: + return (SNode*)qptCreateProjectPhysiNode(nodeType); case QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN: return (SNode*)qptCreateSortMergeJoinPhysiNode(nodeType); case QUERY_NODE_PHYSICAL_PLAN_HASH_AGG: From ff5e7556c4c6399d56dd288d0e20b0d429a6c901 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 9 Oct 2024 14:29:56 +0800 Subject: [PATCH 08/29] enh: add merge join ut --- source/libs/executor/test/queryPlanTests.cpp | 295 ++++++++++++++----- 1 file changed, 217 insertions(+), 78 deletions(-) diff --git a/source/libs/executor/test/queryPlanTests.cpp b/source/libs/executor/test/queryPlanTests.cpp index 3422611ea1..9cfdf931a4 100755 --- a/source/libs/executor/test/queryPlanTests.cpp +++ b/source/libs/executor/test/queryPlanTests.cpp @@ -113,10 +113,11 @@ int32_t QPT_PHYSIC_NODE_LIST[] = { #define QPT_RAND_ORDER_V (QPT_RAND_BOOL_V ? ORDER_ASC : ORDER_DESC) #define QPT_RAND_INT_V (taosRand() * (QPT_RAND_BOOL_V ? 1 : -1)) #define QPT_LOW_PROB() ((taosRand() % 11) == 0) -#define QPT_MID_PROB() ((taosRand() % 11) <= 4) +#define QPT_MID_PROB() ((taosRand() % 11) <= 1) #define QPT_HIGH_PROB() ((taosRand() % 11) <= 7) #define QPT_CORRECT_HIGH_PROB() (qptCtx.param.correctExpected || QPT_HIGH_PROB()) +#define QPT_NCORRECT_LOW_PROB() (!qptCtx.param.correctExpected && QPT_LOW_PROB()) typedef struct { ENodeType type; @@ -125,6 +126,8 @@ typedef struct { typedef struct { bool singlePhysiNode; + uint64_t queryId; + uint64_t taskId; int32_t subplanMaxLevel; int32_t subplanType[QPT_MAX_SUBPLAN_LEVEL]; int32_t physiNodeParamNum; @@ -139,6 +142,7 @@ typedef struct { typedef struct { int32_t vnodeNum; + int32_t vgId; } SQPTVnodeParam; typedef struct { @@ -177,7 +181,10 @@ typedef struct { typedef struct { SPhysiNode* pCurr; - SPhysiNode* pChild; + int32_t childrenNum; + SPhysiNode* pChild; // current child + SPhysiNode* pLeftChild; + SPhysiNode* pRightChild; EOrder currTsOrder; } SQPTBuildPlanCtx; @@ -296,7 +303,7 @@ void qptPrintStatInfo(char* caseName) { bool qptGetDynamicOp() { - if (!qptCtx.param.correctExpected) { + if (QPT_NCORRECT_LOW_PROB()) { return QPT_RAND_BOOL_V; } @@ -308,25 +315,28 @@ bool qptGetDynamicOp() { } EOrder qptGetCurrTsOrder() { - return qptCtx.param.correctExpected ? qptCtx.buildCtx.currTsOrder : QPT_RAND_ORDER_V; + return QPT_CORRECT_HIGH_PROB() ? qptCtx.buildCtx.currTsOrder : QPT_RAND_ORDER_V; } -void qptGetRandValue(uint8_t* pType, int32_t* pLen, void** ppVal) { - int32_t typeMax = TSDB_DATA_TYPE_MAX; - if (!qptCtx.param.correctExpected) { - typeMax++; +void qptGetRandValue(int16_t* pType, int32_t* pLen, void** ppVal) { + if (*pType < 0 || QPT_NCORRECT_LOW_PROB()) { + int32_t typeMax = TSDB_DATA_TYPE_MAX; + if (QPT_NCORRECT_LOW_PROB()) { + typeMax++; + } + + *pType = taosRand() % typeMax; } - *pType = taosRand() % typeMax; switch (*pType) { case TSDB_DATA_TYPE_NULL: - *pLen = qptCtx.param.correctExpected ? 0 : taosRand(); + *pLen = QPT_CORRECT_HIGH_PROB() ? 0 : taosRand(); if (ppVal) { *ppVal = NULL; } break; case TSDB_DATA_TYPE_BOOL: - *pLen = qptCtx.param.correctExpected ? tDataTypes[*pType].bytes : taosRand(); + *pLen = QPT_CORRECT_HIGH_PROB() ? tDataTypes[*pType].bytes : taosRand(); if (ppVal) { *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); assert(*ppVal); @@ -334,7 +344,7 @@ void qptGetRandValue(uint8_t* pType, int32_t* pLen, void** ppVal) { } break; case TSDB_DATA_TYPE_TINYINT: - *pLen = qptCtx.param.correctExpected ? tDataTypes[*pType].bytes : taosRand(); + *pLen = QPT_CORRECT_HIGH_PROB() ? tDataTypes[*pType].bytes : taosRand(); if (ppVal) { *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); assert(*ppVal); @@ -342,7 +352,7 @@ void qptGetRandValue(uint8_t* pType, int32_t* pLen, void** ppVal) { } break; case TSDB_DATA_TYPE_SMALLINT: - *pLen = qptCtx.param.correctExpected ? tDataTypes[*pType].bytes : taosRand(); + *pLen = QPT_CORRECT_HIGH_PROB() ? tDataTypes[*pType].bytes : taosRand(); if (ppVal) { *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); assert(*ppVal); @@ -350,7 +360,7 @@ void qptGetRandValue(uint8_t* pType, int32_t* pLen, void** ppVal) { } break; case TSDB_DATA_TYPE_INT: - *pLen = qptCtx.param.correctExpected ? tDataTypes[*pType].bytes : taosRand(); + *pLen = QPT_CORRECT_HIGH_PROB() ? tDataTypes[*pType].bytes : taosRand(); if (ppVal) { *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); assert(*ppVal); @@ -359,7 +369,7 @@ void qptGetRandValue(uint8_t* pType, int32_t* pLen, void** ppVal) { break; case TSDB_DATA_TYPE_BIGINT: case TSDB_DATA_TYPE_TIMESTAMP: - *pLen = qptCtx.param.correctExpected ? tDataTypes[*pType].bytes : taosRand(); + *pLen = QPT_CORRECT_HIGH_PROB() ? tDataTypes[*pType].bytes : taosRand(); if (ppVal) { *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); assert(*ppVal); @@ -367,7 +377,7 @@ void qptGetRandValue(uint8_t* pType, int32_t* pLen, void** ppVal) { } break; case TSDB_DATA_TYPE_FLOAT: - *pLen = qptCtx.param.correctExpected ? tDataTypes[*pType].bytes : taosRand(); + *pLen = QPT_CORRECT_HIGH_PROB() ? tDataTypes[*pType].bytes : taosRand(); if (ppVal) { *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); assert(*ppVal); @@ -375,7 +385,7 @@ void qptGetRandValue(uint8_t* pType, int32_t* pLen, void** ppVal) { } break; case TSDB_DATA_TYPE_DOUBLE: - *pLen = qptCtx.param.correctExpected ? tDataTypes[*pType].bytes : taosRand(); + *pLen = QPT_CORRECT_HIGH_PROB() ? tDataTypes[*pType].bytes : taosRand(); if (ppVal) { *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); assert(*ppVal); @@ -413,7 +423,7 @@ void qptGetRandValue(uint8_t* pType, int32_t* pLen, void** ppVal) { break; } case TSDB_DATA_TYPE_UTINYINT: - *pLen = qptCtx.param.correctExpected ? tDataTypes[*pType].bytes : taosRand(); + *pLen = QPT_CORRECT_HIGH_PROB() ? tDataTypes[*pType].bytes : taosRand(); if (ppVal) { *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); assert(*ppVal); @@ -421,7 +431,7 @@ void qptGetRandValue(uint8_t* pType, int32_t* pLen, void** ppVal) { } break; case TSDB_DATA_TYPE_USMALLINT: - *pLen = qptCtx.param.correctExpected ? tDataTypes[*pType].bytes : taosRand(); + *pLen = QPT_CORRECT_HIGH_PROB() ? tDataTypes[*pType].bytes : taosRand(); if (ppVal) { *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); assert(*ppVal); @@ -429,7 +439,7 @@ void qptGetRandValue(uint8_t* pType, int32_t* pLen, void** ppVal) { } break; case TSDB_DATA_TYPE_UINT: - *pLen = qptCtx.param.correctExpected ? tDataTypes[*pType].bytes : taosRand(); + *pLen = QPT_CORRECT_HIGH_PROB() ? tDataTypes[*pType].bytes : taosRand(); if (ppVal) { *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); assert(*ppVal); @@ -437,7 +447,7 @@ void qptGetRandValue(uint8_t* pType, int32_t* pLen, void** ppVal) { } break; case TSDB_DATA_TYPE_UBIGINT: - *pLen = qptCtx.param.correctExpected ? tDataTypes[*pType].bytes : taosRand(); + *pLen = QPT_CORRECT_HIGH_PROB() ? tDataTypes[*pType].bytes : taosRand(); if (ppVal) { *ppVal = taosMemoryMalloc(tDataTypes[*pType].bytes); assert(*ppVal); @@ -503,6 +513,13 @@ void qptGetRandRealTableType(int8_t* tableType) { } } +int32_t qptGetInputSlotId(SDataBlockDescNode* pInput) { + if (pInput && pInput->pSlots && pInput->pSlots->length > 0 && QPT_CORRECT_HIGH_PROB()) { + return taosRand() % pInput->pSlots->length; + } + + return taosRand(); +} void qptNodesCalloc(int32_t num, int32_t size, void** pOut) { @@ -550,11 +567,11 @@ int32_t qptNodesListMakeStrictAppend(SNodeList** pList, SNode* pNode) { SNode* qptMakeLimitNode() { - if (QPT_RAND_BOOL_V) { - return NULL; + SNode* pNode = NULL; + if (QPT_NCORRECT_LOW_PROB()) { + return qptMakeRandNode(&pNode); } - SNode* pNode = NULL; assert(0 == nodesMakeNode(QUERY_NODE_LIMIT, &pNode)); assert(pNode); @@ -577,6 +594,26 @@ SNode* qptMakeLimitNode() { return pNode; } + +SNode* qptMakeWindowOffsetNode(SNode** ppNode) { + if (QPT_RAND_BOOL_V) { + return qptMakeRandNode(ppNode); + } + + SNode* pNode = NULL; + SWindowOffsetNode* pWinOffset = NULL; + + assert(0 == nodesMakeNode(QUERY_NODE_WINDOW_OFFSET, &pNode)); + assert(pNode); + + SWindowOffsetNode* pWinOffset = (SWindowOffsetNode*)pNode; + qptMakeValueNode(TSDB_DATA_TYPE_BIGINT, &pWinOffset->pStartOffset); + qptMakeValueNode(TSDB_DATA_TYPE_BIGINT, &pWinOffset->pEndOffset); + + return pNode; +} + + SNode* qptMakeColumnNodeFromTable(int32_t colIdx, EColumnType colType, SScanPhysiNode* pScanPhysiNode) { if (colIdx < 0) { return NULL; @@ -653,6 +690,10 @@ SNode* qptMakeColumnNodeFromTable(int32_t colIdx, EColumnType colType, SScanPhys void qptMakeWhenThenNode(SNode** ppNode) { + if (QPT_NCORRECT_LOW_PROB) { + return qptMakeRandNode(ppNode); + } + assert(0 == nodesMakeNode(QUERY_NODE_WHEN_THEN, ppNode)); assert(*ppNode); SWhenThenNode* pWhenThen = (SWhenThenNode*)*ppNode; @@ -664,6 +705,10 @@ void qptMakeWhenThenNode(SNode** ppNode) { void qptMakeCaseWhenNode(SNode** ppNode) { + if (QPT_NCORRECT_LOW_PROB) { + return qptMakeRandNode(ppNode); + } + assert(0 == nodesMakeNode(QUERY_NODE_CASE_WHEN, ppNode)); assert(*ppNode); @@ -689,6 +734,10 @@ void qptMakeCaseWhenNode(SNode** ppNode) { void qptMakeOperatorNode(SNode** ppNode) { + if (QPT_NCORRECT_LOW_PROB) { + return qptMakeRandNode(ppNode); + } + EOperatorType opType = OPERATOR_ARRAY[taosRand() % (sizeof(OPERATOR_ARRAY)/sizeof(OPERATOR_ARRAY[0]))]; assert(0 == nodesMakeNode(QUERY_NODE_OPERATOR, ppNode)); @@ -697,7 +746,7 @@ void qptMakeOperatorNode(SNode** ppNode) { qptCtx.makeCtx.nodeLevel++; - switch (opType) { + switch (pOp->opType) { case OP_TYPE_ADD: case OP_TYPE_SUB: case OP_TYPE_MULTI: @@ -765,6 +814,10 @@ void qptMakeOperatorNode(SNode** ppNode) { } void qptMakeColumnNode(SNode** ppNode) { + if (QPT_NCORRECT_LOW_PROB) { + return qptMakeRandNode(ppNode); + } + SColumnNode* pCol = NULL; nodesMakeNode(QUERY_NODE_COLUMN, (SNode**)&pCol); SSlotDescNode* pSlot = NULL; @@ -880,11 +933,14 @@ void qptNodesSetValueNodeValue(SValueNode* pNode, void* value) { } -void qptMakeValueNode(SNode** ppNode) { +void qptMakeValueNode(int16_t valType, SNode** ppNode) { + if (QPT_NCORRECT_LOW_PROB) { + return qptMakeRandNode(ppNode); + } + SValueNode* pVal = NULL; nodesMakeNode(QUERY_NODE_VALUE, (SNode**)&pVal); - uint8_t valType; int32_t valBytes; void* pValue = NULL; qptGetRandValue(&valType, &valBytes, &pValue); @@ -898,6 +954,10 @@ void qptMakeValueNode(SNode** ppNode) { } void qptMakeFunctionNode(SNode** ppNode) { + if (QPT_NCORRECT_LOW_PROB) { + return qptMakeRandNode(ppNode); + } + SFunctionNode* pFunc = NULL; nodesMakeNode(QUERY_NODE_FUNCTION, (SNode**)&pFunc); @@ -937,6 +997,10 @@ void qptMakeFunctionNode(SNode** ppNode) { void qptMakeLogicCondNode(SNode** ppNode) { + if (QPT_NCORRECT_LOW_PROB) { + return qptMakeRandNode(ppNode); + } + SLogicConditionNode* pLogic = NULL; nodesMakeNode(QUERY_NODE_LOGIC_CONDITION, (SNode**)&pLogic); @@ -959,6 +1023,10 @@ void qptMakeLogicCondNode(SNode** ppNode) { } void qptMakeNodeListNode(SNode** ppNode) { + if (QPT_NCORRECT_LOW_PROB) { + return qptMakeRandNode(ppNode); + } + SNodeListNode* pList = NULL; nodesMakeNode(QUERY_NODE_NODE_LIST, (SNode**)&pList); @@ -975,6 +1043,10 @@ void qptMakeNodeListNode(SNode** ppNode) { } void qptMakeTempTableNode(SNode** ppNode) { + if (QPT_NCORRECT_LOW_PROB) { + return qptMakeRandNode(ppNode); + } + STempTableNode* pTemp = NULL; assert(0 == nodesMakeNode(QUERY_NODE_TEMP_TABLE, (SNode**)&pTemp)); @@ -986,6 +1058,10 @@ void qptMakeTempTableNode(SNode** ppNode) { } void qptMakeJoinTableNode(SNode** ppNode) { + if (QPT_NCORRECT_LOW_PROB) { + return qptMakeRandNode(ppNode); + } + SJoinTableNode* pJoin = NULL; assert(0 == nodesMakeNode(QUERY_NODE_JOIN_TABLE, (SNode**)&pJoin)); @@ -997,6 +1073,10 @@ void qptMakeJoinTableNode(SNode** ppNode) { } void qptMakeRealTableNode(SNode** ppNode) { + if (QPT_NCORRECT_LOW_PROB) { + return qptMakeRandNode(ppNode); + } + SRealTableNode* pReal = NULL; assert(0 == nodesMakeNode(QUERY_NODE_REAL_TABLE, (SNode**)&pReal)); @@ -1010,6 +1090,10 @@ void qptMakeRealTableNode(SNode** ppNode) { void qptMakeNonRealTableNode(SNode** ppNode) { + if (QPT_NCORRECT_LOW_PROB) { + return qptMakeRandNode(ppNode); + } + if (QPT_CORRECT_HIGH_PROB()) { if (QPT_RAND_BOOL_V) { qptMakeTempTableNode(ppNode); @@ -1021,17 +1105,19 @@ void qptMakeNonRealTableNode(SNode** ppNode) { } } -void qptMakeRandNode(SNode** ppNode) { +SNode* qptMakeRandNode(SNode** ppNode) { nodesMakeNode((ENodeType)taosRand(), ppNode); + return *ppNode; } SNode* qptMakeExprNode(SNode** ppNode) { - if (!qptCtx.param.correctExpected && QPT_LOW_PROB()) { - if (ppNode) { - *ppNode = NULL; - } + SNode* pNode = NULL; + if (NULL == ppNode) { + ppNode = &pNode; + } - return NULL; + if (QPT_NCORRECT_LOW_PROB()) { + return qptMakeRandNode(ppNode); } int32_t nodeTypeMaxValue = 9; @@ -1039,22 +1125,12 @@ SNode* qptMakeExprNode(SNode** ppNode) { nodeTypeMaxValue = 2; } - SNode* pNode = NULL; - if (NULL == ppNode) { - ppNode = &pNode; - } - - if (!qptCtx.param.correctExpected && QPT_LOW_PROB()) { - qptMakeRandNode(ppNode); - return *ppNode; - } - switch (taosRand() % nodeTypeMaxValue) { case 0: qptMakeColumnNode(ppNode); break; case 1: - qptMakeValueNode(ppNode); + qptMakeValueNode(-1, ppNode); break; case 2: qptMakeFunctionNode(ppNode); @@ -1085,6 +1161,7 @@ SNode* qptMakeExprNode(SNode** ppNode) { return *ppNode; } + void qptResetMakeNodeCtx(SDataBlockDescNode* pInput, bool onlyTag) { SQPTMakeNodeCtx* pCtx = &qptCtx.makeCtx; @@ -1240,7 +1317,7 @@ void qptAddDataBlockSlots(SNodeList* pList, SDataBlockDescNode* pDataBlockDesc) } SNode* pExpr = QUERY_NODE_ORDER_BY_EXPR == nodeType(pNode) ? ((SOrderByExprNode*)pNode)->pExpr : pNode; - if (qptCtx.param.correctExpected || QPT_RAND_BOOL_V) { + if (QPT_CORRECT_HIGH_PROB()) { SNode* pDesc = qptCtx.param.correctExpected ? qptMakeSlotDescNode(NULL, pExpr, nextSlotId, output, QPT_RAND_BOOL_V) : qptMakeExprNode(NULL); assert(0 == qptNodesListMakeStrictAppend(&pDataBlockDesc->pSlots, pDesc)); pDataBlockDesc->totalRowSize += qptCtx.param.correctExpected ? ((SExprNode*)pExpr)->resType.bytes : taosRand(); @@ -1252,7 +1329,7 @@ void qptAddDataBlockSlots(SNodeList* pList, SDataBlockDescNode* pDataBlockDesc) slotId = nextSlotId; ++nextSlotId; - if (qptCtx.param.correctExpected || QPT_RAND_BOOL_V) { + if (QPT_CORRECT_HIGH_PROB()) { SNode* pTarget = NULL; qptMakeTargetNode(pNode, pDataBlockDesc->dataBlockId, slotId, &pTarget); REPLACE_NODE(pTarget); @@ -1336,27 +1413,36 @@ SNode* qptCreateProjectPhysiNode(int32_t nodeType) { SNode* qptCreateSortMergeJoinPhysiNode(int32_t nodeType) { SPhysiNode* pPhysiNode = qptCreatePhysiNode(nodeType); - assert(pPhysiNode); - SSortMergeJoinPhysiNode* p = (SSortMergeJoinPhysiNode*)pPhysiNode; + SSortMergeJoinPhysiNode* pJoin = (SSortMergeJoinPhysiNode*)pPhysiNode; -/* - p->joinType = param->joinType; - p->subType = param->subType; - p->asofOpType = param->asofOp; - p->grpJoin = param->grpJoin; - if (p->subType == JOIN_STYPE_WIN || param->jLimit > 1 || taosRand() % 2) { - SLimitNode* limitNode = NULL; - code = nodesMakeNode(QUERY_NODE_LIMIT, (SNode**)&limitNode); - assert(limitNode); - limitNode->limit = param->jLimit; - p->pJLimit = (SNode*)limitNode; - } + pJoin->joinType = taosRand() % JOIN_TYPE_MAX_VALUE + (QPT_CORRECT_HIGH_PROB() ? 0 : 1); + pJoin->subType = taosRand() % JOIN_STYPE_MAX_VALUE + (QPT_CORRECT_HIGH_PROB() ? 0 : 1); + qptMakeWindowOffsetNode(&pJoin->pWindowOffset); + qptMakeLimitNode(&pJoin->pJLimit); + pJoin->asofOpType = OPERATOR_ARRAY[taosRand() % (sizeof(OPERATOR_ARRAY)/sizeof(OPERATOR_ARRAY[0]))] + (QPT_CORRECT_HIGH_PROB() ? 0 : 1); + + qptMakeExprNode(&pJoin->leftPrimExpr); + qptMakeExprNode(&pJoin->rightPrimExpr); + pJoin->leftPrimSlotId = qptGetInputSlotId(qptCtx->buildCtx.pChild ? qptCtx->buildCtx.pChild->pOutputDataBlockDesc : NULL); + pJoin->rightPrimSlotId = qptGetInputSlotId(qptCtx->buildCtx.pChild ? qptCtx->buildCtx.pChild->pOutputDataBlockDesc : NULL); + qptMakeColumnList(&pJoin->pEqLeft); + qptMakeColumnList(&pJoin->pEqRight); + qptMakeExprNode(&pJoin->pPrimKeyCond); + qptMakeExprNode(&pJoin->pColEqCond); + qptMakeExprNode(&pJoin->pColOnCond); + qptMakeExprNode(&pJoin->pFullOnCond); + qptMakeTargetNode(SNode * pNode, int16_t dataBlockId, int16_t slotId, SNode * * pOutput) + + pJoin->seqWinGroup = QPT_RAND_BOOL_V; + pJoin->grpJoin = QPT_RAND_BOOL_V; - p->leftPrimSlotId = JT_PRIM_TS_SLOT_ID; - p->rightPrimSlotId = JT_PRIM_TS_SLOT_ID; - p->node.inputTsOrder = param->asc ? ORDER_ASC : ORDER_DESC; - if (JOIN_STYPE_WIN == p->subType) { + + + pJoin->leftPrimSlotId = JT_PRIM_TS_SLOT_ID; + pJoin->rightPrimSlotId = JT_PRIM_TS_SLOT_ID; + pJoin->node.inputTsOrder = param->asc ? ORDER_ASC : ORDER_DESC; + if (JOIN_STYPE_WIN == pJoin->subType) { SWindowOffsetNode* pOffset = NULL; code = nodesMakeNode(QUERY_NODE_WINDOW_OFFSET, (SNode**)&pOffset); assert(pOffset); @@ -1377,7 +1463,7 @@ SNode* qptCreateSortMergeJoinPhysiNode(int32_t nodeType) { } pOffset->pStartOffset = (SNode*)pStart; pOffset->pEndOffset = (SNode*)pEnd; - p->pWindowOffset = (SNode*)pOffset; + pJoin->pWindowOffset = (SNode*)pOffset; jtCtx.winStartOffset = pStart->datum.i; jtCtx.winEndOffset = pEnd->datum.i; @@ -1393,15 +1479,14 @@ SNode* qptCreateSortMergeJoinPhysiNode(int32_t nodeType) { jtCtx.rightColOnly = (JOIN_TYPE_RIGHT == param->joinType && JOIN_STYPE_SEMI == param->subType); jtCtx.inGrpId = 1; - createColCond(p, param->cond); - createFilterStart(p, param->filter); - createTargetSlotList(p); - createColEqCondEnd(p); - createColOnCondEnd(p); - createFilterEnd(p, param->filter); + createColCond(pJoin, param->cond); + createFilterStart(pJoin, param->filter); + createTargetSlotList(pJoin); + createColEqCondEnd(pJoin); + createColOnCondEnd(pJoin); + createFilterEnd(pJoin, param->filter); updateColRowInfo(); - createBlockDescNode(&p->node.pOutputDataBlockDesc); -*/ + createBlockDescNode(&pJoin->node.pOutputDataBlockDesc); return (SNode*)pPhysiNode; } @@ -1478,6 +1563,8 @@ void qptRerunBlockedHere() { } void qptResetForReRun() { + qptCtx.param.plan.taskId = 1; + qptCtx.param.vnode.vgId = 1; for (int32_t i = 0; i < qptCtx.param.tbl.colNum; ++i) { qptCtx.param.tbl.pCol[i].inUse = 0; } @@ -1526,7 +1613,37 @@ void qptHandleTestEnd() { } -void qptRunPlanTest(char* caseName) { +void qptRunSingleOpTest(char* caseName) { + SNode* pNode = NULL; + SReadHandle readHandle = {0}; + SOperatorInfo* pOperator = NULL; + SExecTaskInfo* pTaskInfo = NULL; + SStorageAPI storageAPI = {0}; + + if (qptCtx.loopIdx > 0) { + qptResetForReRun(); + } + + pNode = (SNode*)qptCreatePhysicalPlanNode(qptCtx.param.plan.subplanType[0]); + + qptPrintBeginInfo(caseName); + + doCreateTask(qptCtx.param.plan.queryId, qptCtx.param.plan.taskId++, qptCtx.param.vnode.vgId, OPTR_EXEC_MODEL_BATCH, &storageAPI, &pTaskInfo); + + qptCtx.startTsUs = taosGetTimestampUs(); + //qptCtx.result.code = createTagScanOperatorInfo(&readHandle, (STagScanPhysiNode*)pNode, NULL, NULL, NULL, pTaskInfo, &pOperator); + //qptCtx.result.code = createProjectOperatorInfo(NULL, (SProjectPhysiNode*)pNode, pTaskInfo, &pOperator); + + doDestroyTask(pTaskInfo); + destroyOperator(pOperator); + nodesDestroyNode((SNode*)pNode); + + qptPrintEndInfo(caseName); + + qptHandleTestEnd(); +} + +void qptRunSubplanTest(char* caseName) { SNode* pNode = NULL; SReadHandle readHandle = {0}; SOperatorInfo* pOperator = NULL; @@ -1535,14 +1652,13 @@ void qptRunPlanTest(char* caseName) { qptResetForReRun(); } - if (qptCtx.param.plan.singlePhysiNode) { - pNode = (SNode*)qptCreatePhysicalPlanNode(qptCtx.param.plan.subplanType[0]); - } + //pNode = (SNode*)qptCreatePhysicalPlanNode(qptCtx.param.plan.subplanType[0]); qptPrintBeginInfo(caseName); qptCtx.startTsUs = taosGetTimestampUs(); //qptCtx.result.code = createTagScanOperatorInfo(&readHandle, (STagScanPhysiNode*)pNode, NULL, NULL, NULL, NULL, &pOperator); + //qptCtx.result.code = createProjectOperatorInfo(NULL, (SProjectPhysiNode*)pNode, NULL, &pOperator); destroyOperator(pOperator); nodesDestroyNode((SNode*)pNode); @@ -1553,6 +1669,13 @@ void qptRunPlanTest(char* caseName) { } +void qptRunPlanTest(char* caseName) { + if (qptCtx.param.plan.singlePhysiNode) { + qptRunSingleOpTest(caseName); + } else { + qptRunSubplanTest(caseName); + } +} SQPTNodeParam* qptInitNodeParam(int32_t nodeType) { return NULL; @@ -1664,7 +1787,7 @@ void qptInitTestCtx(bool correctExpected, bool singleNode, int32_t nodeType, int } // namespace #if 1 -#if 1 +#if 0 TEST(randSingleNodeTest, tagScan) { char* caseName = "randSingleNodeTest:tagScan"; @@ -1677,6 +1800,22 @@ TEST(randSingleNodeTest, tagScan) { qptPrintStatInfo(caseName); } #endif +#if 1 +TEST(randSingleNodeTest, projection) { + char* caseName = "randSingleNodeTest:projection"; + + qptInitTestCtx(false, true, QUERY_NODE_PHYSICAL_PLAN_PROJECT, 0, NULL); + + for (qptCtx.loopIdx = 0; qptCtx.loopIdx < QPT_MAX_LOOP; ++qptCtx.loopIdx) { + qptRunPlanTest(caseName); + } + + qptPrintStatInfo(caseName); +} +#endif + + + #if 0 TEST(correctSingleNodeTest, tagScan) { From 01a4256d0a1625a476efb738ef1735534663a3bb Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 10 Oct 2024 11:32:37 +0800 Subject: [PATCH 09/29] fix: add merge join --- source/libs/executor/test/queryPlanTests.cpp | 146 ++++++++----------- 1 file changed, 64 insertions(+), 82 deletions(-) diff --git a/source/libs/executor/test/queryPlanTests.cpp b/source/libs/executor/test/queryPlanTests.cpp index 9cfdf931a4..e1cd313aae 100755 --- a/source/libs/executor/test/queryPlanTests.cpp +++ b/source/libs/executor/test/queryPlanTests.cpp @@ -1106,8 +1106,9 @@ void qptMakeNonRealTableNode(SNode** ppNode) { } SNode* qptMakeRandNode(SNode** ppNode) { - nodesMakeNode((ENodeType)taosRand(), ppNode); - return *ppNode; + SNode* pNode = NULL; + nodesMakeNode((ENodeType)taosRand(), ppNode ? ppNode : &pNode); + return ppNode ? *ppNode : pNode; } SNode* qptMakeExprNode(SNode** ppNode) { @@ -1179,7 +1180,7 @@ SNode* qptMakeConditionNode(bool onlyTag) { } SNode* qptMakeDataBlockDescNode() { - if (!qptCtx.param.correctExpected && QPT_LOW_PROB()) { + if (QPT_NCORRECT_LOW_PROB()) { return NULL; } @@ -1192,6 +1193,41 @@ SNode* qptMakeDataBlockDescNode() { return (SNode*)pDesc; } +SNode* qptMakeSlotDescNode(const char* pName, const SNode* pNode, int16_t slotId, bool output, bool reserve) { + SSlotDescNode* pSlot = NULL; + if (QPT_NCORRECT_LOW_PROB) { + return qptMakeRandNode((SNode**)&pSlot); + } + + assert(0 == nodesMakeNode(QUERY_NODE_SLOT_DESC, (SNode**)&pSlot)); + + QPT_RAND_BOOL_V ? (pSlot->name[0] = 0) : snprintf(pSlot->name, sizeof(pSlot->name), "%s", pName); + pSlot->slotId = QPT_CORRECT_HIGH_PROB() ? slotId : taosRand(); + if (QPT_CORRECT_HIGH_PROB()) { + pSlot->dataType = ((SExprNode*)pNode)->resType; + } else { + qptGetRandValue(&pSlot->dataType.type, &pSlot->dataType.bytes, NULL); + } + + pSlot->reserve = reserve; + pSlot->output = output; + return (SNode*)pSlot; +} + +void qptMakeTargetNode(SNode* pNode, int16_t dataBlockId, int16_t slotId, SNode** pOutput) { + if (QPT_NCORRECT_LOW_PROB) { + return qptMakeRandNode(pOutput); + } + + STargetNode* pTarget = NULL; + assert(0 == nodesMakeNode(QUERY_NODE_TARGET, (SNode**)&pTarget)); + + pTarget->dataBlockId = QPT_CORRECT_HIGH_PROB() ? dataBlockId : taosRand(); + pTarget->slotId = QPT_CORRECT_HIGH_PROB() ? slotId : taosRand(); + pTarget->pExpr = QPT_CORRECT_HIGH_PROB() ? pNode : qptMakeRandNode(NULL); + + *pOutput = (SNode*)pTarget; +} SPhysiNode* qptCreatePhysiNode(int32_t nodeType) { SPhysiNode* pPhysiNode = NULL; @@ -1274,33 +1310,6 @@ void qptCreateTableScanPseudoCols( SScanPhysiNode* pScanPhysiNode) { qptCreateTableScanColsImpl(pScanPhysiNode, &pScanPhysiNode->pScanPseudoCols, qptCtx.param.tbl.tagNum, qptCtx.param.tbl.pTag); } -SNode* qptMakeSlotDescNode(const char* pName, const SNode* pNode, int16_t slotId, bool output, bool reserve) { - SSlotDescNode* pSlot = NULL; - assert(0 == nodesMakeNode(QUERY_NODE_SLOT_DESC, (SNode**)&pSlot)); - - QPT_RAND_BOOL_V ? (pSlot->name[0] = 0) : snprintf(pSlot->name, sizeof(pSlot->name), "%s", pName); - pSlot->slotId = qptCtx.param.correctExpected ? slotId : taosRand(); - if (qptCtx.param.correctExpected) { - pSlot->dataType = ((SExprNode*)pNode)->resType; - } else { - qptGetRandValue(&pSlot->dataType.type, &pSlot->dataType.bytes, NULL); - } - - pSlot->reserve = reserve; - pSlot->output = output; - return (SNode*)pSlot; -} - -void qptMakeTargetNode(SNode* pNode, int16_t dataBlockId, int16_t slotId, SNode** pOutput) { - STargetNode* pTarget = NULL; - assert(0 == nodesMakeNode(QUERY_NODE_TARGET, (SNode**)&pTarget)); - - pTarget->dataBlockId = dataBlockId; - pTarget->slotId = slotId; - pTarget->pExpr = pNode; - - *pOutput = (SNode*)pTarget; -} void qptAddDataBlockSlots(SNodeList* pList, SDataBlockDescNode* pDataBlockDesc) { if (NULL == pDataBlockDesc) { @@ -1392,6 +1401,26 @@ void qptMakeExprList(SNodeList** ppList) { } } +void qptMakeColumnList(SNodeList** ppList) { + int32_t colNum = taosRand() % QPT_MAX_COLUMN_NUM + (QPT_CORRECT_HIGH_PROB() ? 1 : 0); + qptResetMakeNodeCtx(qptCtx.buildCtx.pChild ? qptCtx.buildCtx.pChild->pOutputDataBlockDesc : NULL, false); + for (int32_t i = 0; i < colNum; ++i) { + SNode* pNode = NULL; + qptMakeColumnNode(&pNode); + qptNodesListMakeStrictAppend(ppList, pNode); + } +} + +void qptMakeTargetList(int16_t datablockId, SNodeList** ppList) { + int32_t tarNum = taosRand() % QPT_MAX_COLUMN_NUM + (QPT_CORRECT_HIGH_PROB() ? 1 : 0); + for (int32_t i = 0; i < tarNum; ++i) { + SNode* pNode = NULL, *pExpr = NULL; + qptMakeColumnNode(&pExpr); + qptMakeTargetNode(pExpr, datablockId, i, &pNode); + qptNodesListMakeStrictAppend(ppList, pNode); + } +} + SNode* qptCreateProjectPhysiNode(int32_t nodeType) { SPhysiNode* pPhysiNode = qptCreatePhysiNode(nodeType); @@ -1432,61 +1461,14 @@ SNode* qptCreateSortMergeJoinPhysiNode(int32_t nodeType) { qptMakeExprNode(&pJoin->pColEqCond); qptMakeExprNode(&pJoin->pColOnCond); qptMakeExprNode(&pJoin->pFullOnCond); - qptMakeTargetNode(SNode * pNode, int16_t dataBlockId, int16_t slotId, SNode * * pOutput) + qptMakeTargetList(pPhysiNode->pOutputDataBlockDesc ? pPhysiNode->pOutputDataBlockDesc->dataBlockId, &pJoin->pTargets); + for (int32_t i = 0; i < 2; i++) { + pJoin->inputStat[i].inputRowNum = taosRand(); + pJoin->inputStat[i].inputRowSize = taosRand(); + } pJoin->seqWinGroup = QPT_RAND_BOOL_V; pJoin->grpJoin = QPT_RAND_BOOL_V; - - - - pJoin->leftPrimSlotId = JT_PRIM_TS_SLOT_ID; - pJoin->rightPrimSlotId = JT_PRIM_TS_SLOT_ID; - pJoin->node.inputTsOrder = param->asc ? ORDER_ASC : ORDER_DESC; - if (JOIN_STYPE_WIN == pJoin->subType) { - SWindowOffsetNode* pOffset = NULL; - code = nodesMakeNode(QUERY_NODE_WINDOW_OFFSET, (SNode**)&pOffset); - assert(pOffset); - SValueNode* pStart = NULL; - code = nodesMakeNode(QUERY_NODE_VALUE, (SNode**)&pStart); - assert(pStart); - SValueNode* pEnd = NULL; - code = nodesMakeNode(QUERY_NODE_VALUE, (SNode**)&pEnd); - assert(pEnd); - pStart->node.resType.type = TSDB_DATA_TYPE_BIGINT; - pStart->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; - pStart->datum.i = (taosRand() % 2) ? (((int32_t)-1) * (int64_t)(taosRand() % JT_MAX_WINDOW_OFFSET)) : (taosRand() % JT_MAX_WINDOW_OFFSET); - pEnd->node.resType.type = TSDB_DATA_TYPE_BIGINT; - pEnd->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; - pEnd->datum.i = (taosRand() % 2) ? (((int32_t)-1) * (int64_t)(taosRand() % JT_MAX_WINDOW_OFFSET)) : (taosRand() % JT_MAX_WINDOW_OFFSET); - if (pStart->datum.i > pEnd->datum.i) { - TSWAP(pStart->datum.i, pEnd->datum.i); - } - pOffset->pStartOffset = (SNode*)pStart; - pOffset->pEndOffset = (SNode*)pEnd; - pJoin->pWindowOffset = (SNode*)pOffset; - - jtCtx.winStartOffset = pStart->datum.i; - jtCtx.winEndOffset = pEnd->datum.i; - } - - jtCtx.grpJoin = param->grpJoin; - jtCtx.joinType = param->joinType; - jtCtx.subType = param->subType; - jtCtx.asc = param->asc; - jtCtx.jLimit = param->jLimit; - jtCtx.asofOpType = param->asofOp; - jtCtx.leftColOnly = (JOIN_TYPE_LEFT == param->joinType && JOIN_STYPE_SEMI == param->subType); - jtCtx.rightColOnly = (JOIN_TYPE_RIGHT == param->joinType && JOIN_STYPE_SEMI == param->subType); - jtCtx.inGrpId = 1; - - createColCond(pJoin, param->cond); - createFilterStart(pJoin, param->filter); - createTargetSlotList(pJoin); - createColEqCondEnd(pJoin); - createColOnCondEnd(pJoin); - createFilterEnd(pJoin, param->filter); - updateColRowInfo(); - createBlockDescNode(&pJoin->node.pOutputDataBlockDesc); return (SNode*)pPhysiNode; } From ab61e051942c658b5172cf5cff36470c8e2f2383 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 15 Oct 2024 19:19:21 +0800 Subject: [PATCH 10/29] enh: add rand query plan --- source/libs/executor/test/queryPlanTests.cpp | 1656 +++++++++++++----- source/libs/nodes/src/nodesUtilFuncs.c | 3 + 2 files changed, 1246 insertions(+), 413 deletions(-) diff --git a/source/libs/executor/test/queryPlanTests.cpp b/source/libs/executor/test/queryPlanTests.cpp index e1cd313aae..2be84a5785 100755 --- a/source/libs/executor/test/queryPlanTests.cpp +++ b/source/libs/executor/test/queryPlanTests.cpp @@ -42,10 +42,11 @@ #include "stub.h" #include "querytask.h" #include "functionMgt.h" +#include "ttime.h" namespace { -#define QPT_MAX_LOOP 100000000 +#define QPT_MAX_LOOP 10000 #define QPT_MAX_SUBPLAN_LEVEL 1000 #define QPT_MAX_WHEN_THEN_NUM 10 #define QPT_MAX_NODE_LEVEL 5 @@ -54,70 +55,22 @@ namespace { #define QPT_MAX_LOGIC_PARAM 5 #define QPT_MAX_NODE_LIST_NUM 5 #define QPT_DEFAULT_VNODE_NUM 5 -#define QPT_MAX_COLUMN_NUM 8192 +#define QPT_MAX_DS_SRC_NUM 10 +#define QPT_MAX_ORDER_BY_NUM 10 +#define QPT_MAX_COLUMN_NUM 6 //8192 +#define QPT_QUERY_NODE_COL 10000 -int32_t QPT_PHYSIC_NODE_LIST[] = { - QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN, - QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, - QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN, - QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN, - QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN, - QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN, - QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN, - QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN, - QUERY_NODE_PHYSICAL_PLAN_PROJECT, - QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN, - QUERY_NODE_PHYSICAL_PLAN_HASH_AGG, - QUERY_NODE_PHYSICAL_PLAN_EXCHANGE, - QUERY_NODE_PHYSICAL_PLAN_MERGE, - QUERY_NODE_PHYSICAL_PLAN_SORT, - QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT, - QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL, - QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL, - QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL, - QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, - QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL, - QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL, - QUERY_NODE_PHYSICAL_PLAN_FILL, - QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL, - QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION, - QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION, - QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION, - QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION, - QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE, - QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE, - QUERY_NODE_PHYSICAL_PLAN_PARTITION, - QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION, - QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC, - QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC, - QUERY_NODE_PHYSICAL_PLAN_DISPATCH, - QUERY_NODE_PHYSICAL_PLAN_INSERT, - QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT, - QUERY_NODE_PHYSICAL_PLAN_DELETE, - QUERY_NODE_PHYSICAL_SUBPLAN, - QUERY_NODE_PHYSICAL_PLAN, - QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN, - QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT, - QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT, - QUERY_NODE_PHYSICAL_PLAN_HASH_JOIN, - QUERY_NODE_PHYSICAL_PLAN_GROUP_CACHE, - QUERY_NODE_PHYSICAL_PLAN_DYN_QUERY_CTRL, - QUERY_NODE_PHYSICAL_PLAN_MERGE_COUNT, - QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT, - QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL -}; +typedef enum { + QPT_NODE_COLUMN, + QPT_NODE_EXPR, + QPT_NODE_FUNCTION, + QPT_NODE_VALUE, + QPT_NODE_MAX_VALUE +} QPT_NODE_TYPE; -#define QPT_PHYSIC_NODE_NUM() (sizeof(QPT_PHYSIC_NODE_LIST)/sizeof(QPT_PHYSIC_NODE_LIST[0])) -#define QPT_RAND_BOOL_V ((taosRand() % 2) ? true : false) -#define QPT_RAND_ORDER_V (QPT_RAND_BOOL_V ? ORDER_ASC : ORDER_DESC) -#define QPT_RAND_INT_V (taosRand() * (QPT_RAND_BOOL_V ? 1 : -1)) -#define QPT_LOW_PROB() ((taosRand() % 11) == 0) -#define QPT_MID_PROB() ((taosRand() % 11) <= 1) -#define QPT_HIGH_PROB() ((taosRand() % 11) <= 7) +typedef SNode* (*planBuildFunc)(int32_t); -#define QPT_CORRECT_HIGH_PROB() (qptCtx.param.correctExpected || QPT_HIGH_PROB()) -#define QPT_NCORRECT_LOW_PROB() (!qptCtx.param.correctExpected && QPT_LOW_PROB()) typedef struct { ENodeType type; @@ -130,6 +83,7 @@ typedef struct { uint64_t taskId; int32_t subplanMaxLevel; int32_t subplanType[QPT_MAX_SUBPLAN_LEVEL]; + int32_t subplanIdx[QPT_MAX_SUBPLAN_LEVEL]; int32_t physiNodeParamNum; SQPTNodeParam* physicNodeParam; } SQPTPlanParam; @@ -143,11 +97,13 @@ typedef struct { typedef struct { int32_t vnodeNum; int32_t vgId; + SEpSet epSet; } SQPTVnodeParam; typedef struct { - char name[TSDB_COL_NAME_LEN]; int32_t type; + char name[TSDB_COL_NAME_LEN]; + int32_t dtype; int32_t len; int8_t inUse; bool hasIndex; @@ -165,13 +121,15 @@ typedef struct { int16_t pkNum; char tblName[TSDB_TABLE_NAME_LEN]; char tblAlias[TSDB_TABLE_NAME_LEN]; - SQPTCol* pCol; - SQPTCol* pTag; + SNodeList* pColList; + SNodeList* pTagList; + SNodeList* pColTagList; } SQPTTblParam; typedef struct { bool correctExpected; + uint64_t schedulerId; SQPTPlanParam plan; SQPTDbParam db; SQPTVnodeParam vnode; @@ -186,13 +144,18 @@ typedef struct { SPhysiNode* pLeftChild; SPhysiNode* pRightChild; EOrder currTsOrder; + int16_t nextBlockId; + int32_t primaryTsSlotId; + SExecTaskInfo* pCurrTask; } SQPTBuildPlanCtx; typedef struct { int32_t nodeLevel; + bool fromTable; bool onlyTag; - int16_t nextBlockId; - SDataBlockDescNode* pInputDataBlockDesc; + bool onlyCol; + int16_t inputBlockId; + SNodeList* pInputList; } SQPTMakeNodeCtx; typedef struct { @@ -201,9 +164,11 @@ typedef struct { typedef struct { int32_t loopIdx; + char caseName[128]; SQPTParam param; SQPTBuildPlanCtx buildCtx; SQPTMakeNodeCtx makeCtx; + SQPTMakeNodeCtx makeCtxBak; SQPTExecResult result; int64_t startTsUs; } SQPTCtx; @@ -216,21 +181,182 @@ typedef struct { bool noKeepResRows; } SQPTCtrl; +typedef struct { + int32_t type; + char* name; + planBuildFunc buildFunc; +} SQPTPlan; + +SNode* qptCreateTagScanPhysiNode(int32_t nodeType); +SNode* qptCreateTableScanPhysiNode(int32_t nodeType); +SNode* qptCreateTableSeqScanPhysiNode(int32_t nodeType); +SNode* qptCreateTableMergeScanPhysiNode(int32_t nodeType); +SNode* qptCreateStreamScanPhysiNode(int32_t nodeType); +SNode* qptCreateSysTableScanPhysiNode(int32_t nodeType); +SNode* qptCreateBlockDistScanPhysiNode(int32_t nodeType); +SNode* qptCreateLastRowScanPhysiNode(int32_t nodeType); +SNode* qptCreateProjectPhysiNode(int32_t nodeType); +SNode* qptCreateMergeJoinPhysiNode(int32_t nodeType); +SNode* qptCreateHashAggPhysiNode(int32_t nodeType); +SNode* qptCreateExchangePhysiNode(int32_t nodeType); +SNode* qptCreateMergePhysiNode(int32_t nodeType); +SNode* qptCreateSortPhysiNode(int32_t nodeType); +SNode* qptCreateGroupSortPhysiNode(int32_t nodeType); +SNode* qptCreateIntervalPhysiNode(int32_t nodeType); +SNode* qptCreateMergeIntervalPhysiNode(int32_t nodeType); +SNode* qptCreateMergeAlignedIntervalPhysiNode(int32_t nodeType); +SNode* qptCreateStreamIntervalPhysiNode(int32_t nodeType); +SNode* qptCreateStreamFinalIntervalPhysiNode(int32_t nodeType); +SNode* qptCreateStreamSemiIntervalPhysiNode(int32_t nodeType); +SNode* qptCreateFillPhysiNode(int32_t nodeType); +SNode* qptCreateStreamFillPhysiNode(int32_t nodeType); +SNode* qptCreateSessionPhysiNode(int32_t nodeType); +SNode* qptCreateStreamSessionPhysiNode(int32_t nodeType); +SNode* qptCreateStreamSemiSessionPhysiNode(int32_t nodeType); +SNode* qptCreateStreamFinalSessionPhysiNode(int32_t nodeType); +SNode* qptCreateStateWindowPhysiNode(int32_t nodeType); +SNode* qptCreateStreamStatePhysiNode(int32_t nodeType); +SNode* qptCreatePartitionPhysiNode(int32_t nodeType); +SNode* qptCreateStreamPartitionPhysiNode(int32_t nodeType); + +SQPTPlan qptPlans[] = { + {QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN, "tagScan", qptCreateTagScanPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, "tableScan", qptCreateTableScanPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN, "tableSeqScan", qptCreateTableSeqScanPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN, "tableMergeScan", qptCreateTableMergeScanPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN, "streamScan", qptCreateStreamScanPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN, "sysTableScan", qptCreateSysTableScanPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN, "blockDistScan", qptCreateBlockDistScanPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN, "lastRowScan", qptCreateLastRowScanPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_PROJECT, "project", qptCreateProjectPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN, "mergeJoin", qptCreateMergeJoinPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_HASH_AGG, "hashAgg", qptCreateHashAggPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_EXCHANGE, "exchange", qptCreateExchangePhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_MERGE, "merge", qptCreateMergePhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_SORT, "sort", qptCreateSortPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT, "groupSort", qptCreateGroupSortPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL, "interval", qptCreateIntervalPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL, "mergeInterval", qptCreateMergeIntervalPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL, "mergeAlignedInterval", qptCreateMergeAlignedIntervalPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, "streamInterval", qptCreateStreamIntervalPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL, "streamFinalInterval", qptCreateStreamFinalIntervalPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL, "streamSemiInterval", qptCreateStreamSemiIntervalPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_FILL, "fill", qptCreateFillPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL, "streamFill", qptCreateStreamFillPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION, "sessionWindow", qptCreateSessionPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION, "streamSession", qptCreateStreamSessionPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION, "streamSemiSession", qptCreateStreamSemiSessionPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION, "streamFinalSession", qptCreateStreamFinalSessionPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE, "stateWindow", qptCreateStateWindowPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE, "streamState", qptCreateStreamStatePhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_PARTITION, "partition", qptCreatePartitionPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION, "streamPartition", qptCreateStreamPartitionPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC, "", NULL}, + {QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC, "", NULL}, + {QUERY_NODE_PHYSICAL_PLAN_DISPATCH, "", NULL}, + {QUERY_NODE_PHYSICAL_PLAN_INSERT, "", NULL}, + {QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT, "", NULL}, + {QUERY_NODE_PHYSICAL_PLAN_DELETE, "", NULL}, + {QUERY_NODE_PHYSICAL_SUBPLAN, "", NULL}, + {QUERY_NODE_PHYSICAL_PLAN, "", NULL}, + {QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN, "", NULL}, + {QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT, "", NULL}, + {QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT, "", NULL}, + {QUERY_NODE_PHYSICAL_PLAN_HASH_JOIN, "", NULL}, + {QUERY_NODE_PHYSICAL_PLAN_GROUP_CACHE, "", NULL}, + {QUERY_NODE_PHYSICAL_PLAN_DYN_QUERY_CTRL, "", NULL}, + {QUERY_NODE_PHYSICAL_PLAN_MERGE_COUNT, "", NULL}, + {QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT, "", NULL}, + {QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL, "", NULL} +}; + + +#define QPT_PHYSIC_NODE_NUM() (sizeof(qptPlans)/sizeof(qptPlans[0])) +#define QPT_RAND_BOOL_V ((taosRand() % 2) ? true : false) +#define QPT_RAND_ORDER_V (QPT_RAND_BOOL_V ? ORDER_ASC : ORDER_DESC) +#define QPT_RAND_INT_V (taosRand() * (QPT_RAND_BOOL_V ? 1 : -1)) +#define QPT_LOW_PROB() ((taosRand() % 11) == 0) +#define QPT_MID_PROB() ((taosRand() % 11) <= 1) +#define QPT_HIGH_PROB() ((taosRand() % 11) <= 7) + +#define QPT_CORRECT_HIGH_PROB() (qptCtx.param.correctExpected || QPT_HIGH_PROB()) +#define QPT_NCORRECT_LOW_PROB() (!qptCtx.param.correctExpected && QPT_LOW_PROB()) + + SQPTCtx qptCtx = {0}; SQPTCtrl qptCtrl = {1, 0, 0, 0, 0}; bool qptErrorRerun = false; bool qptInRerun = false; + SNode* qptMakeExprNode(SNode** ppNode); +void qptMakeNodeList(QPT_NODE_TYPE nodeType, SNodeList** ppList); -void qptPrintBeginInfo(char* caseName) { +int32_t qptGetColumnRandLen(int32_t colType) { + switch (colType) { + case TSDB_DATA_TYPE_NULL: + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: + case TSDB_DATA_TYPE_UTINYINT: + case TSDB_DATA_TYPE_USMALLINT: + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_UBIGINT: + return tDataTypes[colType].bytes; + case TSDB_DATA_TYPE_VARCHAR: + case TSDB_DATA_TYPE_GEOMETRY: + case TSDB_DATA_TYPE_JSON: + case TSDB_DATA_TYPE_VARBINARY: + case TSDB_DATA_TYPE_DECIMAL: + case TSDB_DATA_TYPE_BLOB: + case TSDB_DATA_TYPE_MEDIUMBLOB: + case TSDB_DATA_TYPE_NCHAR: + return taosRand() % TSDB_MAX_BINARY_LEN; + default: + assert(0); + break; + } +} + + +void qptInitSingleTableCol(SQPTCol* pCol, int32_t idx, EColumnType colType) { + if (COLUMN_TYPE_COLUMN == colType && 0 == idx) { + sprintf(pCol->name, "primts%d", idx); + pCol->dtype = TSDB_DATA_TYPE_TIMESTAMP; + pCol->len = qptGetColumnRandLen(pCol->dtype); + pCol->inUse = 0; + pCol->hasIndex = false; + pCol->isPrimTs = true; + pCol->isPk = false; + pCol->colType = colType; + + return; + } + + sprintf(pCol->name, "%s%d", COLUMN_TYPE_COLUMN == colType ? "col" : "tag", idx); + pCol->dtype = taosRand() % TSDB_DATA_TYPE_MAX; + pCol->len = qptGetColumnRandLen(pCol->dtype); + pCol->inUse = 0; + pCol->hasIndex = COLUMN_TYPE_COLUMN == colType ? false : QPT_RAND_BOOL_V; + pCol->isPrimTs = false; + pCol->isPk = COLUMN_TYPE_COLUMN == colType ? QPT_RAND_BOOL_V : false;; + pCol->colType = colType; +} + + +void qptPrintBeginInfo() { if (!qptCtrl.printTestInfo) { return; } - printf("\n%dth TEST [%s] START\n", qptCtx.loopIdx, caseName); + printf("\n%dth TEST [%s] START\n", qptCtx.loopIdx, qptCtx.caseName); /* char inputStat[4] = {0}; @@ -287,17 +413,17 @@ void qptPrintBeginInfo(char* caseName) { */ } -void qptPrintEndInfo(char* caseName) { +void qptPrintEndInfo() { if (!qptCtrl.printTestInfo) { return; } - printf("\n\t%dth TEST [%s] END, result - %s%s\n", qptCtx.loopIdx, caseName, + printf("\n\t%dth TEST [%s] END, result - %s%s\n", qptCtx.loopIdx, qptCtx.caseName, (0 == qptCtx.result.code) ? "succeed" : "failed with error:", (0 == qptCtx.result.code) ? "" : tstrerror(qptCtx.result.code)); } -void qptPrintStatInfo(char* caseName) { +void qptPrintStatInfo() { } @@ -318,8 +444,8 @@ EOrder qptGetCurrTsOrder() { return QPT_CORRECT_HIGH_PROB() ? qptCtx.buildCtx.currTsOrder : QPT_RAND_ORDER_V; } -void qptGetRandValue(int16_t* pType, int32_t* pLen, void** ppVal) { - if (*pType < 0 || QPT_NCORRECT_LOW_PROB()) { +void qptGetRandValue(uint8_t* pType, int32_t* pLen, void** ppVal) { + if (*pType == (uint8_t)-1 || QPT_NCORRECT_LOW_PROB()) { int32_t typeMax = TSDB_DATA_TYPE_MAX; if (QPT_NCORRECT_LOW_PROB()) { typeMax++; @@ -521,6 +647,28 @@ int32_t qptGetInputSlotId(SDataBlockDescNode* pInput) { return taosRand(); } +ENullOrder qptGetRandNullOrder() { + if (QPT_NCORRECT_LOW_PROB()) { + return (ENullOrder)taosRand(); + } + + return (ENullOrder)(taosRand() % NULL_ORDER_LAST + 1); +} + +int8_t qptGetRandTimestampUnit() { + static int8_t units[] = {TIME_UNIT_NANOSECOND, TIME_UNIT_MICROSECOND, TIME_UNIT_MILLISECOND, TIME_UNIT_SECOND, + TIME_UNIT_MINUTE, TIME_UNIT_HOUR, TIME_UNIT_DAY, TIME_UNIT_WEEK, TIME_UNIT_MONTH, TIME_UNIT_YEAR}; + + return units[taosRand() % (sizeof(units) / sizeof(units[0]))]; +} + +int32_t qptGetInputPrimaryTsSlotId() { + if (QPT_CORRECT_HIGH_PROB()) { + return qptCtx.buildCtx.primaryTsSlotId; + } + + return taosRand() % QPT_MAX_COLUMN_NUM; +} void qptNodesCalloc(int32_t num, int32_t size, void** pOut) { void* p = taosMemoryCalloc(num, size); @@ -529,6 +677,32 @@ void qptNodesCalloc(int32_t num, int32_t size, void** pOut) { *pOut = (char*)p + 1; } +void qptNodesFree(void* pNode) { + void* p = (char*)pNode - 1; + taosMemoryFree(p); +} + +EFillMode qptGetRandFillMode() { + if (QPT_CORRECT_HIGH_PROB()) { + return (EFillMode)(taosRand() % FILL_MODE_NEXT + 1); + } + + return (EFillMode)(taosRand()); +} + + +void qptGetRandTimeWindow(STimeWindow* pWindow) { + if (QPT_CORRECT_HIGH_PROB()) { + pWindow->skey = taosRand(); + pWindow->ekey = pWindow->skey + taosRand(); + return; + } + + pWindow->skey = taosRand(); + pWindow->ekey = taosRand(); +} + + int32_t qptNodesListAppend(SNodeList* pList, SNode* pNode) { SListCell* p = NULL; qptNodesCalloc(1, sizeof(SListCell), (void**)&p); @@ -546,7 +720,6 @@ int32_t qptNodesListAppend(SNodeList* pList, SNode* pNode) { return TSDB_CODE_SUCCESS; } - int32_t qptNodesListStrictAppend(SNodeList* pList, SNode* pNode) { int32_t code = qptNodesListAppend(pList, pNode); if (TSDB_CODE_SUCCESS != code) { @@ -565,56 +738,19 @@ int32_t qptNodesListMakeStrictAppend(SNodeList** pList, SNode* pNode) { return qptNodesListStrictAppend(*pList, pNode); } - -SNode* qptMakeLimitNode() { +SNode* qptMakeRandNode(SNode** ppNode) { SNode* pNode = NULL; + nodesMakeNode((ENodeType)taosRand(), ppNode ? ppNode : &pNode); + return ppNode ? *ppNode : pNode; +} + + + +SNode* qptMakeColumnFromTable(int32_t colIdx) { if (QPT_NCORRECT_LOW_PROB()) { - return qptMakeRandNode(&pNode); - } - - assert(0 == nodesMakeNode(QUERY_NODE_LIMIT, &pNode)); - assert(pNode); - - SLimitNode* pLimit = (SLimitNode*)pNode; - - if (!qptCtx.param.correctExpected) { - if (taosRand() % 2) { - pLimit->limit = taosRand() * ((taosRand() % 2) ? 1 : -1); - } - if (taosRand() % 2) { - pLimit->offset = taosRand() * ((taosRand() % 2) ? 1 : -1); - } - } else { - pLimit->limit = taosRand(); - if (taosRand() % 2) { - pLimit->offset = taosRand(); - } + return qptMakeRandNode(NULL); } - return pNode; -} - - -SNode* qptMakeWindowOffsetNode(SNode** ppNode) { - if (QPT_RAND_BOOL_V) { - return qptMakeRandNode(ppNode); - } - - SNode* pNode = NULL; - SWindowOffsetNode* pWinOffset = NULL; - - assert(0 == nodesMakeNode(QUERY_NODE_WINDOW_OFFSET, &pNode)); - assert(pNode); - - SWindowOffsetNode* pWinOffset = (SWindowOffsetNode*)pNode; - qptMakeValueNode(TSDB_DATA_TYPE_BIGINT, &pWinOffset->pStartOffset); - qptMakeValueNode(TSDB_DATA_TYPE_BIGINT, &pWinOffset->pEndOffset); - - return pNode; -} - - -SNode* qptMakeColumnNodeFromTable(int32_t colIdx, EColumnType colType, SScanPhysiNode* pScanPhysiNode) { if (colIdx < 0) { return NULL; } @@ -623,26 +759,34 @@ SNode* qptMakeColumnNodeFromTable(int32_t colIdx, EColumnType colType, SScanPhys assert(0 == nodesMakeNode(QUERY_NODE_COLUMN, (SNode**)&pCol)); assert(pCol); + SQPTCol fakeCol; + fakeCol.type = QPT_QUERY_NODE_COL; + qptInitSingleTableCol(&fakeCol, taosRand(), (EColumnType)(taosRand() % COLUMN_TYPE_GROUP_KEY + 1)); + + SQPTCol* pTbCol = qptCtx.makeCtx.pInputList ? (SQPTCol*)nodesListGetNode(qptCtx.makeCtx.pInputList, colIdx) : &fakeCol; + + int16_t blkId = QPT_CORRECT_HIGH_PROB() ? qptCtx.makeCtx.inputBlockId : taosRand(); + if (QPT_CORRECT_HIGH_PROB()) { - pCol->node.resType.type = qptCtx.param.tbl.pCol[colIdx].type; - pCol->node.resType.bytes = qptCtx.param.tbl.pCol[colIdx].len; + pCol->node.resType.type = pTbCol->dtype; + pCol->node.resType.bytes = pTbCol->len; pCol->tableId = qptCtx.param.tbl.uid; pCol->tableType = qptCtx.param.tbl.tblType; pCol->colId = colIdx; pCol->projIdx = colIdx; - pCol->colType = qptCtx.param.tbl.pCol[colIdx].colType; - pCol->hasIndex = qptCtx.param.tbl.pCol[colIdx].hasIndex; - pCol->isPrimTs = qptCtx.param.tbl.pCol[colIdx].isPrimTs; + pCol->colType = pTbCol->colType; + pCol->hasIndex = pTbCol->hasIndex; + pCol->isPrimTs = pTbCol->isPrimTs; strcpy(pCol->dbName, qptCtx.param.db.dbName); strcpy(pCol->tableName, qptCtx.param.tbl.tblName); strcpy(pCol->tableAlias, qptCtx.param.tbl.tblAlias); - strcpy(pCol->colName, qptCtx.param.tbl.pCol[colIdx].name); - pCol->dataBlockId = pScanPhysiNode->node.pOutputDataBlockDesc ? pScanPhysiNode->node.pOutputDataBlockDesc->dataBlockId : taosRand(); + strcpy(pCol->colName, pTbCol->name); + pCol->dataBlockId = blkId; pCol->slotId = colIdx; pCol->numOfPKs = qptCtx.param.tbl.pkNum; pCol->tableHasPk = qptCtx.param.tbl.pkNum > 0; - pCol->isPk = qptCtx.param.tbl.pCol[colIdx].isPk; + pCol->isPk = pTbCol->isPk; pCol->projRefIdx = 0; pCol->resIdx = 0; } else { @@ -652,7 +796,7 @@ SNode* qptMakeColumnNodeFromTable(int32_t colIdx, EColumnType colType, SScanPhys pCol->tableType = taosRand() % TSDB_TABLE_MAX; pCol->colId = QPT_RAND_BOOL_V ? taosRand() : colIdx; pCol->projIdx = taosRand(); - pCol->colType = QPT_RAND_BOOL_V ? qptCtx.param.tbl.pCol[colIdx].colType : (EColumnType)(taosRand() % (COLUMN_TYPE_GROUP_KEY + 1)); + pCol->colType = QPT_RAND_BOOL_V ? pTbCol->colType : (EColumnType)(taosRand() % (COLUMN_TYPE_GROUP_KEY + 1)); pCol->hasIndex = QPT_RAND_BOOL_V; pCol->isPrimTs = QPT_RAND_BOOL_V; if (QPT_RAND_BOOL_V) { @@ -673,14 +817,14 @@ SNode* qptMakeColumnNodeFromTable(int32_t colIdx, EColumnType colType, SScanPhys if (QPT_RAND_BOOL_V) { pCol->colName[0] = 0; } else { - strcpy(pCol->colName, qptCtx.param.tbl.pCol[colIdx].name); + strcpy(pCol->colName, pTbCol->name); } - pCol->dataBlockId = (QPT_RAND_BOOL_V && pScanPhysiNode->node.pOutputDataBlockDesc) ? pScanPhysiNode->node.pOutputDataBlockDesc->dataBlockId : taosRand(); + pCol->dataBlockId = blkId; pCol->slotId = QPT_RAND_BOOL_V ? taosRand() : colIdx; pCol->numOfPKs = QPT_RAND_BOOL_V ? taosRand() : qptCtx.param.tbl.pkNum; pCol->tableHasPk = QPT_RAND_BOOL_V ? QPT_RAND_BOOL_V : (qptCtx.param.tbl.pkNum > 0); - pCol->isPk = QPT_RAND_BOOL_V ? QPT_RAND_BOOL_V : qptCtx.param.tbl.pCol[colIdx].isPk; + pCol->isPk = QPT_RAND_BOOL_V ? QPT_RAND_BOOL_V : pTbCol->isPk; pCol->projRefIdx = taosRand(); pCol->resIdx = taosRand(); } @@ -689,8 +833,8 @@ SNode* qptMakeColumnNodeFromTable(int32_t colIdx, EColumnType colType, SScanPhys } -void qptMakeWhenThenNode(SNode** ppNode) { - if (QPT_NCORRECT_LOW_PROB) { +SNode* qptMakeWhenThenNode(SNode** ppNode) { + if (QPT_NCORRECT_LOW_PROB()) { return qptMakeRandNode(ppNode); } @@ -701,11 +845,13 @@ void qptMakeWhenThenNode(SNode** ppNode) { qptMakeExprNode(&pWhenThen->pWhen); qptMakeExprNode(&pWhenThen->pThen); + + return *ppNode; } -void qptMakeCaseWhenNode(SNode** ppNode) { - if (QPT_NCORRECT_LOW_PROB) { +SNode* qptMakeCaseWhenNode(SNode** ppNode) { + if (QPT_NCORRECT_LOW_PROB()) { return qptMakeRandNode(ppNode); } @@ -730,11 +876,13 @@ void qptMakeCaseWhenNode(SNode** ppNode) { qptMakeWhenThenNode(&pNode); qptNodesListMakeStrictAppend(&pCaseWhen->pWhenThenList, pNode); } + + return *ppNode; } -void qptMakeOperatorNode(SNode** ppNode) { - if (QPT_NCORRECT_LOW_PROB) { +SNode* qptMakeOperatorNode(SNode** ppNode) { + if (QPT_NCORRECT_LOW_PROB()) { return qptMakeRandNode(ppNode); } @@ -811,36 +959,51 @@ void qptMakeOperatorNode(SNode** ppNode) { } break; } + + return *ppNode; } -void qptMakeColumnNode(SNode** ppNode) { - if (QPT_NCORRECT_LOW_PROB) { +SNode* qptMakeColumnNode(SNode** ppNode) { + if (QPT_NCORRECT_LOW_PROB()) { return qptMakeRandNode(ppNode); } SColumnNode* pCol = NULL; - nodesMakeNode(QUERY_NODE_COLUMN, (SNode**)&pCol); - SSlotDescNode* pSlot = NULL; - if (QPT_CORRECT_HIGH_PROB() && qptCtx.makeCtx.pInputDataBlockDesc && qptCtx.makeCtx.pInputDataBlockDesc->pSlots) { - SNodeList* pColList = qptCtx.makeCtx.pInputDataBlockDesc->pSlots; + if (QPT_CORRECT_HIGH_PROB() && qptCtx.makeCtx.pInputList) { + SNodeList* pColList = qptCtx.makeCtx.pInputList; int32_t colIdx = taosRand() % pColList->length; - SNode* pNode = nodesListGetNode(pColList, colIdx); - if (pNode && nodeType(pNode) == QUERY_NODE_SLOT_DESC) { - pSlot = (SSlotDescNode*)pNode; - pCol->node.resType = pSlot->dataType; - pCol->dataBlockId = qptCtx.makeCtx.pInputDataBlockDesc->dataBlockId; - pCol->slotId = pSlot->slotId; + SQPTCol* pNode = (SQPTCol*)nodesListGetNode(pColList, colIdx); + if (pNode) { + switch (pNode->type) { + case QUERY_NODE_SLOT_DESC: { + nodesMakeNode(QUERY_NODE_COLUMN, (SNode**)&pCol); + SSlotDescNode* pSlot = (SSlotDescNode*)pNode; + pCol->node.resType = pSlot->dataType; + pCol->dataBlockId = qptCtx.makeCtx.inputBlockId; + pCol->slotId = pSlot->slotId; + break; + } + case QPT_QUERY_NODE_COL: { + pCol = (SColumnNode*)qptMakeColumnFromTable(colIdx); + break; + } + default: + break; + } } } - if (NULL == pSlot) { + if (NULL == pCol) { + nodesMakeNode(QUERY_NODE_COLUMN, (SNode**)&pCol); qptGetRandValue(&pCol->node.resType.type, &pCol->node.resType.bytes, NULL); pCol->dataBlockId = taosRand(); pCol->slotId = taosRand(); } *ppNode = (SNode*)pCol; + + return *ppNode; } void qptNodesSetValueNodeValue(SValueNode* pNode, void* value) { @@ -933,8 +1096,8 @@ void qptNodesSetValueNodeValue(SValueNode* pNode, void* value) { } -void qptMakeValueNode(int16_t valType, SNode** ppNode) { - if (QPT_NCORRECT_LOW_PROB) { +SNode* qptMakeValueNode(uint8_t valType, SNode** ppNode) { + if (QPT_NCORRECT_LOW_PROB()) { return qptMakeRandNode(ppNode); } @@ -951,17 +1114,19 @@ void qptMakeValueNode(int16_t valType, SNode** ppNode) { qptNodesSetValueNodeValue(pVal, pValue); *ppNode = (SNode*)pVal; + + return *ppNode; } -void qptMakeFunctionNode(SNode** ppNode) { - if (QPT_NCORRECT_LOW_PROB) { +SNode* qptMakeFunctionNode(SNode** ppNode) { + if (QPT_NCORRECT_LOW_PROB()) { return qptMakeRandNode(ppNode); } SFunctionNode* pFunc = NULL; nodesMakeNode(QUERY_NODE_FUNCTION, (SNode**)&pFunc); - if (qptCtx.param.correctExpected || QPT_HIGH_PROB()) { + if (QPT_CORRECT_HIGH_PROB()) { int32_t funcIdx = taosRand() % funcMgtBuiltinsNum; char* funcName = fmGetFuncName(funcIdx); strcpy(pFunc->functionName, funcName); @@ -991,13 +1156,15 @@ void qptMakeFunctionNode(SNode** ppNode) { } *ppNode = (SNode*)pFunc; + + return *ppNode; } -void qptMakeLogicCondNode(SNode** ppNode) { - if (QPT_NCORRECT_LOW_PROB) { +SNode* qptMakeLogicCondNode(SNode** ppNode) { + if (QPT_NCORRECT_LOW_PROB()) { return qptMakeRandNode(ppNode); } @@ -1020,10 +1187,12 @@ void qptMakeLogicCondNode(SNode** ppNode) { } *ppNode = (SNode*)pLogic; + + return *ppNode; } -void qptMakeNodeListNode(SNode** ppNode) { - if (QPT_NCORRECT_LOW_PROB) { +SNode* qptMakeNodeListNode(QPT_NODE_TYPE nodeType, SNode** ppNode) { + if (QPT_NCORRECT_LOW_PROB()) { return qptMakeRandNode(ppNode); } @@ -1031,19 +1200,16 @@ void qptMakeNodeListNode(SNode** ppNode) { nodesMakeNode(QUERY_NODE_NODE_LIST, (SNode**)&pList); qptCtx.makeCtx.nodeLevel++; - - int32_t nodeNum = QPT_CORRECT_HIGH_PROB() ? (taosRand() % QPT_MAX_NODE_LIST_NUM + 1) : (taosRand() % QPT_MAX_NODE_LIST_NUM); - for (int32_t i = 0; i < nodeNum; ++i) { - SNode* pNode = NULL; - qptMakeExprNode(&pNode); - qptNodesListMakeStrictAppend(&pList->pNodeList, pNode); - } + + qptMakeNodeList(nodeType, &pList->pNodeList); *ppNode = (SNode*)pList; + + return *ppNode; } -void qptMakeTempTableNode(SNode** ppNode) { - if (QPT_NCORRECT_LOW_PROB) { +SNode* qptMakeTempTableNode(SNode** ppNode) { + if (QPT_NCORRECT_LOW_PROB()) { return qptMakeRandNode(ppNode); } @@ -1055,10 +1221,12 @@ void qptMakeTempTableNode(SNode** ppNode) { } *ppNode = (SNode*)pTemp; + + return *ppNode; } -void qptMakeJoinTableNode(SNode** ppNode) { - if (QPT_NCORRECT_LOW_PROB) { +SNode* qptMakeJoinTableNode(SNode** ppNode) { + if (QPT_NCORRECT_LOW_PROB()) { return qptMakeRandNode(ppNode); } @@ -1070,10 +1238,12 @@ void qptMakeJoinTableNode(SNode** ppNode) { } *ppNode = (SNode*)pJoin; + + return *ppNode; } -void qptMakeRealTableNode(SNode** ppNode) { - if (QPT_NCORRECT_LOW_PROB) { +SNode* qptMakeRealTableNode(SNode** ppNode) { + if (QPT_NCORRECT_LOW_PROB()) { return qptMakeRandNode(ppNode); } @@ -1085,12 +1255,14 @@ void qptMakeRealTableNode(SNode** ppNode) { } *ppNode = (SNode*)pReal; + + return *ppNode; } -void qptMakeNonRealTableNode(SNode** ppNode) { - if (QPT_NCORRECT_LOW_PROB) { +SNode* qptMakeNonRealTableNode(SNode** ppNode) { + if (QPT_NCORRECT_LOW_PROB()) { return qptMakeRandNode(ppNode); } @@ -1103,12 +1275,8 @@ void qptMakeNonRealTableNode(SNode** ppNode) { } else { qptMakeRealTableNode(ppNode); } -} -SNode* qptMakeRandNode(SNode** ppNode) { - SNode* pNode = NULL; - nodesMakeNode((ENodeType)taosRand(), ppNode ? ppNode : &pNode); - return ppNode ? *ppNode : pNode; + return *ppNode; } SNode* qptMakeExprNode(SNode** ppNode) { @@ -1140,7 +1308,7 @@ SNode* qptMakeExprNode(SNode** ppNode) { qptMakeLogicCondNode(ppNode); break; case 4: - qptMakeNodeListNode(ppNode); + qptMakeNodeListNode(QPT_NODE_EXPR, ppNode); break; case 5: qptMakeOperatorNode(ppNode); @@ -1163,17 +1331,108 @@ SNode* qptMakeExprNode(SNode** ppNode) { } -void qptResetMakeNodeCtx(SDataBlockDescNode* pInput, bool onlyTag) { - SQPTMakeNodeCtx* pCtx = &qptCtx.makeCtx; +SNode* qptMakeLimitNode(SNode** ppNode) { + SNode* pNode = NULL; + if (QPT_NCORRECT_LOW_PROB()) { + return qptMakeRandNode(&pNode); + } + + assert(0 == nodesMakeNode(QUERY_NODE_LIMIT, &pNode)); + assert(pNode); - pCtx->nodeLevel = 1; - pCtx->onlyTag = onlyTag; - pCtx->pInputDataBlockDesc = pInput; + SLimitNode* pLimit = (SLimitNode*)pNode; + + if (!qptCtx.param.correctExpected) { + if (taosRand() % 2) { + pLimit->limit = taosRand() * ((taosRand() % 2) ? 1 : -1); + } + if (taosRand() % 2) { + pLimit->offset = taosRand() * ((taosRand() % 2) ? 1 : -1); + } + } else { + pLimit->limit = taosRand(); + if (taosRand() % 2) { + pLimit->offset = taosRand(); + } + } + + *ppNode = pNode; + + return pNode; } -SNode* qptMakeConditionNode(bool onlyTag) { + +SNode* qptMakeWindowOffsetNode(SNode** ppNode) { + if (QPT_NCORRECT_LOW_PROB()) { + return qptMakeRandNode(ppNode); + } + + SNode* pNode = NULL; + assert(0 == nodesMakeNode(QUERY_NODE_WINDOW_OFFSET, &pNode)); + assert(pNode); + + SWindowOffsetNode* pWinOffset = (SWindowOffsetNode*)pNode; + qptMakeValueNode(TSDB_DATA_TYPE_BIGINT, &pWinOffset->pStartOffset); + qptMakeValueNode(TSDB_DATA_TYPE_BIGINT, &pWinOffset->pEndOffset); + + *ppNode = pNode; + + return pNode; +} + +void qptSaveMakeNodeCtx() { + qptCtx.makeCtxBak.nodeLevel = qptCtx.makeCtx.nodeLevel; +} + +void qptRestoreMakeNodeCtx() { + qptCtx.makeCtx.nodeLevel = qptCtx.makeCtxBak.nodeLevel; +} + + +void qptResetTableCols() { + SNode* pTmp = NULL; + FOREACH(pTmp, qptCtx.param.tbl.pColList) { + ((SQPTCol*)pTmp)->inUse = 0; + } + FOREACH(pTmp, qptCtx.param.tbl.pTagList) { + ((SQPTCol*)pTmp)->inUse = 0; + } +} + +void qptResetMakeNodeCtx() { + SQPTMakeNodeCtx* pCtx = &qptCtx.makeCtx; + pCtx->nodeLevel = 1; + + if (pCtx->fromTable) { + qptResetTableCols(); + } +} + +void qptInitMakeNodeCtx(bool fromTable, bool onlyTag, bool onlyCol, int16_t inputBlockId, SNodeList* pInputList) { + SQPTMakeNodeCtx* pCtx = &qptCtx.makeCtx; + + pCtx->onlyTag = onlyTag; + pCtx->fromTable = fromTable; + pCtx->onlyCol = onlyCol; + + if (NULL == pInputList) { + if (fromTable) { + inputBlockId = (qptCtx.buildCtx.pCurr && qptCtx.buildCtx.pCurr->pOutputDataBlockDesc) ? qptCtx.buildCtx.pCurr->pOutputDataBlockDesc->dataBlockId : taosRand(); + pInputList = onlyTag ? qptCtx.param.tbl.pTagList : (onlyCol ? qptCtx.param.tbl.pColList : qptCtx.param.tbl.pColTagList); + } else if (qptCtx.buildCtx.pChild && qptCtx.buildCtx.pChild->pOutputDataBlockDesc) { + inputBlockId = qptCtx.buildCtx.pChild->pOutputDataBlockDesc->dataBlockId; + pInputList = qptCtx.buildCtx.pChild->pOutputDataBlockDesc->pSlots; + } + } + + pCtx->inputBlockId = inputBlockId; + pCtx->pInputList = pInputList; + + qptResetMakeNodeCtx(); +} + +SNode* qptMakeConditionNode() { SNode* pNode = NULL; - qptResetMakeNodeCtx(qptCtx.buildCtx.pCurr->pOutputDataBlockDesc, onlyTag); qptMakeExprNode(&pNode); return pNode; @@ -1181,21 +1440,21 @@ SNode* qptMakeConditionNode(bool onlyTag) { SNode* qptMakeDataBlockDescNode() { if (QPT_NCORRECT_LOW_PROB()) { - return NULL; + return qptMakeRandNode(NULL); } SDataBlockDescNode* pDesc = NULL; assert(0 == nodesMakeNode(QUERY_NODE_DATABLOCK_DESC, (SNode**)&pDesc)); - pDesc->dataBlockId = qptCtx.param.correctExpected ? qptCtx.makeCtx.nextBlockId++ : QPT_RAND_INT_V; - pDesc->precision = qptCtx.param.correctExpected ? qptCtx.param.db.precision : QPT_RAND_INT_V; + pDesc->dataBlockId = QPT_CORRECT_HIGH_PROB() ? qptCtx.buildCtx.nextBlockId++ : QPT_RAND_INT_V; + pDesc->precision = QPT_CORRECT_HIGH_PROB() ? qptCtx.param.db.precision : QPT_RAND_INT_V; return (SNode*)pDesc; } SNode* qptMakeSlotDescNode(const char* pName, const SNode* pNode, int16_t slotId, bool output, bool reserve) { SSlotDescNode* pSlot = NULL; - if (QPT_NCORRECT_LOW_PROB) { + if (QPT_NCORRECT_LOW_PROB()) { return qptMakeRandNode((SNode**)&pSlot); } @@ -1211,11 +1470,13 @@ SNode* qptMakeSlotDescNode(const char* pName, const SNode* pNode, int16_t slotId pSlot->reserve = reserve; pSlot->output = output; + return (SNode*)pSlot; } -void qptMakeTargetNode(SNode* pNode, int16_t dataBlockId, int16_t slotId, SNode** pOutput) { - if (QPT_NCORRECT_LOW_PROB) { +SNode* qptMakeTargetNode(SNode* pNode, int16_t dataBlockId, int16_t slotId, SNode** pOutput) { + if (QPT_NCORRECT_LOW_PROB()) { + nodesDestroyNode(pNode); return qptMakeRandNode(pOutput); } @@ -1225,18 +1486,64 @@ void qptMakeTargetNode(SNode* pNode, int16_t dataBlockId, int16_t slotId, SNode* pTarget->dataBlockId = QPT_CORRECT_HIGH_PROB() ? dataBlockId : taosRand(); pTarget->slotId = QPT_CORRECT_HIGH_PROB() ? slotId : taosRand(); pTarget->pExpr = QPT_CORRECT_HIGH_PROB() ? pNode : qptMakeRandNode(NULL); - + if (pTarget->pExpr != pNode) { + nodesDestroyNode(pNode); + } + *pOutput = (SNode*)pTarget; + + return *pOutput; +} + +SNode* qptMakeDownstreamSrcNode(SNode** ppNode) { + if (QPT_NCORRECT_LOW_PROB()) { + return qptMakeRandNode(ppNode); + } + + SDownstreamSourceNode* pDs = NULL; + nodesMakeNode(QUERY_NODE_DOWNSTREAM_SOURCE, (SNode**)&pDs); + + pDs->addr.nodeId = qptCtx.param.vnode.vgId; + memcpy(&pDs->addr.epSet, &qptCtx.param.vnode.epSet, sizeof(pDs->addr.epSet)); + pDs->taskId = (QPT_CORRECT_HIGH_PROB() && qptCtx.buildCtx.pCurrTask) ? qptCtx.buildCtx.pCurrTask->id.taskId : taosRand(); + pDs->schedId = QPT_CORRECT_HIGH_PROB() ? qptCtx.param.schedulerId : taosRand(); + pDs->execId = taosRand(); + pDs->fetchMsgType = QPT_CORRECT_HIGH_PROB() ? (QPT_RAND_BOOL_V ? TDMT_SCH_FETCH : TDMT_SCH_MERGE_FETCH) : taosRand(); + pDs->localExec = QPT_RAND_BOOL_V; + + *ppNode = (SNode*)pDs; + + return *ppNode; +} + +SNode* qptMakeOrderByExprNode(SNode** ppNode) { + if (QPT_NCORRECT_LOW_PROB()) { + return qptMakeRandNode(ppNode); + } + + SOrderByExprNode* pOrder = NULL; + nodesMakeNode(QUERY_NODE_ORDER_BY_EXPR, (SNode**)&pOrder); + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); + qptMakeExprNode(&pOrder->pExpr); + + pOrder->order = (EOrder)(QPT_CORRECT_HIGH_PROB() ? (QPT_RAND_BOOL_V ? ORDER_ASC : ORDER_DESC) : taosRand()); + pOrder->nullOrder = qptGetRandNullOrder(); + + *ppNode = (SNode*)pOrder; + + return *ppNode; } SPhysiNode* qptCreatePhysiNode(int32_t nodeType) { SPhysiNode* pPhysiNode = NULL; assert(0 == nodesMakeNode((ENodeType)nodeType, (SNode**)&pPhysiNode)); - + assert(pPhysiNode); + qptCtx.buildCtx.pCurr = pPhysiNode; - pPhysiNode->pLimit = qptMakeLimitNode(); - pPhysiNode->pSlimit = qptMakeLimitNode(); + qptMakeLimitNode(&pPhysiNode->pLimit); + qptMakeLimitNode(&pPhysiNode->pSlimit); pPhysiNode->dynamicOp = qptGetDynamicOp(); pPhysiNode->inputTsOrder = qptGetCurrTsOrder(); @@ -1249,15 +1556,18 @@ void qptPostCreatePhysiNode(SPhysiNode* pPhysiNode) { pPhysiNode->outputTsOrder = qptGetCurrTsOrder(); if (QPT_RAND_BOOL_V) { - pPhysiNode->pConditions = qptMakeConditionNode(false); + qptInitMakeNodeCtx((QPT_CORRECT_HIGH_PROB() && qptCtx.buildCtx.pChild) ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); + pPhysiNode->pConditions = qptMakeConditionNode(); } } -void qptMarkTableInUseCols(int32_t colNum, int32_t totalColNum, SQPTCol* pCol) { - if (colNum == totalColNum) { - for (int32_t i = 0; i < colNum; ++i) { - pCol[i].inUse = 1; +void qptMarkTableInUseCols(int32_t colNum, int32_t totalColNum) { + if (colNum >= totalColNum) { + for (int32_t i = 0; i < totalColNum; ++i) { + SQPTCol* pNode = (SQPTCol*)nodesListGetNode(qptCtx.makeCtx.pInputList, i); + assert(pNode->type == QPT_QUERY_NODE_COL); + pNode->inUse = 1; } return; } @@ -1265,49 +1575,76 @@ void qptMarkTableInUseCols(int32_t colNum, int32_t totalColNum, SQPTCol* pCol) { int32_t colInUse = 0; do { int32_t colIdx = taosRand() % totalColNum; - if (pCol[colIdx].inUse) { + SQPTCol* pNode = (SQPTCol*)nodesListGetNode(qptCtx.makeCtx.pInputList, colIdx); + assert(pNode->type == QPT_QUERY_NODE_COL); + + if (pNode->inUse) { continue; } - pCol[colIdx].inUse = 1; + pNode->inUse = 1; colInUse++; } while (colInUse < colNum); } -void qptCreateTableScanColsImpl( SScanPhysiNode* pScanPhysiNode, SNodeList** ppCols, int32_t totalColNum, SQPTCol* pCol) { - int32_t colNum = qptCtx.param.correctExpected ? (taosRand() % totalColNum + 1) : (taosRand() % QPT_MAX_COLUMN_NUM + 1); - int32_t colAdded = 0; - - if (qptCtx.param.correctExpected) { - qptMarkTableInUseCols(colNum, totalColNum, pCol); - - for (int32_t i = 0; i < totalColNum && colAdded < colNum; ++i) { - if (0 == pCol[i].inUse) { - continue; - } - - assert(0 == qptNodesListMakeStrictAppend(ppCols, qptMakeColumnNodeFromTable(i, pCol[0].colType, pScanPhysiNode))); +void qptMakeTableScanColList( SNodeList** ppCols) { + if (QPT_NCORRECT_LOW_PROB()) { + if (QPT_RAND_BOOL_V) { + nodesMakeList(ppCols); + } else { + *ppCols = NULL; } - + return; } - for (int32_t i = 0; i < colNum; ++i) { - int32_t colIdx = taosRand(); - colIdx = (colIdx >= totalColNum) ? -1 : colIdx; + int32_t colNum = (QPT_CORRECT_HIGH_PROB() && qptCtx.makeCtx.pInputList) ? (taosRand() % qptCtx.makeCtx.pInputList->length + 1) : (taosRand() % QPT_MAX_COLUMN_NUM); + int32_t colAdded = 0; + + if (qptCtx.makeCtx.pInputList) { + if (QPT_CORRECT_HIGH_PROB()) { + qptMarkTableInUseCols(colNum, qptCtx.makeCtx.pInputList->length); + + for (int32_t i = 0; colAdded < colNum; ++i) { + int32_t idx = (i < qptCtx.makeCtx.pInputList->length) ? i : (taosRand() % qptCtx.makeCtx.pInputList->length); + SQPTCol* pNode = (SQPTCol*)nodesListGetNode(qptCtx.makeCtx.pInputList, idx); + assert(pNode->type == QPT_QUERY_NODE_COL); + + if (0 == pNode->inUse) { + continue; + } + + assert(0 == qptNodesListMakeStrictAppend(ppCols, qptMakeColumnFromTable(idx))); + colAdded++; + } + + return; + } - assert(0 == qptNodesListMakeStrictAppend(ppCols, qptMakeColumnNodeFromTable(colIdx, pCol[0].colType, pScanPhysiNode))); + for (int32_t i = 0; i < colNum; ++i) { + int32_t colIdx = taosRand(); + colIdx = (colIdx >= qptCtx.makeCtx.pInputList->length) ? -1 : colIdx; + + assert(0 == qptNodesListMakeStrictAppend(ppCols, qptMakeColumnFromTable(colIdx))); + } + } else { + for (int32_t i = 0; i < colNum; ++i) { + int32_t colIdx = taosRand(); + assert(0 == qptNodesListMakeStrictAppend(ppCols, qptMakeColumnFromTable(colIdx))); + } } } -void qptCreateTableScanCols( SScanPhysiNode* pScanPhysiNode) { - qptCreateTableScanColsImpl(pScanPhysiNode, &pScanPhysiNode->pScanCols, qptCtx.param.tbl.colNum, qptCtx.param.tbl.pCol); +void qptCreateTableScanCols( int16_t blockId, SNodeList** ppList) { + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? true : false, QPT_CORRECT_HIGH_PROB() ? false : true, QPT_CORRECT_HIGH_PROB() ? true : false, 0, NULL); + qptMakeTableScanColList(ppList); } -void qptCreateTableScanPseudoCols( SScanPhysiNode* pScanPhysiNode) { - qptCreateTableScanColsImpl(pScanPhysiNode, &pScanPhysiNode->pScanPseudoCols, qptCtx.param.tbl.tagNum, qptCtx.param.tbl.pTag); +void qptCreateTableScanPseudoCols( int16_t blockId, SNodeList** ppList) { + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? true : false, QPT_CORRECT_HIGH_PROB() ? true : false, QPT_CORRECT_HIGH_PROB() ? false : true, 0, NULL); + qptMakeTableScanColList(ppList); } @@ -1327,11 +1664,11 @@ void qptAddDataBlockSlots(SNodeList* pList, SDataBlockDescNode* pDataBlockDesc) SNode* pExpr = QUERY_NODE_ORDER_BY_EXPR == nodeType(pNode) ? ((SOrderByExprNode*)pNode)->pExpr : pNode; if (QPT_CORRECT_HIGH_PROB()) { - SNode* pDesc = qptCtx.param.correctExpected ? qptMakeSlotDescNode(NULL, pExpr, nextSlotId, output, QPT_RAND_BOOL_V) : qptMakeExprNode(NULL); + SNode* pDesc = QPT_CORRECT_HIGH_PROB() ? qptMakeSlotDescNode(NULL, pExpr, nextSlotId, output, QPT_RAND_BOOL_V) : qptMakeExprNode(NULL); assert(0 == qptNodesListMakeStrictAppend(&pDataBlockDesc->pSlots, pDesc)); - pDataBlockDesc->totalRowSize += qptCtx.param.correctExpected ? ((SExprNode*)pExpr)->resType.bytes : taosRand(); + pDataBlockDesc->totalRowSize += QPT_CORRECT_HIGH_PROB() ? ((SExprNode*)pExpr)->resType.bytes : taosRand(); if (output && QPT_RAND_BOOL_V) { - pDataBlockDesc->outputRowSize += qptCtx.param.correctExpected ? ((SExprNode*)pExpr)->resType.bytes : taosRand(); + pDataBlockDesc->outputRowSize += QPT_CORRECT_HIGH_PROB() ? ((SExprNode*)pExpr)->resType.bytes : taosRand(); } } @@ -1346,14 +1683,170 @@ void qptAddDataBlockSlots(SNodeList* pList, SDataBlockDescNode* pDataBlockDesc) } } +SNode* qptMakeSpecTypeNode(QPT_NODE_TYPE nodeType, SNode** ppNode) { + switch (nodeType) { + case QPT_NODE_COLUMN: + return qptMakeColumnNode(ppNode); + case QPT_NODE_FUNCTION: + return qptMakeFunctionNode(ppNode); + case QPT_NODE_EXPR: + return qptMakeExprNode(ppNode); + case QPT_NODE_VALUE: + return qptMakeValueNode(-1, ppNode); + default: + break; + } + + return qptMakeRandNode(ppNode); +} + + +void qptMakeRandNodeList(SNodeList** ppList) { + qptSaveMakeNodeCtx(); + + int32_t exprNum = taosRand() % QPT_MAX_COLUMN_NUM + (QPT_CORRECT_HIGH_PROB() ? 1 : 0); + for (int32_t i = 0; i < exprNum; ++i) { + SNode* pNode = NULL; + qptRestoreMakeNodeCtx(); + qptMakeSpecTypeNode((QPT_NODE_TYPE)(taosRand() % (QPT_NODE_MAX_VALUE + 1)), &pNode); + qptNodesListMakeStrictAppend(ppList, pNode); + } +} + + +void qptMakeExprList(SNodeList** ppList) { + qptSaveMakeNodeCtx(); + + int32_t exprNum = taosRand() % QPT_MAX_COLUMN_NUM + (QPT_CORRECT_HIGH_PROB() ? 1 : 0); + for (int32_t i = 0; i < exprNum; ++i) { + SNode* pNode = NULL; + qptRestoreMakeNodeCtx(); + qptMakeExprNode(&pNode); + qptNodesListMakeStrictAppend(ppList, pNode); + } +} + +void qptMakeValueList(SNodeList** ppList) { + qptSaveMakeNodeCtx(); + + int32_t colNum = taosRand() % QPT_MAX_COLUMN_NUM + (QPT_CORRECT_HIGH_PROB() ? 1 : 0); + for (int32_t i = 0; i < colNum; ++i) { + SNode* pNode = NULL; + qptRestoreMakeNodeCtx(); + qptMakeValueNode(-1, &pNode); + qptNodesListMakeStrictAppend(ppList, pNode); + } +} + +void qptMakeColumnList(SNodeList** ppList) { + qptSaveMakeNodeCtx(); + + int32_t colNum = taosRand() % QPT_MAX_COLUMN_NUM + (QPT_CORRECT_HIGH_PROB() ? 1 : 0); + for (int32_t i = 0; i < colNum; ++i) { + SNode* pNode = NULL; + qptRestoreMakeNodeCtx(); + qptMakeColumnNode(&pNode); + qptNodesListMakeStrictAppend(ppList, pNode); + } +} + +void qptMakeTargetList(QPT_NODE_TYPE nodeType, int16_t datablockId, SNodeList** ppList) { + qptSaveMakeNodeCtx(); + + int32_t tarNum = taosRand() % QPT_MAX_COLUMN_NUM + (QPT_CORRECT_HIGH_PROB() ? 1 : 0); + for (int32_t i = 0; i < tarNum; ++i) { + SNode* pNode = NULL, *pExpr = NULL; + qptRestoreMakeNodeCtx(); + qptMakeSpecTypeNode(nodeType, &pExpr); + qptMakeTargetNode(pExpr, datablockId, i, &pNode); + qptNodesListMakeStrictAppend(ppList, pNode); + } +} + +void qptMakeFunctionList(SNodeList** ppList) { + qptSaveMakeNodeCtx(); + + int32_t funcNum = taosRand() % QPT_MAX_COLUMN_NUM + (QPT_CORRECT_HIGH_PROB() ? 1 : 0); + for (int32_t i = 0; i < funcNum; ++i) { + SNode* pNode = NULL; + qptRestoreMakeNodeCtx(); + qptMakeFunctionNode(&pNode); + qptNodesListMakeStrictAppend(ppList, pNode); + } +} + + +void qptMakeDownstreamSrcList(SNodeList** ppList) { + qptSaveMakeNodeCtx(); + + int32_t dsNum = taosRand() % QPT_MAX_DS_SRC_NUM + (QPT_CORRECT_HIGH_PROB() ? 1 : 0); + for (int32_t i = 0; i < dsNum; ++i) { + SNode* pNode = NULL; + qptRestoreMakeNodeCtx(); + qptMakeDownstreamSrcNode(&pNode); + qptNodesListMakeStrictAppend(ppList, pNode); + } +} + + +void qptMakeOrerByExprList(SNodeList** ppList) { + qptSaveMakeNodeCtx(); + + int32_t orderNum = taosRand() % QPT_MAX_ORDER_BY_NUM + (QPT_CORRECT_HIGH_PROB() ? 1 : 0); + for (int32_t i = 0; i < orderNum; ++i) { + SNode* pNode = NULL; + qptRestoreMakeNodeCtx(); + qptMakeOrderByExprNode(&pNode); + qptNodesListMakeStrictAppend(ppList, pNode); + } +} + + +void qptMakeSpecTypeNodeList(QPT_NODE_TYPE nodeType, SNodeList** ppList) { + switch (nodeType) { + case QPT_NODE_COLUMN: + return qptMakeColumnList(ppList); + case QPT_NODE_FUNCTION: + return qptMakeFunctionList(ppList); + case QPT_NODE_EXPR: + return qptMakeExprList(ppList); + case QPT_NODE_VALUE: + return qptMakeValueList(ppList); + default: + break; + } + + return qptMakeRandNodeList(ppList); +} + + +void qptMakeNodeList(QPT_NODE_TYPE nodeType, SNodeList** ppList) { + qptMakeSpecTypeNodeList(nodeType, ppList); +} + + + +void qptMakeAppendToTargetList(SNodeList* pInputList, int16_t blockId, SNodeList** ppOutList) { + SNode* pNode = NULL; + FOREACH(pNode, pInputList) { + if (QPT_CORRECT_HIGH_PROB()) { + SNode* pTarget = NULL; + int16_t slotId = ((*ppOutList) && (*ppOutList)->length) ? (*ppOutList)->length : 0; + qptMakeTargetNode(pNode, blockId, slotId, &pTarget); + qptNodesListMakeStrictAppend(ppOutList, pTarget); + } + } +} void qptCreateScanPhysiNodeImpl( SScanPhysiNode* pScanPhysiNode) { - qptCreateTableScanCols(pScanPhysiNode); + int16_t blockId = (QPT_CORRECT_HIGH_PROB() && pScanPhysiNode->node.pOutputDataBlockDesc) ? pScanPhysiNode->node.pOutputDataBlockDesc->dataBlockId : taosRand(); + qptCreateTableScanCols(blockId, &pScanPhysiNode->pScanCols); qptAddDataBlockSlots(pScanPhysiNode->pScanCols, pScanPhysiNode->node.pOutputDataBlockDesc); if (taosRand() % 2) { - qptCreateTableScanPseudoCols(pScanPhysiNode); + blockId = (QPT_CORRECT_HIGH_PROB() && pScanPhysiNode->node.pOutputDataBlockDesc) ? pScanPhysiNode->node.pOutputDataBlockDesc->dataBlockId : taosRand(); + qptCreateTableScanPseudoCols(blockId, &pScanPhysiNode->pScanPseudoCols); } qptAddDataBlockSlots(pScanPhysiNode->pScanPseudoCols, pScanPhysiNode->node.pOutputDataBlockDesc); @@ -1391,34 +1884,131 @@ SNode* qptCreateTagScanPhysiNode(int32_t nodeType) { return (SNode*)pPhysiNode; } -void qptMakeExprList(SNodeList** ppList) { - int32_t exprNum = taosRand() % QPT_MAX_COLUMN_NUM + (QPT_CORRECT_HIGH_PROB() ? 1 : 0); - for (int32_t i = 0; i < exprNum; ++i) { - SNode* pNode = NULL; - qptResetMakeNodeCtx(qptCtx.buildCtx.pChild ? qptCtx.buildCtx.pChild->pOutputDataBlockDesc : NULL, false); - qptMakeExprNode(&pNode); - qptNodesListMakeStrictAppend(ppList, pNode); - } +SNode* qptCreateTableScanPhysiNode(int32_t nodeType) { + SPhysiNode* pPhysiNode = qptCreatePhysiNode(nodeType); + + STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhysiNode; + pTableScanNode->scanSeq[0] = taosRand() % 4; + pTableScanNode->scanSeq[1] = taosRand() % 4; + pTableScanNode->scanRange.skey = taosRand(); + pTableScanNode->scanRange.ekey = taosRand(); + pTableScanNode->ratio = taosRand(); + pTableScanNode->dataRequired = taosRand(); + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? true : false, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); + qptMakeFunctionList(&pTableScanNode->pDynamicScanFuncs); + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? true : false, QPT_CORRECT_HIGH_PROB() ? true : false, QPT_CORRECT_HIGH_PROB() ? false : true, 0, NULL); + qptMakeColumnList(&pTableScanNode->pGroupTags); + + pTableScanNode->groupSort = QPT_RAND_BOOL_V; + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? true : false, QPT_CORRECT_HIGH_PROB() ? true : false, QPT_CORRECT_HIGH_PROB() ? false : true, 0, NULL); + qptMakeExprList(&pTableScanNode->pTags); + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? true : false, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); + qptMakeExprNode(&pTableScanNode->pSubtable); + pTableScanNode->interval = taosRand(); + pTableScanNode->offset = taosRand(); + pTableScanNode->sliding = taosRand(); + pTableScanNode->intervalUnit = taosRand(); + pTableScanNode->slidingUnit = taosRand(); + pTableScanNode->triggerType = taosRand(); + pTableScanNode->watermark = taosRand(); + pTableScanNode->igExpired = taosRand(); + pTableScanNode->assignBlockUid = QPT_RAND_BOOL_V; + pTableScanNode->igCheckUpdate = taosRand(); + pTableScanNode->filesetDelimited = QPT_RAND_BOOL_V; + pTableScanNode->needCountEmptyTable = QPT_RAND_BOOL_V; + pTableScanNode->paraTablesSort = QPT_RAND_BOOL_V; + pTableScanNode->smallDataTsSort = QPT_RAND_BOOL_V; + + qptCreateScanPhysiNodeImpl(&pTableScanNode->scan); + + qptPostCreatePhysiNode(pPhysiNode); + + return (SNode*)pPhysiNode; } -void qptMakeColumnList(SNodeList** ppList) { - int32_t colNum = taosRand() % QPT_MAX_COLUMN_NUM + (QPT_CORRECT_HIGH_PROB() ? 1 : 0); - qptResetMakeNodeCtx(qptCtx.buildCtx.pChild ? qptCtx.buildCtx.pChild->pOutputDataBlockDesc : NULL, false); - for (int32_t i = 0; i < colNum; ++i) { - SNode* pNode = NULL; - qptMakeColumnNode(&pNode); - qptNodesListMakeStrictAppend(ppList, pNode); - } + +SNode* qptCreateTableSeqScanPhysiNode(int32_t nodeType) { + return qptCreateTableScanPhysiNode(nodeType); } -void qptMakeTargetList(int16_t datablockId, SNodeList** ppList) { - int32_t tarNum = taosRand() % QPT_MAX_COLUMN_NUM + (QPT_CORRECT_HIGH_PROB() ? 1 : 0); - for (int32_t i = 0; i < tarNum; ++i) { - SNode* pNode = NULL, *pExpr = NULL; - qptMakeColumnNode(&pExpr); - qptMakeTargetNode(pExpr, datablockId, i, &pNode); - qptNodesListMakeStrictAppend(ppList, pNode); +SNode* qptCreateTableMergeScanPhysiNode(int32_t nodeType) { + return qptCreateTableScanPhysiNode(nodeType); +} + +SNode* qptCreateStreamScanPhysiNode(int32_t nodeType) { + return qptCreateTableScanPhysiNode(nodeType); +} + +SNode* qptCreateSysTableScanPhysiNode(int32_t nodeType) { + SPhysiNode* pPhysiNode = qptCreatePhysiNode(nodeType); + + SSystemTableScanPhysiNode* pSysScanNode = (SSystemTableScanPhysiNode*)pPhysiNode; + + memcpy(&pSysScanNode->mgmtEpSet, &qptCtx.param.vnode.epSet, sizeof(pSysScanNode->mgmtEpSet)); + pSysScanNode->showRewrite = QPT_RAND_BOOL_V; + pSysScanNode->accountId = QPT_CORRECT_HIGH_PROB() ? 1 : taosRand(); + pSysScanNode->sysInfo = QPT_RAND_BOOL_V; + + qptCreateScanPhysiNodeImpl(&pSysScanNode->scan); + + qptPostCreatePhysiNode(pPhysiNode); + + return (SNode*)pPhysiNode; +} + +SNode* qptCreateBlockDistScanPhysiNode(int32_t nodeType) { + SPhysiNode* pPhysiNode = qptCreatePhysiNode(nodeType); + + SBlockDistScanPhysiNode* pBlkScanNode = (SBlockDistScanPhysiNode*)pPhysiNode; + + qptCreateScanPhysiNodeImpl((SScanPhysiNode*)pBlkScanNode); + + qptPostCreatePhysiNode(pPhysiNode); + + return (SNode*)pPhysiNode; +} + + +SNode* qptCreateLastRowScanPhysiNode(int32_t nodeType) { + SPhysiNode* pPhysiNode = qptCreatePhysiNode(nodeType); + + SLastRowScanPhysiNode* pLRScanNode = (SLastRowScanPhysiNode*)pPhysiNode; + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? true : false, QPT_CORRECT_HIGH_PROB() ? true : false, QPT_CORRECT_HIGH_PROB() ? false : true, 0, NULL); + qptMakeColumnList(&pLRScanNode->pGroupTags); + + pLRScanNode->groupSort = QPT_RAND_BOOL_V; + pLRScanNode->ignoreNull = QPT_RAND_BOOL_V; + + if (QPT_CORRECT_HIGH_PROB()) { + int16_t blockId = (QPT_CORRECT_HIGH_PROB() && pPhysiNode->pOutputDataBlockDesc) ? pPhysiNode->pOutputDataBlockDesc->dataBlockId : taosRand(); + qptMakeAppendToTargetList(pLRScanNode->scan.pScanCols, blockId, &pLRScanNode->pTargets); } + + if (QPT_CORRECT_HIGH_PROB()) { + int16_t blockId = (QPT_CORRECT_HIGH_PROB() && pPhysiNode->pOutputDataBlockDesc) ? pPhysiNode->pOutputDataBlockDesc->dataBlockId : taosRand(); + qptMakeAppendToTargetList(pLRScanNode->scan.pScanPseudoCols, blockId, &pLRScanNode->pTargets); + } + + if (QPT_RAND_BOOL_V) { + int32_t funcNum = taosRand() % QPT_MAX_COLUMN_NUM; + pLRScanNode->pFuncTypes = taosArrayInit(funcNum, sizeof(int32_t)); + assert(pLRScanNode->pFuncTypes); + for (int32_t i = 0; i < funcNum; ++i) { + int32_t funcType = taosRand(); + taosArrayPush(pLRScanNode->pFuncTypes, &funcType); + } + } + + qptCreateScanPhysiNodeImpl(&pLRScanNode->scan); + + qptPostCreatePhysiNode(pPhysiNode); + + return (SNode*)pPhysiNode; } SNode* qptCreateProjectPhysiNode(int32_t nodeType) { @@ -1430,6 +2020,7 @@ SNode* qptCreateProjectPhysiNode(int32_t nodeType) { pProject->ignoreGroupId = QPT_RAND_BOOL_V; pProject->inputIgnoreGroup = QPT_RAND_BOOL_V; + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); qptMakeExprList(&pProject->pProjections); qptAddDataBlockSlots(pProject->pProjections, pProject->node.pOutputDataBlockDesc); @@ -1440,28 +2031,48 @@ SNode* qptCreateProjectPhysiNode(int32_t nodeType) { } -SNode* qptCreateSortMergeJoinPhysiNode(int32_t nodeType) { +SNode* qptCreateMergeJoinPhysiNode(int32_t nodeType) { SPhysiNode* pPhysiNode = qptCreatePhysiNode(nodeType); SSortMergeJoinPhysiNode* pJoin = (SSortMergeJoinPhysiNode*)pPhysiNode; - pJoin->joinType = taosRand() % JOIN_TYPE_MAX_VALUE + (QPT_CORRECT_HIGH_PROB() ? 0 : 1); - pJoin->subType = taosRand() % JOIN_STYPE_MAX_VALUE + (QPT_CORRECT_HIGH_PROB() ? 0 : 1); + pJoin->joinType = (EJoinType)(taosRand() % JOIN_TYPE_MAX_VALUE + (QPT_CORRECT_HIGH_PROB() ? 0 : 1)); + pJoin->subType = (EJoinSubType)(taosRand() % JOIN_STYPE_MAX_VALUE + (QPT_CORRECT_HIGH_PROB() ? 0 : 1)); qptMakeWindowOffsetNode(&pJoin->pWindowOffset); qptMakeLimitNode(&pJoin->pJLimit); pJoin->asofOpType = OPERATOR_ARRAY[taosRand() % (sizeof(OPERATOR_ARRAY)/sizeof(OPERATOR_ARRAY[0]))] + (QPT_CORRECT_HIGH_PROB() ? 0 : 1); + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); qptMakeExprNode(&pJoin->leftPrimExpr); + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); qptMakeExprNode(&pJoin->rightPrimExpr); - pJoin->leftPrimSlotId = qptGetInputSlotId(qptCtx->buildCtx.pChild ? qptCtx->buildCtx.pChild->pOutputDataBlockDesc : NULL); - pJoin->rightPrimSlotId = qptGetInputSlotId(qptCtx->buildCtx.pChild ? qptCtx->buildCtx.pChild->pOutputDataBlockDesc : NULL); + + pJoin->leftPrimSlotId = qptGetInputSlotId(qptCtx.buildCtx.pChild ? qptCtx.buildCtx.pChild->pOutputDataBlockDesc : NULL); + pJoin->rightPrimSlotId = qptGetInputSlotId(qptCtx.buildCtx.pChild ? qptCtx.buildCtx.pChild->pOutputDataBlockDesc : NULL); + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); qptMakeColumnList(&pJoin->pEqLeft); + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); qptMakeColumnList(&pJoin->pEqRight); + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); qptMakeExprNode(&pJoin->pPrimKeyCond); + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); qptMakeExprNode(&pJoin->pColEqCond); + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); qptMakeExprNode(&pJoin->pColOnCond); + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); qptMakeExprNode(&pJoin->pFullOnCond); - qptMakeTargetList(pPhysiNode->pOutputDataBlockDesc ? pPhysiNode->pOutputDataBlockDesc->dataBlockId, &pJoin->pTargets); + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); + int16_t blockId = (QPT_CORRECT_HIGH_PROB() && pPhysiNode->pOutputDataBlockDesc) ? pPhysiNode->pOutputDataBlockDesc->dataBlockId : taosRand(); + qptMakeTargetList(QPT_NODE_EXPR, blockId, &pJoin->pTargets); + for (int32_t i = 0; i < 2; i++) { pJoin->inputStat[i].inputRowNum = taosRand(); pJoin->inputStat[i].inputRowSize = taosRand(); @@ -1473,64 +2084,293 @@ SNode* qptCreateSortMergeJoinPhysiNode(int32_t nodeType) { return (SNode*)pPhysiNode; } -SNode* qptCreatePhysicalPlanNode(int32_t nodeType) { - switch (nodeType) { - case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: - return (SNode*)qptCreateTagScanPhysiNode(nodeType); - case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: - case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN: - case QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN: - case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN: - case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN: - case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN: - case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN: - case QUERY_NODE_PHYSICAL_PLAN_PROJECT: - return (SNode*)qptCreateProjectPhysiNode(nodeType); - case QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN: - return (SNode*)qptCreateSortMergeJoinPhysiNode(nodeType); - case QUERY_NODE_PHYSICAL_PLAN_HASH_AGG: - case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE: - case QUERY_NODE_PHYSICAL_PLAN_MERGE: - case QUERY_NODE_PHYSICAL_PLAN_SORT: - case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT: - case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: - case QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL: - case QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL: - case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: - case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL: - case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL: - case QUERY_NODE_PHYSICAL_PLAN_FILL: - case QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL: - case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION: - case QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION: - case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION: - case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION: - case QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE: - case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE: - case QUERY_NODE_PHYSICAL_PLAN_PARTITION: - case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION: - case QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC: - case QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC: - case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: - case QUERY_NODE_PHYSICAL_PLAN_INSERT: - case QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT: - case QUERY_NODE_PHYSICAL_PLAN_DELETE: - case QUERY_NODE_PHYSICAL_SUBPLAN: - case QUERY_NODE_PHYSICAL_PLAN: - case QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN: - case QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT: - case QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT: - case QUERY_NODE_PHYSICAL_PLAN_HASH_JOIN: - case QUERY_NODE_PHYSICAL_PLAN_GROUP_CACHE: - case QUERY_NODE_PHYSICAL_PLAN_DYN_QUERY_CTRL: - case QUERY_NODE_PHYSICAL_PLAN_MERGE_COUNT: - case QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT: - case QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL: - default: - assert(0); +SNode* qptCreateHashAggPhysiNode(int32_t nodeType) { + SPhysiNode* pPhysiNode = qptCreatePhysiNode(nodeType); + + SAggPhysiNode* pAgg = (SAggPhysiNode*)pPhysiNode; + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); + qptMakeExprList(&pAgg->pExprs); + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); + int16_t blockId = (QPT_CORRECT_HIGH_PROB() && pPhysiNode->pOutputDataBlockDesc) ? pPhysiNode->pOutputDataBlockDesc->dataBlockId : taosRand(); + qptMakeTargetList(QPT_NODE_EXPR, blockId, &pAgg->pGroupKeys); + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); + blockId = (QPT_CORRECT_HIGH_PROB() && pPhysiNode->pOutputDataBlockDesc) ? pPhysiNode->pOutputDataBlockDesc->dataBlockId : taosRand(); + qptMakeTargetList(QPT_NODE_FUNCTION, blockId, &pAgg->pAggFuncs); + + pAgg->mergeDataBlock = QPT_RAND_BOOL_V; + pAgg->groupKeyOptimized = QPT_RAND_BOOL_V; + pAgg->hasCountLikeFunc = QPT_RAND_BOOL_V; + + return (SNode*)pPhysiNode; +} + +SNode* qptCreateExchangePhysiNode(int32_t nodeType) { + SPhysiNode* pPhysiNode = qptCreatePhysiNode(nodeType); + + SExchangePhysiNode* pExc = (SExchangePhysiNode*)pPhysiNode; + + pExc->srcStartGroupId = taosRand(); + pExc->srcEndGroupId = taosRand(); + pExc->singleChannel = QPT_RAND_BOOL_V; + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); + qptMakeDownstreamSrcList(&pExc->pSrcEndPoints); + + pExc->seqRecvData = QPT_RAND_BOOL_V; + + return (SNode*)pPhysiNode; +} + +SNode* qptCreateMergePhysiNode(int32_t nodeType) { + SPhysiNode* pPhysiNode = qptCreatePhysiNode(nodeType); + + SMergePhysiNode* pMerge = (SMergePhysiNode*)pPhysiNode; + + pMerge->type = (EMergeType)(QPT_CORRECT_HIGH_PROB() ? (taosRand() % (MERGE_TYPE_MAX_VALUE - 1) + 1) : taosRand()); + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); + qptMakeOrerByExprList(&pMerge->pMergeKeys); + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); + int16_t blockId = (QPT_CORRECT_HIGH_PROB() && pPhysiNode->pOutputDataBlockDesc) ? pPhysiNode->pOutputDataBlockDesc->dataBlockId : taosRand(); + qptMakeTargetList(QPT_NODE_EXPR, blockId, &pMerge->pTargets); + + pMerge->numOfChannels = taosRand(); + pMerge->numOfSubplans = taosRand(); + pMerge->srcGroupId = taosRand(); + pMerge->srcEndGroupId = taosRand(); + pMerge->groupSort = QPT_RAND_BOOL_V; + pMerge->ignoreGroupId = QPT_RAND_BOOL_V; + pMerge->inputWithGroupId = QPT_RAND_BOOL_V; + + return (SNode*)pPhysiNode; +} + + +SNode* qptCreateSortPhysiNode(int32_t nodeType) { + SPhysiNode* pPhysiNode = qptCreatePhysiNode(nodeType); + + SSortPhysiNode* pSort = (SSortPhysiNode*)pPhysiNode; + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); + qptMakeExprList(&pSort->pExprs); + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); + qptMakeOrerByExprList(&pSort->pSortKeys); + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); + int16_t blockId = (QPT_CORRECT_HIGH_PROB() && pPhysiNode->pOutputDataBlockDesc) ? pPhysiNode->pOutputDataBlockDesc->dataBlockId : taosRand(); + qptMakeTargetList(QPT_NODE_EXPR, blockId, &pSort->pTargets); + + pSort->calcGroupId = QPT_RAND_BOOL_V; + pSort->excludePkCol = QPT_RAND_BOOL_V; + + return (SNode*)pPhysiNode; +} + +SNode* qptCreateGroupSortPhysiNode(int32_t nodeType) { + SPhysiNode* pPhysiNode = qptCreatePhysiNode(nodeType); + + SGroupSortPhysiNode* pSort = (SGroupSortPhysiNode*)pPhysiNode; + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); + qptMakeExprList(&pSort->pExprs); + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); + qptMakeOrerByExprList(&pSort->pSortKeys); + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); + int16_t blockId = (QPT_CORRECT_HIGH_PROB() && pPhysiNode->pOutputDataBlockDesc) ? pPhysiNode->pOutputDataBlockDesc->dataBlockId : taosRand(); + qptMakeTargetList(QPT_NODE_EXPR, blockId, &pSort->pTargets); + + pSort->calcGroupId = QPT_RAND_BOOL_V; + pSort->excludePkCol = QPT_RAND_BOOL_V; + + return (SNode*)pPhysiNode; +} + +void qptCreateWindowPhysiNode(SWindowPhysiNode* pWindow) { + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); + qptMakeExprList(&pWindow->pExprs); + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); + qptMakeFunctionList(&pWindow->pFuncs); + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); + qptMakeColumnNode(&pWindow->pTspk); + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); + qptMakeColumnNode(&pWindow->pTsEnd); + + pWindow->triggerType = taosRand(); + pWindow->watermark = taosRand(); + pWindow->deleteMark = taosRand(); + pWindow->igExpired = taosRand(); + pWindow->destHasPrimayKey = taosRand(); + pWindow->mergeDataBlock = QPT_RAND_BOOL_V; +} + +SNode* qptCreateIntervalPhysiNode(int32_t nodeType) { + SPhysiNode* pPhysiNode = qptCreatePhysiNode(nodeType); + + SIntervalPhysiNode* pInterval = (SIntervalPhysiNode*)pPhysiNode; + + qptCreateWindowPhysiNode(&pInterval->window); + + pInterval->interval = taosRand(); + pInterval->offset = taosRand(); + pInterval->sliding = taosRand(); + pInterval->intervalUnit = qptGetRandTimestampUnit(); + pInterval->slidingUnit = qptGetRandTimestampUnit(); + + return (SNode*)pPhysiNode; +} + +SNode* qptCreateMergeIntervalPhysiNode(int32_t nodeType) { + return qptCreateIntervalPhysiNode(nodeType); +} + +SNode* qptCreateMergeAlignedIntervalPhysiNode(int32_t nodeType) { + return qptCreateIntervalPhysiNode(nodeType); +} + +SNode* qptCreateStreamIntervalPhysiNode(int32_t nodeType) { + return qptCreateIntervalPhysiNode(nodeType); +} + +SNode* qptCreateStreamFinalIntervalPhysiNode(int32_t nodeType) { + return qptCreateIntervalPhysiNode(nodeType); +} + +SNode* qptCreateStreamSemiIntervalPhysiNode(int32_t nodeType) { + return qptCreateIntervalPhysiNode(nodeType); +} + +SNode* qptCreateFillPhysiNode(int32_t nodeType) { + SPhysiNode* pPhysiNode = qptCreatePhysiNode(nodeType); + + SFillPhysiNode* pFill = (SFillPhysiNode*)pPhysiNode; + + pFill->mode = qptGetRandFillMode(); + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); + qptMakeExprList(&pFill->pFillExprs); + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); + qptMakeExprList(&pFill->pNotFillExprs); + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); + qptMakeColumnNode(&pFill->pWStartTs); + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); + qptMakeNodeListNode(QPT_NODE_VALUE, &pFill->pValues); + + qptGetRandTimeWindow(&pFill->timeRange); + + return (SNode*)pPhysiNode; +} + +SNode* qptCreateStreamFillPhysiNode(int32_t nodeType) { + return qptCreateFillPhysiNode(nodeType); +} + + +SNode* qptCreateSessionPhysiNode(int32_t nodeType) { + SPhysiNode* pPhysiNode = qptCreatePhysiNode(nodeType); + + SSessionWinodwPhysiNode* pSession = (SSessionWinodwPhysiNode*)pPhysiNode; + + qptCreateWindowPhysiNode(&pSession->window); + + pSession->gap = taosRand(); + + return (SNode*)pPhysiNode; +} + +SNode* qptCreateStreamSessionPhysiNode(int32_t nodeType) { + return qptCreateSessionPhysiNode(nodeType); +} + +SNode* qptCreateStreamSemiSessionPhysiNode(int32_t nodeType) { + return qptCreateSessionPhysiNode(nodeType); +} + +SNode* qptCreateStreamFinalSessionPhysiNode(int32_t nodeType) { + return qptCreateSessionPhysiNode(nodeType); +} + +SNode* qptCreateStateWindowPhysiNode(int32_t nodeType) { + SPhysiNode* pPhysiNode = qptCreatePhysiNode(nodeType); + + SStateWinodwPhysiNode* pState = (SStateWinodwPhysiNode*)pPhysiNode; + + qptCreateWindowPhysiNode(&pState->window); + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); + qptMakeColumnNode(&pState->pStateKey); + + return (SNode*)pPhysiNode; +} + +SNode* qptCreateStreamStatePhysiNode(int32_t nodeType) { + return qptCreateStateWindowPhysiNode(nodeType); +} + +void qptCreatePartitionPhysiNodeImpl(SPartitionPhysiNode* pPartition) { + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); + qptMakeExprList(&pPartition->pExprs); + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, pPartition->node.pOutputDataBlockDesc ? pPartition->node.pOutputDataBlockDesc->dataBlockId : taosRand(), pPartition->pExprs); + qptMakeColumnList(&pPartition->pPartitionKeys); + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); + qptMakeColumnList(&pPartition->pPartitionKeys); + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); + int16_t blockId = (QPT_CORRECT_HIGH_PROB() && pPartition->node.pOutputDataBlockDesc) ? pPartition->node.pOutputDataBlockDesc->dataBlockId : taosRand(); + qptMakeTargetList(QPT_NODE_EXPR, blockId, &pPartition->pTargets); + + pPartition->needBlockOutputTsOrder = QPT_RAND_BOOL_V; + pPartition->tsSlotId = qptGetInputPrimaryTsSlotId(); +} + +SNode* qptCreatePartitionPhysiNode(int32_t nodeType) { + SPhysiNode* pPhysiNode = qptCreatePhysiNode(nodeType); + + SPartitionPhysiNode* pPartition = (SPartitionPhysiNode*)pPhysiNode; + + qptCreatePartitionPhysiNodeImpl(pPartition); + + return (SNode*)pPhysiNode; +} + +SNode* qptCreateStreamPartitionPhysiNode(int32_t nodeType) { + SPhysiNode* pPhysiNode = qptCreatePhysiNode(nodeType); + + SStreamPartitionPhysiNode* pPartition = (SStreamPartitionPhysiNode*)pPhysiNode; + + qptCreatePartitionPhysiNodeImpl(&pPartition->part); + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_CORRECT_HIGH_PROB() ? true : false, QPT_RAND_BOOL_V, 0, NULL); + qptMakeColumnList(&pPartition->pTags); + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); + qptMakeExprNode(&pPartition->pSubtable); + + return (SNode*)pPhysiNode; +} + + + +SNode* qptCreatePhysicalPlanNode(int32_t nodeIdx) { + if (qptPlans[nodeIdx].buildFunc) { + return (*qptPlans[nodeIdx].buildFunc)(qptPlans[nodeIdx].type); } - return 0; + return NULL; } void qptCreateQueryPlan(SNode** ppPlan) { @@ -1547,12 +2387,8 @@ void qptRerunBlockedHere() { void qptResetForReRun() { qptCtx.param.plan.taskId = 1; qptCtx.param.vnode.vgId = 1; - for (int32_t i = 0; i < qptCtx.param.tbl.colNum; ++i) { - qptCtx.param.tbl.pCol[i].inUse = 0; - } - for (int32_t i = 0; i < qptCtx.param.tbl.tagNum; ++i) { - qptCtx.param.tbl.pTag[i].inUse = 0; - } + + qptResetTableCols(); } void qptSingleTestDone(bool* contLoop) { @@ -1595,7 +2431,7 @@ void qptHandleTestEnd() { } -void qptRunSingleOpTest(char* caseName) { +void qptRunSingleOpTest() { SNode* pNode = NULL; SReadHandle readHandle = {0}; SOperatorInfo* pOperator = NULL; @@ -1605,12 +2441,13 @@ void qptRunSingleOpTest(char* caseName) { if (qptCtx.loopIdx > 0) { qptResetForReRun(); } - - pNode = (SNode*)qptCreatePhysicalPlanNode(qptCtx.param.plan.subplanType[0]); - qptPrintBeginInfo(caseName); - doCreateTask(qptCtx.param.plan.queryId, qptCtx.param.plan.taskId++, qptCtx.param.vnode.vgId, OPTR_EXEC_MODEL_BATCH, &storageAPI, &pTaskInfo); + qptCtx.buildCtx.pCurrTask = pTaskInfo; + + pNode = (SNode*)qptCreatePhysicalPlanNode(qptCtx.param.plan.subplanIdx[0]); + + qptPrintBeginInfo(); qptCtx.startTsUs = taosGetTimestampUs(); //qptCtx.result.code = createTagScanOperatorInfo(&readHandle, (STagScanPhysiNode*)pNode, NULL, NULL, NULL, pTaskInfo, &pOperator); @@ -1620,12 +2457,12 @@ void qptRunSingleOpTest(char* caseName) { destroyOperator(pOperator); nodesDestroyNode((SNode*)pNode); - qptPrintEndInfo(caseName); + qptPrintEndInfo(); qptHandleTestEnd(); } -void qptRunSubplanTest(char* caseName) { +void qptRunSubplanTest() { SNode* pNode = NULL; SReadHandle readHandle = {0}; SOperatorInfo* pOperator = NULL; @@ -1636,7 +2473,7 @@ void qptRunSubplanTest(char* caseName) { //pNode = (SNode*)qptCreatePhysicalPlanNode(qptCtx.param.plan.subplanType[0]); - qptPrintBeginInfo(caseName); + qptPrintBeginInfo(); qptCtx.startTsUs = taosGetTimestampUs(); //qptCtx.result.code = createTagScanOperatorInfo(&readHandle, (STagScanPhysiNode*)pNode, NULL, NULL, NULL, NULL, &pOperator); @@ -1645,17 +2482,17 @@ void qptRunSubplanTest(char* caseName) { destroyOperator(pOperator); nodesDestroyNode((SNode*)pNode); - qptPrintEndInfo(caseName); + qptPrintEndInfo(); qptHandleTestEnd(); } -void qptRunPlanTest(char* caseName) { +void qptRunPlanTest() { if (qptCtx.param.plan.singlePhysiNode) { - qptRunSingleOpTest(caseName); + qptRunSingleOpTest(); } else { - qptRunSubplanTest(caseName); + qptRunSubplanTest(); } } @@ -1663,77 +2500,52 @@ SQPTNodeParam* qptInitNodeParam(int32_t nodeType) { return NULL; } -int32_t qptGetColumnRandLen(int32_t colType) { - switch (colType) { - case TSDB_DATA_TYPE_NULL: - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_FLOAT: - case TSDB_DATA_TYPE_DOUBLE: - case TSDB_DATA_TYPE_UTINYINT: - case TSDB_DATA_TYPE_USMALLINT: - case TSDB_DATA_TYPE_UINT: - case TSDB_DATA_TYPE_UBIGINT: - return tDataTypes[colType].bytes; - case TSDB_DATA_TYPE_VARCHAR: - case TSDB_DATA_TYPE_GEOMETRY: - case TSDB_DATA_TYPE_JSON: - case TSDB_DATA_TYPE_VARBINARY: - case TSDB_DATA_TYPE_DECIMAL: - case TSDB_DATA_TYPE_BLOB: - case TSDB_DATA_TYPE_MEDIUMBLOB: - case TSDB_DATA_TYPE_NCHAR: - return taosRand() % TSDB_MAX_BINARY_LEN; - default: - assert(0); - break; - } -} - -void qptInitTableCols(SQPTCol* pCol, int32_t colNum, EColumnType colType) { +void qptInitTableCols(SNodeList** ppList, int32_t colNum, EColumnType colType) { + SQPTCol* pCol = NULL; int32_t tbnameIdx = -1; if (QPT_RAND_BOOL_V && COLUMN_TYPE_TAG == colType) { tbnameIdx = taosRand() % colNum; } for (int32_t i = 0; i < colNum; ++i) { + qptNodesCalloc(1, sizeof(SQPTCol), (void**)&pCol); + pCol->type = QPT_QUERY_NODE_COL; + if (tbnameIdx >= 0 && i == tbnameIdx) { - strcpy(pCol[i].name, "tbname"); - pCol[i].type = TSDB_DATA_TYPE_VARCHAR; - pCol[i].len = qptGetColumnRandLen(pCol[i].type); - pCol[i].inUse = 0; - pCol[i].hasIndex = QPT_RAND_BOOL_V; - pCol[i].isPrimTs = QPT_RAND_BOOL_V; - pCol[i].isPk = QPT_RAND_BOOL_V; - pCol[i].colType = COLUMN_TYPE_TBNAME; + strcpy(pCol->name, "tbname"); + pCol->dtype = TSDB_DATA_TYPE_VARCHAR; + pCol->len = qptGetColumnRandLen(pCol->dtype); + pCol->inUse = 0; + pCol->hasIndex = QPT_RAND_BOOL_V; + pCol->isPrimTs = QPT_RAND_BOOL_V; + pCol->isPk = QPT_RAND_BOOL_V; + pCol->colType = COLUMN_TYPE_TBNAME; + + qptNodesListMakeStrictAppend(ppList, (SNode *)pCol); continue; } + + qptInitSingleTableCol(pCol, i, colType); - sprintf(pCol[i].name, "col%d", i); - pCol[i].type = taosRand() % TSDB_DATA_TYPE_MAX; - pCol[i].len = qptGetColumnRandLen(pCol[i].type); - pCol[i].inUse = 0; - pCol[i].hasIndex = QPT_RAND_BOOL_V; - pCol[i].isPrimTs = QPT_RAND_BOOL_V; - pCol[i].isPk = QPT_RAND_BOOL_V; - pCol[i].colType = colType; + qptNodesListMakeStrictAppend(ppList, (SNode *)pCol); } } -void qptInitTestCtx(bool correctExpected, bool singleNode, int32_t nodeType, int32_t paramNum, SQPTNodeParam* nodeParam) { +void qptInitTestCtx(bool correctExpected, bool singleNode, int32_t nodeType, int32_t nodeIdx, int32_t paramNum, SQPTNodeParam* nodeParam) { qptCtx.param.correctExpected = correctExpected; + qptCtx.param.schedulerId = taosRand(); qptCtx.param.plan.singlePhysiNode = singleNode; + if (singleNode) { qptCtx.param.plan.subplanMaxLevel = 1; qptCtx.param.plan.subplanType[0] = nodeType; + qptCtx.param.plan.subplanIdx[0] = nodeIdx; } else { qptCtx.param.plan.subplanMaxLevel = taosRand() % QPT_MAX_SUBPLAN_LEVEL + 1; for (int32_t i = 0; i < qptCtx.param.plan.subplanMaxLevel; ++i) { - qptCtx.param.plan.subplanType[i] = QPT_PHYSIC_NODE_LIST[taosRand() % QPT_PHYSIC_NODE_NUM()]; + nodeIdx = taosRand() % QPT_PHYSIC_NODE_NUM(); + qptCtx.param.plan.subplanType[i] = qptPlans[nodeIdx].type; + qptCtx.param.plan.subplanIdx[i] = nodeIdx; } } @@ -1746,59 +2558,77 @@ void qptInitTestCtx(bool correctExpected, bool singleNode, int32_t nodeType, int strcpy(qptCtx.param.db.dbName, "qptdb1"); qptCtx.param.vnode.vnodeNum = QPT_DEFAULT_VNODE_NUM; + qptCtx.param.vnode.vgId = 1; + qptCtx.param.vnode.epSet.numOfEps = 1; + qptCtx.param.vnode.epSet.inUse = 0; + strcpy(qptCtx.param.vnode.epSet.eps[0].fqdn, "127.0.0.1"); + qptCtx.param.vnode.epSet.eps[0].port = 6030; qptCtx.param.tbl.uid = 100; qptCtx.param.tbl.suid = 1; qptGetRandRealTableType(&qptCtx.param.tbl.tblType); - qptCtx.param.tbl.colNum = taosRand() % 4098; - qptCtx.param.tbl.tagNum = taosRand() % 130; + qptCtx.param.tbl.colNum = taosRand() % 4096 + 1; + qptCtx.param.tbl.tagNum = taosRand() % 128 + 1; qptCtx.param.tbl.pkNum = taosRand() % 2; strcpy(qptCtx.param.tbl.tblName, "qpttbl1"); strcpy(qptCtx.param.tbl.tblName, "tbl1"); - qptCtx.param.tbl.pCol = (SQPTCol*)taosMemoryCalloc(qptCtx.param.tbl.colNum, sizeof(*qptCtx.param.tbl.pCol)); - assert(qptCtx.param.tbl.pCol); - qptInitTableCols(qptCtx.param.tbl.pCol, qptCtx.param.tbl.colNum, COLUMN_TYPE_COLUMN); - - qptCtx.param.tbl.pTag = (SQPTCol*)taosMemoryCalloc(qptCtx.param.tbl.tagNum, sizeof(*qptCtx.param.tbl.pTag)); - assert(qptCtx.param.tbl.pTag); - qptInitTableCols(qptCtx.param.tbl.pTag, qptCtx.param.tbl.tagNum, COLUMN_TYPE_TAG); + qptInitTableCols(&qptCtx.param.tbl.pColList, qptCtx.param.tbl.colNum, COLUMN_TYPE_COLUMN); + qptInitTableCols(&qptCtx.param.tbl.pTagList, qptCtx.param.tbl.tagNum, COLUMN_TYPE_TAG); + + SNode* pTmp = NULL; + FOREACH(pTmp, qptCtx.param.tbl.pColList) { + qptNodesListMakeStrictAppend(&qptCtx.param.tbl.pColTagList, pTmp); + } + FOREACH(pTmp, qptCtx.param.tbl.pTagList) { + qptNodesListMakeStrictAppend(&qptCtx.param.tbl.pColTagList, pTmp); + } } +void qptDestroyTestCtx() { + SNode* pTmp = NULL; + FOREACH(pTmp, qptCtx.param.tbl.pColList) { + qptNodesFree(pTmp); + } + FOREACH(pTmp, qptCtx.param.tbl.pTagList) { + qptNodesFree(pTmp); + } + nodesClearList(qptCtx.param.tbl.pColList); + nodesClearList(qptCtx.param.tbl.pTagList); + nodesClearList(qptCtx.param.tbl.pColTagList); + + qptCtx.param.tbl.pColList = NULL; + qptCtx.param.tbl.pTagList = NULL; + qptCtx.param.tbl.pColTagList = NULL; +} } // namespace #if 1 -#if 0 -TEST(randSingleNodeTest, tagScan) { - char* caseName = "randSingleNodeTest:tagScan"; - - qptInitTestCtx(false, true, QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN, 0, NULL); - - for (qptCtx.loopIdx = 0; qptCtx.loopIdx < QPT_MAX_LOOP; ++qptCtx.loopIdx) { - qptRunPlanTest(caseName); - } - - qptPrintStatInfo(caseName); -} -#endif #if 1 -TEST(randSingleNodeTest, projection) { - char* caseName = "randSingleNodeTest:projection"; +TEST(singleNodeTest, randPlan) { + char* caseType = "singleNodeTest:randPlan"; - qptInitTestCtx(false, true, QUERY_NODE_PHYSICAL_PLAN_PROJECT, 0, NULL); - for (qptCtx.loopIdx = 0; qptCtx.loopIdx < QPT_MAX_LOOP; ++qptCtx.loopIdx) { - qptRunPlanTest(caseName); + for (int32_t i = 0; i < sizeof(qptPlans)/sizeof(qptPlans[0]); ++i) { + sprintf(qptCtx.caseName, "%s:%s", caseType, qptPlans[i].name); + qptInitTestCtx(false, true, qptPlans[i].type, i, 0, NULL); + + qptRunPlanTest(); + + qptDestroyTestCtx(); + } } - qptPrintStatInfo(caseName); + qptPrintStatInfo(); } #endif + + #if 0 TEST(correctSingleNodeTest, tagScan) { char* caseName = "correctSingleNodeTest:tagScan"; diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 1b63998513..382584bb76 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -691,6 +691,8 @@ int32_t nodesMakeNode(ENodeType type, SNode** ppNodeOut) { code = makeNode(type, sizeof(SGroupSortPhysiNode), &pNode); break; case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: code = makeNode(type, sizeof(SIntervalPhysiNode), &pNode); break; + case QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL: + code = makeNode(type, sizeof(SMergeIntervalPhysiNode), &pNode); break; case QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL: code = makeNode(type, sizeof(SMergeAlignedIntervalPhysiNode), &pNode); break; case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: @@ -1619,6 +1621,7 @@ void nodesDestroyNode(SNode* pNode) { break; } case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL: From 041f3380bae395c400a9f104c939ac024f5ccc12 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 16 Oct 2024 19:24:42 +0800 Subject: [PATCH 11/29] enh: add more rand plan --- source/libs/executor/test/queryPlanTests.cpp | 406 ++++++++++++++++--- 1 file changed, 356 insertions(+), 50 deletions(-) diff --git a/source/libs/executor/test/queryPlanTests.cpp b/source/libs/executor/test/queryPlanTests.cpp index 2be84a5785..338c734114 100755 --- a/source/libs/executor/test/queryPlanTests.cpp +++ b/source/libs/executor/test/queryPlanTests.cpp @@ -195,6 +195,8 @@ SNode* qptCreateStreamScanPhysiNode(int32_t nodeType); SNode* qptCreateSysTableScanPhysiNode(int32_t nodeType); SNode* qptCreateBlockDistScanPhysiNode(int32_t nodeType); SNode* qptCreateLastRowScanPhysiNode(int32_t nodeType); +SNode* qptCreateTableCountScanPhysiNode(int32_t nodeType); + SNode* qptCreateProjectPhysiNode(int32_t nodeType); SNode* qptCreateMergeJoinPhysiNode(int32_t nodeType); SNode* qptCreateHashAggPhysiNode(int32_t nodeType); @@ -208,6 +210,7 @@ SNode* qptCreateMergeAlignedIntervalPhysiNode(int32_t nodeType); SNode* qptCreateStreamIntervalPhysiNode(int32_t nodeType); SNode* qptCreateStreamFinalIntervalPhysiNode(int32_t nodeType); SNode* qptCreateStreamSemiIntervalPhysiNode(int32_t nodeType); +SNode* qptCreateStreamMidIntervalPhysiNode(int32_t nodeType); SNode* qptCreateFillPhysiNode(int32_t nodeType); SNode* qptCreateStreamFillPhysiNode(int32_t nodeType); SNode* qptCreateSessionPhysiNode(int32_t nodeType); @@ -218,6 +221,19 @@ SNode* qptCreateStateWindowPhysiNode(int32_t nodeType); SNode* qptCreateStreamStatePhysiNode(int32_t nodeType); SNode* qptCreatePartitionPhysiNode(int32_t nodeType); SNode* qptCreateStreamPartitionPhysiNode(int32_t nodeType); +SNode* qptCreateIndefRowsFuncPhysiNode(int32_t nodeType); +SNode* qptCreateInterpFuncPhysiNode(int32_t nodeType); +SNode* qptCreateMergeEventPhysiNode(int32_t nodeType); +SNode* qptCreateStreamEventPhysiNode(int32_t nodeType); +SNode* qptCreateCountWindowPhysiNode(int32_t nodeType); +SNode* qptCreateStreamCountWindowPhysiNode(int32_t nodeType); +SNode* qptCreateHashJoinPhysiNode(int32_t nodeType); +SNode* qptCreateGroupCachePhysiNode(int32_t nodeType); +SNode* qptCreateDynQueryCtrlPhysiNode(int32_t nodeType); +SNode* qptCreateDataDispatchPhysiNode(int32_t nodeType); +SNode* qptCreateDataInsertPhysiNode(int32_t nodeType); +SNode* qptCreateDataQueryInsertPhysiNode(int32_t nodeType); + SQPTPlan qptPlans[] = { {QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN, "tagScan", qptCreateTagScanPhysiNode}, @@ -251,23 +267,23 @@ SQPTPlan qptPlans[] = { {QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE, "streamState", qptCreateStreamStatePhysiNode}, {QUERY_NODE_PHYSICAL_PLAN_PARTITION, "partition", qptCreatePartitionPhysiNode}, {QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION, "streamPartition", qptCreateStreamPartitionPhysiNode}, - {QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC, "", NULL}, - {QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC, "", NULL}, - {QUERY_NODE_PHYSICAL_PLAN_DISPATCH, "", NULL}, - {QUERY_NODE_PHYSICAL_PLAN_INSERT, "", NULL}, - {QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT, "", NULL}, - {QUERY_NODE_PHYSICAL_PLAN_DELETE, "", NULL}, + {QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC, "indefRowsFunc", qptCreateIndefRowsFuncPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC, "interpFunc", qptCreateInterpFuncPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_DISPATCH, "dataDispatch", qptCreateDataDispatchPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_INSERT, "dataInseret", qptCreateDataInsertPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT, "dataQueryInsert", qptCreateDataQueryInsertPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_DELETE, "dataDelete", NULL}, {QUERY_NODE_PHYSICAL_SUBPLAN, "", NULL}, {QUERY_NODE_PHYSICAL_PLAN, "", NULL}, - {QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN, "", NULL}, - {QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT, "", NULL}, - {QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT, "", NULL}, - {QUERY_NODE_PHYSICAL_PLAN_HASH_JOIN, "", NULL}, - {QUERY_NODE_PHYSICAL_PLAN_GROUP_CACHE, "", NULL}, - {QUERY_NODE_PHYSICAL_PLAN_DYN_QUERY_CTRL, "", NULL}, - {QUERY_NODE_PHYSICAL_PLAN_MERGE_COUNT, "", NULL}, - {QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT, "", NULL}, - {QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL, "", NULL} + {QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN, "tableCountScan", qptCreateTableCountScanPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT, "eventWindow", qptCreateMergeEventPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT, "streamEventWindow", qptCreateStreamEventPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_HASH_JOIN, "hashJoin", qptCreateHashJoinPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_GROUP_CACHE, "groupCache", qptCreateGroupCachePhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_DYN_QUERY_CTRL, "dynQueryCtrl", qptCreateDynQueryCtrlPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_MERGE_COUNT, "countWindow", qptCreateCountWindowPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT, "streamCountWindow", qptCreateStreamCountWindowPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL, "streamMidInterval", qptCreateStreamMidIntervalPhysiNode} }; @@ -282,7 +298,7 @@ SQPTPlan qptPlans[] = { #define QPT_CORRECT_HIGH_PROB() (qptCtx.param.correctExpected || QPT_HIGH_PROB()) #define QPT_NCORRECT_LOW_PROB() (!qptCtx.param.correctExpected && QPT_LOW_PROB()) - +#define QPT_VALID_DESC(_desc) ((_desc) && (QUERY_NODE_DATABLOCK_DESC == nodeType(_desc))) SQPTCtx qptCtx = {0}; SQPTCtrl qptCtrl = {1, 0, 0, 0, 0}; @@ -1438,19 +1454,6 @@ SNode* qptMakeConditionNode() { return pNode; } -SNode* qptMakeDataBlockDescNode() { - if (QPT_NCORRECT_LOW_PROB()) { - return qptMakeRandNode(NULL); - } - - SDataBlockDescNode* pDesc = NULL; - assert(0 == nodesMakeNode(QUERY_NODE_DATABLOCK_DESC, (SNode**)&pDesc)); - - pDesc->dataBlockId = QPT_CORRECT_HIGH_PROB() ? qptCtx.buildCtx.nextBlockId++ : QPT_RAND_INT_V; - pDesc->precision = QPT_CORRECT_HIGH_PROB() ? qptCtx.param.db.precision : QPT_RAND_INT_V; - - return (SNode*)pDesc; -} SNode* qptMakeSlotDescNode(const char* pName, const SNode* pNode, int16_t slotId, bool output, bool reserve) { SSlotDescNode* pSlot = NULL; @@ -1474,6 +1477,75 @@ SNode* qptMakeSlotDescNode(const char* pName, const SNode* pNode, int16_t slotId return (SNode*)pSlot; } + +SNode* qptMakeDataBlockDescNode(bool forSink) { + if (QPT_NCORRECT_LOW_PROB()) { + return qptMakeRandNode(NULL); + } + + SDataBlockDescNode* pDesc = NULL; + assert(0 == nodesMakeNode(QUERY_NODE_DATABLOCK_DESC, (SNode**)&pDesc)); + + pDesc->dataBlockId = QPT_CORRECT_HIGH_PROB() ? (forSink ? (qptCtx.buildCtx.nextBlockId - 1) : qptCtx.buildCtx.nextBlockId++) : QPT_RAND_INT_V; + pDesc->precision = QPT_CORRECT_HIGH_PROB() ? qptCtx.param.db.precision : QPT_RAND_INT_V; + + return (SNode*)pDesc; +} + +SNode* qptMakeDataBlockDescNodeFromNode(bool forSink) { + if (QPT_NCORRECT_LOW_PROB()) { + return qptMakeRandNode(NULL); + } + + SDataBlockDescNode* pDesc = NULL; + SDataBlockDescNode* pInput = qptCtx.buildCtx.pCurr ? qptCtx.buildCtx.pCurr->pOutputDataBlockDesc : NULL; + SNode* pTmp = NULL, *pTmp2 = NULL; + + if (QPT_VALID_DESC(pInput)) { + if (QPT_CORRECT_HIGH_PROB()) { + nodesCloneNode((SNode*)pInput, (SNode**)&pDesc); + } else { + assert(0 == nodesMakeNode(QUERY_NODE_DATABLOCK_DESC, (SNode**)&pDesc)); + + pDesc->dataBlockId = QPT_CORRECT_HIGH_PROB() ? pInput->dataBlockId : QPT_RAND_INT_V; + pDesc->precision = QPT_CORRECT_HIGH_PROB() ? pInput->precision : QPT_RAND_INT_V; + pDesc->totalRowSize = QPT_CORRECT_HIGH_PROB() ? pInput->totalRowSize : QPT_RAND_INT_V; + pDesc->outputRowSize = QPT_CORRECT_HIGH_PROB() ? pInput->outputRowSize : QPT_RAND_INT_V; + + FOREACH(pTmp, pInput->pSlots) { + if (QPT_RAND_BOOL_V) { + nodesCloneNode(pTmp, &pTmp2); + qptNodesListMakeStrictAppend(&pDesc->pSlots, pTmp2); + } + } + } + } else { + assert(0 == nodesMakeNode(QUERY_NODE_DATABLOCK_DESC, (SNode**)&pDesc)); + + pDesc->dataBlockId = QPT_CORRECT_HIGH_PROB() ? (forSink ? (qptCtx.buildCtx.nextBlockId - 1) : qptCtx.buildCtx.nextBlockId++) : QPT_RAND_INT_V; + pDesc->precision = QPT_CORRECT_HIGH_PROB() ? qptCtx.param.db.precision : QPT_RAND_INT_V; + pDesc->totalRowSize = QPT_RAND_INT_V; + pDesc->outputRowSize = QPT_RAND_INT_V; + + int32_t slotNum = taosRand() % QPT_MAX_COLUMN_NUM; + for (int32_t i = 0; i < slotNum; ++i) { + pTmp2 = qptMakeExprNode(NULL); + if (QPT_CORRECT_HIGH_PROB()) { + pTmp = qptMakeSlotDescNode(NULL, pTmp2, i, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V); + nodesDestroyNode(pTmp2); + } else { + pTmp = pTmp2; + } + + qptNodesListMakeStrictAppend(&pDesc->pSlots, pTmp); + } + } + + return (SNode*)pDesc; +} + + + SNode* qptMakeTargetNode(SNode* pNode, int16_t dataBlockId, int16_t slotId, SNode** pOutput) { if (QPT_NCORRECT_LOW_PROB()) { nodesDestroyNode(pNode); @@ -1547,7 +1619,7 @@ SPhysiNode* qptCreatePhysiNode(int32_t nodeType) { pPhysiNode->dynamicOp = qptGetDynamicOp(); pPhysiNode->inputTsOrder = qptGetCurrTsOrder(); - pPhysiNode->pOutputDataBlockDesc = (SDataBlockDescNode*)qptMakeDataBlockDescNode(); + pPhysiNode->pOutputDataBlockDesc = (SDataBlockDescNode*)qptMakeDataBlockDescNode(false); return pPhysiNode; } @@ -1649,7 +1721,7 @@ void qptCreateTableScanPseudoCols( int16_t blockId, SNodeList** ppList) { void qptAddDataBlockSlots(SNodeList* pList, SDataBlockDescNode* pDataBlockDesc) { - if (NULL == pDataBlockDesc) { + if (NULL == pDataBlockDesc || QUERY_NODE_DATABLOCK_DESC != nodeType(pDataBlockDesc)) { return; } @@ -1838,35 +1910,36 @@ void qptMakeAppendToTargetList(SNodeList* pInputList, int16_t blockId, SNodeList } } -void qptCreateScanPhysiNodeImpl( SScanPhysiNode* pScanPhysiNode) { - int16_t blockId = (QPT_CORRECT_HIGH_PROB() && pScanPhysiNode->node.pOutputDataBlockDesc) ? pScanPhysiNode->node.pOutputDataBlockDesc->dataBlockId : taosRand(); - qptCreateTableScanCols(blockId, &pScanPhysiNode->pScanCols); +void qptCreateScanPhysiNodeImpl( SScanPhysiNode* pScan) { + SDataBlockDescNode* pDesc = pScan->node.pOutputDataBlockDesc; + int16_t blockId = (QPT_CORRECT_HIGH_PROB() && QPT_VALID_DESC(pDesc)) ? pDesc->dataBlockId : taosRand(); + qptCreateTableScanCols(blockId, &pScan->pScanCols); - qptAddDataBlockSlots(pScanPhysiNode->pScanCols, pScanPhysiNode->node.pOutputDataBlockDesc); + qptAddDataBlockSlots(pScan->pScanCols, pDesc); if (taosRand() % 2) { - blockId = (QPT_CORRECT_HIGH_PROB() && pScanPhysiNode->node.pOutputDataBlockDesc) ? pScanPhysiNode->node.pOutputDataBlockDesc->dataBlockId : taosRand(); - qptCreateTableScanPseudoCols(blockId, &pScanPhysiNode->pScanPseudoCols); + blockId = (QPT_CORRECT_HIGH_PROB() && QPT_VALID_DESC(pDesc)) ? pDesc->dataBlockId : taosRand(); + qptCreateTableScanPseudoCols(blockId, &pScan->pScanPseudoCols); } - qptAddDataBlockSlots(pScanPhysiNode->pScanPseudoCols, pScanPhysiNode->node.pOutputDataBlockDesc); + qptAddDataBlockSlots(pScan->pScanPseudoCols, pDesc); - pScanPhysiNode->uid = qptCtx.param.correctExpected ? qptCtx.param.tbl.uid : taosRand(); - pScanPhysiNode->suid = qptCtx.param.correctExpected ? qptCtx.param.tbl.suid : taosRand(); - pScanPhysiNode->tableType = qptCtx.param.correctExpected ? qptCtx.param.tbl.tblType : taosRand(); - pScanPhysiNode->groupOrderScan = (taosRand() % 2) ? true : false; + pScan->uid = QPT_CORRECT_HIGH_PROB() ? qptCtx.param.tbl.uid : taosRand(); + pScan->suid = QPT_CORRECT_HIGH_PROB() ? qptCtx.param.tbl.suid : taosRand(); + pScan->tableType = QPT_CORRECT_HIGH_PROB() ? qptCtx.param.tbl.tblType : taosRand(); + pScan->groupOrderScan = (taosRand() % 2) ? true : false; SName tblName = {0}; toName(1, qptCtx.param.db.dbName, qptCtx.param.tbl.tblName, &tblName); - if (qptCtx.param.correctExpected || QPT_RAND_BOOL_V) { - memcpy(&pScanPhysiNode->tableName, &tblName, sizeof(SName)); + if (QPT_CORRECT_HIGH_PROB()) { + memcpy(&pScan->tableName, &tblName, sizeof(SName)); } else { - pScanPhysiNode->tableName.acctId = 0; - pScanPhysiNode->tableName.dbname[0] = 0; - pScanPhysiNode->tableName.tname[0] = 0; + pScan->tableName.acctId = 0; + pScan->tableName.dbname[0] = 0; + pScan->tableName.tname[0] = 0; } - qptCtx.buildCtx.currTsOrder = (qptCtx.param.correctExpected) ? qptCtx.buildCtx.currTsOrder : QPT_RAND_ORDER_V; + qptCtx.buildCtx.currTsOrder = QPT_CORRECT_HIGH_PROB() ? qptCtx.buildCtx.currTsOrder : QPT_RAND_ORDER_V; } @@ -2011,6 +2084,11 @@ SNode* qptCreateLastRowScanPhysiNode(int32_t nodeType) { return (SNode*)pPhysiNode; } +SNode* qptCreateTableCountScanPhysiNode(int32_t nodeType) { + return qptCreateLastRowScanPhysiNode(nodeType); +} + + SNode* qptCreateProjectPhysiNode(int32_t nodeType) { SPhysiNode* pPhysiNode = qptCreatePhysiNode(nodeType); @@ -2090,10 +2168,11 @@ SNode* qptCreateHashAggPhysiNode(int32_t nodeType) { SAggPhysiNode* pAgg = (SAggPhysiNode*)pPhysiNode; qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); - qptMakeExprList(&pAgg->pExprs); + int16_t blockId = (QPT_CORRECT_HIGH_PROB() && pPhysiNode->pOutputDataBlockDesc) ? pPhysiNode->pOutputDataBlockDesc->dataBlockId : taosRand(); + qptMakeTargetList(QPT_NODE_EXPR, blockId, &pAgg->pExprs); qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); - int16_t blockId = (QPT_CORRECT_HIGH_PROB() && pPhysiNode->pOutputDataBlockDesc) ? pPhysiNode->pOutputDataBlockDesc->dataBlockId : taosRand(); + blockId = (QPT_CORRECT_HIGH_PROB() && pPhysiNode->pOutputDataBlockDesc) ? pPhysiNode->pOutputDataBlockDesc->dataBlockId : taosRand(); qptMakeTargetList(QPT_NODE_EXPR, blockId, &pAgg->pGroupKeys); qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); @@ -2249,6 +2328,12 @@ SNode* qptCreateStreamSemiIntervalPhysiNode(int32_t nodeType) { return qptCreateIntervalPhysiNode(nodeType); } + +SNode* qptCreateStreamMidIntervalPhysiNode(int32_t nodeType) { + return qptCreateIntervalPhysiNode(nodeType); +} + + SNode* qptCreateFillPhysiNode(int32_t nodeType) { SPhysiNode* pPhysiNode = qptCreatePhysiNode(nodeType); @@ -2364,6 +2449,227 @@ SNode* qptCreateStreamPartitionPhysiNode(int32_t nodeType) { } +SNode* qptCreateIndefRowsFuncPhysiNode(int32_t nodeType) { + SPhysiNode* pPhysiNode = qptCreatePhysiNode(nodeType); + + SIndefRowsFuncPhysiNode* pFunc = (SIndefRowsFuncPhysiNode*)pPhysiNode; + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); + int16_t blockId = (QPT_CORRECT_HIGH_PROB() && pPhysiNode->pOutputDataBlockDesc) ? pPhysiNode->pOutputDataBlockDesc->dataBlockId : taosRand(); + qptMakeTargetList(QPT_NODE_EXPR, blockId, &pFunc->pExprs); + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); + blockId = (QPT_CORRECT_HIGH_PROB() && pPhysiNode->pOutputDataBlockDesc) ? pPhysiNode->pOutputDataBlockDesc->dataBlockId : taosRand(); + qptMakeTargetList(QPT_NODE_FUNCTION, blockId, &pFunc->pFuncs); + + return (SNode*)pPhysiNode; +} + + +SNode* qptCreateInterpFuncPhysiNode(int32_t nodeType) { + SPhysiNode* pPhysiNode = qptCreatePhysiNode(nodeType); + + SInterpFuncPhysiNode* pFunc = (SInterpFuncPhysiNode*)pPhysiNode; + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); + int16_t blockId = (QPT_CORRECT_HIGH_PROB() && pPhysiNode->pOutputDataBlockDesc) ? pPhysiNode->pOutputDataBlockDesc->dataBlockId : taosRand(); + qptMakeTargetList(QPT_NODE_EXPR, blockId, &pFunc->pExprs); + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); + blockId = (QPT_CORRECT_HIGH_PROB() && pPhysiNode->pOutputDataBlockDesc) ? pPhysiNode->pOutputDataBlockDesc->dataBlockId : taosRand(); + qptMakeTargetList(QPT_NODE_FUNCTION, blockId, &pFunc->pFuncs); + + qptGetRandTimeWindow(&pFunc->timeRange); + + pFunc->interval = taosRand(); + pFunc->intervalUnit = qptGetRandTimestampUnit(); + + pFunc->fillMode = qptGetRandFillMode(); + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); + qptMakeNodeListNode(QPT_NODE_VALUE, &pFunc->pFillValues); + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); + qptMakeColumnNode(&pFunc->pTimeSeries); + + return (SNode*)pPhysiNode; +} + +SNode* qptCreateMergeEventPhysiNode(int32_t nodeType) { + SPhysiNode* pPhysiNode = qptCreatePhysiNode(nodeType); + + SEventWinodwPhysiNode* pEvent = (SEventWinodwPhysiNode*)pPhysiNode; + + qptCreateWindowPhysiNode(&pEvent->window); + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); + qptMakeExprNode(&pEvent->pStartCond); + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); + qptMakeExprNode(&pEvent->pEndCond); + + return (SNode*)pPhysiNode; +} + +SNode* qptCreateStreamEventPhysiNode(int32_t nodeType) { + return qptCreateMergeEventPhysiNode(nodeType); +} + +SNode* qptCreateCountWindowPhysiNode(int32_t nodeType) { + SPhysiNode* pPhysiNode = qptCreatePhysiNode(nodeType); + + SCountWinodwPhysiNode* pCount = (SCountWinodwPhysiNode*)pPhysiNode; + + qptCreateWindowPhysiNode(&pCount->window); + + pCount->windowCount = taosRand(); + pCount->windowSliding = taosRand(); + + return (SNode*)pPhysiNode; +} + +SNode* qptCreateStreamCountWindowPhysiNode(int32_t nodeType) { + return qptCreateCountWindowPhysiNode(nodeType); +} + +SNode* qptCreateHashJoinPhysiNode(int32_t nodeType) { + SPhysiNode* pPhysiNode = qptCreatePhysiNode(nodeType); + + SHashJoinPhysiNode* pJoin = (SHashJoinPhysiNode*)pPhysiNode; + + pJoin->joinType = (EJoinType)(taosRand() % JOIN_TYPE_MAX_VALUE + (QPT_CORRECT_HIGH_PROB() ? 0 : 1)); + pJoin->subType = (EJoinSubType)(taosRand() % JOIN_STYPE_MAX_VALUE + (QPT_CORRECT_HIGH_PROB() ? 0 : 1)); + qptMakeWindowOffsetNode(&pJoin->pWindowOffset); + qptMakeLimitNode(&pJoin->pJLimit); + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); + qptMakeColumnList(&pJoin->pOnLeft); + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); + qptMakeColumnList(&pJoin->pOnRight); + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); + qptMakeExprNode(&pJoin->leftPrimExpr); + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); + qptMakeExprNode(&pJoin->rightPrimExpr); + + pJoin->leftPrimSlotId = qptGetInputSlotId(qptCtx.buildCtx.pChild ? qptCtx.buildCtx.pChild->pOutputDataBlockDesc : NULL); + pJoin->rightPrimSlotId = qptGetInputSlotId(qptCtx.buildCtx.pChild ? qptCtx.buildCtx.pChild->pOutputDataBlockDesc : NULL); + + pJoin->timeRangeTarget = QPT_CORRECT_HIGH_PROB() ? (taosRand() % 3) : taosRand(); + qptGetRandTimeWindow(&pJoin->timeRange); + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); + qptMakeExprNode(&pJoin->pLeftOnCond); + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); + qptMakeExprNode(&pJoin->pRightOnCond); + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); + qptMakeExprNode(&pJoin->pFullOnCond); + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); + int16_t blockId = (QPT_CORRECT_HIGH_PROB() && pPhysiNode->pOutputDataBlockDesc) ? pPhysiNode->pOutputDataBlockDesc->dataBlockId : taosRand(); + qptMakeTargetList(QPT_NODE_EXPR, blockId, &pJoin->pTargets); + + for (int32_t i = 0; i < 2; i++) { + pJoin->inputStat[i].inputRowNum = taosRand(); + pJoin->inputStat[i].inputRowSize = taosRand(); + } + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); + qptMakeExprNode(&pJoin->pPrimKeyCond); + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); + qptMakeExprNode(&pJoin->pColEqCond); + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_CORRECT_HIGH_PROB() ? true : false, QPT_RAND_BOOL_V, 0, NULL); + qptMakeExprNode(&pJoin->pTagEqCond); + + return (SNode*)pPhysiNode; +} + +SNode* qptCreateGroupCachePhysiNode(int32_t nodeType) { + SPhysiNode* pPhysiNode = qptCreatePhysiNode(nodeType); + + SGroupCachePhysiNode* pGroup = (SGroupCachePhysiNode*)pPhysiNode; + + pGroup->grpColsMayBeNull = QPT_RAND_BOOL_V; + pGroup->grpByUid = QPT_RAND_BOOL_V; + pGroup->globalGrp = QPT_RAND_BOOL_V; + pGroup->batchFetch = QPT_RAND_BOOL_V; + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); + qptMakeColumnList(&pGroup->pGroupCols); + + return (SNode*)pPhysiNode; +} + +SNode* qptCreateDynQueryCtrlPhysiNode(int32_t nodeType) { + SPhysiNode* pPhysiNode = qptCreatePhysiNode(nodeType); + + SDynQueryCtrlPhysiNode* pDyn = (SDynQueryCtrlPhysiNode*)pPhysiNode; + + pDyn->qType = QPT_CORRECT_HIGH_PROB() ? DYN_QTYPE_STB_HASH : (EDynQueryType)taosRand(); + + SStbJoinDynCtrlBasic* pJoin = &pDyn->stbJoin; + pJoin->batchFetch = QPT_RAND_BOOL_V; + pJoin->vgSlot[0] = taosRand(); + pJoin->vgSlot[1] = taosRand(); + pJoin->uidSlot[0] = taosRand(); + pJoin->uidSlot[1] = taosRand(); + pJoin->srcScan[0] = QPT_RAND_BOOL_V; + pJoin->srcScan[1] = QPT_RAND_BOOL_V; + + return (SNode*)pPhysiNode; +} + +SNode* qptCreateDataSinkNode(int32_t nodeType) { + SDataSinkNode* pSinkNode = NULL; + assert(0 == nodesMakeNode((ENodeType)nodeType, (SNode**)&pSinkNode)); + assert(pSinkNode); + + if (QPT_CORRECT_HIGH_PROB() && qptCtx.buildCtx.pCurr && qptCtx.buildCtx.pCurr->pOutputDataBlockDesc) { + pSinkNode->pInputDataBlockDesc = (SDataBlockDescNode*)qptMakeDataBlockDescNodeFromNode(true); + } else { + pSinkNode->pInputDataBlockDesc = (SDataBlockDescNode*)qptMakeDataBlockDescNode(true); + } + + return (SNode*)pSinkNode; +} + +SNode* qptCreateDataDispatchPhysiNode(int32_t nodeType) { + return (SNode*)qptCreateDataSinkNode(nodeType); +} + +SNode* qptCreateDataInsertPhysiNode(int32_t nodeType) { + SDataInserterNode* pInserter = (SDataInserterNode*)qptCreateDataSinkNode(nodeType); + + pInserter->numOfTables = taosRand(); + pInserter->size = taosRand(); + pInserter->pData = QPT_RAND_BOOL_V ? taosMemoryMalloc(1) : NULL; + + return (SNode*)pInserter; +} + +SNode* qptCreateDataQueryInsertPhysiNode(int32_t nodeType) { + SQueryInserterNode* pInserter = (SQueryInserterNode*)qptCreateDataSinkNode(nodeType); + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); + qptMakeColumnList(&pInserter->pCols); + + pInserter->tableId = QPT_CORRECT_HIGH_PROB() ? qptCtx.param.tbl.uid : taosRand(); + pInserter->stableId = QPT_CORRECT_HIGH_PROB() ? qptCtx.param.tbl.suid : taosRand(); + pInserter->tableType = QPT_CORRECT_HIGH_PROB() ? (QPT_RAND_BOOL_V ? TSDB_CHILD_TABLE : TSDB_NORMAL_TABLE) : (taosRand() % TSDB_TABLE_MAX); + pInserter->tableName[0] = QPT_RAND_BOOL_V ? 'a' : taosRand(); + pInserter->vgId = qptCtx.param.vnode.vgId; + memcpy(&pInserter->epSet, &qptCtx.param.vnode.epSet, sizeof(pInserter->epSet)); + pInserter->explain = QPT_RAND_BOOL_V; + + return (SNode*)pInserter; +} + SNode* qptCreatePhysicalPlanNode(int32_t nodeIdx) { if (qptPlans[nodeIdx].buildFunc) { From 44726fe42a3f6628b57cf428373b881a9f7ca2c6 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 17 Oct 2024 11:28:05 +0800 Subject: [PATCH 12/29] enh: add subplan test --- include/libs/nodes/plannodes.h | 6 +- source/libs/executor/test/queryPlanTests.cpp | 98 ++++++++++++++++++-- 2 files changed, 93 insertions(+), 11 deletions(-) diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index bfe9e9555b..5ba24f08d8 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -766,9 +766,9 @@ typedef struct SDataDeleterNode { char tableFName[TSDB_TABLE_NAME_LEN]; char tsColName[TSDB_COL_NAME_LEN]; STimeWindow deleteTimeRange; - SNode* pAffectedRows; - SNode* pStartTs; - SNode* pEndTs; + SNode* pAffectedRows; // usless + SNode* pStartTs; // usless + SNode* pEndTs; // usless } SDataDeleterNode; typedef struct SSubplan { diff --git a/source/libs/executor/test/queryPlanTests.cpp b/source/libs/executor/test/queryPlanTests.cpp index 338c734114..6faeaed2cf 100755 --- a/source/libs/executor/test/queryPlanTests.cpp +++ b/source/libs/executor/test/queryPlanTests.cpp @@ -146,6 +146,8 @@ typedef struct { EOrder currTsOrder; int16_t nextBlockId; int32_t primaryTsSlotId; + SSubplanId nextSubplanId; + int32_t currSubplanLevel; SExecTaskInfo* pCurrTask; } SQPTBuildPlanCtx; @@ -233,6 +235,7 @@ SNode* qptCreateDynQueryCtrlPhysiNode(int32_t nodeType); SNode* qptCreateDataDispatchPhysiNode(int32_t nodeType); SNode* qptCreateDataInsertPhysiNode(int32_t nodeType); SNode* qptCreateDataQueryInsertPhysiNode(int32_t nodeType); +SNode* qptCreateDataDeletePhysiNode(int32_t nodeType); SQPTPlan qptPlans[] = { @@ -272,9 +275,9 @@ SQPTPlan qptPlans[] = { {QUERY_NODE_PHYSICAL_PLAN_DISPATCH, "dataDispatch", qptCreateDataDispatchPhysiNode}, {QUERY_NODE_PHYSICAL_PLAN_INSERT, "dataInseret", qptCreateDataInsertPhysiNode}, {QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT, "dataQueryInsert", qptCreateDataQueryInsertPhysiNode}, - {QUERY_NODE_PHYSICAL_PLAN_DELETE, "dataDelete", NULL}, - {QUERY_NODE_PHYSICAL_SUBPLAN, "", NULL}, - {QUERY_NODE_PHYSICAL_PLAN, "", NULL}, + {QUERY_NODE_PHYSICAL_PLAN_DELETE, "dataDelete", qptCreateDataDeletePhysiNode}, + {QUERY_NODE_PHYSICAL_SUBPLAN, "subplan", NULL}, + {QUERY_NODE_PHYSICAL_PLAN, "plan", NULL}, {QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN, "tableCountScan", qptCreateTableCountScanPhysiNode}, {QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT, "eventWindow", qptCreateMergeEventPhysiNode}, {QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT, "streamEventWindow", qptCreateStreamEventPhysiNode}, @@ -686,6 +689,12 @@ int32_t qptGetInputPrimaryTsSlotId() { return taosRand() % QPT_MAX_COLUMN_NUM; } +int32_t qptGetRandSubplanMsgType() { + int32_t msgTypeList[] = {TDMT_VND_DELETE, TDMT_SCH_MERGE_QUERY, TDMT_SCH_QUERY, TDMT_VND_SUBMIT}; + + return QPT_CORRECT_HIGH_PROB() ? msgTypeList[taosRand() % (sizeof(msgTypeList)/sizeof(msgTypeList[0]))] : taosRand(); +} + void qptNodesCalloc(int32_t num, int32_t size, void** pOut) { void* p = taosMemoryCalloc(num, size); assert(p); @@ -718,6 +727,11 @@ void qptGetRandTimeWindow(STimeWindow* pWindow) { pWindow->ekey = taosRand(); } +void qptGetSubplanId(SSubplanId* pId) { + pId->queryId = qptCtx.buildCtx.nextSubplanId.queryId; + pId->groupId = qptCtx.buildCtx.nextSubplanId.groupId++; + pId->subplanId = qptCtx.buildCtx.nextSubplanId.subplanId++; +} int32_t qptNodesListAppend(SNodeList* pList, SNode* pNode) { SListCell* p = NULL; @@ -2662,7 +2676,11 @@ SNode* qptCreateDataQueryInsertPhysiNode(int32_t nodeType) { pInserter->tableId = QPT_CORRECT_HIGH_PROB() ? qptCtx.param.tbl.uid : taosRand(); pInserter->stableId = QPT_CORRECT_HIGH_PROB() ? qptCtx.param.tbl.suid : taosRand(); pInserter->tableType = QPT_CORRECT_HIGH_PROB() ? (QPT_RAND_BOOL_V ? TSDB_CHILD_TABLE : TSDB_NORMAL_TABLE) : (taosRand() % TSDB_TABLE_MAX); - pInserter->tableName[0] = QPT_RAND_BOOL_V ? 'a' : taosRand(); + if (QPT_CORRECT_HIGH_PROB()) { + strcpy(pInserter->tableName, qptCtx.param.tbl.tblName); + } else { + pInserter->tableName[0] = QPT_RAND_BOOL_V ? 'a' : 0; + } pInserter->vgId = qptCtx.param.vnode.vgId; memcpy(&pInserter->epSet, &qptCtx.param.vnode.epSet, sizeof(pInserter->epSet)); pInserter->explain = QPT_RAND_BOOL_V; @@ -2671,6 +2689,60 @@ SNode* qptCreateDataQueryInsertPhysiNode(int32_t nodeType) { } +SNode* qptCreateDataDeletePhysiNode(int32_t nodeType) { + SDataDeleterNode* pDeleter = (SDataDeleterNode*)qptCreateDataSinkNode(nodeType); + + pDeleter->tableId = QPT_CORRECT_HIGH_PROB() ? qptCtx.param.tbl.uid : taosRand(); + pDeleter->tableType = QPT_CORRECT_HIGH_PROB() ? (QPT_RAND_BOOL_V ? TSDB_CHILD_TABLE : TSDB_NORMAL_TABLE) : (taosRand() % TSDB_TABLE_MAX); + if (QPT_CORRECT_HIGH_PROB()) { + sprintf(pDeleter->tableFName, "1.%s.%s", qptCtx.param.db.dbName, qptCtx.param.tbl.tblName); + } else { + pDeleter->tableFName[0] = QPT_RAND_BOOL_V ? 'a' : 0; + } + + SQPTCol* pCol = nodesListGetNode(qptCtx.param.tbl.pColList, 0); + if (QPT_CORRECT_HIGH_PROB() && pCol) { + strcpy(pDeleter->tsColName, pCol->name); + } else { + pDeleter->tsColName[0] = QPT_RAND_BOOL_V ? 't' : 0; + } + + qptGetRandTimeWindow(&pDeleter->deleteTimeRange); + + return (SNode*)pDeleter; +} + + +SNode* qptCreateSubplanNode(int32_t nodeType) { + SSubplan* pSubplan = NULL; + assert(0 == nodesMakeNode((ENodeType)nodeType, (SNode**)&pSubplan)); + + qptGetSubplanId(&pSubplan->id); + + pSubplan->subplanType = QPT_CORRECT_HIGH_PROB() ? (taosRand() % SUBPLAN_TYPE_COMPUTE + 1) : (ESubplanType)taosRand(); + pSubplan->msgType = qptGetRandSubplanMsgType(); + pSubplan->level = qptCtx.buildCtx.; + pDeleter->tableId = QPT_CORRECT_HIGH_PROB() ? qptCtx.param.tbl.uid : taosRand(); + pDeleter->tableType = QPT_CORRECT_HIGH_PROB() ? (QPT_RAND_BOOL_V ? TSDB_CHILD_TABLE : TSDB_NORMAL_TABLE) : (taosRand() % TSDB_TABLE_MAX); + if (QPT_CORRECT_HIGH_PROB()) { + sprintf(pDeleter->tableFName, "1.%s.%s", qptCtx.param.db.dbName, qptCtx.param.tbl.tblName); + } else { + pDeleter->tableFName[0] = QPT_RAND_BOOL_V ? 'a' : 0; + } + + SQPTCol* pCol = nodesListGetNode(qptCtx.param.tbl.pColList, 0); + if (QPT_CORRECT_HIGH_PROB() && pCol) { + strcpy(pDeleter->tsColName, pCol->name); + } else { + pDeleter->tsColName[0] = QPT_RAND_BOOL_V ? 't' : 0; + } + + qptGetRandTimeWindow(&pDeleter->deleteTimeRange); + + return (SNode*)pDeleter; +} + + SNode* qptCreatePhysicalPlanNode(int32_t nodeIdx) { if (qptPlans[nodeIdx].buildFunc) { return (*qptPlans[nodeIdx].buildFunc)(qptPlans[nodeIdx].type); @@ -2695,6 +2767,9 @@ void qptResetForReRun() { qptCtx.param.vnode.vgId = 1; qptResetTableCols(); + + qptCtx.buildCtx.pCurr = NULL; + qptCtx.buildCtx.pCurrTask = NULL; } void qptSingleTestDone(bool* contLoop) { @@ -2744,11 +2819,9 @@ void qptRunSingleOpTest() { SExecTaskInfo* pTaskInfo = NULL; SStorageAPI storageAPI = {0}; - if (qptCtx.loopIdx > 0) { - qptResetForReRun(); - } + qptResetForReRun(); - doCreateTask(qptCtx.param.plan.queryId, qptCtx.param.plan.taskId++, qptCtx.param.vnode.vgId, OPTR_EXEC_MODEL_BATCH, &storageAPI, &pTaskInfo); + doCreateTask(qptCtx.param.plan.queryId, qptCtx.param.plan.taskId, qptCtx.param.vnode.vgId, OPTR_EXEC_MODEL_BATCH, &storageAPI, &pTaskInfo); qptCtx.buildCtx.pCurrTask = pTaskInfo; pNode = (SNode*)qptCreatePhysicalPlanNode(qptCtx.param.plan.subplanIdx[0]); @@ -2860,6 +2933,9 @@ void qptInitTestCtx(bool correctExpected, bool singleNode, int32_t nodeType, int qptCtx.param.plan.physicNodeParam = nodeParam; } + qptCtx.param.plan.queryId++; + qptCtx.param.plan.taskId++; + qptCtx.param.db.precision = TSDB_TIME_PRECISION_MILLI; strcpy(qptCtx.param.db.dbName, "qptdb1"); @@ -2889,6 +2965,12 @@ void qptInitTestCtx(bool correctExpected, bool singleNode, int32_t nodeType, int FOREACH(pTmp, qptCtx.param.tbl.pTagList) { qptNodesListMakeStrictAppend(&qptCtx.param.tbl.pColTagList, pTmp); } + + qptCtx.buildCtx.nextBlockId++; + qptCtx.buildCtx.nextSubplanId.queryId = qptCtx.param.plan.queryId; + qptCtx.buildCtx.nextSubplanId.groupId++; + qptCtx.buildCtx.nextSubplanId.subplanId++; + qptCtx.buildCtx.currSubplanLevel = 0; } void qptDestroyTestCtx() { From cde86c2e327af6328b7985deda56655eee5e9f50 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 18 Oct 2024 18:18:10 +0800 Subject: [PATCH 13/29] enh: add plan ut case --- source/libs/executor/test/queryPlanTests.cpp | 321 ++++++++++++++----- source/libs/nodes/src/nodesUtilFuncs.c | 6 +- source/libs/scheduler/src/schJob.c | 17 +- 3 files changed, 261 insertions(+), 83 deletions(-) diff --git a/source/libs/executor/test/queryPlanTests.cpp b/source/libs/executor/test/queryPlanTests.cpp index 6faeaed2cf..52edd59b75 100755 --- a/source/libs/executor/test/queryPlanTests.cpp +++ b/source/libs/executor/test/queryPlanTests.cpp @@ -46,8 +46,10 @@ namespace { -#define QPT_MAX_LOOP 10000 -#define QPT_MAX_SUBPLAN_LEVEL 1000 +#define QPT_MAX_LOOP 1000000 +#define QPT_MAX_LEVEL_SUBPLAN_NUM 10 +#define QPT_MAX_SUBPLAN_LEVEL 5 +#define QPT_MAX_SUBPLAN_GROUP 5 #define QPT_MAX_WHEN_THEN_NUM 10 #define QPT_MAX_NODE_LEVEL 5 #define QPT_MAX_STRING_LEN 1048576 @@ -66,9 +68,17 @@ typedef enum { QPT_NODE_EXPR, QPT_NODE_FUNCTION, QPT_NODE_VALUE, + QPT_NODE_SUBPLAN, QPT_NODE_MAX_VALUE } QPT_NODE_TYPE; +enum { + QPT_PLAN_PHYSIC = 1, + QPT_PLAN_SINK, + QPT_PLAN_SUBPLAN, + QPT_PLAN_PLAN +}; + typedef SNode* (*planBuildFunc)(int32_t); @@ -130,6 +140,7 @@ typedef struct { typedef struct { bool correctExpected; uint64_t schedulerId; + char userName[TSDB_USER_LEN]; SQPTPlanParam plan; SQPTDbParam db; SQPTVnodeParam vnode; @@ -146,8 +157,7 @@ typedef struct { EOrder currTsOrder; int16_t nextBlockId; int32_t primaryTsSlotId; - SSubplanId nextSubplanId; - int32_t currSubplanLevel; + int32_t nextSubplanId; SExecTaskInfo* pCurrTask; } SQPTBuildPlanCtx; @@ -185,6 +195,7 @@ typedef struct { typedef struct { int32_t type; + int32_t classify; char* name; planBuildFunc buildFunc; } SQPTPlan; @@ -236,57 +247,61 @@ SNode* qptCreateDataDispatchPhysiNode(int32_t nodeType); SNode* qptCreateDataInsertPhysiNode(int32_t nodeType); SNode* qptCreateDataQueryInsertPhysiNode(int32_t nodeType); SNode* qptCreateDataDeletePhysiNode(int32_t nodeType); +SNode* qptCreatePhysicalPlanNode(int32_t nodeIdx); +void qptCreatePhysiNodesTree(SPhysiNode** ppRes, SPhysiNode* pParent, int32_t level); +SNode* qptCreateQueryPlanNode(int32_t nodeType); +SNode* qptCreateSubplanNode(int32_t nodeType); SQPTPlan qptPlans[] = { - {QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN, "tagScan", qptCreateTagScanPhysiNode}, - {QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, "tableScan", qptCreateTableScanPhysiNode}, - {QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN, "tableSeqScan", qptCreateTableSeqScanPhysiNode}, - {QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN, "tableMergeScan", qptCreateTableMergeScanPhysiNode}, - {QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN, "streamScan", qptCreateStreamScanPhysiNode}, - {QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN, "sysTableScan", qptCreateSysTableScanPhysiNode}, - {QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN, "blockDistScan", qptCreateBlockDistScanPhysiNode}, - {QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN, "lastRowScan", qptCreateLastRowScanPhysiNode}, - {QUERY_NODE_PHYSICAL_PLAN_PROJECT, "project", qptCreateProjectPhysiNode}, - {QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN, "mergeJoin", qptCreateMergeJoinPhysiNode}, - {QUERY_NODE_PHYSICAL_PLAN_HASH_AGG, "hashAgg", qptCreateHashAggPhysiNode}, - {QUERY_NODE_PHYSICAL_PLAN_EXCHANGE, "exchange", qptCreateExchangePhysiNode}, - {QUERY_NODE_PHYSICAL_PLAN_MERGE, "merge", qptCreateMergePhysiNode}, - {QUERY_NODE_PHYSICAL_PLAN_SORT, "sort", qptCreateSortPhysiNode}, - {QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT, "groupSort", qptCreateGroupSortPhysiNode}, - {QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL, "interval", qptCreateIntervalPhysiNode}, - {QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL, "mergeInterval", qptCreateMergeIntervalPhysiNode}, - {QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL, "mergeAlignedInterval", qptCreateMergeAlignedIntervalPhysiNode}, - {QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, "streamInterval", qptCreateStreamIntervalPhysiNode}, - {QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL, "streamFinalInterval", qptCreateStreamFinalIntervalPhysiNode}, - {QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL, "streamSemiInterval", qptCreateStreamSemiIntervalPhysiNode}, - {QUERY_NODE_PHYSICAL_PLAN_FILL, "fill", qptCreateFillPhysiNode}, - {QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL, "streamFill", qptCreateStreamFillPhysiNode}, - {QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION, "sessionWindow", qptCreateSessionPhysiNode}, - {QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION, "streamSession", qptCreateStreamSessionPhysiNode}, - {QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION, "streamSemiSession", qptCreateStreamSemiSessionPhysiNode}, - {QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION, "streamFinalSession", qptCreateStreamFinalSessionPhysiNode}, - {QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE, "stateWindow", qptCreateStateWindowPhysiNode}, - {QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE, "streamState", qptCreateStreamStatePhysiNode}, - {QUERY_NODE_PHYSICAL_PLAN_PARTITION, "partition", qptCreatePartitionPhysiNode}, - {QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION, "streamPartition", qptCreateStreamPartitionPhysiNode}, - {QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC, "indefRowsFunc", qptCreateIndefRowsFuncPhysiNode}, - {QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC, "interpFunc", qptCreateInterpFuncPhysiNode}, - {QUERY_NODE_PHYSICAL_PLAN_DISPATCH, "dataDispatch", qptCreateDataDispatchPhysiNode}, - {QUERY_NODE_PHYSICAL_PLAN_INSERT, "dataInseret", qptCreateDataInsertPhysiNode}, - {QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT, "dataQueryInsert", qptCreateDataQueryInsertPhysiNode}, - {QUERY_NODE_PHYSICAL_PLAN_DELETE, "dataDelete", qptCreateDataDeletePhysiNode}, - {QUERY_NODE_PHYSICAL_SUBPLAN, "subplan", NULL}, - {QUERY_NODE_PHYSICAL_PLAN, "plan", NULL}, - {QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN, "tableCountScan", qptCreateTableCountScanPhysiNode}, - {QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT, "eventWindow", qptCreateMergeEventPhysiNode}, - {QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT, "streamEventWindow", qptCreateStreamEventPhysiNode}, - {QUERY_NODE_PHYSICAL_PLAN_HASH_JOIN, "hashJoin", qptCreateHashJoinPhysiNode}, - {QUERY_NODE_PHYSICAL_PLAN_GROUP_CACHE, "groupCache", qptCreateGroupCachePhysiNode}, - {QUERY_NODE_PHYSICAL_PLAN_DYN_QUERY_CTRL, "dynQueryCtrl", qptCreateDynQueryCtrlPhysiNode}, - {QUERY_NODE_PHYSICAL_PLAN_MERGE_COUNT, "countWindow", qptCreateCountWindowPhysiNode}, - {QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT, "streamCountWindow", qptCreateStreamCountWindowPhysiNode}, - {QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL, "streamMidInterval", qptCreateStreamMidIntervalPhysiNode} + {QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN, QPT_PLAN_PHYSIC, "tagScan", qptCreateTagScanPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, QPT_PLAN_PHYSIC, "tableScan", qptCreateTableScanPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN, QPT_PLAN_PHYSIC, "tableSeqScan", qptCreateTableSeqScanPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN, QPT_PLAN_PHYSIC, "tableMergeScan", qptCreateTableMergeScanPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN, QPT_PLAN_PHYSIC, "streamScan", qptCreateStreamScanPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN, QPT_PLAN_PHYSIC, "sysTableScan", qptCreateSysTableScanPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN, QPT_PLAN_PHYSIC, "blockDistScan", qptCreateBlockDistScanPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN, QPT_PLAN_PHYSIC, "lastRowScan", qptCreateLastRowScanPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_PROJECT, QPT_PLAN_PHYSIC, "project", qptCreateProjectPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN, QPT_PLAN_PHYSIC, "mergeJoin", qptCreateMergeJoinPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_HASH_AGG, QPT_PLAN_PHYSIC, "hashAgg", qptCreateHashAggPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_EXCHANGE, QPT_PLAN_PHYSIC, "exchange", qptCreateExchangePhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_MERGE, QPT_PLAN_PHYSIC, "merge", qptCreateMergePhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_SORT, QPT_PLAN_PHYSIC, "sort", qptCreateSortPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT, QPT_PLAN_PHYSIC, "groupSort", qptCreateGroupSortPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL, QPT_PLAN_PHYSIC, "interval", qptCreateIntervalPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL, QPT_PLAN_PHYSIC, "mergeInterval", qptCreateMergeIntervalPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL, QPT_PLAN_PHYSIC, "mergeAlignedInterval", qptCreateMergeAlignedIntervalPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, QPT_PLAN_PHYSIC, "streamInterval", qptCreateStreamIntervalPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL, QPT_PLAN_PHYSIC, "streamFinalInterval", qptCreateStreamFinalIntervalPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL, QPT_PLAN_PHYSIC, "streamSemiInterval", qptCreateStreamSemiIntervalPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_FILL, QPT_PLAN_PHYSIC, "fill", qptCreateFillPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL, QPT_PLAN_PHYSIC, "streamFill", qptCreateStreamFillPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION, QPT_PLAN_PHYSIC, "sessionWindow", qptCreateSessionPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION, QPT_PLAN_PHYSIC, "streamSession", qptCreateStreamSessionPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION, QPT_PLAN_PHYSIC, "streamSemiSession", qptCreateStreamSemiSessionPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION, QPT_PLAN_PHYSIC, "streamFinalSession", qptCreateStreamFinalSessionPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE, QPT_PLAN_PHYSIC, "stateWindow", qptCreateStateWindowPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE, QPT_PLAN_PHYSIC, "streamState", qptCreateStreamStatePhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_PARTITION, QPT_PLAN_PHYSIC, "partition", qptCreatePartitionPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION, QPT_PLAN_PHYSIC, "streamPartition", qptCreateStreamPartitionPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC, QPT_PLAN_PHYSIC, "indefRowsFunc", qptCreateIndefRowsFuncPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC, QPT_PLAN_PHYSIC, "interpFunc", qptCreateInterpFuncPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_DISPATCH, QPT_PLAN_SINK, "dataDispatch", qptCreateDataDispatchPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_INSERT, QPT_PLAN_SINK, "dataInseret", qptCreateDataInsertPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT, QPT_PLAN_SINK, "dataQueryInsert", qptCreateDataQueryInsertPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_DELETE, QPT_PLAN_SINK, "dataDelete", qptCreateDataDeletePhysiNode}, + {QUERY_NODE_PHYSICAL_SUBPLAN, QPT_PLAN_SUBPLAN, "subplan", qptCreateSubplanNode}, + {QUERY_NODE_PHYSICAL_PLAN, QPT_PLAN_PLAN, "plan", qptCreateQueryPlanNode}, + {QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN, QPT_PLAN_PHYSIC, "tableCountScan", qptCreateTableCountScanPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT, QPT_PLAN_PHYSIC, "eventWindow", qptCreateMergeEventPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT, QPT_PLAN_PHYSIC, "streamEventWindow", qptCreateStreamEventPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_HASH_JOIN, QPT_PLAN_PHYSIC, "hashJoin", qptCreateHashJoinPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_GROUP_CACHE, QPT_PLAN_PHYSIC, "groupCache", qptCreateGroupCachePhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_DYN_QUERY_CTRL, QPT_PLAN_PHYSIC, "dynQueryCtrl", qptCreateDynQueryCtrlPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_MERGE_COUNT, QPT_PLAN_PHYSIC, "countWindow", qptCreateCountWindowPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT, QPT_PLAN_PHYSIC, "streamCountWindow", qptCreateStreamCountWindowPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL, QPT_PLAN_PHYSIC, "streamMidInterval", qptCreateStreamMidIntervalPhysiNode} }; @@ -307,6 +322,7 @@ SQPTCtx qptCtx = {0}; SQPTCtrl qptCtrl = {1, 0, 0, 0, 0}; bool qptErrorRerun = false; bool qptInRerun = false; +int32_t qptSink[] = {QUERY_NODE_PHYSICAL_PLAN_DISPATCH, QUERY_NODE_PHYSICAL_PLAN_INSERT, QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT, QUERY_NODE_PHYSICAL_PLAN_DELETE}; SNode* qptMakeExprNode(SNode** ppNode); @@ -727,10 +743,28 @@ void qptGetRandTimeWindow(STimeWindow* pWindow) { pWindow->ekey = taosRand(); } -void qptGetSubplanId(SSubplanId* pId) { - pId->queryId = qptCtx.buildCtx.nextSubplanId.queryId; - pId->groupId = qptCtx.buildCtx.nextSubplanId.groupId++; - pId->subplanId = qptCtx.buildCtx.nextSubplanId.subplanId++; +int32_t qptGetSubplanNum(SNodeList* pList) { + if (QPT_NCORRECT_LOW_PROB()) { + return taosRand(); + } + + int32_t subplanNum = 0; + SNode* pNode = NULL; + FOREACH(pNode, pList) { + if (NULL == pNode || QUERY_NODE_NODE_LIST != nodeType(pNode)) { + continue; + } + + SNodeListNode* pNodeListNode = (SNodeListNode*)pNode; + + if (NULL == pNodeListNode->pNodeList) { + continue; + } + + subplanNum += pNodeListNode->pNodeList->length; + } + + return subplanNum; } int32_t qptNodesListAppend(SNodeList* pList, SNode* pNode) { @@ -1226,6 +1260,11 @@ SNode* qptMakeNodeListNode(QPT_NODE_TYPE nodeType, SNode** ppNode) { return qptMakeRandNode(ppNode); } + SNode* pTmp = NULL; + if (NULL == ppNode) { + ppNode = &pTmp; + } + SNodeListNode* pList = NULL; nodesMakeNode(QUERY_NODE_NODE_LIST, (SNode**)&pList); @@ -2700,7 +2739,7 @@ SNode* qptCreateDataDeletePhysiNode(int32_t nodeType) { pDeleter->tableFName[0] = QPT_RAND_BOOL_V ? 'a' : 0; } - SQPTCol* pCol = nodesListGetNode(qptCtx.param.tbl.pColList, 0); + SQPTCol* pCol = (SQPTCol*)nodesListGetNode(qptCtx.param.tbl.pColList, 0); if (QPT_CORRECT_HIGH_PROB() && pCol) { strcpy(pDeleter->tsColName, pCol->name); } else { @@ -2712,34 +2751,58 @@ SNode* qptCreateDataDeletePhysiNode(int32_t nodeType) { return (SNode*)pDeleter; } +void qptBuildSinkIdx(int32_t* pSinkIdx) { + +} + +void qptCreateSubplanDataSink(SDataSinkNode** ppOutput) { + static int32_t sinkIdx[sizeof(qptSink) / sizeof(qptSink[0])] = {-1}; + int32_t nodeIdx = 0; + + if (sinkIdx[0] < 0) { + qptBuildSinkIdx(sinkIdx); + } + + nodeIdx = taosRand() % (sizeof(sinkIdx)/sizeof(sinkIdx[0])); + + *ppOutput = (SDataSinkNode*)qptCreatePhysicalPlanNode(nodeIdx); +} + SNode* qptCreateSubplanNode(int32_t nodeType) { SSubplan* pSubplan = NULL; assert(0 == nodesMakeNode((ENodeType)nodeType, (SNode**)&pSubplan)); - qptGetSubplanId(&pSubplan->id); + pSubplan->id.queryId = qptCtx.param.plan.queryId; + pSubplan->id.groupId = taosRand() % QPT_MAX_SUBPLAN_GROUP; + pSubplan->id.subplanId = qptCtx.buildCtx.nextSubplanId++; - pSubplan->subplanType = QPT_CORRECT_HIGH_PROB() ? (taosRand() % SUBPLAN_TYPE_COMPUTE + 1) : (ESubplanType)taosRand(); + pSubplan->subplanType = QPT_CORRECT_HIGH_PROB() ? (ESubplanType)(taosRand() % SUBPLAN_TYPE_COMPUTE + 1) : (ESubplanType)taosRand(); pSubplan->msgType = qptGetRandSubplanMsgType(); - pSubplan->level = qptCtx.buildCtx.; - pDeleter->tableId = QPT_CORRECT_HIGH_PROB() ? qptCtx.param.tbl.uid : taosRand(); - pDeleter->tableType = QPT_CORRECT_HIGH_PROB() ? (QPT_RAND_BOOL_V ? TSDB_CHILD_TABLE : TSDB_NORMAL_TABLE) : (taosRand() % TSDB_TABLE_MAX); - if (QPT_CORRECT_HIGH_PROB()) { - sprintf(pDeleter->tableFName, "1.%s.%s", qptCtx.param.db.dbName, qptCtx.param.tbl.tblName); - } else { - pDeleter->tableFName[0] = QPT_RAND_BOOL_V ? 'a' : 0; - } + pSubplan->level = taosRand() % QPT_MAX_SUBPLAN_LEVEL; + sprintf(pSubplan->dbFName, "1.%s", qptCtx.param.db.dbName); + strcpy(pSubplan->user, qptCtx.param.userName); + pSubplan->execNode.nodeId = qptCtx.param.vnode.vgId; + memcpy(&pSubplan->execNode.epSet, &qptCtx.param.vnode.epSet, sizeof(pSubplan->execNode.epSet)); + pSubplan->execNodeStat.tableNum = taosRand(); - SQPTCol* pCol = nodesListGetNode(qptCtx.param.tbl.pColList, 0); - if (QPT_CORRECT_HIGH_PROB() && pCol) { - strcpy(pDeleter->tsColName, pCol->name); - } else { - pDeleter->tsColName[0] = QPT_RAND_BOOL_V ? 't' : 0; - } + qptCreatePhysiNodesTree(&pSubplan->pNode, NULL, 0); - qptGetRandTimeWindow(&pDeleter->deleteTimeRange); + qptCreateSubplanDataSink(&pSubplan->pDataSink); + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_CORRECT_HIGH_PROB() ? true : false, QPT_RAND_BOOL_V, 0, NULL); + qptMakeExprNode(&pSubplan->pTagCond); + + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_CORRECT_HIGH_PROB() ? true : false, QPT_RAND_BOOL_V, 0, NULL); + qptMakeExprNode(&pSubplan->pTagIndexCond); + + pSubplan->showRewrite = QPT_RAND_BOOL_V; + pSubplan->isView = QPT_RAND_BOOL_V; + pSubplan->isAudit = QPT_RAND_BOOL_V; + pSubplan->dynamicRowThreshold = QPT_RAND_BOOL_V; + pSubplan->rowsThreshold = taosRand(); - return (SNode*)pDeleter; + return (SNode*)pSubplan; } @@ -2751,8 +2814,108 @@ SNode* qptCreatePhysicalPlanNode(int32_t nodeIdx) { return NULL; } -void qptCreateQueryPlan(SNode** ppPlan) { +SNode* qptCreateRealPhysicalPlanNode() { + int32_t nodeIdx = 0; + do { + nodeIdx = taosRand() % (sizeof(qptPlans) / sizeof(qptPlans[0])); + if (QPT_PLAN_PHYSIC != qptPlans[nodeIdx].classify) { + continue; + } + + return qptCreatePhysicalPlanNode(nodeIdx); + } while (true); +} + +void qptCreatePhysiNodesTree(SPhysiNode** ppRes, SPhysiNode* pParent, int32_t level) { + SPhysiNode* pNew = NULL; + if (level < QPT_MAX_SUBPLAN_LEVEL && (NULL == pParent || QPT_RAND_BOOL_V)) { + pNew = (SPhysiNode*)qptCreateRealPhysicalPlanNode(); + pNew->pParent = pParent; + int32_t childrenNum = taosRand() % QPT_MAX_LEVEL_SUBPLAN_NUM; + for (int32_t i = 0; i < childrenNum; ++i) { + qptCreatePhysiNodesTree(NULL, pNew, level + 1); + } + } else if (QPT_RAND_BOOL_V) { + return; + } + + if (pParent) { + qptNodesListMakeStrictAppend(&pParent->pChildren, (SNode*)pNew); + } else { + *ppRes = pNew; + } +} + +void qptAppendParentsSubplan(SNodeList* pParents, SNodeList* pNew) { + SNode* pNode = NULL; + FOREACH(pNode, pNew) { + if (NULL == pNode || QUERY_NODE_PHYSICAL_SUBPLAN != nodeType(pNode)) { + continue; + } + + qptNodesListMakeStrictAppend(&pParents, pNode); + } +} + +void qptSetSubplansRelation(SNodeList* pParents, SNodeList* pNew) { + int32_t parentIdx = 0; + SNode* pNode = NULL; + SSubplan* pChild = NULL; + SSubplan* pParent = NULL; + FOREACH(pNode, pNew) { + if (QPT_CORRECT_HIGH_PROB()) { + pChild = (SSubplan*)pNode; + parentIdx = taosRand() % pParents->length; + pParent = (SSubplan*)nodesListGetNode(pParents, parentIdx); + qptNodesListMakeStrictAppend(&pParent->pChildren, pNode); + qptNodesListMakeStrictAppend(&pChild->pParents, (SNode*)pParent); + } + } +} + +void qptBuildSubplansRelation(SNodeList* pList) { + SNode* pNode = NULL; + SNodeList* pParents = NULL; + FOREACH(pNode, pList) { + if (NULL == pNode || QUERY_NODE_NODE_LIST != nodeType(pNode)) { + continue; + } + + SNodeListNode* pNodeList = (SNodeListNode*)pNode; + + if (NULL == pParents) { + qptAppendParentsSubplan(pParents, pNodeList->pNodeList); + continue; + } + + qptSetSubplansRelation(pParents, pNodeList->pNodeList); + qptAppendParentsSubplan(pParents, pNodeList->pNodeList); + } +} + +SNode* qptCreateQueryPlanNode(int32_t nodeType) { + SQueryPlan* pPlan = NULL; + assert(0 == nodesMakeNode((ENodeType)nodeType, (SNode**)&pPlan)); + + int32_t subplanNum = 0, subplanLevelNum = taosRand() % QPT_MAX_SUBPLAN_LEVEL; + pPlan->queryId = QPT_CORRECT_HIGH_PROB() ? qptCtx.param.plan.queryId : taosRand(); + + for (int32_t l = 0; l < subplanLevelNum; ++l) { + qptInitMakeNodeCtx(QPT_CORRECT_HIGH_PROB() ? false : true, QPT_RAND_BOOL_V, QPT_RAND_BOOL_V, 0, NULL); + qptNodesListMakeStrictAppend(&pPlan->pSubplans, qptMakeNodeListNode(QPT_NODE_SUBPLAN, NULL)); + } + + pPlan->numOfSubplans = qptGetSubplanNum(pPlan->pSubplans); + qptBuildSubplansRelation(pPlan->pSubplans); + + pPlan->explainInfo.mode = (EExplainMode)(taosRand() % EXPLAIN_MODE_ANALYZE + 1); + pPlan->explainInfo.verbose = QPT_RAND_BOOL_V; + pPlan->explainInfo.ratio = taosRand(); + + pPlan->pPostPlan = QPT_RAND_BOOL_V ? NULL : (void*)0x1; + + return (SNode*)pPlan; } @@ -2913,6 +3076,7 @@ void qptInitTableCols(SNodeList** ppList, int32_t colNum, EColumnType colType) { void qptInitTestCtx(bool correctExpected, bool singleNode, int32_t nodeType, int32_t nodeIdx, int32_t paramNum, SQPTNodeParam* nodeParam) { qptCtx.param.correctExpected = correctExpected; qptCtx.param.schedulerId = taosRand(); + strcpy(qptCtx.param.userName, "user1"); qptCtx.param.plan.singlePhysiNode = singleNode; if (singleNode) { @@ -2967,10 +3131,7 @@ void qptInitTestCtx(bool correctExpected, bool singleNode, int32_t nodeType, int } qptCtx.buildCtx.nextBlockId++; - qptCtx.buildCtx.nextSubplanId.queryId = qptCtx.param.plan.queryId; - qptCtx.buildCtx.nextSubplanId.groupId++; - qptCtx.buildCtx.nextSubplanId.subplanId++; - qptCtx.buildCtx.currSubplanLevel = 0; + qptCtx.buildCtx.nextSubplanId++; } void qptDestroyTestCtx() { diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 77b39b8f0d..acb5c41d38 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -1575,10 +1575,14 @@ void nodesDestroyNode(SNode* pNode) { case QUERY_NODE_PHYSICAL_PLAN_HASH_JOIN: { SHashJoinPhysiNode* pPhyNode = (SHashJoinPhysiNode*)pNode; destroyPhysiNode((SPhysiNode*)pPhyNode); + nodesDestroyNode(pPhyNode->pWindowOffset); + nodesDestroyNode(pPhyNode->pJLimit); nodesDestroyList(pPhyNode->pOnLeft); nodesDestroyList(pPhyNode->pOnRight); nodesDestroyNode(pPhyNode->leftPrimExpr); nodesDestroyNode(pPhyNode->rightPrimExpr); + nodesDestroyNode(pPhyNode->pLeftOnCond); + nodesDestroyNode(pPhyNode->pRightOnCond); nodesDestroyNode(pPhyNode->pFullOnCond); nodesDestroyList(pPhyNode->pTargets); @@ -1586,8 +1590,6 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyNode(pPhyNode->pColEqCond); nodesDestroyNode(pPhyNode->pTagEqCond); - nodesDestroyNode(pPhyNode->pLeftOnCond); - nodesDestroyNode(pPhyNode->pRightOnCond); break; } case QUERY_NODE_PHYSICAL_PLAN_HASH_AGG: { diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index 03145da939..b888b826bb 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -363,6 +363,11 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); } + if (QUERY_NODE_NODE_LIST != nodeType(plans)) { + SCH_JOB_ELOG("invalid level plan, level:%d, planNodeType:%d", i, nodeType(plans)); + SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); + } + taskNum = (int32_t)LIST_LENGTH(plans->pNodeList); if (taskNum <= 0) { SCH_JOB_ELOG("invalid level plan number:%d, level:%d", taskNum, i); @@ -380,10 +385,20 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { for (int32_t n = 0; n < taskNum; ++n) { SSubplan *plan = (SSubplan *)nodesListGetNode(plans->pNodeList, n); if (NULL == plan) { - SCH_JOB_ELOG("fail to get the %dth subplan, taskNum: %d", n, taskNum); + SCH_JOB_ELOG("fail to get the %dth subplan, taskNum: %d, level: %d", n, taskNum, i); SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } + if (QUERY_NODE_PHYSICAL_SUBPLAN != nodeType(plan)) { + SCH_JOB_ELOG("invalid subplan type, level:%d, subplanNodeType:%d", i, nodeType(plan)); + SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); + } + + if (plan->subplanType < SUBPLAN_TYPE_MERGE || plan->subplanType > SUBPLAN_TYPE_COMPUTE) { + SCH_JOB_ELOG("invalid subplanType %d, level:%d, subplan idx:%d", plan->subplanType, i, n); + SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); + } + SCH_SET_JOB_TYPE(pJob, plan->subplanType); SSchTask task = {0}; From 6a9fd0c95e2a61f4d8775e40c22088569d711168 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 21 Oct 2024 19:24:52 +0800 Subject: [PATCH 14/29] enh: add operator --- include/common/tmsg.h | 4 +- source/libs/executor/test/queryPlanTests.cpp | 182 ++++++++++++++++--- source/libs/scheduler/inc/schInt.h | 5 +- source/libs/scheduler/src/schJob.c | 26 ++- source/libs/scheduler/src/schTask.c | 2 +- source/libs/scheduler/src/schUtil.c | 47 +++++ 6 files changed, 222 insertions(+), 44 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 1a10f02c96..11c9be4d06 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -412,7 +412,7 @@ typedef enum ENodeType { // physical plan node QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN = 1100, QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, - QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN, + QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN, // INACTIVE QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN, QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN, @@ -426,7 +426,7 @@ typedef enum ENodeType { QUERY_NODE_PHYSICAL_PLAN_SORT, QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT, QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL, - QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL, + QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL, // INACTIVE QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL, QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL, diff --git a/source/libs/executor/test/queryPlanTests.cpp b/source/libs/executor/test/queryPlanTests.cpp index 52edd59b75..f03d347a78 100755 --- a/source/libs/executor/test/queryPlanTests.cpp +++ b/source/libs/executor/test/queryPlanTests.cpp @@ -46,7 +46,7 @@ namespace { -#define QPT_MAX_LOOP 1000000 +#define QPT_MAX_LOOP 1000 #define QPT_MAX_LEVEL_SUBPLAN_NUM 10 #define QPT_MAX_SUBPLAN_LEVEL 5 #define QPT_MAX_SUBPLAN_GROUP 5 @@ -256,7 +256,7 @@ SNode* qptCreateSubplanNode(int32_t nodeType); SQPTPlan qptPlans[] = { {QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN, QPT_PLAN_PHYSIC, "tagScan", qptCreateTagScanPhysiNode}, {QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, QPT_PLAN_PHYSIC, "tableScan", qptCreateTableScanPhysiNode}, - {QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN, QPT_PLAN_PHYSIC, "tableSeqScan", qptCreateTableSeqScanPhysiNode}, + {QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN, QPT_PLAN_PHYSIC, "tableSeqScan", NULL /*qptCreateTableSeqScanPhysiNode*/ }, {QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN, QPT_PLAN_PHYSIC, "tableMergeScan", qptCreateTableMergeScanPhysiNode}, {QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN, QPT_PLAN_PHYSIC, "streamScan", qptCreateStreamScanPhysiNode}, {QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN, QPT_PLAN_PHYSIC, "sysTableScan", qptCreateSysTableScanPhysiNode}, @@ -2975,6 +2975,159 @@ void qptHandleTestEnd() { } +void qptExecPlan(SReadHandle* pReadHandle, SNode* pNode, SExecTaskInfo* pTaskInfo, SOperatorInfo** ppOperaotr) { + switch (nodeType(pNode)) { + case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: + qptCtx.result.code = createTagScanOperatorInfo(pReadHandle, (STagScanPhysiNode*)pNode, NULL, NULL, NULL, pTaskInfo, ppOperaotr); + break; + case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: + qptCtx.result.code = createTableScanOperatorInfo((STableScanPhysiNode*)pNode, pReadHandle, NULL, pTaskInfo, ppOperaotr); + break; + case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN: + //qptCtx.result.code = createTableSeqScanOperatorInfo((STableScanPhysiNode*)pNode, pReadHandle, NULL, pTaskInfo, ppOperaotr); + break; + case QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN: + qptCtx.result.code = createTableMergeScanOperatorInfo((STableScanPhysiNode*)pNode, pReadHandle, NULL, pTaskInfo, ppOperaotr); + break; + case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN: + qptCtx.result.code = createStreamScanOperatorInfo(pReadHandle, (STableScanPhysiNode*)pNode, NULL, NULL, pTaskInfo, ppOperaotr); + break; + case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN: + qptCtx.result.code = createSysTableScanOperatorInfo(pReadHandle, (SSystemTableScanPhysiNode*)pNode, NULL, pTaskInfo, ppOperaotr); + break; + case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN: + qptCtx.result.code = createDataBlockInfoScanOperator(pReadHandle, (SBlockDistScanPhysiNode*)pNode, NULL, pTaskInfo, ppOperaotr); + break; + case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN: + qptCtx.result.code = createCacherowsScanOperator((SLastRowScanPhysiNode*)pNode, pReadHandle, NULL, pTaskInfo, ppOperaotr); + break; + case QUERY_NODE_PHYSICAL_PLAN_PROJECT: + qptCtx.result.code = createProjectOperatorInfo(NULL, (SProjectPhysiNode*)pNode, pTaskInfo, ppOperaotr); + break; + case QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN: + qptCtx.result.code = createMergeJoinOperatorInfo(NULL, 0, (SSortMergeJoinPhysiNode*)pNode, pTaskInfo, ppOperaotr); + break; + case QUERY_NODE_PHYSICAL_PLAN_HASH_AGG: { + SAggPhysiNode* pAggNode = (SAggPhysiNode*)pNode; + if (pAggNode->pGroupKeys != NULL) { + qptCtx.result.code = createGroupOperatorInfo(NULL, pAggNode, pTaskInfo, ppOperaotr); + } else { + qptCtx.result.code = createAggregateOperatorInfo(NULL, pAggNode, pTaskInfo, ppOperaotr); + } + break; + } + case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE: + qptCtx.result.code = createExchangeOperatorInfo(NULL, (SExchangePhysiNode*)pNode, pTaskInfo, ppOperaotr); + break; + case QUERY_NODE_PHYSICAL_PLAN_MERGE: + qptCtx.result.code = createMultiwayMergeOperatorInfo(NULL, 0, (SMergePhysiNode*)pNode, pTaskInfo, ppOperaotr); + break; + case QUERY_NODE_PHYSICAL_PLAN_SORT: + qptCtx.result.code = createSortOperatorInfo(NULL, (SSortPhysiNode*)pNode, pTaskInfo, ppOperaotr); + break; + case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT: + qptCtx.result.code = createGroupSortOperatorInfo(NULL, (SGroupSortPhysiNode*)pNode, pTaskInfo, ppOperaotr); + break; + case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: + qptCtx.result.code = createIntervalOperatorInfo(NULL, (SIntervalPhysiNode*)pNode, pTaskInfo, ppOperaotr); + break; + case QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL: + //qptCtx.result.code = createMergeIntervalOperatorInfo(NULL, (SMergeIntervalPhysiNode*)pNode, pTaskInfo, ppOperaotr); + break; + case QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL: + qptCtx.result.code = createMergeAlignedIntervalOperatorInfo(NULL, (SMergeAlignedIntervalPhysiNode*)pNode, pTaskInfo, ppOperaotr); + break; + case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: + qptCtx.result.code = createStreamIntervalOperatorInfo(NULL, (SPhysiNode*)pNode, pTaskInfo, pReadHandle, ppOperaotr); + break; + case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_MID_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL: + qptCtx.result.code = createStreamFinalIntervalOperatorInfo(NULL, (SPhysiNode*)pNode, pTaskInfo, 0, pReadHandle, ppOperaotr); + break; + case QUERY_NODE_PHYSICAL_PLAN_FILL: + qptCtx.result.code = createFillOperatorInfo(NULL, (SFillPhysiNode*)pNode, pTaskInfo, ppOperaotr); + break; + case QUERY_NODE_PHYSICAL_PLAN_STREAM_FILL: + qptCtx.result.code = createStreamFillOperatorInfo(NULL, (SStreamFillPhysiNode*)pNode, pTaskInfo, ppOperaotr); + break; + case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION: + qptCtx.result.code = createSessionAggOperatorInfo(NULL, (SSessionWinodwPhysiNode*)pNode, pTaskInfo, ppOperaotr); + break; + case QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION: + qptCtx.result.code = createStreamSessionAggOperatorInfo(NULL, (SPhysiNode*)pNode, pTaskInfo, pReadHandle, ppOperaotr); + break; + case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION: + qptCtx.result.code = createStreamFinalSessionAggOperatorInfo(NULL, (SPhysiNode*)pNode, pTaskInfo, 0, pReadHandle, ppOperaotr); + break; + case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION: + qptCtx.result.code = createStreamFinalSessionAggOperatorInfo(NULL, (SPhysiNode*)pNode, pTaskInfo, 0, pReadHandle, ppOperaotr); + break; + case QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE: + qptCtx.result.code = createStatewindowOperatorInfo(NULL, (SStateWinodwPhysiNode*)pNode, pTaskInfo, ppOperaotr); + break; + case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE: + qptCtx.result.code = createStreamStateAggOperatorInfo(NULL, (SPhysiNode*)pNode, pTaskInfo, pReadHandle, ppOperaotr); + break; + case QUERY_NODE_PHYSICAL_PLAN_PARTITION: + qptCtx.result.code = createPartitionOperatorInfo(NULL, (SPartitionPhysiNode*)pNode, pTaskInfo, ppOperaotr); + break; + case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION: + qptCtx.result.code = createStreamPartitionOperatorInfo(NULL, (SStreamPartitionPhysiNode*)pNode, pTaskInfo, ppOperaotr); + break; + case QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC: + qptCtx.result.code = createIndefinitOutputOperatorInfo(NULL, (SPhysiNode*)pNode, pTaskInfo, ppOperaotr); + break; + case QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC: + qptCtx.result.code = createTimeSliceOperatorInfo(NULL, (SPhysiNode*)pNode, pTaskInfo, ppOperaotr); + break; + case QUERY_NODE_PHYSICAL_PLAN_INSERT: + break; + case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: + case QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT: + case QUERY_NODE_PHYSICAL_PLAN_DELETE: { + DataSinkHandle handle = NULL; + qptCtx.result.code = dsCreateDataSinker(NULL, (SDataSinkNode*)pNode, &handle, NULL, NULL); + dsDestroyDataSinker(handle); + break; + } + case QUERY_NODE_PHYSICAL_SUBPLAN: + break; + case QUERY_NODE_PHYSICAL_PLAN: { + SSchJob job = {0}; + qptCtx.result.code = schValidateAndBuildJob((SQueryPlan*)pNode, &job); + schFreeJobImpl(&job); + break; + } + case QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN: + qptCtx.result.code = createTableCountScanOperatorInfo(pReadHandle, (STableCountScanPhysiNode*)pNode, pTaskInfo, ppOperaotr); + break; + case QUERY_NODE_PHYSICAL_PLAN_MERGE_EVENT: + qptCtx.result.code = createEventwindowOperatorInfo(NULL, (SPhysiNode*)pNode, pTaskInfo, ppOperaotr); + break; + case QUERY_NODE_PHYSICAL_PLAN_STREAM_EVENT: + qptCtx.result.code = createStreamEventAggOperatorInfo(NULL, (SPhysiNode*)pNode, pTaskInfo, pReadHandle, ppOperaotr); + break; + case QUERY_NODE_PHYSICAL_PLAN_HASH_JOIN: + qptCtx.result.code = createHashJoinOperatorInfo(NULL, 0, (SHashJoinPhysiNode*)pNode, pTaskInfo, ppOperaotr); + break; + case QUERY_NODE_PHYSICAL_PLAN_GROUP_CACHE: + qptCtx.result.code = createGroupCacheOperatorInfo(NULL, 0, (SGroupCachePhysiNode*)pNode, pTaskInfo, ppOperaotr); + break; + case QUERY_NODE_PHYSICAL_PLAN_DYN_QUERY_CTRL: + qptCtx.result.code = createDynQueryCtrlOperatorInfo(NULL, 0, (SDynQueryCtrlPhysiNode*)pNode, pTaskInfo, ppOperaotr); + break; + case QUERY_NODE_PHYSICAL_PLAN_MERGE_COUNT: + qptCtx.result.code = createCountwindowOperatorInfo(NULL, (SPhysiNode*)pNode, pTaskInfo, ppOperaotr); + break; + case QUERY_NODE_PHYSICAL_PLAN_STREAM_COUNT: + qptCtx.result.code = createStreamCountAggOperatorInfo(NULL, (SPhysiNode*)pNode, pTaskInfo, pReadHandle, ppOperaotr); + break; + default: + assert(0); + } +} + void qptRunSingleOpTest() { SNode* pNode = NULL; SReadHandle readHandle = {0}; @@ -2992,8 +3145,8 @@ void qptRunSingleOpTest() { qptPrintBeginInfo(); qptCtx.startTsUs = taosGetTimestampUs(); - //qptCtx.result.code = createTagScanOperatorInfo(&readHandle, (STagScanPhysiNode*)pNode, NULL, NULL, NULL, pTaskInfo, &pOperator); - //qptCtx.result.code = createProjectOperatorInfo(NULL, (SProjectPhysiNode*)pNode, pTaskInfo, &pOperator); + + qptExecPlan(&readHandle, pNode, pTaskInfo, &pOperator); doDestroyTask(pTaskInfo); destroyOperator(pOperator); @@ -3173,27 +3326,6 @@ TEST(singleNodeTest, randPlan) { } #endif - - - - - -#if 0 -TEST(correctSingleNodeTest, tagScan) { - char* caseName = "correctSingleNodeTest:tagScan"; - - qptInitTestCtx(true, true, QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN, 0, NULL); - - for (qptCtx.loopIdx = 0; qptCtx.loopIdx < QPT_MAX_LOOP; ++qptCtx.loopIdx) { - qptRunPlanTest(caseName); - } - - qptPrintStatInfo(caseName); -} -#endif - - - #endif diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index 8a156e8a06..52c7c64887 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -330,8 +330,8 @@ extern SSchedulerMgmt schMgmt; #define SCH_TASK_EID(_task) ((_task) ? (_task)->execId : -1) #define SCH_IS_DATA_BIND_QRY_TASK(task) ((task)->plan->subplanType == SUBPLAN_TYPE_SCAN) -#define SCH_IS_DATA_BIND_TASK(task) \ - (((task)->plan->subplanType == SUBPLAN_TYPE_SCAN) || ((task)->plan->subplanType == SUBPLAN_TYPE_MODIFY)) +#define SCH_IS_DATA_BIND_PLAN(_plan) (((_plan)->subplanType == SUBPLAN_TYPE_SCAN) || ((_plan)->subplanType == SUBPLAN_TYPE_MODIFY)) +#define SCH_IS_DATA_BIND_TASK(task) SCH_IS_DATA_BIND_PLAN((task)->plan) #define SCH_IS_LEAF_TASK(_job, _task) (((_task)->level->level + 1) == (_job)->levelNum) #define SCH_IS_DATA_MERGE_TASK(task) (!SCH_IS_DATA_BIND_TASK(task)) #define SCH_IS_LOCAL_EXEC_TASK(_job, _task) \ @@ -641,6 +641,7 @@ void schDropTaskInHashList(SSchJob *pJob, SHashObj *list); int32_t schNotifyTaskInHashList(SSchJob *pJob, SHashObj *list, ETaskNotifyType type, SSchTask *pTask); int32_t schLaunchLevelTasks(SSchJob *pJob, SSchLevel *level); void schGetTaskFromList(SHashObj *pTaskList, uint64_t taskId, SSchTask **pTask); +int32_t schValidateSubplan(SSchJob *pJob, SSubplan* pSubplan, int32_t level, int32_t idx, int32_t taskNum); int32_t schInitTask(SSchJob *pJob, SSchTask *pTask, SSubplan *pPlan, SSchLevel *pLevel); int32_t schSwitchTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask); void schDirectPostJobRes(SSchedulerReq *pReq, int32_t errCode); diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index b888b826bb..1b6e086472 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -200,7 +200,12 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { SSubplan *child = (SSubplan *)nodesListGetNode(pPlan->pChildren, n); if (NULL == child) { SCH_JOB_ELOG("fail to get the %dth child subplan, childNum: %d", n, childNum); - SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + if (QUERY_NODE_PHYSICAL_SUBPLAN != nodeType(child)) { + SCH_JOB_ELOG("invalid subplan type for the %dth child, level:%d, subplanNodeType:%d", n, i, nodeType(child)); + SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } SSchTask **childTask = taosHashGet(planToTask, &child, POINTER_BYTES); @@ -242,6 +247,11 @@ int32_t schBuildTaskRalation(SSchJob *pJob, SHashObj *planToTask) { SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } + if (QUERY_NODE_PHYSICAL_SUBPLAN != nodeType(parent)) { + SCH_JOB_ELOG("invalid subplan type for the %dth parent, level:%d, subplanNodeType:%d", n, i, nodeType(parent)); + SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + SSchTask **parentTask = taosHashGet(planToTask, &parent, POINTER_BYTES); if (NULL == parentTask || NULL == *parentTask) { SCH_TASK_ELOG("subplan parent relationship error, level:%d, taskIdx:%d, childIdx:%d", i, m, n); @@ -384,20 +394,8 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { for (int32_t n = 0; n < taskNum; ++n) { SSubplan *plan = (SSubplan *)nodesListGetNode(plans->pNodeList, n); - if (NULL == plan) { - SCH_JOB_ELOG("fail to get the %dth subplan, taskNum: %d, level: %d", n, taskNum, i); - SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); - } - if (QUERY_NODE_PHYSICAL_SUBPLAN != nodeType(plan)) { - SCH_JOB_ELOG("invalid subplan type, level:%d, subplanNodeType:%d", i, nodeType(plan)); - SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); - } - - if (plan->subplanType < SUBPLAN_TYPE_MERGE || plan->subplanType > SUBPLAN_TYPE_COMPUTE) { - SCH_JOB_ELOG("invalid subplanType %d, level:%d, subplan idx:%d", plan->subplanType, i, n); - SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); - } + SCH_ERR_RET(schValidateSubplan(pJob, plan, pLevel->level, n, taskNum)); SCH_SET_JOB_TYPE(pJob, plan->subplanType); diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index e6b68051f9..fe24633c12 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -831,7 +831,7 @@ int32_t schSetTaskCandidateAddrs(SSchJob *pJob, SSchTask *pTask) { if (SCH_IS_DATA_BIND_TASK(pTask)) { SCH_TASK_ELOG("no execNode specifed for data src task, numOfEps:%d", pTask->plan->execNode.epSet.numOfEps); - SCH_ERR_RET(TSDB_CODE_MND_INVALID_SCHEMA_VER); + SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); } SCH_ERR_RET(schSetAddrsFromNodeList(pJob, pTask)); diff --git a/source/libs/scheduler/src/schUtil.c b/source/libs/scheduler/src/schUtil.c index b68f665200..4840108638 100644 --- a/source/libs/scheduler/src/schUtil.c +++ b/source/libs/scheduler/src/schUtil.c @@ -360,3 +360,50 @@ void schGetTaskFromList(SHashObj *pTaskList, uint64_t taskId, SSchTask **pTask) *pTask = *task; } + +int32_t schValidateSubplan(SSchJob *pJob, SSubplan* pSubplan, int32_t level, int32_t idx, int32_t taskNum) { + if (NULL == pSubplan) { + SCH_JOB_ELOG("fail to get the %dth subplan, taskNum: %d, level: %d", idx, taskNum, level); + SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + if (QUERY_NODE_PHYSICAL_SUBPLAN != nodeType(pSubplan)) { + SCH_JOB_ELOG("invalid subplan type, level:%d, subplanNodeType:%d", level, nodeType(pSubplan)); + SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + if (pSubplan->subplanType < SUBPLAN_TYPE_MERGE || pSubplan->subplanType > SUBPLAN_TYPE_COMPUTE) { + SCH_JOB_ELOG("invalid subplanType %d, level:%d, subplan idx:%d", pSubplan->subplanType, level, idx); + SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + if (pSubplan->level != level) { + SCH_JOB_ELOG("plan level %d mis-match with current level %d", pSubplan->level, level); + SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + if (SCH_IS_DATA_BIND_PLAN(pSubplan)) { + if (pSubplan->execNode.epSet.numOfEps <= 0) { + SCH_JOB_ELOG("no execNode specifed for data src plan %d, numOfEps:%d", pSubplan->subplanType, pSubplan->execNode.epSet.numOfEps); + SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + if (pSubplan->execNode.epSet.inUse >= pSubplan->execNode.epSet.numOfEps) { + SCH_JOB_ELOG("invalid epset inUse %d for data src plan %d, numOfEps:%d", pSubplan->execNode.epSet.inUse, pSubplan->subplanType, pSubplan->execNode.epSet.numOfEps); + SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + } + + if (NULL == pSubplan->pNode) { + SCH_JOB_ELOG("empty plan root node, level:%d, subplan idx:%d", level, idx); + SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + if (NULL == pSubplan->pDataSink) { + SCH_JOB_ELOG("empty plan dataSink, level:%d, subplan idx:%d", level, idx); + SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + } + + return TSDB_CODE_SUCCESS; +} + + From a2665ebc82fb6a749989151663aced5eebc2cbc5 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 22 Oct 2024 09:04:12 +0800 Subject: [PATCH 15/29] fix: add plan validation in scheduler --- include/libs/scheduler/scheduler.h | 4 ++-- source/libs/executor/test/queryPlanTests.cpp | 8 +++---- source/libs/scheduler/src/schJob.c | 10 +++++---- source/libs/scheduler/src/scheduler.c | 23 ++++++++++++++++++++ 4 files changed, 35 insertions(+), 10 deletions(-) diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index b98170f168..af8deff1a0 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -76,8 +76,6 @@ int32_t schedulerExecJob(SSchedulerReq* pReq, int64_t* pJob); int32_t schedulerFetchRows(int64_t jobId, SSchedulerReq* pReq); -void schedulerFetchRowsA(int64_t job, schedulerFetchFp fp, void* param); - int32_t schedulerGetTasksStatus(int64_t job, SArray* pSub); void schedulerStopQueryHb(void* pTrans); @@ -100,6 +98,8 @@ void schedulerFreeJob(int64_t* job, int32_t errCode); void schedulerDestroy(void); +int32_t schedulerValidatePlan(SQueryPlan* pPlan); + #ifdef __cplusplus } #endif diff --git a/source/libs/executor/test/queryPlanTests.cpp b/source/libs/executor/test/queryPlanTests.cpp index f03d347a78..176af60944 100755 --- a/source/libs/executor/test/queryPlanTests.cpp +++ b/source/libs/executor/test/queryPlanTests.cpp @@ -43,6 +43,7 @@ #include "querytask.h" #include "functionMgt.h" #include "ttime.h" +#include "scheduler.h" namespace { @@ -3091,12 +3092,11 @@ void qptExecPlan(SReadHandle* pReadHandle, SNode* pNode, SExecTaskInfo* pTaskInf dsDestroyDataSinker(handle); break; } - case QUERY_NODE_PHYSICAL_SUBPLAN: + case QUERY_NODE_PHYSICAL_SUBPLAN: { break; + } case QUERY_NODE_PHYSICAL_PLAN: { - SSchJob job = {0}; - qptCtx.result.code = schValidateAndBuildJob((SQueryPlan*)pNode, &job); - schFreeJobImpl(&job); + qptCtx.result.code = schedulerValidatePlan((SQueryPlan*)pNode); break; } case QUERY_NODE_PHYSICAL_PLAN_TABLE_COUNT_SCAN: diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index 1b6e086472..d1c970b9fe 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -794,9 +794,11 @@ void schFreeJobImpl(void *job) { } taosMemoryFree(pJob); - int32_t jobNum = atomic_sub_fetch_32(&schMgmt.jobNum, 1); - if (jobNum == 0) { - schCloseJobRef(); + if (refId > 0) { + int32_t jobNum = atomic_sub_fetch_32(&schMgmt.jobNum, 1); + if (jobNum == 0) { + schCloseJobRef(); + } } qDebug("QID:0x%" PRIx64 " sch job freed, refId:0x%" PRIx64 ", pointer:%p", queryId, refId, pJob); @@ -917,7 +919,7 @@ _return: if (NULL == pJob) { qDestroyQueryPlan(pReq->pDag); - } else if (pJob->refId < 0) { + } else if (pJob->refId <= 0) { schFreeJobImpl(pJob); } else { code = taosRemoveRef(schMgmt.jobRef, pJob->refId); diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 091de5c048..01d3204dc6 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -224,3 +224,26 @@ void schedulerDestroy(void) { qWorkerDestroy(&schMgmt.queryMgmt); schMgmt.queryMgmt = NULL; } + +int32_t schedulerValidatePlan(SQueryPlan* pPlan) { + int32_t code = TSDB_CODE_SUCCESS; + SSchJob *pJob = taosMemoryCalloc(1, sizeof(SSchJob)); + if (NULL == pJob) { + qError("QID:0x%" PRIx64 " calloc %d failed", pPlan->queryId, (int32_t)sizeof(SSchJob)); + SCH_ERR_RET(terrno); + } + + SCH_ERR_JRET(schValidateAndBuildJob(pPlan, pJob)); + + if (SCH_IS_EXPLAIN_JOB(pJob)) { + SCH_ERR_JRET(qExecExplainBegin(pPlan, &pJob->explainCtx, 0)); + } + +_return: + + schFreeJobImpl(pJob); + + return code; +} + + From e52628ab57371c62db0d003ff526a3cac2d7adef Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 23 Oct 2024 19:13:15 +0800 Subject: [PATCH 16/29] fix: scheduler validate plan issues --- source/client/inc/clientInt.h | 2 + source/client/src/clientImpl.c | 13 ++- source/libs/executor/test/queryPlanTests.cpp | 87 +++++++++++++++++--- source/libs/scheduler/inc/schInt.h | 3 + source/libs/scheduler/src/schJob.c | 36 +++++--- source/libs/scheduler/src/scheduler.c | 7 ++ 6 files changed, 125 insertions(+), 23 deletions(-) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 8d45e8b4a8..114bc00125 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -58,6 +58,8 @@ enum { #define TD_RES_TMQ_METADATA(res) (*(int8_t*)(res) == RES_TYPE__TMQ_METADATA) #define TD_RES_TMQ_BATCH_META(res) (*(int8_t*)(res) == RES_TYPE__TMQ_BATCH_META) +#define TSC_MAX_SUBPLAN_CAPACITY_NUM 1000 + typedef struct SAppInstInfo SAppInstInfo; typedef struct { diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 2c67cafdf5..c55764aaf4 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1251,6 +1251,7 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) { void launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQuery, void** res) { int32_t code = 0; + int32_t subplanNum = 0; if (pQuery->pRoot) { pRequest->stmtType = pQuery->pRoot->type; @@ -1292,7 +1293,7 @@ void launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQuery, void SQueryPlan* pDag = NULL; code = getPlan(pRequest, pQuery, &pDag, pMnodeList); if (TSDB_CODE_SUCCESS == code) { - pRequest->body.subplanNum = pDag->numOfSubplans; + subplanNum = pDag->numOfSubplans; if (!pRequest->validateOnly) { SArray* pNodeList = NULL; code = buildSyncExecNodeList(pRequest, &pNodeList, pMnodeList); @@ -1301,6 +1302,10 @@ void launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQuery, void } taosArrayDestroy(pNodeList); } + + if (TSDB_CODE_SUCCESS == code) { + pRequest->body.subplanNum = subplanNum; + } } taosArrayDestroy(pMnodeList); break; @@ -1343,6 +1348,7 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat pRequest->type = pQuery->msgType; SArray* pMnodeList = NULL; SQueryPlan* pDag = NULL; + int32_t subplanNum = 0; int64_t st = taosGetTimestampUs(); if (!pRequest->parseOnly) { @@ -1369,7 +1375,7 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat tscError("0x%" PRIx64 " failed to create query plan, code:%s 0x%" PRIx64, pRequest->self, tstrerror(code), pRequest->requestId); } else { - pRequest->body.subplanNum = pDag->numOfSubplans; + subplanNum = pDag->numOfSubplans; TSWAP(pRequest->pPostPlan, pDag->pPostPlan); } } @@ -1406,6 +1412,9 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat if (TSDB_CODE_SUCCESS == code) { code = schedulerExecJob(&req, &pRequest->body.queryJob); } + if (TSDB_CODE_SUCCESS == code) { + pRequest->body.subplanNum = subplanNum; + } taosArrayDestroy(pNodeList); } else { qDestroyQueryPlan(pDag); diff --git a/source/libs/executor/test/queryPlanTests.cpp b/source/libs/executor/test/queryPlanTests.cpp index 176af60944..c367391f8e 100755 --- a/source/libs/executor/test/queryPlanTests.cpp +++ b/source/libs/executor/test/queryPlanTests.cpp @@ -47,9 +47,9 @@ namespace { -#define QPT_MAX_LOOP 1000 +#define QPT_MAX_LOOP 100000 #define QPT_MAX_LEVEL_SUBPLAN_NUM 10 -#define QPT_MAX_SUBPLAN_LEVEL 5 +#define QPT_MAX_SUBPLAN_LEVEL 2 #define QPT_MAX_SUBPLAN_GROUP 5 #define QPT_MAX_WHEN_THEN_NUM 10 #define QPT_MAX_NODE_LEVEL 5 @@ -172,7 +172,10 @@ typedef struct { } SQPTMakeNodeCtx; typedef struct { + int64_t startTsUs; int32_t code; + int64_t succeedTimes; + int64_t failedTimes; } SQPTExecResult; typedef struct { @@ -183,7 +186,6 @@ typedef struct { SQPTMakeNodeCtx makeCtx; SQPTMakeNodeCtx makeCtxBak; SQPTExecResult result; - int64_t startTsUs; } SQPTCtx; typedef struct { @@ -257,7 +259,7 @@ SNode* qptCreateSubplanNode(int32_t nodeType); SQPTPlan qptPlans[] = { {QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN, QPT_PLAN_PHYSIC, "tagScan", qptCreateTagScanPhysiNode}, {QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN, QPT_PLAN_PHYSIC, "tableScan", qptCreateTableScanPhysiNode}, - {QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN, QPT_PLAN_PHYSIC, "tableSeqScan", NULL /*qptCreateTableSeqScanPhysiNode*/ }, + {QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN, QPT_PLAN_PHYSIC, "tableSeqScan", qptCreateTableSeqScanPhysiNode}, {QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN, QPT_PLAN_PHYSIC, "tableMergeScan", qptCreateTableMergeScanPhysiNode}, {QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN, QPT_PLAN_PHYSIC, "streamScan", qptCreateStreamScanPhysiNode}, {QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN, QPT_PLAN_PHYSIC, "sysTableScan", qptCreateSysTableScanPhysiNode}, @@ -329,6 +331,16 @@ int32_t qptSink[] = {QUERY_NODE_PHYSICAL_PLAN_DISPATCH, QUERY_NODE_PHYSICAL_PLAN SNode* qptMakeExprNode(SNode** ppNode); void qptMakeNodeList(QPT_NODE_TYPE nodeType, SNodeList** ppList); +int32_t qptGetSpecificPlanIndex(int32_t type) { + int32_t planNum = sizeof(qptPlans) / sizeof(qptPlans[0]); + for (int32_t i = 0; i < planNum; ++i) { + if (qptPlans[i].type == type) { + return i; + } + } + + return -1; +} int32_t qptGetColumnRandLen(int32_t colType) { switch (colType) { @@ -460,7 +472,8 @@ void qptPrintEndInfo() { } void qptPrintStatInfo() { - + printf("\n\tAll %d times TEST [%s] END, result - succeed:%" PRId64 " failed:%" PRId64 "\n", qptCtx.loopIdx + 1, qptCtx.caseName, + qptCtx.result.succeedTimes, qptCtx.result.failedTimes); } @@ -1661,6 +1674,19 @@ SNode* qptMakeOrderByExprNode(SNode** ppNode) { return *ppNode; } +SNode* qptMakeSubplanNode(SNode** ppNode) { + if (QPT_NCORRECT_LOW_PROB()) { + return qptMakeRandNode(ppNode); + } + + *ppNode = (SNode*)qptCreateSubplanNode(QUERY_NODE_PHYSICAL_SUBPLAN); + + return *ppNode; +} + + + + SPhysiNode* qptCreatePhysiNode(int32_t nodeType) { SPhysiNode* pPhysiNode = NULL; assert(0 == nodesMakeNode((ENodeType)nodeType, (SNode**)&pPhysiNode)); @@ -1928,6 +1954,18 @@ void qptMakeOrerByExprList(SNodeList** ppList) { } +void qptMakeSubplanList(SNodeList** ppList) { + qptSaveMakeNodeCtx(); + + int32_t planNum = taosRand() % QPT_MAX_LEVEL_SUBPLAN_NUM + (QPT_CORRECT_HIGH_PROB() ? 1 : 0); + for (int32_t i = 0; i < planNum; ++i) { + SNode* pNode = NULL; + qptRestoreMakeNodeCtx(); + qptMakeSubplanNode(&pNode); + qptNodesListMakeStrictAppend(ppList, pNode); + } +} + void qptMakeSpecTypeNodeList(QPT_NODE_TYPE nodeType, SNodeList** ppList) { switch (nodeType) { case QPT_NODE_COLUMN: @@ -1938,6 +1976,8 @@ void qptMakeSpecTypeNodeList(QPT_NODE_TYPE nodeType, SNodeList** ppList) { return qptMakeExprList(ppList); case QPT_NODE_VALUE: return qptMakeValueList(ppList); + case QPT_NODE_SUBPLAN: + return qptMakeSubplanList(ppList); default: break; } @@ -2934,6 +2974,8 @@ void qptResetForReRun() { qptCtx.buildCtx.pCurr = NULL; qptCtx.buildCtx.pCurrTask = NULL; + + qptCtx.result.code = 0; } void qptSingleTestDone(bool* contLoop) { @@ -3126,6 +3168,13 @@ void qptExecPlan(SReadHandle* pReadHandle, SNode* pNode, SExecTaskInfo* pTaskInf default: assert(0); } + + if (qptCtx.result.code) { + qptCtx.result.failedTimes++; + } else { + qptCtx.result.succeedTimes++; + } + } void qptRunSingleOpTest() { @@ -3144,7 +3193,7 @@ void qptRunSingleOpTest() { qptPrintBeginInfo(); - qptCtx.startTsUs = taosGetTimestampUs(); + qptCtx.result.startTsUs = taosGetTimestampUs(); qptExecPlan(&readHandle, pNode, pTaskInfo, &pOperator); @@ -3170,7 +3219,7 @@ void qptRunSubplanTest() { qptPrintBeginInfo(); - qptCtx.startTsUs = taosGetTimestampUs(); + qptCtx.result.startTsUs = taosGetTimestampUs(); //qptCtx.result.code = createTagScanOperatorInfo(&readHandle, (STagScanPhysiNode*)pNode, NULL, NULL, NULL, NULL, &pOperator); //qptCtx.result.code = createProjectOperatorInfo(NULL, (SProjectPhysiNode*)pNode, NULL, &pOperator); @@ -3307,9 +3356,9 @@ void qptDestroyTestCtx() { } // namespace #if 1 -#if 1 -TEST(singleNodeTest, randPlan) { - char* caseType = "singleNodeTest:randPlan"; +#if 0 +TEST(singleRandNodeTest, loopPlans) { + char* caseType = "singleRandNodeTest:loopPlans"; for (qptCtx.loopIdx = 0; qptCtx.loopIdx < QPT_MAX_LOOP; ++qptCtx.loopIdx) { for (int32_t i = 0; i < sizeof(qptPlans)/sizeof(qptPlans[0]); ++i) { @@ -3325,6 +3374,24 @@ TEST(singleNodeTest, randPlan) { qptPrintStatInfo(); } #endif +#if 1 +TEST(singleRandNodeTest, specificPlan) { + char* caseType = "singleRandNodeTest:specificPlan"; + + int32_t idx = qptGetSpecificPlanIndex(QUERY_NODE_PHYSICAL_PLAN); + for (qptCtx.loopIdx = 0; qptCtx.loopIdx < QPT_MAX_LOOP; ++qptCtx.loopIdx) { + sprintf(qptCtx.caseName, "%s:%s", caseType, qptPlans[idx].name); + qptInitTestCtx(false, true, qptPlans[idx].type, idx, 0, NULL); + + qptRunPlanTest(); + + qptDestroyTestCtx(); + } + + qptPrintStatInfo(); +} +#endif + #endif diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index 52c7c64887..96b9d2da8d 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -62,6 +62,7 @@ typedef enum { #define SCH_DEFAULT_MAX_RETRY_NUM 6 #define SCH_MIN_AYSNC_EXEC_NUM 3 #define SCH_DEFAULT_RETRY_TOTAL_ROUND 3 +#define SCH_DEFAULT_TASK_CAPACITY_NUM 1000 typedef struct SSchDebug { bool lockEnable; @@ -318,6 +319,8 @@ typedef struct SSchTaskCtx { extern SSchedulerMgmt schMgmt; +#define SCH_GET_TASK_CAPACITY(_n) ((_n) > SCH_DEFAULT_TASK_CAPACITY_NUM ? SCH_DEFAULT_TASK_CAPACITY_NUM : (_n)) + #define SCH_TASK_TIMEOUT(_task) \ ((taosGetTimestampUs() - *(int64_t *)taosArrayGet((_task)->profile.execTime, (_task)->execId)) > (_task)->timeoutUsec) diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index d1c970b9fe..c52f779b75 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -317,7 +317,7 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - pJob->dataSrcTasks = taosArrayInit(pDag->numOfSubplans, POINTER_BYTES); + pJob->dataSrcTasks = taosArrayInit(SCH_GET_TASK_CAPACITY(pDag->numOfSubplans), POINTER_BYTES); if (NULL == pJob->dataSrcTasks) { SCH_ERR_RET(terrno); } @@ -329,7 +329,7 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { } SHashObj *planToTask = taosHashInit( - pDag->numOfSubplans, + SCH_GET_TASK_CAPACITY(pDag->numOfSubplans), taosGetDefaultHashFunction(POINTER_BYTES == sizeof(int64_t) ? TSDB_DATA_TYPE_BIGINT : TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK); if (NULL == planToTask) { @@ -349,6 +349,7 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { SSchLevel level = {0}; SNodeListNode *plans = NULL; int32_t taskNum = 0; + int32_t totalTaskNum = 0; SSchLevel *pLevel = NULL; level.status = JOB_TASK_STATUS_INIT; @@ -362,7 +363,7 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { pLevel = taosArrayGet(pJob->levels, i); if (NULL == pLevel) { SCH_JOB_ELOG("fail to get the %dth level, levelNum: %d", i, levelNum); - SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } pLevel->level = i; @@ -384,6 +385,12 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); } + totalTaskNum += taskNum; + if (totalTaskNum > pDag->numOfSubplans) { + SCH_JOB_ELOG("current totalTaskNum %d is bigger than numOfSubplans %d, level:%d", totalTaskNum, pDag->numOfSubplans, i); + SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); + } + pLevel->taskNum = taskNum; pLevel->subTasks = taosArrayInit(taskNum, sizeof(SSchTask)); @@ -410,14 +417,16 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { SCH_ERR_JRET(schAppendJobDataSrc(pJob, pTask)); - if (0 != taosHashPut(planToTask, &plan, POINTER_BYTES, &pTask, POINTER_BYTES)) { - SCH_TASK_ELOG("taosHashPut to planToTaks failed, taskIdx:%d", n); - SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + code = taosHashPut(planToTask, &plan, POINTER_BYTES, &pTask, POINTER_BYTES); + if (0 != code) { + SCH_TASK_ELOG("taosHashPut to planToTaks failed, taskIdx:%d, error:%s", n, tstrerror(code)); + SCH_ERR_JRET(code); } - if (0 != taosHashPut(pJob->taskList, &pTask->taskId, sizeof(pTask->taskId), &pTask, POINTER_BYTES)) { - SCH_TASK_ELOG("taosHashPut to taskList failed, taskIdx:%d", n); - SCH_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); + code = taosHashPut(pJob->taskList, &pTask->taskId, sizeof(pTask->taskId), &pTask, POINTER_BYTES); + if (0 != code) { + SCH_TASK_ELOG("taosHashPut to taskList failed, taskIdx:%d, error:%s", n, tstrerror(code)); + SCH_ERR_JRET(code); } ++pJob->taskNum; @@ -426,6 +435,11 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { SCH_JOB_DLOG("level %d initialized, taskNum:%d", i, taskNum); } + if (totalTaskNum != pDag->numOfSubplans) { + SCH_JOB_ELOG("totalTaskNum %d mis-match with numOfSubplans %d", totalTaskNum, pDag->numOfSubplans); + SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); + } + SCH_ERR_JRET(schBuildTaskRalation(pJob, planToTask)); _return: @@ -876,10 +890,10 @@ int32_t schInitJob(int64_t *pJobId, SSchedulerReq *pReq) { } } - pJob->taskList = taosHashInit(pReq->pDag->numOfSubplans, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, + pJob->taskList = taosHashInit(SCH_GET_TASK_CAPACITY(pReq->pDag->numOfSubplans), taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, HASH_ENTRY_LOCK); if (NULL == pJob->taskList) { - SCH_JOB_ELOG("taosHashInit %d taskList failed", pReq->pDag->numOfSubplans); + SCH_JOB_ELOG("taosHashInit %d taskList failed", SCH_GET_TASK_CAPACITY(pReq->pDag->numOfSubplans)); SCH_ERR_JRET(terrno); } diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 01d3204dc6..db9ecd6025 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -233,6 +233,13 @@ int32_t schedulerValidatePlan(SQueryPlan* pPlan) { SCH_ERR_RET(terrno); } + pJob->taskList = taosHashInit(100, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), false, + HASH_ENTRY_LOCK); + if (NULL == pJob->taskList) { + SCH_JOB_ELOG("taosHashInit %d taskList failed", 100); + SCH_ERR_JRET(terrno); + } + SCH_ERR_JRET(schValidateAndBuildJob(pPlan, pJob)); if (SCH_IS_EXPLAIN_JOB(pJob)) { From f795e415e5e97b527c2e5f8bfb6755b32425133b Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 24 Oct 2024 15:01:49 +0800 Subject: [PATCH 17/29] fix: add subplan test --- source/libs/executor/test/queryPlanTests.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/test/queryPlanTests.cpp b/source/libs/executor/test/queryPlanTests.cpp index c367391f8e..720251fdfd 100755 --- a/source/libs/executor/test/queryPlanTests.cpp +++ b/source/libs/executor/test/queryPlanTests.cpp @@ -3027,7 +3027,7 @@ void qptExecPlan(SReadHandle* pReadHandle, SNode* pNode, SExecTaskInfo* pTaskInf qptCtx.result.code = createTableScanOperatorInfo((STableScanPhysiNode*)pNode, pReadHandle, NULL, pTaskInfo, ppOperaotr); break; case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN: - //qptCtx.result.code = createTableSeqScanOperatorInfo((STableScanPhysiNode*)pNode, pReadHandle, NULL, pTaskInfo, ppOperaotr); + qptCtx.result.code = createTableSeqScanOperatorInfo((STableScanPhysiNode*)pNode, pReadHandle, NULL, pTaskInfo, ppOperaotr); break; case QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN: qptCtx.result.code = createTableMergeScanOperatorInfo((STableScanPhysiNode*)pNode, pReadHandle, NULL, pTaskInfo, ppOperaotr); @@ -3075,7 +3075,7 @@ void qptExecPlan(SReadHandle* pReadHandle, SNode* pNode, SExecTaskInfo* pTaskInf qptCtx.result.code = createIntervalOperatorInfo(NULL, (SIntervalPhysiNode*)pNode, pTaskInfo, ppOperaotr); break; case QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL: - //qptCtx.result.code = createMergeIntervalOperatorInfo(NULL, (SMergeIntervalPhysiNode*)pNode, pTaskInfo, ppOperaotr); + qptCtx.result.code = createMergeIntervalOperatorInfo(NULL, (SMergeIntervalPhysiNode*)pNode, pTaskInfo, ppOperaotr); break; case QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL: qptCtx.result.code = createMergeAlignedIntervalOperatorInfo(NULL, (SMergeAlignedIntervalPhysiNode*)pNode, pTaskInfo, ppOperaotr); @@ -3125,6 +3125,7 @@ void qptExecPlan(SReadHandle* pReadHandle, SNode* pNode, SExecTaskInfo* pTaskInf qptCtx.result.code = createTimeSliceOperatorInfo(NULL, (SPhysiNode*)pNode, pTaskInfo, ppOperaotr); break; case QUERY_NODE_PHYSICAL_PLAN_INSERT: + qptCtx.result.code = 0; break; case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: case QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT: @@ -3135,6 +3136,9 @@ void qptExecPlan(SReadHandle* pReadHandle, SNode* pNode, SExecTaskInfo* pTaskInf break; } case QUERY_NODE_PHYSICAL_SUBPLAN: { + DataSinkHandle handle = NULL; + qptCtx.result.code = qCreateExecTask(pReadHandle, qptCtx.param.vnode.vgId, pTaskInfo->id.taskId, (SSubplan*)pNode, &pTaskInfo, &handle, + QPT_RAND_BOOL_V ? 0 : 1, taosStrdup("sql string"), OPTR_EXEC_MODEL_BATCH); break; } case QUERY_NODE_PHYSICAL_PLAN: { From 7a45984c96b9a96c228308080d596f598ca32e32 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 24 Oct 2024 15:28:09 +0800 Subject: [PATCH 18/29] fix: table seq scan issue --- source/libs/executor/inc/operator.h | 2 ++ source/libs/executor/test/queryPlanTests.cpp | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/inc/operator.h b/source/libs/executor/inc/operator.h index fce0b05fde..ddb9a5e0ec 100644 --- a/source/libs/executor/inc/operator.h +++ b/source/libs/executor/inc/operator.h @@ -85,6 +85,8 @@ int32_t createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode* pExNo int32_t createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, STableListInfo* pTableList, SExecTaskInfo* pTaskInfo, SOperatorInfo** pInfo); +int32_t createTableSeqScanOperatorInfo(void* pReadHandle, SExecTaskInfo* pTaskInfo, SOperatorInfo** pOptrInfo); + int32_t createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo, SOperatorInfo** pInfo); int32_t createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, STableListInfo* pTableListInfo, SNode* pTagCond, SNode*pTagIndexCond, SExecTaskInfo* pTaskInfo, SOperatorInfo** pInfo); diff --git a/source/libs/executor/test/queryPlanTests.cpp b/source/libs/executor/test/queryPlanTests.cpp index 720251fdfd..ced1f964c6 100755 --- a/source/libs/executor/test/queryPlanTests.cpp +++ b/source/libs/executor/test/queryPlanTests.cpp @@ -3027,7 +3027,7 @@ void qptExecPlan(SReadHandle* pReadHandle, SNode* pNode, SExecTaskInfo* pTaskInf qptCtx.result.code = createTableScanOperatorInfo((STableScanPhysiNode*)pNode, pReadHandle, NULL, pTaskInfo, ppOperaotr); break; case QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN: - qptCtx.result.code = createTableSeqScanOperatorInfo((STableScanPhysiNode*)pNode, pReadHandle, NULL, pTaskInfo, ppOperaotr); + qptCtx.result.code = createTableSeqScanOperatorInfo(pReadHandle, pTaskInfo, ppOperaotr); // usless break; case QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN: qptCtx.result.code = createTableMergeScanOperatorInfo((STableScanPhysiNode*)pNode, pReadHandle, NULL, pTaskInfo, ppOperaotr); From 8cd994645b0f132bd5d7618573088e5a2e6ddb5c Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 24 Oct 2024 15:30:47 +0800 Subject: [PATCH 19/29] fix: subplan test compile issue --- source/libs/executor/test/queryPlanTests.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/libs/executor/test/queryPlanTests.cpp b/source/libs/executor/test/queryPlanTests.cpp index ced1f964c6..1e0129ce5b 100755 --- a/source/libs/executor/test/queryPlanTests.cpp +++ b/source/libs/executor/test/queryPlanTests.cpp @@ -3137,7 +3137,7 @@ void qptExecPlan(SReadHandle* pReadHandle, SNode* pNode, SExecTaskInfo* pTaskInf } case QUERY_NODE_PHYSICAL_SUBPLAN: { DataSinkHandle handle = NULL; - qptCtx.result.code = qCreateExecTask(pReadHandle, qptCtx.param.vnode.vgId, pTaskInfo->id.taskId, (SSubplan*)pNode, &pTaskInfo, &handle, + qptCtx.result.code = qCreateExecTask(pReadHandle, qptCtx.param.vnode.vgId, pTaskInfo->id.taskId, (SSubplan*)pNode, (qTaskInfo_t*)&pTaskInfo, &handle, QPT_RAND_BOOL_V ? 0 : 1, taosStrdup("sql string"), OPTR_EXEC_MODEL_BATCH); break; } From 33a0fd551984fa5f0c48d0798d08a356f8784d9e Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Thu, 24 Oct 2024 18:34:53 +0800 Subject: [PATCH 20/29] fix: modify subplan validation issue --- source/libs/scheduler/src/schUtil.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/scheduler/src/schUtil.c b/source/libs/scheduler/src/schUtil.c index 4840108638..59951ab618 100644 --- a/source/libs/scheduler/src/schUtil.c +++ b/source/libs/scheduler/src/schUtil.c @@ -393,8 +393,8 @@ int32_t schValidateSubplan(SSchJob *pJob, SSubplan* pSubplan, int32_t level, int } } - if (NULL == pSubplan->pNode) { - SCH_JOB_ELOG("empty plan root node, level:%d, subplan idx:%d", level, idx); + if (NULL == pSubplan->pNode && pSubplan->subplanType != SUBPLAN_TYPE_MODIFY) { + SCH_JOB_ELOG("empty plan root node, level:%d, subplan idx:%d, subplanType:%d", level, idx, pSubplan->subplanType); SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } From c31eadea217db12c921b9a3500e952a0e49545a8 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Fri, 25 Oct 2024 08:42:14 +0800 Subject: [PATCH 21/29] fix: request subplan number setting issue --- source/client/src/clientImpl.c | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index c55764aaf4..ac164dc484 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1293,7 +1293,7 @@ void launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQuery, void SQueryPlan* pDag = NULL; code = getPlan(pRequest, pQuery, &pDag, pMnodeList); if (TSDB_CODE_SUCCESS == code) { - subplanNum = pDag->numOfSubplans; + pRequest->body.subplanNum = pDag->numOfSubplans; if (!pRequest->validateOnly) { SArray* pNodeList = NULL; code = buildSyncExecNodeList(pRequest, &pNodeList, pMnodeList); @@ -1302,10 +1302,6 @@ void launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQuery, void } taosArrayDestroy(pNodeList); } - - if (TSDB_CODE_SUCCESS == code) { - pRequest->body.subplanNum = subplanNum; - } } taosArrayDestroy(pMnodeList); break; @@ -1348,7 +1344,6 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat pRequest->type = pQuery->msgType; SArray* pMnodeList = NULL; SQueryPlan* pDag = NULL; - int32_t subplanNum = 0; int64_t st = taosGetTimestampUs(); if (!pRequest->parseOnly) { @@ -1375,7 +1370,7 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat tscError("0x%" PRIx64 " failed to create query plan, code:%s 0x%" PRIx64, pRequest->self, tstrerror(code), pRequest->requestId); } else { - subplanNum = pDag->numOfSubplans; + pRequest->body.subplanNum = pDag->numOfSubplans; TSWAP(pRequest->pPostPlan, pDag->pPostPlan); } } @@ -1412,9 +1407,7 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat if (TSDB_CODE_SUCCESS == code) { code = schedulerExecJob(&req, &pRequest->body.queryJob); } - if (TSDB_CODE_SUCCESS == code) { - pRequest->body.subplanNum = subplanNum; - } + taosArrayDestroy(pNodeList); } else { qDestroyQueryPlan(pDag); From cea25e7f9d9da27d55da219b4b2cf06a94a14e4b Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 28 Oct 2024 18:44:41 +0800 Subject: [PATCH 22/29] fix: scheduler ut issues --- source/libs/executor/test/queryPlanTests.cpp | 2 ++ source/libs/scheduler/src/schJob.c | 4 ++-- source/libs/scheduler/test/schedulerTests.cpp | 19 ++++++++++++++++++- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/source/libs/executor/test/queryPlanTests.cpp b/source/libs/executor/test/queryPlanTests.cpp index 1e0129ce5b..2b0c0e1118 100755 --- a/source/libs/executor/test/queryPlanTests.cpp +++ b/source/libs/executor/test/queryPlanTests.cpp @@ -371,6 +371,8 @@ int32_t qptGetColumnRandLen(int32_t colType) { assert(0); break; } + + return 0; } diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index c52f779b75..ab9bc6a98f 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -334,7 +334,7 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { HASH_NO_LOCK); if (NULL == planToTask) { SCH_JOB_ELOG("taosHashInit %d failed", SCHEDULE_DEFAULT_MAX_TASK_NUM); - SCH_ERR_RET(terrno); + SCH_ERR_JRET(terrno); } pJob->levels = taosArrayInit(levelNum, sizeof(SSchLevel)); @@ -363,7 +363,7 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { pLevel = taosArrayGet(pJob->levels, i); if (NULL == pLevel) { SCH_JOB_ELOG("fail to get the %dth level, levelNum: %d", i, levelNum); - SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + SCH_ERR_JRET(TSDB_CODE_QRY_INVALID_INPUT); } pLevel->level = i; diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 44d32b9480..6e13e37e88 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -203,6 +203,10 @@ void schtBuildQueryDag(SQueryPlan *dag) { return; } scanPlan->msgType = TDMT_SCH_QUERY; + code = nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_DISPATCH, (SNode**)&scanPlan->pDataSink); + if (NULL == scanPlan->pDataSink) { + return; + } mergePlan->id.queryId = qId; mergePlan->id.groupId = schtMergeTemplateId; @@ -223,6 +227,10 @@ void schtBuildQueryDag(SQueryPlan *dag) { return; } mergePlan->msgType = TDMT_SCH_QUERY; + code = nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_DISPATCH, (SNode**)&mergePlan->pDataSink); + if (NULL == mergePlan->pDataSink) { + return; + } merge->pNodeList = NULL; code = nodesMakeList(&merge->pNodeList); @@ -235,6 +243,7 @@ void schtBuildQueryDag(SQueryPlan *dag) { return; } + (void)nodesListAppend(merge->pNodeList, (SNode *)mergePlan); (void)nodesListAppend(scan->pNodeList, (SNode *)scanPlan); @@ -250,7 +259,7 @@ void schtBuildQueryFlowCtrlDag(SQueryPlan *dag) { int32_t scanPlanNum = 20; dag->queryId = qId; - dag->numOfSubplans = 2; + dag->numOfSubplans = scanPlanNum + 1; dag->pSubplans = NULL; int32_t code = nodesMakeList(&dag->pSubplans); if (NULL == dag->pSubplans) { @@ -289,6 +298,10 @@ void schtBuildQueryFlowCtrlDag(SQueryPlan *dag) { if (NULL == mergePlan->pChildren) { return; } + code = nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_DISPATCH, (SNode**)&mergePlan->pDataSink); + if (NULL == mergePlan->pDataSink) { + return; + } for (int32_t i = 0; i < scanPlanNum; ++i) { SSubplan *scanPlan = NULL; @@ -322,6 +335,10 @@ void schtBuildQueryFlowCtrlDag(SQueryPlan *dag) { return; } scanPlan->msgType = TDMT_SCH_QUERY; + code = nodesMakeNode(QUERY_NODE_PHYSICAL_PLAN_DISPATCH, (SNode**)&scanPlan->pDataSink); + if (NULL == scanPlan->pDataSink) { + return; + } (void)nodesListAppend(scanPlan->pParents, (SNode *)mergePlan); (void)nodesListAppend(mergePlan->pChildren, (SNode *)scanPlan); From 7d89545d3ceb35dd50925f94fc0d2c6561192d65 Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Wed, 30 Oct 2024 10:26:49 +0800 Subject: [PATCH 23/29] enh: alter taosCheckAndSetDebugFlag log level --- source/common/src/tglobal.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 5ab5500fa6..4076679ff4 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -2346,8 +2346,13 @@ static void taosCheckAndSetDebugFlag(int32_t *pFlagPtr, char *name, int32_t flag if (noNeedToSetVars != NULL && taosArraySearch(noNeedToSetVars, name, taosLogVarComp, TD_EQ) != NULL) { return; } - if (taosSetDebugFlag(pFlagPtr, name, flag) != 0) { - uError("failed to set flag %s to %d", name, flag); + int32_t code = 0; + if ((code = taosSetDebugFlag(pFlagPtr, name, flag)) != 0) { + if (code != TSDB_CODE_CFG_NOT_FOUND) { + uError("failed to set flag %s to %d", name, flag); + } else { + uDebug("failed to set flag %s to %d", name, flag); + } } return; } From 74065f91d805588d51d94a7dba5737fe1be3d722 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Wed, 30 Oct 2024 17:39:33 +0800 Subject: [PATCH 24/29] fix: data source ep issue --- include/libs/qcom/query.h | 2 +- include/util/taoserror.h | 1 + source/libs/scheduler/src/schJob.c | 2 +- source/libs/scheduler/src/schUtil.c | 2 +- source/util/src/terror.c | 1 + 5 files changed, 5 insertions(+), 3 deletions(-) diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 81a3952463..d2f714f400 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -364,7 +364,7 @@ void* getTaskPoolWorkerCb(); #define NEED_CLIENT_REFRESH_VG_ERROR(_code) \ ((_code) == TSDB_CODE_VND_HASH_MISMATCH || (_code) == TSDB_CODE_VND_INVALID_VGROUP_ID) #define NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code) \ - ((_code) == TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER || (_code) == TSDB_CODE_MND_INVALID_SCHEMA_VER) + ((_code) == TSDB_CODE_TDB_INVALID_TABLE_SCHEMA_VER || (_code) == TSDB_CODE_MND_INVALID_SCHEMA_VER || (_code) == TSDB_CODE_SCH_DATA_SRC_EP_MISS) #define NEED_CLIENT_HANDLE_ERROR(_code) \ (NEED_CLIENT_RM_TBLMETA_ERROR(_code) || NEED_CLIENT_REFRESH_VG_ERROR(_code) || \ NEED_CLIENT_REFRESH_TBLMETA_ERROR(_code)) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index a53923b904..2a67a3cae1 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -771,6 +771,7 @@ int32_t taosGetErrSize(); #define TSDB_CODE_SCH_TIMEOUT_ERROR TAOS_DEF_ERROR_CODE(0, 0x2504) #define TSDB_CODE_SCH_JOB_IS_DROPPING TAOS_DEF_ERROR_CODE(0, 0x2505) #define TSDB_CODE_SCH_JOB_NOT_EXISTS TAOS_DEF_ERROR_CODE(0, 0x2506) +#define TSDB_CODE_SCH_DATA_SRC_EP_MISS TAOS_DEF_ERROR_CODE(0, 0x2507) //parser #define TSDB_CODE_PAR_SYNTAX_ERROR TAOS_DEF_ERROR_CODE(0, 0x2600) diff --git a/source/libs/scheduler/src/schJob.c b/source/libs/scheduler/src/schJob.c index ab9bc6a98f..375a316185 100644 --- a/source/libs/scheduler/src/schJob.c +++ b/source/libs/scheduler/src/schJob.c @@ -402,7 +402,7 @@ int32_t schValidateAndBuildJob(SQueryPlan *pDag, SSchJob *pJob) { for (int32_t n = 0; n < taskNum; ++n) { SSubplan *plan = (SSubplan *)nodesListGetNode(plans->pNodeList, n); - SCH_ERR_RET(schValidateSubplan(pJob, plan, pLevel->level, n, taskNum)); + SCH_ERR_JRET(schValidateSubplan(pJob, plan, pLevel->level, n, taskNum)); SCH_SET_JOB_TYPE(pJob, plan->subplanType); diff --git a/source/libs/scheduler/src/schUtil.c b/source/libs/scheduler/src/schUtil.c index 59951ab618..4697de6f28 100644 --- a/source/libs/scheduler/src/schUtil.c +++ b/source/libs/scheduler/src/schUtil.c @@ -385,7 +385,7 @@ int32_t schValidateSubplan(SSchJob *pJob, SSubplan* pSubplan, int32_t level, int if (SCH_IS_DATA_BIND_PLAN(pSubplan)) { if (pSubplan->execNode.epSet.numOfEps <= 0) { SCH_JOB_ELOG("no execNode specifed for data src plan %d, numOfEps:%d", pSubplan->subplanType, pSubplan->execNode.epSet.numOfEps); - SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); + SCH_ERR_RET(TSDB_CODE_SCH_DATA_SRC_EP_MISS); } if (pSubplan->execNode.epSet.inUse >= pSubplan->execNode.epSet.numOfEps) { SCH_JOB_ELOG("invalid epset inUse %d for data src plan %d, numOfEps:%d", pSubplan->execNode.epSet.inUse, pSubplan->subplanType, pSubplan->execNode.epSet.numOfEps); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 52a3be120d..4d8b3627ef 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -618,6 +618,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SCH_INTERNAL_ERROR, "scheduler internal er TAOS_DEFINE_ERROR(TSDB_CODE_SCH_TIMEOUT_ERROR, "Task timeout") TAOS_DEFINE_ERROR(TSDB_CODE_SCH_JOB_IS_DROPPING, "Job is dropping") TAOS_DEFINE_ERROR(TSDB_CODE_SCH_JOB_NOT_EXISTS, "Job no longer exist") +TAOS_DEFINE_ERROR(TSDB_CODE_SCH_DATA_SRC_EP_MISS, "No valid epSet for data source node") // parser TAOS_DEFINE_ERROR(TSDB_CODE_PAR_SYNTAX_ERROR, "syntax error near") From fd1f3ba048e90a750961acab8a3257a7b9538f4c Mon Sep 17 00:00:00 2001 From: Shungang Li Date: Thu, 31 Oct 2024 11:40:39 +0800 Subject: [PATCH 25/29] enh: taosCheckAndSetDebugFlag print errstr --- source/common/src/tglobal.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 4076679ff4..00052cc810 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -2349,9 +2349,9 @@ static void taosCheckAndSetDebugFlag(int32_t *pFlagPtr, char *name, int32_t flag int32_t code = 0; if ((code = taosSetDebugFlag(pFlagPtr, name, flag)) != 0) { if (code != TSDB_CODE_CFG_NOT_FOUND) { - uError("failed to set flag %s to %d", name, flag); + uError("failed to set flag %s to %d, since:%s", name, flag, tstrerror(code)); } else { - uDebug("failed to set flag %s to %d", name, flag); + uDebug("failed to set flag %s to %d, since:%s", name, flag, tstrerror(code)); } } return; From 25f07c6e342c2f5746c99ded7dc4ea50a6ab9505 Mon Sep 17 00:00:00 2001 From: qevolg <2227465945@qq.com> Date: Thu, 31 Oct 2024 16:42:52 +0800 Subject: [PATCH 26/29] fix(.github): fix taoskeeper ci --- .github/workflows/taoskeeper-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/taoskeeper-ci.yml b/.github/workflows/taoskeeper-ci.yml index 38dfb6d43c..ba43937029 100644 --- a/.github/workflows/taoskeeper-ci.yml +++ b/.github/workflows/taoskeeper-ci.yml @@ -28,7 +28,7 @@ jobs: run: | mkdir debug cd debug - cmake .. -DBUILD_HTTP=false -DBUILD_JDBC=false -DBUILD_TOOLS=false -DBUILD_TEST=off -DBUILD_KEEPER=true + cmake .. -DBUILD_HTTP=false -DBUILD_JDBC=false -DBUILD_TOOLS=false -DBUILD_TEST=off -DBUILD_KEEPER=true -DBUILD_DEPENDENCY_TESTS=false make -j 4 sudo make install which taosd From 9b743c10bdbf8ebf7517831ead093562402099f8 Mon Sep 17 00:00:00 2001 From: qevolg <2227465945@qq.com> Date: Thu, 31 Oct 2024 16:59:43 +0800 Subject: [PATCH 27/29] chore(*): trigger taoskeeper ci --- examples/c/asyncdemo.c | 1 + tools/keeper/api/adapter2_test.go | 1 + 2 files changed, 2 insertions(+) diff --git a/examples/c/asyncdemo.c b/examples/c/asyncdemo.c index 91ec6f24b1..ef0bbc1e55 100644 --- a/examples/c/asyncdemo.c +++ b/examples/c/asyncdemo.c @@ -17,6 +17,7 @@ // this example opens multiple tables, insert/retrieve multiple tables // it is used by TAOS internally for one performance testing // to compiple: gcc -o asyncdemo asyncdemo.c -ltaos +// #include #include diff --git a/tools/keeper/api/adapter2_test.go b/tools/keeper/api/adapter2_test.go index e6fd263c43..5456dfeecf 100644 --- a/tools/keeper/api/adapter2_test.go +++ b/tools/keeper/api/adapter2_test.go @@ -14,6 +14,7 @@ import ( ) func TestAdapter2(t *testing.T) { + // c := &config.Config{ InstanceID: 64, Port: 6043, From ad6fe3f1ed706ded00e2259f39400d5c4b1114ee Mon Sep 17 00:00:00 2001 From: qevolg <2227465945@qq.com> Date: Thu, 31 Oct 2024 17:00:29 +0800 Subject: [PATCH 28/29] chore(*): trigger taoskeeper ci --- examples/c/asyncdemo.c | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/c/asyncdemo.c b/examples/c/asyncdemo.c index ef0bbc1e55..91ec6f24b1 100644 --- a/examples/c/asyncdemo.c +++ b/examples/c/asyncdemo.c @@ -17,7 +17,6 @@ // this example opens multiple tables, insert/retrieve multiple tables // it is used by TAOS internally for one performance testing // to compiple: gcc -o asyncdemo asyncdemo.c -ltaos -// #include #include From c9ee2a8565db7196782870521c81031be6ef4d03 Mon Sep 17 00:00:00 2001 From: qevolg <2227465945@qq.com> Date: Thu, 31 Oct 2024 17:03:14 +0800 Subject: [PATCH 29/29] chore(*): trigger taoskeeper ci --- tools/keeper/api/adapter2_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/keeper/api/adapter2_test.go b/tools/keeper/api/adapter2_test.go index 5456dfeecf..e6fd263c43 100644 --- a/tools/keeper/api/adapter2_test.go +++ b/tools/keeper/api/adapter2_test.go @@ -14,7 +14,6 @@ import ( ) func TestAdapter2(t *testing.T) { - // c := &config.Config{ InstanceID: 64, Port: 6043,