From 15014c2e19e51353e1a2f49a5d1bf6d3a9729733 Mon Sep 17 00:00:00 2001 From: wpan Date: Thu, 24 Jun 2021 10:50:10 +0800 Subject: [PATCH 01/16] fix bug --- src/client/src/tscSubquery.c | 2 +- src/query/inc/qExecutor.h | 1 + src/query/src/qExecutor.c | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index c3df4773e1..4d97fef52f 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -3604,10 +3604,10 @@ void* createQInfoFromQueryNode(SQueryInfo* pQueryInfo, STableGroupInfo* pTableGr // todo refactor: filter should not be applied here. createFilterInfo(pQueryAttr, 0); - pQueryAttr->numOfFilterCols = 0; SArray* pa = NULL; if (stage == MASTER_SCAN) { + pQueryAttr->createFilterOperator = false; // no need for parent query pa = createExecOperatorPlan(pQueryAttr); } else { pa = createGlobalMergePlan(pQueryAttr); diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index 9cd1c5b033..dbee1a7812 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -192,6 +192,7 @@ typedef struct SQueryAttr { bool needReverseScan; // need reverse scan bool distinctTag; // distinct tag query bool stateWindow; // window State on sub/normal table + bool createFilterOperator; // if filter operator is needed int32_t interBufSize; // intermediate buffer sizse int32_t havingNum; // having expr number diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 97a6cf807c..63e0025550 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -7045,6 +7045,8 @@ int32_t createFilterInfo(SQueryAttr* pQueryAttr, uint64_t qId) { doCreateFilterInfo(pQueryAttr->tableCols, pQueryAttr->numOfCols, pQueryAttr->numOfFilterCols, &pQueryAttr->pFilterInfo, qId); + pQueryAttr->createFilterOperator = true; + return TSDB_CODE_SUCCESS; } From eed57b8a5b46aeaed9231973fcd214eef03603ff Mon Sep 17 00:00:00 2001 From: wpan Date: Thu, 24 Jun 2021 15:28:26 +0800 Subject: [PATCH 02/16] fix bug --- src/query/src/qPlan.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/src/qPlan.c b/src/query/src/qPlan.c index ee587a515d..d2eb9afb22 100644 --- a/src/query/src/qPlan.c +++ b/src/query/src/qPlan.c @@ -623,7 +623,7 @@ SArray* createExecOperatorPlan(SQueryAttr* pQueryAttr) { taosArrayPush(plan, &op); } } else { // diff/add/multiply/subtract/division - if (pQueryAttr->numOfFilterCols > 0 && pQueryAttr->vgId == 0) { // todo refactor + if (pQueryAttr->numOfFilterCols > 0 && pQueryAttr->createFilterOperator && pQueryAttr->vgId == 0) { // todo refactor op = OP_Filter; taosArrayPush(plan, &op); } else { From b945a7fd333dbabb6bf0f8216e5a9cea03c0e5c2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 24 Jun 2021 16:33:57 +0800 Subject: [PATCH 03/16] [td-4791] check sql function in outer query. --- src/client/src/tscSQLParser.c | 58 ++++++++++-------- src/query/inc/qExecutor.h | 8 +-- src/query/src/qAggMain.c | 1 - src/query/src/qExecutor.c | 54 ++++++++--------- src/query/src/qPlan.c | 14 ++--- tests/script/general/parser/nestquery.sim | 72 +++++++++++++++++++++-- 6 files changed, 138 insertions(+), 69 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index e94f77d254..815442a038 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -2069,33 +2069,29 @@ static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS const char* name, int32_t resColIdx, SColumnIndex* pColIndex, bool finalResult) { const char* msg1 = "not support column types"; - int16_t type = 0; - int16_t bytes = 0; - int32_t functionID = cvtFunc.execFuncId; - - if (functionID == TSDB_FUNC_SPREAD) { + int32_t f = cvtFunc.execFuncId; + if (f == TSDB_FUNC_SPREAD) { int32_t t1 = pSchema->type; - if (t1 == TSDB_DATA_TYPE_BINARY || t1 == TSDB_DATA_TYPE_NCHAR || t1 == TSDB_DATA_TYPE_BOOL) { + if (IS_VAR_DATA_TYPE(t1) || t1 == TSDB_DATA_TYPE_BOOL) { invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1); return -1; - } else { - type = TSDB_DATA_TYPE_DOUBLE; - bytes = tDataTypes[type].bytes; } - } else { - type = pSchema->type; - bytes = pSchema->bytes; } - SExprInfo* pExpr = tscExprAppend(pQueryInfo, functionID, pColIndex, type, bytes, getNewResColId(pCmd), bytes, false); + int16_t resType = 0; + int16_t resBytes = 0; + int32_t interBufSize = 0; + + getResultDataInfo(pSchema->type, pSchema->bytes, f, 0, &resType, &resBytes, &interBufSize, 0, false); + SExprInfo* pExpr = tscExprAppend(pQueryInfo, f, pColIndex, resType, resBytes, getNewResColId(pCmd), interBufSize, false); tstrncpy(pExpr->base.aliasName, name, tListLen(pExpr->base.aliasName)); - if (cvtFunc.originFuncId == TSDB_FUNC_LAST_ROW && cvtFunc.originFuncId != functionID) { + if (cvtFunc.originFuncId == TSDB_FUNC_LAST_ROW && cvtFunc.originFuncId != f) { pExpr->base.colInfo.flag |= TSDB_COL_NULL; } // set reverse order scan data blocks for last query - if (functionID == TSDB_FUNC_LAST) { + if (f == TSDB_FUNC_LAST) { pExpr->base.numOfParams = 1; pExpr->base.param[0].i64 = TSDB_ORDER_DESC; pExpr->base.param[0].nType = TSDB_DATA_TYPE_INT; @@ -2108,7 +2104,7 @@ static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS // if it is not in the final result, do not add it SColumnList ids = createColumnList(1, pColIndex->tableIndex, pColIndex->columnIndex); if (finalResult) { - insertResultField(pQueryInfo, resColIdx, &ids, bytes, (int8_t)type, pExpr->base.aliasName, pExpr); + insertResultField(pQueryInfo, resColIdx, &ids, pSchema->bytes, (int8_t)pSchema->type, pExpr->base.aliasName, pExpr); } else { tscColumnListInsert(pQueryInfo->colList, ids.ids[0].columnIndex, pExpr->base.uid, pSchema); } @@ -2557,8 +2553,9 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col tVariant* pVariant = &pParamElem[1].pNode->value; - int8_t resultType = pSchema->type; - int16_t resultSize = pSchema->bytes; + int16_t resultType = pSchema->type; + int16_t resultSize = pSchema->bytes; + int32_t interResult = 0; char val[8] = {0}; @@ -2571,8 +2568,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg5); } - resultSize = sizeof(double); - resultType = TSDB_DATA_TYPE_DOUBLE; + getResultDataInfo(pSchema->type, pSchema->bytes, functionId, 0, &resultType, &resultSize, &interResult, 0, false); /* * sql function transformation @@ -2582,7 +2578,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col tscInsertPrimaryTsSourceColumn(pQueryInfo, pTableMetaInfo->pTableMeta->id.uid); colIndex += 1; // the first column is ts - pExpr = tscExprAppend(pQueryInfo, functionId, &index, resultType, resultSize, getNewResColId(pCmd), resultSize, false); + pExpr = tscExprAppend(pQueryInfo, functionId, &index, resultType, resultSize, getNewResColId(pCmd), interResult, false); tscExprAddParams(&pExpr->base, val, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); } else { tVariantDump(pVariant, val, TSDB_DATA_TYPE_BIGINT, true); @@ -7784,8 +7780,8 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf const char* msg3 = "start(end) time of query range required or time range too large"; const char* msg4 = "interval query not supported, since the result of sub query not include valid timestamp column"; const char* msg5 = "only tag query not compatible with normal column filter"; - const char* msg6 = "not support stddev/percentile in outer query yet"; - const char* msg7 = "drivative requires timestamp column exists in subquery"; + const char* msg6 = "not support stddev/percentile/interp in the outer query yet"; + const char* msg7 = "derivative/twa/irate requires timestamp column exists in subquery"; int32_t code = TSDB_CODE_SUCCESS; @@ -7828,15 +7824,17 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf if (validateSelectNodeList(pCmd, pQueryInfo, pSqlNode->pSelNodeList, false, false, timeWindowQuery) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_OPERATION; } + // parse the window_state if (validateStateWindowNode(pCmd, pQueryInfo, pSqlNode, false) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_OPERATION; } + // todo NOT support yet for(int32_t i = 0; i < tscNumOfExprs(pQueryInfo); ++i) { SExprInfo* pExpr = tscExprGet(pQueryInfo, i); int32_t f = pExpr->base.functionId; - if (f == TSDB_FUNC_STDDEV || f == TSDB_FUNC_PERCT) { + if (f == TSDB_FUNC_STDDEV || f == TSDB_FUNC_PERCT || f == TSDB_FUNC_INTERP) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg6); } @@ -7851,9 +7849,17 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta; SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, 0); - if (tscNumOfExprs(pQueryInfo) > 1) { + int32_t numOfExprs = tscNumOfExprs(pQueryInfo); + if (numOfExprs == 1) { + SExprInfo* pExpr = tscExprGet(pQueryInfo, 0); + int32_t f = pExpr->base.functionId; + if (f == TSDB_FUNC_DERIVATIVE || f == TSDB_FUNC_TWA || f == TSDB_FUNC_IRATE) { + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); + } + } else { SExprInfo* pExpr = tscExprGet(pQueryInfo, 1); - if (pExpr->base.functionId == TSDB_FUNC_DERIVATIVE && pSchema->type != TSDB_DATA_TYPE_TIMESTAMP) { + int32_t f = pExpr->base.functionId; + if ((f == TSDB_FUNC_DERIVATIVE || f == TSDB_FUNC_TWA || f == TSDB_FUNC_IRATE) && pSchema->type != TSDB_DATA_TYPE_TIMESTAMP) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); } } diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index 9cd1c5b033..1f1f9f3e60 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -285,7 +285,7 @@ enum OPERATOR_TYPE_E { OP_TagScan = 4, OP_TableBlockInfoScan= 5, OP_Aggregate = 6, - OP_Arithmetic = 7, + OP_Project = 7, OP_Groupby = 8, OP_Limit = 9, OP_SLimit = 10, @@ -413,13 +413,13 @@ typedef struct SAggOperatorInfo { uint32_t seed; } SAggOperatorInfo; -typedef struct SArithOperatorInfo { +typedef struct SProjectOperatorInfo { SOptrBasicInfo binfo; int32_t bufCapacity; uint32_t seed; SSDataBlock *existDataBlock; -} SArithOperatorInfo; +} SProjectOperatorInfo; typedef struct SLimitOperatorInfo { int64_t limit; @@ -513,7 +513,7 @@ SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv); SOperatorInfo* createAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createArithOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createProjectOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); SOperatorInfo* createLimitOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream); SOperatorInfo* createTimeIntervalOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); SOperatorInfo* createSWindowOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); diff --git a/src/query/src/qAggMain.c b/src/query/src/qAggMain.c index 8efc4aad4c..676e5b6ce6 100644 --- a/src/query/src/qAggMain.c +++ b/src/query/src/qAggMain.c @@ -74,7 +74,6 @@ } while (0); void noop1(SQLFunctionCtx *UNUSED_PARAM(pCtx)) {} -void noop2(SQLFunctionCtx *UNUSED_PARAM(pCtx), int32_t UNUSED_PARAM(index)) {} void doFinalizer(SQLFunctionCtx *pCtx) { RESET_RESULT_INFO(GET_RES_INFO(pCtx)); } diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 97a6cf807c..49ab0af680 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -184,7 +184,7 @@ static int32_t getNumOfScanTimes(SQueryAttr* pQueryAttr); static void destroyBasicOperatorInfo(void* param, int32_t numOfOutput); static void destroySFillOperatorInfo(void* param, int32_t numOfOutput); static void destroyGroupbyOperatorInfo(void* param, int32_t numOfOutput); -static void destroyArithOperatorInfo(void* param, int32_t numOfOutput); +static void destroyProjectOperatorInfo(void* param, int32_t numOfOutput); static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput); static void destroySWindowOperatorInfo(void* param, int32_t numOfOutput); static void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput); @@ -912,7 +912,7 @@ void setInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlo doSetInputDataBlockInfo(pOperator, pCtx, pBlock, order); } } else { - if (/*pCtx[0].pInput == NULL && */pBlock->pDataBlock != NULL) { + if (pBlock->pDataBlock != NULL) { doSetInputDataBlock(pOperator, pCtx, pBlock, order); } else { doSetInputDataBlockInfo(pOperator, pCtx, pBlock, order); @@ -978,7 +978,7 @@ static void doAggregateImpl(SOperatorInfo* pOperator, TSKEY startTs, SQLFunction } } -static void arithmeticApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t numOfOutput) { +static void projectApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t numOfOutput) { SQueryAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; for (int32_t k = 0; k < numOfOutput; ++k) { @@ -1806,17 +1806,17 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOf break; } - case OP_Arithmetic: { // TODO refactor to remove arith operator. + case OP_Project: { // TODO refactor to remove arith operator. SOperatorInfo* prev = pRuntimeEnv->proot; if (i == 0) { - pRuntimeEnv->proot = createArithOperatorInfo(pRuntimeEnv, prev, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); + pRuntimeEnv->proot = createProjectOperatorInfo(pRuntimeEnv, prev, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); if (pRuntimeEnv->proot != NULL && prev->operatorType != OP_DummyInput && prev->operatorType != OP_Join) { // TODO refactor setTableScanFilterOperatorInfo(prev->info, pRuntimeEnv->proot); } } else { prev = pRuntimeEnv->proot; assert(pQueryAttr->pExpr2 != NULL); - pRuntimeEnv->proot = createArithOperatorInfo(pRuntimeEnv, prev, pQueryAttr->pExpr2, pQueryAttr->numOfExpr2); + pRuntimeEnv->proot = createProjectOperatorInfo(pRuntimeEnv, prev, pQueryAttr->pExpr2, pQueryAttr->numOfExpr2); } break; } @@ -4578,8 +4578,8 @@ void setTableScanFilterOperatorInfo(STableScanInfo* pTableScanInfo, SOperatorInf pTableScanInfo->pResultRowInfo = &pInfo->resultRowInfo; pTableScanInfo->rowCellInfoOffset = pInfo->rowCellInfoOffset; - } else if (pDownstream->operatorType == OP_Arithmetic) { - SArithOperatorInfo *pInfo = pDownstream->info; + } else if (pDownstream->operatorType == OP_Project) { + SProjectOperatorInfo *pInfo = pDownstream->info; pTableScanInfo->pCtx = pInfo->binfo.pCtx; pTableScanInfo->pResultRowInfo = &pInfo->binfo.resultRowInfo; @@ -4934,23 +4934,23 @@ static SSDataBlock* doSTableAggregate(void* param, bool* newgroup) { return pInfo->pRes; } -static SSDataBlock* doArithmeticOperation(void* param, bool* newgroup) { +static SSDataBlock* doProjectOperation(void* param, bool* newgroup) { SOperatorInfo* pOperator = (SOperatorInfo*) param; - SArithOperatorInfo* pArithInfo = pOperator->info; + SProjectOperatorInfo* pProjectInfo = pOperator->info; SQueryRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; - SOptrBasicInfo *pInfo = &pArithInfo->binfo; + SOptrBasicInfo *pInfo = &pProjectInfo->binfo; SSDataBlock* pRes = pInfo->pRes; int32_t order = pRuntimeEnv->pQueryAttr->order.order; pRes->info.rows = 0; - if (pArithInfo->existDataBlock) { // TODO refactor + if (pProjectInfo->existDataBlock) { // TODO refactor STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; - SSDataBlock* pBlock = pArithInfo->existDataBlock; - pArithInfo->existDataBlock = NULL; + SSDataBlock* pBlock = pProjectInfo->existDataBlock; + pProjectInfo->existDataBlock = NULL; *newgroup = true; // todo dynamic set tags @@ -4960,9 +4960,9 @@ static SSDataBlock* doArithmeticOperation(void* param, bool* newgroup) { // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order); - updateOutputBuf(&pArithInfo->binfo, &pArithInfo->bufCapacity, pBlock->info.rows); + updateOutputBuf(&pProjectInfo->binfo, &pProjectInfo->bufCapacity, pBlock->info.rows); - arithmeticApplyFunctions(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput); + projectApplyFunctions(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput); if (pTableQueryInfo != NULL) { updateTableIdInfo(pTableQueryInfo, pBlock, pRuntimeEnv->pTableRetrieveTsMap, order); } @@ -4990,7 +4990,7 @@ static SSDataBlock* doArithmeticOperation(void* param, bool* newgroup) { // Return result of the previous group in the firstly. if (*newgroup) { if (pRes->info.rows > 0) { - pArithInfo->existDataBlock = pBlock; + pProjectInfo->existDataBlock = pBlock; clearNumOfRes(pInfo->pCtx, pOperator->numOfOutput); return pInfo->pRes; } else { // init output buffer for a new group data @@ -5010,9 +5010,9 @@ static SSDataBlock* doArithmeticOperation(void* param, bool* newgroup) { // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order); - updateOutputBuf(&pArithInfo->binfo, &pArithInfo->bufCapacity, pBlock->info.rows); + updateOutputBuf(&pProjectInfo->binfo, &pProjectInfo->bufCapacity, pBlock->info.rows); - arithmeticApplyFunctions(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput); + projectApplyFunctions(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput); if (pTableQueryInfo != NULL) { updateTableIdInfo(pTableQueryInfo, pBlock, pRuntimeEnv->pTableRetrieveTsMap, order); } @@ -5649,8 +5649,8 @@ static void destroyGroupbyOperatorInfo(void* param, int32_t numOfOutput) { tfree(pInfo->prevData); } -static void destroyArithOperatorInfo(void* param, int32_t numOfOutput) { - SArithOperatorInfo* pInfo = (SArithOperatorInfo*) param; +static void destroyProjectOperatorInfo(void* param, int32_t numOfOutput) { + SProjectOperatorInfo* pInfo = (SProjectOperatorInfo*) param; doDestroyBasicInfo(&pInfo->binfo, numOfOutput); } @@ -5696,8 +5696,8 @@ SOperatorInfo* createMultiTableAggOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SO return pOperator; } -SOperatorInfo* createArithOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { - SArithOperatorInfo* pInfo = calloc(1, sizeof(SArithOperatorInfo)); +SOperatorInfo* createProjectOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { + SProjectOperatorInfo* pInfo = calloc(1, sizeof(SProjectOperatorInfo)); pInfo->seed = rand(); pInfo->bufCapacity = pRuntimeEnv->resultInfo.capacity; @@ -5710,8 +5710,8 @@ SOperatorInfo* createArithOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorI setDefaultOutputBuf(pRuntimeEnv, pBInfo, pInfo->seed, MASTER_SCAN); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "ArithmeticOperator"; - pOperator->operatorType = OP_Arithmetic; + pOperator->name = "ProjectOperator"; + pOperator->operatorType = OP_Project; pOperator->blockingOptr = false; pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; @@ -5719,8 +5719,8 @@ SOperatorInfo* createArithOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorI pOperator->numOfOutput = numOfOutput; pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->exec = doArithmeticOperation; - pOperator->cleanup = destroyArithOperatorInfo; + pOperator->exec = doProjectOperation; + pOperator->cleanup = destroyProjectOperatorInfo; appendUpstream(pOperator, upstream); return pOperator; diff --git a/src/query/src/qPlan.c b/src/query/src/qPlan.c index ee587a515d..214ef32cd4 100644 --- a/src/query/src/qPlan.c +++ b/src/query/src/qPlan.c @@ -565,7 +565,7 @@ SArray* createExecOperatorPlan(SQueryAttr* pQueryAttr) { taosArrayPush(plan, &op); if (pQueryAttr->pExpr2 != NULL) { - op = OP_Arithmetic; + op = OP_Project; taosArrayPush(plan, &op); } @@ -585,7 +585,7 @@ SArray* createExecOperatorPlan(SQueryAttr* pQueryAttr) { } if (pQueryAttr->pExpr2 != NULL) { - op = OP_Arithmetic; + op = OP_Project; taosArrayPush(plan, &op); } } else if (pQueryAttr->sw.gap > 0) { @@ -593,7 +593,7 @@ SArray* createExecOperatorPlan(SQueryAttr* pQueryAttr) { taosArrayPush(plan, &op); if (pQueryAttr->pExpr2 != NULL) { - op = OP_Arithmetic; + op = OP_Project; taosArrayPush(plan, &op); } } else if (pQueryAttr->stateWindow) { @@ -601,7 +601,7 @@ SArray* createExecOperatorPlan(SQueryAttr* pQueryAttr) { taosArrayPush(plan, &op); if (pQueryAttr->pExpr2 != NULL) { - op = OP_Arithmetic; + op = OP_Project; taosArrayPush(plan, &op); } } else if (pQueryAttr->simpleAgg) { @@ -619,7 +619,7 @@ SArray* createExecOperatorPlan(SQueryAttr* pQueryAttr) { } if (pQueryAttr->pExpr2 != NULL && !pQueryAttr->stableQuery) { - op = OP_Arithmetic; + op = OP_Project; taosArrayPush(plan, &op); } } else { // diff/add/multiply/subtract/division @@ -627,7 +627,7 @@ SArray* createExecOperatorPlan(SQueryAttr* pQueryAttr) { op = OP_Filter; taosArrayPush(plan, &op); } else { - op = OP_Arithmetic; + op = OP_Project; taosArrayPush(plan, &op); } } @@ -665,7 +665,7 @@ SArray* createGlobalMergePlan(SQueryAttr* pQueryAttr) { } if (pQueryAttr->pExpr2 != NULL) { - op = OP_Arithmetic; + op = OP_Project; taosArrayPush(plan, &op); } } diff --git a/tests/script/general/parser/nestquery.sim b/tests/script/general/parser/nestquery.sim index 8249d9197f..fd56a91dd6 100644 --- a/tests/script/general/parser/nestquery.sim +++ b/tests/script/general/parser/nestquery.sim @@ -180,20 +180,82 @@ if $data21 != 49.500000000 then endi #define TSDB_FUNC_APERCT 7 -#define TSDB_FUNC_LAST_ROW 10 #define TSDB_FUNC_TWA 14 #define TSDB_FUNC_LEASTSQR 15 -#define TSDB_FUNC_ARITHM 23 #define TSDB_FUNC_DIFF 24 #define TSDB_FUNC_INTERP 28 -#define TSDB_FUNC_RATE 29 #define TSDB_FUNC_IRATE 30 #define TSDB_FUNC_DERIVATIVE 32 sql_error select stddev(c1) from (select c1 from nest_tb0); sql_error select percentile(c1, 20) from (select * from nest_tb0); +sql_error select interp(c1) from (select * from nest_tb0); +sql_error select derivative(val, 1s, 0) from (select c1 val from nest_tb0); +sql_error select twa(c1) from (select c1 from nest_tb0); +sql_error select irate(c1) from (select c1 from nest_tb0); +sql_error select diff(c1), twa(c1) from (select * from nest_tb0); +sql_error select irate(c1), interp(c1), twa(c1) from (select * from nest_tb0); + +sql select apercentile(c1, 50) from (select * from nest_tb0) interval(1d) +sql select twa(c1) from (select * from nest_tb0); +sql select leastsquares(c1, 1, 1) from (select * from nest_tb0); +sql select irate(c1) from (select * from nest_tb0); sql select avg(c1),sum(c2), max(c3), min(c4), count(*), first(c7), last(c7),spread(c6) from (select * from nest_tb0) interval(1d); +if $rows != 7 then + return -1 +endi + +if $data00 != @20-09-15 00:00:00.000@ then + return -1 +endi + +if $data01 != 48.666666667 then + print expect 48.666666667, actual: $data01 + return -1 +endi + +if $data02 != 70080.000000000 then + return -1 +endi + +if $data03 != 99 then + return -1 +endi + +if $data04 != 0 then + return -1 +endi + +if $data05 != 1440 then + return -1 +endi + +if $data06 != 0 then + print $data06 + return -1 +endi + +if $data07 != 1 then + return -1 +endi + +if $data08 != 99.000000000 then + print expect 99.000000000, actual: $data08 + return -1 +endi + +if $data10 != @20-09-16 00:00:00.000@ then + return -1 +endi + +if $data11 != 49.777777778 then + return -1 +endi + +if $data12 != 71680.000000000 then + return -1 +endi sql select top(x, 20) from (select c1 x from nest_tb0); @@ -207,6 +269,9 @@ print ===================> group by + having +print =========================> ascending order/descending order + + print =========================> nest query join @@ -273,7 +338,6 @@ if $data03 != @20-09-15 00:00:00.000@ then return -1 endi -sql_error select derivative(val, 1s, 0) from (select c1 val from nest_tb0); sql select diff(val) from (select c1 val from nest_tb0); if $rows != 9999 then return -1 From 795bd8c07676bac3d0a163b862be7ed2d2f4c1e6 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Fri, 25 Jun 2021 10:39:17 +0800 Subject: [PATCH 04/16] [TD-4098] unsigned date type not found --- src/client/src/tscSQLParser.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index e94f77d254..5107884422 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -165,6 +165,7 @@ bool serializeExprListToVariant(SArray* pList, tVariant **dst, int16_t colType, } else { tbufWriteUint32(&bw, colType); } + tbufWriteInt32(&bw, (int32_t)(pList->size)); for (int32_t i = 0; i < (int32_t)pList->size; i++) { @@ -181,10 +182,11 @@ bool serializeExprListToVariant(SArray* pList, tVariant **dst, int16_t colType, } tbufWriteInt64(&bw, var->i64); } else if (IS_UNSIGNED_NUMERIC_TYPE(colType)) { - if (IS_SIGNED_NUMERIC_TYPE(var->nType) && IS_UNSIGNED_NUMERIC_TYPE(var->nType)) { + if (IS_SIGNED_NUMERIC_TYPE(var->nType) || IS_UNSIGNED_NUMERIC_TYPE(var->nType)) { + tbufWriteUint64(&bw, var->u64); + } else { break; - } - tbufWriteUint64(&bw, var->u64); + } } else if (colType == TSDB_DATA_TYPE_DOUBLE || colType == TSDB_DATA_TYPE_FLOAT) { if (IS_SIGNED_NUMERIC_TYPE(var->nType) || IS_UNSIGNED_NUMERIC_TYPE(var->nType)) { tbufWriteDouble(&bw, (double)(var->i64)); From 2555665a8e33a8a8774b41acbb181f6fd265f036 Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Fri, 25 Jun 2021 15:22:22 +0800 Subject: [PATCH 05/16] [TD-4897] : clients should process result set ASAP after calling "taos_consume()". --- documentation20/cn/08.connector/docs.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/documentation20/cn/08.connector/docs.md b/documentation20/cn/08.connector/docs.md index 2d76c866d1..aa5fa50b66 100644 --- a/documentation20/cn/08.connector/docs.md +++ b/documentation20/cn/08.connector/docs.md @@ -427,12 +427,15 @@ TDengine提供时间驱动的实时流式计算API。可以每隔一指定的时 * res:查询结果集,注意结果集中可能没有记录 * param:调用 `taos_subscribe`时客户程序提供的附加参数 * code:错误码 + **注意**:在这个回调函数里不可以做耗时过长的处理,尤其是对于返回的结果集中数据较多的情况,否则有可能导致客户端阻塞等异常状态。如果必须进行复杂计算,则建议在另外的线程中进行处理。 * `TAOS_RES *taos_consume(TAOS_SUB *tsub)` 同步模式下,该函数用来获取订阅的结果。 用户应用程序将其置于一个循环之中。 如两次调用`taos_consume`的间隔小于订阅的轮询周期,API将会阻塞,直到时间间隔超过此周期。 如果数据库有新记录到达,该API将返回该最新的记录,否则返回一个没有记录的空结果集。 如果返回值为 `NULL`,说明系统出错。 异步模式下,用户程序不应调用此API。 + **注意**:在调用 `taos_consume()` 之后,用户应用应确保尽快调用 `taos_fetch_row()` 或 `taos_fetch_block()` 来处理订阅结果,否则服务端会持续缓存查询结果数据等待客户端读取,极端情况下会导致服务端内存消耗殆尽,影响服务稳定性。 + * `void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress)` 取消订阅。 如参数 `keepProgress` 不为0,API会保留订阅的进度信息,后续调用 `taos_subscribe` 时可以基于此进度继续;否则将删除进度信息,后续只能重新开始读取数据。 From 0ecaba7e9ea4050d2ab813e35469164d6c0efb8a Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 21 Jun 2021 11:18:17 +0800 Subject: [PATCH 06/16] [TD-4756]: nanosecond precison on gcBuildQueryJson in http plugin --- src/plugins/http/src/httpGcJson.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/plugins/http/src/httpGcJson.c b/src/plugins/http/src/httpGcJson.c index 397791706d..f33a994465 100644 --- a/src/plugins/http/src/httpGcJson.c +++ b/src/plugins/http/src/httpGcJson.c @@ -228,13 +228,11 @@ bool gcBuildQueryJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result, case TSDB_DATA_TYPE_NCHAR: httpJsonStringForTransMean(jsonBuf, (char *)row[i], fields[i].bytes); break; - case TSDB_DATA_TYPE_TIMESTAMP: - if (precision == TSDB_TIME_PRECISION_MILLI) { // ms - httpJsonInt64(jsonBuf, *((int64_t *)row[i])); - } else { - httpJsonInt64(jsonBuf, *((int64_t *)row[i]) / 1000); - } + case TSDB_DATA_TYPE_TIMESTAMP: { + int64_t ts = convertTimePrecision(*((int64_t *)row[i]), precision, TSDB_TIME_PRECISION_MILLI); + httpJsonInt64(jsonBuf, ts); break; + } default: httpJsonString(jsonBuf, "-", 1); break; From 2361040efe22c4abeeb17f5f2ceef6e60e6c387b Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Fri, 25 Jun 2021 18:09:37 +0800 Subject: [PATCH 07/16] [TD-4793] : describe SQL function IRATE(). --- documentation20/cn/12.taos-sql/docs.md | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/documentation20/cn/12.taos-sql/docs.md b/documentation20/cn/12.taos-sql/docs.md index 5904abbbaa..5ab055bcda 100644 --- a/documentation20/cn/12.taos-sql/docs.md +++ b/documentation20/cn/12.taos-sql/docs.md @@ -854,7 +854,23 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 应用字段:不能应用在timestamp、binary、nchar、bool类型字段。 - 适用于:**表**。 + 适用于:**表、(超级表)**。 + + 说明:从 2.1.3.0 版本开始,TWA 函数可以在由 GROUP BY 划分出单独时间线的情况下用于超级表(也即 GROUP BY tbname)。 + +- **IRATE** + ```mysql + SELECT IRATE(field_name) FROM tb_name WHERE clause; + ``` + 功能说明:计算瞬时增长率。使用时间区间中最后两个样本数据来计算瞬时增长速率;如果这两个值呈递减关系,那么只取最后一个数用于计算,而不是使用二者差值。 + + 返回结果数据类型:双精度浮点数Double。 + + 应用字段:不能应用在timestamp、binary、nchar、bool类型字段。 + + 适用于:**表、(超级表)**。 + + 说明:从 2.1.3.0 版本开始新增此函数。IRATE 可以在由 GROUP BY 划分出单独时间线的情况下用于超级表(也即 GROUP BY tbname)。 - **SUM** ```mysql From 41e1d2dd9689cd93fc1a65c88ef5f8d3f9d54953 Mon Sep 17 00:00:00 2001 From: Elias Soong Date: Fri, 25 Jun 2021 18:55:01 +0800 Subject: [PATCH 08/16] [TD-4555] : describe SQL function DERIVATIVE(). --- documentation20/cn/12.taos-sql/docs.md | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/documentation20/cn/12.taos-sql/docs.md b/documentation20/cn/12.taos-sql/docs.md index 5ab055bcda..844865f6db 100644 --- a/documentation20/cn/12.taos-sql/docs.md +++ b/documentation20/cn/12.taos-sql/docs.md @@ -870,7 +870,7 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 适用于:**表、(超级表)**。 - 说明:从 2.1.3.0 版本开始新增此函数。IRATE 可以在由 GROUP BY 划分出单独时间线的情况下用于超级表(也即 GROUP BY tbname)。 + 说明:(从 2.1.3.0 版本开始新增此函数)IRATE 可以在由 GROUP BY 划分出单独时间线的情况下用于超级表(也即 GROUP BY tbname)。 - **SUM** ```mysql @@ -1219,13 +1219,14 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 ``` ### 计算函数 + - **DIFF** ```mysql SELECT DIFF(field_name) FROM tb_name [WHERE clause]; ``` 功能说明:统计表中某列的值与前一行对应值的差。 - 返回结果数据类型: 同应用字段。 + 返回结果数据类型:同应用字段。 应用字段:不能应用在timestamp、binary、nchar、bool类型字段。 @@ -1243,13 +1244,27 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 Query OK, 2 row(s) in set (0.001162s) ``` +- **DERIVATIVE** + ```mysql + SELECT DERIVATIVE(field_name, time_interval, ignore_negative) FROM tb_name [WHERE clause]; + ``` + 功能说明:统计表中某列数值的单位变化率。其中单位时间区间的长度可以通过 time_interval 参数指定,最小可以是 1 秒(1s);ignore_negative 参数的值可以是 0 或 1,为 1 时表示忽略负值。 + + 返回结果数据类型:双精度浮点数。 + + 应用字段:不能应用在 timestamp、binary、nchar、bool 类型字段。 + + 适用于:**表、(超级表)**。 + + 说明:(从 2.1.3.0 版本开始新增此函数)输出结果行数是范围内总行数减一,第一行没有结果输出。DERIVATIVE 函数可以在由 GROUP BY 划分出单独时间线的情况下用于超级表(也即 GROUP BY tbname)。 + - **SPREAD** ```mysql SELECT SPREAD(field_name) FROM { tb_name | stb_name } [WHERE clause]; ``` 功能说明:统计表/超级表中某列的最大值和最小值之差。 - 返回结果数据类型: 双精度浮点数。 + 返回结果数据类型:双精度浮点数。 应用字段:不能应用在binary、nchar、bool类型字段。 From 11761d016d01159a0c5bb56c1b646bd9ffa996a9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 25 Jun 2021 23:10:10 +0800 Subject: [PATCH 09/16] [td-4894]:fix bug in group by normal columns. --- src/query/src/qExecutor.c | 18 ++++------- tests/script/general/parser/groupby.sim | 40 +++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 12 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 0742d3eb81..e7f63a027d 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -1282,11 +1282,8 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SGroupbyOperatorInfo *pIn return; } - int64_t* tsList = NULL; SColumnInfoData* pFirstColData = taosArrayGet(pSDataBlock->pDataBlock, 0); - if (pFirstColData->info.type == TSDB_DATA_TYPE_TIMESTAMP) { - tsList = (int64_t*) pFirstColData->pData; - } + int64_t* tsList = (pFirstColData->info.type == TSDB_DATA_TYPE_TIMESTAMP)? (int64_t*) pFirstColData->pData:NULL; STimeWindow w = TSWINDOW_INITIALIZER; @@ -1319,12 +1316,10 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SGroupbyOperatorInfo *pIn } if (pQueryAttr->stableQuery && pQueryAttr->stabledev && (pRuntimeEnv->prevResult != NULL)) { - setParamForStableStddevByColData(pRuntimeEnv, pInfo->binfo.pCtx, pOperator->numOfOutput, pOperator->pExpr, pInfo->prevData, - bytes); + setParamForStableStddevByColData(pRuntimeEnv, pInfo->binfo.pCtx, pOperator->numOfOutput, pOperator->pExpr, pInfo->prevData, bytes); } - int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, &(pInfo->binfo), pOperator->numOfOutput, pInfo->prevData, type, bytes, - item->groupIndex); + int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, &(pInfo->binfo), pOperator->numOfOutput, pInfo->prevData, type, bytes, item->groupIndex); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); } @@ -1340,17 +1335,16 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SGroupbyOperatorInfo *pIn memcpy(pInfo->prevData, val, bytes); if (pQueryAttr->stableQuery && pQueryAttr->stabledev && (pRuntimeEnv->prevResult != NULL)) { - setParamForStableStddevByColData(pRuntimeEnv, pInfo->binfo.pCtx, pOperator->numOfOutput, pOperator->pExpr, val, - bytes); + setParamForStableStddevByColData(pRuntimeEnv, pInfo->binfo.pCtx, pOperator->numOfOutput, pOperator->pExpr, val, bytes); } - int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, &(pInfo->binfo), pOperator->numOfOutput, val, type, bytes, - item->groupIndex); + int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, &(pInfo->binfo), pOperator->numOfOutput, val, type, bytes, item->groupIndex); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); } doApplyFunctions(pRuntimeEnv, pInfo->binfo.pCtx, &w, pSDataBlock->info.rows - num, num, tsList, pSDataBlock->info.rows, pOperator->numOfOutput); + tfree(pInfo->prevData); } } diff --git a/tests/script/general/parser/groupby.sim b/tests/script/general/parser/groupby.sim index 507431f536..6ae5d420d8 100644 --- a/tests/script/general/parser/groupby.sim +++ b/tests/script/general/parser/groupby.sim @@ -741,4 +741,44 @@ if $data14 != 2 then return -1 endi +sql create table m1 (ts timestamp, k int, f1 int) tags(a int); +sql create table tm0 using m1 tags(0); +sql create table tm1 using m1 tags(1); + +sql insert into tm0 values('2020-1-1 1:1:1', 1, 10); +sql insert into tm0 values('2020-1-1 1:1:2', 1, 20); +sql insert into tm1 values('2020-2-1 1:1:1', 2, 10); +sql insert into tm1 values('2020-2-1 1:1:2', 2, 20); + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +sleep 100 +system sh/exec.sh -n dnode1 -s start +sleep 100 + +sql connect +sleep 100 +sql use group_db0; + +print =========================>TD-4894 +sql select count(*),k from m1 group by k; +if $rows != 2 then + return -1 +endi + +if $data00 != 2 then + return -1 +endi + +if $data01 != 1 then + return -1 +endi + +if $data10 != 2 then + return -1 +endi + +if $data11 != 2 then + return -1 +endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT From 782e498d1fac32243305ea646f45da8ba2935e8c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 25 Jun 2021 23:22:22 +0800 Subject: [PATCH 10/16] [td-225]fix compiler error. --- src/client/src/tscSQLParser.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 815442a038..805fd9dfe4 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -2613,7 +2613,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col SColumnList ids = createColumnList(1, index.tableIndex, index.columnIndex); if (finalResult) { - insertResultField(pQueryInfo, colIndex, &ids, resultSize, resultType, pExpr->base.aliasName, pExpr); + insertResultField(pQueryInfo, colIndex, &ids, resultSize, (int8_t)resultType, pExpr->base.aliasName, pExpr); } else { assert(ids.num == 1); tscColumnListInsert(pQueryInfo->colList, ids.ids[0].columnIndex, pExpr->base.uid, pSchema); From 9094bf8849d50721dc1996d2798ad938864b97ae Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 25 Jun 2021 23:42:15 +0800 Subject: [PATCH 11/16] [td-225]fix compiler error. --- src/client/src/tscSQLParser.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 805fd9dfe4..da02d63f1e 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -7849,7 +7849,7 @@ int32_t validateSqlNode(SSqlObj* pSql, SSqlNode* pSqlNode, SQueryInfo* pQueryInf STableMeta* pTableMeta = tscGetMetaInfo(pQueryInfo, 0)->pTableMeta; SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, 0); - int32_t numOfExprs = tscNumOfExprs(pQueryInfo); + int32_t numOfExprs = (int32_t) tscNumOfExprs(pQueryInfo); if (numOfExprs == 1) { SExprInfo* pExpr = tscExprGet(pQueryInfo, 0); int32_t f = pExpr->base.functionId; From 14d9473b5a56200e43a4c86c8349b6a7bb3cb115 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 26 Jun 2021 15:43:08 +0800 Subject: [PATCH 12/16] [td-225]fix the bug found by regression test. --- src/client/src/tscSQLParser.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index da02d63f1e..cf72b7abd9 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -2104,7 +2104,7 @@ static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS // if it is not in the final result, do not add it SColumnList ids = createColumnList(1, pColIndex->tableIndex, pColIndex->columnIndex); if (finalResult) { - insertResultField(pQueryInfo, resColIdx, &ids, pSchema->bytes, (int8_t)pSchema->type, pExpr->base.aliasName, pExpr); + insertResultField(pQueryInfo, resColIdx, &ids, resBytes, (int8_t)resType, pExpr->base.aliasName, pExpr); } else { tscColumnListInsert(pQueryInfo->colList, ids.ids[0].columnIndex, pExpr->base.uid, pSchema); } From 09ae4a83f0209c4809fbbf56fd96044d40c655ec Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sat, 26 Jun 2021 22:34:50 +0800 Subject: [PATCH 13/16] Hotfix/sangshuduo/td 4892 taosdemo sub fetch for develop (#6635) * fix crash if no result file. * make taosdemo same as master branch. --- src/kit/taosdemo/taosdemo.c | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/kit/taosdemo/taosdemo.c b/src/kit/taosdemo/taosdemo.c index 0eac1518a7..b2ac11b2ca 100644 --- a/src/kit/taosdemo/taosdemo.c +++ b/src/kit/taosdemo/taosdemo.c @@ -1204,23 +1204,24 @@ static void fetchResult(TAOS_RES *res, threadInfo* pThreadInfo) { return ; } - int totalLen = 0; + int64_t totalLen = 0; // fetch the records row by row while((row = taos_fetch_row(res))) { - if ((strlen(pThreadInfo->filePath) > 0) - && (totalLen >= 100*1024*1024 - 32000)) { - appendResultBufToFile(databuf, pThreadInfo); + if (totalLen >= 100*1024*1024 - 32000) { + if (strlen(pThreadInfo->filePath) > 0) + appendResultBufToFile(databuf, pThreadInfo); totalLen = 0; memset(databuf, 0, 100*1024*1024); } num_rows++; - char temp[16000] = {0}; + char temp[16000] = {0}; int len = taos_print_row(temp, row, fields, num_fields); len += sprintf(temp + len, "\n"); //printf("query result:%s\n", temp); memcpy(databuf + totalLen, temp, len); totalLen += len; + debugPrint("totalLen: %"PRId64"\n", totalLen); } verbosePrint("%s() LN%d, databuf=%s resultFile=%s\n", From ae972e2fb44e63bf56a08441dd2db9eac3cc5169 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Sun, 27 Jun 2021 09:12:14 +0800 Subject: [PATCH 14/16] [TD-4926]:fix core during operator time usage calculate --- src/query/src/qExecutor.c | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 1cd50df46b..336925c797 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -3792,7 +3792,9 @@ void publishOperatorProfEvent(SOperatorInfo* operatorInfo, EQueryProfEventType e event.operatorType = operatorInfo->operatorType; SQInfo* qInfo = operatorInfo->pRuntimeEnv->qinfo; - taosArrayPush(qInfo->summary.queryProfEvents, &event); + if (qInfo->summary.queryProfEvents) { + taosArrayPush(qInfo->summary.queryProfEvents, &event); + } } void publishQueryAbortEvent(SQInfo* pQInfo, int32_t code) { @@ -3801,7 +3803,9 @@ void publishQueryAbortEvent(SQInfo* pQInfo, int32_t code) { event.eventTime = taosGetTimestampUs(); event.abortCode = code; - taosArrayPush(pQInfo->summary.queryProfEvents, &event); + if (pQInfo->summary.queryProfEvents) { + taosArrayPush(pQInfo->summary.queryProfEvents, &event); + } } typedef struct { @@ -3837,10 +3841,21 @@ static void doOperatorExecProfOnce(SOperatorStackItem* item, SQueryProfEvent* ev } void calculateOperatorProfResults(SQInfo* pQInfo) { + if (pQInfo->summary.queryProfEvents == NULL) { + qDebug("query prof events array is null"); + return; + } + + if (pQInfo->summary.operatorProfResults == NULL) { + qDebug("operator prof results hash is null"); + return; + } + SArray* opStack = taosArrayInit(32, sizeof(SOperatorStackItem)); if (opStack == NULL) { return; } + size_t size = taosArrayGetSize(pQInfo->summary.queryProfEvents); SHashObj* profResults = pQInfo->summary.operatorProfResults; @@ -3897,11 +3912,13 @@ void queryCostStatis(SQInfo *pQInfo) { qDebug("QInfo:0x%"PRIx64" :cost summary: winResPool size:%.2f Kb, numOfWin:%"PRId64", tableInfoSize:%.2f Kb, hashTable:%.2f Kb", pQInfo->qId, pSummary->winInfoSize/1024.0, pSummary->numOfTimeWindows, pSummary->tableInfoSize/1024.0, pSummary->hashSize/1024.0); - SOperatorProfResult* opRes = taosHashIterate(pSummary->operatorProfResults, NULL); - while (opRes != NULL) { - qDebug("QInfo:0x%"PRIx64" :cost summary: operator : %d, exec times: %"PRId64", self time: %"PRId64, pQInfo->qId, - opRes->operatorType, opRes->sumRunTimes, opRes->sumSelfTime ); - opRes = taosHashIterate(pSummary->operatorProfResults, opRes); + if (pSummary->operatorProfResults) { + SOperatorProfResult* opRes = taosHashIterate(pSummary->operatorProfResults, NULL); + while (opRes != NULL) { + qDebug("QInfo:0x%" PRIx64 " :cost summary: operator : %d, exec times: %" PRId64 ", self time: %" PRId64, + pQInfo->qId, opRes->operatorType, opRes->sumRunTimes, opRes->sumSelfTime); + opRes = taosHashIterate(pSummary->operatorProfResults, opRes); + } } } @@ -4305,8 +4322,14 @@ int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr int32_t numOfTables = (int32_t)pQueryAttr->tableGroupInfo.numOfTables; pQInfo->summary.tableInfoSize += (numOfTables * sizeof(STableQueryInfo)); pQInfo->summary.queryProfEvents = taosArrayInit(512, sizeof(SQueryProfEvent)); + if (pQInfo->summary.queryProfEvents == NULL) { + qDebug("failed to allocate query prof events array"); + } pQInfo->summary.operatorProfResults = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_TINYINT), true, HASH_NO_LOCK); + if (pQInfo->summary.operatorProfResults == NULL) { + qDebug("failed to allocate operator prof results hash"); + } code = setupQueryRuntimeEnv(pRuntimeEnv, (int32_t) pQueryAttr->tableGroupInfo.numOfTables, pOperator, param); if (code != TSDB_CODE_SUCCESS) { From 9a06248432a70b29ce76b4d77331e1056ba21726 Mon Sep 17 00:00:00 2001 From: Jun Li Date: Sun, 27 Jun 2021 06:13:33 -0700 Subject: [PATCH 15/16] Rename file and add some comment/changes (#6587) 1. rename semphone to semaphore 2. add comment for tsched.h 3. change the function signature for taosSchedulerTask, changing from return int to void. We currently don't check any return code of the function. 4. add some error handlings. For fatal error, just exit the process because the program may run into a random state. --- src/os/inc/os.h | 2 +- src/os/inc/{osSemphone.h => osSemaphore.h} | 4 +- .../darwin/{dwSemphone.c => dwSemaphore.c} | 0 .../detail/{osSemphone.c => osSemaphore.c} | 0 .../src/windows/{wSemphone.c => wSemaphore.c} | 0 src/util/inc/tsched.h | 39 +++++++++-- src/util/src/tsched.c | 67 ++++++++++++------- 7 files changed, 79 insertions(+), 33 deletions(-) rename src/os/inc/{osSemphone.h => osSemaphore.h} (97%) rename src/os/src/darwin/{dwSemphone.c => dwSemaphore.c} (100%) rename src/os/src/detail/{osSemphone.c => osSemaphore.c} (100%) rename src/os/src/windows/{wSemphone.c => wSemaphore.c} (100%) diff --git a/src/os/inc/os.h b/src/os/inc/os.h index 6731ca6d7d..903e80d5c7 100644 --- a/src/os/inc/os.h +++ b/src/os/inc/os.h @@ -29,7 +29,7 @@ extern "C" { #include "osMath.h" #include "osMemory.h" #include "osRand.h" -#include "osSemphone.h" +#include "osSemaphore.h" #include "osSignal.h" #include "osSleep.h" #include "osSocket.h" diff --git a/src/os/inc/osSemphone.h b/src/os/inc/osSemaphore.h similarity index 97% rename from src/os/inc/osSemphone.h rename to src/os/inc/osSemaphore.h index fe59095205..10d14700e0 100644 --- a/src/os/inc/osSemphone.h +++ b/src/os/inc/osSemaphore.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef TDENGINE_OS_SEMPHONE_H -#define TDENGINE_OS_SEMPHONE_H +#ifndef TDENGINE_OS_SEMAPHORE_H +#define TDENGINE_OS_SEMAPHORE_H #ifdef __cplusplus extern "C" { diff --git a/src/os/src/darwin/dwSemphone.c b/src/os/src/darwin/dwSemaphore.c similarity index 100% rename from src/os/src/darwin/dwSemphone.c rename to src/os/src/darwin/dwSemaphore.c diff --git a/src/os/src/detail/osSemphone.c b/src/os/src/detail/osSemaphore.c similarity index 100% rename from src/os/src/detail/osSemphone.c rename to src/os/src/detail/osSemaphore.c diff --git a/src/os/src/windows/wSemphone.c b/src/os/src/windows/wSemaphore.c similarity index 100% rename from src/os/src/windows/wSemphone.c rename to src/os/src/windows/wSemaphore.c diff --git a/src/util/inc/tsched.h b/src/util/inc/tsched.h index 3e481cbc32..a1591512c1 100644 --- a/src/util/inc/tsched.h +++ b/src/util/inc/tsched.h @@ -28,10 +28,41 @@ typedef struct SSchedMsg { void *thandle; } SSchedMsg; -void *taosInitScheduler(int queueSize, int numOfThreads, const char *label); -void *taosInitSchedulerWithInfo(int queueSize, int numOfThreads, const char *label, void *tmrCtrl); -int taosScheduleTask(void *qhandle, SSchedMsg *pMsg); -void taosCleanUpScheduler(void *param); +/** + * Create a thread-safe ring-buffer based task queue and return the instance. A thread + * pool will be created to consume the messages in the queue. + * @param capacity the queue capacity + * @param numOfThreads the number of threads for the thread pool + * @param label the label of the queue + * @return the created queue scheduler + */ +void *taosInitScheduler(int capacity, int numOfThreads, const char *label); + +/** + * Create a thread-safe ring-buffer based task queue and return the instance. + * Same as taosInitScheduler, and it also print the queue status every 1 minite. + * @param capacity the queue capacity + * @param numOfThreads the number of threads for the thread pool + * @param label the label of the queue + * @param tmrCtrl the timer controller, tmr_ctrl_t* + * @return the created queue scheduler + */ +void *taosInitSchedulerWithInfo(int capacity, int numOfThreads, const char *label, void *tmrCtrl); + +/** + * Clean up the queue scheduler instance and free the memory. + * @param queueScheduler the queue scheduler to free + */ +void taosCleanUpScheduler(void *queueScheduler); + +/** + * Schedule a new task to run, the task is described by pMsg. + * The function may be blocked if no thread is available to execute the task. + * That may happen when all threads are busy. + * @param queueScheduler the queue scheduler instance + * @param pMsg the message for the task + */ +void taosScheduleTask(void *queueScheduler, SSchedMsg *pMsg); #ifdef __cplusplus } diff --git a/src/util/src/tsched.c b/src/util/src/tsched.c index f014dd0fab..16142470c9 100644 --- a/src/util/src/tsched.c +++ b/src/util/src/tsched.c @@ -108,39 +108,47 @@ void *taosInitScheduler(int queueSize, int numOfThreads, const char *label) { void *taosInitSchedulerWithInfo(int queueSize, int numOfThreads, const char *label, void *tmrCtrl) { SSchedQueue* pSched = taosInitScheduler(queueSize, numOfThreads, label); - + if (tmrCtrl != NULL && pSched != NULL) { pSched->pTmrCtrl = tmrCtrl; taosTmrReset(taosDumpSchedulerStatus, DUMP_SCHEDULER_TIME_WINDOW, pSched, pSched->pTmrCtrl, &pSched->pTimer); } - + return pSched; } -void *taosProcessSchedQueue(void *param) { +void *taosProcessSchedQueue(void *scheduler) { SSchedMsg msg; - SSchedQueue *pSched = (SSchedQueue *)param; + SSchedQueue *pSched = (SSchedQueue *)scheduler; + int ret = 0; while (1) { - if (tsem_wait(&pSched->fullSem) != 0) { - uError("wait %s fullSem failed(%s)", pSched->label, strerror(errno)); + if ((ret = tsem_wait(&pSched->fullSem)) != 0) { + uFatal("wait %s fullSem failed(%s)", pSched->label, strerror(errno)); + exit(ret); } if (pSched->stop) { break; } - if (pthread_mutex_lock(&pSched->queueMutex) != 0) - uError("lock %s queueMutex failed(%s)", pSched->label, strerror(errno)); + if ((ret = pthread_mutex_lock(&pSched->queueMutex)) != 0) { + uFatal("lock %s queueMutex failed(%s)", pSched->label, strerror(errno)); + exit(ret); + } msg = pSched->queue[pSched->fullSlot]; memset(pSched->queue + pSched->fullSlot, 0, sizeof(SSchedMsg)); pSched->fullSlot = (pSched->fullSlot + 1) % pSched->queueSize; - if (pthread_mutex_unlock(&pSched->queueMutex) != 0) - uError("unlock %s queueMutex failed(%s)", pSched->label, strerror(errno)); + if ((ret = pthread_mutex_unlock(&pSched->queueMutex)) != 0) { + uFatal("unlock %s queueMutex failed(%s)", pSched->label, strerror(errno)); + exit(ret); + } - if (tsem_post(&pSched->emptySem) != 0) - uError("post %s emptySem failed(%s)", pSched->label, strerror(errno)); + if ((ret = tsem_post(&pSched->emptySem)) != 0) { + uFatal("post %s emptySem failed(%s)", pSched->label, strerror(errno)); + exit(ret); + } if (msg.fp) (*(msg.fp))(&msg); @@ -151,30 +159,37 @@ void *taosProcessSchedQueue(void *param) { return NULL; } -int taosScheduleTask(void *qhandle, SSchedMsg *pMsg) { - SSchedQueue *pSched = (SSchedQueue *)qhandle; +void taosScheduleTask(void *queueScheduler, SSchedMsg *pMsg) { + SSchedQueue *pSched = (SSchedQueue *)queueScheduler; + int ret = 0; + if (pSched == NULL) { uError("sched is not ready, msg:%p is dropped", pMsg); - return 0; + return; } - if (tsem_wait(&pSched->emptySem) != 0) { - uError("wait %s emptySem failed(%s)", pSched->label, strerror(errno)); + if ((ret = tsem_wait(&pSched->emptySem)) != 0) { + uFatal("wait %s emptySem failed(%s)", pSched->label, strerror(errno)); + exit(ret); } - if (pthread_mutex_lock(&pSched->queueMutex) != 0) - uError("lock %s queueMutex failed(%s)", pSched->label, strerror(errno)); + if ((ret = pthread_mutex_lock(&pSched->queueMutex)) != 0) { + uFatal("lock %s queueMutex failed(%s)", pSched->label, strerror(errno)); + exit(ret); + } pSched->queue[pSched->emptySlot] = *pMsg; pSched->emptySlot = (pSched->emptySlot + 1) % pSched->queueSize; - if (pthread_mutex_unlock(&pSched->queueMutex) != 0) - uError("unlock %s queueMutex failed(%s)", pSched->label, strerror(errno)); + if ((ret = pthread_mutex_unlock(&pSched->queueMutex)) != 0) { + uFatal("unlock %s queueMutex failed(%s)", pSched->label, strerror(errno)); + exit(ret); + } - if (tsem_post(&pSched->fullSem) != 0) - uError("post %s fullSem failed(%s)", pSched->label, strerror(errno)); - - return 0; + if ((ret = tsem_post(&pSched->fullSem)) != 0) { + uFatal("post %s fullSem failed(%s)", pSched->label, strerror(errno)); + exit(ret); + } } void taosCleanUpScheduler(void *param) { @@ -219,4 +234,4 @@ void taosDumpSchedulerStatus(void *qhandle, void *tmrId) { } taosTmrReset(taosDumpSchedulerStatus, DUMP_SCHEDULER_TIME_WINDOW, pSched, pSched->pTmrCtrl, &pSched->pTimer); -} +} \ No newline at end of file From f48783df4f7b20bf291f6ba9bb4e6a86b66d4040 Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Sun, 27 Jun 2021 22:45:38 +0800 Subject: [PATCH 16/16] fix converity scan issue. (#6641) --- src/kit/taosdump/taosdump.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/kit/taosdump/taosdump.c b/src/kit/taosdump/taosdump.c index 05c6b1efbb..33118ce311 100644 --- a/src/kit/taosdump/taosdump.c +++ b/src/kit/taosdump/taosdump.c @@ -1126,7 +1126,7 @@ int taosGetTableDes( strncpy(tableDes->cols[count].field, (char *)row[TSDB_DESCRIBE_METRIC_FIELD_INDEX], fields[TSDB_DESCRIBE_METRIC_FIELD_INDEX].bytes); strncpy(tableDes->cols[count].type, (char *)row[TSDB_DESCRIBE_METRIC_TYPE_INDEX], - min(16, fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes)); + min(15, fields[TSDB_DESCRIBE_METRIC_TYPE_INDEX].bytes)); tableDes->cols[count].length = *((int *)row[TSDB_DESCRIBE_METRIC_LENGTH_INDEX]); strncpy(tableDes->cols[count].note, (char *)row[TSDB_DESCRIBE_METRIC_NOTE_INDEX], fields[TSDB_DESCRIBE_METRIC_NOTE_INDEX].bytes);