From a39e6f7d92a4da91abf1827755148b6c3a7a718a Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Mon, 29 Aug 2022 14:01:18 +0800 Subject: [PATCH 1/5] enh: super table scan requires project operator to merge packets to improve performance --- source/client/inc/clientInt.h | 1 + source/client/src/clientEnv.c | 5 +++-- source/client/src/clientImpl.c | 2 ++ source/libs/planner/src/planOptimizer.c | 5 ++++- source/libs/planner/test/planTestUtil.cpp | 4 ++-- 5 files changed, 12 insertions(+), 5 deletions(-) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 07e5f75e87..d57175d31b 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -101,6 +101,7 @@ typedef struct SQueryExecMetric { int64_t ctgStart; // start to parse, us int64_t ctgEnd; // end to parse, us int64_t semanticEnd; + int64_t planEnd; int64_t execEnd; int64_t send; // start to send to server, us int64_t rsp; // receive response from server, us diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 1342e89b52..9117287b32 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -83,11 +83,12 @@ static void deregisterRequest(SRequestObj *pRequest) { pRequest->metric.execEnd - pRequest->metric.semanticEnd); atomic_add_fetch_64((int64_t *)&pActivity->insertElapsedTime, duration); } else if (QUERY_NODE_SELECT_STMT == pRequest->stmtType) { - tscPerf("select duration %" PRId64 "us: syntax:%" PRId64 "us, ctg:%" PRId64 "us, semantic:%" PRId64 "us, exec:%" PRId64 "us", + tscPerf("select duration %" PRId64 "us: syntax:%" PRId64 "us, ctg:%" PRId64 "us, semantic:%" PRId64 "us, planner:%" PRId64 "us, exec:%" PRId64 "us", duration, pRequest->metric.syntaxEnd - pRequest->metric.syntaxStart, pRequest->metric.ctgEnd - pRequest->metric.ctgStart, pRequest->metric.semanticEnd - pRequest->metric.ctgEnd, - pRequest->metric.execEnd - pRequest->metric.semanticEnd); + pRequest->metric.planEnd - pRequest->metric.semanticEnd, + nowUs - pRequest->metric.semanticEnd); atomic_add_fetch_64((int64_t *)&pActivity->queryElapsedTime, duration); } diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 2baca1288f..97429e099b 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1030,6 +1030,8 @@ void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultM pRequest->body.subplanNum = pDag->numOfSubplans; } + pRequest->metric.planEnd = taosGetTimestampUs(); + if (TSDB_CODE_SUCCESS == code && !pRequest->validateOnly) { SArray* pNodeList = NULL; buildAsyncExecNodeList(pRequest, &pNodeList, pMnodeList, pResultMeta); diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 45ab3903a9..120b33dd13 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1665,7 +1665,10 @@ static bool eliminateProjOptMayBeOptimized(SLogicNode* pNode) { return false; } - if (QUERY_NODE_LOGIC_PLAN_PROJECT != nodeType(pNode) || 1 != LIST_LENGTH(pNode->pChildren)) { + // Super table scan requires project operator to merge packets to improve performance. + if (QUERY_NODE_LOGIC_PLAN_PROJECT != nodeType(pNode) || 1 != LIST_LENGTH(pNode->pChildren) || + (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(nodesListGetNode(pNode->pChildren, 0)) && + TSDB_SUPER_TABLE == ((SScanLogicNode*)nodesListGetNode(pNode->pChildren, 0))->tableType)) { return false; } diff --git a/source/libs/planner/test/planTestUtil.cpp b/source/libs/planner/test/planTestUtil.cpp index f904643be9..96f7d29230 100644 --- a/source/libs/planner/test/planTestUtil.cpp +++ b/source/libs/planner/test/planTestUtil.cpp @@ -278,12 +278,12 @@ class PlannerTestBaseImpl { } void dump(DumpModule module) { + cout << "========================================== " << sqlNo_ << " sql : [" << stmtEnv_.sql_ << "]" << endl; + if (DUMP_MODULE_NOTHING == module) { return; } - cout << "========================================== " << sqlNo_ << " sql : [" << stmtEnv_.sql_ << "]" << endl; - if (DUMP_MODULE_ALL == module || DUMP_MODULE_PARSER == module) { if (res_.prepareAst_.empty()) { cout << "+++++++++++++++++++++syntax tree : " << endl; From a861537bc075f8b1dc2ebccd4ad4cdfaa7ceb1a6 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Mon, 29 Aug 2022 15:44:10 +0800 Subject: [PATCH 2/5] fix: fix filter group merge error --- source/libs/scalar/inc/filterInt.h | 2 +- source/libs/scalar/src/filter.c | 82 ++++++++++++++++++++++++----- tests/script/jenkins/basic.txt | 1 + tests/script/tsim/scalar/filter.sim | 38 +++++++++++++ 4 files changed, 110 insertions(+), 13 deletions(-) create mode 100644 tests/script/tsim/scalar/filter.sim diff --git a/source/libs/scalar/inc/filterInt.h b/source/libs/scalar/inc/filterInt.h index 23693c785a..23d88d1a26 100644 --- a/source/libs/scalar/inc/filterInt.h +++ b/source/libs/scalar/inc/filterInt.h @@ -276,7 +276,7 @@ struct SFilterInfo { #define FILTER_CLR_FLAG(st, f) st &= (~f) #define SIMPLE_COPY_VALUES(dst, src) *((int64_t *)dst) = *((int64_t *)src) -#define FILTER_PACKAGE_UNIT_HASH_KEY(v, optr, idx1, idx2) do { char *_t = (char *)v; _t[0] = optr; *(uint32_t *)(_t + 1) = idx1; *(uint32_t *)(_t + 3) = idx2; } while (0) +#define FLT_PACKAGE_UNIT_HASH_KEY(v, op1, op2, lidx, ridx, ridx2) do { char *_t = (char *)(v); _t[0] = (op1); _t[1] = (op2); *(uint32_t *)(_t + 2) = (lidx); *(uint32_t *)(_t + 2 + sizeof(uint32_t)) = (ridx); } while (0) #define FILTER_GREATER(cr,sflag,eflag) ((cr > 0) || ((cr == 0) && (FILTER_GET_FLAG(sflag,RANGE_FLG_EXCLUDE) || FILTER_GET_FLAG(eflag,RANGE_FLG_EXCLUDE)))) #define FILTER_COPY_RA(dst, src) do { (dst)->sflag = (src)->sflag; (dst)->eflag = (src)->eflag; (dst)->s = (src)->s; (dst)->e = (src)->e; } while (0) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index 4377dbf14e..b27a61b8bd 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -910,14 +910,14 @@ int32_t filterAddFieldFromNode(SFilterInfo *info, SNode *node, SFilterFieldId *f return TSDB_CODE_SUCCESS; } -int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFilterFieldId *right, uint32_t *uidx) { +int32_t filterAddUnitImpl(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFilterFieldId *right, uint8_t optr2, SFilterFieldId *right2, uint32_t *uidx) { if (FILTER_GET_FLAG(info->options, FLT_OPTION_NEED_UNIQE)) { if (info->pctx.unitHash == NULL) { info->pctx.unitHash = taosHashInit(FILTER_DEFAULT_GROUP_SIZE * FILTER_DEFAULT_UNIT_SIZE, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, false); } else { - int64_t v = 0; - FILTER_PACKAGE_UNIT_HASH_KEY(&v, optr, left->idx, right ? right->idx : -1); - void *hu = taosHashGet(info->pctx.unitHash, &v, sizeof(v)); + char v[14] = {0}; + FLT_PACKAGE_UNIT_HASH_KEY(&v, optr, optr2, left->idx, (right ? right->idx : -1), (right2 ? right2->idx : -1)); + void *hu = taosHashGet(info->pctx.unitHash, v, sizeof(v)); if (hu) { *uidx = *(uint32_t *)hu; return TSDB_CODE_SUCCESS; @@ -939,7 +939,11 @@ int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFi if (right) { u->right = *right; } - + u->compare.optr2 = optr2; + if (right2) { + u->right2 = *right2; + } + if (u->right.type == FLD_TYPE_VALUE) { SFilterField *val = FILTER_UNIT_RIGHT_FIELD(info, u); assert(FILTER_GET_FLAG(val->flag, FLD_TYPE_VALUE)); @@ -960,9 +964,9 @@ int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFi *uidx = info->unitNum; if (FILTER_GET_FLAG(info->options, FLT_OPTION_NEED_UNIQE)) { - int64_t v = 0; - FILTER_PACKAGE_UNIT_HASH_KEY(&v, optr, left->idx, right ? right->idx : -1); - taosHashPut(info->pctx.unitHash, &v, sizeof(v), uidx, sizeof(*uidx)); + char v[14] = {0}; + FLT_PACKAGE_UNIT_HASH_KEY(&v, optr, optr2, left->idx, (right ? right->idx : -1), (right2 ? right2->idx : -1)); + taosHashPut(info->pctx.unitHash, v, sizeof(v), uidx, sizeof(*uidx)); } ++info->unitNum; @@ -971,6 +975,9 @@ int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFi } +int32_t filterAddUnit(SFilterInfo *info, uint8_t optr, SFilterFieldId *left, SFilterFieldId *right, uint32_t *uidx) { + return filterAddUnitImpl(info, optr, left, right, 0, NULL, uidx); +} int32_t filterAddUnitToGroup(SFilterGroup *group, uint32_t unitIdx) { if (group->unitNum >= group->unitSize) { @@ -1147,8 +1154,8 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan SIMPLE_COPY_VALUES(data2, &ra->e); filterAddField(dst, NULL, &data2, FLD_TYPE_VALUE, &right2, tDataTypes[type].bytes, true); - filterAddUnit(dst, FILTER_GET_FLAG(ra->sflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_GREATER_THAN : OP_TYPE_GREATER_EQUAL, &left, &right, &uidx); - filterAddUnitRight(dst, FILTER_GET_FLAG(ra->eflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_LOWER_THAN : OP_TYPE_LOWER_EQUAL, &right2, uidx); + filterAddUnitImpl(dst, FILTER_GET_FLAG(ra->sflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_GREATER_THAN : OP_TYPE_GREATER_EQUAL, &left, &right, + FILTER_GET_FLAG(ra->eflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_LOWER_THAN : OP_TYPE_LOWER_EQUAL, &right2, &uidx); filterAddUnitToGroup(g, uidx); return TSDB_CODE_SUCCESS; } @@ -1222,8 +1229,8 @@ int32_t filterAddGroupUnitFromCtx(SFilterInfo *dst, SFilterInfo *src, SFilterRan SIMPLE_COPY_VALUES(data2, &r->ra.e); filterAddField(dst, NULL, &data2, FLD_TYPE_VALUE, &right2, tDataTypes[type].bytes, true); - filterAddUnit(dst, FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_GREATER_THAN : OP_TYPE_GREATER_EQUAL, &left, &right, &uidx); - filterAddUnitRight(dst, FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_LOWER_THAN : OP_TYPE_LOWER_EQUAL, &right2, uidx); + filterAddUnitImpl(dst, FILTER_GET_FLAG(r->ra.sflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_GREATER_THAN : OP_TYPE_GREATER_EQUAL, &left, &right, + FILTER_GET_FLAG(r->ra.eflag, RANGE_FLG_EXCLUDE) ? OP_TYPE_LOWER_THAN : OP_TYPE_LOWER_EQUAL, &right2, &uidx); filterAddUnitToGroup(g, uidx); } @@ -2073,6 +2080,44 @@ int32_t filterMergeGroupUnits(SFilterInfo *info, SFilterGroupCtx** gRes, int32_t return TSDB_CODE_SUCCESS; } +bool filterIsSameUnits(SFilterColInfo* pCol1, SFilterColInfo* pCol2) { + if (pCol1->type != pCol2->type) { + return false; + } + + if (RANGE_TYPE_MR_CTX == pCol1->type) { + SFilterRangeCtx* pCtx1 = (SFilterRangeCtx*)pCol1->info; + SFilterRangeCtx* pCtx2 = (SFilterRangeCtx*)pCol2->info; + + if ((pCtx1->isnull != pCtx2->isnull) || (pCtx1->notnull != pCtx2->notnull) || (pCtx1->isrange != pCtx2->isrange)) { + return false; + } + + + SFilterRangeNode* pNode1 = pCtx1->rs; + SFilterRangeNode* pNode2 = pCtx2->rs; + + while (true) { + if (NULL == pNode1 && NULL == pNode2) { + break; + } + + if (NULL == pNode1 || NULL == pNode2) { + return false; + } + + if (pNode1->ra.s != pNode2->ra.s || pNode1->ra.e != pNode2->ra.e || pNode1->ra.sflag != pNode2->ra.sflag || pNode1->ra.eflag != pNode2->ra.eflag) { + return false; + } + + pNode1 = pNode1->next; + pNode2 = pNode2->next; + } + } + + return true; +} + void filterCheckColConflict(SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool *conflict) { uint32_t idx1 = 0, idx2 = 0, m = 0, n = 0; bool equal = false; @@ -2098,6 +2143,11 @@ void filterCheckColConflict(SFilterGroupCtx* gRes1, SFilterGroupCtx* gRes2, bool return; } + if (!filterIsSameUnits(&gRes1->colInfo[idx1], &gRes2->colInfo[idx2])) { + *conflict = true; + return; + } + // for long in operation if (gRes1->colInfo[idx1].optr == OP_TYPE_EQUAL && gRes2->colInfo[idx2].optr == OP_TYPE_EQUAL) { SFilterRangeCtx* ctx = gRes1->colInfo[idx1].info; @@ -2711,17 +2761,22 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SColumnDataAgg *pDataStatis, int3 for (uint32_t g = 0; g < info->groupNum; ++g) { SFilterGroup *group = &info->groups[g]; + // first is block unint num for a group, following append unitNum blkUnitIdx for this group *unitNum = group->unitNum; all = 0; empty = 0; + // save group idx start pointer + uint32_t * pGroupIdx = unitIdx; for (uint32_t u = 0; u < group->unitNum; ++u) { uint32_t uidx = group->unitIdxs[u]; if (info->blkUnitRes[uidx] == 1) { + // blkUnitRes == 1 is always true, so need not compare every time, delete this unit from group --(*unitNum); all = 1; continue; } else if (info->blkUnitRes[uidx] == -1) { + // blkUnitRes == -1 is alwary false, so in group is alwary false, need delete this group from blkGroupNum *unitNum = 0; empty = 1; break; @@ -2731,6 +2786,9 @@ int32_t filterRmUnitByRange(SFilterInfo *info, SColumnDataAgg *pDataStatis, int3 } if (*unitNum == 0) { + // if unit num is zero, reset unitIdx to start on this group + unitIdx = pGroupIdx; + --info->blkGroupNum; assert(empty || all); diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 97295d75e0..46bae734ea 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -344,6 +344,7 @@ # --- scalar ---- ./test.sh -f tsim/scalar/in.sim ./test.sh -f tsim/scalar/scalar.sim +./test.sh -f tsim/scalar/filter.sim # ---- alter ---- ./test.sh -f tsim/alter/cached_schema_after_alter.sim diff --git a/tests/script/tsim/scalar/filter.sim b/tests/script/tsim/scalar/filter.sim new file mode 100644 index 0000000000..9231662278 --- /dev/null +++ b/tests/script/tsim/scalar/filter.sim @@ -0,0 +1,38 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print ======== step1 +sql drop database if exists db1; +sql create database db1 vgroups 3; +sql use db1; +sql create stable st1 (fts timestamp, fbool bool, ftiny tinyint, fsmall smallint, fint int, fbig bigint, futiny tinyint unsigned, fusmall smallint unsigned, fuint int unsigned, fubig bigint unsigned, ffloat float, fdouble double, fbin binary(10), fnchar nchar(10)) tags(tts timestamp, tbool bool, ttiny tinyint, tsmall smallint, tint int, tbig bigint, tutiny tinyint unsigned, tusmall smallint unsigned, tuint int unsigned, tubig bigint unsigned, tfloat float, tdouble double, tbin binary(10), tnchar nchar(10)); +sql create table tb1 using st1 tags('2022-07-10 16:31:00', true, 1, 1, 1, 1, 1, 1, 1, 1, 1.0, 1.0, 'a', 'a'); +sql create table tb2 using st1 tags('2022-07-10 16:32:00', false, 2, 2, 2, 2, 2, 2, 2, 2, 2.0, 2.0, 'b', 'b'); +sql create table tb3 using st1 tags('2022-07-10 16:33:00', true, 3, 3, 3, 3, 3, 3, 3, 3, 3.0, 3.0, 'c', 'c'); + +sql insert into tb1 values ('2022-07-10 16:31:01', false, 1, 1, 1, 1, 1, 1, 1, 1, 1.0, 1.0, 'a', 'a'); +sql insert into tb1 values ('2022-07-10 16:31:02', true, 2, 2, 2, 2, 2, 2, 2, 2, 2.0, 2.0, 'b', 'b'); +sql insert into tb1 values ('2022-07-10 16:31:03', false, 3, 3, 3, 3, 3, 3, 3, 3, 3.0, 3.0, 'c', 'c'); +sql insert into tb1 values ('2022-07-10 16:31:04', true, 4, 4, 4, 4, 4, 4, 4, 4, 4.0, 4.0, 'd', 'd'); +sql insert into tb1 values ('2022-07-10 16:31:05', false, 5, 5, 5, 5, 5, 5, 5, 5, 5.0, 5.0, 'e', 'e'); + +sql insert into tb2 values ('2022-07-10 16:32:01', false, 1, 1, 1, 1, 1, 1, 1, 1, 1.0, 1.0, 'a', 'a'); +sql insert into tb2 values ('2022-07-10 16:32:02', true, 2, 2, 2, 2, 2, 2, 2, 2, 2.0, 2.0, 'b', 'b'); +sql insert into tb2 values ('2022-07-10 16:32:03', false, 3, 3, 3, 3, 3, 3, 3, 3, 3.0, 3.0, 'c', 'c'); +sql insert into tb2 values ('2022-07-10 16:32:04', true, 4, 4, 4, 4, 4, 4, 4, 4, 4.0, 4.0, 'd', 'd'); +sql insert into tb2 values ('2022-07-10 16:32:05', false, 5, 5, 5, 5, 5, 5, 5, 5, 5.0, 5.0, 'e', 'e'); + +sql insert into tb3 values ('2022-07-10 16:33:01', false, 1, 1, 1, 1, 1, 1, 1, 1, 1.0, 1.0, 'a', 'a'); +sql insert into tb3 values ('2022-07-10 16:33:02', true, 2, 2, 2, 2, 2, 2, 2, 2, 2.0, 2.0, 'b', 'b'); +sql insert into tb3 values ('2022-07-10 16:33:03', false, 3, 3, 3, 3, 3, 3, 3, 3, 3.0, 3.0, 'c', 'c'); +sql insert into tb3 values ('2022-07-10 16:33:04', true, 4, 4, 4, 4, 4, 4, 4, 4, 4.0, 4.0, 'd', 'd'); +sql insert into tb3 values ('2022-07-10 16:33:05', false, 5, 5, 5, 5, 5, 5, 5, 5, 5.0, 5.0, 'e', 'e'); + +sql select * from st1 where (ttiny > 2 or ftiny < 5) and ftiny > 2; +if $rows != 7 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT From cc1d37cfe5881e38a85841a893f8d1746e518a27 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Mon, 29 Aug 2022 17:23:29 +0800 Subject: [PATCH 3/5] fix: some problems of parser --- source/common/src/tvariant.c | 8 ++-- source/libs/parser/src/parInsert.c | 4 ++ source/libs/parser/src/parTranslater.c | 53 +++++++++++++++++++++++++- source/util/src/terror.c | 2 +- 4 files changed, 60 insertions(+), 7 deletions(-) diff --git a/source/common/src/tvariant.c b/source/common/src/tvariant.c index 0810be1497..a01c393441 100644 --- a/source/common/src/tvariant.c +++ b/source/common/src/tvariant.c @@ -155,8 +155,8 @@ void taosVariantCreateFromBinary(SVariant *pVar, const char *pz, size_t len, uin void taosVariantDestroy(SVariant *pVar) { if (pVar == NULL) return; - if (pVar->nType == TSDB_DATA_TYPE_BINARY || pVar->nType == TSDB_DATA_TYPE_NCHAR - || pVar->nType == TSDB_DATA_TYPE_JSON) { + if (pVar->nType == TSDB_DATA_TYPE_BINARY || pVar->nType == TSDB_DATA_TYPE_NCHAR || + pVar->nType == TSDB_DATA_TYPE_JSON) { taosMemoryFreeClear(pVar->pz); pVar->nLen = 0; } @@ -185,8 +185,8 @@ void taosVariantAssign(SVariant *pDst, const SVariant *pSrc) { if (pSrc == NULL || pDst == NULL) return; pDst->nType = pSrc->nType; - if (pSrc->nType == TSDB_DATA_TYPE_BINARY || pSrc->nType == TSDB_DATA_TYPE_NCHAR - || pSrc->nType == TSDB_DATA_TYPE_JSON) { + if (pSrc->nType == TSDB_DATA_TYPE_BINARY || pSrc->nType == TSDB_DATA_TYPE_NCHAR || + pSrc->nType == TSDB_DATA_TYPE_JSON) { int32_t len = pSrc->nLen + TSDB_NCHAR_SIZE; char *p = taosMemoryRealloc(pDst->pz, len); assert(p); diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index c9115d90e1..56fbafe76d 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -502,6 +502,10 @@ static int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int return func(pMsgBuf, NULL, 0, param); } + if (IS_NUMERIC_TYPE(pSchema->type) && pToken->n == 0) { + return buildSyntaxErrMsg(pMsgBuf, "invalid numeric data", pToken->z); + } + switch (pSchema->type) { case TSDB_DATA_TYPE_BOOL: { if ((pToken->type == TK_NK_BOOL || pToken->type == TK_NK_STRING) && (pToken->n != 0)) { diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 5a32c87fc3..fbffaba3f0 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2475,13 +2475,62 @@ static int32_t translateOrderBy(STranslateContext* pCxt, SSelectStmt* pSelect) { return code; } +static EDealRes needFillImpl(SNode* pNode, void* pContext) { + if (isAggFunc(pNode)) { + *(bool*)pContext = true; + return DEAL_RES_END; + } + return DEAL_RES_CONTINUE; +} + +static bool needFill(SNode* pNode) { + bool hasFillFunc = false; + nodesWalkExpr(pNode, needFillImpl, &hasFillFunc); + return hasFillFunc; +} + +static bool mismatchFillDataType(SDataType origDt, SDataType fillDt) { + if (IS_NUMERIC_TYPE(origDt.type) && !IS_NUMERIC_TYPE(fillDt.type)) { + return true; + } + if (IS_VAR_DATA_TYPE(origDt.type) && !IS_VAR_DATA_TYPE(fillDt.type)) { + return true; + } + return false; +} + +static int32_t checkFillValues(STranslateContext* pCxt, SFillNode* pFill, SNodeList* pProjectionList) { + if (FILL_MODE_VALUE != pFill->mode) { + return TSDB_CODE_SUCCESS; + } + + int32_t fillNo = 0; + SNodeListNode* pFillValues = (SNodeListNode*)pFill->pValues; + SNode* pProject = NULL; + FOREACH(pProject, pProjectionList) { + if (needFill(pProject)) { + if (fillNo >= LIST_LENGTH(pFillValues->pNodeList)) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Filled values number mismatch"); + } + if (mismatchFillDataType(((SExprNode*)pProject)->resType, + ((SExprNode*)nodesListGetNode(pFillValues->pNodeList, fillNo))->resType)) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Filled data type mismatch"); + } + ++fillNo; + } + } + if (fillNo != LIST_LENGTH(pFillValues->pNodeList)) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Filled values number mismatch"); + } + return TSDB_CODE_SUCCESS; +} + static int32_t translateFillValues(STranslateContext* pCxt, SSelectStmt* pSelect) { if (NULL == pSelect->pWindow || QUERY_NODE_INTERVAL_WINDOW != nodeType(pSelect->pWindow) || NULL == ((SIntervalWindowNode*)pSelect->pWindow)->pFill) { return TSDB_CODE_SUCCESS; } - SFillNode* pFill = (SFillNode*)((SIntervalWindowNode*)pSelect->pWindow)->pFill; - return TSDB_CODE_SUCCESS; + return checkFillValues(pCxt, (SFillNode*)((SIntervalWindowNode*)pSelect->pWindow)->pFill, pSelect->pProjectionList); } static int32_t rewriteProjectAlias(SNodeList* pProjectionList) { diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 662a3f0c88..f99d126892 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -121,7 +121,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_CONN_KILLED, "Connection killed") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_SQL_SYNTAX_ERROR, "Syntax error in SQL") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_DB_NOT_SELECTED, "Database not specified or available") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_TABLE_NAME, "Table does not exist") -TAOS_DEFINE_ERROR(TSDB_CODE_TSC_EXCEED_SQL_LIMIT, "SQL statement too long, check maxSQLLength config") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_EXCEED_SQL_LIMIT, "SQL statement too long") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_FILE_EMPTY, "File is empty") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_LINE_SYNTAX_ERROR, "Syntax error in Line") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NO_META_CACHED, "No table meta cached") From b5e177f151cabb00a5214ce4918401041c36e069 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Mon, 29 Aug 2022 18:52:00 +0800 Subject: [PATCH 4/5] fix: some problems of parser --- source/libs/parser/src/parTranslater.c | 5 +- tests/script/tsim/compute/interval.sim | 4 +- .../script/tsim/parser/columnValue_bigint.sim | 4 +- .../script/tsim/parser/columnValue_double.sim | 4 +- .../script/tsim/parser/columnValue_float.sim | 4 +- tests/script/tsim/parser/columnValue_int.sim | 4 +- .../tsim/parser/columnValue_smallint.sim | 4 +- .../tsim/parser/columnValue_tinyint.sim | 4 +- tests/script/tsim/parser/fill.sim | 77 +++++-------------- tests/script/tsim/parser/fill_us.sim | 66 ++++------------ tests/script/tsim/parser/slimit_query.sim | 10 +-- tests/script/tsim/parser/timestamp_query.sim | 4 +- tests/script/tsim/valgrind/checkError6.sim | 18 ++--- tests/system-test/2-query/last_row.py | 6 +- 14 files changed, 70 insertions(+), 144 deletions(-) diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index fbffaba3f0..b6b677858d 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -2476,7 +2476,7 @@ static int32_t translateOrderBy(STranslateContext* pCxt, SSelectStmt* pSelect) { } static EDealRes needFillImpl(SNode* pNode, void* pContext) { - if (isAggFunc(pNode)) { + if (isAggFunc(pNode) && FUNCTION_TYPE_GROUP_KEY != ((SFunctionNode*)pNode)->funcType) { *(bool*)pContext = true; return DEAL_RES_END; } @@ -2490,6 +2490,9 @@ static bool needFill(SNode* pNode) { } static bool mismatchFillDataType(SDataType origDt, SDataType fillDt) { + if (TSDB_DATA_TYPE_NULL == fillDt.type) { + return false; + } if (IS_NUMERIC_TYPE(origDt.type) && !IS_NUMERIC_TYPE(fillDt.type)) { return true; } diff --git a/tests/script/tsim/compute/interval.sim b/tests/script/tsim/compute/interval.sim index 4e7960ac4a..dc11c20ec9 100644 --- a/tests/script/tsim/compute/interval.sim +++ b/tests/script/tsim/compute/interval.sim @@ -101,7 +101,7 @@ $ms = 1601481600000 + $cc $cc = 1 * 60000 $ms2 = 1601481600000 - $cc -sql select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms and ts > $ms2 interval(1m) fill(value,0) +sql select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), count(tbcol) from $tb where ts <= $ms and ts > $ms2 interval(1m) fill(value,0,0,0,0,0) print ===> $rows if $rows < 30 then print expect greater than 30, actual: $rows @@ -180,7 +180,7 @@ $ms1 = 1601481600000 + $cc $cc = 1 * 60000 $ms2 = 1601481600000 - $cc -sql select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms1 and ts > $ms2 interval(1m) fill(value, 0) +sql select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), count(tbcol) from $mt where ts <= $ms1 and ts > $ms2 interval(1m) fill(value, 0,0,0,0,0) if $rows < 30 then return -1 endi diff --git a/tests/script/tsim/parser/columnValue_bigint.sim b/tests/script/tsim/parser/columnValue_bigint.sim index 2cf0151a05..0a024029a5 100644 --- a/tests/script/tsim/parser/columnValue_bigint.sim +++ b/tests/script/tsim/parser/columnValue_bigint.sim @@ -373,7 +373,7 @@ sql_error insert into st_bigint_e7 values (now, "123abc") sql_error insert into st_bigint_e9 values (now, abc) sql_error insert into st_bigint_e10 values (now, "abc") sql_error insert into st_bigint_e11 values (now, " ") -sql insert into st_bigint_e12 values (now, '') +sql_error insert into st_bigint_e12 values (now, '') sql_error insert into st_bigint_e13 using mt_bigint tags (033) values (now, 9223372036854775808) sql insert into st_bigint_e14 using mt_bigint tags (033) values (now, -9223372036854775808) @@ -386,7 +386,7 @@ sql_error insert into st_bigint_e20 using mt_bigint tags (033) values (now, "123 sql_error insert into st_bigint_e22 using mt_bigint tags (033) values (now, abc) sql_error insert into st_bigint_e23 using mt_bigint tags (033) values (now, "abc") sql_error insert into st_bigint_e24 using mt_bigint tags (033) values (now, " ") -sql insert into st_bigint_e25 using mt_bigint tags (033) values (now, '') +sql_error insert into st_bigint_e25 using mt_bigint tags (033) values (now, '') sql_error insert into st_bigint_e13_0 using mt_bigint tags (9223372036854775808) values (now, -033) sql insert into st_bigint_e14_0 using mt_bigint tags (-9223372036854775808) values (now, -033) diff --git a/tests/script/tsim/parser/columnValue_double.sim b/tests/script/tsim/parser/columnValue_double.sim index da09b77353..bfcf338fac 100644 --- a/tests/script/tsim/parser/columnValue_double.sim +++ b/tests/script/tsim/parser/columnValue_double.sim @@ -476,7 +476,7 @@ sql_error insert into st_double_e7 values (now, "123abc") sql_error insert into st_double_e9 values (now, abc) sql_error insert into st_double_e10 values (now, "abc") sql_error insert into st_double_e11 values (now, " ") -sql insert into st_double_e12 values (now, '') +sql_error insert into st_double_e12 values (now, '') sql_error insert into st_double_e13 using mt_double tags (033) values (now, 11.7976931348623157e+308) sql_error insert into st_double_e14 using mt_double tags (033) values (now, -11.7976931348623157e+308) @@ -489,7 +489,7 @@ sql_error insert into st_double_e20 using mt_double tags (033) values (now, "123 sql_error insert into st_double_e22 using mt_double tags (033) values (now, abc) sql_error insert into st_double_e23 using mt_double tags (033) values (now, "abc") sql_error insert into st_double_e24 using mt_double tags (033) values (now, " ") -sql insert into st_double_e25_1 using mt_double tags (033) values (now, '') +sql_error insert into st_double_e25_1 using mt_double tags (033) values (now, '') sql_error insert into st_double_e13 using mt_double tags (31.7976931348623157e+308) values (now, -033) sql_error insert into st_double_e14 using mt_double tags (-31.7976931348623157e+308) values (now, -033) diff --git a/tests/script/tsim/parser/columnValue_float.sim b/tests/script/tsim/parser/columnValue_float.sim index 3e20e178c3..b2db7dff2b 100644 --- a/tests/script/tsim/parser/columnValue_float.sim +++ b/tests/script/tsim/parser/columnValue_float.sim @@ -506,7 +506,7 @@ sql_error insert into st_float_e7 values (now, "123abc") sql_error insert into st_float_e9 values (now, abc) sql_error insert into st_float_e10 values (now, "abc") sql_error insert into st_float_e11 values (now, " ") -sql insert into st_float_e12 values (now, '') +sql_error insert into st_float_e12 values (now, '') sql_error insert into st_float_e13 using mt_float tags (033) values (now, 3.50282347e+38) sql_error insert into st_float_e14 using mt_float tags (033) values (now, -3.50282347e+38) @@ -519,7 +519,7 @@ sql_error insert into st_float_e20 using mt_float tags (033) values (now, "123ab sql_error insert into st_float_e22 using mt_float tags (033) values (now, abc) sql_error insert into st_float_e23 using mt_float tags (033) values (now, "abc") sql_error insert into st_float_e24 using mt_float tags (033) values (now, " ") -sql insert into st_float_e25_1 using mt_float tags (033) values (now, '') +sql_error insert into st_float_e25_1 using mt_float tags (033) values (now, '') sql_error insert into st_float_e13 using mt_float tags (3.50282347e+38) values (now, -033) sql_error insert into st_float_e14 using mt_float tags (-3.50282347e+38) values (now, -033) diff --git a/tests/script/tsim/parser/columnValue_int.sim b/tests/script/tsim/parser/columnValue_int.sim index 009fbd1ede..4a3b8ebd0b 100644 --- a/tests/script/tsim/parser/columnValue_int.sim +++ b/tests/script/tsim/parser/columnValue_int.sim @@ -371,7 +371,7 @@ sql_error insert into st_int_e7 values (now, "123abc") sql_error insert into st_int_e9 values (now, abc) sql_error insert into st_int_e10 values (now, "abc") sql_error insert into st_int_e11 values (now, " ") -sql insert into st_int_e12 values (now, '') +sql_error insert into st_int_e12 values (now, '') sql_error insert into st_int_e13 using mt_int tags (033) values (now, 2147483648) sql insert into st_int_e14 using mt_int tags (033) values (now, -2147483648) @@ -384,7 +384,7 @@ sql_error insert into st_int_e20 using mt_int tags (033) values (now, "123abc") sql_error insert into st_int_e22 using mt_int tags (033) values (now, abc) sql_error insert into st_int_e23 using mt_int tags (033) values (now, "abc") sql_error insert into st_int_e24 using mt_int tags (033) values (now, " ") -sql insert into st_int_e25 using mt_int tags (033) values (now, '') +sql_error insert into st_int_e25 using mt_int tags (033) values (now, '') sql_error insert into st_int_e13 using mt_int tags (2147483648) values (now, -033) sql insert into st_int_e14_1 using mt_int tags (-2147483648) values (now, -033) diff --git a/tests/script/tsim/parser/columnValue_smallint.sim b/tests/script/tsim/parser/columnValue_smallint.sim index 0dcb0d85f4..eb364f3630 100644 --- a/tests/script/tsim/parser/columnValue_smallint.sim +++ b/tests/script/tsim/parser/columnValue_smallint.sim @@ -374,7 +374,7 @@ sql_error insert into st_smallint_e7 values (now, "123abc") sql_error insert into st_smallint_e9 values (now, abc) sql_error insert into st_smallint_e10 values (now, "abc") sql_error insert into st_smallint_e11 values (now, " ") -sql insert into st_smallint_e12 values (now, '') +sql_error insert into st_smallint_e12 values (now, '') sql_error insert into st_smallint_e13 using mt_smallint tags (033) values (now, 32768) sql insert into st_smallint_e14_1 using mt_smallint tags (033) values (now, -32768) @@ -387,7 +387,7 @@ sql_error insert into st_smallint_e20 using mt_smallint tags (033) values (now, sql_error insert into st_smallint_e22 using mt_smallint tags (033) values (now, abc) sql_error insert into st_smallint_e23 using mt_smallint tags (033) values (now, "abc") sql_error insert into st_smallint_e24 using mt_smallint tags (033) values (now, " ") -sql insert into st_smallint_e25_1 using mt_smallint tags (033) values (now, '') +sql_error insert into st_smallint_e25_1 using mt_smallint tags (033) values (now, '') sql_error insert into st_smallint_e13 using mt_smallint tags (32768) values (now, -033) sql insert into st_smallint_e14 using mt_smallint tags (-32768) values (now, -033) diff --git a/tests/script/tsim/parser/columnValue_tinyint.sim b/tests/script/tsim/parser/columnValue_tinyint.sim index 62ae4e5228..d7938aa739 100644 --- a/tests/script/tsim/parser/columnValue_tinyint.sim +++ b/tests/script/tsim/parser/columnValue_tinyint.sim @@ -372,7 +372,7 @@ sql_error insert into st_tinyint_e7 values (now, "123abc") sql_error insert into st_tinyint_e9 values (now, abc) sql_error insert into st_tinyint_e10 values (now, "abc") sql_error insert into st_tinyint_e11 values (now, " ") -sql insert into st_tinyint_e12 values (now, '') +sql_error insert into st_tinyint_e12 values (now, '') sql_error insert into st_tinyint_e13 using mt_tinyint tags (033) values (now, 128) sql insert into st_tinyint_e14_1 using mt_tinyint tags (033) values (now, -128) @@ -385,7 +385,7 @@ sql_error insert into st_tinyint_e20 using mt_tinyint tags (033) values (now, "1 sql_error insert into st_tinyint_e22 using mt_tinyint tags (033) values (now, abc) sql_error insert into st_tinyint_e23 using mt_tinyint tags (033) values (now, "abc") sql_error insert into st_tinyint_e24 using mt_tinyint tags (033) values (now, " ") -sql insert into st_tinyint_e25_2 using mt_tinyint tags (033) values (now, '') +sql_error insert into st_tinyint_e25_2 using mt_tinyint tags (033) values (now, '') sql_error insert into st_tinyint_e13 using mt_tinyint tags (128) values (now, -033) sql insert into st_tinyint_e14 using mt_tinyint tags (-128) values (now, -033) diff --git a/tests/script/tsim/parser/fill.sim b/tests/script/tsim/parser/fill.sim index f688d815e7..4892345e12 100644 --- a/tests/script/tsim/parser/fill.sim +++ b/tests/script/tsim/parser/fill.sim @@ -47,31 +47,10 @@ $tsu = $tsu + $ts0 ## fill syntax test # number of fill values exceeds number of selected columns -sql select _wstart, max(c1), max(c2), max(c3), max(c4), max(c5) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 6, 6, 6, 6, 6, 6, 6, 6) -if $data11 != 6 then - return -1 -endi -if $data12 != 6 then - return -1 -endi -if $data13 != 6.00000 then - return -1 -endi -if $data14 != 6.000000000 then - return -1 -endi +sql_error select _wstart, max(c1), max(c2), max(c3), max(c4), max(c5) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 6, 6, 6, 6, 6, 6, 6, 6) # number of fill values is smaller than number of selected columns -sql select _wstart, max(c1), max(c2), max(c3) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 6, 6) -if $data11 != 6 then - return -1 -endi -if $data12 != 6 then - return -1 -endi -if $data13 != 6.00000 then - return -1 -endi +sql_error select _wstart, max(c1), max(c2), max(c3) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 6, 6) # unspecified filling method sql_error select _wstart, max(c1), max(c2), max(c3), max(c4), max(c5) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill (6, 6, 6, 6, 6) @@ -182,7 +161,7 @@ endi # min_with_fill print min_with_fill -sql select _wstart, min(c1), min(c2), min(c3), min(c4), min(c5) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 6, 6, 6, 6, 6, 6, 6, 6) +sql select _wstart, min(c1), min(c2), min(c3), min(c4), min(c5) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 6, 6, 6, 6, 6) if $rows != 9 then return -1 endi @@ -216,7 +195,7 @@ endi # first_with_fill print first_with_fill -sql select _wstart, first(c1), first(c2), first(c3), first(c4), first(c5), first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 6, 6, 6, 6, 6, 6, 6, 6) +sql select _wstart, first(c1), first(c2), first(c3), first(c4), first(c5), first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 6, 6, 6, 6, 6, 6, '6', '6') if $rows != 9 then return -1 endi @@ -305,7 +284,7 @@ endi # last_with_fill print last_with_fill -sql select _wstart, last(c1), last(c2), last(c3), last(c4), last(c5), last(c6), last(c7), last(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 6, 6, 6, 6, 6, 6, 6, 6) +sql select _wstart, last(c1), last(c2), last(c3), last(c4), last(c5), last(c6), last(c7), last(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 6, 6, 6, 6, 6, 6, '6', '6') if $rows != 9 then return -1 endi @@ -351,7 +330,7 @@ if $data11 != -1 then endi # fill_char_values_to_arithmetic_fields -sql select sum(c1), avg(c2), max(c3), min(c4), avg(c4), count(c6), last(c7), last(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c') +sql_error select sum(c1), avg(c2), max(c3), min(c4), avg(c4), count(c6), last(c7), last(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c') # fill_multiple_columns sql_error select sum(c1), avg(c2), min(c3), max(c4), count(c6), first(c7), last(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 99, 99, 99, 99, 99, abc, abc) @@ -376,37 +355,25 @@ endi # fill_into_nonarithmetic_fieds print select _wstart, first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 20000000, 20000000, 20000000) -sql select _wstart, first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 20000000, 20000000, 20000000) -if $data01 != 1 then - return -1 -endi -if $data11 != NULL then - return -1 -endi +sql_error select _wstart, first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 20000000, 20000000, 20000000) -sql select first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 1, 1, 1) -sql select first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 1.1, 1.1, 1.1) -sql select first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 1e1, 1e1, 1e1) +sql_error select first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 1, 1, 1) +sql_error select first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 1.1, 1.1, 1.1) +sql_error select first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 1e1, 1e1, 1e1) sql select first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, '1e', '1e1') # fill quoted values into bool column will throw error unless the value is 'true' or 'false' Note:2018-10-24 # fill values into binary or nchar columns will be set to NULL automatically Note:2018-10-24 sql select first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, '1e', '1e1','1e1') -sql select first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, true, true, true) +sql_error select first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, true, true, true) sql select first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 'true', 'true','true') # fill nonarithmetic values into arithmetic fields sql_error select count(*) where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, abc); -sql select count(*) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 'true'); +sql_error select count(*) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 'true'); print select _wstart, count(*) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, '1e1'); -sql select _wstart, count(*) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, '1e1'); -if $rows != 9 then - return -1 -endi -if $data01 != 1 then - return -1 -endi +sql_error select _wstart, count(*) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, '1e1'); sql select _wstart, count(*) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 1e1); if $rows != 9 then @@ -416,13 +383,7 @@ if $data01 != 1 then return -1 endi -sql select _wstart, count(*) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, '10'); -if $rows != 9 then - return -1 -endi -if $data01 != 1 then - return -1 -endi +sql_error select _wstart, count(*) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, '10'); ## linear fill # feature currently switched off 2018/09/29 @@ -859,8 +820,8 @@ sql insert into tm0 values('2020-1-1 1:3:8', 8); sql insert into tm0 values('2020-1-1 1:3:9', 9); sql insert into tm0 values('2020-1-1 1:4:10', 10); -print select _wstart, max(k)-min(k),last(k)-first(k),0-spread(k) from tm0 where ts>='2020-1-1 1:1:1' and ts<='2020-1-1 1:2:15' interval(10s) fill(value, 99,91,90,89,88,87,86,85); -sql select _wstart, max(k)-min(k),last(k)-first(k),0-spread(k) from tm0 where ts>='2020-1-1 1:1:1' and ts<='2020-1-1 1:2:15' interval(10s) fill(value, 99,91,90,89,88,87,86,85); +print select _wstart, max(k)-min(k),last(k)-first(k),0-spread(k) from tm0 where ts>='2020-1-1 1:1:1' and ts<='2020-1-1 1:2:15' interval(10s) fill(value, 99,91,90); +sql select _wstart, max(k)-min(k),last(k)-first(k),0-spread(k) from tm0 where ts>='2020-1-1 1:1:1' and ts<='2020-1-1 1:2:15' interval(10s) fill(value, 99,91,90); if $rows != 8 then return -1 endi @@ -958,14 +919,14 @@ if $data12 != NULL then return -1 endi -sql select _wstart, max(k)-min(k),last(k)-first(k),0-spread(k) from tm0 where ts>='2020-1-1 1:1:1' and ts<='2020-1-1 4:2:15' interval(500a) fill(value, 99,91,90,89,88,87,86,85) ; +sql select _wstart, max(k)-min(k),last(k)-first(k),0-spread(k) from tm0 where ts>='2020-1-1 1:1:1' and ts<='2020-1-1 4:2:15' interval(500a) fill(value, 99,91,90) ; if $rows != 21749 then print expect 21749, actual: $rows return -1 endi -print select _wstart, max(k)-min(k),last(k)-first(k),0-spread(k),count(1) from m1 where ts>='2020-1-1 1:1:1' and ts<='2020-1-1 1:2:15' interval(10s) fill(value, 99,91,90,89,88,87,86,85) ; -sql select _wstart, max(k)-min(k),last(k)-first(k),0-spread(k),count(1) from m1 where ts>='2020-1-1 1:1:1' and ts<='2020-1-1 1:2:15' interval(10s) fill(value, 99,91,90,89,88,87,86,85) ; +print select _wstart, max(k)-min(k),last(k)-first(k),0-spread(k),count(1) from m1 where ts>='2020-1-1 1:1:1' and ts<='2020-1-1 1:2:15' interval(10s) fill(value, 99,91,90,89) ; +sql select _wstart, max(k)-min(k),last(k)-first(k),0-spread(k),count(1) from m1 where ts>='2020-1-1 1:1:1' and ts<='2020-1-1 1:2:15' interval(10s) fill(value, 99,91,90,89) ; if $rows != 8 then return -1 endi diff --git a/tests/script/tsim/parser/fill_us.sim b/tests/script/tsim/parser/fill_us.sim index 0a45c02f58..f760ba3577 100644 --- a/tests/script/tsim/parser/fill_us.sim +++ b/tests/script/tsim/parser/fill_us.sim @@ -48,32 +48,11 @@ $tsu = $tsu + $ts0 ## fill syntax test # number of fill values exceeds number of selected columns print select _wstart, max(c1), max(c2), max(c3), max(c4), max(c5) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 6, 6, 6, 6, 6, 6, 6, 6) -sql select _wstart, max(c1), max(c2), max(c3), max(c4), max(c5) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 6, 6, 6, 6, 6, 6, 6, 6) -if $data11 != 6 then - return -1 -endi -if $data12 != 6 then - return -1 -endi -if $data13 != 6.00000 then - return -1 -endi -if $data14 != 6.000000000 then - return -1 -endi +sql_error select _wstart, max(c1), max(c2), max(c3), max(c4), max(c5) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 6, 6, 6, 6, 6, 6, 6, 6) # number of fill values is smaller than number of selected columns print sql select _wstart, max(c1), max(c2), max(c3) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 6, 6) -sql select _wstart, max(c1), max(c2), max(c3) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 6, 6) -if $data11 != 6 then - return -1 -endi -if $data12 != 6 then - return -1 -endi -if $data13 != 6.00000 then - return -1 -endi +sql_error select _wstart, max(c1), max(c2), max(c3) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 6, 6) # unspecified filling method sql_error select max(c1), max(c2), max(c3), max(c4), max(c5) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill (6, 6, 6, 6, 6) @@ -185,7 +164,7 @@ endi # min_with_fill print min_with_fill -sql select min(c1), min(c2), min(c3), min(c4), min(c5) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 6, 6, 6, 6, 6, 6, 6, 6) +sql select min(c1), min(c2), min(c3), min(c4), min(c5) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 6, 6, 6, 6, 6) if $rows != 9 then return -1 endi @@ -219,7 +198,7 @@ endi # first_with_fill print first_with_fill -sql select _wstart, first(c1), first(c2), first(c3), first(c4), first(c5), first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 6, 6, 6, 6, 6, 6, 6, 6) +sql select _wstart, first(c1), first(c2), first(c3), first(c4), first(c5), first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 6, 6, 6, 6, 6, 6, '6', '6') if $rows != 9 then return -1 endi @@ -308,7 +287,7 @@ endi # last_with_fill print last_with_fill -sql select last(c1), last(c2), last(c3), last(c4), last(c5), last(c6), last(c7), last(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 6, 6, 6, 6, 6, 6, 6, 6) +sql select last(c1), last(c2), last(c3), last(c4), last(c5), last(c6), last(c7), last(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 6, 6, 6, 6, 6, 6, '6', '6') if $rows != 9 then return -1 endi @@ -353,7 +332,7 @@ if $data11 != -1 then endi # fill_char_values_to_arithmetic_fields -sql select sum(c1), avg(c2), max(c3), min(c4), avg(c4), count(c6), last(c7), last(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c') +sql_error select sum(c1), avg(c2), max(c3), min(c4), avg(c4), count(c6), last(c7), last(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 'c', 'c', 'c', 'c', 'c', 'c', 'c', 'c') # fill_multiple_columns sql_error select _wstart, sum(c1), avg(c2), min(c3), max(c4), count(c6), first(c7), last(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 99, 99, 99, 99, 99, abc, abc) @@ -379,34 +358,24 @@ endi # fill_into_nonarithmetic_fieds -sql select _wstart, first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 20000000, 20000000, 20000000) -#if $data11 != 20000000 then -if $data11 != NULL then - return -1 -endi +sql_error select _wstart, first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 20000000, 20000000, 20000000) -sql select first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 1, 1, 1) -sql select first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 1.1, 1.1, 1.1) -sql select first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 1e1, 1e1, 1e1) +sql_error select first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 1, 1, 1) +sql_error select first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 1.1, 1.1, 1.1) +sql_error select first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 1e1, 1e1, 1e1) sql select first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, '1e', '1e1') # fill quoted values into bool column will throw error unless the value is 'true' or 'false' Note:2018-10-24 # fill values into binary or nchar columns will be set to null automatically Note:2018-10-24 sql select first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, '1e', '1e1','1e1') -sql select first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, true, true, true) +sql_error select first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, true, true, true) sql select first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 'true', 'true','true') # fill nonarithmetic values into arithmetic fields sql_error select count(*) where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, abc); -sql select count(*) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 'true'); +sql_error select count(*) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 'true'); -sql select _wstart, count(*) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, '1e1'); -if $rows != 9 then - return -1 -endi -if $data01 != 1 then - return -1 -endi +sql_error select _wstart, count(*) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, '1e1'); sql select _wstart, count(*) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 1e1); if $rows != 9 then @@ -416,14 +385,7 @@ if $data01 != 1 then return -1 endi -sql select _wstart, count(*) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, '10'); -if $rows != 9 then - return -1 -endi -if $data01 != 1 then - return -1 -endi - +sql_error select _wstart, count(*) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, '10'); ## linear fill # feature currently switched off 2018/09/29 diff --git a/tests/script/tsim/parser/slimit_query.sim b/tests/script/tsim/parser/slimit_query.sim index 1e04a31099..acf0489d3c 100644 --- a/tests/script/tsim/parser/slimit_query.sim +++ b/tests/script/tsim/parser/slimit_query.sim @@ -93,25 +93,25 @@ if $rows != 3 then endi ### slimit + fill -sql select max(c1), min(c2), avg(c3), sum(c4), sum(c6), count(c7), first(c8), last(c9) from $stb where ts >= $ts0 and ts <= $tsu and t2 >= 2 and t3 <= 5 partition by t1 interval(5m) fill(value, -1, -2) slimit 4 soffset 4 limit 0 offset 0 +sql select max(c1), min(c2), avg(c3), sum(c4), sum(c6), count(c7), first(c8), last(c9) from $stb where ts >= $ts0 and ts <= $tsu and t2 >= 2 and t3 <= 5 partition by t1 interval(5m) fill(linear) slimit 4 soffset 4 limit 0 offset 0 if $rows != 0 then return -1 endi -sql select max(c1), min(c2), avg(c3), sum(c4), sum(c6), count(c7), first(c8), last(c9) from $stb where ts >= $ts0 and ts <= $tsu and t2 >= 2 and t3 <= 9 partition by t1 interval(5m) fill(value, -1, -2) slimit 4 soffset 4 limit 2 offset 0 -print select max(c1), min(c2), avg(c3), sum(c4), sum(c6), count(c7), first(c8), last(c9) from $stb where ts >= $ts0 and ts <= $tsu and t2 >= 2 and t3 <= 9 partition by t1 interval(5m) fill(value, -1, -2) slimit 4 soffset 4 limit 2 offset 0 +sql select max(c1), min(c2), avg(c3), sum(c4), sum(c6), count(c7), first(c8), last(c9) from $stb where ts >= $ts0 and ts <= $tsu and t2 >= 2 and t3 <= 9 partition by t1 interval(5m) fill(linear) slimit 4 soffset 4 limit 2 offset 0 +print select max(c1), min(c2), avg(c3), sum(c4), sum(c6), count(c7), first(c8), last(c9) from $stb where ts >= $ts0 and ts <= $tsu and t2 >= 2 and t3 <= 9 partition by t1 interval(5m) fill(linear) slimit 4 soffset 4 limit 2 offset 0 print $rows $data00 $data01 $data02 $data03 if $rows != 8 then return -1 endi # desc -sql select max(c1), min(c2), avg(c3), sum(c4), sum(c6), count(c7), first(c8), last(c9) from $stb where ts >= $ts0 and ts <= $tsu and t2 >= 2 and t3 <= 9 partition by t1 interval(5m) fill(value, -1, -2) slimit 4 soffset 4 limit 2 offset 0 +sql select max(c1), min(c2), avg(c3), sum(c4), sum(c6), count(c7), first(c8), last(c9) from $stb where ts >= $ts0 and ts <= $tsu and t2 >= 2 and t3 <= 9 partition by t1 interval(5m) fill(linear) slimit 4 soffset 4 limit 2 offset 0 if $rows != 8 then return -1 endi -sql select max(c1), min(c2), avg(c3), sum(c4), sum(c6), count(c7), first(c8), last(c9) from $stb where ts >= $ts0 and ts <= $tsu and t2 >= 2 and t3 <= 9 partition by t1 interval(5m) fill(value, -1, -2) slimit 4 soffset 4 limit 2 offset 598 +sql select max(c1), min(c2), avg(c3), sum(c4), sum(c6), count(c7), first(c8), last(c9) from $stb where ts >= $ts0 and ts <= $tsu and t2 >= 2 and t3 <= 9 partition by t1 interval(5m) fill(linear) slimit 4 soffset 4 limit 2 offset 598 if $rows != 4 then return -1 endi diff --git a/tests/script/tsim/parser/timestamp_query.sim b/tests/script/tsim/parser/timestamp_query.sim index 6e92dbcb3a..24058cbc84 100644 --- a/tests/script/tsim/parser/timestamp_query.sim +++ b/tests/script/tsim/parser/timestamp_query.sim @@ -28,7 +28,7 @@ sql select * from ts_stb0 where ts <> $ts0 ##### select from supertable $tb = $tbPrefix . 0 -sql select _wstart, first(c1), last(c1), (1537325400 - 1537146000)/(5*60) v from $tb where ts >= $ts0 and ts < $tsu interval(5m) fill(value, -1) +sql select _wstart, first(c1), last(c1), (1537325400 - 1537146000)/(5*60) v from $tb where ts >= $ts0 and ts < $tsu interval(5m) fill(value, -1, -1) $res = $rowNum * 2 $n = $res - 2 print ============>$n @@ -47,7 +47,7 @@ if $data13 != 598.000000000 then return -1 endi -sql select _wstart, first(c1), last(c1), (1537325400 - 1537146000)/(5*60) v from $tb where ts >= $ts0 and ts < $tsu interval(5m) fill(value, NULL) +sql select _wstart, first(c1), last(c1), (1537325400 - 1537146000)/(5*60) v from $tb where ts >= $ts0 and ts < $tsu interval(5m) fill(value, NULL, NULL) if $data13 != 598.000000000 then print expect 598.000000000, actual $data03 return -1 diff --git a/tests/script/tsim/valgrind/checkError6.sim b/tests/script/tsim/valgrind/checkError6.sim index fcc5b04c90..d85a1bebc8 100644 --- a/tests/script/tsim/valgrind/checkError6.sim +++ b/tests/script/tsim/valgrind/checkError6.sim @@ -67,17 +67,17 @@ sql select diff(tbcol) from tb1 where tbcol > 5 and tbcol < 20 order by ts sql select first(tbcol), last(tbcol) as b from tb1 where ts <= 1601481840000 interval(1m) sql select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), sum(tbcol), stddev(tbcol) from tb1 where ts <= 1601481840000 partition by tgcol order by tgcol sql select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), sum(tbcol), stddev(tbcol) from tb1 where ts <= 1601481840000 partition by tgcol interval(1m) -sql select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), count(tbcol) from tb1 where ts <= 1601481840000 and ts >= 1601481800000 partition by tgcol interval(1m) fill(value, 0) +sql select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), count(tbcol) from tb1 where ts <= 1601481840000 and ts >= 1601481800000 partition by tgcol interval(1m) fill(value, 0,0,0,0,0) sql select last_row(*) from tb1 where tbcol > 5 and tbcol < 20 sql select _wstart, _wend, _wduration, _qstart, _qend, count(*) from tb1 interval(10s, 2s) sliding(10s) -sql select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), count(tbcol) from tb1 where ts <= 1601481840000 and ts >= 1601481800000 partition by tgcol interval(1m) fill(value, 0) -sql select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), count(tbcol) from tb1 where ts <= 1601481840000 and ts >= 1601481800000 partition by tgcol interval(1m) fill(value, 0) order by tgcol desc +sql select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), count(tbcol) from tb1 where ts <= 1601481840000 and ts >= 1601481800000 partition by tgcol interval(1m) fill(value, 0,0,0,0,0) +sql select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), count(tbcol) from tb1 where ts <= 1601481840000 and ts >= 1601481800000 partition by tgcol interval(1m) fill(value, 0,0,0,0,0) order by tgcol desc sql select log(tbcol), abs(tbcol), pow(tbcol, 2), sqrt(tbcol), sin(tbcol), cos(tbcol), tan(tbcol), asin(tbcol), acos(tbcol), atan(tbcol), ceil(tbcol), floor(tbcol), round(tbcol), atan(tbcol) from tb1 sql select length("abcd1234"), char_length("abcd1234=-+*") from tb1 sql select tbcol4, length(tbcol4), lower(tbcol4), upper(tbcol4), ltrim(tbcol4), rtrim(tbcol4), concat(tbcol4, tbcol5), concat_ws('_', tbcol4, tbcol5), substr(tbcol4, 1, 4) from tb1 sql select * from tb1 where tbcol not in (1,2,3,null); sql select * from tb1 where tbcol + 3 <> null; -sql select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), count(tbcol) from tb1 where ts <= 1601481840000 and ts >= 1601481800000 partition by tgcol interval(1m) fill(value, 0) +sql select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), count(tbcol) from tb1 where ts <= 1601481840000 and ts >= 1601481800000 partition by tgcol interval(1m) fill(value, 0,0,0,0,0) sql select tbcol5 - tbcol3 from tb1 print =============== step4: stb @@ -97,8 +97,8 @@ sql select first(tbcol), last(tbcol) as c from stb group by tgcol sql select first(tbcol), last(tbcol) as b from stb where ts <= 1601481840000 and tbcol2 is null partition by tgcol interval(1m) sql select first(tbcol), last(tbcol) as b from stb where ts <= 1601481840000 partition by tgcol interval(1m) sql select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), sum(tbcol), stddev(tbcol) from stb where ts <= 1601481840000 partition by tgcol interval(1m) -sql select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), count(tbcol) from stb where ts <= 1601481840000 and ts >= 1601481800000 partition by tgcol interval(1m) fill(value, 0) -sql select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), count(tbcol) from stb where ts <= 1601481840000 and ts >= 1601481800000 and tgcol = 1 partition by tgcol interval(1m) fill(value, 0) order by tgcol desc +sql select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), count(tbcol) from stb where ts <= 1601481840000 and ts >= 1601481800000 partition by tgcol interval(1m) fill(value, 0,0,0,0,0) +sql select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), count(tbcol) from stb where ts <= 1601481840000 and ts >= 1601481800000 and tgcol = 1 partition by tgcol interval(1m) fill(value, 0,0,0,0,0) order by tgcol desc sql select last_row(tbcol), stddev(tbcol) from stb where tbcol > 5 and tbcol < 20 group by tgcol sql select _wstart, _wend, _wduration, _qstart, _qend, count(*) from stb interval(10s, 2s) sliding(10s) sql select log(tbcol), abs(tbcol), pow(tbcol, 2), sqrt(tbcol), sin(tbcol), cos(tbcol), tan(tbcol), asin(tbcol), acos(tbcol), atan(tbcol), ceil(tbcol), floor(tbcol), round(tbcol), atan(tbcol) from stb @@ -108,7 +108,7 @@ sql select * from stb where tbcol not in (1,2,3,null); sql select * from stb where tbcol + 3 <> null; sql select count(tbcol), avg(tbcol), sum(tbcol), min(tbcol), max(tbcol), first(tbcol), last(tbcol) from stb where tbcol = 1 and tbcol2 = 1 and tbcol3 = 1 partition by tgcol interval(1d) sql select _wstart, count(*) from tb1 session(ts, 1m) -sql select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), count(tbcol) from stb where ts <= 1601481840000 and ts >= 1601481800000 partition by tgcol interval(1m) fill(value, 0) +sql select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), count(tbcol) from stb where ts <= 1601481840000 and ts >= 1601481800000 partition by tgcol interval(1m) fill(value, 0,0,0,0,0) sql select tbcol5 - tbcol3 from stb sql select spread( tbcol2 )/44, spread(tbcol2), 0.204545455 * 44 from stb; @@ -127,8 +127,8 @@ sql explain analyze select count(*),sum(tbcol) from stb; sql explain analyze select count(*),sum(tbcol) from stb group by tbcol; sql explain analyze select * from information_schema.ins_stables; sql explain analyze verbose true select * from information_schema.ins_stables where db_name='db2'; -sql explain analyze verbose true select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), count(tbcol) from stb where ts <= 1601481840000 and ts >= 1601481800000 partition by tgcol interval(1m) fill(value, 0) -sql explain select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), count(tbcol) from stb where ts <= 1601481840000 and ts >= 1601481800000 partition by tgcol interval(1m) fill(value, 0) +sql explain analyze verbose true select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), count(tbcol) from stb where ts <= 1601481840000 and ts >= 1601481800000 partition by tgcol interval(1m) fill(value, 0,0,0,0,0) +sql explain select count(tbcol), avg(tbcol), max(tbcol), min(tbcol), count(tbcol) from stb where ts <= 1601481840000 and ts >= 1601481800000 partition by tgcol interval(1m) fill(value, 0,0,0,0,0) print =============== step6: in cast sql select 1+1n; diff --git a/tests/system-test/2-query/last_row.py b/tests/system-test/2-query/last_row.py index f65744a0b7..5d435b068f 100644 --- a/tests/system-test/2-query/last_row.py +++ b/tests/system-test/2-query/last_row.py @@ -638,13 +638,13 @@ class TDTestCase: tdSql.query(f"select ts , last_row(c1) ,c1 from (select ts , c1 ,t1 from {dbname}.stb1)") tdSql.checkData(0,1,None) - tdSql.query(f"select ts , last_row(c1) ,c1 from (select ts , max(c1) c1 ,t1 from {dbname}.stb1 where ts >now -1h and ts now -1h and ts now -1h and ts now -1h and ts now -1h and ts now -1h and ts Date: Mon, 29 Aug 2022 19:14:12 +0800 Subject: [PATCH 5/5] avoid deadlock --- source/libs/transport/src/thttp.c | 2 + source/libs/transport/src/transCli.c | 182 ++++++++++++++++----------- source/libs/transport/src/transSvr.c | 3 +- 3 files changed, 113 insertions(+), 74 deletions(-) diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index 386ea95dd7..e880a5abdb 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -155,6 +155,8 @@ static void clientSentCb(uv_write_t* req, int32_t status) { if (status != 0) { terrno = TAOS_SYSTEM_ERROR(status); uError("http-report failed to send data %s", uv_strerror(status)); + uv_close((uv_handle_t*)&cli->tcp, clientCloseCb); + return; } else { uTrace("http-report succ to send data"); } diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 7052b0b915..41688c7330 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -16,7 +16,7 @@ #include "transComm.h" typedef struct SConnList { - queue conn; + queue conns; int32_t size; } SConnList; @@ -107,11 +107,11 @@ static void doCloseIdleConn(void* param); static void cliReadTimeoutCb(uv_timer_t* handle); // register timer in each thread to clear expire conn // static void cliTimeoutCb(uv_timer_t* handle); -// alloc buf for recv +// alloc buffer for recv static void cliAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); -// callback after read nbytes from socket +// callback after recv nbytes from socket static void cliRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf); -// callback after write data to socket +// callback after send data to socket static void cliSendCb(uv_write_t* req, int status); // callback after conn to server static void cliConnCb(uv_connect_t* req, int status); @@ -129,19 +129,14 @@ static SCliConn* cliCreateConn(SCliThrd* thrd); static void cliDestroyConn(SCliConn* pConn, bool clear /*clear tcp handle or not*/); static void cliDestroy(uv_handle_t* handle); static void cliSend(SCliConn* pConn); +static void cliDestroyConnMsgs(SCliConn* conn, bool destroy); -static bool cliIsEpsetUpdated(int32_t code, STransConnCtx* pCtx) { - if (code != 0) return false; - if (pCtx->retryCnt == 0) return false; - if (transEpSetIsEqual(&pCtx->epSet, &pCtx->origEpSet)) return false; - return true; -} +// cli util func +static bool cliIsEpsetUpdated(int32_t code, STransConnCtx* pCtx); +static void cliMayCvtFqdnToIp(SEpSet* pEpSet, SCvtAddr* pCvtAddr); + +static int32_t cliBuildExceptResp(SCliMsg* pMsg, STransMsg* resp); -void cliMayCvtFqdnToIp(SEpSet* pEpSet, SCvtAddr* pCvtAddr); -/* - * set TCP connection timeout per-socket level - */ -static int cliCreateSocket(); // process data read from server, add decompress etc later static void cliHandleResp(SCliConn* conn); // handle except about conn @@ -169,15 +164,14 @@ static void destroyThrdObj(SCliThrd* pThrd); static void cliWalkCb(uv_handle_t* handle, void* arg); static void cliReleaseUnfinishedMsg(SCliConn* conn) { - SCliMsg* pMsg = NULL; for (int i = 0; i < transQueueSize(&conn->cliMsgs); i++) { - pMsg = transQueueGet(&conn->cliMsgs, i); - if (pMsg != NULL && pMsg->ctx != NULL) { - if (conn->ctx.freeFunc != NULL) { - conn->ctx.freeFunc(pMsg->ctx->ahandle); + SCliMsg* msg = transQueueGet(&conn->cliMsgs, i); + if (msg != NULL && msg->ctx != NULL) { + if (conn->ctx.freeFunc != NULL && msg->ctx->ahandle != NULL) { + conn->ctx.freeFunc(msg->ctx->ahandle); } } - destroyCmsg(pMsg); + destroyCmsg(msg); } } #define CLI_RELEASE_UV(loop) \ @@ -217,8 +211,10 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { } \ if (i == sz) { \ pMsg = NULL; \ + tDebug("msg not found, %" PRIu64 "", ahandle); \ } else { \ pMsg = transQueueRm(&conn->cliMsgs, i); \ + tDebug("msg found, %" PRIu64 "", ahandle); \ } \ } while (0) #define CONN_GET_NEXT_SENDMSG(conn) \ @@ -470,8 +466,8 @@ void* createConnPool(int size) { void* destroyConnPool(void* pool) { SConnList* connList = taosHashIterate((SHashObj*)pool, NULL); while (connList != NULL) { - while (!QUEUE_IS_EMPTY(&connList->conn)) { - queue* h = QUEUE_HEAD(&connList->conn); + while (!QUEUE_IS_EMPTY(&connList->conns)) { + queue* h = QUEUE_HEAD(&connList->conns); SCliConn* c = QUEUE_DATA(h, SCliConn, q); cliDestroyConn(c, true); } @@ -484,21 +480,21 @@ void* destroyConnPool(void* pool) { static SCliConn* getConnFromPool(void* pool, char* ip, uint32_t port) { char key[32] = {0}; CONN_CONSTRUCT_HASH_KEY(key, ip, port); - SHashObj* pPool = pool; - SConnList* plist = taosHashGet(pPool, key, strlen(key)); + + SConnList* plist = taosHashGet((SHashObj*)pool, key, strlen(key)); if (plist == NULL) { SConnList list = {0}; - taosHashPut(pPool, key, strlen(key), (void*)&list, sizeof(list)); - plist = taosHashGet(pPool, key, strlen(key)); - QUEUE_INIT(&plist->conn); + taosHashPut((SHashObj*)pool, key, strlen(key), (void*)&list, sizeof(list)); + plist = taosHashGet((SHashObj*)pool, key, strlen(key)); + QUEUE_INIT(&plist->conns); } - if (QUEUE_IS_EMPTY(&plist->conn)) { + if (QUEUE_IS_EMPTY(&plist->conns)) { return NULL; } plist->size -= 1; - queue* h = QUEUE_HEAD(&plist->conn); + queue* h = QUEUE_HEAD(&plist->conns); SCliConn* conn = QUEUE_DATA(h, SCliConn, q); conn->status = ConnNormal; QUEUE_REMOVE(&conn->q); @@ -514,22 +510,21 @@ static void addConnToPool(void* pool, SCliConn* conn) { if (conn->status == ConnInPool) { return; } - SCliThrd* thrd = conn->hostThrd; - CONN_HANDLE_THREAD_QUIT(thrd); - allocConnRef(conn, true); + SCliThrd* thrd = conn->hostThrd; if (conn->timer != NULL) { uv_timer_stop(conn->timer); taosArrayPush(thrd->timerList, &conn->timer); conn->timer->data = NULL; conn->timer = NULL; } + if (T_REF_VAL_GET(conn) > 1) { + transUnrefCliHandle(conn); + } + + cliDestroyConnMsgs(conn, false); - STrans* pTransInst = thrd->pTransInst; - cliReleaseUnfinishedMsg(conn); - transQueueClear(&conn->cliMsgs); - transCtxCleanup(&conn->ctx); conn->status = ConnInPool; if (conn->list == NULL) { @@ -540,18 +535,15 @@ static void addConnToPool(void* pool, SCliConn* conn) { } else { tTrace("%s conn %p added to conn pool, read buf cap:%d", CONN_GET_INST_LABEL(conn), conn, conn->readBuf.cap); } - assert(conn->list != NULL); - QUEUE_INIT(&conn->q); - QUEUE_PUSH(&conn->list->conn, &conn->q); + QUEUE_PUSH(&conn->list->conns, &conn->q); conn->list->size += 1; - conn->task = NULL; - assert(!QUEUE_IS_EMPTY(&conn->list->conn)); - if (conn->list->size >= 50) { STaskArg* arg = taosMemoryCalloc(1, sizeof(STaskArg)); arg->param1 = conn; arg->param2 = thrd; + + STrans* pTransInst = thrd->pTransInst; conn->task = transDQSched(thrd->timeoutQueue, doCloseIdleConn, arg, CONN_PERSIST_TIME(pTransInst->idleTime)); } } @@ -691,11 +683,10 @@ static void cliDestroy(uv_handle_t* handle) { transRemoveExHandle(transGetRefMgt(), conn->refId); taosMemoryFree(conn->ip); - conn->stream->data = NULL; taosMemoryFree(conn->stream); - transCtxCleanup(&conn->ctx); - cliReleaseUnfinishedMsg(conn); - transQueueDestroy(&conn->cliMsgs); + + cliDestroyConnMsgs(conn, true); + tTrace("%s conn %p destroy successfully", CONN_GET_INST_LABEL(conn), conn); transReqQueueClear(&conn->wreqQueue); transDestroyBuffer(&conn->readBuf); @@ -738,8 +729,6 @@ static void cliSendCb(uv_write_t* req, int status) { } void cliSend(SCliConn* pConn) { - CONN_HANDLE_BROKEN(pConn); - assert(!transQueueEmpty(&pConn->cliMsgs)); SCliMsg* pCliMsg = NULL; @@ -756,8 +745,8 @@ void cliSend(SCliConn* pConn) { pMsg->pCont = (void*)rpcMallocCont(0); pMsg->contLen = 0; } - int msgLen = transMsgLenFromCont(pMsg->contLen); + int msgLen = transMsgLenFromCont(pMsg->contLen); STransMsgHead* pHead = transHeadFromCont(pMsg->pCont); pHead->ahandle = pCtx != NULL ? (uint64_t)pCtx->ahandle : 0; pHead->noResp = REQUEST_NO_RESP(pMsg) ? 1 : 0; @@ -769,8 +758,6 @@ void cliSend(SCliConn* pConn) { pHead->traceId = pMsg->info.traceId; pHead->magicNum = htonl(TRANS_MAGIC_NUM); - uv_buf_t wb = uv_buf_init((char*)pHead, msgLen); - STraceId* trace = &pMsg->info.traceId; tGDebug("%s conn %p %s is sent to %s, local info %s, len:%d", CONN_GET_INST_LABEL(pConn), pConn, TMSG_INFO(pHead->msgType), pConn->dst, pConn->src, pMsg->contLen); @@ -792,6 +779,8 @@ void cliSend(SCliConn* pConn) { tGTrace("%s conn %p start timer for msg:%s", CONN_GET_INST_LABEL(pConn), pConn, TMSG_INFO(pMsg->msgType)); uv_timer_start((uv_timer_t*)pConn->timer, cliReadTimeoutCb, TRANS_READ_TIMEOUT, 0); } + + uv_buf_t wb = uv_buf_init((char*)pHead, msgLen); uv_write_t* req = transReqQueuePush(&pConn->wreqQueue); uv_write(req, (uv_stream_t*)pConn->stream, &wb, 1, cliSendCb); return; @@ -807,7 +796,6 @@ void cliConnCb(uv_connect_t* req, int status) { cliHandleExcept(pConn); return; } - // int addrlen = sizeof(pConn->addr); struct sockaddr peername, sockname; int addrlen = sizeof(peername); @@ -840,7 +828,7 @@ static void cliHandleRelease(SCliMsg* pMsg, SCliThrd* pThrd) { int64_t refId = (int64_t)(pMsg->msg.info.handle); SExHandle* exh = transAcquireExHandle(transGetRefMgt(), refId); if (exh == NULL) { - tDebug("%" PRId64 " already release", refId); + tDebug("%" PRId64 " already released", refId); destroyCmsg(pMsg); return; } @@ -856,6 +844,9 @@ static void cliHandleRelease(SCliMsg* pMsg, SCliThrd* pThrd) { return; } cliSend(conn); + } else { + tError("%s conn %p already released", CONN_GET_INST_LABEL(conn), conn); + destroyCmsg(pMsg); } } static void cliHandleUpdate(SCliMsg* pMsg, SCliThrd* pThrd) { @@ -905,6 +896,27 @@ void cliMayCvtFqdnToIp(SEpSet* pEpSet, SCvtAddr* pCvtAddr) { } } } + +bool cliIsEpsetUpdated(int32_t code, STransConnCtx* pCtx) { + if (code != 0) return false; + if (pCtx->retryCnt == 0) return false; + if (transEpSetIsEqual(&pCtx->epSet, &pCtx->origEpSet)) return false; + return true; +} + +int32_t cliBuildExceptResp(SCliMsg* pMsg, STransMsg* pResp) { + if (pMsg == NULL) return -1; + + memset(pResp, 0, sizeof(STransMsg)); + + pResp->code = TSDB_CODE_RPC_BROKEN_LINK; + pResp->msgType = pMsg->msg.msgType + 1; + pResp->info.ahandle = pMsg->ctx ? pMsg->ctx->ahandle : NULL; + pResp->info.traceId = pMsg->msg.info.traceId; + + return 0; +} + void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) { STrans* pTransInst = pThrd->pTransInst; STransConnCtx* pCtx = pMsg->ctx; @@ -920,13 +932,8 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) { SCliConn* conn = cliGetConn(pMsg, pThrd, &ignore); if (ignore == true) { // persist conn already release by server - STransMsg resp = {0}; - resp.code = TSDB_CODE_RPC_BROKEN_LINK; - resp.msgType = pMsg->msg.msgType + 1; - - resp.info.ahandle = pMsg && pMsg->ctx ? pMsg->ctx->ahandle : NULL; - resp.info.traceId = pMsg->msg.info.traceId; - + STransMsg resp; + cliBuildExceptResp(pMsg, &resp); pTransInst->cfp(pTransInst->parent, &resp, NULL); destroyCmsg(pMsg); return; @@ -991,9 +998,6 @@ static void cliAsyncCb(uv_async_t* handle) { QUEUE_REMOVE(h); SCliMsg* pMsg = QUEUE_DATA(h, SCliMsg, q); - if (pMsg == NULL) { - continue; - } (*cliAsyncHandle[pMsg->type])(pMsg, pThrd); count++; } @@ -1035,24 +1039,58 @@ static void cliPrepareCb(uv_prepare_t* handle) { if (thrd->stopMsg != NULL) cliHandleQuit(thrd->stopMsg, thrd); } +void cliDestroyConnMsgs(SCliConn* conn, bool destroy) { + transCtxCleanup(&conn->ctx); + cliReleaseUnfinishedMsg(conn); + if (destroy == 1) { + transQueueDestroy(&conn->cliMsgs); + } else { + transQueueClear(&conn->cliMsgs); + } +} + +void cliIteraConnMsgs(SCliConn* conn) { + SCliThrd* pThrd = conn->hostThrd; + STrans* pTransInst = pThrd->pTransInst; + + for (int i = 0; i < transQueueSize(&conn->cliMsgs); i++) { + SCliMsg* cmsg = transQueueGet(&conn->cliMsgs, i); + if (cmsg->type == Release || REQUEST_NO_RESP(&cmsg->msg) || cmsg->msg.msgType == TDMT_SCH_DROP_TASK) { + continue; + } + + STransMsg resp = {0}; + if (-1 == cliBuildExceptResp(cmsg, &resp)) { + continue; + } + pTransInst->cfp(pTransInst->parent, &resp, NULL); + + cmsg->ctx->ahandle = NULL; + } +} bool cliRecvReleaseReq(SCliConn* conn, STransMsgHead* pHead) { if (pHead->release == 1 && (pHead->msgLen) == sizeof(*pHead)) { uint64_t ahandle = pHead->ahandle; + tDebug("ahandle = %" PRIu64 "", ahandle); SCliMsg* pMsg = NULL; CONN_GET_MSGCTX_BY_AHANDLE(conn, ahandle); + transClearBuffer(&conn->readBuf); transFreeMsg(transContFromHead((char*)pHead)); - if (transQueueSize(&conn->cliMsgs) > 0 && ahandle == 0) { - SCliMsg* cliMsg = transQueueGet(&conn->cliMsgs, 0); - if (cliMsg->type == Release) return true; + + for (int i = 0; ahandle == 0 && i < transQueueSize(&conn->cliMsgs); i++) { + SCliMsg* cliMsg = transQueueGet(&conn->cliMsgs, i); + if (cliMsg->type == Release) { + assert(pMsg == NULL); + return true; + } } + + cliIteraConnMsgs(conn); + tDebug("%s conn %p receive release request, refId:%" PRId64 "", CONN_GET_INST_LABEL(conn), conn, conn->refId); - if (T_REF_VAL_GET(conn) > 1) { - transUnrefCliHandle(conn); - } destroyCmsg(pMsg); - cliReleaseUnfinishedMsg(conn); - transQueueClear(&conn->cliMsgs); + addConnToPool(((SCliThrd*)conn->hostThrd)->pool, conn); return true; } diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 207b967923..46046b2a95 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -492,7 +492,6 @@ void uvWorkerAsyncCb(uv_async_t* handle) { // release handle to rpc init if (msg->type == Quit) { (*transAsyncHandle[msg->type])(msg, pThrd); - continue; } else { STransMsg transMsg = msg->msg; @@ -771,7 +770,7 @@ static bool addHandleToWorkloop(SWorkThrd* pThrd, char* pipeName) { // conn set QUEUE_INIT(&pThrd->conn); - pThrd->asyncPool = transAsyncPoolCreate(pThrd->loop, 1, pThrd, uvWorkerAsyncCb); + pThrd->asyncPool = transAsyncPoolCreate(pThrd->loop, 5, pThrd, uvWorkerAsyncCb); uv_pipe_connect(&pThrd->connect_req, pThrd->pipe, pipeName, uvOnPipeConnectionCb); // uv_read_start((uv_stream_t*)pThrd->pipe, uvAllocConnBufferCb, uvOnConnectionCb); return true;