From b77b767a02b6f461b3352834763639e5683b5a92 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 3 Sep 2020 13:53:09 +0800 Subject: [PATCH 1/3] [td-1317] --- src/query/src/qExecutor.c | 38 +++++++++---------- src/tsdb/src/tsdbRead.c | 9 ++++- tests/script/general/parser/lastrow_query.sim | 7 +++- 3 files changed, 32 insertions(+), 22 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 41daed087c..df4bb009bc 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -1951,36 +1951,36 @@ static void changeExecuteScanOrder(SQInfo *pQInfo, SQueryTableMsg* pQueryMsg, bo // todo handle the case the the order irrelevant query type mixed up with order critical query type // descending order query for last_row query - if (isFirstLastRowQuery(pQuery)) { + if (isFirstLastRowQuery(pQuery) && !QUERY_IS_ASC_QUERY(pQuery)) { qDebug("QInfo:%p scan order changed for last_row query, old:%d, new:%d", GET_QINFO_ADDR(pQuery), pQuery->order.order, TSDB_ORDER_ASC); + SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); pQuery->order.order = TSDB_ORDER_ASC; - if (pQuery->window.skey > pQuery->window.ekey) { - SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); - } - - return; - } - - if (isGroupbyNormalCol(pQuery->pGroupbyExpr) && pQuery->order.order == TSDB_ORDER_DESC) { - pQuery->order.order = TSDB_ORDER_ASC; - if (pQuery->window.skey > pQuery->window.ekey) { - SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); - } + assert (pQuery->window.skey <= pQuery->window.ekey); doExchangeTimeWindow(pQInfo, &pQuery->window); return; } - if (isPointInterpoQuery(pQuery) && pQuery->intervalTime == 0) { - if (!QUERY_IS_ASC_QUERY(pQuery)) { - qDebug(msg, GET_QINFO_ADDR(pQuery), "interp", pQuery->order.order, TSDB_ORDER_ASC, pQuery->window.skey, - pQuery->window.ekey, pQuery->window.ekey, pQuery->window.skey); - SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); - } + if (isGroupbyNormalCol(pQuery->pGroupbyExpr) && !QUERY_IS_ASC_QUERY(pQuery)) { + pQuery->order.order = TSDB_ORDER_ASC; + SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); + assert (pQuery->window.skey <= pQuery->window.ekey); + + doExchangeTimeWindow(pQInfo, &pQuery->window); + return; + } + + if (isPointInterpoQuery(pQuery) && (pQuery->intervalTime == 0) && !QUERY_IS_ASC_QUERY(pQuery)) { + qDebug(msg, GET_QINFO_ADDR(pQuery), "interp", pQuery->order.order, TSDB_ORDER_ASC, pQuery->window.skey, + pQuery->window.ekey, pQuery->window.ekey, pQuery->window.skey); + SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY); pQuery->order.order = TSDB_ORDER_ASC; + + assert (pQuery->window.skey <= pQuery->window.ekey); + doExchangeTimeWindow(pQInfo, &pQuery->window); return; } diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index f8ff25ddab..d8f8d8f9f1 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -295,9 +295,10 @@ out_of_memory: } TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, void* qinfo) { - pCond->order = TSDB_ORDER_ASC; pCond->twindow = changeTableGroupByLastrow(groupList); STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo); + + assert(pCond->order == TSDB_ORDER_ASC && pCond->twindow.skey <= pCond->twindow.ekey); return pQueryHandle; } @@ -1982,7 +1983,6 @@ STimeWindow changeTableGroupByLastrow(STableGroupInfo *groupList) { STimeWindow window = {INT64_MAX, INT64_MIN}; // NOTE: starts from the buffer in case of descending timestamp order check data blocks - // todo consider the query time window, current last_row does not apply the query time window size_t numOfGroups = taosArrayGetSize(groupList->pGroupList); for(int32_t j = 0; j < numOfGroups; ++j) { SArray* pGroup = taosArrayGetP(groupList->pGroupList, j); @@ -2019,6 +2019,11 @@ STimeWindow changeTableGroupByLastrow(STableGroupInfo *groupList) { } } + // window does not being updated, so set the original + if (window.skey == INT64_MAX && window.ekey == INT64_MIN) { + window = TSWINDOW_INITIALIZER; + } + return window; } diff --git a/tests/script/general/parser/lastrow_query.sim b/tests/script/general/parser/lastrow_query.sim index 1459b7b470..ef00af3210 100644 --- a/tests/script/general/parser/lastrow_query.sim +++ b/tests/script/general/parser/lastrow_query.sim @@ -153,4 +153,9 @@ if $rows != 46 then return -1 endi - +print ========>td-1317, empty table last_row query crashed +sql create table t1(ts timestamp, k int) +sql select last_row(*) from t1 +if $rows != 0 then + return -1 +endi From ec5673952758ab33c211c41942bb92dd8615933f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 3 Sep 2020 16:17:48 +0800 Subject: [PATCH 2/3] [td-1317] --- src/query/src/qExecutor.c | 76 ++++++++++++------- src/tsdb/src/tsdbRead.c | 20 ++++- tests/script/general/parser/lastrow_query.sim | 15 +++- tests/script/general/parser/timestamp.sim | 2 +- .../script/general/parser/timestamp_query.sim | 25 +++++- 5 files changed, 100 insertions(+), 38 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index df4bb009bc..9e71a7421a 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -4354,6 +4354,32 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) { return true; } +static void freeTableQueryInfo(STableGroupInfo* pTableGroupInfo) { + if (pTableGroupInfo->pGroupList == NULL) { + assert(pTableGroupInfo->numOfTables == 0); + } else { + size_t numOfGroups = taosArrayGetSize(pTableGroupInfo->pGroupList); + for (int32_t i = 0; i < numOfGroups; ++i) { + SArray *p = taosArrayGetP(pTableGroupInfo->pGroupList, i); + + size_t num = taosArrayGetSize(p); + for(int32_t j = 0; j < num; ++j) { + STableQueryInfo* item = taosArrayGetP(p, j); + destroyTableQueryInfo(item); + } + + taosArrayDestroy(p); + } + + taosArrayDestroy(pTableGroupInfo->pGroupList); + pTableGroupInfo->pGroupList = NULL; + pTableGroupInfo->numOfTables = 0; + } + + taosHashCleanup(pTableGroupInfo->map); + pTableGroupInfo->map = NULL; +} + static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQuery *pQuery = pQInfo->runtimeEnv.pQuery; @@ -4389,20 +4415,22 @@ static int32_t setupQueryHandle(void* tsdb, SQInfo* pQInfo, bool isSTableQuery) terrno = TSDB_CODE_SUCCESS; if (isFirstLastRowQuery(pQuery)) { pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(tsdb, &cond, &pQInfo->tableGroupInfo, pQInfo); + if (pRuntimeEnv->pQueryHandle == NULL) { // no data in current stable, clear all + freeTableQueryInfo(&pQInfo->tableqinfoGroupInfo); + } else { // update the query time window + pQuery->window = cond.twindow; - // update the query time window - pQuery->window = cond.twindow; + size_t numOfGroups = GET_NUM_OF_TABLEGROUP(pQInfo); + for (int32_t i = 0; i < numOfGroups; ++i) { + SArray *group = GET_TABLEGROUP(pQInfo, i); - size_t numOfGroups = GET_NUM_OF_TABLEGROUP(pQInfo); - for(int32_t i = 0; i < numOfGroups; ++i) { - SArray *group = GET_TABLEGROUP(pQInfo, i); + size_t t = taosArrayGetSize(group); + for (int32_t j = 0; j < t; ++j) { + STableQueryInfo *pCheckInfo = taosArrayGetP(group, j); - size_t t = taosArrayGetSize(group); - for (int32_t j = 0; j < t; ++j) { - STableQueryInfo *pCheckInfo = taosArrayGetP(group, j); - - pCheckInfo->win = pQuery->window; - pCheckInfo->lastKey = pCheckInfo->win.skey; + pCheckInfo->win = pQuery->window; + pCheckInfo->lastKey = pCheckInfo->win.skey; + } } } } else if (isPointInterpoQuery(pQuery)) { @@ -4456,6 +4484,12 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo return code; } + if (pQInfo->tableqinfoGroupInfo.numOfTables == 0) { + qDebug("QInfo:%p no table qualified for tag filter, abort query", pQInfo); + setQueryStatus(pQuery, QUERY_COMPLETED); + return TSDB_CODE_SUCCESS; + } + pQInfo->tsdb = tsdb; pQInfo->vgId = vgId; @@ -6349,29 +6383,13 @@ static void freeQInfo(SQInfo *pQInfo) { taosTFree(pQuery); } - // todo refactor, extract method to destroytableDataInfo - if (pQInfo->tableqinfoGroupInfo.pGroupList != NULL) { - int32_t numOfGroups = (int32_t)(GET_NUM_OF_TABLEGROUP(pQInfo)); - for (int32_t i = 0; i < numOfGroups; ++i) { - SArray *p = GET_TABLEGROUP(pQInfo, i); - - size_t num = taosArrayGetSize(p); - for(int32_t j = 0; j < num; ++j) { - STableQueryInfo* item = taosArrayGetP(p, j); - destroyTableQueryInfo(item); - } - - taosArrayDestroy(p); - } - } + freeTableQueryInfo(&pQInfo->tableqinfoGroupInfo); taosTFree(pQInfo->pBuf); - taosArrayDestroy(pQInfo->tableqinfoGroupInfo.pGroupList); - taosHashCleanup(pQInfo->tableqinfoGroupInfo.map); + tsdbDestroyTableGroup(&pQInfo->tableGroupInfo); taosArrayDestroy(pQInfo->arrTableIdInfo); - pQInfo->signature = 0; qDebug("QInfo:%p QInfo is freed", pQInfo); diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index d8f8d8f9f1..ac7eba72b2 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -296,6 +296,12 @@ out_of_memory: TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, void* qinfo) { pCond->twindow = changeTableGroupByLastrow(groupList); + + // no qualified table + if (groupList->numOfTables == 0) { + return NULL; + } + STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo); assert(pCond->order == TSDB_ORDER_ASC && pCond->twindow.skey <= pCond->twindow.ekey); @@ -1982,6 +1988,8 @@ bool tsdbNextDataBlock(TsdbQueryHandleT* pHandle) { STimeWindow changeTableGroupByLastrow(STableGroupInfo *groupList) { STimeWindow window = {INT64_MAX, INT64_MIN}; + int32_t totalNumOfTable = 0; + // NOTE: starts from the buffer in case of descending timestamp order check data blocks size_t numOfGroups = taosArrayGetSize(groupList->pGroupList); for(int32_t j = 0; j < numOfGroups; ++j) { @@ -1993,8 +2001,9 @@ STimeWindow changeTableGroupByLastrow(STableGroupInfo *groupList) { size_t numOfTables = taosArrayGetSize(pGroup); for(int32_t i = 0; i < numOfTables; ++i) { STableKeyInfo* pKeyInfo = (STableKeyInfo*) taosArrayGet(pGroup, i); - TSKEY lastKey = ((STable*)(pKeyInfo->pTable))->lastKey; + // if the lastKey equals to INT64_MIN, there is no data in this table + TSKEY lastKey = ((STable*)(pKeyInfo->pTable))->lastKey; if (key < lastKey) { key = lastKey; @@ -2012,9 +2021,12 @@ STimeWindow changeTableGroupByLastrow(STableGroupInfo *groupList) { } } + // clear current group + taosArrayClear(pGroup); + // more than one table in each group, only one table left for each group - if (numOfTables > 1) { - taosArrayClear(pGroup); + if (keyInfo.pTable != NULL) { + totalNumOfTable++; taosArrayPush(pGroup, &keyInfo); } } @@ -2022,8 +2034,10 @@ STimeWindow changeTableGroupByLastrow(STableGroupInfo *groupList) { // window does not being updated, so set the original if (window.skey == INT64_MAX && window.ekey == INT64_MIN) { window = TSWINDOW_INITIALIZER; + assert(totalNumOfTable == 0); } + groupList->numOfTables = totalNumOfTable; return window; } diff --git a/tests/script/general/parser/lastrow_query.sim b/tests/script/general/parser/lastrow_query.sim index ef00af3210..5fc47ed15d 100644 --- a/tests/script/general/parser/lastrow_query.sim +++ b/tests/script/general/parser/lastrow_query.sim @@ -154,8 +154,21 @@ if $rows != 46 then endi print ========>td-1317, empty table last_row query crashed -sql create table t1(ts timestamp, k int) +sql create table m1(ts timestamp, k int) tags (a int); +sql create table t1 using m1 tags(1); +sql create table t2 using m1 tags(2); + sql select last_row(*) from t1 if $rows != 0 then return -1 endi + +sql select last_row(*) from m1 +if $rows != 0 then + return -1 +endi + +sql select last_row(*) from m1 where tbname in ('t1') +if $rows != 0 then + return -1 +endi diff --git a/tests/script/general/parser/timestamp.sim b/tests/script/general/parser/timestamp.sim index 0a86e39de0..28bbc9df0e 100644 --- a/tests/script/general/parser/timestamp.sim +++ b/tests/script/general/parser/timestamp.sim @@ -20,7 +20,7 @@ $db = $dbPrefix . $i $stb = $stbPrefix . $i sql drop database if exists $db -sql create database $db maxrows 200 cache 1024 tblocks 200 maxTables 4 +sql create database $db maxrows 200 maxTables 4 print ====== create tables sql use $db sql create table $stb (ts timestamp, c1 timestamp, c2 int) tags(t1 binary(20)) diff --git a/tests/script/general/parser/timestamp_query.sim b/tests/script/general/parser/timestamp_query.sim index 63e40d0bf7..6994b2d295 100644 --- a/tests/script/general/parser/timestamp_query.sim +++ b/tests/script/general/parser/timestamp_query.sim @@ -22,12 +22,29 @@ $tsu = $tsu - $delta $tsu = $tsu + $ts0 ##### select from supertable - $tb = $tbPrefix . 0 -sql select first(c1), last(c1) from $tb where ts >= $ts0 and ts < $tsu interval(5m) fill(value, -1) +sql select first(c1), last(c1), (1537325400 - 1537146000)/(5*60) v from $tb where ts >= $ts0 and ts < $tsu interval(5m) fill(value, -1) $res = $rowNum * 2 -$res = $res - 1 -if $rows != $res then +$n = $res - 2 +print ============>$n +if $rows != $n then + print expect $n, actual $rows return -1 endi +if $data03 != 598.000000000 then + print expect 598.000000000, actual $data03 + return -1 +endi + + +if $data13 != 598.000000000 then + print expect 598.000000000, actual $data03 + return -1 +endi + +sql select first(c1), last(c1), (1537325400 - 1537146000)/(5*60) v from $tb where ts >= $ts0 and ts < $tsu interval(5m) fill(value, NULL) +if $data13 != 598.000000000 then + print expect 598.000000000, actual $data03 + return -1 +endi \ No newline at end of file From 19b0a7ccb70dca87ca8d86669c5f8a25413b6a2a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 3 Sep 2020 17:05:07 +0800 Subject: [PATCH 3/3] [td-1322] --- src/query/src/qPercentile.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/query/src/qPercentile.c b/src/query/src/qPercentile.c index c6eb836c61..3a484bf955 100644 --- a/src/query/src/qPercentile.c +++ b/src/query/src/qPercentile.c @@ -154,9 +154,14 @@ int32_t tBucketBigIntHash(tMemBucket *pBucket, const void *value) { // todo refactor to more generic int32_t tBucketIntHash(tMemBucket *pBucket, const void *value) { - int32_t v = *(int32_t *)value; - int32_t index = -1; + int32_t v = 0; + switch(pBucket->type) { + case TSDB_DATA_TYPE_SMALLINT: v = *(int16_t*) value; break; + case TSDB_DATA_TYPE_TINYINT: v = *(int8_t*) value; break; + default: v = *(int32_t*) value;break; + } + int32_t index = -1; if (pBucket->range.iMaxVal == INT32_MIN) { /* * taking negative integer into consideration,