From aa706fd5e7439173a108a2bae5d07945c24214b3 Mon Sep 17 00:00:00 2001 From: bryanchang0603 Date: Tue, 1 Jun 2021 13:43:19 +0800 Subject: [PATCH 01/25] [TD-4463] add test case for checking keep modification --- tests/pytest/alter/alter_keep.py | 70 ++++++++++++++++++++++++++++++++ tests/pytest/fulltest.sh | 2 +- 2 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 tests/pytest/alter/alter_keep.py diff --git a/tests/pytest/alter/alter_keep.py b/tests/pytest/alter/alter_keep.py new file mode 100644 index 0000000000..e7c58a09c5 --- /dev/null +++ b/tests/pytest/alter/alter_keep.py @@ -0,0 +1,70 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +from util.log import * +from util.cases import * +from util.sql import * + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + + def run(self): + tdSql.prepare() + tdSql.execute('create table tb (ts timestamp, speed int)') + + tdSql.query('show databases') + tdSql.checkData(0,7,'3650,3650,3650') + tdSql.execute('alter database db keep 10') + tdSql.query('show databases') + tdSql.checkData(0,7,'10,10,10') + tdSql.execute('alter database db keep 50') + tdSql.query('show databases') + tdSql.checkData(0,7,'50,50,50') + tdSql.error('alter database db keep !)') + + ##TODO: test the illegal alter keep input where input < days + # tdSql.error('alter database db keep -10') + # tdSql.error('alter database db keep 0') + + ##TODO: test keep keep hot alter, cannot be tested for now as test.py's output + ## is inconsistent with the actual output. + + # tdSql.execute('insert into tb values (now, 10)') + # tdSql.execute('insert into tb values (now + 10m, 10)') + # tdSql.query('select * from tb') + # tdSql.checkRows(2) + # tdSql.execute('alter database db keep 40,40,40') + # os.system('systemctl restart taosd') + # tdSql.execute('insert into tb values (now-60d, 10)') + # tdSql.execute('insert into tb values (now-30d, 10)') + # tdSql.query('select * from tb') + # tdSql.showQueryResult() + # tdSql.checkRows(2) + # tdSql.execute('alter database db keep 20,20,20') + # tdSql.checkRows(3) + # os.system('systemctl restart taosd') + # tdSql.query('select * from tb') + # tdSql.checkRows(2) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/fulltest.sh b/tests/pytest/fulltest.sh index c93fbc5eb3..a546c74da6 100755 --- a/tests/pytest/fulltest.sh +++ b/tests/pytest/fulltest.sh @@ -335,5 +335,5 @@ python3 ./test.py -f tag_lite/alter_tag.py python3 test.py -f tools/taosdemoAllTest/taosdemoTestInsertWithJson.py python3 test.py -f tools/taosdemoAllTest/taosdemoTestQueryWithJson.py python3 test.py -f insert/insert_before_use_db.py - +python3 test.py -f alter/alter_keep.py #======================p4-end=============== From c55f6121a0a9c3b4f9e3146cfed43e485209eb05 Mon Sep 17 00:00:00 2001 From: bryanchang0603 Date: Tue, 1 Jun 2021 16:13:40 +0800 Subject: [PATCH 02/25] [TD-4463] moldifying alter_keep.py --- tests/pytest/alter/alter_keep.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/pytest/alter/alter_keep.py b/tests/pytest/alter/alter_keep.py index e7c58a09c5..5e1378e249 100644 --- a/tests/pytest/alter/alter_keep.py +++ b/tests/pytest/alter/alter_keep.py @@ -37,9 +37,16 @@ class TDTestCase: tdSql.checkData(0,7,'50,50,50') tdSql.error('alter database db keep !)') - ##TODO: test the illegal alter keep input where input < days - # tdSql.error('alter database db keep -10') - # tdSql.error('alter database db keep 0') + tdSql.error('alter database db keep 1') + + ## the following sql will not raise error, but will not cause error either + # based on Li Chuang's explaination, <= 0 will not cause keep>days error + tdSql.execute('alter database db keep -10') + tdSql.query('show databases') + tdSql.checkData(0,7,'50,50,50') + tdSql.execute('alter database db keep 0') + tdSql.query('show databases') + tdSql.checkData(0,7,'50,50,50') ##TODO: test keep keep hot alter, cannot be tested for now as test.py's output ## is inconsistent with the actual output. From e9a3969b8e3a4ff1ce53cf0d6610aeff336f1d8e Mon Sep 17 00:00:00 2001 From: lichuang Date: Tue, 1 Jun 2021 19:36:59 +0800 Subject: [PATCH 03/25] [TD-3963]async commit a msg to save config --- src/tsdb/inc/tsdbMemTable.h | 1 + src/tsdb/src/tsdbCommitQueue.c | 9 ++++++++- src/tsdb/src/tsdbMain.c | 2 +- src/tsdb/src/tsdbMemTable.c | 17 ++++++++++++++++- 4 files changed, 26 insertions(+), 3 deletions(-) diff --git a/src/tsdb/inc/tsdbMemTable.h b/src/tsdb/inc/tsdbMemTable.h index 6046274af4..868a10d361 100644 --- a/src/tsdb/inc/tsdbMemTable.h +++ b/src/tsdb/inc/tsdbMemTable.h @@ -66,6 +66,7 @@ int tsdbTakeMemSnapshot(STsdbRepo* pRepo, SMemSnapshot* pSnapshot, SArray* pAT void tsdbUnTakeMemSnapShot(STsdbRepo* pRepo, SMemSnapshot* pSnapshot); void* tsdbAllocBytes(STsdbRepo* pRepo, int bytes); int tsdbAsyncCommit(STsdbRepo* pRepo); +int tsdbAsyncCommitConfig(STsdbRepo* pRepo); int tsdbLoadDataFromCache(STable* pTable, SSkipListIterator* pIter, TSKEY maxKey, int maxRowsToRead, SDataCols* pCols, TKEY* filterKeys, int nFilterKeys, bool keepDup, SMergeInfo* pMergeInfo); void* tsdbCommitData(STsdbRepo* pRepo); diff --git a/src/tsdb/src/tsdbCommitQueue.c b/src/tsdb/src/tsdbCommitQueue.c index abea79bc4f..4d4585421c 100644 --- a/src/tsdb/src/tsdbCommitQueue.c +++ b/src/tsdb/src/tsdbCommitQueue.c @@ -154,6 +154,7 @@ static void *tsdbLoopCommit(void *arg) { SCommitQueue *pQueue = &tsCommitQueue; SListNode * pNode = NULL; STsdbRepo * pRepo = NULL; + bool config_changed = false; while (true) { pthread_mutex_lock(&(pQueue->lock)); @@ -177,11 +178,17 @@ static void *tsdbLoopCommit(void *arg) { pRepo = ((SCommitReq *)pNode->data)->pRepo; // check if need to apply new config + config_changed = pRepo->config_changed; if (pRepo->config_changed) { tsdbApplyRepoConfig(pRepo); } - tsdbCommitData(pRepo); + if (config_changed && pRepo->imem == NULL) { + tsem_post(&(pRepo->readyToCommit)); + } else { + tsdbCommitData(pRepo); + } + listNodeFree(pNode); } diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index afbedd5b2f..3a375ec59c 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -271,7 +271,7 @@ int32_t tsdbConfigRepo(STsdbRepo *repo, STsdbCfg *pCfg) { pthread_mutex_unlock(&repo->save_mutex); // schedule a commit msg then the new config will be applied immediatly - tsdbAsyncCommit(repo); + tsdbAsyncCommitConfig(repo); return 0; #if 0 diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 79dbb8be5d..4ca7b3b820 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -271,10 +271,25 @@ void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) { return ptr; } +int tsdbAsyncCommitConfig(STsdbRepo* pRepo) { + ASSERT(pRepo->config_changed == true); + tsem_wait(&(pRepo->readyToCommit)); + + if (pRepo->code != TSDB_CODE_SUCCESS) { + tsdbWarn("vgId:%d try to commit config when TSDB not in good state: %s", REPO_ID(pRepo), tstrerror(terrno)); + } + + if (tsdbLockRepo(pRepo) < 0) return -1; + tsdbScheduleCommit(pRepo); + if (tsdbUnlockRepo(pRepo) < 0) return -1; + + return 0; +} + int tsdbAsyncCommit(STsdbRepo *pRepo) { tsem_wait(&(pRepo->readyToCommit)); - //ASSERT(pRepo->imem == NULL); + ASSERT(pRepo->imem == NULL); if (pRepo->mem == NULL) { tsem_post(&(pRepo->readyToCommit)); return 0; From e0b4bb2442f114b9d6a71e12d0be5dc90e1a8f90 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 1 Jun 2021 22:30:12 +0800 Subject: [PATCH 04/25] [td-255]fix memory leak in test cases. --- tests/examples/c/apitest.c | 39 +++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/tests/examples/c/apitest.c b/tests/examples/c/apitest.c index b2411d1212..2961750efc 100644 --- a/tests/examples/c/apitest.c +++ b/tests/examples/c/apitest.c @@ -402,10 +402,6 @@ void verify_prepare(TAOS* taos) { taos_stmt_close(stmt); } - - - - void verify_prepare2(TAOS* taos) { TAOS_RES* result = taos_query(taos, "drop database if exists test;"); taos_free_result(result); @@ -531,10 +527,9 @@ void verify_prepare2(TAOS* taos) { params[9].is_null = is_null; params[9].num = 10; - - sql = "insert into ? values(?,?,?,?,?,?,?,?,?,?)"; + sql = "insert into ? (ts, b, v1, v2, v4, v8, f4, f8, bin, blob) values(?,?,?,?,?,?,?,?,?,?)"; code = taos_stmt_prepare(stmt, sql, 0); - if (code != 0){ + if (code != 0) { printf("\033[31mfailed to execute taos_stmt_prepare. code:0x%x\033[0m\n", code); } @@ -577,10 +572,9 @@ void verify_prepare2(TAOS* taos) { printf("\033[31mfailed to execute insert statement.\033[0m\n"); return; } + taos_stmt_close(stmt); - - // query the records stmt = taos_stmt_init(taos); taos_stmt_prepare(stmt, "SELECT * FROM m1 WHERE v1 > ? AND v2 < ?", 0); @@ -623,10 +617,17 @@ void verify_prepare2(TAOS* taos) { taos_free_result(result); taos_stmt_close(stmt); + + free(t8_len); + free(t16_len); + free(t32_len); + free(t64_len); + free(float_len); + free(double_len); + free(bin_len); + free(blob_len); } - - void verify_prepare3(TAOS* taos) { TAOS_RES* result = taos_query(taos, "drop database if exists test;"); taos_free_result(result); @@ -810,7 +811,6 @@ void verify_prepare3(TAOS* taos) { blob_len[i] = (int32_t)strlen(v.blob[i]); } - taos_stmt_bind_param_batch(stmt, params); taos_stmt_add_batch(stmt); @@ -852,10 +852,12 @@ void verify_prepare3(TAOS* taos) { int rows = 0; int num_fields = taos_num_fields(result); TAOS_FIELD *fields = taos_fetch_fields(result); - char temp[256]; + char temp[256] = {0}; // fetch the records row by row while ((row = taos_fetch_row(result))) { + memset(temp, 0, sizeof(temp)/sizeof(temp[0])); + rows++; taos_print_row(temp, row, fields, num_fields); printf("%s\n", temp); @@ -863,10 +865,17 @@ void verify_prepare3(TAOS* taos) { taos_free_result(result); taos_stmt_close(stmt); + + free(t8_len); + free(t16_len); + free(t32_len); + free(t64_len); + free(float_len); + free(double_len); + free(bin_len); + free(blob_len); } - - void retrieve_callback(void *param, TAOS_RES *tres, int numOfRows) { if (numOfRows > 0) { From cdf2d3f4378f3c94664e9cf17f96aa885c3a0959 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 2 Jun 2021 15:20:44 +0800 Subject: [PATCH 05/25] [td-4111]: fix bug for irate function. --- src/client/src/tscSQLParser.c | 14 +-- src/query/inc/qAggMain.h | 28 ++--- src/query/src/qAggMain.c | 218 +++++++--------------------------- src/query/src/qExecutor.c | 17 --- 4 files changed, 59 insertions(+), 218 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index b28ff5e3e9..c3a7c7c89c 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -2144,10 +2144,6 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col case TSDB_FUNC_AVG: case TSDB_FUNC_RATE: case TSDB_FUNC_IRATE: - case TSDB_FUNC_SUM_RATE: - case TSDB_FUNC_SUM_IRATE: - case TSDB_FUNC_AVG_RATE: - case TSDB_FUNC_AVG_IRATE: case TSDB_FUNC_TWA: case TSDB_FUNC_MIN: case TSDB_FUNC_MAX: @@ -2212,8 +2208,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col SExprInfo* pExpr = tscExprAppend(pQueryInfo, functionId, &index, resultType, resultSize, getNewResColId(pCmd), resultSize, false); - if (functionId == TSDB_FUNC_LEASTSQR) { - /* set the leastsquares parameters */ + if (functionId == TSDB_FUNC_LEASTSQR) { // set the leastsquares parameters char val[8] = {0}; if (tVariantDump(&pParamElem[1].pNode->value, val, TSDB_DATA_TYPE_DOUBLE, true) < 0) { return TSDB_CODE_TSC_INVALID_OPERATION; @@ -2227,6 +2222,11 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col } tscExprAddParams(&pExpr->base, val, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); + } else if (functionId == TSDB_FUNC_IRATE) { + STableComInfo info = tscGetTableInfo(pTableMetaInfo->pTableMeta); + int64_t prec = info.precision; + + tscExprAddParams(&pExpr->base, (char*)&prec, TSDB_DATA_TYPE_BIGINT, LONG_BYTES); } SColumnList ids = createColumnList(1, index.tableIndex, index.columnIndex); @@ -2875,7 +2875,7 @@ int32_t tscTansformFuncForSTableQuery(SQueryInfo* pQueryInfo) { if ((functionId >= TSDB_FUNC_SUM && functionId <= TSDB_FUNC_TWA) || (functionId >= TSDB_FUNC_FIRST_DST && functionId <= TSDB_FUNC_STDDEV_DST) || - (functionId >= TSDB_FUNC_RATE && functionId <= TSDB_FUNC_AVG_IRATE)) { + (functionId >= TSDB_FUNC_RATE && functionId <= TSDB_FUNC_IRATE)) { if (getResultDataInfo(pSrcSchema->type, pSrcSchema->bytes, functionId, (int32_t)pExpr->base.param[0].i64, &type, &bytes, &interBytes, 0, true) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_OPERATION; diff --git a/src/query/inc/qAggMain.h b/src/query/inc/qAggMain.h index bdccd4eb3c..47c61fc444 100644 --- a/src/query/inc/qAggMain.h +++ b/src/query/inc/qAggMain.h @@ -65,24 +65,18 @@ extern "C" { #define TSDB_FUNC_RATE 29 #define TSDB_FUNC_IRATE 30 -#define TSDB_FUNC_SUM_RATE 31 -#define TSDB_FUNC_SUM_IRATE 32 -#define TSDB_FUNC_AVG_RATE 33 -#define TSDB_FUNC_AVG_IRATE 34 - -#define TSDB_FUNC_TID_TAG 35 -#define TSDB_FUNC_BLKINFO 36 - -#define TSDB_FUNC_HISTOGRAM 37 -#define TSDB_FUNC_HLL 38 -#define TSDB_FUNC_MODE 39 -#define TSDB_FUNC_SAMPLE 40 -#define TSDB_FUNC_CEIL 41 -#define TSDB_FUNC_FLOOR 42 -#define TSDB_FUNC_ROUND 43 -#define TSDB_FUNC_MAVG 44 -#define TSDB_FUNC_CSUM 45 +#define TSDB_FUNC_TID_TAG 31 +#define TSDB_FUNC_BLKINFO 32 +#define TSDB_FUNC_HISTOGRAM 33 +#define TSDB_FUNC_HLL 34 +#define TSDB_FUNC_MODE 35 +#define TSDB_FUNC_SAMPLE 36 +#define TSDB_FUNC_CEIL 37 +#define TSDB_FUNC_FLOOR 38 +#define TSDB_FUNC_ROUND 39 +#define TSDB_FUNC_MAVG 40 +#define TSDB_FUNC_CSUM 41 #define TSDB_FUNCSTATE_SO 0x1u // single output #define TSDB_FUNCSTATE_MO 0x2u // dynamic number of output, not multinumber of output e.g., TOP/BOTTOM diff --git a/src/query/src/qAggMain.c b/src/query/src/qAggMain.c index be0716ce99..de8ffea06a 100644 --- a/src/query/src/qAggMain.c +++ b/src/query/src/qAggMain.c @@ -159,8 +159,6 @@ typedef struct SRateInfo { TSKEY lastKey; int8_t hasResult; // flag to denote has value bool isIRate; // true for IRate functions, false for Rate functions - int64_t num; // for sum/avg - double sum; // for sum/avg } SRateInfo; int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionId, int32_t param, int16_t *type, @@ -238,7 +236,7 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI *interBytes = *bytes; return TSDB_CODE_SUCCESS; - } else if (functionId >= TSDB_FUNC_RATE && functionId <= TSDB_FUNC_AVG_IRATE) { + } else if (functionId >= TSDB_FUNC_RATE && functionId <= TSDB_FUNC_IRATE) { *type = TSDB_DATA_TYPE_DOUBLE; *bytes = sizeof(SRateInfo); *interBytes = sizeof(SRateInfo); @@ -304,7 +302,7 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI *type = TSDB_DATA_TYPE_DOUBLE; *bytes = sizeof(double); *interBytes = sizeof(SAvgInfo); - } else if (functionId >= TSDB_FUNC_RATE && functionId <= TSDB_FUNC_AVG_IRATE) { + } else if (functionId >= TSDB_FUNC_RATE && functionId <= TSDB_FUNC_IRATE) { *type = TSDB_DATA_TYPE_DOUBLE; *bytes = sizeof(double); *interBytes = sizeof(SRateInfo); @@ -4479,16 +4477,18 @@ static void ts_comp_finalize(SQLFunctionCtx *pCtx) { } ////////////////////////////////////////////////////////////////////////////////////////////// -// RATE functions - -static double do_calc_rate(const SRateInfo* pRateInfo) { - if ((INT64_MIN == pRateInfo->lastKey) || (INT64_MIN == pRateInfo->firstKey) || (pRateInfo->firstKey >= pRateInfo->lastKey)) { +// rate functions +static double do_calc_rate(const SRateInfo* pRateInfo, int64_t tickPerSec) { + if ((INT64_MIN == pRateInfo->lastKey) || (INT64_MIN == pRateInfo->firstKey) || + (pRateInfo->firstKey >= pRateInfo->lastKey)) { return 0; } - + int64_t diff = 0; if (pRateInfo->isIRate) { + // If the previous value of the last is greater than the last value, only keep the last point instead of the delta + // value between two values. diff = pRateInfo->lastValue; if (diff >= pRateInfo->firstValue) { diff -= pRateInfo->firstValue; @@ -4501,14 +4501,11 @@ static double do_calc_rate(const SRateInfo* pRateInfo) { } int64_t duration = pRateInfo->lastKey - pRateInfo->firstKey; - duration = (duration + 500) / 1000; - - double resultVal = ((double)diff) / duration; - - qDebug("do_calc_rate() isIRate:%d firstKey:%" PRId64 " lastKey:%" PRId64 " firstValue:%" PRId64 " lastValue:%" PRId64 " CorrectionValue:%" PRId64 " resultVal:%f", - pRateInfo->isIRate, pRateInfo->firstKey, pRateInfo->lastKey, pRateInfo->firstValue, pRateInfo->lastValue, pRateInfo->CorrectionValue, resultVal); - - return resultVal; + if (duration == 0) { + return 0; + } + + return (duration > 0)? ((double)diff) / (duration/((double) tickPerSec)):0; } static bool rate_function_setup(SQLFunctionCtx *pCtx) { @@ -4516,19 +4513,17 @@ static bool rate_function_setup(SQLFunctionCtx *pCtx) { return false; } - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); //->pOutput + pCtx->outputBytes; - SRateInfo * pInfo = GET_ROWCELL_INTERBUF(pResInfo); - + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + + SRateInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); pInfo->CorrectionValue = 0; pInfo->firstKey = INT64_MIN; pInfo->lastKey = INT64_MIN; pInfo->firstValue = INT64_MIN; pInfo->lastValue = INT64_MIN; - pInfo->num = 0; - pInfo->sum = 0; - + pInfo->hasResult = 0; - pInfo->isIRate = ((pCtx->functionId == TSDB_FUNC_IRATE) || (pCtx->functionId == TSDB_FUNC_SUM_IRATE) || (pCtx->functionId == TSDB_FUNC_AVG_IRATE)); + pInfo->isIRate = (pCtx->functionId == TSDB_FUNC_IRATE); return true; } @@ -4647,18 +4642,13 @@ static void rate_finalizer(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); - qDebug("%p isIRate:%d firstKey:%" PRId64 " lastKey:%" PRId64 " firstValue:%" PRId64 " lastValue:%" PRId64 " CorrectionValue:%" PRId64 " hasResult:%d", - pCtx, pRateInfo->isIRate, pRateInfo->firstKey, pRateInfo->lastKey, pRateInfo->firstValue, pRateInfo->lastValue, pRateInfo->CorrectionValue, pRateInfo->hasResult); - if (pRateInfo->hasResult != DATA_SET_FLAG) { setNull(pCtx->pOutput, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); return; } - *(double*)pCtx->pOutput = do_calc_rate(pRateInfo); - - qDebug("rate_finalizer() output result:%f", *(double *)pCtx->pOutput); - + *(double*)pCtx->pOutput = do_calc_rate(pRateInfo, TSDB_TICK_PER_SECOND(pCtx->param[0].i64)); + // cannot set the numOfIteratedElems again since it is set during previous iteration pResInfo->numOfRes = 1; pResInfo->hasResult = DATA_SET_FLAG; @@ -4667,22 +4657,15 @@ static void rate_finalizer(SQLFunctionCtx *pCtx) { } static void irate_function(SQLFunctionCtx *pCtx) { - - int32_t notNullElems = 0; - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); - TSKEY *primaryKey = GET_TS_LIST(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - qDebug("%p irate_function() size:%d, hasNull:%d", pCtx, pCtx->size, pCtx->hasNull); - - if (pCtx->size < 1) { - return; - } + int32_t notNullElems = 0; + SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); + TSKEY *primaryKey = GET_TS_LIST(pCtx); for (int32_t i = pCtx->size - 1; i >= 0; --i) { char *pData = GET_INPUT_DATA(pCtx, i); if (pCtx->hasNull && isNull(pData, pCtx->inputType)) { - qDebug("%p irate_function() index of null data:%d", pCtx, i); continue; } @@ -4690,21 +4673,16 @@ static void irate_function(SQLFunctionCtx *pCtx) { int64_t v = 0; GET_TYPED_DATA(v, int64_t, pCtx->inputType, pData); - - // TODO: calc once if only call this function once ???? - if ((INT64_MIN == pRateInfo->lastKey) || (INT64_MIN == pRateInfo->lastValue)) { + + if ((INT64_MIN == pRateInfo->lastKey) || primaryKey[i] > pRateInfo->lastKey) { pRateInfo->lastValue = v; pRateInfo->lastKey = primaryKey[i]; - - qDebug("%p irate_function() lastValue:%" PRId64 " lastKey:%" PRId64, pCtx, pRateInfo->lastValue, pRateInfo->lastKey); continue; } - if ((INT64_MIN == pRateInfo->firstKey) || (INT64_MIN == pRateInfo->firstValue)){ + if ((INT64_MIN == pRateInfo->firstKey) || primaryKey[i] > pRateInfo->firstKey) { pRateInfo->firstValue = v; pRateInfo->firstKey = primaryKey[i]; - - qDebug("%p irate_function() firstValue:%" PRId64 " firstKey:%" PRId64, pCtx, pRateInfo->firstValue, pRateInfo->firstKey); break; } } @@ -4756,68 +4734,6 @@ static void irate_function_f(SQLFunctionCtx *pCtx, int32_t index) { } } -static void do_sumrate_merge(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - assert(pCtx->stableQuery); - - SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); - char * input = GET_INPUT_DATA_LIST(pCtx); - - for (int32_t i = 0; i < pCtx->size; ++i, input += pCtx->inputBytes) { - SRateInfo *pInput = (SRateInfo *)input; - - qDebug("%p do_sumrate_merge() hasResult:%d input num:%" PRId64 " input sum:%f total num:%" PRId64 " total sum:%f", pCtx, pInput->hasResult, pInput->num, pInput->sum, pRateInfo->num, pRateInfo->sum); - - if (pInput->hasResult != DATA_SET_FLAG) { - continue; - } else if (pInput->num == 0) { - pRateInfo->sum += do_calc_rate(pInput); - pRateInfo->num++; - } else { - pRateInfo->sum += pInput->sum; - pRateInfo->num += pInput->num; - } - pRateInfo->hasResult = DATA_SET_FLAG; - } - - // if the data set hasResult is not set, the result is null - if (DATA_SET_FLAG == pRateInfo->hasResult) { - pResInfo->hasResult = DATA_SET_FLAG; - SET_VAL(pCtx, pRateInfo->num, 1); - memcpy(pCtx->pOutput, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo)); - } -} - -static void sumrate_func_merge(SQLFunctionCtx *pCtx) { - qDebug("%p sumrate_func_merge() process ...", pCtx); - do_sumrate_merge(pCtx); -} - -static void sumrate_finalizer(SQLFunctionCtx *pCtx) { - SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); - - qDebug("%p sumrate_finalizer() superTableQ:%d num:%" PRId64 " sum:%f hasResult:%d", pCtx, pCtx->stableQuery, pRateInfo->num, pRateInfo->sum, pRateInfo->hasResult); - - if (pRateInfo->hasResult != DATA_SET_FLAG) { - setNull(pCtx->pOutput, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); - return; - } - - if (pRateInfo->num == 0) { - // from meter - *(double*)pCtx->pOutput = do_calc_rate(pRateInfo); - } else if (pCtx->functionId == TSDB_FUNC_SUM_RATE || pCtx->functionId == TSDB_FUNC_SUM_IRATE) { - *(double*)pCtx->pOutput = pRateInfo->sum; - } else { - *(double*)pCtx->pOutput = pRateInfo->sum / pRateInfo->num; - } - - pResInfo->numOfRes = 1; - pResInfo->hasResult = DATA_SET_FLAG; - doFinalizer(pCtx); -} - void blockInfo_func(SQLFunctionCtx* pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); STableBlockDist* pDist = (STableBlockDist*) GET_ROWCELL_INTERBUF(pResInfo); @@ -4987,8 +4903,8 @@ int32_t functionCompatList[] = { 4, -1, -1, 1, 1, 1, 1, 1, 1, -1, // tag, colprj, tagprj, arithmetic, diff, first_dist, last_dist, interp rate irate 1, 1, 1, 1, -1, 1, 1, 5, 1, 1, - // sum_rate, sum_irate, avg_rate, avg_irate, tid_tag, blk_info - 1, 1, 1, 1, 6, 7 + // tid_tag, blk_info + 6, 7 }; SAggFunctionInfo aAggs[] = {{ @@ -5400,58 +5316,6 @@ SAggFunctionInfo aAggs[] = {{ }, { // 31 - "sum_rate", - TSDB_FUNC_SUM_RATE, - TSDB_FUNC_SUM_RATE, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, - rate_function_setup, - rate_function, - rate_function_f, - sumrate_finalizer, - sumrate_func_merge, - dataBlockRequired, - }, - { - // 32 - "sum_irate", - TSDB_FUNC_SUM_IRATE, - TSDB_FUNC_SUM_IRATE, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, - rate_function_setup, - irate_function, - irate_function_f, - sumrate_finalizer, - sumrate_func_merge, - dataBlockRequired, - }, - { - // 33 - "avg_rate", - TSDB_FUNC_AVG_RATE, - TSDB_FUNC_AVG_RATE, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, - rate_function_setup, - rate_function, - rate_function_f, - sumrate_finalizer, - sumrate_func_merge, - dataBlockRequired, - }, - { - // 34 - "avg_irate", - TSDB_FUNC_AVG_IRATE, - TSDB_FUNC_AVG_IRATE, - TSDB_BASE_FUNC_SO | TSDB_FUNCSTATE_NEED_TS, - rate_function_setup, - irate_function, - irate_function_f, - sumrate_finalizer, - sumrate_func_merge, - dataBlockRequired, - }, - { - // 35 "tbid", // return table id and the corresponding tags for join match and subscribe TSDB_FUNC_TID_TAG, TSDB_FUNC_TID_TAG, @@ -5464,15 +5328,15 @@ SAggFunctionInfo aAggs[] = {{ dataBlockRequired, }, { - // 35 - "_block_dist", // return table id and the corresponding tags for join match and subscribe - TSDB_FUNC_BLKINFO, - TSDB_FUNC_BLKINFO, - TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STABLE, - function_setup, - blockInfo_func, - noop2, - blockinfo_func_finalizer, - block_func_merge, - dataBlockRequired, + // 32 + "_block_dist", // return table id and the corresponding tags for join match and subscribe + TSDB_FUNC_BLKINFO, + TSDB_FUNC_BLKINFO, + TSDB_FUNCSTATE_SO | TSDB_FUNCSTATE_STABLE, + function_setup, + blockInfo_func, + noop2, + blockinfo_func_finalizer, + block_func_merge, + dataBlockRequired, }}; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 25e7e446bd..c1087db8b7 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -1990,23 +1990,6 @@ void setQueryKilled(SQInfo *pQInfo) { pQInfo->code = TSDB_CODE_TSC_QUERY_CANCELL // return false; //} -// TODO REFACTOR:MERGE WITH CLIENT-SIDE FUNCTION -static UNUSED_FUNC bool isSumAvgRateQuery(SQueryAttr *pQueryAttr) { - for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { - int32_t functionId = pQueryAttr->pExpr1[i].base.functionId; - if (functionId == TSDB_FUNC_TS) { - continue; - } - - if (functionId == TSDB_FUNC_SUM_RATE || functionId == TSDB_FUNC_SUM_IRATE || functionId == TSDB_FUNC_AVG_RATE || - functionId == TSDB_FUNC_AVG_IRATE) { - return true; - } - } - - return false; -} - static bool isFirstLastRowQuery(SQueryAttr *pQueryAttr) { for (int32_t i = 0; i < pQueryAttr->numOfOutput; ++i) { int32_t functionID = pQueryAttr->pExpr1[i].base.functionId; From 49e0643bf2bf4cdf8d95d09ddf70fb04cdcce46e Mon Sep 17 00:00:00 2001 From: bryanchang0603 Date: Wed, 2 Jun 2021 16:02:06 +0800 Subject: [PATCH 06/25] [TD-4463] update boundary case and error case --- tests/pytest/alter/alter_keep.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/pytest/alter/alter_keep.py b/tests/pytest/alter/alter_keep.py index 5e1378e249..37a8a8da55 100644 --- a/tests/pytest/alter/alter_keep.py +++ b/tests/pytest/alter/alter_keep.py @@ -41,16 +41,17 @@ class TDTestCase: ## the following sql will not raise error, but will not cause error either # based on Li Chuang's explaination, <= 0 will not cause keep>days error - tdSql.execute('alter database db keep -10') - tdSql.query('show databases') - tdSql.checkData(0,7,'50,50,50') - tdSql.execute('alter database db keep 0') + # tdSql.error('alter database db keep -10') + # tdSql.query('show databases') + # tdSql.checkData(0,7,'50,50,50') + # tdSql.error('alter database db keep 0') + # tdSql.error('alter database db keep 0.1') + tdSql.error('alter database db keep 10.1') tdSql.query('show databases') tdSql.checkData(0,7,'50,50,50') ##TODO: test keep keep hot alter, cannot be tested for now as test.py's output ## is inconsistent with the actual output. - # tdSql.execute('insert into tb values (now, 10)') # tdSql.execute('insert into tb values (now + 10m, 10)') # tdSql.query('select * from tb') From 41befca45de4dffb697bc6b070ef068728a90746 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 2 Jun 2021 16:12:26 +0800 Subject: [PATCH 07/25] [td-255]fix bug in session window. --- src/query/src/qExecutor.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index c1087db8b7..9e1534eb24 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -1347,8 +1347,8 @@ static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSWindowOperatorInf pInfo->start = j; } else if (tsList[j] - pInfo->prevTs <= gap) { pInfo->curWindow.ekey = tsList[j]; - //pInfo->prevTs = tsList[j]; pInfo->numOfRows += 1; + pInfo->prevTs = tsList[j]; if (j == 0 && pInfo->start != 0) { pInfo->numOfRows = 1; pInfo->start = 0; From 0d7a58f0ca8d1d820c2590508cfa30bedf44d4d3 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 2 Jun 2021 17:52:16 +0800 Subject: [PATCH 08/25] [td-255] add test cases, fix compiler error. --- src/query/src/qAggMain.c | 29 ++---- tests/script/general/parser/function.sim | 118 +++++++++++++++++++++++ 2 files changed, 127 insertions(+), 20 deletions(-) diff --git a/src/query/src/qAggMain.c b/src/query/src/qAggMain.c index de8ffea06a..08c057ecc6 100644 --- a/src/query/src/qAggMain.c +++ b/src/query/src/qAggMain.c @@ -153,9 +153,9 @@ typedef struct STSCompInfo { typedef struct SRateInfo { int64_t CorrectionValue; - int64_t firstValue; + double firstValue; TSKEY firstKey; - int64_t lastValue; + double lastValue; TSKEY lastKey; int8_t hasResult; // flag to denote has value bool isIRate; // true for IRate functions, false for Rate functions @@ -4484,8 +4484,7 @@ static double do_calc_rate(const SRateInfo* pRateInfo, int64_t tickPerSec) { return 0; } - int64_t diff = 0; - + double diff = 0; if (pRateInfo->isIRate) { // If the previous value of the last is greater than the last value, only keep the last point instead of the delta // value between two values. @@ -4551,8 +4550,6 @@ static void rate_function(SQLFunctionCtx *pCtx) { if ((INT64_MIN == pRateInfo->firstValue) || (INT64_MIN == pRateInfo->firstKey)) { pRateInfo->firstValue = v; pRateInfo->firstKey = primaryKey[i]; - - qDebug("firstValue:%" PRId64 " firstKey:%" PRId64, pRateInfo->firstValue, pRateInfo->firstKey); } if (INT64_MIN == pRateInfo->lastValue) { @@ -4564,7 +4561,6 @@ static void rate_function(SQLFunctionCtx *pCtx) { pRateInfo->lastValue = v; pRateInfo->lastKey = primaryKey[i]; - qDebug("lastValue:%" PRId64 " lastKey:%" PRId64, pRateInfo->lastValue, pRateInfo->lastKey); } if (!pCtx->hasNull) { @@ -4612,8 +4608,6 @@ static void rate_function_f(SQLFunctionCtx *pCtx, int32_t index) { pRateInfo->lastValue = v; pRateInfo->lastKey = primaryKey[index]; - qDebug("====%p rate_function_f() index:%d lastValue:%" PRId64 " lastKey:%" PRId64 " CorrectionValue:%" PRId64, pCtx, index, pRateInfo->lastValue, pRateInfo->lastKey, pRateInfo->CorrectionValue); - SET_VAL(pCtx, 1, 1); // set has result flag @@ -4632,10 +4626,6 @@ static void rate_func_copy(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); memcpy(GET_ROWCELL_INTERBUF(pResInfo), pCtx->pInput, (size_t)pCtx->inputBytes); pResInfo->hasResult = ((SRateInfo*)pCtx->pInput)->hasResult; - - SRateInfo* pRateInfo = (SRateInfo*)pCtx->pInput; - qDebug("%p rate_func_merge() firstKey:%" PRId64 " lastKey:%" PRId64 " firstValue:%" PRId64 " lastValue:%" PRId64 " CorrectionValue:%" PRId64 " hasResult:%d", - pCtx, pRateInfo->firstKey, pRateInfo->lastKey, pRateInfo->firstValue, pRateInfo->lastValue, pRateInfo->CorrectionValue, pRateInfo->hasResult); } static void rate_finalizer(SQLFunctionCtx *pCtx) { @@ -4671,8 +4661,8 @@ static void irate_function(SQLFunctionCtx *pCtx) { notNullElems++; - int64_t v = 0; - GET_TYPED_DATA(v, int64_t, pCtx->inputType, pData); + double v = 0; + GET_TYPED_DATA(v, double, pCtx->inputType, pData); if ((INT64_MIN == pRateInfo->lastKey) || primaryKey[i] > pRateInfo->lastKey) { pRateInfo->lastValue = v; @@ -4720,8 +4710,7 @@ static void irate_function_f(SQLFunctionCtx *pCtx, int32_t index) { pRateInfo->lastValue = v; pRateInfo->lastKey = primaryKey[index]; - qDebug("====%p irate_function_f() index:%d lastValue:%" PRId64 " lastKey:%" PRId64 " firstValue:%" PRId64 " firstKey:%" PRId64, pCtx, index, pRateInfo->lastValue, pRateInfo->lastKey, pRateInfo->firstValue , pRateInfo->firstKey); - +// qDebug("====%p irate_function_f() index:%d lastValue:%" PRId64 " lastKey:%" PRId64 " firstValue:%" PRId64 " firstKey:%" PRId64, pCtx, index, pRateInfo->lastValue, pRateInfo->lastKey, pRateInfo->firstValue , pRateInfo->firstKey); SET_VAL(pCtx, 1, 1); // set has result flag @@ -4899,10 +4888,10 @@ void blockinfo_func_finalizer(SQLFunctionCtx* pCtx) { int32_t functionCompatList[] = { // count, sum, avg, min, max, stddev, percentile, apercentile, first, last 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - // last_row,top, bottom, spread, twa, leastsqr, ts, ts_dummy, tag_dummy, ts_z + // last_row,top, bottom, spread, twa, leastsqr, ts, ts_dummy, tag_dummy, ts_comp 4, -1, -1, 1, 1, 1, 1, 1, 1, -1, - // tag, colprj, tagprj, arithmetic, diff, first_dist, last_dist, interp rate irate - 1, 1, 1, 1, -1, 1, 1, 5, 1, 1, + // tag, colprj, tagprj, arithmetic, diff, first_dist, last_dist, stddev_dst, interp rate irate + 1, 1, 1, 1, -1, 1, 1, 1, 5, 1, 1, // tid_tag, blk_info 6, 7 }; diff --git a/tests/script/general/parser/function.sim b/tests/script/general/parser/function.sim index ee5a750c88..591d5d1535 100644 --- a/tests/script/general/parser/function.sim +++ b/tests/script/general/parser/function.sim @@ -814,3 +814,121 @@ if $data00 != 1 then endi print ====================> TODO stddev + normal column filter + + +print ====================> irate +sql select irate(k) from t1 +if $rows != 1 then + return -1 +endi + +if $data00 != 0.000027778 then + return -1 +endi + +sql select irate(k) from t1 where ts>='2015-8-18 00:30:00.000' +if $rows != 1 then + return -1 +endi + +if $data00 != 0.000000000 then + print expect 0.000000000, actual $data00 + return -1 +endi + +sql select irate(k) from t1 where ts>='2015-8-18 00:06:00.000' and ts<='2015-8-18 00:12:000'; +if $rows != 1 then + return -1 +endi + +if $data00 != 0.005633334 then + return -1 +endi + +sql select irate(k) from t1 interval(10a) +if $rows != 6 then + return -1 +endi + +if $data01 != 0.000000000 then + return -1 +endi + +if $data11 != 0.000000000 then + return -1 +endi + +if $data51 != 0.000000000 then + return -1 +endi + +sql select count(*),irate(k) from t1 interval(10m) +if $rows != 4 then + return -1 +endi + +if $data00 != @15-08-18 00:00:00.000@ then + return -1 +endi + +if $data01 != 2 then + return -1 +endi + +if $data02 != 0.000144445 then + return -1 +endi + +if $data10 != @15-08-18 00:10:00.000@ then + return -1 +endi + +if $data11 != 2 then + return -1 +endi + +if $data12 != 0.000272222 then + return -1 +endi + +if $data20 != @15-08-18 00:20:00.000@ then + return -1 +endi + +if $data21 != 1 then + return -1 +endi + +if $data22 != 0.000000000 then + return -1 +endi + +if $data30 != @15-08-18 00:30:00.000@ then + return -1 +endi + +if $data31 != 1 then + return -1 +endi + +if $data32 != 0.000000000 then + return -1 +endi + +sql select count(*),irate(k) from t1 interval(10m) order by ts desc +if $rows != 4 then + return -1 +endi + +if $data30 != @15-08-18 00:00:00.000@ then + return -1 +endi + +if $data31 != 2 then + return -1 +endi + +if $data32 != 0.000144445 then + return -1 +endi + From d09d9a2e5687fdd01fdf8388592d5df85fd7f3e2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 2 Jun 2021 19:43:10 +0800 Subject: [PATCH 09/25] [td-4520]: support the tbname column alias name. --- src/client/src/tscSQLParser.c | 22 ++++++++++++++++------ src/query/src/qAggMain.c | 12 ++++++------ tests/script/general/parser/nestquery.sim | 6 ++++++ 3 files changed, 28 insertions(+), 12 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index c3a7c7c89c..f2558086ed 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -1931,8 +1931,10 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t } if (index.columnIndex == TSDB_TBNAME_COLUMN_INDEX) { - SSchema* colSchema = tGetTbnameColumnSchema(); - tscAddFuncInSelectClause(pQueryInfo, startPos, TSDB_FUNC_TAGPRJ, &index, colSchema, TSDB_COL_TAG, getNewResColId(pCmd)); + SSchema colSchema = *tGetTbnameColumnSchema(); + getColumnName(pItem, colSchema.name, colSchema.name, sizeof(colSchema.name) - 1); + + /*SExprInfo* pExpr = */tscAddFuncInSelectClause(pQueryInfo, startPos, TSDB_FUNC_TAGPRJ, &index, &colSchema, TSDB_COL_TAG, getNewResColId(pCmd)); } else { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; @@ -3887,7 +3889,8 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql return code; } - if (index.columnIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX) { // query on time range + SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, index.columnIndex); + if (pSchema->colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { // query on time range if (!validateJoinExprNode(pCmd, pQueryInfo, *pExpr, &index)) { return TSDB_CODE_TSC_INVALID_OPERATION; } @@ -6754,6 +6757,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { const char* msg5 = "sql too long"; // todo ADD support const char* msg6 = "from missing in subclause"; const char* msg7 = "time interval is required"; + const char* msg8 = "the first column should be primary timestamp column"; SSqlCmd* pCmd = &pSql->cmd; SQueryInfo* pQueryInfo = tscGetQueryInfo(pCmd); @@ -6807,13 +6811,19 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { return TSDB_CODE_TSC_INVALID_OPERATION; } - if (isTimeWindowQuery(pQueryInfo) && (validateFunctionsInIntervalOrGroupbyQuery(pCmd, pQueryInfo) != TSDB_CODE_SUCCESS)) { return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg2); } - if (!tscIsProjectionQuery(pQueryInfo) && pQueryInfo->interval.interval == 0) { - return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); + if (tscIsProjectionQuery(pQueryInfo)) { + SExprInfo* pExpr = tscExprGet(pQueryInfo, 0); + if (pExpr->base.colInfo.colId != PRIMARYKEY_TIMESTAMP_COL_INDEX) { + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg8); + } + } else { + if (pQueryInfo->interval.interval == 0) { + return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg7); + } } // set the created table[stream] name diff --git a/src/query/src/qAggMain.c b/src/query/src/qAggMain.c index 08c057ecc6..7cc5803f2e 100644 --- a/src/query/src/qAggMain.c +++ b/src/query/src/qAggMain.c @@ -4481,7 +4481,7 @@ static void ts_comp_finalize(SQLFunctionCtx *pCtx) { static double do_calc_rate(const SRateInfo* pRateInfo, int64_t tickPerSec) { if ((INT64_MIN == pRateInfo->lastKey) || (INT64_MIN == pRateInfo->firstKey) || (pRateInfo->firstKey >= pRateInfo->lastKey)) { - return 0; + return 0.0; } double diff = 0; @@ -4504,7 +4504,7 @@ static double do_calc_rate(const SRateInfo* pRateInfo, int64_t tickPerSec) { return 0; } - return (duration > 0)? ((double)diff) / (duration/((double) tickPerSec)):0; + return (duration > 0)? ((double)diff) / (duration/((double) tickPerSec)):0.0; } static bool rate_function_setup(SQLFunctionCtx *pCtx) { @@ -4545,7 +4545,7 @@ static void rate_function(SQLFunctionCtx *pCtx) { notNullElems++; int64_t v = 0; - GET_TYPED_DATA(v, int64_t, pCtx->inputType, pData); + GET_TYPED_DATA(v, double, pCtx->inputType, pData); if ((INT64_MIN == pRateInfo->firstValue) || (INT64_MIN == pRateInfo->firstKey)) { pRateInfo->firstValue = v; @@ -4592,7 +4592,7 @@ static void rate_function_f(SQLFunctionCtx *pCtx, int32_t index) { TSKEY *primaryKey = GET_TS_LIST(pCtx); int64_t v = 0; - GET_TYPED_DATA(v, int64_t, pCtx->inputType, pData); + GET_TYPED_DATA(v, double, pCtx->inputType, pData); if ((INT64_MIN == pRateInfo->firstValue) || (INT64_MIN == pRateInfo->firstKey)) { pRateInfo->firstValue = v; @@ -4637,7 +4637,7 @@ static void rate_finalizer(SQLFunctionCtx *pCtx) { return; } - *(double*)pCtx->pOutput = do_calc_rate(pRateInfo, TSDB_TICK_PER_SECOND(pCtx->param[0].i64)); + *(double*) pCtx->pOutput = do_calc_rate(pRateInfo, TSDB_TICK_PER_SECOND(pCtx->param[0].i64)); // cannot set the numOfIteratedElems again since it is set during previous iteration pResInfo->numOfRes = 1; @@ -4702,7 +4702,7 @@ static void irate_function_f(SQLFunctionCtx *pCtx, int32_t index) { TSKEY *primaryKey = GET_TS_LIST(pCtx); int64_t v = 0; - GET_TYPED_DATA(v, int64_t, pCtx->inputType, pData); + GET_TYPED_DATA(v, double, pCtx->inputType, pData); pRateInfo->firstKey = pRateInfo->lastKey; pRateInfo->firstValue = pRateInfo->lastValue; diff --git a/tests/script/general/parser/nestquery.sim b/tests/script/general/parser/nestquery.sim index 3d13ff504d..ee0f7fd0fe 100644 --- a/tests/script/general/parser/nestquery.sim +++ b/tests/script/general/parser/nestquery.sim @@ -124,6 +124,12 @@ if $rows != 2 then return -1 endi +sql select * from (select count(*) a from nest_mt0 group by tbname) t where t.a<0 +if $rows != 0 then + return -1 +endi + + print ===================> nest query interval From a69cede0ab53fd61e36e5cea999726ed5465e32c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 2 Jun 2021 19:59:54 +0800 Subject: [PATCH 10/25] [td-255] update sim test. --- src/client/src/tscSQLParser.c | 2 +- tests/script/general/parser/nestquery.sim | 17 ++++++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index f2558086ed..15e40d5918 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -3890,7 +3890,7 @@ static int32_t handleExprInQueryCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSql } SSchema* pSchema = tscGetTableColumnSchema(pTableMeta, index.columnIndex); - if (pSchema->colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { // query on time range + if (pSchema->type == TSDB_DATA_TYPE_TIMESTAMP && index.columnIndex == PRIMARYKEY_TIMESTAMP_COL_INDEX) { // query on time range if (!validateJoinExprNode(pCmd, pQueryInfo, *pExpr, &index)) { return TSDB_CODE_TSC_INVALID_OPERATION; } diff --git a/tests/script/general/parser/nestquery.sim b/tests/script/general/parser/nestquery.sim index ee0f7fd0fe..16d190d3fe 100644 --- a/tests/script/general/parser/nestquery.sim +++ b/tests/script/general/parser/nestquery.sim @@ -124,12 +124,27 @@ if $rows != 2 then return -1 endi -sql select * from (select count(*) a from nest_mt0 group by tbname) t where t.a<0 +sql select * from (select count(*) a, tbname f1 from nest_mt0 group by tbname) t where t.a<0 and f1 = 'nest_tb0'; if $rows != 0 then return -1 endi +sql select * from (select count(*) a, tbname f1 from nest_mt0 group by tbname) t where t.a>0 and f1 = 'nest_tb0'; +if $rows != 1 then + return -1 +endi +if $data00 != 10000 then + return -1 +endi + +if $data01 != @nest_tb0@ then + return -1 +endi + +if $data02 != @nest_tb0@ then + return -1 +endi print ===================> nest query interval From 35febb15191f15fc1b1bdfd5c06d0cc38b2b69b7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 2 Jun 2021 22:18:05 +0800 Subject: [PATCH 11/25] [td-255] fix compiler error. --- src/query/src/qAggMain.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/query/src/qAggMain.c b/src/query/src/qAggMain.c index 7cc5803f2e..370e9e2b79 100644 --- a/src/query/src/qAggMain.c +++ b/src/query/src/qAggMain.c @@ -4544,7 +4544,7 @@ static void rate_function(SQLFunctionCtx *pCtx) { notNullElems++; - int64_t v = 0; + double v = 0; GET_TYPED_DATA(v, double, pCtx->inputType, pData); if ((INT64_MIN == pRateInfo->firstValue) || (INT64_MIN == pRateInfo->firstKey)) { @@ -4591,7 +4591,7 @@ static void rate_function_f(SQLFunctionCtx *pCtx, int32_t index) { SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); TSKEY *primaryKey = GET_TS_LIST(pCtx); - int64_t v = 0; + double v = 0; GET_TYPED_DATA(v, double, pCtx->inputType, pData); if ((INT64_MIN == pRateInfo->firstValue) || (INT64_MIN == pRateInfo->firstKey)) { From 60f3f7ed7106205492cc328f8cb040587a492ca6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 2 Jun 2021 22:19:51 +0800 Subject: [PATCH 12/25] [td-255] fix compiler error. --- src/query/src/qAggMain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/src/qAggMain.c b/src/query/src/qAggMain.c index 370e9e2b79..c598679a91 100644 --- a/src/query/src/qAggMain.c +++ b/src/query/src/qAggMain.c @@ -4701,7 +4701,7 @@ static void irate_function_f(SQLFunctionCtx *pCtx, int32_t index) { SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); TSKEY *primaryKey = GET_TS_LIST(pCtx); - int64_t v = 0; + double v = 0; GET_TYPED_DATA(v, double, pCtx->inputType, pData); pRateInfo->firstKey = pRateInfo->lastKey; From bed28fca982311d10b22beaf2340db380bbb4972 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 3 Jun 2021 10:38:18 +0800 Subject: [PATCH 13/25] [td-255] fix compiler error on windows. --- src/query/src/qAggMain.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/query/src/qAggMain.c b/src/query/src/qAggMain.c index c598679a91..4c7c7d6970 100644 --- a/src/query/src/qAggMain.c +++ b/src/query/src/qAggMain.c @@ -152,7 +152,7 @@ typedef struct STSCompInfo { } STSCompInfo; typedef struct SRateInfo { - int64_t CorrectionValue; + double correctionValue; double firstValue; TSKEY firstKey; double lastValue; @@ -4493,7 +4493,7 @@ static double do_calc_rate(const SRateInfo* pRateInfo, int64_t tickPerSec) { diff -= pRateInfo->firstValue; } } else { - diff = pRateInfo->CorrectionValue + pRateInfo->lastValue - pRateInfo->firstValue; + diff = pRateInfo->correctionValue + pRateInfo->lastValue - pRateInfo->firstValue; if (diff <= 0) { return 0; } @@ -4515,7 +4515,7 @@ static bool rate_function_setup(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); SRateInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); - pInfo->CorrectionValue = 0; + pInfo->correctionValue = 0; pInfo->firstKey = INT64_MIN; pInfo->lastKey = INT64_MIN; pInfo->firstValue = INT64_MIN; @@ -4555,8 +4555,8 @@ static void rate_function(SQLFunctionCtx *pCtx) { if (INT64_MIN == pRateInfo->lastValue) { pRateInfo->lastValue = v; } else if (v < pRateInfo->lastValue) { - pRateInfo->CorrectionValue += pRateInfo->lastValue; - qDebug("CorrectionValue:%" PRId64, pRateInfo->CorrectionValue); + pRateInfo->correctionValue += pRateInfo->lastValue; + qDebug("correctionValue:%" PRId64, pRateInfo->correctionValue); } pRateInfo->lastValue = v; @@ -4602,7 +4602,7 @@ static void rate_function_f(SQLFunctionCtx *pCtx, int32_t index) { if (INT64_MIN == pRateInfo->lastValue) { pRateInfo->lastValue = v; } else if (v < pRateInfo->lastValue) { - pRateInfo->CorrectionValue += pRateInfo->lastValue; + pRateInfo->correctionValue += pRateInfo->lastValue; } pRateInfo->lastValue = v; @@ -4637,7 +4637,7 @@ static void rate_finalizer(SQLFunctionCtx *pCtx) { return; } - *(double*) pCtx->pOutput = do_calc_rate(pRateInfo, TSDB_TICK_PER_SECOND(pCtx->param[0].i64)); + *(double*) pCtx->pOutput = (double) do_calc_rate(pRateInfo, TSDB_TICK_PER_SECOND(pCtx->param[0].i64)); // cannot set the numOfIteratedElems again since it is set during previous iteration pResInfo->numOfRes = 1; From 9216a9f2011705c8e5d6525e768a4fe76a2a5fd6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 3 Jun 2021 14:01:15 +0800 Subject: [PATCH 14/25] [td-255] --- src/query/src/qAggMain.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/query/src/qAggMain.c b/src/query/src/qAggMain.c index 4c7c7d6970..a533cc8255 100644 --- a/src/query/src/qAggMain.c +++ b/src/query/src/qAggMain.c @@ -4556,7 +4556,6 @@ static void rate_function(SQLFunctionCtx *pCtx) { pRateInfo->lastValue = v; } else if (v < pRateInfo->lastValue) { pRateInfo->correctionValue += pRateInfo->lastValue; - qDebug("correctionValue:%" PRId64, pRateInfo->correctionValue); } pRateInfo->lastValue = v; From 91d2d3b85a37e46098cbe277d3728e0385a0ec02 Mon Sep 17 00:00:00 2001 From: bryanchang0603 Date: Thu, 3 Jun 2021 15:29:23 +0800 Subject: [PATCH 15/25] [TD-4463]rewrite the case to fit develop and community --- tests/pytest/alter/alter_keep.py | 112 +++++++++++++++++++++++-------- 1 file changed, 84 insertions(+), 28 deletions(-) diff --git a/tests/pytest/alter/alter_keep.py b/tests/pytest/alter/alter_keep.py index 37a8a8da55..49b82c7d15 100644 --- a/tests/pytest/alter/alter_keep.py +++ b/tests/pytest/alter/alter_keep.py @@ -22,34 +22,6 @@ class TDTestCase: tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) - - def run(self): - tdSql.prepare() - tdSql.execute('create table tb (ts timestamp, speed int)') - - tdSql.query('show databases') - tdSql.checkData(0,7,'3650,3650,3650') - tdSql.execute('alter database db keep 10') - tdSql.query('show databases') - tdSql.checkData(0,7,'10,10,10') - tdSql.execute('alter database db keep 50') - tdSql.query('show databases') - tdSql.checkData(0,7,'50,50,50') - tdSql.error('alter database db keep !)') - - tdSql.error('alter database db keep 1') - - ## the following sql will not raise error, but will not cause error either - # based on Li Chuang's explaination, <= 0 will not cause keep>days error - # tdSql.error('alter database db keep -10') - # tdSql.query('show databases') - # tdSql.checkData(0,7,'50,50,50') - # tdSql.error('alter database db keep 0') - # tdSql.error('alter database db keep 0.1') - tdSql.error('alter database db keep 10.1') - tdSql.query('show databases') - tdSql.checkData(0,7,'50,50,50') - ##TODO: test keep keep hot alter, cannot be tested for now as test.py's output ## is inconsistent with the actual output. # tdSql.execute('insert into tb values (now, 10)') @@ -68,6 +40,90 @@ class TDTestCase: # os.system('systemctl restart taosd') # tdSql.query('select * from tb') # tdSql.checkRows(2) + + def alterKeepCommunity(self): + ## community accepts both 1 paramater and 3 paramaters + ## comunity should not accept 2 paramaters + tdSql.query('show databases') + tdSql.checkData(0,7,'3650,3650,3650') + + tdSql.execute('alter database db keep 10') + tdSql.query('show databases') + tdSql.checkData(0,7,'10,10,10') + + tdSql.execute('alter database db keep 50') + tdSql.query('show databases') + tdSql.checkData(0,7,'50,50,50') + + tdSql.execute('alter database db keep 20') + tdSql.query('show databases') + tdSql.checkData(0,7,'20,20,20') + + ## the order for altering keep is keep(D), keep0, keep1. + ## if the order is changed, please modify the following test + ## to make sure the the test is accurate + tdSql.execute('alter database db keep 100, 98 ,99') + tdSql.query('show databases') + tdSql.checkData(0,7,'98,99,100') + + tdSql.execute('alter database db keep 200, 200 ,200') + tdSql.query('show databases') + tdSql.checkData(0,7,'200,200,200') + + tdSql.error('alter database db keep 198, 199 ,200') + tdSql.query('show databases') + tdSql.checkData(0,7,'200,200,200') + + tdSql.execute('alter database db keep 3650,3650,3650') + tdSql.error('alter database db keep 4000,4000') + tdSql.error('alter database db keep 5000,50') + tdSql.query('show databases') + tdSql.checkData(0,7,'3650,3650,3650') + + def alterKeepEnterprise(self): + ## enterprise only accept three inputs + ## does not accept 1 paramaters nor 3 paramaters + tdSql.query('show databases') + tdSql.checkData(0,7,'3650,3650,3650') + + tdSql.error('alter database db keep 10') + tdSql.query('show databases') + tdSql.checkData(0,7,'3650,3650,3650') + + ## the order for altering keep is keep(D), keep0, keep1. + ## if the order is changed, please modify the following test + ## to make sure the the test is accurate + + tdSql.execute('alter database db keep 100, 98 ,99') + tdSql.query('show databases') + tdSql.checkData(0,7,'98,99,100') + + tdSql.execute('alter database db keep 200, 200 ,200') + tdSql.query('show databases') + tdSql.checkData(0,7,'200,200,200') + + tdSql.error('alter database db keep 198, 199 ,200') + tdSql.query('show databases') + tdSql.checkData(0,7,'200,200,200') + + tdSql.execute('alter database db keep 3650,3650,3650') + tdSql.error('alter database db keep 4000,3640') + tdSql.error('alter database db keep 10,10') + tdSql.query('show databases') + tdSql.checkData(0,7,'3650,3650,3650') + + def run(self): + tdSql.prepare() + tdSql.execute('create table tb (ts timestamp, speed int)') + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + tdLog.debug('running enterprise test') + self.alterKeepEnterprise() + else: + tdLog.debug('running community test') + self.alterKeepCommunity() + def stop(self): tdSql.close() From 904bf2381e1a8cbd55652f4b14ff361c93b9b42d Mon Sep 17 00:00:00 2001 From: bryanchang0603 Date: Thu, 3 Jun 2021 16:04:48 +0800 Subject: [PATCH 16/25] [TD-4463] modifying test case for community version --- tests/pytest/alter/alter_keep.py | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/tests/pytest/alter/alter_keep.py b/tests/pytest/alter/alter_keep.py index 49b82c7d15..88f3529539 100644 --- a/tests/pytest/alter/alter_keep.py +++ b/tests/pytest/alter/alter_keep.py @@ -42,8 +42,9 @@ class TDTestCase: # tdSql.checkRows(2) def alterKeepCommunity(self): - ## community accepts both 1 paramater and 3 paramaters - ## comunity should not accept 2 paramaters + ## community accepts both 1 paramater, 2 parmaters and 3 paramaters + ## but paramaters other than paramater 1 will be ignored + ## only paramater 1 will be used tdSql.query('show databases') tdSql.checkData(0,7,'3650,3650,3650') @@ -59,26 +60,30 @@ class TDTestCase: tdSql.query('show databases') tdSql.checkData(0,7,'20,20,20') - ## the order for altering keep is keep(D), keep0, keep1. - ## if the order is changed, please modify the following test - ## to make sure the the test is accurate tdSql.execute('alter database db keep 100, 98 ,99') tdSql.query('show databases') - tdSql.checkData(0,7,'98,99,100') + tdSql.checkData(0,7,'100,100,100') - tdSql.execute('alter database db keep 200, 200 ,200') + tdSql.execute('alter database db keep 99, 100 ,101') + tdSql.query('show databases') + tdSql.checkData(0,7,'99,99,99') + + tdSql.execute('alter database db keep 200, 199 ,198') tdSql.query('show databases') tdSql.checkData(0,7,'200,200,200') - tdSql.error('alter database db keep 198, 199 ,200') + tdSql.execute('alter database db keep 4000,4001') tdSql.query('show databases') - tdSql.checkData(0,7,'200,200,200') + tdSql.checkData(0,7,'4000,4000,4000') - tdSql.execute('alter database db keep 3650,3650,3650') - tdSql.error('alter database db keep 4000,4000') - tdSql.error('alter database db keep 5000,50') + tdSql.execute('alter database db keep 5000,50') tdSql.query('show databases') - tdSql.checkData(0,7,'3650,3650,3650') + tdSql.checkData(0,7,'5000,5000,5000') + + tdSql.execute('alter database db keep 50,5000') + tdSql.query('show databases') + tdSql.checkData(0,7,'50,50,50') + def alterKeepEnterprise(self): ## enterprise only accept three inputs From f797c397e436d1d39536fc1adf5adabc50eee028 Mon Sep 17 00:00:00 2001 From: bryanchang0603 Date: Thu, 3 Jun 2021 16:54:15 +0800 Subject: [PATCH 17/25] [TD-4463] enterprise test pass --- tests/pytest/alter/alter_keep.py | 57 ++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 25 deletions(-) diff --git a/tests/pytest/alter/alter_keep.py b/tests/pytest/alter/alter_keep.py index 88f3529539..cb630963a7 100644 --- a/tests/pytest/alter/alter_keep.py +++ b/tests/pytest/alter/alter_keep.py @@ -21,25 +21,6 @@ class TDTestCase: def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) - - ##TODO: test keep keep hot alter, cannot be tested for now as test.py's output - ## is inconsistent with the actual output. - # tdSql.execute('insert into tb values (now, 10)') - # tdSql.execute('insert into tb values (now + 10m, 10)') - # tdSql.query('select * from tb') - # tdSql.checkRows(2) - # tdSql.execute('alter database db keep 40,40,40') - # os.system('systemctl restart taosd') - # tdSql.execute('insert into tb values (now-60d, 10)') - # tdSql.execute('insert into tb values (now-30d, 10)') - # tdSql.query('select * from tb') - # tdSql.showQueryResult() - # tdSql.checkRows(2) - # tdSql.execute('alter database db keep 20,20,20') - # tdSql.checkRows(3) - # os.system('systemctl restart taosd') - # tdSql.query('select * from tb') - # tdSql.checkRows(2) def alterKeepCommunity(self): ## community accepts both 1 paramater, 2 parmaters and 3 paramaters @@ -99,6 +80,10 @@ class TDTestCase: ## if the order is changed, please modify the following test ## to make sure the the test is accurate + tdSql.execute('alter database db keep 10, 10 ,10') + tdSql.query('show databases') + tdSql.checkData(0,7,'10,10,10') + tdSql.execute('alter database db keep 100, 98 ,99') tdSql.query('show databases') tdSql.checkData(0,7,'98,99,100') @@ -111,15 +96,14 @@ class TDTestCase: tdSql.query('show databases') tdSql.checkData(0,7,'200,200,200') - tdSql.execute('alter database db keep 3650,3650,3650') - tdSql.error('alter database db keep 4000,3640') - tdSql.error('alter database db keep 10,10') - tdSql.query('show databases') - tdSql.checkData(0,7,'3650,3650,3650') + # tdSql.execute('alter database db keep 3650,3650,3650') + # tdSql.error('alter database db keep 4000,3640') + # tdSql.error('alter database db keep 10,10') + # tdSql.query('show databases') + # tdSql.checkData(0,7,'3650,3650,3650') def run(self): tdSql.prepare() - tdSql.execute('create table tb (ts timestamp, speed int)') selfPath = os.path.dirname(os.path.realpath(__file__)) if ("community" in selfPath): @@ -128,6 +112,29 @@ class TDTestCase: else: tdLog.debug('running community test') self.alterKeepCommunity() + + + ##TODO: need to wait for TD-4445 to implement the following + ## tests + # tdSql.prepare() + # tdSql.execute('create table tb (ts timestamp, speed int)') + # tdSql.execute('alter database db keep 10,10,10') + # tdSql.execute('insert into tb values (now, 10)') + # tdSql.execute('insert into tb values (now + 10m, 10)') + # tdSql.query('select * from tb') + # tdSql.checkRows(2) + # tdSql.execute('alter database db keep 40,40,40') + # tdSql.query('show databases') + # tdSql.checkData(0,7,'40,40,40') + # tdSql.error('insert into tb values (now-60d, 10)') + # tdSql.execute('insert into tb values (now-30d, 10)') + # tdSql.query('select * from tb') + # tdSql.checkRows(3) + # tdSql.execute('alter database db keep 20,20,20') + # tdSql.query('show databases') + # tdSql.checkData(0,7,'20,20,20') + # tdSql.query('select * from tb') + # tdSql.checkRows(2) def stop(self): From 7082d8a5e05ce72c7dd693cd88e9c1d5ca5b42a3 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 3 Jun 2021 18:12:56 +0800 Subject: [PATCH 18/25] [td-255] fix compiler error on windows. --- src/query/src/qAggMain.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/query/src/qAggMain.c b/src/query/src/qAggMain.c index a533cc8255..ba6efcabb2 100644 --- a/src/query/src/qAggMain.c +++ b/src/query/src/qAggMain.c @@ -4478,7 +4478,7 @@ static void ts_comp_finalize(SQLFunctionCtx *pCtx) { ////////////////////////////////////////////////////////////////////////////////////////////// // rate functions -static double do_calc_rate(const SRateInfo* pRateInfo, int64_t tickPerSec) { +static double do_calc_rate(const SRateInfo* pRateInfo, double tickPerSec) { if ((INT64_MIN == pRateInfo->lastKey) || (INT64_MIN == pRateInfo->firstKey) || (pRateInfo->firstKey >= pRateInfo->lastKey)) { return 0.0; @@ -4504,7 +4504,7 @@ static double do_calc_rate(const SRateInfo* pRateInfo, int64_t tickPerSec) { return 0; } - return (duration > 0)? ((double)diff) / (duration/((double) tickPerSec)):0.0; + return (duration > 0)? ((double)diff) / (duration/tickPerSec):0.0; } static bool rate_function_setup(SQLFunctionCtx *pCtx) { @@ -4636,7 +4636,7 @@ static void rate_finalizer(SQLFunctionCtx *pCtx) { return; } - *(double*) pCtx->pOutput = (double) do_calc_rate(pRateInfo, TSDB_TICK_PER_SECOND(pCtx->param[0].i64)); + *(double*) pCtx->pOutput = do_calc_rate(pRateInfo, TSDB_TICK_PER_SECOND(pCtx->param[0].i64)); // cannot set the numOfIteratedElems again since it is set during previous iteration pResInfo->numOfRes = 1; From 61f8139178c24ca2d8888b24e85aab890155cc57 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Thu, 3 Jun 2021 18:17:22 +0800 Subject: [PATCH 19/25] update test case --- tests/pytest/query/last_row_cache.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/pytest/query/last_row_cache.py b/tests/pytest/query/last_row_cache.py index a0e8147709..f116d0254e 100644 --- a/tests/pytest/query/last_row_cache.py +++ b/tests/pytest/query/last_row_cache.py @@ -25,7 +25,7 @@ class TDTestCase: self.tables = 10 self.rows = 20 - self.columns = 50 + self.columns = 5 self.perfix = 't' self.ts = 1601481600000 @@ -34,7 +34,7 @@ class TDTestCase: sql = "create table st(ts timestamp, " for i in range(self.columns - 1): sql += "c%d int, " % (i + 1) - sql += "c50 int) tags(t1 int)" + sql += "c5 int) tags(t1 int)" tdSql.execute(sql) for i in range(self.tables): From e81c7324dbd109d19b0eff14681b8058a4cf3eaf Mon Sep 17 00:00:00 2001 From: lichuang Date: Fri, 4 Jun 2021 11:23:01 +0800 Subject: [PATCH 20/25] [TD-3963]fix cache_last_row bug --- src/tsdb/src/tsdbMain.c | 3 +-- src/tsdb/src/tsdbMemTable.c | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 3a375ec59c..5a4c785f02 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -907,13 +907,12 @@ int tsdbCacheLastData(STsdbRepo *pRepo, STsdbCfg* oldCfg) { atomic_store_8(&pRepo->hasCachedLastColumn, 0); } tsdbInfo("free cache last data since cacheLast option changed"); - for (int i = 1; i < maxTableIdx; i++) { + for (int i = 1; i <= maxTableIdx; i++) { STable *pTable = pMeta->tables[i]; if (pTable == NULL) continue; if (need_free_last_row) { taosTZfree(pTable->lastRow); pTable->lastRow = NULL; - pTable->lastKey = TSKEY_INITIAL_VAL; } if (need_free_last_col) { tsdbFreeLastColumns(pTable); diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 1f88474f8e..ec773dbb38 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -1030,7 +1030,6 @@ static int tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable, SDataRow taosTZfree(pTable->lastRow); TSDB_WLOCK_TABLE(pTable); pTable->lastRow = NULL; - pTable->lastKey = TSKEY_INITIAL_VAL; TSDB_WUNLOCK_TABLE(pTable); } From d7219d1aaaa48d4c86337671d4505fdf59726832 Mon Sep 17 00:00:00 2001 From: lichuang Date: Fri, 4 Jun 2021 14:50:47 +0800 Subject: [PATCH 21/25] [TD-3963]sync config change msg and wait for the config applied --- src/tsdb/inc/tsdbMemTable.h | 2 +- src/tsdb/src/tsdbMain.c | 4 ++-- src/tsdb/src/tsdbMemTable.c | 11 ++++++++++- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/tsdb/inc/tsdbMemTable.h b/src/tsdb/inc/tsdbMemTable.h index 868a10d361..babb7024b2 100644 --- a/src/tsdb/inc/tsdbMemTable.h +++ b/src/tsdb/inc/tsdbMemTable.h @@ -66,7 +66,7 @@ int tsdbTakeMemSnapshot(STsdbRepo* pRepo, SMemSnapshot* pSnapshot, SArray* pAT void tsdbUnTakeMemSnapShot(STsdbRepo* pRepo, SMemSnapshot* pSnapshot); void* tsdbAllocBytes(STsdbRepo* pRepo, int bytes); int tsdbAsyncCommit(STsdbRepo* pRepo); -int tsdbAsyncCommitConfig(STsdbRepo* pRepo); +int tsdbSyncCommitConfig(STsdbRepo* pRepo); int tsdbLoadDataFromCache(STable* pTable, SSkipListIterator* pIter, TSKEY maxKey, int maxRowsToRead, SDataCols* pCols, TKEY* filterKeys, int nFilterKeys, bool keepDup, SMergeInfo* pMergeInfo); void* tsdbCommitData(STsdbRepo* pRepo); diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 5a4c785f02..ce20bf1821 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -270,8 +270,8 @@ int32_t tsdbConfigRepo(STsdbRepo *repo, STsdbCfg *pCfg) { pthread_mutex_unlock(&repo->save_mutex); - // schedule a commit msg then the new config will be applied immediatly - tsdbAsyncCommitConfig(repo); + // schedule a commit msg and wait for the new config applied + tsdbSyncCommitConfig(repo); return 0; #if 0 diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index ec773dbb38..9d8b1ca7f2 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -271,7 +271,7 @@ void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) { return ptr; } -int tsdbAsyncCommitConfig(STsdbRepo* pRepo) { +int tsdbSyncCommitConfig(STsdbRepo* pRepo) { ASSERT(pRepo->config_changed == true); tsem_wait(&(pRepo->readyToCommit)); @@ -283,6 +283,15 @@ int tsdbAsyncCommitConfig(STsdbRepo* pRepo) { tsdbScheduleCommit(pRepo, COMMIT_CONFIG_REQ); if (tsdbUnlockRepo(pRepo) < 0) return -1; + tsem_wait(&(pRepo->readyToCommit)); + tsem_post(&(pRepo->readyToCommit)); + + if (pRepo->code != TSDB_CODE_SUCCESS) { + terrno = pRepo->code; + return -1; + } + + terrno = TSDB_CODE_SUCCESS; return 0; } From 15aecd488241f00822440920605d17bcd9f5fc3e Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Fri, 4 Jun 2021 16:52:18 +0800 Subject: [PATCH 22/25] [TD-4505]: add test case for last_row() --- tests/pytest/query/last_row_cache.py | 60 +++++++++++++++++++++++++--- 1 file changed, 55 insertions(+), 5 deletions(-) diff --git a/tests/pytest/query/last_row_cache.py b/tests/pytest/query/last_row_cache.py index a0e8147709..4aae4ce487 100644 --- a/tests/pytest/query/last_row_cache.py +++ b/tests/pytest/query/last_row_cache.py @@ -25,7 +25,7 @@ class TDTestCase: self.tables = 10 self.rows = 20 - self.columns = 50 + self.columns = 5 self.perfix = 't' self.ts = 1601481600000 @@ -34,7 +34,7 @@ class TDTestCase: sql = "create table st(ts timestamp, " for i in range(self.columns - 1): sql += "c%d int, " % (i + 1) - sql += "c50 int) tags(t1 int)" + sql += "c5 int) tags(t1 int)" tdSql.execute(sql) for i in range(self.tables): @@ -148,15 +148,38 @@ class TDTestCase: self.executeQueries() self.insertData2() self.executeQueries2() - + print("============== alter last cache") tdSql.execute("alter database test1 cachelast 1") self.executeQueries2() + + tdSql.execute("alter database test1 cachelast 2") + self.executeQueries2() + + tdSql.execute("alter database test1 cachelast 3") + self.executeQueries2() + + + print("============== alter last cache") + tdSql.execute("alter database test1 cachelast 0") + self.executeQueries2() tdDnodes.stop(1) tdDnodes.start(1) self.executeQueries2() - tdSql.execute("alter database test1 cachelast 0") + tdSql.execute("alter database test1 cachelast 1") + self.executeQueries2() + tdDnodes.stop(1) + tdDnodes.start(1) + self.executeQueries2() + + tdSql.execute("alter database test1 cachelast 2") + self.executeQueries2() + tdDnodes.stop(1) + tdDnodes.start(1) + self.executeQueries2() + + tdSql.execute("alter database test1 cachelast 3") self.executeQueries2() tdDnodes.stop(1) tdDnodes.start(1) @@ -174,10 +197,22 @@ class TDTestCase: self.executeQueries2() tdSql.execute("alter database test2 cachelast 0") + self.executeQueries2() + + tdSql.execute("alter database test2 cachelast 1") self.executeQueries2() + + tdSql.execute("alter database test2 cachelast 2") + self.executeQueries2() + + tdSql.execute("alter database test2 cachelast 3") + self.executeQueries2() + + tdSql.execute("alter database test2 cachelast 0") + self.executeQueries2() tdDnodes.stop(1) tdDnodes.start(1) - self.executeQueries2() + self.executeQueries2() tdSql.execute("alter database test2 cachelast 1") self.executeQueries2() @@ -185,6 +220,21 @@ class TDTestCase: tdDnodes.start(1) self.executeQueries2() + tdSql.execute("alter database test2 cachelast 2") + self.executeQueries2() + tdDnodes.stop(1) + tdDnodes.start(1) + self.executeQueries2() + + tdSql.execute("alter database test2 cachelast 3") + self.executeQueries2() + tdDnodes.stop(1) + tdDnodes.start(1) + self.executeQueries2() + + tdSql.query("select last_row(*) from st group by tbname") + tdSql.checkRows(10) + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) From 7f77f0362ff2e90a29b05df442ea931aa58cf51c Mon Sep 17 00:00:00 2001 From: Shuduo Sang Date: Fri, 4 Jun 2021 21:48:37 +0800 Subject: [PATCH 23/25] Hotfix/sangshuduo/td 4025 fix travis ci broken for develop (#6376) * [TD-4025]: travis ci broken due to valgrind dependency missed. modify smoketest.sh to python3.8 * [TD-4025]: travis ci broken due to ubuntu 20.04 repo issue. * [TD-4025]: travis ci broken due to ubuntu 20.04 repo issue. * [TD-4025]: travis ci broken due to ubuntu 20.04 repo issue. * [TD-4025]: travis ci broken due to valgrind dependency missed. change focal to bionic. * [TD-4025]: travis ci broken due to ubuntu 20.04 repo issue. cherry pick from master. * don't check maven on windows for appveyor. * fix timezone build error with clang/linux. * add .appveyor.yml back. Co-authored-by: Shuduo Sang --- src/os/src/detail/osTime.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/os/src/detail/osTime.c b/src/os/src/detail/osTime.c index 67e0c2642e..2956dd29ad 100644 --- a/src/os/src/detail/osTime.c +++ b/src/os/src/detail/osTime.c @@ -43,7 +43,7 @@ */ int64_t user_mktime64(const unsigned int year0, const unsigned int mon0, const unsigned int day, const unsigned int hour, - const unsigned int min, const unsigned int sec, int64_t timezone) + const unsigned int min, const unsigned int sec, int64_t time_zone) { unsigned int mon = mon0, year = year0; @@ -61,7 +61,7 @@ int64_t user_mktime64(const unsigned int year0, const unsigned int mon0, res = res*24; res = ((res + hour) * 60 + min) * 60 + sec; - return (res + timezone); + return (res + time_zone); } // ==== mktime() kernel code =================// From 2241f2f2cb941d8733381b0fc1f16bbb481fbf95 Mon Sep 17 00:00:00 2001 From: lichuang Date: Sat, 5 Jun 2021 10:04:07 +0800 Subject: [PATCH 24/25] [TD-3963]fix tsdbGetCachedLastRow:when lastRow == NULL return TSDB_CODE_TDB_NO_CACHE_LAST_ROW --- src/inc/taoserror.h | 1 + src/tsdb/inc/tsdbint.h | 1 - src/tsdb/src/tsdbMain.c | 11 +---------- src/tsdb/src/tsdbRead.c | 28 +++++++++++++++------------- 4 files changed, 17 insertions(+), 24 deletions(-) diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index ab15e851e7..431c9116cc 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -244,6 +244,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TDB_NO_AVAIL_DISK TAOS_DEF_ERROR_CODE(0, 0x0613) //"No available disk") #define TSDB_CODE_TDB_MESSED_MSG TAOS_DEF_ERROR_CODE(0, 0x0614) //"TSDB messed message") #define TSDB_CODE_TDB_IVLD_TAG_VAL TAOS_DEF_ERROR_CODE(0, 0x0615) //"TSDB invalid tag value") +#define TSDB_CODE_TDB_NO_CACHE_LAST_ROW TAOS_DEF_ERROR_CODE(0, 0x0616) //"TSDB no cache last row data") // query #define TSDB_CODE_QRY_INVALID_QHANDLE TAOS_DEF_ERROR_CODE(0, 0x0700) //"Invalid handle") diff --git a/src/tsdb/inc/tsdbint.h b/src/tsdb/inc/tsdbint.h index 049c1bdb6e..7cf8882631 100644 --- a/src/tsdb/inc/tsdbint.h +++ b/src/tsdb/inc/tsdbint.h @@ -78,7 +78,6 @@ struct STsdbRepo { bool config_changed; // config changed flag pthread_mutex_t save_mutex; // protect save config - uint8_t hasCachedLastRow; uint8_t hasCachedLastColumn; STsdbAppH appH; diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index ce20bf1821..bb02e01283 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -553,7 +553,6 @@ static STsdbRepo *tsdbNewRepo(STsdbCfg *pCfg, STsdbAppH *pAppH) { return NULL; } pRepo->config_changed = false; - atomic_store_8(&pRepo->hasCachedLastRow, 0); atomic_store_8(&pRepo->hasCachedLastColumn, 0); code = tsem_init(&(pRepo->readyToCommit), 0, 1); @@ -857,9 +856,7 @@ int tsdbRestoreInfo(STsdbRepo *pRepo) { } tsdbDestroyReadH(&readh); - if (CACHE_LAST_ROW(pCfg)) { - atomic_store_8(&pRepo->hasCachedLastRow, 1); - } + if (CACHE_LAST_NULL_COLUMN(pCfg)) { atomic_store_8(&pRepo->hasCachedLastColumn, 1); } @@ -900,9 +897,6 @@ int tsdbCacheLastData(STsdbRepo *pRepo, STsdbCfg* oldCfg) { // if close last option,need to free data if (need_free_last_row || need_free_last_col) { - if (need_free_last_row) { - atomic_store_8(&pRepo->hasCachedLastRow, 0); - } if (need_free_last_col) { atomic_store_8(&pRepo->hasCachedLastColumn, 0); } @@ -982,9 +976,6 @@ int tsdbCacheLastData(STsdbRepo *pRepo, STsdbCfg* oldCfg) { tsdbDestroyReadH(&readh); - if (cacheLastRow) { - atomic_store_8(&pRepo->hasCachedLastRow, 1); - } if (cacheLastCol) { atomic_store_8(&pRepo->hasCachedLastColumn, 1); } diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 1545d44395..bbfa723cb5 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -2469,7 +2469,6 @@ static bool loadCachedLastRow(STsdbQueryHandle* pQueryHandle) { if (ret != TSDB_CODE_SUCCESS) { return false; } - copyOneRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, 0, pRow, numOfCols, pCheckInfo->pTableObj, NULL); tfree(pRow); @@ -2860,24 +2859,27 @@ bool tsdbGetExternalRow(TsdbQueryHandleT pHandle) { } /* - * 1. no data at all (pTable->lastKey = TSKEY_INITIAL_VAL), just return TSKEY_INITIAL_VAL - * 2. has data but not loaded, just return lastKey but not set pRes - * 3. has data and loaded, return lastKey and set pRes + * if lastRow == NULL, return TSDB_CODE_TDB_NO_CACHE_LAST_ROW + * else set pRes and return TSDB_CODE_SUCCESS */ int32_t tsdbGetCachedLastRow(STable* pTable, SDataRow* pRes, TSKEY* lastKey) { - TSDB_RLOCK_TABLE(pTable); - *lastKey = pTable->lastKey; + int32_t code = TSDB_CODE_SUCCESS; - if ((*lastKey) != TSKEY_INITIAL_VAL && pTable->lastRow) { - *pRes = tdDataRowDup(pTable->lastRow); - if (*pRes == NULL) { - TSDB_RUNLOCK_TABLE(pTable); - return TSDB_CODE_TDB_OUT_OF_MEMORY; - } + TSDB_RLOCK_TABLE(pTable); + + if (!pTable->lastRow) { + code = TSDB_CODE_TDB_NO_CACHE_LAST_ROW; + goto out; } + *pRes = tdDataRowDup(pTable->lastRow); + if (*pRes == NULL) { + code = TSDB_CODE_TDB_OUT_OF_MEMORY; + } + +out: TSDB_RUNLOCK_TABLE(pTable); - return TSDB_CODE_SUCCESS; + return code; } bool isTsdbCacheLastRow(TsdbQueryHandleT* pQueryHandle) { From 9d353971b7816488eb78dad2cf8dab7d10cf5510 Mon Sep 17 00:00:00 2001 From: lichuang Date: Sat, 5 Jun 2021 12:14:57 +0800 Subject: [PATCH 25/25] [TD-3963]no need to save last row in checkForCachedLastRow --- src/tsdb/src/tsdbRead.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index bbfa723cb5..c333294179 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -2860,7 +2860,7 @@ bool tsdbGetExternalRow(TsdbQueryHandleT pHandle) { /* * if lastRow == NULL, return TSDB_CODE_TDB_NO_CACHE_LAST_ROW - * else set pRes and return TSDB_CODE_SUCCESS + * else set pRes and return TSDB_CODE_SUCCESS and save lastKey */ int32_t tsdbGetCachedLastRow(STable* pTable, SDataRow* pRes, TSKEY* lastKey) { int32_t code = TSDB_CODE_SUCCESS; @@ -2872,9 +2872,11 @@ int32_t tsdbGetCachedLastRow(STable* pTable, SDataRow* pRes, TSKEY* lastKey) { goto out; } - *pRes = tdDataRowDup(pTable->lastRow); - if (*pRes == NULL) { - code = TSDB_CODE_TDB_OUT_OF_MEMORY; + if (pRes) { + *pRes = tdDataRowDup(pTable->lastRow); + if (*pRes == NULL) { + code = TSDB_CODE_TDB_OUT_OF_MEMORY; + } } out: @@ -2889,7 +2891,6 @@ bool isTsdbCacheLastRow(TsdbQueryHandleT* pQueryHandle) { int32_t checkForCachedLastRow(STsdbQueryHandle* pQueryHandle, STableGroupInfo *groupList) { assert(pQueryHandle != NULL && groupList != NULL); - SDataRow pRow = NULL; TSKEY key = TSKEY_INITIAL_VAL; SArray* group = taosArrayGetP(groupList->pGroupList, 0); @@ -2900,7 +2901,7 @@ int32_t checkForCachedLastRow(STsdbQueryHandle* pQueryHandle, STableGroupInfo *g int32_t code = 0; if (((STable*)pInfo->pTable)->lastRow) { - code = tsdbGetCachedLastRow(pInfo->pTable, &pRow, &key); + code = tsdbGetCachedLastRow(pInfo->pTable, NULL, &key); if (code != TSDB_CODE_SUCCESS) { pQueryHandle->cachelastrow = 0; } else { @@ -2915,7 +2916,6 @@ int32_t checkForCachedLastRow(STsdbQueryHandle* pQueryHandle, STableGroupInfo *g pQueryHandle->activeIndex = -1; // start from -1 } - tfree(pRow); return code; }