From 56d0d69a487bd91bfe5ce293bcbc8187e2aba048 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 13 Aug 2021 17:05:30 +0800 Subject: [PATCH 1/7] [TD-5538]: add --force-keep-file option --- src/common/src/tglobal.c | 3 +++ src/dnode/src/dnodeSystem.c | 2 ++ src/tsdb/inc/tsdbFS.h | 3 +++ src/tsdb/src/tsdbFS.c | 38 +++++++++++++++++++++++++++++++++++++ src/util/inc/tconfig.h | 1 + 5 files changed, 47 insertions(+) diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index ec7249cef5..198d57b6b9 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -26,6 +26,9 @@ #include "tlocale.h" #include "ttimezone.h" +// TSDB +bool tsdbForceKeepFile = false; + // cluster char tsFirst[TSDB_EP_LEN] = {0}; char tsSecond[TSDB_EP_LEN] = {0}; diff --git a/src/dnode/src/dnodeSystem.c b/src/dnode/src/dnodeSystem.c index e49b3eba99..c3e4dae206 100644 --- a/src/dnode/src/dnodeSystem.c +++ b/src/dnode/src/dnodeSystem.c @@ -42,6 +42,8 @@ int32_t main(int32_t argc, char *argv[]) { } } else if (strcmp(argv[i], "-C") == 0) { dump_config = 1; + } else if (strcmp(argv[i], "--force-keep-file") == 0) { + tsdbForceKeepFile = true; } else if (strcmp(argv[i], "-V") == 0) { #ifdef _ACCT char *versionStr = "enterprise"; diff --git a/src/tsdb/inc/tsdbFS.h b/src/tsdb/inc/tsdbFS.h index d63aeb14ac..367d905522 100644 --- a/src/tsdb/inc/tsdbFS.h +++ b/src/tsdb/inc/tsdbFS.h @@ -18,6 +18,9 @@ #define TSDB_FS_VERSION 0 +// ================== TSDB global config +extern bool tsdbForceKeepFile; + // ================== CURRENT file header info typedef struct { uint32_t version; // Current file system version (relating to code) diff --git a/src/tsdb/src/tsdbFS.c b/src/tsdb/src/tsdbFS.c index c9f087a5cf..37f9f0af98 100644 --- a/src/tsdb/src/tsdbFS.c +++ b/src/tsdb/src/tsdbFS.c @@ -982,6 +982,26 @@ static int tsdbRestoreMeta(STsdbRepo *pRepo) { return -1; } + if (tsdbForceKeepFile) { + struct stat tfstat; + + // Get real file size + if (fstat(pfs->cstatus->pmf->fd, &tfstat) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + tsdbCloseMFile(pfs->cstatus->pmf); + tfsClosedir(tdir); + regfree(®ex); + return -1; + } + + if (pfs->cstatus->pmf->info.size != tfstat.st_size) { + int64_t tfsize = pfs->cstatus->pmf->info.size; + pfs->cstatus->pmf->info.size = tfstat.st_size; + tsdbInfo("vgId:%d file %s header size is changed from %" PRId64 " to %" PRId64, REPO_ID(pRepo), + TSDB_FILE_FULL_NAME(pfs->cstatus->pmf), tfsize, pfs->cstatus->pmf->info.size); + } + } + tsdbCloseMFile(pfs->cstatus->pmf); } } else if (code == REG_NOMATCH) { @@ -1141,6 +1161,24 @@ static int tsdbRestoreDFileSet(STsdbRepo *pRepo) { return -1; } + if (tsdbForceKeepFile) { + struct stat tfstat; + + // Get real file size + if (fstat(pDFile->fd, &tfstat) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + taosArrayDestroy(fArray); + return -1; + } + + if (pDFile->info.size != tfstat.st_size) { + int64_t tfsize = pDFile->info.size; + pDFile->info.size = tfstat.st_size; + tsdbInfo("vgId:%d file %s header size is changed from %" PRId64 " to %" PRId64, REPO_ID(pRepo), + TSDB_FILE_FULL_NAME(pDFile), tfsize, pDFile->info.size); + } + } + tsdbCloseDFile(pDFile); index++; } diff --git a/src/util/inc/tconfig.h b/src/util/inc/tconfig.h index fdb2595fd8..d6e0f4ca46 100644 --- a/src/util/inc/tconfig.h +++ b/src/util/inc/tconfig.h @@ -77,6 +77,7 @@ typedef struct { extern SGlobalCfg tsGlobalConfig[]; extern int32_t tsGlobalConfigNum; extern char * tsCfgStatusStr[]; +extern bool tsdbForceKeepFile; void taosReadGlobalLogCfg(); bool taosReadGlobalCfg(); From 7d0e9c515e1902df66fb4a97f8234a100fae7883 Mon Sep 17 00:00:00 2001 From: liuyq-617 Date: Fri, 13 Aug 2021 17:21:36 +0800 Subject: [PATCH 2/7] speed up drone in 2.0 --- .drone.yml | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/.drone.yml b/.drone.yml index b520f308ba..085a07acf9 100644 --- a/.drone.yml +++ b/.drone.yml @@ -15,7 +15,7 @@ steps: - mkdir debug - cd debug - cmake .. - - make + - make -j4 trigger: event: - pull_request @@ -23,6 +23,7 @@ steps: branch: - develop - master + - 2.0 --- kind: pipeline name: test_arm64_bionic @@ -39,7 +40,7 @@ steps: - mkdir debug - cd debug - cmake .. -DCPUTYPE=aarch64 > /dev/null - - make + - make -j4 trigger: event: - pull_request @@ -66,7 +67,7 @@ steps: - mkdir debug - cd debug - cmake .. -DCPUTYPE=aarch64 > /dev/null - - make + - make -j4 trigger: event: - pull_request @@ -91,7 +92,7 @@ steps: - mkdir debug - cd debug - cmake .. -DCPUTYPE=aarch64 > /dev/null - - make + - make -j4 trigger: event: - pull_request @@ -116,7 +117,7 @@ steps: - mkdir debug - cd debug - cmake .. -DCPUTYPE=aarch64 > /dev/null - - make + - make -j4 trigger: event: - pull_request @@ -142,7 +143,7 @@ steps: - mkdir debug - cd debug - cmake .. -DCPUTYPE=aarch32 > /dev/null - - make + - make -j4 trigger: event: - pull_request @@ -150,6 +151,7 @@ steps: branch: - develop - master + - 2.0 --- kind: pipeline name: build_trusty @@ -168,7 +170,7 @@ steps: - mkdir debug - cd debug - cmake .. - - make + - make -j4 trigger: event: - pull_request @@ -176,6 +178,7 @@ steps: branch: - develop - master + - 2.0 --- kind: pipeline name: build_xenial @@ -193,7 +196,7 @@ steps: - mkdir debug - cd debug - cmake .. - - make + - make -j4 trigger: event: - pull_request @@ -201,7 +204,7 @@ steps: branch: - develop - master - + - 2.0 --- kind: pipeline name: build_bionic @@ -218,7 +221,7 @@ steps: - mkdir debug - cd debug - cmake .. - - make + - make -j4 trigger: event: - pull_request @@ -226,6 +229,7 @@ steps: branch: - develop - master + - 2.0 --- kind: pipeline name: build_centos7 @@ -241,7 +245,7 @@ steps: - mkdir debug - cd debug - cmake .. - - make + - make -j4 trigger: event: - pull_request @@ -249,4 +253,4 @@ steps: branch: - develop - master - + - 2.0 \ No newline at end of file From 28a50c47b96dcd3dcc684099050553349b443791 Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Mon, 16 Aug 2021 15:29:38 +0800 Subject: [PATCH 3/7] [TD-6054]: Filtered by tag with nchar value not as expected --- src/util/src/tcompare.c | 22 ++++++++- tests/pytest/query/filterWithinMultiNchar.py | 51 ++++++++++++++++++++ 2 files changed, 71 insertions(+), 2 deletions(-) create mode 100644 tests/pytest/query/filterWithinMultiNchar.py diff --git a/src/util/src/tcompare.c b/src/util/src/tcompare.c index 354e7899c2..3619dad83b 100644 --- a/src/util/src/tcompare.c +++ b/src/util/src/tcompare.c @@ -129,7 +129,16 @@ int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight) { if (len1 != len2) { return len1 > len2? 1:-1; } else { - int32_t ret = wcsncmp(varDataVal(pLeft), varDataVal(pRight), len1/TSDB_NCHAR_SIZE); + char *pLeftTerm = (char *)tcalloc(len1 + 1, sizeof(char)); + char *pRightTerm = (char *)tcalloc(len1 + 1, sizeof(char)); + memcpy(pLeftTerm, varDataVal(pLeft), len1); + memcpy(pRightTerm, varDataVal(pRight), len2); + + int32_t ret = wcsncmp((wchar_t*) pLeftTerm, (wchar_t*) pRightTerm, len1/TSDB_NCHAR_SIZE); + + tfree(pLeftTerm); + tfree(pRightTerm); + if (ret == 0) { return 0; } else { @@ -410,7 +419,16 @@ int32_t doCompare(const char* f1, const char* f2, int32_t type, size_t size) { return t1->len > t2->len? 1:-1; } - int32_t ret = wcsncmp((wchar_t*) t1->data, (wchar_t*) t2->data, t2->len/TSDB_NCHAR_SIZE); + char *t1_term = (char *)tcalloc(t1->len + 1, sizeof(char)); + char *t2_term = (char *)tcalloc(t2->len + 1, sizeof(char)); + memcpy(t1_term, t1->data, t1->len); + memcpy(t2_term, t2->data, t2->len); + + int32_t ret = wcsncmp((wchar_t*) t1_term, (wchar_t*) t2_term, t2->len/TSDB_NCHAR_SIZE); + + tfree(t1_term); + tfree(t2_term); + if (ret == 0) { return ret; } diff --git a/tests/pytest/query/filterWithinMultiNchar.py b/tests/pytest/query/filterWithinMultiNchar.py new file mode 100644 index 0000000000..15fcf4e24b --- /dev/null +++ b/tests/pytest/query/filterWithinMultiNchar.py @@ -0,0 +1,51 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import sys +import taos +from util.log import tdLog +from util.cases import tdCases +from util.sql import tdSql + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def run(self): + tdSql.prepare() + + print("==============step1") + tdSql.execute( + "create stable t6 (ts timestamp,val int,flow nchar(36)) tags(dev nchar(36),dev1 nchar(36),dev2 nchar(36))") + tdSql.execute("insert into t6004 using t6 (dev,dev1,dev2) tags ('b50c79bc-b102-48e6-bda1-4212263e46d0','b50c79bc-b102-48e6-bda1-4212263e46d0', 'b50c79bc-b102-48e6-bda1-4212263e46d0') values(now,1,'b50c79bc-b102-48e6-bda1-4212263e46d0')") + + + print("==============step2") + tdSql.query("select * from t6 where dev='b50c79bc-b102-48e6-bda1-4212263e46d0'") + tdSql.checkRows(1) + + tdSql.query("select * from t6 where dev1='b50c79bc-b102-48e6-bda1-4212263e46d0'") + tdSql.checkRows(1) + + tdSql.query("select * from t6 where dev2='b50c79bc-b102-48e6-bda1-4212263e46d0'") + tdSql.checkRows(1) + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From a6011d55a20fe9542480ccb66635a01d4d8f214c Mon Sep 17 00:00:00 2001 From: tomchon Date: Mon, 16 Aug 2021 18:03:17 +0800 Subject: [PATCH 4/7] change version number --- cmake/version.inc | 2 +- snap/snapcraft.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/version.inc b/cmake/version.inc index d34080aa43..84ef060196 100755 --- a/cmake/version.inc +++ b/cmake/version.inc @@ -4,7 +4,7 @@ PROJECT(TDengine) IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "2.0.20.12") + SET(TD_VER_NUMBER "2.0.20.13") ENDIF () IF (DEFINED VERCOMPATIBLE) diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 7538765d2c..b08ae53ecd 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,6 +1,6 @@ name: tdengine base: core18 -version: '2.0.20.12' +version: '2.0.20.13' icon: snap/gui/t-dengine.svg summary: an open-source big data platform designed and optimized for IoT. description: | @@ -72,7 +72,7 @@ parts: - usr/bin/taosd - usr/bin/taos - usr/bin/taosdemo - - usr/lib/libtaos.so.2.0.20.12 + - usr/lib/libtaos.so.2.0.20.13 - usr/lib/libtaos.so.1 - usr/lib/libtaos.so From 296376df1fb92f5bbcd137d759c00689d56215e4 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Mon, 16 Aug 2021 18:36:09 +0800 Subject: [PATCH 5/7] [TD-6086]:num of tags taken from output cols instead of groupby expr --- src/client/src/tscLocalMerge.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c index 97d52cc684..5fe020bb33 100644 --- a/src/client/src/tscLocalMerge.c +++ b/src/client/src/tscLocalMerge.c @@ -396,7 +396,16 @@ void tscCreateLocalMerger(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrde if (pQueryInfo->fillType != TSDB_FILL_NONE) { SFillColInfo* pFillCol = createFillColInfo(pQueryInfo); - pReducer->pFillInfo = taosCreateFillInfo(pQueryInfo->order.order, revisedSTime, pQueryInfo->groupbyExpr.numOfGroupCols, + // support sql like: select selective_function, tag1... where ... group by tag3... fill(not fill none) + // the group by expr columns and select tags are different + int32_t numOfCols = tscNumOfFields(pQueryInfo); + int32_t numOfTags = 0; + for (int32_t i = 0; i < numOfCols; ++i) { + if (TSDB_COL_IS_TAG(pFillCol[i].flag)) { + numOfTags++; + } + } + pReducer->pFillInfo = taosCreateFillInfo(pQueryInfo->order.order, revisedSTime, numOfTags, 4096, (int32_t)pQueryInfo->fieldsInfo.numOfOutput, pQueryInfo->interval.sliding, pQueryInfo->interval.slidingUnit, tinfo.precision, pQueryInfo->fillType, pFillCol, pSql); } From 2bd11ebfd66bf9525124b4c316674b365b2a1c11 Mon Sep 17 00:00:00 2001 From: shenglian zhou Date: Tue, 17 Aug 2021 09:12:16 +0800 Subject: [PATCH 6/7] [TD-6086]:add test case --- tests/script/general/parser/function.sim | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/script/general/parser/function.sim b/tests/script/general/parser/function.sim index 65058333fb..c00f6fc898 100644 --- a/tests/script/general/parser/function.sim +++ b/tests/script/general/parser/function.sim @@ -310,6 +310,12 @@ if $rows != 6 then return -1 endi +print =============================> TD-6086 +sql create stable td6086st(ts timestamp, d double) tags(t nchar(50)); +sql create table td6086ct1 using td6086st tags("ct1"); +sql create table td6086ct2 using td6086st tags("ct2"); +sql SELECT LAST(d),t FROM td6086st WHERE tbname in ('td6086ct1', 'td6086ct2') and ts>="2019-07-30 00:00:00" and ts<="2021-08-31 00:00:00" interval(1800s) fill(prev) GROUP BY tbname; + print ==================> td-2624 sql create table tm2(ts timestamp, k int, b binary(12)); sql insert into tm2 values('2011-01-02 18:42:45.326', -1,'abc'); From 7e7b67146a4b91b33f3590efe39d20dc4c21801b Mon Sep 17 00:00:00 2001 From: Ganlin Zhao Date: Tue, 17 Aug 2021 17:04:10 +0800 Subject: [PATCH 7/7] [TD-6165]: use memcpy to replace wcsncmp for nchar type comparision --- src/util/src/tcompare.c | 23 ++--------------------- 1 file changed, 2 insertions(+), 21 deletions(-) diff --git a/src/util/src/tcompare.c b/src/util/src/tcompare.c index 3619dad83b..f8deb65d48 100644 --- a/src/util/src/tcompare.c +++ b/src/util/src/tcompare.c @@ -129,16 +129,7 @@ int32_t compareLenPrefixedWStr(const void *pLeft, const void *pRight) { if (len1 != len2) { return len1 > len2? 1:-1; } else { - char *pLeftTerm = (char *)tcalloc(len1 + 1, sizeof(char)); - char *pRightTerm = (char *)tcalloc(len1 + 1, sizeof(char)); - memcpy(pLeftTerm, varDataVal(pLeft), len1); - memcpy(pRightTerm, varDataVal(pRight), len2); - - int32_t ret = wcsncmp((wchar_t*) pLeftTerm, (wchar_t*) pRightTerm, len1/TSDB_NCHAR_SIZE); - - tfree(pLeftTerm); - tfree(pRightTerm); - + int32_t ret = memcmp((wchar_t*) pLeft, (wchar_t*) pRight, len1); if (ret == 0) { return 0; } else { @@ -418,17 +409,7 @@ int32_t doCompare(const char* f1, const char* f2, int32_t type, size_t size) { if (t1->len != t2->len) { return t1->len > t2->len? 1:-1; } - - char *t1_term = (char *)tcalloc(t1->len + 1, sizeof(char)); - char *t2_term = (char *)tcalloc(t2->len + 1, sizeof(char)); - memcpy(t1_term, t1->data, t1->len); - memcpy(t2_term, t2->data, t2->len); - - int32_t ret = wcsncmp((wchar_t*) t1_term, (wchar_t*) t2_term, t2->len/TSDB_NCHAR_SIZE); - - tfree(t1_term); - tfree(t2_term); - + int32_t ret = memcmp((wchar_t*) t1, (wchar_t*) t2, t2->len); if (ret == 0) { return ret; }