From 2d7c1a206f6f44913b8deebda1f6e509a839130b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 27 Dec 2024 15:10:21 +0800 Subject: [PATCH 01/14] enh: add more cases --- source/libs/executor/src/sysscanoperator.c | 2 + tests/parallel_test/cases.task | 1 + tests/system-test/0-others/ins_filesets.py | 54 ++++++++++++++++++++++ 3 files changed, 57 insertions(+) create mode 100644 tests/system-test/0-others/ins_filesets.py diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 7862b36f27..56060b0061 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -2269,6 +2269,8 @@ static SSDataBlock* sysTableBuildUserFileSets(SOperatorInfo* pOperator) { if (ret) { if (ret == TSDB_CODE_NOT_FOUND) { // no more scan entry + setOperatorCompleted(pOperator); + pAPI->tsdReader.fileSetReaderClose(&pInfo->pFileSetReader); break; } else { code = ret; diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 38dd080ef5..8903ef27a4 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -484,6 +484,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/show.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/show_tag_index.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/information_schema.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/ins_filesets.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/grant.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py -R diff --git a/tests/system-test/0-others/ins_filesets.py b/tests/system-test/0-others/ins_filesets.py new file mode 100644 index 0000000000..e5bc4b61b7 --- /dev/null +++ b/tests/system-test/0-others/ins_filesets.py @@ -0,0 +1,54 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import re +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.execute('create database db vgroups 1') + tdSql.execute('use db') + tdSql.execute('create table t1 (ts timestamp, a int, b int)') + tdSql.execute('insert into t1 values(\'2024-12-27 14:00:00\', 1, 2)') + tdSql.execute('flush database db') + + tdLog.sleep(5) + + rows = tdSql.query('select * from information_schema.ins_filesets') + tdSql.checkRows(1) + tdSql.checkEqual(tdSql.getData(0, 0), 'db') + tdSql.checkEqual(tdSql.getData(0, 1), 2) + tdSql.checkEqual(tdSql.getData(0, 2), 2008) + # tdSql.CheckEqual(str(tdSql.getData(0, 3)), '2024-12-23 08:00:00.000') + # tdSql.CheckEqual(str(tdSql.getData(0, 4)), '2025-01-02 07:59:59.999') + # tdSql.CheckEqual(tdSql.getData(0, 6), '1970-01-01 08:00:00.000') + # tdSql.CheckEqual(tdSql.getData(0, 7), False) + + tdDnodes.stopAll() + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From a72c46a7fc28cb38d078c715956b742d68e3a255 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 27 Dec 2024 17:44:05 +0800 Subject: [PATCH 02/14] more cases --- source/dnode/vnode/src/tsdb/tsdbFS2.c | 25 +++++++++++++++---- source/dnode/vnode/src/tsdb/tsdbFSet2.c | 32 +++++++++++++++++++++++++ source/dnode/vnode/src/tsdb/tsdbFSet2.h | 2 ++ 3 files changed, 55 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index 82dd49b0e2..aa68a8af5c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -680,7 +680,7 @@ static int32_t fset_cmpr_fn(const struct STFileSet *pSet1, const struct STFileSe return 0; } -static int32_t edit_fs(STFileSystem *fs, const TFileOpArray *opArray) { +static int32_t edit_fs(STFileSystem *fs, const TFileOpArray *opArray, EFEditT etype) { int32_t code = 0; int32_t lino = 0; @@ -690,6 +690,8 @@ static int32_t edit_fs(STFileSystem *fs, const TFileOpArray *opArray) { TFileSetArray *fsetArray = fs->fSetArrTmp; STFileSet *fset = NULL; const STFileOp *op; + int32_t fid = INT32_MIN; + TSKEY now = taosGetTimestampMs(); TARRAY2_FOREACH_PTR(opArray, op) { if (!fset || fset->fid != op->fid) { STFileSet tfset = {.fid = op->fid}; @@ -708,6 +710,15 @@ static int32_t edit_fs(STFileSystem *fs, const TFileOpArray *opArray) { code = tsdbTFileSetEdit(fs->tsdb, fset, op); TSDB_CHECK_CODE(code, lino, _exit); + + if (fid != op->fid) { + fid = op->fid; + if (etype == TSDB_FEDIT_COMMIT) { + fset->lastCommit = now; + } else if (etype == TSDB_FEDIT_COMPACT) { + fset->lastCompact = now; + } + } } // remove empty empty stt level and empty file set @@ -864,7 +875,7 @@ int32_t tsdbFSEditBegin(STFileSystem *fs, const TFileOpArray *opArray, EFEditT e fs->etype = etype; // edit - code = edit_fs(fs, opArray); + code = edit_fs(fs, opArray, etype); TSDB_CHECK_CODE(code, lino, _exit); // save fs @@ -1288,6 +1299,12 @@ int32_t tsdbFileSetReaderOpen(void *pVnode, struct SFileSetReader **ppReader) { return TSDB_CODE_SUCCESS; } +extern bool tsdbShouldCompact(const STFileSet *pFileSet); + +#ifndef TD_ENTERPRISE +bool tsdbShouldCompact(const STFileSet *pFileSet) { return false; } +#endif + static int32_t tsdbFileSetReaderNextNoLock(struct SFileSetReader *pReader) { STsdb *pTsdb = pReader->pTsdb; int32_t code = TSDB_CODE_SUCCESS; @@ -1311,7 +1328,7 @@ static int32_t tsdbFileSetReaderNextNoLock(struct SFileSetReader *pReader) { // get file set details pReader->fid = pReader->pFileSet->fid; tsdbFidKeyRange(pReader->fid, pTsdb->keepCfg.days, pTsdb->keepCfg.precision, &pReader->startTime, &pReader->endTime); - pReader->lastCompactTime = 0; // TODO + pReader->lastCompactTime = pReader->pFileSet->lastCompact; pReader->totalSize = 0; for (int32_t i = 0; i < TSDB_FTYPE_MAX; i++) { STFileObj *fobj = pReader->pFileSet->farr[i]; @@ -1375,7 +1392,7 @@ int32_t tsdbFileSetGetEntryField(struct SFileSetReader *pReader, const char *fie fieldName = "should_compact"; if (strncmp(field, fieldName, strlen(fieldName) + 1) == 0) { - *(char *)value = 0; // TODO + *(char *)value = tsdbShouldCompact(pReader->pFileSet); return TSDB_CODE_SUCCESS; } diff --git a/source/dnode/vnode/src/tsdb/tsdbFSet2.c b/source/dnode/vnode/src/tsdb/tsdbFSet2.c index 68914300e4..6530ee2269 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSet2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.c @@ -273,6 +273,15 @@ int32_t tsdbTFileSetToJson(const STFileSet *fset, cJSON *json) { if (code) return code; } + // about compact and commit + if (cJSON_AddNumberToObject(json, "last compact", fset->lastCompact) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + if (cJSON_AddNumberToObject(json, "last commit", fset->lastCommit) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + return 0; } @@ -324,6 +333,20 @@ int32_t tsdbJsonToTFileSet(STsdb *pTsdb, const cJSON *json, STFileSet **fset) { } else { return TSDB_CODE_FILE_CORRUPTED; } + // about compact and commit + item1 = cJSON_GetObjectItem(json, "last compact"); + if (cJSON_IsNumber(item1)) { + (*fset)->lastCompact = item1->valuedouble; + } else { + (*fset)->lastCompact = 0; + } + + item1 = cJSON_GetObjectItem(json, "last commit"); + if (cJSON_IsNumber(item1)) { + (*fset)->lastCommit = item1->valuedouble; + } else { + (*fset)->lastCommit = 0; + } return 0; } @@ -467,6 +490,9 @@ int32_t tsdbTFileSetApplyEdit(STsdb *pTsdb, const STFileSet *fset1, STFileSet *f } } + fset2->lastCompact = fset1->lastCompact; + fset2->lastCommit = fset1->lastCommit; + return 0; } @@ -522,6 +548,9 @@ int32_t tsdbTFileSetInitCopy(STsdb *pTsdb, const STFileSet *fset1, STFileSet **f if (code) return code; } + (*fset)->lastCompact = fset1->lastCompact; + (*fset)->lastCommit = fset1->lastCommit; + return 0; } @@ -617,6 +646,9 @@ int32_t tsdbTFileSetInitRef(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fs } } + (*fset)->lastCompact = fset1->lastCompact; + (*fset)->lastCommit = fset1->lastCommit; + return 0; } diff --git a/source/dnode/vnode/src/tsdb/tsdbFSet2.h b/source/dnode/vnode/src/tsdb/tsdbFSet2.h index 83ef32e5e5..ca9c133e9c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSet2.h +++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.h @@ -92,6 +92,8 @@ struct STFileSet { int64_t maxVerValid; STFileObj *farr[TSDB_FTYPE_MAX]; // file array TSttLvlArray lvlArr[1]; // level array + TSKEY lastCompact; + TSKEY lastCommit; bool mergeScheduled; SVATaskID mergeTask; From dc1ab696d4a320f1c26017b87c224fe8005db599 Mon Sep 17 00:00:00 2001 From: haoranchen Date: Fri, 27 Dec 2024 19:40:34 +0800 Subject: [PATCH 03/14] Create taosd-ci-build.yml --- .github/workflows/taosd-ci-build.yml | 65 ++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 .github/workflows/taosd-ci-build.yml diff --git a/.github/workflows/taosd-ci-build.yml b/.github/workflows/taosd-ci-build.yml new file mode 100644 index 0000000000..3ac3caaa7f --- /dev/null +++ b/.github/workflows/taosd-ci-build.yml @@ -0,0 +1,65 @@ +name: TDengine CI + +on: + pull_request: + branches: + - 'main' + - '3.0' + - '3.1' + push: + branches: + - 'enh/main/TD-33380' + - '3.0' + - '3.1' + - 'main' + +jobs: + build: + runs-on: ubuntu-latest + name: Run unit tests + + steps: + - name: Checkout the repository + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: 1.18 + + - name: Install system dependencies + run: | + sudo apt update -y + sudo apt install -y build-essential cmake libgeos-dev + + - name: Install TDengine + run: | + mkdir debug + cd debug + cmake .. -DBUILD_HTTP=false -DBUILD_JDBC=false -DBUILD_TOOLS=false -DBUILD_TEST=off -DBUILD_KEEPER=true -DBUILD_DEPENDENCY_TESTS=false + make -j 4 + sudo make install + which taosd + which taosadapter + which taoskeeper + + - name: Start taosd + run: | + cp /etc/taos/taos.cfg ./ + sudo echo "supportVnodes 256" >> taos.cfg + nohup sudo taosd -c taos.cfg & + + - name: Start taosadapter + run: nohup sudo taosadapter & + + - name: Run tests with coverage + working-directory: tools/keeper + run: | + taosBenchmark -t 10 -n 10 -y + taos -s "select count(*) from test.meters" + + - name: Clean up + if: always() + run: | + if pgrep taosd; then sudo pkill taosd; fi + if pgrep taosadapter; then sudo pkill taosadapter; fi From 0928590519fff3d01e6f1c2c99932680088813fa Mon Sep 17 00:00:00 2001 From: haoranchen Date: Fri, 27 Dec 2024 19:43:09 +0800 Subject: [PATCH 04/14] Update taosd-ci-build.yml --- .github/workflows/taosd-ci-build.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/taosd-ci-build.yml b/.github/workflows/taosd-ci-build.yml index 3ac3caaa7f..4495fd737e 100644 --- a/.github/workflows/taosd-ci-build.yml +++ b/.github/workflows/taosd-ci-build.yml @@ -36,7 +36,7 @@ jobs: run: | mkdir debug cd debug - cmake .. -DBUILD_HTTP=false -DBUILD_JDBC=false -DBUILD_TOOLS=false -DBUILD_TEST=off -DBUILD_KEEPER=true -DBUILD_DEPENDENCY_TESTS=false + cmake .. -DBUILD_HTTP=false -DBUILD_JDBC=false -DBUILD_TOOLS=true -DBUILD_TEST=off -DBUILD_KEEPER=true -DBUILD_DEPENDENCY_TESTS=false make -j 4 sudo make install which taosd @@ -52,8 +52,7 @@ jobs: - name: Start taosadapter run: nohup sudo taosadapter & - - name: Run tests with coverage - working-directory: tools/keeper + - name: Run tests with taosBenchmark run: | taosBenchmark -t 10 -n 10 -y taos -s "select count(*) from test.meters" From a675882d14c898b080796b418ab36d851128ea41 Mon Sep 17 00:00:00 2001 From: haoranchen Date: Fri, 27 Dec 2024 19:53:37 +0800 Subject: [PATCH 05/14] fix: add dependencies for taos-tools compile --- .github/workflows/taosd-ci-build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/taosd-ci-build.yml b/.github/workflows/taosd-ci-build.yml index 4495fd737e..8a0294a0a6 100644 --- a/.github/workflows/taosd-ci-build.yml +++ b/.github/workflows/taosd-ci-build.yml @@ -30,7 +30,8 @@ jobs: - name: Install system dependencies run: | sudo apt update -y - sudo apt install -y build-essential cmake libgeos-dev + sudo apt install -y build-essential cmake libgeos-dev libjansson-dev libsnappy-dev liblzma-dev libz-dev zlib1g pkg-config libssl-dev gawk + - name: Install TDengine run: | From a005a132d9e75d3c98e0e491314279d79f60be4b Mon Sep 17 00:00:00 2001 From: haoranchen Date: Fri, 27 Dec 2024 20:15:52 +0800 Subject: [PATCH 06/14] Update taosd-ci-build.yml --- .github/workflows/taosd-ci-build.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/taosd-ci-build.yml b/.github/workflows/taosd-ci-build.yml index 8a0294a0a6..3adfde3460 100644 --- a/.github/workflows/taosd-ci-build.yml +++ b/.github/workflows/taosd-ci-build.yml @@ -8,7 +8,6 @@ on: - '3.1' push: branches: - - 'enh/main/TD-33380' - '3.0' - '3.1' - 'main' From a73ad9642a3c4352e6443a60c043532ed8b71858 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 26 Dec 2024 17:24:17 +0800 Subject: [PATCH 07/14] fix(stream): check if the altered tag value table should be added into stream/tq scan table list. --- source/dnode/vnode/src/vnd/vnodeSvr.c | 31 ++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index f8b75eb15c..4eac1cd5c9 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -1415,7 +1415,8 @@ static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t ver, void *pReq, i SVAlterTbReq vAlterTbReq = {0}; SVAlterTbRsp vAlterTbRsp = {0}; SDecoder dc = {0}; - int32_t rcode = 0; + int32_t code = 0; + int32_t lino = 0; int32_t ret; SEncoder ec = {0}; STableMetaRsp vMetaRsp = {0}; @@ -1431,7 +1432,6 @@ static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t ver, void *pReq, i if (tDecodeSVAlterTbReq(&dc, &vAlterTbReq) < 0) { vAlterTbRsp.code = TSDB_CODE_INVALID_MSG; tDecoderClear(&dc); - rcode = -1; goto _exit; } @@ -1439,7 +1439,6 @@ static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t ver, void *pReq, i if (metaAlterTable(pVnode->pMeta, ver, &vAlterTbReq, &vMetaRsp) < 0) { vAlterTbRsp.code = terrno; tDecoderClear(&dc); - rcode = -1; goto _exit; } tDecoderClear(&dc); @@ -1449,6 +1448,31 @@ static int32_t vnodeProcessAlterTbReq(SVnode *pVnode, int64_t ver, void *pReq, i vAlterTbRsp.pMeta = &vMetaRsp; } + if (vAlterTbReq.action == TSDB_ALTER_TABLE_UPDATE_TAG_VAL || vAlterTbReq.action == TSDB_ALTER_TABLE_UPDATE_MULTI_TAG_VAL) { + int64_t uid = metaGetTableEntryUidByName(pVnode->pMeta, vAlterTbReq.tbName); + if (uid == 0) { + vError("vgId:%d, %s failed at %s:%d since table %s not found", TD_VID(pVnode), __func__, __FILE__, __LINE__, + vAlterTbReq.tbName); + goto _exit; + } + + SArray* tbUids = taosArrayInit(4, sizeof(int64_t)); + void* p = taosArrayPush(tbUids, &uid); + TSDB_CHECK_NULL(p, code, lino, _exit, terrno); + + vDebug("vgId:%d, remove tags value altered table:%s from query table list", TD_VID(pVnode), vAlterTbReq.tbName); + if ((code = tqUpdateTbUidList(pVnode->pTq, tbUids, false)) < 0) { + vError("vgId:%d, failed to remove tbUid list since %s", TD_VID(pVnode), tstrerror(code)); + } + + vDebug("vgId:%d, try to add table:%s in query table list", TD_VID(pVnode), vAlterTbReq.tbName); + if ((code = tqUpdateTbUidList(pVnode->pTq, tbUids, true)) < 0) { + vError("vgId:%d, failed to add tbUid list since %s", TD_VID(pVnode), tstrerror(code)); + } + + taosArrayDestroy(tbUids); + } + _exit: taosArrayDestroy(vAlterTbReq.pMultiTag); tEncodeSize(tEncodeSVAlterTbRsp, &vAlterTbRsp, pRsp->contLen, ret); @@ -1457,6 +1481,7 @@ _exit: if (tEncodeSVAlterTbRsp(&ec, &vAlterTbRsp) != 0) { vError("vgId:%d, failed to encode alter table response", TD_VID(pVnode)); } + tEncoderClear(&ec); if (vMetaRsp.pSchemas) { taosMemoryFree(vMetaRsp.pSchemas); From ba4d47b8b680723748170ac2344058a7c4812d6a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 29 Dec 2024 00:15:58 +0800 Subject: [PATCH 08/14] enh(stream): extend wait time upto 300s --- source/libs/stream/src/streamCheckStatus.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/stream/src/streamCheckStatus.c b/source/libs/stream/src/streamCheckStatus.c index 118cb1cfb6..5250e96a73 100644 --- a/source/libs/stream/src/streamCheckStatus.c +++ b/source/libs/stream/src/streamCheckStatus.c @@ -18,7 +18,7 @@ #include "streamBackendRocksdb.h" #include "streamInt.h" -#define CHECK_NOT_RSP_DURATION 10 * 1000 // 10 sec +#define CHECK_NOT_RSP_DURATION 30 * 1000 // 10 sec static void processDownstreamReadyRsp(SStreamTask* pTask); static void rspMonitorFn(void* param, void* tmrId); @@ -660,7 +660,7 @@ void handleTimeoutDownstreamTasks(SStreamTask* pTask, SArray* pTimeoutList) { pInfo->timeoutRetryCount += 1; - // timeout more than 100 sec, add into node update list + // timeout more than 300 sec, add into node update list if (pInfo->timeoutRetryCount > 10) { pInfo->timeoutRetryCount = 0; @@ -674,7 +674,7 @@ void handleTimeoutDownstreamTasks(SStreamTask* pTask, SArray* pTimeoutList) { findCheckRspStatus(pInfo, *pTaskId, &p); if (p != NULL) { code = streamTaskAddIntoNodeUpdateList(pTask, p->vgId); - stDebug("s-task:%s vgId:%d downstream task:0x%x (vgId:%d) timeout more than 100sec, add into nodeUpdate list", + stDebug("s-task:%s vgId:%d downstream task:0x%x (vgId:%d) timeout more than 300sec, add into nodeUpdate list", id, vgId, p->taskId, p->vgId); } } From 0c8ac0f2ecd2b202432b4e41b626b5aeebffbf7a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 29 Dec 2024 00:42:39 +0800 Subject: [PATCH 09/14] enh(stream): inc the timeout threshold to be 600s. --- source/libs/stream/src/streamCheckStatus.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/stream/src/streamCheckStatus.c b/source/libs/stream/src/streamCheckStatus.c index 5250e96a73..f880526541 100644 --- a/source/libs/stream/src/streamCheckStatus.c +++ b/source/libs/stream/src/streamCheckStatus.c @@ -18,7 +18,7 @@ #include "streamBackendRocksdb.h" #include "streamInt.h" -#define CHECK_NOT_RSP_DURATION 30 * 1000 // 10 sec +#define CHECK_NOT_RSP_DURATION 60 * 1000 // 60 sec static void processDownstreamReadyRsp(SStreamTask* pTask); static void rspMonitorFn(void* param, void* tmrId); @@ -660,7 +660,7 @@ void handleTimeoutDownstreamTasks(SStreamTask* pTask, SArray* pTimeoutList) { pInfo->timeoutRetryCount += 1; - // timeout more than 300 sec, add into node update list + // timeout more than 600 sec, add into node update list if (pInfo->timeoutRetryCount > 10) { pInfo->timeoutRetryCount = 0; @@ -674,7 +674,7 @@ void handleTimeoutDownstreamTasks(SStreamTask* pTask, SArray* pTimeoutList) { findCheckRspStatus(pInfo, *pTaskId, &p); if (p != NULL) { code = streamTaskAddIntoNodeUpdateList(pTask, p->vgId); - stDebug("s-task:%s vgId:%d downstream task:0x%x (vgId:%d) timeout more than 300sec, add into nodeUpdate list", + stDebug("s-task:%s vgId:%d downstream task:0x%x (vgId:%d) timeout more than 600sec, add into nodeUpdate list", id, vgId, p->taskId, p->vgId); } } From 28342a0e4472d2ee644b0a1714d8fb82cdab7603 Mon Sep 17 00:00:00 2001 From: WANG Xu Date: Sun, 29 Dec 2024 15:44:34 +0800 Subject: [PATCH 10/14] refactor: break long lines with backslash --- .github/workflows/taosd-ci-build.yml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/taosd-ci-build.yml b/.github/workflows/taosd-ci-build.yml index 3adfde3460..d0915b9547 100644 --- a/.github/workflows/taosd-ci-build.yml +++ b/.github/workflows/taosd-ci-build.yml @@ -1,4 +1,4 @@ -name: TDengine CI +name: TDengine Build on: pull_request: @@ -29,14 +29,17 @@ jobs: - name: Install system dependencies run: | sudo apt update -y - sudo apt install -y build-essential cmake libgeos-dev libjansson-dev libsnappy-dev liblzma-dev libz-dev zlib1g pkg-config libssl-dev gawk + sudo apt install -y build-essential cmake \ + libgeos-dev libjansson-dev libsnappy-dev liblzma-dev libz-dev \ + zlib1g pkg-config libssl-dev gawk - - name: Install TDengine + - name: Build and install TDengine run: | - mkdir debug - cd debug - cmake .. -DBUILD_HTTP=false -DBUILD_JDBC=false -DBUILD_TOOLS=true -DBUILD_TEST=off -DBUILD_KEEPER=true -DBUILD_DEPENDENCY_TESTS=false + mkdir debug && cd debug + cmake .. -DBUILD_HTTP=false -DBUILD_JDBC=false \ + -DBUILD_TOOLS=true -DBUILD_TEST=off \ + -DBUILD_KEEPER=true -DBUILD_DEPENDENCY_TESTS=false make -j 4 sudo make install which taosd From 6987f3692ca1715263435c70e6cdadcc5eaa7115 Mon Sep 17 00:00:00 2001 From: WANG Xu Date: Sun, 29 Dec 2024 15:57:41 +0800 Subject: [PATCH 11/14] chore: add build badge --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index f827c38975..33a3f948ce 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@

+[![Build Status](https://github.com/taosdata/TDengine/actions/workflows/taosd-ci-build.yml/badge.svg)] [![Coverage Status](https://coveralls.io/repos/github/taosdata/TDengine/badge.svg?branch=3.0)](https://coveralls.io/github/taosdata/TDengine?branch=3.0) [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/4201/badge)](https://bestpractices.coreinfrastructure.org/projects/4201)
From d855b4f59a7991271161da70771abe7a07bf591a Mon Sep 17 00:00:00 2001 From: WANG Xu Date: Sun, 29 Dec 2024 16:04:48 +0800 Subject: [PATCH 12/14] add link to the build workflow --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 33a3f948ce..ff72412434 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@

-[![Build Status](https://github.com/taosdata/TDengine/actions/workflows/taosd-ci-build.yml/badge.svg)] +[![Build Status](https://github.com/taosdata/TDengine/actions/workflows/taosd-ci-build.yml/badge.svg)](https://github.com/taosdata/TDengine/actions/workflows/taosd-ci-build.yml) [![Coverage Status](https://coveralls.io/repos/github/taosdata/TDengine/badge.svg?branch=3.0)](https://coveralls.io/github/taosdata/TDengine?branch=3.0) [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/4201/badge)](https://bestpractices.coreinfrastructure.org/projects/4201)
From 7c2c437a633effe90082ca0c66db85c81ee033ec Mon Sep 17 00:00:00 2001 From: WANG Xu Date: Sun, 29 Dec 2024 16:40:57 +0800 Subject: [PATCH 13/14] refactor: build trigger [skip ci] --- .github/workflows/taosd-ci-build.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/taosd-ci-build.yml b/.github/workflows/taosd-ci-build.yml index d0915b9547..0876f5b731 100644 --- a/.github/workflows/taosd-ci-build.yml +++ b/.github/workflows/taosd-ci-build.yml @@ -6,12 +6,11 @@ on: - 'main' - '3.0' - '3.1' - push: - branches: - - '3.0' - - '3.1' - - 'main' - + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: build: runs-on: ubuntu-latest From 8978b7d97feb557922f57e4157905bb444ba935d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 29 Dec 2024 17:52:43 +0800 Subject: [PATCH 14/14] test: add test cases. --- tests/parallel_test/cases.task | 2 + tests/script/tsim/stream/tag.sim | 110 +++++++++++++++++++++++++++++++ tests/script/tsim/testsuit.sim | 4 ++ 3 files changed, 116 insertions(+) create mode 100644 tests/script/tsim/stream/tag.sim diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index ed73a2742c..8cfb37a189 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -1382,6 +1382,8 @@ ,,y,script,./test.sh -f tsim/stream/basic2.sim ,,y,script,./test.sh -f tsim/stream/basic3.sim ,,y,script,./test.sh -f tsim/stream/basic4.sim +,,y,script,./test.sh -f tsim/stream/basic5.sim +,,y,script,./test.sh -f tsim/stream/tag.sim ,,y,script,./test.sh -f tsim/stream/snodeCheck.sim ,,y,script,./test.sh -f tsim/stream/concurrentcheckpt.sim ,,y,script,./test.sh -f tsim/stream/checkpointInterval0.sim diff --git a/tests/script/tsim/stream/tag.sim b/tests/script/tsim/stream/tag.sim new file mode 100644 index 0000000000..f293f4ac05 --- /dev/null +++ b/tests/script/tsim/stream/tag.sim @@ -0,0 +1,110 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start + +sleep 100 +sql connect + +print step1 +print =============== create database +sql create database test vgroups 2; +sql use test; + +sql create table st1(ts timestamp, a int, b int , c int, d double) tags(x int); +sql create table t1 using st1 tags(1); +sql create table t2 using st1 tags(2); +sql create stream streams1 trigger at_once IGNORE EXPIRED 1 IGNORE UPDATE 0 WATERMARK 100s into streamt as select _wstart as s, count(*) c1 from st1 where x>=2 interval(60s) ; + +run tsim/stream/checkTaskStatus.sim + + +sql insert into t2 values(1648791213000,0,1,1,1.0); +sql insert into t2 values(1648791213001,9,2,2,1.1); +sql insert into t2 values(1648791213009,0,3,3,1.0); + +sql insert into t1 values(1648791223000,0,1,1,1.0); +sql insert into t1 values(1648791223001,9,2,2,1.1); +sql insert into t1 values(1648791223009,0,3,3,1.0); + +sleep 300 + +sql select * from streamt; +if $data01 != 3 then + return -1 +endi + +sql alter table t1 set tag x=3; + +sql insert into t1 values(1648791233000,0,1,1,1.0); +sql insert into t1 values(1648791233001,9,2,2,1.1); +sql insert into t1 values(1648791233009,0,3,3,1.0); + +sleep 1000 +sql select * from streamt; + +if $data01 != 6 then + return -1 +endi + +sql alter table t1 set tag x=1; +sql alter table t2 set tag x=1; + +sql insert into t1 values(1648791243000,0,1,1,1.0); +sql insert into t1 values(1648791243001,9,2,2,1.1); + + +sql select * from streamt; +if $data01 != 6 then + return -1 +endi + +#$loop_count = 0 +#loop2: +# +#sleep 300 +#print 1 sql select * from streamt; +#sql select * from streamt; +# +#print $data00 $data01 $data02 $data03 +#print $data10 $data11 $data12 $data13 +# +#$loop_count = $loop_count + 1 +#if $loop_count == 10 then +# return -1 +#endi +# +## row 0 +#if $data01 != 3 then +# print ======data01=$data01 +# goto loop2 +#endi +# +#if $data02 != 6 then +# print ======data02=$data02 +# goto loop2 +#endi +# +#if $data03 != 3 then +# print ======data03=$data03 +# goto loop2 +#endi +# +## row 1 +#if $data11 != 3 then +# print ======data11=$data11 +# goto loop2 +#endi +# +#if $data12 != 6 then +# print ======data12=$data12 +# goto loop2 +#endi +# +#if $data13 != 3 then +# print ======data13=$data13 +# goto loop2 +#endi +# + +print tag end +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/testsuit.sim b/tests/script/tsim/testsuit.sim index ec52b8c234..67b84c24df 100644 --- a/tests/script/tsim/testsuit.sim +++ b/tests/script/tsim/testsuit.sim @@ -111,6 +111,10 @@ run tsim/stream/distributeInterval0.sim run tsim/stream/distributeSession0.sim run tsim/stream/state0.sim run tsim/stream/basic2.sim +run tsim/stream/basic3.sim +run tsim/stream/basic4.sim +run tsim/stream/basic5.sim +run tsim/stream/tag.sim run tsim/stream/concurrentcheckpt.sim run tsim/insert/basic1.sim run tsim/insert/commit-merge0.sim