From 2d7c1a206f6f44913b8deebda1f6e509a839130b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 27 Dec 2024 15:10:21 +0800 Subject: [PATCH 1/2] enh: add more cases --- source/libs/executor/src/sysscanoperator.c | 2 + tests/parallel_test/cases.task | 1 + tests/system-test/0-others/ins_filesets.py | 54 ++++++++++++++++++++++ 3 files changed, 57 insertions(+) create mode 100644 tests/system-test/0-others/ins_filesets.py diff --git a/source/libs/executor/src/sysscanoperator.c b/source/libs/executor/src/sysscanoperator.c index 7862b36f27..56060b0061 100644 --- a/source/libs/executor/src/sysscanoperator.c +++ b/source/libs/executor/src/sysscanoperator.c @@ -2269,6 +2269,8 @@ static SSDataBlock* sysTableBuildUserFileSets(SOperatorInfo* pOperator) { if (ret) { if (ret == TSDB_CODE_NOT_FOUND) { // no more scan entry + setOperatorCompleted(pOperator); + pAPI->tsdReader.fileSetReaderClose(&pInfo->pFileSetReader); break; } else { code = ret; diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 38dd080ef5..8903ef27a4 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -484,6 +484,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/show.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/show_tag_index.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/information_schema.py +,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/ins_filesets.py ,,y,system-test,./pytest.sh python3 ./test.py -f 0-others/grant.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py ,,y,system-test,./pytest.sh python3 ./test.py -f 2-query/abs.py -R diff --git a/tests/system-test/0-others/ins_filesets.py b/tests/system-test/0-others/ins_filesets.py new file mode 100644 index 0000000000..e5bc4b61b7 --- /dev/null +++ b/tests/system-test/0-others/ins_filesets.py @@ -0,0 +1,54 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import re +from util.log import * +from util.cases import * +from util.sql import * +from util.common import * +from util.sqlset import * + +class TDTestCase: + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + + def run(self): + tdSql.execute('create database db vgroups 1') + tdSql.execute('use db') + tdSql.execute('create table t1 (ts timestamp, a int, b int)') + tdSql.execute('insert into t1 values(\'2024-12-27 14:00:00\', 1, 2)') + tdSql.execute('flush database db') + + tdLog.sleep(5) + + rows = tdSql.query('select * from information_schema.ins_filesets') + tdSql.checkRows(1) + tdSql.checkEqual(tdSql.getData(0, 0), 'db') + tdSql.checkEqual(tdSql.getData(0, 1), 2) + tdSql.checkEqual(tdSql.getData(0, 2), 2008) + # tdSql.CheckEqual(str(tdSql.getData(0, 3)), '2024-12-23 08:00:00.000') + # tdSql.CheckEqual(str(tdSql.getData(0, 4)), '2025-01-02 07:59:59.999') + # tdSql.CheckEqual(tdSql.getData(0, 6), '1970-01-01 08:00:00.000') + # tdSql.CheckEqual(tdSql.getData(0, 7), False) + + tdDnodes.stopAll() + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) From a72c46a7fc28cb38d078c715956b742d68e3a255 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 27 Dec 2024 17:44:05 +0800 Subject: [PATCH 2/2] more cases --- source/dnode/vnode/src/tsdb/tsdbFS2.c | 25 +++++++++++++++---- source/dnode/vnode/src/tsdb/tsdbFSet2.c | 32 +++++++++++++++++++++++++ source/dnode/vnode/src/tsdb/tsdbFSet2.h | 2 ++ 3 files changed, 55 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbFS2.c b/source/dnode/vnode/src/tsdb/tsdbFS2.c index 82dd49b0e2..aa68a8af5c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS2.c @@ -680,7 +680,7 @@ static int32_t fset_cmpr_fn(const struct STFileSet *pSet1, const struct STFileSe return 0; } -static int32_t edit_fs(STFileSystem *fs, const TFileOpArray *opArray) { +static int32_t edit_fs(STFileSystem *fs, const TFileOpArray *opArray, EFEditT etype) { int32_t code = 0; int32_t lino = 0; @@ -690,6 +690,8 @@ static int32_t edit_fs(STFileSystem *fs, const TFileOpArray *opArray) { TFileSetArray *fsetArray = fs->fSetArrTmp; STFileSet *fset = NULL; const STFileOp *op; + int32_t fid = INT32_MIN; + TSKEY now = taosGetTimestampMs(); TARRAY2_FOREACH_PTR(opArray, op) { if (!fset || fset->fid != op->fid) { STFileSet tfset = {.fid = op->fid}; @@ -708,6 +710,15 @@ static int32_t edit_fs(STFileSystem *fs, const TFileOpArray *opArray) { code = tsdbTFileSetEdit(fs->tsdb, fset, op); TSDB_CHECK_CODE(code, lino, _exit); + + if (fid != op->fid) { + fid = op->fid; + if (etype == TSDB_FEDIT_COMMIT) { + fset->lastCommit = now; + } else if (etype == TSDB_FEDIT_COMPACT) { + fset->lastCompact = now; + } + } } // remove empty empty stt level and empty file set @@ -864,7 +875,7 @@ int32_t tsdbFSEditBegin(STFileSystem *fs, const TFileOpArray *opArray, EFEditT e fs->etype = etype; // edit - code = edit_fs(fs, opArray); + code = edit_fs(fs, opArray, etype); TSDB_CHECK_CODE(code, lino, _exit); // save fs @@ -1288,6 +1299,12 @@ int32_t tsdbFileSetReaderOpen(void *pVnode, struct SFileSetReader **ppReader) { return TSDB_CODE_SUCCESS; } +extern bool tsdbShouldCompact(const STFileSet *pFileSet); + +#ifndef TD_ENTERPRISE +bool tsdbShouldCompact(const STFileSet *pFileSet) { return false; } +#endif + static int32_t tsdbFileSetReaderNextNoLock(struct SFileSetReader *pReader) { STsdb *pTsdb = pReader->pTsdb; int32_t code = TSDB_CODE_SUCCESS; @@ -1311,7 +1328,7 @@ static int32_t tsdbFileSetReaderNextNoLock(struct SFileSetReader *pReader) { // get file set details pReader->fid = pReader->pFileSet->fid; tsdbFidKeyRange(pReader->fid, pTsdb->keepCfg.days, pTsdb->keepCfg.precision, &pReader->startTime, &pReader->endTime); - pReader->lastCompactTime = 0; // TODO + pReader->lastCompactTime = pReader->pFileSet->lastCompact; pReader->totalSize = 0; for (int32_t i = 0; i < TSDB_FTYPE_MAX; i++) { STFileObj *fobj = pReader->pFileSet->farr[i]; @@ -1375,7 +1392,7 @@ int32_t tsdbFileSetGetEntryField(struct SFileSetReader *pReader, const char *fie fieldName = "should_compact"; if (strncmp(field, fieldName, strlen(fieldName) + 1) == 0) { - *(char *)value = 0; // TODO + *(char *)value = tsdbShouldCompact(pReader->pFileSet); return TSDB_CODE_SUCCESS; } diff --git a/source/dnode/vnode/src/tsdb/tsdbFSet2.c b/source/dnode/vnode/src/tsdb/tsdbFSet2.c index 68914300e4..6530ee2269 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSet2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.c @@ -273,6 +273,15 @@ int32_t tsdbTFileSetToJson(const STFileSet *fset, cJSON *json) { if (code) return code; } + // about compact and commit + if (cJSON_AddNumberToObject(json, "last compact", fset->lastCompact) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + if (cJSON_AddNumberToObject(json, "last commit", fset->lastCommit) == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + return 0; } @@ -324,6 +333,20 @@ int32_t tsdbJsonToTFileSet(STsdb *pTsdb, const cJSON *json, STFileSet **fset) { } else { return TSDB_CODE_FILE_CORRUPTED; } + // about compact and commit + item1 = cJSON_GetObjectItem(json, "last compact"); + if (cJSON_IsNumber(item1)) { + (*fset)->lastCompact = item1->valuedouble; + } else { + (*fset)->lastCompact = 0; + } + + item1 = cJSON_GetObjectItem(json, "last commit"); + if (cJSON_IsNumber(item1)) { + (*fset)->lastCommit = item1->valuedouble; + } else { + (*fset)->lastCommit = 0; + } return 0; } @@ -467,6 +490,9 @@ int32_t tsdbTFileSetApplyEdit(STsdb *pTsdb, const STFileSet *fset1, STFileSet *f } } + fset2->lastCompact = fset1->lastCompact; + fset2->lastCommit = fset1->lastCommit; + return 0; } @@ -522,6 +548,9 @@ int32_t tsdbTFileSetInitCopy(STsdb *pTsdb, const STFileSet *fset1, STFileSet **f if (code) return code; } + (*fset)->lastCompact = fset1->lastCompact; + (*fset)->lastCommit = fset1->lastCommit; + return 0; } @@ -617,6 +646,9 @@ int32_t tsdbTFileSetInitRef(STsdb *pTsdb, const STFileSet *fset1, STFileSet **fs } } + (*fset)->lastCompact = fset1->lastCompact; + (*fset)->lastCommit = fset1->lastCommit; + return 0; } diff --git a/source/dnode/vnode/src/tsdb/tsdbFSet2.h b/source/dnode/vnode/src/tsdb/tsdbFSet2.h index 83ef32e5e5..ca9c133e9c 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFSet2.h +++ b/source/dnode/vnode/src/tsdb/tsdbFSet2.h @@ -92,6 +92,8 @@ struct STFileSet { int64_t maxVerValid; STFileObj *farr[TSDB_FTYPE_MAX]; // file array TSttLvlArray lvlArr[1]; // level array + TSKEY lastCompact; + TSKEY lastCommit; bool mergeScheduled; SVATaskID mergeTask;