From 7034cbee22790d67cba9b19437f97f75cdebbdf4 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 17 Apr 2024 19:48:50 +0800 Subject: [PATCH 1/3] cos/multi-writing: init fid & cid for s3 page cache --- include/util/tdef.h | 6 +++--- source/dnode/vnode/src/tsdb/tsdbFile2.c | 4 ++++ source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 4 ++++ 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/include/util/tdef.h b/include/util/tdef.h index 15d3bb0087..9dd0ef65be 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -149,7 +149,7 @@ typedef enum EOperatorType { OP_TYPE_BIT_OR, // binary comparison operator - OP_TYPE_GREATER_THAN = 40, // MUST KEEP IT FIRST AT COMPARE SECTION + OP_TYPE_GREATER_THAN = 40, // MUST KEEP IT FIRST AT COMPARE SECTION OP_TYPE_GREATER_EQUAL, OP_TYPE_LOWER_THAN, OP_TYPE_LOWER_EQUAL, @@ -170,7 +170,7 @@ typedef enum EOperatorType { OP_TYPE_IS_NOT_TRUE, OP_TYPE_IS_NOT_FALSE, OP_TYPE_IS_NOT_UNKNOWN, - OP_TYPE_COMPARE_MAX_VALUE = 149, // MUST KEEP IT LAST AT COMPARE SECTION + OP_TYPE_COMPARE_MAX_VALUE = 149, // MUST KEEP IT LAST AT COMPARE SECTION // json operator OP_TYPE_JSON_GET_VALUE = 150, @@ -417,7 +417,7 @@ typedef enum ELogicConditionType { #define TSDB_MAX_HASH_SUFFIX (TSDB_TABLE_NAME_LEN - 2) #define TSDB_DEFAULT_HASH_SUFFIX 0 -#define TSDB_MIN_S3_CHUNK_SIZE (32 * 1024) +#define TSDB_MIN_S3_CHUNK_SIZE (128 * 1024) #define TSDB_MAX_S3_CHUNK_SIZE (1024 * 1024) #define TSDB_DEFAULT_S3_CHUNK_SIZE (256 * 1024) #define TSDB_MIN_S3_KEEP_LOCAL (1 * 1440) // unit minute diff --git a/source/dnode/vnode/src/tsdb/tsdbFile2.c b/source/dnode/vnode/src/tsdb/tsdbFile2.c index 792d94ce73..041da5ef24 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile2.c @@ -291,6 +291,10 @@ static void tsdbTFileObjRemoveLC(STFileObj *fobj, bool remove_all) { remove_file(lc_path); } } else { + if (fobj->f->lcn < 1) { + remove_file(fobj->fname); + return; + } // delete by data file prefix char lc_path[TSDB_FILENAME_LEN]; tstrncpy(lc_path, fobj->fname, TSDB_FQDN_LEN); diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index d271690bd1..ad668f77c6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -30,6 +30,10 @@ static int32_t tsdbOpenFileImpl(STsdbFD *pFD) { char lc_path[TSDB_FILENAME_LEN]; tstrncpy(lc_path, path, TSDB_FQDN_LEN); + int32_t vid = 0; + const char *object_name = taosDirEntryBaseName((char *)path); + sscanf(object_name, "v%df%dver%" PRId64 ".data", &vid, &pFD->fid, &pFD->cid); + char *dot = strrchr(lc_path, '.'); if (!dot) { tsdbError("unexpected path: %s", lc_path); From 8ca2b08d5c22475e7a12da7e04b53199f6163052 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 18 Apr 2024 08:48:44 +0800 Subject: [PATCH 2/3] cos/multi-writing: fix remove_file with -1 lcn --- source/dnode/vnode/src/tsdb/tsdbFile2.c | 32 ++++++++++--------------- 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbFile2.c b/source/dnode/vnode/src/tsdb/tsdbFile2.c index 041da5ef24..cde74f5e36 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile2.c @@ -267,34 +267,26 @@ int32_t tsdbTFileObjUnref(STFileObj *fobj) { } static void tsdbTFileObjRemoveLC(STFileObj *fobj, bool remove_all) { - if (fobj->f->type != TSDB_FTYPE_DATA) { + if (fobj->f->type != TSDB_FTYPE_DATA || fobj->f->lcn < 1) { remove_file(fobj->fname); return; } if (!remove_all) { - if (fobj->f->lcn < 1) { - remove_file(fobj->fname); + // remove local last chunk file + char lc_path[TSDB_FILENAME_LEN]; + tstrncpy(lc_path, fobj->fname, TSDB_FQDN_LEN); + + char *dot = strrchr(lc_path, '.'); + if (!dot) { + tsdbError("unexpected path: %s", lc_path); return; - } else { - // remove local last chunk file - char lc_path[TSDB_FILENAME_LEN]; - tstrncpy(lc_path, fobj->fname, TSDB_FQDN_LEN); - - char *dot = strrchr(lc_path, '.'); - if (!dot) { - tsdbError("unexpected path: %s", lc_path); - return; - } - snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - lc_path), "%d.data", fobj->f->lcn); - - remove_file(lc_path); } + snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - lc_path), "%d.data", fobj->f->lcn); + + remove_file(lc_path); + } else { - if (fobj->f->lcn < 1) { - remove_file(fobj->fname); - return; - } // delete by data file prefix char lc_path[TSDB_FILENAME_LEN]; tstrncpy(lc_path, fobj->fname, TSDB_FQDN_LEN); From a39a6660e30741448db2f3efbaabfa1fceaf7b6d Mon Sep 17 00:00:00 2001 From: Alex Duan <51781608+DuanKuanJun@users.noreply.github.com> Date: Thu, 18 Apr 2024 10:19:30 +0800 Subject: [PATCH 3/3] Revert "test: set the par DCMAKE_BUILD_TYPE to release" --- tests/parallel_test/container_build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/parallel_test/container_build.sh b/tests/parallel_test/container_build.sh index c908265b42..f4db7edb8b 100755 --- a/tests/parallel_test/container_build.sh +++ b/tests/parallel_test/container_build.sh @@ -69,7 +69,7 @@ docker run \ -v ${REP_REAL_PATH}/community/contrib/libuv/:${REP_DIR}/community/contrib/libuv \ -v ${REP_REAL_PATH}/community/contrib/lz4/:${REP_DIR}/community/contrib/lz4 \ -v ${REP_REAL_PATH}/community/contrib/zlib/:${REP_DIR}/community/contrib/zlib \ - --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DCMAKE_BUILD_TYPE=Release -DWEBSOCKET=true -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0;make -j 10|| exit 1" + --rm --ulimit core=-1 taos_test:v1.0 sh -c "pip uninstall taospy -y;pip3 install taospy==2.7.2;cd $REP_DIR;rm -rf debug;mkdir -p debug;cd debug;cmake .. -DBUILD_HTTP=false -DBUILD_TOOLS=true -DBUILD_TEST=true -DWEBSOCKET=true -DBUILD_TAOSX=false -DJEMALLOC_ENABLED=0;make -j 10|| exit 1" # -v ${REP_REAL_PATH}/community/contrib/jemalloc/:${REP_DIR}/community/contrib/jemalloc \ if [[ -d ${WORKDIR}/debugNoSan ]] ;then