From 7034cbee22790d67cba9b19437f97f75cdebbdf4 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 17 Apr 2024 19:48:50 +0800 Subject: [PATCH 1/2] cos/multi-writing: init fid & cid for s3 page cache --- include/util/tdef.h | 6 +++--- source/dnode/vnode/src/tsdb/tsdbFile2.c | 4 ++++ source/dnode/vnode/src/tsdb/tsdbReaderWriter.c | 4 ++++ 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/include/util/tdef.h b/include/util/tdef.h index 15d3bb0087..9dd0ef65be 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -149,7 +149,7 @@ typedef enum EOperatorType { OP_TYPE_BIT_OR, // binary comparison operator - OP_TYPE_GREATER_THAN = 40, // MUST KEEP IT FIRST AT COMPARE SECTION + OP_TYPE_GREATER_THAN = 40, // MUST KEEP IT FIRST AT COMPARE SECTION OP_TYPE_GREATER_EQUAL, OP_TYPE_LOWER_THAN, OP_TYPE_LOWER_EQUAL, @@ -170,7 +170,7 @@ typedef enum EOperatorType { OP_TYPE_IS_NOT_TRUE, OP_TYPE_IS_NOT_FALSE, OP_TYPE_IS_NOT_UNKNOWN, - OP_TYPE_COMPARE_MAX_VALUE = 149, // MUST KEEP IT LAST AT COMPARE SECTION + OP_TYPE_COMPARE_MAX_VALUE = 149, // MUST KEEP IT LAST AT COMPARE SECTION // json operator OP_TYPE_JSON_GET_VALUE = 150, @@ -417,7 +417,7 @@ typedef enum ELogicConditionType { #define TSDB_MAX_HASH_SUFFIX (TSDB_TABLE_NAME_LEN - 2) #define TSDB_DEFAULT_HASH_SUFFIX 0 -#define TSDB_MIN_S3_CHUNK_SIZE (32 * 1024) +#define TSDB_MIN_S3_CHUNK_SIZE (128 * 1024) #define TSDB_MAX_S3_CHUNK_SIZE (1024 * 1024) #define TSDB_DEFAULT_S3_CHUNK_SIZE (256 * 1024) #define TSDB_MIN_S3_KEEP_LOCAL (1 * 1440) // unit minute diff --git a/source/dnode/vnode/src/tsdb/tsdbFile2.c b/source/dnode/vnode/src/tsdb/tsdbFile2.c index 792d94ce73..041da5ef24 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile2.c @@ -291,6 +291,10 @@ static void tsdbTFileObjRemoveLC(STFileObj *fobj, bool remove_all) { remove_file(lc_path); } } else { + if (fobj->f->lcn < 1) { + remove_file(fobj->fname); + return; + } // delete by data file prefix char lc_path[TSDB_FILENAME_LEN]; tstrncpy(lc_path, fobj->fname, TSDB_FQDN_LEN); diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index d271690bd1..ad668f77c6 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -30,6 +30,10 @@ static int32_t tsdbOpenFileImpl(STsdbFD *pFD) { char lc_path[TSDB_FILENAME_LEN]; tstrncpy(lc_path, path, TSDB_FQDN_LEN); + int32_t vid = 0; + const char *object_name = taosDirEntryBaseName((char *)path); + sscanf(object_name, "v%df%dver%" PRId64 ".data", &vid, &pFD->fid, &pFD->cid); + char *dot = strrchr(lc_path, '.'); if (!dot) { tsdbError("unexpected path: %s", lc_path); From 8ca2b08d5c22475e7a12da7e04b53199f6163052 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Thu, 18 Apr 2024 08:48:44 +0800 Subject: [PATCH 2/2] cos/multi-writing: fix remove_file with -1 lcn --- source/dnode/vnode/src/tsdb/tsdbFile2.c | 32 ++++++++++--------------- 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbFile2.c b/source/dnode/vnode/src/tsdb/tsdbFile2.c index 041da5ef24..cde74f5e36 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile2.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile2.c @@ -267,34 +267,26 @@ int32_t tsdbTFileObjUnref(STFileObj *fobj) { } static void tsdbTFileObjRemoveLC(STFileObj *fobj, bool remove_all) { - if (fobj->f->type != TSDB_FTYPE_DATA) { + if (fobj->f->type != TSDB_FTYPE_DATA || fobj->f->lcn < 1) { remove_file(fobj->fname); return; } if (!remove_all) { - if (fobj->f->lcn < 1) { - remove_file(fobj->fname); + // remove local last chunk file + char lc_path[TSDB_FILENAME_LEN]; + tstrncpy(lc_path, fobj->fname, TSDB_FQDN_LEN); + + char *dot = strrchr(lc_path, '.'); + if (!dot) { + tsdbError("unexpected path: %s", lc_path); return; - } else { - // remove local last chunk file - char lc_path[TSDB_FILENAME_LEN]; - tstrncpy(lc_path, fobj->fname, TSDB_FQDN_LEN); - - char *dot = strrchr(lc_path, '.'); - if (!dot) { - tsdbError("unexpected path: %s", lc_path); - return; - } - snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - lc_path), "%d.data", fobj->f->lcn); - - remove_file(lc_path); } + snprintf(dot + 1, TSDB_FQDN_LEN - (dot + 1 - lc_path), "%d.data", fobj->f->lcn); + + remove_file(lc_path); + } else { - if (fobj->f->lcn < 1) { - remove_file(fobj->fname); - return; - } // delete by data file prefix char lc_path[TSDB_FILENAME_LEN]; tstrncpy(lc_path, fobj->fname, TSDB_FQDN_LEN);