From 5eca4ebd278330a56ba2c9479a2d6107e9c02f39 Mon Sep 17 00:00:00 2001 From: lichuang Date: Fri, 18 Jun 2021 11:33:32 +0800 Subject: [PATCH 01/22] [TD-4352]refactor duplicate insert new table actions to function --- src/tsdb/src/tsdbMeta.c | 45 ++++++++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 621be04e21..6cf4ef28d2 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -44,6 +44,7 @@ static int tsdbRemoveTableFromStore(STsdbRepo *pRepo, STable *pTable); static int tsdbRmTableFromMeta(STsdbRepo *pRepo, STable *pTable); static int tsdbAdjustMetaTables(STsdbRepo *pRepo, int tid); static int tsdbCheckTableTagVal(SKVRow *pKVRow, STSchema *pSchema); +static int tsdbInsertNewTableAction(STsdbRepo *pRepo, STable* pTable); // ------------------ OUTER FUNCTIONS ------------------ int tsdbCreateTable(STsdbRepo *repo, STableCfg *pCfg) { @@ -127,21 +128,16 @@ int tsdbCreateTable(STsdbRepo *repo, STableCfg *pCfg) { tsdbUnlockRepoMeta(pRepo); // Write to memtable action - // TODO: refactor duplicate codes - int tlen = 0; - void *pBuf = NULL; if (newSuper || superChanged) { - tlen = tsdbGetTableEncodeSize(TSDB_UPDATE_META, super); - pBuf = tsdbAllocBytes(pRepo, tlen); - if (pBuf == NULL) goto _err; - void *tBuf = tsdbInsertTableAct(pRepo, TSDB_UPDATE_META, pBuf, super); - ASSERT(POINTER_DISTANCE(tBuf, pBuf) == tlen); + // add insert new super table action + if (tsdbInsertNewTableAction(pRepo, super) != 0) { + goto _err; + } + } + // add insert new table action + if (tsdbInsertNewTableAction(pRepo, table) != 0) { + goto _err; } - tlen = tsdbGetTableEncodeSize(TSDB_UPDATE_META, table); - pBuf = tsdbAllocBytes(pRepo, tlen); - if (pBuf == NULL) goto _err; - void *tBuf = tsdbInsertTableAct(pRepo, TSDB_UPDATE_META, pBuf, table); - ASSERT(POINTER_DISTANCE(tBuf, pBuf) == tlen); if (tsdbCheckCommit(pRepo) < 0) return -1; @@ -382,7 +378,7 @@ int tsdbUpdateTableTagValue(STsdbRepo *repo, SUpdateTableTagValMsg *pMsg) { tdDestroyTSchemaBuilder(&schemaBuilder); } - // Chage in memory + // Change in memory if (pNewSchema != NULL) { // change super table tag schema TSDB_WLOCK_TABLE(pTable->pSuper); STSchema *pOldSchema = pTable->pSuper->tagSchema; @@ -425,6 +421,21 @@ int tsdbUpdateTableTagValue(STsdbRepo *repo, SUpdateTableTagValMsg *pMsg) { } // ------------------ INTERNAL FUNCTIONS ------------------ +static int tsdbInsertNewTableAction(STsdbRepo *pRepo, STable* pTable) { + int tlen = 0; + void *pBuf = NULL; + + tlen = tsdbGetTableEncodeSize(TSDB_UPDATE_META, pTable); + pBuf = tsdbAllocBytes(pRepo, tlen); + if (pBuf == NULL) { + return -1; + } + void *tBuf = tsdbInsertTableAct(pRepo, TSDB_UPDATE_META, pBuf, pTable); + ASSERT(POINTER_DISTANCE(tBuf, pBuf) == tlen); + + return 0; +} + STsdbMeta *tsdbNewMeta(STsdbCfg *pCfg) { STsdbMeta *pMeta = (STsdbMeta *)calloc(1, sizeof(*pMeta)); if (pMeta == NULL) { @@ -616,6 +627,7 @@ int16_t tsdbGetLastColumnsIndexByColId(STable* pTable, int16_t colId) { if (pTable->lastCols == NULL) { return -1; } + // TODO: use binary search instead for (int16_t i = 0; i < pTable->maxColNum; ++i) { if (pTable->lastCols[i].colId == colId) { return i; @@ -740,10 +752,7 @@ void tsdbUpdateTableSchema(STsdbRepo *pRepo, STable *pTable, STSchema *pSchema, TSDB_WUNLOCK_TABLE(pCTable); if (insertAct) { - int tlen = tsdbGetTableEncodeSize(TSDB_UPDATE_META, pCTable); - void *buf = tsdbAllocBytes(pRepo, tlen); - ASSERT(buf != NULL); - tsdbInsertTableAct(pRepo, TSDB_UPDATE_META, buf, pCTable); + ASSERT(tsdbInsertNewTableAction(pRepo, pCTable) == 0); } } From 5f5a802bb939c31c21cd800156adbff81600de33 Mon Sep 17 00:00:00 2001 From: lichuang Date: Fri, 18 Jun 2021 11:35:06 +0800 Subject: [PATCH 02/22] [TD-4352]update meta maxCols and maxRowBytes after remove table --- src/tsdb/src/tsdbMeta.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 6cf4ef28d2..9d5df9fb32 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -1043,6 +1043,8 @@ static void tsdbRemoveTableFromMeta(STsdbRepo *pRepo, STable *pTable, bool rmFro maxRowBytes = MAX(maxRowBytes, schemaTLen(pSchema)); } } + pMeta->maxCols = maxCols; + pMeta->maxRowBytes = maxRowBytes; } if (lock) tsdbUnlockRepoMeta(pRepo); From b50154343ddc1cc144ffb507151a9d8929da11f7 Mon Sep 17 00:00:00 2001 From: lichuang Date: Tue, 22 Jun 2021 16:41:11 +0800 Subject: [PATCH 03/22] [TD-4352]compact tsdb meta data implementation --- src/tsdb/inc/tsdbFile.h | 5 +- src/tsdb/src/tsdbCommit.c | 107 ++++++++++++++++++++++++++++++++++++-- src/tsdb/src/tsdbFS.c | 2 +- src/tsdb/src/tsdbFile.c | 31 ++++++++--- src/tsdb/src/tsdbSync.c | 2 +- 5 files changed, 132 insertions(+), 15 deletions(-) diff --git a/src/tsdb/inc/tsdbFile.h b/src/tsdb/inc/tsdbFile.h index dcb5eadfab..9ebf1253cb 100644 --- a/src/tsdb/inc/tsdbFile.h +++ b/src/tsdb/inc/tsdbFile.h @@ -38,7 +38,7 @@ #define TSDB_FILE_IS_OK(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_OK) #define TSDB_FILE_IS_BAD(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_BAD) -typedef enum { TSDB_FILE_HEAD = 0, TSDB_FILE_DATA, TSDB_FILE_LAST, TSDB_FILE_MAX, TSDB_FILE_META } TSDB_FILE_T; +typedef enum { TSDB_FILE_HEAD = 0, TSDB_FILE_DATA, TSDB_FILE_LAST, TSDB_FILE_MAX, TSDB_FILE_META, TSDB_FILE_META_TMP} TSDB_FILE_T; // =============== SMFile typedef struct { @@ -56,7 +56,8 @@ typedef struct { uint8_t state; } SMFile; -void tsdbInitMFile(SMFile* pMFile, SDiskID did, int vid, uint32_t ver); +void tsdbInitMFile(SMFile* pMFile, SDiskID did, int vid, uint32_t ver, bool tmp); +void tsdbRenameOrDeleleTempMetaFile(SMFile* pMFile, SDiskID did, int vid, uint32_t ver, int code); void tsdbInitMFileEx(SMFile* pMFile, const SMFile* pOMFile); int tsdbEncodeSMFile(void** buf, SMFile* pMFile); void* tsdbDecodeSMFile(void* buf, SMFile* pMFile); diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index 82cc6f07f7..6c42311abc 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -55,8 +55,9 @@ typedef struct { #define TSDB_COMMIT_TXN_VERSION(ch) FS_TXN_VERSION(REPO_FS(TSDB_COMMIT_REPO(ch))) static int tsdbCommitMeta(STsdbRepo *pRepo); -static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void *cont, int contLen); +static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void *cont, int contLen, bool updateMeta); static int tsdbDropMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid); +static int tsdbCompactMetaFile(STsdbRepo *pRepo, STsdbFS *pfs, SMFile *pMFile); static int tsdbCommitTSData(STsdbRepo *pRepo); static void tsdbStartCommit(STsdbRepo *pRepo); static void tsdbEndCommit(STsdbRepo *pRepo, int eno); @@ -283,7 +284,7 @@ static int tsdbCommitMeta(STsdbRepo *pRepo) { // Create a new meta file did.level = TFS_PRIMARY_LEVEL; did.id = TFS_PRIMARY_ID; - tsdbInitMFile(&mf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo))); + tsdbInitMFile(&mf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo)), false); if (tsdbCreateMFile(&mf, true) < 0) { tsdbError("vgId:%d failed to create META file since %s", REPO_ID(pRepo), tstrerror(terrno)); @@ -305,7 +306,7 @@ static int tsdbCommitMeta(STsdbRepo *pRepo) { pAct = (SActObj *)pNode->data; if (pAct->act == TSDB_UPDATE_META) { pCont = (SActCont *)POINTER_SHIFT(pAct, sizeof(SActObj)); - if (tsdbUpdateMetaRecord(pfs, &mf, pAct->uid, (void *)(pCont->cont), pCont->len) < 0) { + if (tsdbUpdateMetaRecord(pfs, &mf, pAct->uid, (void *)(pCont->cont), pCont->len, true) < 0) { tsdbError("vgId:%d failed to update META record, uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid, tstrerror(terrno)); tsdbCloseMFile(&mf); @@ -338,6 +339,10 @@ static int tsdbCommitMeta(STsdbRepo *pRepo) { tsdbCloseMFile(&mf); tsdbUpdateMFile(pfs, &mf); + if (tsdbCompactMetaFile(pRepo, pfs, &mf) < 0) { + tsdbError("compact meta file error"); + } + return 0; } @@ -375,7 +380,7 @@ void tsdbGetRtnSnap(STsdbRepo *pRepo, SRtn *pRtn) { pRtn->minFid, pRtn->midFid, pRtn->maxFid); } -static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void *cont, int contLen) { +static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void *cont, int contLen, bool updateMeta) { char buf[64] = "\0"; void * pBuf = buf; SKVRecord rInfo; @@ -401,6 +406,11 @@ static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void } tsdbUpdateMFileMagic(pMFile, POINTER_SHIFT(cont, contLen - sizeof(TSCKSUM))); + if (!updateMeta) { + pMFile->info.nRecords++; + return 0; + } + SKVRecord *pRecord = taosHashGet(pfs->metaCache, (void *)&uid, sizeof(uid)); if (pRecord != NULL) { pMFile->info.tombSize += (pRecord->size + sizeof(SKVRecord)); @@ -442,6 +452,95 @@ static int tsdbDropMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid) { return 0; } +static int tsdbCompactMetaFile(STsdbRepo *pRepo, STsdbFS *pfs, SMFile *pMFile) { + float delPercent = pMFile->info.nDels * 1.0 / pMFile->info.nRecords; + float tombPercent = pMFile->info.tombSize * 1.0 / pMFile->info.size; + + if (delPercent < 0.33 && tombPercent < 0.33) { + return 0; + } + + tsdbInfo("begin compact tsdb meta file, nDels:%" PRId64 ",nRecords:%" PRId64 ",tombSize:%" PRId64 ",size:%" PRId64, + pMFile->info.nDels,pMFile->info.nRecords,pMFile->info.tombSize,pMFile->info.size); + + SMFile mf; + SDiskID did; + + // first create tmp meta file + did.level = TFS_PRIMARY_LEVEL; + did.id = TFS_PRIMARY_ID; + tsdbInitMFile(&mf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo)), true); + + if (tsdbCreateMFile(&mf, true) < 0) { + tsdbError("vgId:%d failed to create META file since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + tsdbInfo("vgId:%d meta file %s is created to compact meta data", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(&mf)); + + // second iterator metaCache + int code = -1; + int64_t maxBufSize = 1024; + SKVRecord *pRecord; + void *pBuf = NULL; + + pBuf = malloc((size_t)maxBufSize); + if (pBuf == NULL) { + goto _err; + } + + pRecord = taosHashIterate(pfs->metaCache, NULL); + while (pRecord) { + if (tsdbSeekMFile(pMFile, pRecord->offset + sizeof(SKVRecord), SEEK_SET) < 0) { + tsdbError("vgId:%d failed to seek file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), + tstrerror(terrno)); + break; + } + if (pRecord->size > maxBufSize) { + maxBufSize = pRecord->size; + void* tmp = realloc(pBuf, maxBufSize); + if (tmp == NULL) { + break; + } + pBuf = tmp; + } + int nread = (int)tsdbReadMFile(pMFile, pBuf, pRecord->size); + if (nread < 0) { + tsdbError("vgId:%d failed to read file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), + tstrerror(terrno)); + break; + } + + if (nread < pRecord->size) { + tsdbError("vgId:%d failed to read file %s since file corrupted, expected read:%" PRId64 " actual read:%d", + REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), pRecord->size, nread); + break; + } + + if (tsdbUpdateMetaRecord(pfs, &mf, pRecord->uid, pBuf, pRecord->size, false) < 0) { + tsdbError("vgId:%d failed to update META record, uid %" PRIu64 " since %s", REPO_ID(pRepo), pRecord->uid, + tstrerror(terrno)); + break; + } + + pRecord = taosHashIterate(pfs->metaCache, pRecord); + } + code = 0; + +_err: + TSDB_FILE_FSYNC(&mf); + tsdbCloseMFile(&mf); + + tsdbRenameOrDeleleTempMetaFile(&mf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo)), code); + if (code == 0) { + tsdbUpdateMFile(pfs, &mf); + } + + tfree(pBuf); + tsdbInfo("end compact tsdb meta file, code:%d", code); + return code; +} + // =================== Commit Time-Series Data static int tsdbCommitTSData(STsdbRepo *pRepo) { SMemTable *pMem = pRepo->imem; diff --git a/src/tsdb/src/tsdbFS.c b/src/tsdb/src/tsdbFS.c index 54372ae8c2..35e9998323 100644 --- a/src/tsdb/src/tsdbFS.c +++ b/src/tsdb/src/tsdbFS.c @@ -272,7 +272,7 @@ static int tsdbCreateMeta(STsdbRepo *pRepo) { // Create a new meta file did.level = TFS_PRIMARY_LEVEL; did.id = TFS_PRIMARY_ID; - tsdbInitMFile(&mf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo))); + tsdbInitMFile(&mf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo)), false); if (tsdbCreateMFile(&mf, true) < 0) { tsdbError("vgId:%d failed to create META file since %s", REPO_ID(pRepo), tstrerror(terrno)); diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 50fa393e9f..b1ff79bfef 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -16,11 +16,12 @@ #include "tsdbint.h" static const char *TSDB_FNAME_SUFFIX[] = { - "head", // TSDB_FILE_HEAD - "data", // TSDB_FILE_DATA - "last", // TSDB_FILE_LAST - "", // TSDB_FILE_MAX - "meta" // TSDB_FILE_META + "head", // TSDB_FILE_HEAD + "data", // TSDB_FILE_DATA + "last", // TSDB_FILE_LAST + "", // TSDB_FILE_MAX + "meta" // TSDB_FILE_META + "meta.tmp" // TSDB_FILE_META_TMP }; static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, char *fname); @@ -30,7 +31,7 @@ static void *tsdbDecodeDFInfo(void *buf, SDFInfo *pInfo); static int tsdbRollBackDFile(SDFile *pDFile); // ============== SMFile -void tsdbInitMFile(SMFile *pMFile, SDiskID did, int vid, uint32_t ver) { +void tsdbInitMFile(SMFile *pMFile, SDiskID did, int vid, uint32_t ver, bool tmp) { char fname[TSDB_FILENAME_LEN]; TSDB_FILE_SET_STATE(pMFile, TSDB_FILE_STATE_OK); @@ -38,10 +39,26 @@ void tsdbInitMFile(SMFile *pMFile, SDiskID did, int vid, uint32_t ver) { memset(&(pMFile->info), 0, sizeof(pMFile->info)); pMFile->info.magic = TSDB_FILE_INIT_MAGIC; - tsdbGetFilename(vid, 0, ver, TSDB_FILE_META, fname); + tsdbGetFilename(vid, 0, ver, tmp ? TSDB_FILE_META_TMP : TSDB_FILE_META, fname); tfsInitFile(TSDB_FILE_F(pMFile), did.level, did.id, fname); } +void tsdbRenameOrDeleleTempMetaFile(SMFile* pMFile, SDiskID did, int vid, uint32_t ver, int code) { + char mfname[TSDB_FILENAME_LEN] = {'\0'}; + char tfname[TSDB_FILENAME_LEN] = {'\0'}; + + tsdbGetFilename(vid, 0, ver, TSDB_FILE_META_TMP, tfname); + + if (code != 0) { + remove(tfname); + return; + } + + tsdbGetFilename(vid, 0, ver, TSDB_FILE_META, mfname); + + (void)taosRename(tfname, mfname); +} + void tsdbInitMFileEx(SMFile *pMFile, const SMFile *pOMFile) { *pMFile = *pOMFile; TSDB_FILE_SET_CLOSED(pMFile); diff --git a/src/tsdb/src/tsdbSync.c b/src/tsdb/src/tsdbSync.c index edcb84d091..f06943bc37 100644 --- a/src/tsdb/src/tsdbSync.c +++ b/src/tsdb/src/tsdbSync.c @@ -209,7 +209,7 @@ static int32_t tsdbSyncRecvMeta(SSyncH *pSynch) { // Recv from remote SMFile mf; SDiskID did = {.level = TFS_PRIMARY_LEVEL, .id = TFS_PRIMARY_ID}; - tsdbInitMFile(&mf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo))); + tsdbInitMFile(&mf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo)), false); if (tsdbCreateMFile(&mf, false) < 0) { tsdbError("vgId:%d, failed to create file while recv metafile since %s", REPO_ID(pRepo), tstrerror(terrno)); return -1; From ea75e443ed80905718e01a61b091f4fc6589f944 Mon Sep 17 00:00:00 2001 From: lichuang Date: Tue, 22 Jun 2021 16:43:57 +0800 Subject: [PATCH 04/22] [TD-4352]compact tsdb meta data implementation --- src/tsdb/src/tsdbCommit.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index 6c42311abc..a9a2b2ded3 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -537,7 +537,9 @@ _err: } tfree(pBuf); - tsdbInfo("end compact tsdb meta file, code:%d", code); + + tsdbInfo("end compact tsdb meta file, code:%d, nDels:%" PRId64 ",nRecords:%" PRId64 ",tombSize:%" PRId64 ",size:%" PRId64, + code, mf.info.nDels,mf.info.nRecords,mf.info.tombSize,mf.info.size); return code; } From dd5642260737f250ea472ab1ed2ecf7c23aeadff Mon Sep 17 00:00:00 2001 From: lichuang Date: Tue, 22 Jun 2021 16:47:13 +0800 Subject: [PATCH 05/22] [TD-4352]compact tsdb meta data implementation --- src/tsdb/src/tsdbCommit.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index a9a2b2ded3..68d0fb9ae1 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -538,8 +538,10 @@ _err: tfree(pBuf); - tsdbInfo("end compact tsdb meta file, code:%d, nDels:%" PRId64 ",nRecords:%" PRId64 ",tombSize:%" PRId64 ",size:%" PRId64, - code, mf.info.nDels,mf.info.nRecords,mf.info.tombSize,mf.info.size); + ASSERT(mf.info.nDels == 0); + ASSERT(mf.info.tombSize == 0); + tsdbInfo("end compact tsdb meta file,code:%d,nRecords:%" PRId64 ",size:%" PRId64, + code,mf.info.nRecords,mf.info.size); return code; } From 9b6edbe7e75814325c8482cb3264677cdc383ff4 Mon Sep 17 00:00:00 2001 From: lichuang Date: Tue, 22 Jun 2021 21:03:31 +0800 Subject: [PATCH 06/22] [TD-4352]fix compile error --- src/tsdb/src/tsdbCommit.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index 68d0fb9ae1..bb08c159b8 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -453,8 +453,8 @@ static int tsdbDropMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid) { } static int tsdbCompactMetaFile(STsdbRepo *pRepo, STsdbFS *pfs, SMFile *pMFile) { - float delPercent = pMFile->info.nDels * 1.0 / pMFile->info.nRecords; - float tombPercent = pMFile->info.tombSize * 1.0 / pMFile->info.size; + float delPercent = (float)(pMFile->info.nDels) / (float)(pMFile->info.nRecords); + float tombPercent = (float)(pMFile->info.tombSize) / (float)(pMFile->info.size); if (delPercent < 0.33 && tombPercent < 0.33) { return 0; @@ -540,6 +540,7 @@ _err: ASSERT(mf.info.nDels == 0); ASSERT(mf.info.tombSize == 0); + tsdbInfo("end compact tsdb meta file,code:%d,nRecords:%" PRId64 ",size:%" PRId64, code,mf.info.nRecords,mf.info.size); return code; From 64ade71d7504c0e680ab8637919a7e353dd5117d Mon Sep 17 00:00:00 2001 From: lichuang Date: Tue, 22 Jun 2021 21:11:48 +0800 Subject: [PATCH 07/22] [TD-4352]fix compile error --- src/tsdb/src/tsdbCommit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index bb08c159b8..9ff4f673da 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -517,7 +517,7 @@ static int tsdbCompactMetaFile(STsdbRepo *pRepo, STsdbFS *pfs, SMFile *pMFile) { break; } - if (tsdbUpdateMetaRecord(pfs, &mf, pRecord->uid, pBuf, pRecord->size, false) < 0) { + if (tsdbUpdateMetaRecord(pfs, &mf, pRecord->uid, pBuf, (int)pRecord->size, false) < 0) { tsdbError("vgId:%d failed to update META record, uid %" PRIu64 " since %s", REPO_ID(pRepo), pRecord->uid, tstrerror(terrno)); break; From 3237ee5c5b0a6fd3587d6e2e09b87e2bc18fcca8 Mon Sep 17 00:00:00 2001 From: lichuang Date: Tue, 22 Jun 2021 22:30:19 +0800 Subject: [PATCH 08/22] [TD-4352]fix compile error --- src/tsdb/src/tsdbCommit.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index 9ff4f673da..df0f8b4d78 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -498,7 +498,7 @@ static int tsdbCompactMetaFile(STsdbRepo *pRepo, STsdbFS *pfs, SMFile *pMFile) { } if (pRecord->size > maxBufSize) { maxBufSize = pRecord->size; - void* tmp = realloc(pBuf, maxBufSize); + void* tmp = realloc(pBuf, (size_t)maxBufSize); if (tmp == NULL) { break; } From eaf84f0fb7db5ba379c298e4c4dead8ae5d81312 Mon Sep 17 00:00:00 2001 From: tomchon Date: Wed, 14 Jul 2021 18:14:01 +0800 Subject: [PATCH 09/22] [TD-4840]add testcase of compacting sdb --- tests/pytest/tsdb/insertDataDb1.json | 87 +++++++++++ tests/pytest/tsdb/insertDataDb1Replica2.json | 87 +++++++++++ tests/pytest/tsdb/insertDataDb2.json | 86 +++++++++++ tests/pytest/tsdb/insertDataDb2Newstab.json | 86 +++++++++++ .../tsdb/insertDataDb2NewstabReplica2.json | 86 +++++++++++ tests/pytest/tsdb/insertDataDb2Replica2.json | 86 +++++++++++ tests/pytest/tsdb/sdbComp.py | 121 ++++++++++++++++ tests/pytest/tsdb/sdbCompCluster.py | 135 +++++++++++++++++ tests/pytest/tsdb/sdbCompClusterReplica2.py | 136 ++++++++++++++++++ 9 files changed, 910 insertions(+) create mode 100644 tests/pytest/tsdb/insertDataDb1.json create mode 100644 tests/pytest/tsdb/insertDataDb1Replica2.json create mode 100644 tests/pytest/tsdb/insertDataDb2.json create mode 100644 tests/pytest/tsdb/insertDataDb2Newstab.json create mode 100644 tests/pytest/tsdb/insertDataDb2NewstabReplica2.json create mode 100644 tests/pytest/tsdb/insertDataDb2Replica2.json create mode 100644 tests/pytest/tsdb/sdbComp.py create mode 100644 tests/pytest/tsdb/sdbCompCluster.py create mode 100644 tests/pytest/tsdb/sdbCompClusterReplica2.py diff --git a/tests/pytest/tsdb/insertDataDb1.json b/tests/pytest/tsdb/insertDataDb1.json new file mode 100644 index 0000000000..f01cc35a1b --- /dev/null +++ b/tests/pytest/tsdb/insertDataDb1.json @@ -0,0 +1,87 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 10, + "num_of_records_per_req": 1000, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "db1", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 50, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"no", + "childtable_count": 100000, + "childtable_prefix": "stb00_", + "auto_create_table": "no", + "batch_create_tbl_num": 1000, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 100, + "childtable_limit": 0, + "childtable_offset":0, + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}] + }, + { + "name": "stb1", + "child_table_exists":"no", + "childtable_count": 100000, + "childtable_prefix": "stb01_", + "auto_create_table": "no", + "batch_create_tbl_num": 1000, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 200, + "childtable_limit": 0, + "childtable_offset":0, + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}] + }] + }] +} + diff --git a/tests/pytest/tsdb/insertDataDb1Replica2.json b/tests/pytest/tsdb/insertDataDb1Replica2.json new file mode 100644 index 0000000000..fec38bcdec --- /dev/null +++ b/tests/pytest/tsdb/insertDataDb1Replica2.json @@ -0,0 +1,87 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 10, + "num_of_records_per_req": 1000, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "db1", + "drop": "yes", + "replica": 2, + "days": 10, + "cache": 50, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"no", + "childtable_count": 1000, + "childtable_prefix": "stb00_", + "auto_create_table": "no", + "batch_create_tbl_num": 100, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 100, + "childtable_limit": 0, + "childtable_offset":0, + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}] + }, + { + "name": "stb1", + "child_table_exists":"no", + "childtable_count": 1000, + "childtable_prefix": "stb01_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 200, + "childtable_limit": 0, + "childtable_offset":0, + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}] + }] + }] +} + diff --git a/tests/pytest/tsdb/insertDataDb2.json b/tests/pytest/tsdb/insertDataDb2.json new file mode 100644 index 0000000000..89536418a2 --- /dev/null +++ b/tests/pytest/tsdb/insertDataDb2.json @@ -0,0 +1,86 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 0, + "num_of_records_per_req": 3000, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "db2", + "drop": "yes", + "replica": 1, + "days": 10, + "cache": 50, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"no", + "childtable_count": 200000, + "childtable_prefix": "stb0_", + "auto_create_table": "no", + "batch_create_tbl_num": 1000, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 2000, + "childtable_limit": 0, + "childtable_offset":0, + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}] + }, + { + "name": "stb1", + "child_table_exists":"no", + "childtable_count": 500000, + "childtable_prefix": "stb1_", + "auto_create_table": "no", + "batch_create_tbl_num": 1000, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 5, + "childtable_limit": 0, + "childtable_offset":0, + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}] + }] + }] +} diff --git a/tests/pytest/tsdb/insertDataDb2Newstab.json b/tests/pytest/tsdb/insertDataDb2Newstab.json new file mode 100644 index 0000000000..f9d0713385 --- /dev/null +++ b/tests/pytest/tsdb/insertDataDb2Newstab.json @@ -0,0 +1,86 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 0, + "num_of_records_per_req": 3000, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "db2", + "drop": "no", + "replica": 1, + "days": 10, + "cache": 50, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"no", + "childtable_count": 1, + "childtable_prefix": "stb0_", + "auto_create_table": "no", + "batch_create_tbl_num": 100, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 0, + "childtable_limit": -1, + "childtable_offset":0, + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}] + }, + { + "name": "stb1", + "child_table_exists":"yes", + "childtable_count": 1, + "childtable_prefix": "stb01_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 10, + "childtable_limit": -1, + "childtable_offset":0, + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-11-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}] + }] + }] +} diff --git a/tests/pytest/tsdb/insertDataDb2NewstabReplica2.json b/tests/pytest/tsdb/insertDataDb2NewstabReplica2.json new file mode 100644 index 0000000000..e052f2850f --- /dev/null +++ b/tests/pytest/tsdb/insertDataDb2NewstabReplica2.json @@ -0,0 +1,86 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 0, + "num_of_records_per_req": 3000, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "db2", + "drop": "no", + "replica": 2, + "days": 10, + "cache": 50, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"no", + "childtable_count": 1, + "childtable_prefix": "stb0_", + "auto_create_table": "no", + "batch_create_tbl_num": 100, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 0, + "childtable_limit": -1, + "childtable_offset":0, + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}] + }, + { + "name": "stb1", + "child_table_exists":"yes", + "childtable_count": 1, + "childtable_prefix": "stb01_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 10, + "childtable_limit": -1, + "childtable_offset":0, + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-11-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}] + }] + }] +} diff --git a/tests/pytest/tsdb/insertDataDb2Replica2.json b/tests/pytest/tsdb/insertDataDb2Replica2.json new file mode 100644 index 0000000000..121f70956a --- /dev/null +++ b/tests/pytest/tsdb/insertDataDb2Replica2.json @@ -0,0 +1,86 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 0, + "num_of_records_per_req": 3000, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "db2", + "drop": "yes", + "replica": 2, + "days": 10, + "cache": 50, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"no", + "childtable_count": 2000, + "childtable_prefix": "stb0_", + "auto_create_table": "no", + "batch_create_tbl_num": 100, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 2000, + "childtable_limit": 0, + "childtable_offset":0, + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}] + }, + { + "name": "stb1", + "child_table_exists":"no", + "childtable_count": 2, + "childtable_prefix": "stb1_", + "auto_create_table": "no", + "batch_create_tbl_num": 10, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 5, + "childtable_limit": 0, + "childtable_offset":0, + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}] + }] + }] +} diff --git a/tests/pytest/tsdb/sdbComp.py b/tests/pytest/tsdb/sdbComp.py new file mode 100644 index 0000000000..6ee7a09fdb --- /dev/null +++ b/tests/pytest/tsdb/sdbComp.py @@ -0,0 +1,121 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +from distutils.log import debug +import sys +import os +import taos +from util.log import * +from util.cases import * +from util.sql import * +from util.dnodes import * +import subprocess + + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor(), logSql) + + def getBuildPath(self): + global selfPath + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + def run(self): + + # set path para + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + + binPath = buildPath+ "/build/bin/" + testPath = selfPath+ "/../../../" + walFilePath = testPath + "/sim/dnode1/data/mnode_bak/wal/" + + #new db and insert data + tdSql.execute("drop database if exists db2") + os.system("%staosdemo -f tsdb/insertDataDb1.json -y " % binPath) + tdSql.execute("drop database if exists db1") + os.system("%staosdemo -f tsdb/insertDataDb2.json -y " % binPath) + tdSql.execute("drop table if exists db2.stb0") + os.system("%staosdemo -f wal/insertDataDb2Newstab.json -y " % binPath) + # query_pid1 = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1]) + # print(query_pid1) + tdSql.execute("use db2") + tdSql.execute("drop table if exists stb1_0") + tdSql.execute("drop table if exists stb1_1") + tdSql.execute("insert into stb0_0 values(1614218412000,8637,78.861045,'R','bf3')(1614218422000,8637,98.861045,'R','bf3')") + tdSql.execute("alter table db2.stb0 add column col4 int") + tdSql.execute("alter table db2.stb0 drop column col2") + tdSql.execute("alter table db2.stb0 add tag t3 int;") + tdSql.execute("alter table db2.stb0 drop tag t1") + tdSql.execute("create table if not exists stb2_0 (ts timestamp, col0 int, col1 float) ") + tdSql.execute("insert into stb2_0 values(1614218412000,8637,78.861045)") + tdSql.execute("alter table stb2_0 add column col2 binary(4)") + tdSql.execute("alter table stb2_0 drop column col1") + tdSql.execute("insert into stb2_0 values(1614218422000,8638,'R')") + + # stop taosd and compact wal file + tdDnodes.stop(1) + sleep(10) + # assert os.path.exists(walFilePath) , "%s is not generated, compact didn't take effect " % walFilePath + + # use new wal file to start taosd + tdDnodes.start(1) + sleep(5) + tdSql.execute("reset query cache") + query_pid2 = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1]) + print(query_pid2) + + # verify that the data is correct + tdSql.execute("use db2") + tdSql.query("select count (tbname) from stb0") + tdSql.checkData(0, 0, 1) + tdSql.query("select count (tbname) from stb1") + tdSql.checkRows(0) + tdSql.query("select count(*) from stb0_0") + tdSql.checkData(0, 0, 2) + tdSql.query("select count(*) from stb0") + tdSql.checkData(0, 0, 2) + tdSql.query("select count(*) from stb2_0") + tdSql.checkData(0, 0, 2) + + # delete useless file + testcaseFilename = os.path.split(__file__)[-1] + os.system("rm -rf ./insert_res.txt") + os.system("rm -rf wal/%s.sql" % testcaseFilename ) + + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/pytest/tsdb/sdbCompCluster.py b/tests/pytest/tsdb/sdbCompCluster.py new file mode 100644 index 0000000000..4fa84817ec --- /dev/null +++ b/tests/pytest/tsdb/sdbCompCluster.py @@ -0,0 +1,135 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import os +import sys +sys.path.insert(0, os.getcwd()) +from util.log import * +from util.sql import * +from util.dnodes import * +import taos +import threading + + +class TwoClients: + def initConnection(self): + self.host = "chenhaoran02" + self.user = "root" + self.password = "taosdata" + self.config = "/etc/taos/" + self.port =6030 + self.rowNum = 10 + self.ts = 1537146000000 + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + def run(self): + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + binPath = buildPath+ "/build/bin/" + walFilePath = "/var/lib/taos/mnode_bak/wal/" + + # new taos client + conn1 = taos.connect(host=self.host, user=self.user, password=self.password, config=self.config ) + print(conn1) + cur1 = conn1.cursor() + tdSql.init(cur1, True) + + # new db and insert data + os.system("rm -rf /var/lib/taos/mnode_bak/") + os.system("rm -rf /var/lib/taos/mnode_temp/") + tdSql.execute("drop database if exists db2") + os.system("%staosdemo -f wal/insertDataDb1.json -y " % binPath) + tdSql.execute("drop database if exists db1") + os.system("%staosdemo -f wal/insertDataDb2.json -y " % binPath) + tdSql.execute("drop table if exists db2.stb0") + os.system("%staosdemo -f wal/insertDataDb2Newstab.json -y " % binPath) + query_pid1 = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1]) + print(query_pid1) + tdSql.execute("use db2") + tdSql.execute("drop table if exists stb1_0") + tdSql.execute("drop table if exists stb1_1") + tdSql.execute("insert into stb0_0 values(1614218412000,8637,78.861045,'R','bf3')(1614218422000,8637,98.861045,'R','bf3')") + tdSql.execute("alter table db2.stb0 add column col4 int") + tdSql.execute("alter table db2.stb0 drop column col2") + tdSql.execute("alter table db2.stb0 add tag t3 int") + tdSql.execute("alter table db2.stb0 drop tag t1") + tdSql.execute("create table if not exists stb2_0 (ts timestamp, col0 int, col1 float) ") + tdSql.execute("insert into stb2_0 values(1614218412000,8637,78.861045)") + tdSql.execute("alter table stb2_0 add column col2 binary(4)") + tdSql.execute("alter table stb2_0 drop column col1") + tdSql.execute("insert into stb2_0 values(1614218422000,8638,'R')") + + # stop taosd and compact wal file + os.system("ps -ef |grep taosd |grep -v 'grep' |awk '{print $2}'|xargs kill -2") + sleep(10) + os.system("nohup taosd --compact-mnode-wal -c /etc/taos & ") + sleep(10) + os.system("nohup /usr/bin/taosd > /dev/null 2>&1 &") + sleep(4) + tdSql.execute("reset query cache") + query_pid2 = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1]) + print(query_pid2) + assert os.path.exists(walFilePath) , "%s is not generated " % walFilePath + + # new taos connecting to server + conn2 = taos.connect(host=self.host, user=self.user, password=self.password, config=self.config ) + print(conn2) + cur2 = conn2.cursor() + tdSql.init(cur2, True) + + # use new wal file to start up tasod + tdSql.query("show databases") + for i in range(tdSql.queryRows): + if tdSql.queryResult[i][0]=="db2": + assert tdSql.queryResult[i][4]==1 , "replica is wrong" + tdSql.execute("use db2") + tdSql.query("select count (tbname) from stb0") + tdSql.checkData(0, 0, 1) + tdSql.query("select count (tbname) from stb1") + tdSql.checkRows(0) + tdSql.query("select count(*) from stb0_0") + tdSql.checkData(0, 0, 2) + tdSql.query("select count(*) from stb0") + tdSql.checkData(0, 0, 2) + tdSql.query("select count(*) from stb2_0") + tdSql.checkData(0, 0, 2) + tdSql.query("select * from stb2_0") + tdSql.checkData(1, 2, 'R') + + # delete useless file + testcaseFilename = os.path.split(__file__)[-1] + os.system("rm -rf ./insert_res.txt") + os.system("rm -rf wal/%s.sql" % testcaseFilename ) + +clients = TwoClients() +clients.initConnection() +# clients.getBuildPath() +clients.run() \ No newline at end of file diff --git a/tests/pytest/tsdb/sdbCompClusterReplica2.py b/tests/pytest/tsdb/sdbCompClusterReplica2.py new file mode 100644 index 0000000000..117da8ca2f --- /dev/null +++ b/tests/pytest/tsdb/sdbCompClusterReplica2.py @@ -0,0 +1,136 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import os +import sys +sys.path.insert(0, os.getcwd()) +from util.log import * +from util.sql import * +from util.dnodes import * +import taos +import threading + + +class TwoClients: + def initConnection(self): + self.host = "chenhaoran02" + self.user = "root" + self.password = "taosdata" + self.config = "/etc/taos/" + self.port =6030 + self.rowNum = 10 + self.ts = 1537146000000 + + def getBuildPath(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + + for root, dirs, files in os.walk(projPath): + if ("taosd" in files): + rootRealPath = os.path.dirname(os.path.realpath(root)) + if ("packaging" not in rootRealPath): + buildPath = root[:len(root)-len("/build/bin")] + break + return buildPath + + def run(self): + buildPath = self.getBuildPath() + if (buildPath == ""): + tdLog.exit("taosd not found!") + else: + tdLog.info("taosd found in %s" % buildPath) + binPath = buildPath+ "/build/bin/" + walFilePath = "/var/lib/taos/mnode_bak/wal/" + + # new taos client + conn1 = taos.connect(host=self.host, user=self.user, password=self.password, config=self.config ) + print(conn1) + cur1 = conn1.cursor() + tdSql.init(cur1, True) + + # new db and insert data + os.system("rm -rf /var/lib/taos/mnode_bak/") + os.system("rm -rf /var/lib/taos/mnode_temp/") + tdSql.execute("drop database if exists db2") + os.system("%staosdemo -f wal/insertDataDb1Replica2.json -y " % binPath) + tdSql.execute("drop database if exists db1") + os.system("%staosdemo -f wal/insertDataDb2Replica2.json -y " % binPath) + tdSql.execute("drop table if exists db2.stb0") + os.system("%staosdemo -f wal/insertDataDb2NewstabReplica2.json -y " % binPath) + query_pid1 = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1]) + print(query_pid1) + tdSql.execute("use db2") + tdSql.execute("drop table if exists stb1_0") + tdSql.execute("drop table if exists stb1_1") + tdSql.execute("insert into stb0_0 values(1614218412000,8637,78.861045,'R','bf3')(1614218422000,8637,98.861045,'R','bf3')") + tdSql.execute("alter table db2.stb0 add column col4 int") + tdSql.execute("alter table db2.stb0 drop column col2") + tdSql.execute("alter table db2.stb0 add tag t3 int") + tdSql.execute("alter table db2.stb0 drop tag t1") + tdSql.execute("create table if not exists stb2_0 (ts timestamp, col0 int, col1 float) ") + tdSql.execute("insert into stb2_0 values(1614218412000,8637,78.861045)") + tdSql.execute("alter table stb2_0 add column col2 binary(4)") + tdSql.execute("alter table stb2_0 drop column col1") + tdSql.execute("insert into stb2_0 values(1614218422000,8638,'R')") + + + # stop taosd and compact wal file + os.system("ps -ef |grep taosd |grep -v 'grep' |awk '{print $2}'|xargs kill -2") + sleep(10) + os.system("nohup taosd --compact-mnode-wal -c /etc/taos & ") + sleep(10) + os.system("nohup /usr/bin/taosd > /dev/null 2>&1 &") + sleep(4) + tdSql.execute("reset query cache") + query_pid2 = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1]) + print(query_pid2) + assert os.path.exists(walFilePath) , "%s is not generated " % walFilePath + + # new taos connecting to server + conn2 = taos.connect(host=self.host, user=self.user, password=self.password, config=self.config ) + print(conn2) + cur2 = conn2.cursor() + tdSql.init(cur2, True) + + # use new wal file to start up tasod + tdSql.query("show databases") + for i in range(tdSql.queryRows): + if tdSql.queryResult[i][0]=="db2": + assert tdSql.queryResult[i][4]==2 , "replica is wrong" + tdSql.execute("use db2") + tdSql.query("select count (tbname) from stb0") + tdSql.checkData(0, 0, 1) + tdSql.query("select count (tbname) from stb1") + tdSql.checkRows(0) + tdSql.query("select count(*) from stb0_0") + tdSql.checkData(0, 0, 2) + tdSql.query("select count(*) from stb0") + tdSql.checkData(0, 0, 2) + tdSql.query("select count(*) from stb2_0") + tdSql.checkData(0, 0, 2) + tdSql.query("select * from stb2_0") + tdSql.checkData(1, 2, 'R') + + # delete useless file + testcaseFilename = os.path.split(__file__)[-1] + os.system("rm -rf ./insert_res.txt") + os.system("rm -rf wal/%s.sql" % testcaseFilename ) + +clients = TwoClients() +clients.initConnection() +# clients.getBuildPath() +clients.run() \ No newline at end of file From 2d0bca4290c9329508cef5bdefcbd99461f2cb59 Mon Sep 17 00:00:00 2001 From: tomchon Date: Wed, 14 Jul 2021 18:36:59 +0800 Subject: [PATCH 10/22] [TD-4840]add testcase of compacting sdb --- tests/pytest/tsdb/insertDataDb2.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pytest/tsdb/insertDataDb2.json b/tests/pytest/tsdb/insertDataDb2.json index 89536418a2..4cde724a82 100644 --- a/tests/pytest/tsdb/insertDataDb2.json +++ b/tests/pytest/tsdb/insertDataDb2.json @@ -38,7 +38,7 @@ "childtable_count": 200000, "childtable_prefix": "stb0_", "auto_create_table": "no", - "batch_create_tbl_num": 1000, + "batch_create_tbl_num": 100, "data_source": "rand", "insert_mode": "taosc", "insert_rows": 2000, From 543b17cb84d7689be334b8173b9a06f10ad99fbd Mon Sep 17 00:00:00 2001 From: lichuang Date: Thu, 15 Jul 2021 10:17:58 +0800 Subject: [PATCH 11/22] [TD-4352]fix tsdb meta compact bug --- src/tsdb/inc/tsdbFile.h | 1 - src/tsdb/src/tsdbCommit.c | 16 +++++++++++++++- src/tsdb/src/tsdbFile.c | 20 ++------------------ 3 files changed, 17 insertions(+), 20 deletions(-) diff --git a/src/tsdb/inc/tsdbFile.h b/src/tsdb/inc/tsdbFile.h index 9ebf1253cb..a2be99e279 100644 --- a/src/tsdb/inc/tsdbFile.h +++ b/src/tsdb/inc/tsdbFile.h @@ -57,7 +57,6 @@ typedef struct { } SMFile; void tsdbInitMFile(SMFile* pMFile, SDiskID did, int vid, uint32_t ver, bool tmp); -void tsdbRenameOrDeleleTempMetaFile(SMFile* pMFile, SDiskID did, int vid, uint32_t ver, int code); void tsdbInitMFileEx(SMFile* pMFile, const SMFile* pOMFile); int tsdbEncodeSMFile(void** buf, SMFile* pMFile); void* tsdbDecodeSMFile(void* buf, SMFile* pMFile); diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index df0f8b4d78..98aef13948 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -460,6 +460,11 @@ static int tsdbCompactMetaFile(STsdbRepo *pRepo, STsdbFS *pfs, SMFile *pMFile) { return 0; } + if (tsdbOpenMFile(pMFile, O_RDONLY) < 0) { + tsdbError("open meta file %s compact fail", pMFile->f.rname); + return -1; + } + tsdbInfo("begin compact tsdb meta file, nDels:%" PRId64 ",nRecords:%" PRId64 ",tombSize:%" PRId64 ",size:%" PRId64, pMFile->info.nDels,pMFile->info.nRecords,pMFile->info.tombSize,pMFile->info.size); @@ -530,10 +535,19 @@ static int tsdbCompactMetaFile(STsdbRepo *pRepo, STsdbFS *pfs, SMFile *pMFile) { _err: TSDB_FILE_FSYNC(&mf); tsdbCloseMFile(&mf); + tsdbCloseMFile(pMFile); - tsdbRenameOrDeleleTempMetaFile(&mf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo)), code); if (code == 0) { + // rename meta.tmp -> meta + taosRename(mf.f.aname,pMFile->f.aname); + tstrncpy(mf.f.aname, pMFile->f.aname, TSDB_FILENAME_LEN); + tstrncpy(mf.f.rname, pMFile->f.rname, TSDB_FILENAME_LEN); + // update current meta file info + pfs->nstatus->pmf = NULL; tsdbUpdateMFile(pfs, &mf); + } else { + // remove meta.tmp file + remove(mf.f.aname); } tfree(pBuf); diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index b1ff79bfef..cc6fbb632f 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -20,8 +20,8 @@ static const char *TSDB_FNAME_SUFFIX[] = { "data", // TSDB_FILE_DATA "last", // TSDB_FILE_LAST "", // TSDB_FILE_MAX - "meta" // TSDB_FILE_META - "meta.tmp" // TSDB_FILE_META_TMP + "meta", // TSDB_FILE_META + "meta.tmp", // TSDB_FILE_META_TMP }; static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, char *fname); @@ -43,22 +43,6 @@ void tsdbInitMFile(SMFile *pMFile, SDiskID did, int vid, uint32_t ver, bool tmp) tfsInitFile(TSDB_FILE_F(pMFile), did.level, did.id, fname); } -void tsdbRenameOrDeleleTempMetaFile(SMFile* pMFile, SDiskID did, int vid, uint32_t ver, int code) { - char mfname[TSDB_FILENAME_LEN] = {'\0'}; - char tfname[TSDB_FILENAME_LEN] = {'\0'}; - - tsdbGetFilename(vid, 0, ver, TSDB_FILE_META_TMP, tfname); - - if (code != 0) { - remove(tfname); - return; - } - - tsdbGetFilename(vid, 0, ver, TSDB_FILE_META, mfname); - - (void)taosRename(tfname, mfname); -} - void tsdbInitMFileEx(SMFile *pMFile, const SMFile *pOMFile) { *pMFile = *pOMFile; TSDB_FILE_SET_CLOSED(pMFile); From 674992e016e9e2149ad0001d13d4c5f8868f5087 Mon Sep 17 00:00:00 2001 From: lichuang Date: Tue, 27 Jul 2021 11:26:40 +0800 Subject: [PATCH 12/22] [TD-4352]fix code bug --- src/tsdb/src/tsdbCommit.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index 98aef13948..289bb518e9 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -499,13 +499,13 @@ static int tsdbCompactMetaFile(STsdbRepo *pRepo, STsdbFS *pfs, SMFile *pMFile) { if (tsdbSeekMFile(pMFile, pRecord->offset + sizeof(SKVRecord), SEEK_SET) < 0) { tsdbError("vgId:%d failed to seek file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), tstrerror(terrno)); - break; + goto _err; } if (pRecord->size > maxBufSize) { maxBufSize = pRecord->size; void* tmp = realloc(pBuf, (size_t)maxBufSize); if (tmp == NULL) { - break; + goto _err; } pBuf = tmp; } @@ -513,19 +513,19 @@ static int tsdbCompactMetaFile(STsdbRepo *pRepo, STsdbFS *pfs, SMFile *pMFile) { if (nread < 0) { tsdbError("vgId:%d failed to read file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), tstrerror(terrno)); - break; + goto _err; } if (nread < pRecord->size) { tsdbError("vgId:%d failed to read file %s since file corrupted, expected read:%" PRId64 " actual read:%d", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), pRecord->size, nread); - break; + goto _err; } if (tsdbUpdateMetaRecord(pfs, &mf, pRecord->uid, pBuf, (int)pRecord->size, false) < 0) { tsdbError("vgId:%d failed to update META record, uid %" PRIu64 " since %s", REPO_ID(pRepo), pRecord->uid, tstrerror(terrno)); - break; + goto _err; } pRecord = taosHashIterate(pfs->metaCache, pRecord); From 5602fef8f62dfa059dee00c9cda7751e4d8a2039 Mon Sep 17 00:00:00 2001 From: tomchon Date: Tue, 27 Jul 2021 14:59:32 +0800 Subject: [PATCH 13/22] [TD-4840]: add testcase of compressing tsdb meta data --- tests/pytest/tsdb/insertDataDb1.json | 6 +- tests/pytest/tsdb/insertDataDb2.json | 6 +- tests/pytest/tsdb/{sdbComp.py => tsdbComp.py} | 62 ++++++++++++++++--- 3 files changed, 60 insertions(+), 14 deletions(-) rename tests/pytest/tsdb/{sdbComp.py => tsdbComp.py} (64%) diff --git a/tests/pytest/tsdb/insertDataDb1.json b/tests/pytest/tsdb/insertDataDb1.json index f01cc35a1b..60c6def92c 100644 --- a/tests/pytest/tsdb/insertDataDb1.json +++ b/tests/pytest/tsdb/insertDataDb1.json @@ -35,13 +35,13 @@ "super_tables": [{ "name": "stb0", "child_table_exists":"no", - "childtable_count": 100000, + "childtable_count": 1000, "childtable_prefix": "stb00_", "auto_create_table": "no", "batch_create_tbl_num": 1000, "data_source": "rand", "insert_mode": "taosc", - "insert_rows": 100, + "insert_rows": 1000, "childtable_limit": 0, "childtable_offset":0, "interlace_rows": 0, @@ -60,7 +60,7 @@ { "name": "stb1", "child_table_exists":"no", - "childtable_count": 100000, + "childtable_count": 10000, "childtable_prefix": "stb01_", "auto_create_table": "no", "batch_create_tbl_num": 1000, diff --git a/tests/pytest/tsdb/insertDataDb2.json b/tests/pytest/tsdb/insertDataDb2.json index 4cde724a82..ead5f19716 100644 --- a/tests/pytest/tsdb/insertDataDb2.json +++ b/tests/pytest/tsdb/insertDataDb2.json @@ -38,10 +38,10 @@ "childtable_count": 200000, "childtable_prefix": "stb0_", "auto_create_table": "no", - "batch_create_tbl_num": 100, + "batch_create_tbl_num": 1000, "data_source": "rand", "insert_mode": "taosc", - "insert_rows": 2000, + "insert_rows": 0, "childtable_limit": 0, "childtable_offset":0, "interlace_rows": 0, @@ -60,7 +60,7 @@ { "name": "stb1", "child_table_exists":"no", - "childtable_count": 500000, + "childtable_count": 2, "childtable_prefix": "stb1_", "auto_create_table": "no", "batch_create_tbl_num": 1000, diff --git a/tests/pytest/tsdb/sdbComp.py b/tests/pytest/tsdb/tsdbComp.py similarity index 64% rename from tests/pytest/tsdb/sdbComp.py rename to tests/pytest/tsdb/tsdbComp.py index 6ee7a09fdb..b2ea36d239 100644 --- a/tests/pytest/tsdb/sdbComp.py +++ b/tests/pytest/tsdb/tsdbComp.py @@ -20,13 +20,13 @@ from util.cases import * from util.sql import * from util.dnodes import * import subprocess - +from random import choice class TDTestCase: def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor(), logSql) - + def getBuildPath(self): global selfPath selfPath = os.path.dirname(os.path.realpath(__file__)) @@ -63,7 +63,7 @@ class TDTestCase: tdSql.execute("drop database if exists db1") os.system("%staosdemo -f tsdb/insertDataDb2.json -y " % binPath) tdSql.execute("drop table if exists db2.stb0") - os.system("%staosdemo -f wal/insertDataDb2Newstab.json -y " % binPath) + os.system("%staosdemo -f tsdb/insertDataDb2Newstab.json -y " % binPath) # query_pid1 = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1]) # print(query_pid1) tdSql.execute("use db2") @@ -80,12 +80,54 @@ class TDTestCase: tdSql.execute("alter table stb2_0 drop column col1") tdSql.execute("insert into stb2_0 values(1614218422000,8638,'R')") - # stop taosd and compact wal file + # create db utest + + + dataType= [ "tinyint", "smallint", "int", "bigint", "float", "double", "bool", " binary(20)", "nchar(20)", "tinyint unsigned", "smallint unsigned", "int unsigned", "bigint unsigned"] + + tdSql.execute("drop database if exists utest") + tdSql.execute("create database utest keep 3650") + tdSql.execute("use utest") + tdSql.execute('''create table test(ts timestamp, col0 tinyint, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20), col10 tinyint unsigned, col11 smallint unsigned, col12 int unsigned, col13 bigint unsigned) tags(loc nchar(200), tag1 int)''') + + # rowNum1 = 13 + # for i in range(rowNum1): + # columnName= "col" + str(i+1) + # tdSql.execute("alter table test drop column %s ;" % columnName ) + + rowNum2= 988 + for i in range(rowNum2): + tdSql.execute("alter table test add column col%d %s ;" %( i+14, choice(dataType)) ) + + rowNum3= 988 + for i in range(rowNum3): + tdSql.execute("alter table test drop column col%d ;" %( i+14) ) + + + self.rowNum = 1 + self.rowNum2 = 100 + self.rowNum3 = 20 + self.ts = 1537146000000 + self.ts1 = 1537146000000000 + self.ts2 = 1597146000000 + # tdSql.execute("create table test1 using test tags('beijing', 10)") + # tdSql.execute("create table test2 using test tags('tianjing', 20)") + # tdSql.execute("create table test3 using test tags('shanghai', 20)") + + for j in range(self.rowNum2): + tdSql.execute("create table test%d using test tags('beijing%d', 10)" % (j,j) ) + for i in range(self.rowNum): + tdSql.execute("insert into test%d values(%d, %d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (j, self.ts + i*1000, i + 1, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) + + for j in range(self.rowNum2): + tdSql.execute("drop table if exists test%d" % (j+1)) + + + # stop taosd and restart taosd tdDnodes.stop(1) sleep(10) - # assert os.path.exists(walFilePath) , "%s is not generated, compact didn't take effect " % walFilePath - - # use new wal file to start taosd tdDnodes.start(1) sleep(5) tdSql.execute("reset query cache") @@ -104,7 +146,11 @@ class TDTestCase: tdSql.checkData(0, 0, 2) tdSql.query("select count(*) from stb2_0") tdSql.checkData(0, 0, 2) - + + tdSql.execute("use utest") + tdSql.query("select count (tbname) from test") + tdSql.checkData(0, 0, 1) + # delete useless file testcaseFilename = os.path.split(__file__)[-1] os.system("rm -rf ./insert_res.txt") From 705d6e3eb3d2a70d097d33a1c830cf2d6c4ee3bd Mon Sep 17 00:00:00 2001 From: tomchon Date: Wed, 28 Jul 2021 16:11:09 +0800 Subject: [PATCH 14/22] [TD-4840]: update testcase of tsdb meta compressed --- tests/pytest/tsdb/tsdbComp.py | 10 +-- .../{sdbCompCluster.py => tsdbCompCluster.py} | 63 +++++++++----- ...Replica2.py => tsdbCompClusterReplica2.py} | 84 +++++++++++++------ 3 files changed, 105 insertions(+), 52 deletions(-) rename tests/pytest/tsdb/{sdbCompCluster.py => tsdbCompCluster.py} (65%) rename tests/pytest/tsdb/{sdbCompClusterReplica2.py => tsdbCompClusterReplica2.py} (57%) diff --git a/tests/pytest/tsdb/tsdbComp.py b/tests/pytest/tsdb/tsdbComp.py index b2ea36d239..a46b3280fb 100644 --- a/tests/pytest/tsdb/tsdbComp.py +++ b/tests/pytest/tsdb/tsdbComp.py @@ -64,8 +64,7 @@ class TDTestCase: os.system("%staosdemo -f tsdb/insertDataDb2.json -y " % binPath) tdSql.execute("drop table if exists db2.stb0") os.system("%staosdemo -f tsdb/insertDataDb2Newstab.json -y " % binPath) - # query_pid1 = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1]) - # print(query_pid1) + tdSql.execute("use db2") tdSql.execute("drop table if exists stb1_0") tdSql.execute("drop table if exists stb1_1") @@ -109,11 +108,6 @@ class TDTestCase: self.rowNum2 = 100 self.rowNum3 = 20 self.ts = 1537146000000 - self.ts1 = 1537146000000000 - self.ts2 = 1597146000000 - # tdSql.execute("create table test1 using test tags('beijing', 10)") - # tdSql.execute("create table test2 using test tags('tianjing', 20)") - # tdSql.execute("create table test3 using test tags('shanghai', 20)") for j in range(self.rowNum2): tdSql.execute("create table test%d using test tags('beijing%d', 10)" % (j,j) ) @@ -154,7 +148,7 @@ class TDTestCase: # delete useless file testcaseFilename = os.path.split(__file__)[-1] os.system("rm -rf ./insert_res.txt") - os.system("rm -rf wal/%s.sql" % testcaseFilename ) + os.system("rm -rf tsdb/%s.sql" % testcaseFilename ) diff --git a/tests/pytest/tsdb/sdbCompCluster.py b/tests/pytest/tsdb/tsdbCompCluster.py similarity index 65% rename from tests/pytest/tsdb/sdbCompCluster.py rename to tests/pytest/tsdb/tsdbCompCluster.py index 4fa84817ec..39c8b563c5 100644 --- a/tests/pytest/tsdb/sdbCompCluster.py +++ b/tests/pytest/tsdb/tsdbCompCluster.py @@ -19,6 +19,8 @@ from util.sql import * from util.dnodes import * import taos import threading +import subprocess +from random import choice class TwoClients: @@ -62,17 +64,15 @@ class TwoClients: cur1 = conn1.cursor() tdSql.init(cur1, True) - # new db and insert data - os.system("rm -rf /var/lib/taos/mnode_bak/") - os.system("rm -rf /var/lib/taos/mnode_temp/") + # new db ,new super tables , child tables, and insert data tdSql.execute("drop database if exists db2") - os.system("%staosdemo -f wal/insertDataDb1.json -y " % binPath) + os.system("%staosdemo -f tsdb/insertDataDb1.json -y " % binPath) tdSql.execute("drop database if exists db1") - os.system("%staosdemo -f wal/insertDataDb2.json -y " % binPath) + os.system("%staosdemo -f tsdb/insertDataDb2.json -y " % binPath) tdSql.execute("drop table if exists db2.stb0") - os.system("%staosdemo -f wal/insertDataDb2Newstab.json -y " % binPath) - query_pid1 = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1]) - print(query_pid1) + os.system("%staosdemo -f tsdb/insertDataDb2Newstab.json -y " % binPath) + + # new general tables and modify general tables; tdSql.execute("use db2") tdSql.execute("drop table if exists stb1_0") tdSql.execute("drop table if exists stb1_1") @@ -87,17 +87,41 @@ class TwoClients: tdSql.execute("alter table stb2_0 drop column col1") tdSql.execute("insert into stb2_0 values(1614218422000,8638,'R')") - # stop taosd and compact wal file + # create db utest and modify super tables; + dataType= [ "tinyint", "smallint", "int", "bigint", "float", "double", "bool", " binary(20)", "nchar(20)", "tinyint unsigned", "smallint unsigned", "int unsigned", "bigint unsigned"] + tdSql.execute("drop database if exists utest") + tdSql.execute("create database utest keep 3650") + tdSql.execute("use utest") + tdSql.execute('''create table test(ts timestamp, col0 tinyint, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20), col10 tinyint unsigned, col11 smallint unsigned, col12 int unsigned, col13 bigint unsigned) tags(loc nchar(200), tag1 int)''') + rowNum2= 988 + for i in range(rowNum2): + tdSql.execute("alter table test add column col%d %s ;" %( i+14, choice(dataType)) ) + rowNum3= 988 + for i in range(rowNum3): + tdSql.execute("alter table test drop column col%d ;" %( i+14) ) + + self.rowNum = 1 + self.rowNum2 = 100 + self.ts = 1537146000000 + for j in range(self.rowNum2): + tdSql.execute("create table test%d using test tags('beijing%d', 10)" % (j,j) ) + for i in range(self.rowNum): + tdSql.execute("insert into test%d values(%d, %d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (j, self.ts + i*1000, i + 1, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) + # delete child tables; + for j in range(self.rowNum2): + tdSql.execute("drop table if exists test%d" % (j+1)) + + #restart taosd os.system("ps -ef |grep taosd |grep -v 'grep' |awk '{print $2}'|xargs kill -2") - sleep(10) - os.system("nohup taosd --compact-mnode-wal -c /etc/taos & ") - sleep(10) + sleep(20) + print("123") os.system("nohup /usr/bin/taosd > /dev/null 2>&1 &") sleep(4) tdSql.execute("reset query cache") query_pid2 = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1]) print(query_pid2) - assert os.path.exists(walFilePath) , "%s is not generated " % walFilePath # new taos connecting to server conn2 = taos.connect(host=self.host, user=self.user, password=self.password, config=self.config ) @@ -105,11 +129,9 @@ class TwoClients: cur2 = conn2.cursor() tdSql.init(cur2, True) - # use new wal file to start up tasod + + # check data correct tdSql.query("show databases") - for i in range(tdSql.queryRows): - if tdSql.queryResult[i][0]=="db2": - assert tdSql.queryResult[i][4]==1 , "replica is wrong" tdSql.execute("use db2") tdSql.query("select count (tbname) from stb0") tdSql.checkData(0, 0, 1) @@ -123,11 +145,14 @@ class TwoClients: tdSql.checkData(0, 0, 2) tdSql.query("select * from stb2_0") tdSql.checkData(1, 2, 'R') - + tdSql.execute("use utest") + tdSql.query("select count (tbname) from test") + tdSql.checkData(0, 0, 1) + # delete useless file testcaseFilename = os.path.split(__file__)[-1] os.system("rm -rf ./insert_res.txt") - os.system("rm -rf wal/%s.sql" % testcaseFilename ) + os.system("rm -rf tsdb/%s.sql" % testcaseFilename ) clients = TwoClients() clients.initConnection() diff --git a/tests/pytest/tsdb/sdbCompClusterReplica2.py b/tests/pytest/tsdb/tsdbCompClusterReplica2.py similarity index 57% rename from tests/pytest/tsdb/sdbCompClusterReplica2.py rename to tests/pytest/tsdb/tsdbCompClusterReplica2.py index 117da8ca2f..9b43b7ce80 100644 --- a/tests/pytest/tsdb/sdbCompClusterReplica2.py +++ b/tests/pytest/tsdb/tsdbCompClusterReplica2.py @@ -19,7 +19,8 @@ from util.sql import * from util.dnodes import * import taos import threading - +import subprocess +from random import choice class TwoClients: def initConnection(self): @@ -62,17 +63,15 @@ class TwoClients: cur1 = conn1.cursor() tdSql.init(cur1, True) - # new db and insert data - os.system("rm -rf /var/lib/taos/mnode_bak/") - os.system("rm -rf /var/lib/taos/mnode_temp/") + # new db ,new super tables , child tables, and insert data tdSql.execute("drop database if exists db2") - os.system("%staosdemo -f wal/insertDataDb1Replica2.json -y " % binPath) + os.system("%staosdemo -f tsdb/insertDataDb1Replica2.json -y " % binPath) tdSql.execute("drop database if exists db1") - os.system("%staosdemo -f wal/insertDataDb2Replica2.json -y " % binPath) + os.system("%staosdemo -f tsdb/insertDataDb2Replica2.json -y " % binPath) tdSql.execute("drop table if exists db2.stb0") - os.system("%staosdemo -f wal/insertDataDb2NewstabReplica2.json -y " % binPath) - query_pid1 = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1]) - print(query_pid1) + os.system("%staosdemo -f tsdb/insertDataDb2NewstabReplica2.json -y " % binPath) + + # new general tables and modify general tables; tdSql.execute("use db2") tdSql.execute("drop table if exists stb1_0") tdSql.execute("drop table if exists stb1_1") @@ -86,19 +85,53 @@ class TwoClients: tdSql.execute("alter table stb2_0 add column col2 binary(4)") tdSql.execute("alter table stb2_0 drop column col1") tdSql.execute("insert into stb2_0 values(1614218422000,8638,'R')") - - # stop taosd and compact wal file - os.system("ps -ef |grep taosd |grep -v 'grep' |awk '{print $2}'|xargs kill -2") - sleep(10) - os.system("nohup taosd --compact-mnode-wal -c /etc/taos & ") - sleep(10) + + # create db utest replica 2 and modify super tables; + dataType= [ "tinyint", "smallint", "int", "bigint", "float", "double", "bool", " binary(20)", "nchar(20)", "tinyint unsigned", "smallint unsigned", "int unsigned", "bigint unsigned"] + tdSql.execute("drop database if exists utest") + tdSql.execute("create database utest keep 3650 replica 2 ") + tdSql.execute("use utest") + tdSql.execute('''create table test(ts timestamp, col0 tinyint, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + col7 bool, col8 binary(20), col9 nchar(20), col10 tinyint unsigned, col11 smallint unsigned, col12 int unsigned, col13 bigint unsigned) tags(loc nchar(200), tag1 int)''') + rowNum2= 988 + for i in range(rowNum2): + tdSql.execute("alter table test add column col%d %s ;" %( i+14, choice(dataType)) ) + rowNum3= 988 + for i in range(rowNum3): + tdSql.execute("alter table test drop column col%d ;" %( i+14) ) + self.rowNum = 1 + self.rowNum2 = 100 + self.ts = 1537146000000 + for j in range(self.rowNum2): + tdSql.execute("create table test%d using test tags('beijing%d', 10)" % (j,j) ) + for i in range(self.rowNum): + tdSql.execute("insert into test%d values(%d, %d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + % (j, self.ts + i*1000, i + 1, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) + # delete child tables; + for j in range(self.rowNum2): + tdSql.execute("drop table if exists test%d" % (j+1)) + + # drop dnodes and restart taosd; + sleep(3) + tdSql.execute(" drop dnode 'chenhaoran02:6030'; ") + sleep(20) + os.system("rm -rf /var/lib/taos/*") + print("clear dnode chenhaoran02'data files") os.system("nohup /usr/bin/taosd > /dev/null 2>&1 &") - sleep(4) - tdSql.execute("reset query cache") - query_pid2 = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1]) - print(query_pid2) - assert os.path.exists(walFilePath) , "%s is not generated " % walFilePath + print("start taosd") + sleep(10) + tdSql.execute("reset query cache ;") + tdSql.execute("create dnode chenhaoran02 ;") + + # # + # os.system("ps -ef |grep taosd |grep -v 'grep' |awk '{print $2}'|xargs kill -2") + # sleep(20) + # os.system("nohup /usr/bin/taosd > /dev/null 2>&1 &") + # sleep(4) + # tdSql.execute("reset query cache") + # query_pid2 = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1]) + # print(query_pid2) # new taos connecting to server conn2 = taos.connect(host=self.host, user=self.user, password=self.password, config=self.config ) @@ -106,11 +139,8 @@ class TwoClients: cur2 = conn2.cursor() tdSql.init(cur2, True) - # use new wal file to start up tasod + # check data correct tdSql.query("show databases") - for i in range(tdSql.queryRows): - if tdSql.queryResult[i][0]=="db2": - assert tdSql.queryResult[i][4]==2 , "replica is wrong" tdSql.execute("use db2") tdSql.query("select count (tbname) from stb0") tdSql.checkData(0, 0, 1) @@ -125,10 +155,14 @@ class TwoClients: tdSql.query("select * from stb2_0") tdSql.checkData(1, 2, 'R') + tdSql.execute("use utest") + tdSql.query("select count (tbname) from test") + tdSql.checkData(0, 0, 1) + # delete useless file testcaseFilename = os.path.split(__file__)[-1] os.system("rm -rf ./insert_res.txt") - os.system("rm -rf wal/%s.sql" % testcaseFilename ) + os.system("rm -rf tsdb/%s.sql" % testcaseFilename ) clients = TwoClients() clients.initConnection() From 3fa6787382082cbb37227a5abf5c86c01d33aa49 Mon Sep 17 00:00:00 2001 From: tomchon Date: Wed, 28 Jul 2021 18:30:00 +0800 Subject: [PATCH 15/22] [TD-4840]: update testcase of tsdb meta compressed --- tests/pytest/tsdb/tsdbComp.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/pytest/tsdb/tsdbComp.py b/tests/pytest/tsdb/tsdbComp.py index a46b3280fb..3563655efe 100644 --- a/tests/pytest/tsdb/tsdbComp.py +++ b/tests/pytest/tsdb/tsdbComp.py @@ -69,14 +69,14 @@ class TDTestCase: tdSql.execute("drop table if exists stb1_0") tdSql.execute("drop table if exists stb1_1") tdSql.execute("insert into stb0_0 values(1614218412000,8637,78.861045,'R','bf3')(1614218422000,8637,98.861045,'R','bf3')") - tdSql.execute("alter table db2.stb0 add column col4 int") - tdSql.execute("alter table db2.stb0 drop column col2") + tdSql.execute("alter table db2.stb0 add column c4 int") + tdSql.execute("alter table db2.stb0 drop column c2") tdSql.execute("alter table db2.stb0 add tag t3 int;") tdSql.execute("alter table db2.stb0 drop tag t1") - tdSql.execute("create table if not exists stb2_0 (ts timestamp, col0 int, col1 float) ") + tdSql.execute("create table if not exists stb2_0 (ts timestamp, c0 int, c1 float) ") tdSql.execute("insert into stb2_0 values(1614218412000,8637,78.861045)") - tdSql.execute("alter table stb2_0 add column col2 binary(4)") - tdSql.execute("alter table stb2_0 drop column col1") + tdSql.execute("alter table stb2_0 add column c2 binary(4)") + tdSql.execute("alter table stb2_0 drop column c1") tdSql.execute("insert into stb2_0 values(1614218422000,8638,'R')") # create db utest From 7346cdeba5c1880f90132a710eb878064b072741 Mon Sep 17 00:00:00 2001 From: tomchon Date: Thu, 5 Aug 2021 18:22:50 +0800 Subject: [PATCH 16/22] [TD-4840]:modify testcase of tsdb meta compressed --- tests/pytest/tsdb/tsdbCompCluster.py | 10 +++++----- tests/pytest/tsdb/tsdbCompClusterReplica2.py | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/pytest/tsdb/tsdbCompCluster.py b/tests/pytest/tsdb/tsdbCompCluster.py index 39c8b563c5..3df4c9a9d4 100644 --- a/tests/pytest/tsdb/tsdbCompCluster.py +++ b/tests/pytest/tsdb/tsdbCompCluster.py @@ -77,14 +77,14 @@ class TwoClients: tdSql.execute("drop table if exists stb1_0") tdSql.execute("drop table if exists stb1_1") tdSql.execute("insert into stb0_0 values(1614218412000,8637,78.861045,'R','bf3')(1614218422000,8637,98.861045,'R','bf3')") - tdSql.execute("alter table db2.stb0 add column col4 int") - tdSql.execute("alter table db2.stb0 drop column col2") + tdSql.execute("alter table db2.stb0 add column c4 int") + tdSql.execute("alter table db2.stb0 drop column c2") tdSql.execute("alter table db2.stb0 add tag t3 int") tdSql.execute("alter table db2.stb0 drop tag t1") - tdSql.execute("create table if not exists stb2_0 (ts timestamp, col0 int, col1 float) ") + tdSql.execute("create table if not exists stb2_0 (ts timestamp, c0 int, c1 float) ") tdSql.execute("insert into stb2_0 values(1614218412000,8637,78.861045)") - tdSql.execute("alter table stb2_0 add column col2 binary(4)") - tdSql.execute("alter table stb2_0 drop column col1") + tdSql.execute("alter table stb2_0 add column c2 binary(4)") + tdSql.execute("alter table stb2_0 drop column c1") tdSql.execute("insert into stb2_0 values(1614218422000,8638,'R')") # create db utest and modify super tables; diff --git a/tests/pytest/tsdb/tsdbCompClusterReplica2.py b/tests/pytest/tsdb/tsdbCompClusterReplica2.py index 9b43b7ce80..2e016deea0 100644 --- a/tests/pytest/tsdb/tsdbCompClusterReplica2.py +++ b/tests/pytest/tsdb/tsdbCompClusterReplica2.py @@ -76,14 +76,14 @@ class TwoClients: tdSql.execute("drop table if exists stb1_0") tdSql.execute("drop table if exists stb1_1") tdSql.execute("insert into stb0_0 values(1614218412000,8637,78.861045,'R','bf3')(1614218422000,8637,98.861045,'R','bf3')") - tdSql.execute("alter table db2.stb0 add column col4 int") - tdSql.execute("alter table db2.stb0 drop column col2") + tdSql.execute("alter table db2.stb0 add column c4 int") + tdSql.execute("alter table db2.stb0 drop column c2") tdSql.execute("alter table db2.stb0 add tag t3 int") tdSql.execute("alter table db2.stb0 drop tag t1") - tdSql.execute("create table if not exists stb2_0 (ts timestamp, col0 int, col1 float) ") + tdSql.execute("create table if not exists stb2_0 (ts timestamp, c0 int, c1 float) ") tdSql.execute("insert into stb2_0 values(1614218412000,8637,78.861045)") - tdSql.execute("alter table stb2_0 add column col2 binary(4)") - tdSql.execute("alter table stb2_0 drop column col1") + tdSql.execute("alter table stb2_0 add column c2 binary(4)") + tdSql.execute("alter table stb2_0 drop column c1") tdSql.execute("insert into stb2_0 values(1614218422000,8638,'R')") From 67040b64cc83973faff2361677ad5ba16b31ce66 Mon Sep 17 00:00:00 2001 From: tomchon Date: Thu, 5 Aug 2021 18:23:54 +0800 Subject: [PATCH 17/22] [TD-5114]: add testcase of rollUpgrading --- .../manualTest/TD-5114/continueCreateDn.py | 97 ++++++ .../TD-5114/insertDataDb3Replica2.json | 61 ++++ .../manualTest/TD-5114/rollingUpgrade.py | 275 ++++++++++++++++++ 3 files changed, 433 insertions(+) create mode 100644 tests/pytest/manualTest/TD-5114/continueCreateDn.py create mode 100644 tests/pytest/manualTest/TD-5114/insertDataDb3Replica2.json create mode 100644 tests/pytest/manualTest/TD-5114/rollingUpgrade.py diff --git a/tests/pytest/manualTest/TD-5114/continueCreateDn.py b/tests/pytest/manualTest/TD-5114/continueCreateDn.py new file mode 100644 index 0000000000..4b724f0587 --- /dev/null +++ b/tests/pytest/manualTest/TD-5114/continueCreateDn.py @@ -0,0 +1,97 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +import os +import sys +sys.path.insert(0, os.getcwd()) +from util.log import * +from util.sql import * +from util.dnodes import * +import taos +import threading +import subprocess +from random import choice + +class TwoClients: + def initConnection(self): + self.host = "chr03" + self.user = "root" + self.password = "taosdata" + self.config = "/etc/taos/" + self.port =6030 + self.rowNum = 10 + self.ts = 1537146000000 + + def run(self): + + # new taos client + conn1 = taos.connect(host=self.host, user=self.user, password=self.password, config=self.config ) + print(conn1) + cur1 = conn1.cursor() + tdSql.init(cur1, True) + + tdSql.execute("drop database if exists db3") + + # insert data with taosc + for i in range(10): + os.system("taosdemo -f manualTest/TD-5114/insertDataDb3Replica2.json -y ") + # # check data correct + tdSql.execute("show databases") + tdSql.execute("use db3") + tdSql.query("select count (tbname) from stb0") + tdSql.checkData(0, 0, 20000) + tdSql.query("select count (*) from stb0") + tdSql.checkData(0, 0, 4000000) + + # insert data with python connector , if you want to use this case ,cancel note. + + # for x in range(10): + # dataType= [ "tinyint", "smallint", "int", "bigint", "float", "double", "bool", " binary(20)", "nchar(20)", "tinyint unsigned", "smallint unsigned", "int unsigned", "bigint unsigned"] + # tdSql.execute("drop database if exists db3") + # tdSql.execute("create database db3 keep 3650 replica 2 ") + # tdSql.execute("use db3") + # tdSql.execute('''create table test(ts timestamp, col0 tinyint, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 float, col6 double, + # col7 bool, col8 binary(20), col9 nchar(20), col10 tinyint unsigned, col11 smallint unsigned, col12 int unsigned, col13 bigint unsigned) tags(loc nchar(3000), tag1 int)''') + # rowNum2= 988 + # for i in range(rowNum2): + # tdSql.execute("alter table test add column col%d %s ;" %( i+14, choice(dataType)) ) + # rowNum3= 988 + # for i in range(rowNum3): + # tdSql.execute("alter table test drop column col%d ;" %( i+14) ) + # self.rowNum = 50 + # self.rowNum2 = 2000 + # self.ts = 1537146000000 + # for j in range(self.rowNum2): + # tdSql.execute("create table test%d using test tags('beijing%d', 10)" % (j,j) ) + # for i in range(self.rowNum): + # tdSql.execute("insert into test%d values(%d, %d, %d, %d, %d, %d, %f, %f, %d, 'taosdata%d', '涛思数据%d', %d, %d, %d, %d)" + # % (j, self.ts + i*1000, i + 1, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1)) + + # # check data correct + # tdSql.execute("show databases") + # tdSql.execute("use db3") + # tdSql.query("select count (tbname) from test") + # tdSql.checkData(0, 0, 200) + # tdSql.query("select count (*) from test") + # tdSql.checkData(0, 0, 200000) + + + # delete useless file + testcaseFilename = os.path.split(__file__)[-1] + os.system("rm -rf ./insert_res.txt") + os.system("rm -rf manualTest/TD-5114/%s.sql" % testcaseFilename ) + +clients = TwoClients() +clients.initConnection() +# clients.getBuildPath() +clients.run() \ No newline at end of file diff --git a/tests/pytest/manualTest/TD-5114/insertDataDb3Replica2.json b/tests/pytest/manualTest/TD-5114/insertDataDb3Replica2.json new file mode 100644 index 0000000000..b2755823ef --- /dev/null +++ b/tests/pytest/manualTest/TD-5114/insertDataDb3Replica2.json @@ -0,0 +1,61 @@ +{ + "filetype": "insert", + "cfgdir": "/etc/taos", + "host": "127.0.0.1", + "port": 6030, + "user": "root", + "password": "taosdata", + "thread_count": 4, + "thread_count_create_tbl": 4, + "result_file": "./insert_res.txt", + "confirm_parameter_prompt": "no", + "insert_interval": 0, + "interlace_rows": 0, + "num_of_records_per_req": 3000, + "max_sql_len": 1024000, + "databases": [{ + "dbinfo": { + "name": "db3", + "drop": "yes", + "replica": 2, + "days": 10, + "cache": 50, + "blocks": 8, + "precision": "ms", + "keep": 365, + "minRows": 100, + "maxRows": 4096, + "comp":2, + "walLevel":1, + "cachelast":0, + "quorum":1, + "fsync":3000, + "update": 0 + }, + "super_tables": [{ + "name": "stb0", + "child_table_exists":"no", + "childtable_count": 20000, + "childtable_prefix": "stb0_", + "auto_create_table": "no", + "batch_create_tbl_num": 1000, + "data_source": "rand", + "insert_mode": "taosc", + "insert_rows": 2000, + "childtable_limit": 0, + "childtable_offset":0, + "interlace_rows": 0, + "insert_interval":0, + "max_sql_len": 1024000, + "disorder_ratio": 0, + "disorder_range": 1000, + "timestamp_step": 1, + "start_timestamp": "2020-10-01 00:00:00.000", + "sample_format": "csv", + "sample_file": "./sample.csv", + "tags_file": "", + "columns": [{"type": "INT"}, {"type": "DOUBLE", "count":1}, {"type": "BINARY", "len": 16, "count":1}, {"type": "BINARY", "len": 32, "count":1}], + "tags": [{"type": "TINYINT", "count":2}, {"type": "BINARY", "len": 16, "count":1}] + }] + }] +} diff --git a/tests/pytest/manualTest/TD-5114/rollingUpgrade.py b/tests/pytest/manualTest/TD-5114/rollingUpgrade.py new file mode 100644 index 0000000000..f634eb1208 --- /dev/null +++ b/tests/pytest/manualTest/TD-5114/rollingUpgrade.py @@ -0,0 +1,275 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +from sys import version +from fabric import Connection +import random +import time +import datetime +import logging +import subprocess +import os +import sys + +class Node: + def __init__(self, index, username, hostIP, password, version): + self.index = index + self.username = username + self.hostIP = hostIP + # self.hostName = hostName + # self.homeDir = homeDir + self.version = version + self.verName = "TDengine-enterprise-server-%s-Linux-x64.tar.gz" % self.version + self.installPath = "TDengine-enterprise-server-%s" % self.version + # self.corePath = '/coredump' + self.conn = Connection("{}@{}".format(username, hostIP), connect_kwargs={"password": "{}".format(password)}) + + + def buildTaosd(self): + try: + print(self.conn) + # self.conn.run('echo "1234" > /home/chr/installtest/test.log') + self.conn.run("cd /home/chr/installtest/ && tar -xvf %s " %self.verName) + self.conn.run("cd /home/chr/installtest/%s && ./install.sh " % self.installPath) + except Exception as e: + print("Build Taosd error for node %d " % self.index) + logging.exception(e) + pass + + def rebuildTaosd(self): + try: + print(self.conn) + # self.conn.run('echo "1234" > /home/chr/installtest/test.log') + self.conn.run("cd /home/chr/installtest/%s && ./install.sh " % self.installPath) + except Exception as e: + print("Build Taosd error for node %d " % self.index) + logging.exception(e) + pass + + def startTaosd(self): + try: + self.conn.run("sudo systemctl start taosd") + except Exception as e: + print("Start Taosd error for node %d " % self.index) + logging.exception(e) + + def restartTarbi(self): + try: + self.conn.run("sudo systemctl restart tarbitratord ") + except Exception as e: + print("Start Taosd error for node %d " % self.index) + logging.exception(e) + + def clearData(self): + timeNow = datetime.datetime.now() + # timeYes = datetime.datetime.now() + datetime.timedelta(days=-1) + timStr = timeNow.strftime('%Y%m%d%H%M%S') + # timStr = timeNow.strftime('%Y%m%d%H%M%S') + try: + # self.conn.run("mv /var/lib/taos/ /var/lib/taos%s " % timStr) + self.conn.run("rm -rf /home/chr/data/taos*") + except Exception as e: + print("rm -rf /var/lib/taos error %d " % self.index) + logging.exception(e) + + def stopTaosd(self): + try: + self.conn.run("sudo systemctl stop taosd") + except Exception as e: + print("Stop Taosd error for node %d " % self.index) + logging.exception(e) + + def restartTaosd(self): + try: + self.conn.run("sudo systemctl restart taosd") + except Exception as e: + print("Stop Taosd error for node %d " % self.index) + logging.exception(e) + +class oneNode: + + def FirestStartNode(self, id, username, IP, passwd, version): + # get installPackage + verName = "TDengine-enterprise-server-%s-Linux-x64.tar.gz" % version + # installPath = "TDengine-enterprise-server-%s" % self.version + node131 = Node(131, 'ubuntu', '192.168.1.131', 'tbase125!', '2.0.20.0') + node131.conn.run('sshpass -p tbase125! scp /nas/TDengine/v%s/enterprise/%s root@192.168.1.%d:/home/chr/installtest/' % (version,verName,id)) + node131.conn.close() + # install TDengine at 192.168.103/104/141 + try: + node = Node(id, username, IP, passwd, version) + node.conn.run('echo "start taosd"') + node.buildTaosd() + # clear DataPath , if need clear data + node.clearData() + node.startTaosd() + if id == 103 : + node.restartTarbi() + print("start taosd ver:%s node:%d successfully " % (version,id)) + node.conn.close() + + # query_pid = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1]) + # assert query_pid == 1 , "node %d: start taosd failed " % id + except Exception as e: + print("Stop Taosd error for node %d " % id) + logging.exception(e) + + def startNode(self, id, username, IP, passwd, version): + # start TDengine + try: + node = Node(id, username, IP, passwd, version) + node.conn.run('echo "restart taosd"') + # clear DataPath , if need clear data + node.clearData() + node.restartTaosd() + time.sleep(5) + if id == 103 : + node.restartTarbi() + print("start taosd ver:%s node:%d successfully " % (version,id)) + node.conn.close() + + # query_pid = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1]) + # assert query_pid == 1 , "node %d: start taosd failed " % id + except Exception as e: + print("Stop Taosd error for node %d " % id) + logging.exception(e) + + def firstUpgradeNode(self, id, username, IP, passwd, version): + # get installPackage + verName = "TDengine-enterprise-server-%s-Linux-x64.tar.gz" % version + # installPath = "TDengine-enterprise-server-%s" % self.version + node131 = Node(131, 'ubuntu', '192.168.1.131', 'tbase125!', '2.0.20.0') + node131.conn.run('echo "upgrade cluster"') + node131.conn.run('sshpass -p tbase125! scp /nas/TDengine/v%s/enterprise/%s root@192.168.1.%d:/home/chr/installtest/' % (version,verName,id)) + node131.conn.close() + # upgrade TDengine at 192.168.103/104/141 + try: + node = Node(id, username, IP, passwd, version) + node.conn.run('echo "start taosd"') + node.conn.run('echo "1234" > /home/chr/test.log') + node.buildTaosd() + time.sleep(5) + node.startTaosd() + if id == 103 : + node.restartTarbi() + print("start taosd ver:%s node:%d successfully " % (version,id)) + node.conn.close() + + # query_pid = int(subprocess.getstatusoutput('ps aux|grep taosd |grep -v "grep"|awk \'{print $2}\'')[1]) + # assert query_pid == 1 , "node %d: start taosd failed " % id + except Exception as e: + print("Stop Taosd error for node %d " % id) + logging.exception(e) + + def upgradeNode(self, id, username, IP, passwd, version): + + # backCluster TDengine at 192.168.103/104/141 + try: + node = Node(id, username, IP, passwd, version) + node.conn.run('echo "rollback taos"') + node.rebuildTaosd() + time.sleep(5) + node.startTaosd() + if id == 103 : + node.restartTarbi() + print("start taosd ver:%s node:%d successfully " % (version,id)) + node.conn.close() + except Exception as e: + print("Stop Taosd error for node %d " % id) + logging.exception(e) + + +# how to use : cd TDinternal/commumity/test/pytest && python3 manualTest/rollingUpgrade.py ,when inserting data, we can start " python3 manualTest/rollingUpagrade.py". add example "oneNode().FirestStartNode(103,'root','192.168.1.103','tbase125!','2.0.20.0')" + + +# node103=oneNode().FirestStartNode(103,'root','192.168.1.103','tbase125!','2.0.20.0') +# node104=oneNode().FirestStartNode(104,'root','192.168.1.104','tbase125!','2.0.20.0') +# node141=oneNode().FirestStartNode(141,'root','192.168.1.141','tbase125!','2.0.20.0') + +# node103=oneNode().startNode(103,'root','192.168.1.103','tbase125!','2.0.20.0') +# time.sleep(30) +# node141=oneNode().startNode(141,'root','192.168.1.141','tbase125!','2.0.20.0') +# time.sleep(30) +# node104=oneNode().startNode(104,'root','192.168.1.104','tbase125!','2.0.20.0') +# time.sleep(30) + +# node103=oneNode().firstUpgradeNode(103,'root','192.168.1.103','tbase125!','2.0.20.5') +# time.sleep(30) +# node104=oneNode().firstUpgradeNode(104,'root','192.168.1.104','tbase125!','2.0.20.5') +# time.sleep(30) +# node141=oneNode().firstUpgradeNode(141,'root','192.168.1.141','tbase125!','2.0.20.5') +# time.sleep(30) + +# node141=oneNode().firstUpgradeNode(141,'root','192.168.1.141','tbase125!','2.0.20.10') +# time.sleep(30) +# node103=oneNode().firstUpgradeNode(103,'root','192.168.1.103','tbase125!','2.0.20.10') +# time.sleep(30) +# node104=oneNode().firstUpgradeNode(104,'root','192.168.1.104','tbase125!','2.0.20.10') +# time.sleep(30) + +# node141=oneNode().firstUpgradeNode(141,'root','192.168.1.141','tbase125!','2.0.20.12') +# time.sleep(30) +# node103=oneNode().firstUpgradeNode(103,'root','192.168.1.103','tbase125!','2.0.20.12') +# time.sleep(30) +# node104=oneNode().firstUpgradeNode(104,'root','192.168.1.104','tbase125!','2.0.20.12') +# time.sleep(30) + + + +# node103=oneNode().upgradeNode(103,'root','192.168.1.103','tbase125!','2.0.20.0') +# time.sleep(120) +# node104=oneNode().upgradeNode(104,'root','192.168.1.104','tbase125!','2.0.20.0') +# time.sleep(180) +# node141=oneNode().upgradeNode(141,'root','192.168.1.141','tbase125!','2.0.20.0') +# time.sleep(240) + +# node104=oneNode().upgradeNode(104,'root','192.168.1.104','tbase125!','2.0.20.5') +# time.sleep(120) +# node103=oneNode().upgradeNode(103,'root','192.168.1.103','tbase125!','2.0.20.5') +# time.sleep(120) +# node141=oneNode().upgradeNode(141,'root','192.168.1.141','tbase125!','2.0.20.5') +# time.sleep(180) + +# node141=oneNode().upgradeNode(141,'root','192.168.1.141','tbase125!','2.0.20.10') +# time.sleep(120) +# node103=oneNode().upgradeNode(103,'root','192.168.1.103','tbase125!','2.0.20.10') +# time.sleep(120) +# node104=oneNode().upgradeNode(104,'root','192.168.1.104','tbase125!','2.0.20.10') +# time.sleep(180) + +# node103=oneNode().upgradeNode(103,'root','192.168.1.103','tbase125!','2.0.20.12') +# time.sleep(180) +# node141=oneNode().upgradeNode(141,'root','192.168.1.141','tbase125!','2.0.20.12') +# time.sleep(180) +# node104=oneNode().upgradeNode(104,'root','192.168.1.104','tbase125!','2.0.20.12') + + +# node141=oneNode().firstUpgradeNode(141,'root','192.168.1.141','tbase125!','2.0.20.9') +# time.sleep(5) +# node103=oneNode().firstUpgradeNode(103,'root','192.168.1.103','tbase125!','2.0.20.9') +# time.sleep(5) +# node104=oneNode().firstUpgradeNode(104,'root','192.168.1.104','tbase125!','2.0.20.9') +# time.sleep(30) + +# node141=oneNode().upgradeNode(141,'root','192.168.1.141','tbase125!','2.0.20.10') +# time.sleep(12) +# node103=oneNode().upgradeNode(103,'root','192.168.1.103','tbase125!','2.0.20.10') +# time.sleep(12) +# node104=oneNode().upgradeNode(104,'root','192.168.1.104','tbase125!','2.0.20.10') +# time.sleep(180) + +# node103=oneNode().upgradeNode(103,'root','192.168.1.103','tbase125!','2.0.20.12') +# time.sleep(120) +# node141=oneNode().upgradeNode(141,'root','192.168.1.141','tbase125!','2.0.20.12') +# time.sleep(120) +# node104=oneNode().upgradeNode(104,'root','192.168.1.104','tbase125!','2.0.20.12') From c180e69290124fac84468b5dae096804f33b98df Mon Sep 17 00:00:00 2001 From: lichuang Date: Tue, 17 Aug 2021 10:31:11 +0800 Subject: [PATCH 18/22] [TD-4352]add tsdbMetaCompactRatio config item --- packaging/cfg/taos.cfg | 2 ++ src/common/src/tglobal.c | 11 +++++++++++ src/inc/taosdef.h | 1 + src/tsdb/src/tsdbCommit.c | 11 +++++++---- 4 files changed, 21 insertions(+), 4 deletions(-) diff --git a/packaging/cfg/taos.cfg b/packaging/cfg/taos.cfg index 3ae4e9941e..fdbb5829f8 100644 --- a/packaging/cfg/taos.cfg +++ b/packaging/cfg/taos.cfg @@ -284,3 +284,5 @@ keepColumnName 1 # 0 no query allowed, queries are disabled # queryBufferSize -1 +# percent of redundant data in tsdb meta will compact meta data,0 means donot compact +# tsdbMetaCompactRatio 30 diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 44b3e87e7d..6c51ba22a1 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -150,6 +150,7 @@ int32_t tsMaxVgroupsPerDb = 0; int32_t tsMinTablePerVnode = TSDB_TABLES_STEP; int32_t tsMaxTablePerVnode = TSDB_DEFAULT_TABLES; int32_t tsTableIncStepPerVnode = TSDB_TABLES_STEP; +int32_t tsTsdbMetaCompactRatio = TSDB_META_COMPACT_RATIO; // balance int8_t tsEnableBalance = 1; @@ -1576,6 +1577,16 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); + cfg.option = "tsdbMetaCompactRatio"; + cfg.ptr = &tsTsdbMetaCompactRatio; + cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG; + cfg.minValue = 0; + cfg.maxValue = 100; + cfg.ptrLength = 0; + cfg.unitType = TAOS_CFG_UTYPE_NONE; + taosInitConfigOption(cfg); + assert(tsGlobalConfigNum <= TSDB_CFG_MAX_NUM); #ifdef TD_TSZ // lossy compress diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index fceeaea0ae..66811326ee 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -275,6 +275,7 @@ do { \ #define TSDB_MAX_TABLES 10000000 #define TSDB_DEFAULT_TABLES 1000000 #define TSDB_TABLES_STEP 1000 +#define TSDB_META_COMPACT_RATIO 30 #define TSDB_MIN_DAYS_PER_FILE 1 #define TSDB_MAX_DAYS_PER_FILE 3650 diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index 9e441c1770..6bd47247a2 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -14,6 +14,8 @@ */ #include "tsdbint.h" +extern int32_t tsTsdbMetaCompactRatio; + #define TSDB_MAX_SUBBLOCKS 8 static FORCE_INLINE int TSDB_KEY_FID(TSKEY key, int32_t days, int8_t precision) { if (key < 0) { @@ -339,7 +341,7 @@ static int tsdbCommitMeta(STsdbRepo *pRepo) { tsdbCloseMFile(&mf); tsdbUpdateMFile(pfs, &mf); - if (tsdbCompactMetaFile(pRepo, pfs, &mf) < 0) { + if (tsTsdbMetaCompactRatio > 0 && tsdbCompactMetaFile(pRepo, pfs, &mf) < 0) { tsdbError("compact meta file error"); } @@ -455,8 +457,9 @@ static int tsdbDropMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid) { static int tsdbCompactMetaFile(STsdbRepo *pRepo, STsdbFS *pfs, SMFile *pMFile) { float delPercent = (float)(pMFile->info.nDels) / (float)(pMFile->info.nRecords); float tombPercent = (float)(pMFile->info.tombSize) / (float)(pMFile->info.size); + float compactRatio = (float)(tsTsdbMetaCompactRatio)/100; - if (delPercent < 0.33 && tombPercent < 0.33) { + if (delPercent < compactRatio && tombPercent < compactRatio) { return 0; } @@ -465,8 +468,8 @@ static int tsdbCompactMetaFile(STsdbRepo *pRepo, STsdbFS *pfs, SMFile *pMFile) { return -1; } - tsdbInfo("begin compact tsdb meta file, nDels:%" PRId64 ",nRecords:%" PRId64 ",tombSize:%" PRId64 ",size:%" PRId64, - pMFile->info.nDels,pMFile->info.nRecords,pMFile->info.tombSize,pMFile->info.size); + tsdbInfo("begin compact tsdb meta file, ratio:%d, nDels:%" PRId64 ",nRecords:%" PRId64 ",tombSize:%" PRId64 ",size:%" PRId64, + tsTsdbMetaCompactRatio, pMFile->info.nDels,pMFile->info.nRecords,pMFile->info.tombSize,pMFile->info.size); SMFile mf; SDiskID did; From 1a3448ffbcb3db1afa9441e407435f68eb10d993 Mon Sep 17 00:00:00 2001 From: lichuang Date: Tue, 17 Aug 2021 14:08:50 +0800 Subject: [PATCH 19/22] [TD-4352]disable tsdbMetaCompactRatio by default --- packaging/cfg/taos.cfg | 2 +- src/inc/taosdef.h | 2 +- src/tsdb/src/tsdbCommit.c | 2 +- src/tsdb/src/tsdbMeta.c | 5 ++++- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/packaging/cfg/taos.cfg b/packaging/cfg/taos.cfg index fdbb5829f8..06e7cb7da0 100644 --- a/packaging/cfg/taos.cfg +++ b/packaging/cfg/taos.cfg @@ -285,4 +285,4 @@ keepColumnName 1 # queryBufferSize -1 # percent of redundant data in tsdb meta will compact meta data,0 means donot compact -# tsdbMetaCompactRatio 30 +# tsdbMetaCompactRatio 0 diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 66811326ee..4f70fd71f0 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -275,7 +275,7 @@ do { \ #define TSDB_MAX_TABLES 10000000 #define TSDB_DEFAULT_TABLES 1000000 #define TSDB_TABLES_STEP 1000 -#define TSDB_META_COMPACT_RATIO 30 +#define TSDB_META_COMPACT_RATIO 0 // disable tsdb meta compact by default #define TSDB_MIN_DAYS_PER_FILE 1 #define TSDB_MAX_DAYS_PER_FILE 3650 diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index 6bd47247a2..dd5dec81e0 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -536,7 +536,7 @@ static int tsdbCompactMetaFile(STsdbRepo *pRepo, STsdbFS *pfs, SMFile *pMFile) { code = 0; _err: - TSDB_FILE_FSYNC(&mf); + if (code == 0) TSDB_FILE_FSYNC(&mf); tsdbCloseMFile(&mf); tsdbCloseMFile(pMFile); diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 62c3a06321..eba14cbb87 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -746,7 +746,10 @@ void tsdbUpdateTableSchema(STsdbRepo *pRepo, STable *pTable, STSchema *pSchema, TSDB_WUNLOCK_TABLE(pCTable); if (insertAct) { - ASSERT(tsdbInsertNewTableAction(pRepo, pCTable) == 0); + if (tsdbInsertNewTableAction(pRepo, pCTable) != 0) { + tsdbError("vgId:%d table %s tid %d uid %" PRIu64 " tsdbInsertNewTableAction fail", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), + TABLE_TID(pTable), TABLE_UID(pTable)); + } } } From dc778057deed2073916321f9b9c2609618ccebe1 Mon Sep 17 00:00:00 2001 From: lichuang Date: Tue, 17 Aug 2021 15:22:45 +0800 Subject: [PATCH 20/22] run ci again From a1ac1c63488506db233641298719809f554dfbbb Mon Sep 17 00:00:00 2001 From: lichuang Date: Thu, 19 Aug 2021 11:10:10 +0800 Subject: [PATCH 21/22] [TD-4352]refactor code,use meta-ver instead of tmp file --- src/tsdb/inc/tsdbFile.h | 4 +-- src/tsdb/src/tsdbCommit.c | 63 ++++++++++++++++++++++++++------------- src/tsdb/src/tsdbFS.c | 2 +- src/tsdb/src/tsdbFile.c | 5 ++-- src/tsdb/src/tsdbSync.c | 2 +- 5 files changed, 48 insertions(+), 28 deletions(-) diff --git a/src/tsdb/inc/tsdbFile.h b/src/tsdb/inc/tsdbFile.h index 3913a97c3c..b9d5431de6 100644 --- a/src/tsdb/inc/tsdbFile.h +++ b/src/tsdb/inc/tsdbFile.h @@ -38,7 +38,7 @@ #define TSDB_FILE_IS_OK(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_OK) #define TSDB_FILE_IS_BAD(tf) (TSDB_FILE_STATE(tf) == TSDB_FILE_STATE_BAD) -typedef enum { TSDB_FILE_HEAD = 0, TSDB_FILE_DATA, TSDB_FILE_LAST, TSDB_FILE_MAX, TSDB_FILE_META, TSDB_FILE_META_TMP} TSDB_FILE_T; +typedef enum { TSDB_FILE_HEAD = 0, TSDB_FILE_DATA, TSDB_FILE_LAST, TSDB_FILE_MAX, TSDB_FILE_META } TSDB_FILE_T; // =============== SMFile typedef struct { @@ -56,7 +56,7 @@ typedef struct { uint8_t state; } SMFile; -void tsdbInitMFile(SMFile* pMFile, SDiskID did, int vid, uint32_t ver, bool tmp); +void tsdbInitMFile(SMFile* pMFile, SDiskID did, int vid, uint32_t ver); void tsdbInitMFileEx(SMFile* pMFile, const SMFile* pOMFile); int tsdbEncodeSMFile(void** buf, SMFile* pMFile); void* tsdbDecodeSMFile(void* buf, SMFile* pMFile); diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index dd5dec81e0..45fe6f6936 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -264,6 +264,35 @@ int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf) { // =================== Commit Meta Data +static int tsdbInitCommitMetaFile(STsdbRepo *pRepo, SMFile* pMf, bool open) { + STsdbFS * pfs = REPO_FS(pRepo); + SMFile * pOMFile = pfs->cstatus->pmf; + SDiskID did; + + // Create/Open a meta file or open the existing file + if (pOMFile == NULL) { + // Create a new meta file + did.level = TFS_PRIMARY_LEVEL; + did.id = TFS_PRIMARY_ID; + tsdbInitMFile(pMf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo))); + + if (open && tsdbCreateMFile(pMf, true) < 0) { + tsdbError("vgId:%d failed to create META file since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + + tsdbInfo("vgId:%d meta file %s is created to commit", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMf)); + } else { + tsdbInitMFileEx(pMf, pOMFile); + if (open && tsdbOpenMFile(pMf, O_WRONLY) < 0) { + tsdbError("vgId:%d failed to open META file since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + } + + return 0; +} + static int tsdbCommitMeta(STsdbRepo *pRepo) { STsdbFS * pfs = REPO_FS(pRepo); SMemTable *pMem = pRepo->imem; @@ -272,34 +301,25 @@ static int tsdbCommitMeta(STsdbRepo *pRepo) { SActObj * pAct = NULL; SActCont * pCont = NULL; SListNode *pNode = NULL; - SDiskID did; ASSERT(pOMFile != NULL || listNEles(pMem->actList) > 0); if (listNEles(pMem->actList) <= 0) { // no meta data to commit, just keep the old meta file tsdbUpdateMFile(pfs, pOMFile); + if (tsTsdbMetaCompactRatio > 0) { + if (tsdbInitCommitMetaFile(pRepo, &mf, false) < 0) { + return -1; + } + int ret = tsdbCompactMetaFile(pRepo, pfs, &mf); + if (ret < 0) tsdbError("compact meta file error"); + + return ret; + } return 0; } else { - // Create/Open a meta file or open the existing file - if (pOMFile == NULL) { - // Create a new meta file - did.level = TFS_PRIMARY_LEVEL; - did.id = TFS_PRIMARY_ID; - tsdbInitMFile(&mf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo)), false); - - if (tsdbCreateMFile(&mf, true) < 0) { - tsdbError("vgId:%d failed to create META file since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - tsdbInfo("vgId:%d meta file %s is created to commit", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(&mf)); - } else { - tsdbInitMFileEx(&mf, pOMFile); - if (tsdbOpenMFile(&mf, O_WRONLY) < 0) { - tsdbError("vgId:%d failed to open META file since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } + if (tsdbInitCommitMetaFile(pRepo, &mf, true) < 0) { + return -1; } } @@ -477,7 +497,7 @@ static int tsdbCompactMetaFile(STsdbRepo *pRepo, STsdbFS *pfs, SMFile *pMFile) { // first create tmp meta file did.level = TFS_PRIMARY_LEVEL; did.id = TFS_PRIMARY_ID; - tsdbInitMFile(&mf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo)), true); + tsdbInitMFile(&mf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo)) + 1); if (tsdbCreateMFile(&mf, true) < 0) { tsdbError("vgId:%d failed to create META file since %s", REPO_ID(pRepo), tstrerror(terrno)); @@ -542,6 +562,7 @@ _err: if (code == 0) { // rename meta.tmp -> meta + tsdbInfo("vgId:%d meta file rename %s -> %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(&mf), TSDB_FILE_FULL_NAME(pMFile)); taosRename(mf.f.aname,pMFile->f.aname); tstrncpy(mf.f.aname, pMFile->f.aname, TSDB_FILENAME_LEN); tstrncpy(mf.f.rname, pMFile->f.rname, TSDB_FILENAME_LEN); diff --git a/src/tsdb/src/tsdbFS.c b/src/tsdb/src/tsdbFS.c index 5f34764538..68450301d8 100644 --- a/src/tsdb/src/tsdbFS.c +++ b/src/tsdb/src/tsdbFS.c @@ -274,7 +274,7 @@ static int tsdbCreateMeta(STsdbRepo *pRepo) { // Create a new meta file did.level = TFS_PRIMARY_LEVEL; did.id = TFS_PRIMARY_ID; - tsdbInitMFile(&mf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo)), false); + tsdbInitMFile(&mf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo))); if (tsdbCreateMFile(&mf, true) < 0) { tsdbError("vgId:%d failed to create META file since %s", REPO_ID(pRepo), tstrerror(terrno)); diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index cc6fbb632f..0f13b6108f 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -21,7 +21,6 @@ static const char *TSDB_FNAME_SUFFIX[] = { "last", // TSDB_FILE_LAST "", // TSDB_FILE_MAX "meta", // TSDB_FILE_META - "meta.tmp", // TSDB_FILE_META_TMP }; static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, char *fname); @@ -31,7 +30,7 @@ static void *tsdbDecodeDFInfo(void *buf, SDFInfo *pInfo); static int tsdbRollBackDFile(SDFile *pDFile); // ============== SMFile -void tsdbInitMFile(SMFile *pMFile, SDiskID did, int vid, uint32_t ver, bool tmp) { +void tsdbInitMFile(SMFile *pMFile, SDiskID did, int vid, uint32_t ver) { char fname[TSDB_FILENAME_LEN]; TSDB_FILE_SET_STATE(pMFile, TSDB_FILE_STATE_OK); @@ -39,7 +38,7 @@ void tsdbInitMFile(SMFile *pMFile, SDiskID did, int vid, uint32_t ver, bool tmp) memset(&(pMFile->info), 0, sizeof(pMFile->info)); pMFile->info.magic = TSDB_FILE_INIT_MAGIC; - tsdbGetFilename(vid, 0, ver, tmp ? TSDB_FILE_META_TMP : TSDB_FILE_META, fname); + tsdbGetFilename(vid, 0, ver, TSDB_FILE_META, fname); tfsInitFile(TSDB_FILE_F(pMFile), did.level, did.id, fname); } diff --git a/src/tsdb/src/tsdbSync.c b/src/tsdb/src/tsdbSync.c index f06943bc37..edcb84d091 100644 --- a/src/tsdb/src/tsdbSync.c +++ b/src/tsdb/src/tsdbSync.c @@ -209,7 +209,7 @@ static int32_t tsdbSyncRecvMeta(SSyncH *pSynch) { // Recv from remote SMFile mf; SDiskID did = {.level = TFS_PRIMARY_LEVEL, .id = TFS_PRIMARY_ID}; - tsdbInitMFile(&mf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo)), false); + tsdbInitMFile(&mf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo))); if (tsdbCreateMFile(&mf, false) < 0) { tsdbError("vgId:%d, failed to create file while recv metafile since %s", REPO_ID(pRepo), tstrerror(terrno)); return -1; From 23cbcdd0b6d97055f40babeb0b3777ef2dd5b0c8 Mon Sep 17 00:00:00 2001 From: lichuang Date: Thu, 19 Aug 2021 15:11:11 +0800 Subject: [PATCH 22/22] [TD-4352]fix bug:save new file content in new meta cache --- src/tsdb/inc/tsdbFS.h | 5 +++-- src/tsdb/src/tsdbCommit.c | 33 +++++++++++++++++++++++---------- src/tsdb/src/tsdbFS.c | 1 + 3 files changed, 27 insertions(+), 12 deletions(-) diff --git a/src/tsdb/inc/tsdbFS.h b/src/tsdb/inc/tsdbFS.h index d63aeb14ac..3b6b6449f6 100644 --- a/src/tsdb/inc/tsdbFS.h +++ b/src/tsdb/inc/tsdbFS.h @@ -42,8 +42,9 @@ typedef struct { typedef struct { pthread_rwlock_t lock; - SFSStatus* cstatus; // current status - SHashObj* metaCache; // meta cache + SFSStatus* cstatus; // current status + SHashObj* metaCache; // meta cache + SHashObj* metaCacheComp; // meta cache for compact bool intxn; SFSStatus* nstatus; // new status } STsdbFS; diff --git a/src/tsdb/src/tsdbCommit.c b/src/tsdb/src/tsdbCommit.c index 45fe6f6936..15fc3cc47d 100644 --- a/src/tsdb/src/tsdbCommit.c +++ b/src/tsdb/src/tsdbCommit.c @@ -57,7 +57,7 @@ typedef struct { #define TSDB_COMMIT_TXN_VERSION(ch) FS_TXN_VERSION(REPO_FS(TSDB_COMMIT_REPO(ch))) static int tsdbCommitMeta(STsdbRepo *pRepo); -static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void *cont, int contLen, bool updateMeta); +static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void *cont, int contLen, bool compact); static int tsdbDropMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid); static int tsdbCompactMetaFile(STsdbRepo *pRepo, STsdbFS *pfs, SMFile *pMFile); static int tsdbCommitTSData(STsdbRepo *pRepo); @@ -328,7 +328,7 @@ static int tsdbCommitMeta(STsdbRepo *pRepo) { pAct = (SActObj *)pNode->data; if (pAct->act == TSDB_UPDATE_META) { pCont = (SActCont *)POINTER_SHIFT(pAct, sizeof(SActObj)); - if (tsdbUpdateMetaRecord(pfs, &mf, pAct->uid, (void *)(pCont->cont), pCont->len, true) < 0) { + if (tsdbUpdateMetaRecord(pfs, &mf, pAct->uid, (void *)(pCont->cont), pCont->len, false) < 0) { tsdbError("vgId:%d failed to update META record, uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid, tstrerror(terrno)); tsdbCloseMFile(&mf); @@ -402,7 +402,7 @@ void tsdbGetRtnSnap(STsdbRepo *pRepo, SRtn *pRtn) { pRtn->minFid, pRtn->midFid, pRtn->maxFid); } -static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void *cont, int contLen, bool updateMeta) { +static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void *cont, int contLen, bool compact) { char buf[64] = "\0"; void * pBuf = buf; SKVRecord rInfo; @@ -428,18 +428,18 @@ static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void } tsdbUpdateMFileMagic(pMFile, POINTER_SHIFT(cont, contLen - sizeof(TSCKSUM))); - if (!updateMeta) { - pMFile->info.nRecords++; - return 0; - } - SKVRecord *pRecord = taosHashGet(pfs->metaCache, (void *)&uid, sizeof(uid)); + SHashObj* cache = compact ? pfs->metaCacheComp : pfs->metaCache; + + pMFile->info.nRecords++; + + SKVRecord *pRecord = taosHashGet(cache, (void *)&uid, sizeof(uid)); if (pRecord != NULL) { pMFile->info.tombSize += (pRecord->size + sizeof(SKVRecord)); } else { pMFile->info.nRecords++; } - taosHashPut(pfs->metaCache, (void *)(&uid), sizeof(uid), (void *)(&rInfo), sizeof(rInfo)); + taosHashPut(cache, (void *)(&uid), sizeof(uid), (void *)(&rInfo), sizeof(rInfo)); return 0; } @@ -517,6 +517,13 @@ static int tsdbCompactMetaFile(STsdbRepo *pRepo, STsdbFS *pfs, SMFile *pMFile) { goto _err; } + // init Comp + assert(pfs->metaCacheComp == NULL); + pfs->metaCacheComp = taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + if (pfs->metaCacheComp == NULL) { + goto _err; + } + pRecord = taosHashIterate(pfs->metaCache, NULL); while (pRecord) { if (tsdbSeekMFile(pMFile, pRecord->offset + sizeof(SKVRecord), SEEK_SET) < 0) { @@ -545,7 +552,7 @@ static int tsdbCompactMetaFile(STsdbRepo *pRepo, STsdbFS *pfs, SMFile *pMFile) { goto _err; } - if (tsdbUpdateMetaRecord(pfs, &mf, pRecord->uid, pBuf, (int)pRecord->size, false) < 0) { + if (tsdbUpdateMetaRecord(pfs, &mf, pRecord->uid, pBuf, (int)pRecord->size, true) < 0) { tsdbError("vgId:%d failed to update META record, uid %" PRIu64 " since %s", REPO_ID(pRepo), pRecord->uid, tstrerror(terrno)); goto _err; @@ -569,9 +576,15 @@ _err: // update current meta file info pfs->nstatus->pmf = NULL; tsdbUpdateMFile(pfs, &mf); + + taosHashCleanup(pfs->metaCache); + pfs->metaCache = pfs->metaCacheComp; + pfs->metaCacheComp = NULL; } else { // remove meta.tmp file remove(mf.f.aname); + taosHashCleanup(pfs->metaCacheComp); + pfs->metaCacheComp = NULL; } tfree(pBuf); diff --git a/src/tsdb/src/tsdbFS.c b/src/tsdb/src/tsdbFS.c index 63f89c1957..a3d6c59f72 100644 --- a/src/tsdb/src/tsdbFS.c +++ b/src/tsdb/src/tsdbFS.c @@ -215,6 +215,7 @@ STsdbFS *tsdbNewFS(STsdbCfg *pCfg) { } pfs->intxn = false; + pfs->metaCacheComp = NULL; pfs->nstatus = tsdbNewFSStatus(maxFSet); if (pfs->nstatus == NULL) {