From b02219027916e9f55f8564c78f2723f465112943 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 29 Jul 2021 03:27:58 +0800 Subject: [PATCH 1/4] [TD-5664] add more state of compact --- src/inc/tsdb.h | 2 +- src/tsdb/inc/tsdbint.h | 4 ++-- src/tsdb/src/tsdbCompact.c | 8 +++++--- src/tsdb/src/tsdbMain.c | 4 ++-- src/vnode/src/vnodeMgmt.c | 2 +- 5 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index 79d9029dbc..7880dc43b2 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -94,7 +94,7 @@ STsdbRepo *tsdbOpenRepo(STsdbCfg *pCfg, STsdbAppH *pAppH); int tsdbCloseRepo(STsdbRepo *repo, int toCommit); int32_t tsdbConfigRepo(STsdbRepo *repo, STsdbCfg *pCfg); int tsdbGetState(STsdbRepo *repo); -bool tsdbInCompact(STsdbRepo *repo); +int8_t tsdbGetCompactState(STsdbRepo *repo); // --------- TSDB TABLE DEFINITION typedef struct { uint64_t uid; // the unique table ID diff --git a/src/tsdb/inc/tsdbint.h b/src/tsdb/inc/tsdbint.h index dd43e39310..84c7ba4e4b 100644 --- a/src/tsdb/inc/tsdbint.h +++ b/src/tsdb/inc/tsdbint.h @@ -92,7 +92,7 @@ struct STsdbRepo { pthread_mutex_t mutex; bool repoLocked; int32_t code; // Commit code - bool inCompact; // is in compact process? + int8_t compactState; // compact state: inCompact/noCompact/waitingCompact? }; #define REPO_ID(r) (r)->config.tsdbId @@ -139,4 +139,4 @@ static FORCE_INLINE int tsdbGetNextMaxTables(int tid) { } #endif -#endif /* _TD_TSDB_INT_H_ */ \ No newline at end of file +#endif /* _TD_TSDB_INT_H_ */ diff --git a/src/tsdb/src/tsdbCompact.c b/src/tsdb/src/tsdbCompact.c index 5211ee3c61..a85fdc25c5 100644 --- a/src/tsdb/src/tsdbCompact.c +++ b/src/tsdb/src/tsdbCompact.c @@ -58,6 +58,7 @@ static int tsdbCompactFSetImpl(SCompactH *pComph); static int tsdbWriteBlockToRightFile(SCompactH *pComph, STable *pTable, SDataCols *pDataCols, void **ppBuf, void **ppCBuf); +enum {TSDB_NO_COMPACT, TSDB_IN_COMPACT, TSDB_WAITING_COMPACT}; int tsdbCompact(STsdbRepo *pRepo) { return tsdbAsyncCompact(pRepo); } void *tsdbCompactImpl(STsdbRepo *pRepo) { @@ -89,16 +90,17 @@ _err: } static int tsdbAsyncCompact(STsdbRepo *pRepo) { + pRepo->compactState = TSDB_WAITING_COMPACT; tsem_wait(&(pRepo->readyToCommit)); return tsdbScheduleCommit(pRepo, COMPACT_REQ); } static void tsdbStartCompact(STsdbRepo *pRepo) { - ASSERT(!pRepo->inCompact); + assert(pRepo->compactState != TSDB_IN_COMPACT); tsdbInfo("vgId:%d start to compact!", REPO_ID(pRepo)); tsdbStartFSTxn(pRepo, 0, 0); pRepo->code = TSDB_CODE_SUCCESS; - pRepo->inCompact = true; + pRepo->compactState = TSDB_IN_COMPACT; } static void tsdbEndCompact(STsdbRepo *pRepo, int eno) { @@ -107,7 +109,7 @@ static void tsdbEndCompact(STsdbRepo *pRepo, int eno) { } else { tsdbEndFSTxn(pRepo); } - pRepo->inCompact = false; + pRepo->compactState = TSDB_NO_COMPACT; tsdbInfo("vgId:%d compact over, %s", REPO_ID(pRepo), (eno == TSDB_CODE_SUCCESS) ? "succeed" : "failed"); tsem_post(&(pRepo->readyToCommit)); } diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index f3a7c4b7ee..44460a7db3 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -197,7 +197,7 @@ STsdbRepoInfo *tsdbGetStatus(STsdbRepo *pRepo) { return NULL; } int tsdbGetState(STsdbRepo *repo) { return repo->state; } -bool tsdbInCompact(STsdbRepo *repo) { return repo->inCompact; } +int8_t tsdbGetCompactState(STsdbRepo *repo) { return (int8_t)(repo->compactState); } void tsdbReportStat(void *repo, int64_t *totalPoints, int64_t *totalStorage, int64_t *compStorage) { ASSERT(repo != NULL); @@ -537,7 +537,7 @@ static STsdbRepo *tsdbNewRepo(STsdbCfg *pCfg, STsdbAppH *pAppH) { pRepo->state = TSDB_STATE_OK; pRepo->code = TSDB_CODE_SUCCESS; - pRepo->inCompact = false; + pRepo->compactState = 0; pRepo->config = *pCfg; if (pAppH) { pRepo->appH = *pAppH; diff --git a/src/vnode/src/vnodeMgmt.c b/src/vnode/src/vnodeMgmt.c index e14b5a385e..7e427988b5 100644 --- a/src/vnode/src/vnodeMgmt.c +++ b/src/vnode/src/vnodeMgmt.c @@ -160,7 +160,7 @@ static void vnodeBuildVloadMsg(SVnodeObj *pVnode, SStatusMsg *pStatus) { pLoad->status = pVnode->status; pLoad->role = pVnode->role; pLoad->replica = pVnode->syncCfg.replica; - pLoad->compact = (pVnode->tsdb != NULL) && tsdbInCompact(pVnode->tsdb) ? 1 : 0; + pLoad->compact = (pVnode->tsdb != NULL) ? tsdbGetCompactState(pVnode->tsdb) : 0; } int32_t vnodeGetVnodeList(int32_t vnodeList[], int32_t *numOfVnodes) { From f25a01f83a2d4277c2adfb37cfa5b6f838d60573 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 29 Jul 2021 05:00:30 +0800 Subject: [PATCH 2/4] [TD-5664] add more state of compact --- src/tsdb/src/tsdbCompact.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/tsdb/src/tsdbCompact.c b/src/tsdb/src/tsdbCompact.c index 6ff0693d46..98888924ec 100644 --- a/src/tsdb/src/tsdbCompact.c +++ b/src/tsdb/src/tsdbCompact.c @@ -90,6 +90,10 @@ _err: } static int tsdbAsyncCompact(STsdbRepo *pRepo) { + if (pRepo->compactState != TSDB_NO_COMPACT) { + tsdbInfo("vgId:%d not compact tsdb again", REPO_ID(pRepo)); + return 0; + } pRepo->compactState = TSDB_WAITING_COMPACT; tsem_wait(&(pRepo->readyToCommit)); return tsdbScheduleCommit(pRepo, COMPACT_REQ); From 95f304e4762064d24cb3964c84d86517694f3b69 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Thu, 29 Jul 2021 11:49:18 +0800 Subject: [PATCH 3/4] [TD-5664] add more state of compact --- src/tsdb/src/tsdbCompact.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tsdb/src/tsdbCompact.c b/src/tsdb/src/tsdbCompact.c index 98888924ec..62f9e41119 100644 --- a/src/tsdb/src/tsdbCompact.c +++ b/src/tsdb/src/tsdbCompact.c @@ -58,7 +58,7 @@ static int tsdbCompactFSetImpl(SCompactH *pComph); static int tsdbWriteBlockToRightFile(SCompactH *pComph, STable *pTable, SDataCols *pDataCols, void **ppBuf, void **ppCBuf); -enum {TSDB_NO_COMPACT, TSDB_IN_COMPACT, TSDB_WAITING_COMPACT}; +enum { TSDB_NO_COMPACT, TSDB_IN_COMPACT, TSDB_WAITING_COMPACT}; int tsdbCompact(STsdbRepo *pRepo) { return tsdbAsyncCompact(pRepo); } void *tsdbCompactImpl(STsdbRepo *pRepo) { @@ -91,7 +91,7 @@ _err: static int tsdbAsyncCompact(STsdbRepo *pRepo) { if (pRepo->compactState != TSDB_NO_COMPACT) { - tsdbInfo("vgId:%d not compact tsdb again", REPO_ID(pRepo)); + tsdbInfo("vgId:%d not compact tsdb again ", REPO_ID(pRepo)); return 0; } pRepo->compactState = TSDB_WAITING_COMPACT; From 86a6a87666d0437ae774595941a3d4bda2a78983 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Mon, 2 Aug 2021 12:57:20 +0800 Subject: [PATCH 4/4] [TD-5618] fix distinct(tbname) error --- src/query/src/qExecutor.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 3f6df2ec07..93a2535d56 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -6661,19 +6661,20 @@ static SSDataBlock* hashDistinct(void* param, bool* newgroup) { if (isNull(val, type)) { continue; } - + char* p = val; size_t keyLen = 0; if (IS_VAR_DATA_TYPE(pOperator->pExpr->base.colType)) { tstr* var = (tstr*)(val); + p = var->data; keyLen = varDataLen(var); } else { keyLen = bytes; } int dummy; - void* res = taosHashGet(pInfo->pSet, val, keyLen); + void* res = taosHashGet(pInfo->pSet, p, keyLen); if (res == NULL) { - taosHashPut(pInfo->pSet, val, keyLen, &dummy, sizeof(dummy)); + taosHashPut(pInfo->pSet, p, keyLen, &dummy, sizeof(dummy)); char* start = pResultColInfoData->pData + bytes * pInfo->pRes->info.rows; memcpy(start, val, bytes); pRes->info.rows += 1;