From 53cc394523df2386844a124544940109b04d597c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 11 Jun 2020 02:11:51 +0000 Subject: [PATCH 01/65] TD-353 --- src/inc/tsdb.h | 46 ++++++++++++++++++------------------ src/tsdb/inc/tsdbMain.h | 14 +++++------ src/tsdb/src/tsdbCache.c | 8 +++---- src/tsdb/src/tsdbMain.c | 44 +++++++++++++++++----------------- src/tsdb/src/tsdbMeta.c | 8 +++---- src/tsdb/src/tsdbRead.c | 12 +++++----- src/tsdb/tests/tsdbTests.cpp | 4 ++-- 7 files changed, 68 insertions(+), 68 deletions(-) diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index add85d9438..f9b048f495 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -72,19 +72,19 @@ typedef struct { int64_t pointsWritten; // total data points written } STsdbStat; -typedef void TsdbRepoT; // use void to hide implementation details from outside +typedef void TSDB_REPO_T; // use void to hide implementation details from outside void tsdbSetDefaultCfg(STsdbCfg *pCfg); STsdbCfg *tsdbCreateDefaultCfg(); void tsdbFreeCfg(STsdbCfg *pCfg); -STsdbCfg *tsdbGetCfg(const TsdbRepoT *repo); +STsdbCfg *tsdbGetCfg(const TSDB_REPO_T *repo); // --------- TSDB REPOSITORY DEFINITION int tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg, void *limiter); -int32_t tsdbDropRepo(TsdbRepoT *repo); -TsdbRepoT *tsdbOpenRepo(char *rootDir, STsdbAppH *pAppH); -int32_t tsdbCloseRepo(TsdbRepoT *repo, int toCommit); -int32_t tsdbConfigRepo(TsdbRepoT *repo, STsdbCfg *pCfg); +int32_t tsdbDropRepo(TSDB_REPO_T *repo); +TSDB_REPO_T *tsdbOpenRepo(char *rootDir, STsdbAppH *pAppH); +int32_t tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit); +int32_t tsdbConfigRepo(TSDB_REPO_T *repo, STsdbCfg *pCfg); // --------- TSDB TABLE DEFINITION typedef struct { @@ -116,18 +116,18 @@ int tsdbTableSetSName(STableCfg *config, char *sname, bool dup); int tsdbTableSetStreamSql(STableCfg *config, char *sql, bool dup); void tsdbClearTableCfg(STableCfg *config); -void* tsdbGetTableTagVal(TsdbRepoT* repo, const STableId* id, int32_t colId, int16_t type, int16_t bytes); -char* tsdbGetTableName(TsdbRepoT *repo, const STableId *id); +void* tsdbGetTableTagVal(TSDB_REPO_T* repo, const STableId* id, int32_t colId, int16_t type, int16_t bytes); +char* tsdbGetTableName(TSDB_REPO_T *repo, const STableId *id); STableCfg *tsdbCreateTableCfgFromMsg(SMDCreateTableMsg *pMsg); -int tsdbCreateTable(TsdbRepoT *repo, STableCfg *pCfg); -int tsdbDropTable(TsdbRepoT *pRepo, STableId tableId); -int tsdbAlterTable(TsdbRepoT *repo, STableCfg *pCfg); -int tsdbUpdateTagValue(TsdbRepoT *repo, SUpdateTableTagValMsg *pMsg); -TSKEY tsdbGetTableLastKey(TsdbRepoT *repo, uint64_t uid); -void tsdbStartStream(TsdbRepoT *repo); +int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg); +int tsdbDropTable(TSDB_REPO_T *pRepo, STableId tableId); +int tsdbAlterTable(TSDB_REPO_T *repo, STableCfg *pCfg); +int tsdbUpdateTagValue(TSDB_REPO_T *repo, SUpdateTableTagValMsg *pMsg); +TSKEY tsdbGetTableLastKey(TSDB_REPO_T *repo, uint64_t uid); +void tsdbStartStream(TSDB_REPO_T *repo); -uint32_t tsdbGetFileInfo(TsdbRepoT *repo, char *name, uint32_t *index, uint32_t eindex, int32_t *size); +uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_t eindex, int32_t *size); // the TSDB repository info typedef struct STsdbRepoInfo { @@ -137,7 +137,7 @@ typedef struct STsdbRepoInfo { int64_t tsdbTotalDiskSize; // the total disk size taken by this TSDB repository // TODO: Other informations to add } STsdbRepoInfo; -STsdbRepoInfo *tsdbGetStatus(TsdbRepoT *pRepo); +STsdbRepoInfo *tsdbGetStatus(TSDB_REPO_T *pRepo); // the meter information report structure typedef struct { @@ -146,7 +146,7 @@ typedef struct { int64_t tableTotalDataSize; // In bytes int64_t tableTotalDiskSize; // In bytes } STableInfo; -STableInfo *tsdbGetTableInfo(TsdbRepoT *pRepo, STableId tid); +STableInfo *tsdbGetTableInfo(TSDB_REPO_T *pRepo, STableId tid); // -- FOR INSERT DATA /** @@ -156,7 +156,7 @@ STableInfo *tsdbGetTableInfo(TsdbRepoT *pRepo, STableId tid); * * @return the number of points inserted, -1 for failure and the error number is set */ -int32_t tsdbInsertData(TsdbRepoT *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg * pRsp) ; +int32_t tsdbInsertData(TSDB_REPO_T *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg * pRsp) ; // -- FOR QUERY TIME SERIES DATA @@ -199,7 +199,7 @@ typedef void *TsdbPosT; * @param qinfo query info handle from query processor * @return */ -TsdbQueryHandleT *tsdbQueryTables(TsdbRepoT *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupInfo, void* qinfo); +TsdbQueryHandleT *tsdbQueryTables(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupInfo, void* qinfo); /** * Get the last row of the given query time window for all the tables in STableGroupInfo object. @@ -211,11 +211,11 @@ TsdbQueryHandleT *tsdbQueryTables(TsdbRepoT *tsdb, STsdbQueryCond *pCond, STable * @param groupInfo tableId list. * @return */ -TsdbQueryHandleT tsdbQueryLastRow(TsdbRepoT *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupInfo, void* qinfo); +TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupInfo, void* qinfo); SArray* tsdbGetQueriedTableIdList(TsdbQueryHandleT *pHandle); -TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TsdbRepoT *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, void* qinfo); +TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TSDB_REPO_T *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, void* qinfo); /** * move to next block if exists @@ -293,7 +293,7 @@ SArray *tsdbGetTableList(TsdbQueryHandleT *pQueryHandle); * @param stableid. super table sid * @param pTagCond. tag query condition */ -int32_t tsdbQuerySTableByTagCond(TsdbRepoT *tsdb, uint64_t uid, const char *pTagCond, size_t len, +int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T *tsdb, uint64_t uid, const char *pTagCond, size_t len, int16_t tagNameRelType, const char *tbnameCond, STableGroupInfo *pGroupList, SColIndex *pColIndex, int32_t numOfCols); @@ -305,7 +305,7 @@ int32_t tsdbQuerySTableByTagCond(TsdbRepoT *tsdb, uint64_t uid, const char *pTag * @param pGroupInfo the generated result * @return */ -int32_t tsdbGetOneTableGroup(TsdbRepoT *tsdb, uint64_t uid, STableGroupInfo *pGroupInfo); +int32_t tsdbGetOneTableGroup(TSDB_REPO_T *tsdb, uint64_t uid, STableGroupInfo *pGroupInfo); /** * clean up the query handle diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index f52022d79d..d697ed7926 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -142,7 +142,7 @@ STSchema * tsdbGetTableTagSchema(STsdbMeta *pMeta, STable *pTable); #define TSDB_TABLE_OF_ID(pHandle, id) ((pHandle)->pTables)[id] #define TSDB_GET_TABLE_OF_NAME(pHandle, name) /* TODO */ -STsdbMeta *tsdbGetMeta(TsdbRepoT *pRepo); +STsdbMeta *tsdbGetMeta(TSDB_REPO_T *pRepo); STable *tsdbIsValidTableToInsert(STsdbMeta *pMeta, STableId tableId); // int32_t tsdbInsertRowToTableImpl(SSkipListNode *pNode, STable *pTable); @@ -177,10 +177,10 @@ typedef struct { STsdbCacheBlock *curBlock; SCacheMem * mem; SCacheMem * imem; - TsdbRepoT * pRepo; + TSDB_REPO_T * pRepo; } STsdbCache; -STsdbCache *tsdbInitCache(int cacheBlockSize, int totalBlocks, TsdbRepoT *pRepo); +STsdbCache *tsdbInitCache(int cacheBlockSize, int totalBlocks, TSDB_REPO_T *pRepo); void tsdbFreeCache(STsdbCache *pCache); void * tsdbAllocFromCache(STsdbCache *pCache, int bytes, TSKEY key); @@ -342,7 +342,7 @@ typedef struct { SCompCol cols[]; } SCompData; -STsdbFileH *tsdbGetFile(TsdbRepoT *pRepo); +STsdbFileH *tsdbGetFile(TSDB_REPO_T *pRepo); int tsdbCopyBlockDataInFile(SFile *pOutFile, SFile *pInFile, SCompInfo *pCompInfo, int idx, int isLast, SDataCols *pCols); @@ -404,9 +404,9 @@ typedef struct { int tsdbInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter); SSubmitBlk *tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter); -int32_t tsdbTriggerCommit(TsdbRepoT *repo); -int32_t tsdbLockRepo(TsdbRepoT *repo); -int32_t tsdbUnLockRepo(TsdbRepoT *repo); +int32_t tsdbTriggerCommit(TSDB_REPO_T *repo); +int32_t tsdbLockRepo(TSDB_REPO_T *repo); +int32_t tsdbUnLockRepo(TSDB_REPO_T *repo); typedef enum { TSDB_WRITE_HELPER, TSDB_READ_HELPER } tsdb_rw_helper_t; diff --git a/src/tsdb/src/tsdbCache.c b/src/tsdb/src/tsdbCache.c index 24476d8997..56e1b4e52a 100644 --- a/src/tsdb/src/tsdbCache.c +++ b/src/tsdb/src/tsdbCache.c @@ -22,7 +22,7 @@ static void tsdbFreeBlockList(SList *list); static void tsdbFreeCacheMem(SCacheMem *mem); static int tsdbAddCacheBlockToPool(STsdbCache *pCache); -STsdbCache *tsdbInitCache(int cacheBlockSize, int totalBlocks, TsdbRepoT *pRepo) { +STsdbCache *tsdbInitCache(int cacheBlockSize, int totalBlocks, TSDB_REPO_T *pRepo) { STsdbCache *pCache = (STsdbCache *)calloc(1, sizeof(STsdbCache)); if (pCache == NULL) return NULL; @@ -143,7 +143,7 @@ int tsdbAlterCacheTotalBlocks(STsdbRepo *pRepo, int totalBlocks) { STsdbCache *pCache = pRepo->tsdbCache; int oldNumOfBlocks = pCache->totalCacheBlocks; - tsdbLockRepo((TsdbRepoT *)pRepo); + tsdbLockRepo((TSDB_REPO_T *)pRepo); ASSERT(pCache->totalCacheBlocks != totalBlocks); @@ -153,7 +153,7 @@ int tsdbAlterCacheTotalBlocks(STsdbRepo *pRepo, int totalBlocks) { pCache->totalCacheBlocks = totalBlocks; for (int i = 0; i < blocksToAdd; i++) { if (tsdbAddCacheBlockToPool(pCache) < 0) { - tsdbUnLockRepo((TsdbRepoT *)pRepo); + tsdbUnLockRepo((TSDB_REPO_T *)pRepo); tsdbError("tsdbId:%d, failed to add cache block to cache pool", pRepo->config.tsdbId); return -1; } @@ -164,7 +164,7 @@ int tsdbAlterCacheTotalBlocks(STsdbRepo *pRepo, int totalBlocks) { } pRepo->config.totalBlocks = totalBlocks; - tsdbUnLockRepo((TsdbRepoT *)pRepo); + tsdbUnLockRepo((TSDB_REPO_T *)pRepo); tsdbTrace("vgId:%d, tsdb total cache blocks changed from %d to %d", pRepo->config.tsdbId, oldNumOfBlocks, totalBlocks); return 0; } diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 5526ad0d6e..aa3be47164 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -26,7 +26,7 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg); static int32_t tsdbSetRepoEnv(STsdbRepo *pRepo); static int32_t tsdbDestroyRepoEnv(STsdbRepo *pRepo); // static int tsdbOpenMetaFile(char *tsdbDir); -static int32_t tsdbInsertDataToTable(TsdbRepoT *repo, SSubmitBlk *pBlock, TSKEY now, int * affectedrows); +static int32_t tsdbInsertDataToTable(TSDB_REPO_T *repo, SSubmitBlk *pBlock, TSKEY now, int * affectedrows); static int32_t tsdbRestoreCfg(STsdbRepo *pRepo, STsdbCfg *pCfg); static int32_t tsdbGetDataDirName(STsdbRepo *pRepo, char *fname); static void * tsdbCommitData(void *arg); @@ -78,7 +78,7 @@ void tsdbFreeCfg(STsdbCfg *pCfg) { if (pCfg != NULL) free(pCfg); } -STsdbCfg *tsdbGetCfg(const TsdbRepoT *repo) { +STsdbCfg *tsdbGetCfg(const TSDB_REPO_T *repo) { assert(repo != NULL); return &((STsdbRepo*)repo)->config; } @@ -134,7 +134,7 @@ int32_t tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg, void *limiter /* TODO */) * * @return 0 for success, -1 for failure and the error number is set */ -int32_t tsdbDropRepo(TsdbRepoT *repo) { +int32_t tsdbDropRepo(TSDB_REPO_T *repo) { STsdbRepo *pRepo = (STsdbRepo *)repo; int id = pRepo->config.tsdbId; @@ -192,7 +192,7 @@ _err: * * @return a TSDB repository handle on success, NULL for failure and the error number is set */ -TsdbRepoT *tsdbOpenRepo(char *rootDir, STsdbAppH *pAppH) { +TSDB_REPO_T *tsdbOpenRepo(char *rootDir, STsdbAppH *pAppH) { char dataDir[128] = "\0"; if (access(rootDir, F_OK | W_OK | R_OK) < 0) { return NULL; @@ -215,7 +215,7 @@ TsdbRepoT *tsdbOpenRepo(char *rootDir, STsdbAppH *pAppH) { return NULL; } - pRepo->tsdbCache = tsdbInitCache(pRepo->config.cacheBlockSize, pRepo->config.totalBlocks, (TsdbRepoT *)pRepo); + pRepo->tsdbCache = tsdbInitCache(pRepo->config.cacheBlockSize, pRepo->config.totalBlocks, (TSDB_REPO_T *)pRepo); if (pRepo->tsdbCache == NULL) { tsdbFreeMeta(pRepo->tsdbMeta); free(pRepo->rootDir); @@ -246,7 +246,7 @@ TsdbRepoT *tsdbOpenRepo(char *rootDir, STsdbAppH *pAppH) { pRepo->state = TSDB_REPO_STATE_ACTIVE; tsdbTrace("vgId:%d, open tsdb repository successfully!", pRepo->config.tsdbId); - return (TsdbRepoT *)pRepo; + return (TSDB_REPO_T *)pRepo; } // static int32_t tsdbFlushCache(STsdbRepo *pRepo) { @@ -261,7 +261,7 @@ TsdbRepoT *tsdbOpenRepo(char *rootDir, STsdbAppH *pAppH) { * * @return 0 for success, -1 for failure and the error number is set */ -int32_t tsdbCloseRepo(TsdbRepoT *repo, int toCommit) { +int32_t tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit) { STsdbRepo *pRepo = (STsdbRepo *)repo; if (pRepo == NULL) return 0; int id = pRepo->config.tsdbId; @@ -310,7 +310,7 @@ int32_t tsdbCloseRepo(TsdbRepoT *repo, int toCommit) { * * @return 0 for success, -1 for failure and the error number is set */ -int32_t tsdbConfigRepo(TsdbRepoT *repo, STsdbCfg *pCfg) { +int32_t tsdbConfigRepo(TSDB_REPO_T *repo, STsdbCfg *pCfg) { STsdbRepo *pRepo = (STsdbRepo *)repo; STsdbCfg * pRCfg = &pRepo->config; @@ -346,7 +346,7 @@ int32_t tsdbConfigRepo(TsdbRepoT *repo, STsdbCfg *pCfg) { return TSDB_CODE_SUCCESS; } -int32_t tsdbTriggerCommit(TsdbRepoT *repo) { +int32_t tsdbTriggerCommit(TSDB_REPO_T *repo) { STsdbRepo *pRepo = (STsdbRepo *)repo; if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_START); @@ -381,12 +381,12 @@ int32_t tsdbTriggerCommit(TsdbRepoT *repo) { return 0; } -int32_t tsdbLockRepo(TsdbRepoT *repo) { +int32_t tsdbLockRepo(TSDB_REPO_T *repo) { STsdbRepo *pRepo = (STsdbRepo *)repo; return pthread_mutex_lock(&(pRepo->mutex)); } -int32_t tsdbUnLockRepo(TsdbRepoT *repo) { +int32_t tsdbUnLockRepo(TSDB_REPO_T *repo) { STsdbRepo *pRepo = (STsdbRepo *)repo; return pthread_mutex_unlock(&(pRepo->mutex)); } @@ -399,17 +399,17 @@ int32_t tsdbUnLockRepo(TsdbRepoT *repo) { * @return a info struct handle on success, NULL for failure and the error number is set. The upper * layers should free the info handle themselves or memory leak will occur */ -STsdbRepoInfo *tsdbGetStatus(TsdbRepoT *pRepo) { +STsdbRepoInfo *tsdbGetStatus(TSDB_REPO_T *pRepo) { // TODO return NULL; } -int tsdbAlterTable(TsdbRepoT *pRepo, STableCfg *pCfg) { +int tsdbAlterTable(TSDB_REPO_T *pRepo, STableCfg *pCfg) { // TODO return 0; } -int tsdbUpdateTagValue(TsdbRepoT *repo, SUpdateTableTagValMsg *pMsg) { +int tsdbUpdateTagValue(TSDB_REPO_T *repo, SUpdateTableTagValMsg *pMsg) { STsdbRepo *pRepo = (STsdbRepo *)repo; STsdbMeta *pMeta = pRepo->tsdbMeta; int16_t tversion = htons(pMsg->tversion); @@ -464,7 +464,7 @@ int tsdbUpdateTagValue(TsdbRepoT *repo, SUpdateTableTagValMsg *pMsg) { return TSDB_CODE_SUCCESS; } -TSKEY tsdbGetTableLastKey(TsdbRepoT *repo, uint64_t uid) { +TSKEY tsdbGetTableLastKey(TSDB_REPO_T *repo, uint64_t uid) { STsdbRepo *pRepo = (STsdbRepo *)repo; STable *pTable = tsdbGetTableByUid(pRepo->tsdbMeta, uid); @@ -473,7 +473,7 @@ TSKEY tsdbGetTableLastKey(TsdbRepoT *repo, uint64_t uid) { return TSDB_GET_TABLE_LAST_KEY(pTable); } -void tsdbStartStream(TsdbRepoT *repo) { +void tsdbStartStream(TSDB_REPO_T *repo) { STsdbRepo *pRepo = (STsdbRepo *)repo; STsdbMeta *pMeta = pRepo->tsdbMeta; @@ -485,13 +485,13 @@ void tsdbStartStream(TsdbRepoT *repo) { } } -STableInfo *tsdbGetTableInfo(TsdbRepoT *pRepo, STableId tableId) { +STableInfo *tsdbGetTableInfo(TSDB_REPO_T *pRepo, STableId tableId) { // TODO return NULL; } // TODO: need to return the number of data inserted -int32_t tsdbInsertData(TsdbRepoT *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg * pRsp) { +int32_t tsdbInsertData(TSDB_REPO_T *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg * pRsp) { SSubmitMsgIter msgIter; STsdbRepo *pRepo = (STsdbRepo *)repo; @@ -698,12 +698,12 @@ SSubmitBlk *tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter) { return pBlock; } -STsdbMeta* tsdbGetMeta(TsdbRepoT* pRepo) { +STsdbMeta* tsdbGetMeta(TSDB_REPO_T* pRepo) { STsdbRepo *tsdb = (STsdbRepo *)pRepo; return tsdb->tsdbMeta; } -STsdbFileH* tsdbGetFile(TsdbRepoT* pRepo) { +STsdbFileH* tsdbGetFile(TSDB_REPO_T* pRepo) { STsdbRepo* tsdb = (STsdbRepo*) pRepo; return tsdb->tsdbFileH; } @@ -950,7 +950,7 @@ static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable return 0; } -static int32_t tsdbInsertDataToTable(TsdbRepoT *repo, SSubmitBlk *pBlock, TSKEY now, int32_t *affectedrows) { +static int32_t tsdbInsertDataToTable(TSDB_REPO_T *repo, SSubmitBlk *pBlock, TSKEY now, int32_t *affectedrows) { STsdbRepo *pRepo = (STsdbRepo *)repo; STsdbMeta *pMeta = pRepo->tsdbMeta; int64_t points = 0; @@ -1318,7 +1318,7 @@ static void tsdbAlterMaxTables(STsdbRepo *pRepo, int32_t maxTables) { } #define TSDB_META_FILE_INDEX 10000000 -uint32_t tsdbGetFileInfo(TsdbRepoT *repo, char *name, uint32_t *index, uint32_t eindex, int32_t *size) { +uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_t eindex, int32_t *size) { STsdbRepo *pRepo = (STsdbRepo *)repo; // STsdbMeta *pMeta = pRepo->tsdbMeta; STsdbFileH *pFileH = pRepo->tsdbFileH; diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 8eb259d38c..70314cb33b 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -280,7 +280,7 @@ STSchema * tsdbGetTableTagSchema(STsdbMeta *pMeta, STable *pTable) { } } -void* tsdbGetTableTagVal(TsdbRepoT* repo, const STableId* id, int32_t colId, int16_t type, int16_t bytes) { +void* tsdbGetTableTagVal(TSDB_REPO_T* repo, const STableId* id, int32_t colId, int16_t type, int16_t bytes) { STsdbMeta* pMeta = tsdbGetMeta(repo); STable* pTable = tsdbGetTableByUid(pMeta, id->uid); @@ -300,7 +300,7 @@ void* tsdbGetTableTagVal(TsdbRepoT* repo, const STableId* id, int32_t colId, int return val; } -char* tsdbGetTableName(TsdbRepoT *repo, const STableId* id) { +char* tsdbGetTableName(TSDB_REPO_T *repo, const STableId* id) { STsdbMeta* pMeta = tsdbGetMeta(repo); STable* pTable = tsdbGetTableByUid(pMeta, id->uid); @@ -438,7 +438,7 @@ int tsdbUpdateTable(STsdbMeta *pMeta, STable *pTable, STableCfg *pCfg) { return TSDB_CODE_SUCCESS; } -int tsdbCreateTable(TsdbRepoT *repo, STableCfg *pCfg) { +int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg) { STsdbRepo *pRepo = (STsdbRepo *)repo; STsdbMeta *pMeta = pRepo->tsdbMeta; @@ -581,7 +581,7 @@ _err: } // int32_t tsdbDropTableImpl(STsdbMeta *pMeta, STableId tableId) { -int tsdbDropTable(TsdbRepoT *repo, STableId tableId) { +int tsdbDropTable(TSDB_REPO_T *repo, STableId tableId) { STsdbRepo *pRepo = (STsdbRepo *)repo; if (pRepo == NULL) return -1; diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index d3bc45e2e4..70ee6d1c3e 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -135,7 +135,7 @@ static void tsdbInitCompBlockLoadInfo(SLoadCompBlockInfo* pCompBlockLoadInfo) { pCompBlockLoadInfo->fileId = -1; } -TsdbQueryHandleT* tsdbQueryTables(TsdbRepoT* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, void* qinfo) { +TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, void* qinfo) { // todo 1. filter not exist table // todo 2. add the reference count for each table that is involved in query @@ -203,7 +203,7 @@ TsdbQueryHandleT* tsdbQueryTables(TsdbRepoT* tsdb, STsdbQueryCond* pCond, STable return (TsdbQueryHandleT) pQueryHandle; } -TsdbQueryHandleT tsdbQueryLastRow(TsdbRepoT *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, void* qinfo) { +TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, void* qinfo) { STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo); pQueryHandle->type = TSDB_QUERY_TYPE_LAST; @@ -229,7 +229,7 @@ SArray* tsdbGetQueriedTableIdList(TsdbQueryHandleT *pHandle) { return res; } -TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TsdbRepoT *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, void* qinfo) { +TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TSDB_REPO_T *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, void* qinfo) { STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo); pQueryHandle->type = TSDB_QUERY_TYPE_EXTERNAL; @@ -1990,7 +1990,7 @@ void createTableGroupImpl(SArray* pGroups, SArray* pTableIdList, size_t numOfTab } SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pCols, int32_t numOfOrderCols, - TsdbRepoT* tsdb) { + TSDB_REPO_T* tsdb) { assert(pTableList != NULL); SArray* pTableGroup = taosArrayInit(1, POINTER_BYTES); @@ -2102,7 +2102,7 @@ static int32_t doQueryTableList(STable* pSTable, SArray* pRes, tExprNode* pExpr) return TSDB_CODE_SUCCESS; } -int32_t tsdbQuerySTableByTagCond(TsdbRepoT* tsdb, uint64_t uid, const char* pTagCond, size_t len, +int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, const char* pTagCond, size_t len, int16_t tagNameRelType, const char* tbnameCond, STableGroupInfo* pGroupInfo, SColIndex* pColIndex, int32_t numOfCols) { STable* pTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid); @@ -2176,7 +2176,7 @@ int32_t tsdbQuerySTableByTagCond(TsdbRepoT* tsdb, uint64_t uid, const char* pTag return ret; } -int32_t tsdbGetOneTableGroup(TsdbRepoT* tsdb, uint64_t uid, STableGroupInfo* pGroupInfo) { +int32_t tsdbGetOneTableGroup(TSDB_REPO_T* tsdb, uint64_t uid, STableGroupInfo* pGroupInfo) { STable* pTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid); if (pTable == NULL) { return TSDB_CODE_TDB_INVALID_TABLE_ID; diff --git a/src/tsdb/tests/tsdbTests.cpp b/src/tsdb/tests/tsdbTests.cpp index e26caa4ae7..0c42ca1ee3 100644 --- a/src/tsdb/tests/tsdbTests.cpp +++ b/src/tsdb/tests/tsdbTests.cpp @@ -13,7 +13,7 @@ static double getCurTime() { } typedef struct { - TsdbRepoT *pRepo; + TSDB_REPO_T *pRepo; bool isAscend; int tid; uint64_t uid; @@ -136,7 +136,7 @@ TEST(TsdbTest, createRepo) { tsdbSetDefaultCfg(&config); ASSERT_EQ(tsdbCreateRepo("/home/ubuntu/work/ttest/vnode0", &config, NULL), 0); - TsdbRepoT *pRepo = tsdbOpenRepo("/home/ubuntu/work/ttest/vnode0", NULL); + TSDB_REPO_T *pRepo = tsdbOpenRepo("/home/ubuntu/work/ttest/vnode0", NULL); ASSERT_NE(pRepo, nullptr); // 2. Create a normal table From cca6f87a38df0397eedfd045a5745a6e0031fb86 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 11 Jun 2020 02:23:32 +0000 Subject: [PATCH 02/65] refactor --- src/inc/tsdb.h | 58 ++++++++---------- src/tsdb/src/tsdbMain.c | 118 ------------------------------------ src/tsdb/src/tsdbMeta.c | 121 ++++++++++++++++++++++++++++++++++++- src/vnode/src/vnodeWrite.c | 6 +- 4 files changed, 144 insertions(+), 159 deletions(-) diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index f9b048f495..6f2e48f801 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -19,11 +19,11 @@ #include #include -#include "tdataformat.h" -#include "tname.h" #include "taosdef.h" #include "taosmsg.h" #include "tarray.h" +#include "tdataformat.h" +#include "tname.h" #ifdef __cplusplus extern "C" { @@ -35,7 +35,7 @@ extern "C" { #define TSDB_INVALID_SUPER_TABLE_ID -1 #define TSDB_STATUS_COMMIT_START 1 -#define TSDB_STATUS_COMMIT_OVER 2 +#define TSDB_STATUS_COMMIT_OVER 2 // --------- TSDB APPLICATION HANDLE DEFINITION typedef struct { @@ -53,9 +53,9 @@ typedef struct { int32_t tsdbId; int32_t cacheBlockSize; int32_t totalBlocks; - int32_t maxTables; // maximum number of tables this repository can have - int32_t daysPerFile; // day per file sharding policy - int32_t keep; // day of data to keep + int32_t maxTables; // maximum number of tables this repository can have + int32_t daysPerFile; // day per file sharding policy + int32_t keep; // day of data to keep int32_t keep1; int32_t keep2; int32_t minRowsPerFileBlock; // minimum rows per file block @@ -74,17 +74,14 @@ typedef struct { typedef void TSDB_REPO_T; // use void to hide implementation details from outside -void tsdbSetDefaultCfg(STsdbCfg *pCfg); -STsdbCfg *tsdbCreateDefaultCfg(); -void tsdbFreeCfg(STsdbCfg *pCfg); STsdbCfg *tsdbGetCfg(const TSDB_REPO_T *repo); // --------- TSDB REPOSITORY DEFINITION -int tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg, void *limiter); -int32_t tsdbDropRepo(TSDB_REPO_T *repo); +int tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg, void *limiter); +int32_t tsdbDropRepo(TSDB_REPO_T *repo); TSDB_REPO_T *tsdbOpenRepo(char *rootDir, STsdbAppH *pAppH); -int32_t tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit); -int32_t tsdbConfigRepo(TSDB_REPO_T *repo, STsdbCfg *pCfg); +int32_t tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit); +int32_t tsdbConfigRepo(TSDB_REPO_T *repo, STsdbCfg *pCfg); // --------- TSDB TABLE DEFINITION typedef struct { @@ -106,23 +103,14 @@ typedef struct { char * sql; } STableCfg; -int tsdbInitTableCfg(STableCfg *config, ETableType type, uint64_t uid, int32_t tid); -int tsdbTableSetSuperUid(STableCfg *config, uint64_t uid); -int tsdbTableSetSchema(STableCfg *config, STSchema *pSchema, bool dup); -int tsdbTableSetTagSchema(STableCfg *config, STSchema *pSchema, bool dup); -int tsdbTableSetTagValue(STableCfg *config, SKVRow row, bool dup); -int tsdbTableSetName(STableCfg *config, char *name, bool dup); -int tsdbTableSetSName(STableCfg *config, char *sname, bool dup); -int tsdbTableSetStreamSql(STableCfg *config, char *sql, bool dup); void tsdbClearTableCfg(STableCfg *config); -void* tsdbGetTableTagVal(TSDB_REPO_T* repo, const STableId* id, int32_t colId, int16_t type, int16_t bytes); -char* tsdbGetTableName(TSDB_REPO_T *repo, const STableId *id); +void * tsdbGetTableTagVal(TSDB_REPO_T *repo, const STableId *id, int32_t colId, int16_t type, int16_t bytes); +char * tsdbGetTableName(TSDB_REPO_T *repo, const STableId *id); STableCfg *tsdbCreateTableCfgFromMsg(SMDCreateTableMsg *pMsg); int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg); int tsdbDropTable(TSDB_REPO_T *pRepo, STableId tableId); -int tsdbAlterTable(TSDB_REPO_T *repo, STableCfg *pCfg); int tsdbUpdateTagValue(TSDB_REPO_T *repo, SUpdateTableTagValMsg *pMsg); TSKEY tsdbGetTableLastKey(TSDB_REPO_T *repo, uint64_t uid); void tsdbStartStream(TSDB_REPO_T *repo); @@ -156,7 +144,7 @@ STableInfo *tsdbGetTableInfo(TSDB_REPO_T *pRepo, STableId tid); * * @return the number of points inserted, -1 for failure and the error number is set */ -int32_t tsdbInsertData(TSDB_REPO_T *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg * pRsp) ; +int32_t tsdbInsertData(TSDB_REPO_T *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg *pRsp); // -- FOR QUERY TIME SERIES DATA @@ -164,10 +152,10 @@ typedef void *TsdbQueryHandleT; // Use void to hide implementation details // query condition to build vnode iterator typedef struct STsdbQueryCond { - STimeWindow twindow; - int32_t order; // desc|asc order to iterate the data block - int32_t numOfCols; - SColumnInfo *colList; + STimeWindow twindow; + int32_t order; // desc|asc order to iterate the data block + int32_t numOfCols; + SColumnInfo *colList; } STsdbQueryCond; typedef struct SDataBlockInfo { @@ -199,7 +187,7 @@ typedef void *TsdbPosT; * @param qinfo query info handle from query processor * @return */ -TsdbQueryHandleT *tsdbQueryTables(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupInfo, void* qinfo); +TsdbQueryHandleT *tsdbQueryTables(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupInfo, void *qinfo); /** * Get the last row of the given query time window for all the tables in STableGroupInfo object. @@ -207,15 +195,17 @@ TsdbQueryHandleT *tsdbQueryTables(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STab * all tables in this group. * * @param tsdb tsdb handle - * @param pCond query condition, including time window, result set order, and basic required columns for each block + * @param pCond query condition, including time window, result set order, and basic required columns for each + * block * @param groupInfo tableId list. * @return */ -TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupInfo, void* qinfo); +TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupInfo, void *qinfo); -SArray* tsdbGetQueriedTableIdList(TsdbQueryHandleT *pHandle); +SArray *tsdbGetQueriedTableIdList(TsdbQueryHandleT *pHandle); -TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TSDB_REPO_T *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, void* qinfo); +TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, + void *qinfo); /** * move to next block if exists diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index aa3be47164..0be07d6179 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -404,11 +404,6 @@ STsdbRepoInfo *tsdbGetStatus(TSDB_REPO_T *pRepo) { return NULL; } -int tsdbAlterTable(TSDB_REPO_T *pRepo, STableCfg *pCfg) { - // TODO - return 0; -} - int tsdbUpdateTagValue(TSDB_REPO_T *repo, SUpdateTableTagValMsg *pMsg) { STsdbRepo *pRepo = (STsdbRepo *)repo; STsdbMeta *pMeta = pRepo->tsdbMeta; @@ -511,119 +506,6 @@ int32_t tsdbInsertData(TSDB_REPO_T *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg * return code; } -/** - * Initialize a table configuration - */ -int tsdbInitTableCfg(STableCfg *config, ETableType type, uint64_t uid, int32_t tid) { - if (config == NULL) return -1; - if (type != TSDB_CHILD_TABLE && type != TSDB_NORMAL_TABLE && type != TSDB_STREAM_TABLE) return -1; - - memset((void *)config, 0, sizeof(STableCfg)); - - config->type = type; - config->superUid = TSDB_INVALID_SUPER_TABLE_ID; - config->tableId.uid = uid; - config->tableId.tid = tid; - config->name = NULL; - config->sql = NULL; - return 0; -} - -/** - * Set the super table UID of the created table - */ -int tsdbTableSetSuperUid(STableCfg *config, uint64_t uid) { - if (config->type != TSDB_CHILD_TABLE) return -1; - if (uid == TSDB_INVALID_SUPER_TABLE_ID) return -1; - - config->superUid = uid; - return 0; -} - -/** - * Set the table schema in the configuration - * @param config the configuration to set - * @param pSchema the schema to set - * @param dup use the schema directly or duplicate one for use - * - * @return 0 for success and -1 for failure - */ -int tsdbTableSetSchema(STableCfg *config, STSchema *pSchema, bool dup) { - if (dup) { - config->schema = tdDupSchema(pSchema); - } else { - config->schema = pSchema; - } - return 0; -} - -/** - * Set the table schema in the configuration - * @param config the configuration to set - * @param pSchema the schema to set - * @param dup use the schema directly or duplicate one for use - * - * @return 0 for success and -1 for failure - */ -int tsdbTableSetTagSchema(STableCfg *config, STSchema *pSchema, bool dup) { - if (config->type != TSDB_CHILD_TABLE) return -1; - - if (dup) { - config->tagSchema = tdDupSchema(pSchema); - } else { - config->tagSchema = pSchema; - } - return 0; -} - -int tsdbTableSetTagValue(STableCfg *config, SKVRow row, bool dup) { - if (config->type != TSDB_CHILD_TABLE) return -1; - - if (dup) { - config->tagValues = tdKVRowDup(row); - } else { - config->tagValues = row; - } - - return 0; -} - -int tsdbTableSetName(STableCfg *config, char *name, bool dup) { - if (dup) { - config->name = strdup(name); - if (config->name == NULL) return -1; - } else { - config->name = name; - } - - return 0; -} - -int tsdbTableSetSName(STableCfg *config, char *sname, bool dup) { - if (config->type != TSDB_CHILD_TABLE) return -1; - - if (dup) { - config->sname = strdup(sname); - if (config->sname == NULL) return -1; - } else { - config->sname = sname; - } - return 0; -} - -int tsdbTableSetStreamSql(STableCfg *config, char *sql, bool dup) { - if (config->type != TSDB_STREAM_TABLE) return -1; - - if (dup) { - config->sql = strdup(sql); - if (config->sql == NULL) return -1; - } else { - config->sql = sql; - } - - return 0; -} - void tsdbClearTableCfg(STableCfg *config) { if (config) { if (config->schema) tdFreeSchema(config->schema); diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 70314cb33b..d0343999c1 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -8,11 +8,18 @@ #define TSDB_SUPER_TABLE_SL_LEVEL 5 // TODO: may change here // #define TSDB_META_FILE_NAME "META" - static int tsdbFreeTable(STable *pTable); static int32_t tsdbCheckTableCfg(STableCfg *pCfg); static int tsdbAddTableToMeta(STsdbMeta *pMeta, STable *pTable, bool addIdx); static int tsdbRemoveTableFromMeta(STsdbMeta *pMeta, STable *pTable, bool rmFromIdx); +static int tsdbInitTableCfg(STableCfg *config, ETableType type, uint64_t uid, int32_t tid); +static int tsdbTableSetSuperUid(STableCfg *config, uint64_t uid); +static int tsdbTableSetSchema(STableCfg *config, STSchema *pSchema, bool dup); +static int tsdbTableSetTagSchema(STableCfg *config, STSchema *pSchema, bool dup); +static int tsdbTableSetTagValue(STableCfg *config, SKVRow row, bool dup); +static int tsdbTableSetName(STableCfg *config, char *name, bool dup); +static int tsdbTableSetSName(STableCfg *config, char *sname, bool dup); +static int tsdbTableSetStreamSql(STableCfg *config, char *sql, bool dup); /** * Encode a TSDB table object as a binary content @@ -796,4 +803,114 @@ int tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable) { char *getTSTupleKey(const void * data) { SDataRow row = (SDataRow)data; return POINTER_SHIFT(row, TD_DATA_ROW_HEAD_SIZE); -} \ No newline at end of file +} + +static int tsdbInitTableCfg(STableCfg *config, ETableType type, uint64_t uid, int32_t tid) { + if (config == NULL) return -1; + if (type != TSDB_CHILD_TABLE && type != TSDB_NORMAL_TABLE && type != TSDB_STREAM_TABLE) return -1; + + memset((void *)config, 0, sizeof(STableCfg)); + + config->type = type; + config->superUid = TSDB_INVALID_SUPER_TABLE_ID; + config->tableId.uid = uid; + config->tableId.tid = tid; + config->name = NULL; + config->sql = NULL; + return 0; +} + +/** + * Set the super table UID of the created table + */ +static int tsdbTableSetSuperUid(STableCfg *config, uint64_t uid) { + if (config->type != TSDB_CHILD_TABLE) return -1; + if (uid == TSDB_INVALID_SUPER_TABLE_ID) return -1; + + config->superUid = uid; + return 0; +} + +/** + * Set the table schema in the configuration + * @param config the configuration to set + * @param pSchema the schema to set + * @param dup use the schema directly or duplicate one for use + * + * @return 0 for success and -1 for failure + */ +static int tsdbTableSetSchema(STableCfg *config, STSchema *pSchema, bool dup) { + if (dup) { + config->schema = tdDupSchema(pSchema); + } else { + config->schema = pSchema; + } + return 0; +} + +/** + * Set the table schema in the configuration + * @param config the configuration to set + * @param pSchema the schema to set + * @param dup use the schema directly or duplicate one for use + * + * @return 0 for success and -1 for failure + */ +static int tsdbTableSetTagSchema(STableCfg *config, STSchema *pSchema, bool dup) { + if (config->type != TSDB_CHILD_TABLE) return -1; + + if (dup) { + config->tagSchema = tdDupSchema(pSchema); + } else { + config->tagSchema = pSchema; + } + return 0; +} + +static int tsdbTableSetTagValue(STableCfg *config, SKVRow row, bool dup) { + if (config->type != TSDB_CHILD_TABLE) return -1; + + if (dup) { + config->tagValues = tdKVRowDup(row); + } else { + config->tagValues = row; + } + + return 0; +} + +static int tsdbTableSetName(STableCfg *config, char *name, bool dup) { + if (dup) { + config->name = strdup(name); + if (config->name == NULL) return -1; + } else { + config->name = name; + } + + return 0; +} + +static int tsdbTableSetSName(STableCfg *config, char *sname, bool dup) { + if (config->type != TSDB_CHILD_TABLE) return -1; + + if (dup) { + config->sname = strdup(sname); + if (config->sname == NULL) return -1; + } else { + config->sname = sname; + } + return 0; +} + +static int tsdbTableSetStreamSql(STableCfg *config, char *sql, bool dup) { + if (config->type != TSDB_STREAM_TABLE) return -1; + + if (dup) { + config->sql = strdup(sql); + if (config->sql == NULL) return -1; + } else { + config->sql = sql; + } + + return 0; +} diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index fa322757e2..e57188b479 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -131,11 +131,7 @@ static int32_t vnodeProcessDropTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet } static int32_t vnodeProcessAlterTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pRet) { - STableCfg *pCfg = tsdbCreateTableCfgFromMsg((SMDCreateTableMsg *)pCont); - if (pCfg == NULL) return terrno; - int32_t code = tsdbAlterTable(pVnode->tsdb, pCfg); - tsdbClearTableCfg(pCfg); - return code; + return TSDB_CODE_SUCCESS; } static int32_t vnodeProcessDropStableMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pRet) { From 4e77ffd28369499bd21110507c7b1580404cdec5 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 11 Jun 2020 11:16:09 +0800 Subject: [PATCH 03/65] TD-353 --- src/inc/tsdb.h | 2 +- src/tsdb/src/tsdbMain.c | 4 ++-- src/vnode/src/vnodeMain.c | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index 6f2e48f801..96a04ea5ed 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -77,7 +77,7 @@ typedef void TSDB_REPO_T; // use void to hide implementation details from outsi STsdbCfg *tsdbGetCfg(const TSDB_REPO_T *repo); // --------- TSDB REPOSITORY DEFINITION -int tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg, void *limiter); +int tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg); int32_t tsdbDropRepo(TSDB_REPO_T *repo); TSDB_REPO_T *tsdbOpenRepo(char *rootDir, STsdbAppH *pAppH); int32_t tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit); diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 0be07d6179..e2af459ae3 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -91,7 +91,7 @@ STsdbCfg *tsdbGetCfg(const TSDB_REPO_T *repo) { * * @return a TSDB repository handle on success, NULL for failure */ -int32_t tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg, void *limiter /* TODO */) { +int32_t tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg) { if (mkdir(rootDir, 0755) != 0) { tsdbError("vgId:%d, failed to create rootDir! rootDir:%s, reason:%s", pCfg->tsdbId, rootDir, strerror(errno)); if (errno == EACCES) { @@ -117,7 +117,7 @@ int32_t tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg, void *limiter /* TODO */) pRepo->rootDir = strdup(rootDir); pRepo->config = *pCfg; - pRepo->limiter = limiter; + // pRepo->limiter = limiter; // Create the environment files and directories int32_t code = tsdbSetRepoEnv(pRepo); diff --git a/src/vnode/src/vnodeMain.c b/src/vnode/src/vnodeMain.c index b952c07616..cbebbfd4e2 100644 --- a/src/vnode/src/vnodeMain.c +++ b/src/vnode/src/vnodeMain.c @@ -123,7 +123,7 @@ int32_t vnodeCreate(SMDCreateVnodeMsg *pVnodeCfg) { char tsdbDir[TSDB_FILENAME_LEN] = {0}; sprintf(tsdbDir, "%s/vnode%d/tsdb", tsVnodeDir, pVnodeCfg->cfg.vgId); - code = tsdbCreateRepo(tsdbDir, &tsdbCfg, NULL); + code = tsdbCreateRepo(tsdbDir, &tsdbCfg); if (code != TSDB_CODE_SUCCESS) { vError("vgId:%d, failed to create tsdb in vnode, reason:%s", pVnodeCfg->cfg.vgId, tstrerror(code)); return TSDB_CODE_VND_INIT_FAILED; From 34b17dc013ddb87ac1f73c8170f37bf7df67364a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 12 Jun 2020 10:52:48 +0000 Subject: [PATCH 04/65] TD-353 --- src/inc/taoserror.h | 2 + src/tsdb/inc/tsdbMain.h | 494 +++---- src/tsdb/src/tsdbBuffer.c | 159 +++ src/tsdb/src/tsdbCache.c | 207 --- src/tsdb/src/tsdbCompactor.c | 0 src/tsdb/src/tsdbFile.c | 132 +- src/tsdb/src/tsdbMain.c | 2394 ++++++++++++++++++---------------- src/tsdb/src/tsdbMemTable.c | 282 ++++ src/tsdb/src/tsdbMeta.c | 818 ++++++------ src/tsdb/src/tsdbMetaFile.c | 281 ---- src/tsdb/src/tsdbRWHelper.c | 675 +++++----- src/util/inc/tlist.h | 1 + src/util/src/tlist.c | 7 + src/util/src/tskiplist.c | 2 + 14 files changed, 2643 insertions(+), 2811 deletions(-) create mode 100644 src/tsdb/src/tsdbBuffer.c delete mode 100644 src/tsdb/src/tsdbCache.c delete mode 100644 src/tsdb/src/tsdbCompactor.c create mode 100644 src/tsdb/src/tsdbMemTable.c delete mode 100644 src/tsdb/src/tsdbMetaFile.c diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index ac2af75742..715a45d8e3 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -188,6 +188,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_FILE_CORRUPTED, 0, 0x0608, "tsdb file TAOS_DEFINE_ERROR(TSDB_CODE_TDB_OUT_OF_MEMORY, 0, 0x0609, "tsdb out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TAG_VER_OUT_OF_DATE, 0, 0x060A, "tsdb tag version is out of date") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE, 0, 0x060B, "tsdb timestamp is out of range") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP, 0, 0x060C, "tsdb submit message is messed up") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_ACTION, 0, 0x060D, "tsdb invalid action") // query TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_QHANDLE, 0, 0x0700, "query invalid handle") diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index d697ed7926..6055f30aad 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -15,13 +15,16 @@ #ifndef _TD_TSDB_MAIN_H_ #define _TD_TSDB_MAIN_H_ +#include "hash.h" +#include "tcoding.h" #include "tglobal.h" +#include "tkvstore.h" #include "tlist.h" +#include "tlog.h" +#include "tref.h" #include "tsdb.h" #include "tskiplist.h" #include "tutil.h" -#include "tlog.h" -#include "tcoding.h" #ifdef __cplusplus extern "C" { @@ -34,223 +37,106 @@ extern int tsdbDebugFlag; #define tsdbTrace(...) { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} #define tsdbPrint(...) { taosPrintLog("TDB ", 255, __VA_ARGS__); } -// ------------------------------ TSDB META FILE INTERFACES ------------------------------ -#define TSDB_META_FILE_NAME "meta" -#define TSDB_META_HASH_FRACTION 1.1 - -typedef int (*iterFunc)(void *, void *cont, int contLen); -typedef void (*afterFunc)(void *); - -typedef struct { - int fd; // File descriptor - int nDel; // number of deletions - int tombSize; // deleted size - int64_t size; // Total file size - void * map; // Map from uid ==> position - iterFunc iFunc; - afterFunc aFunc; - void * appH; -} SMetaFile; - -SMetaFile *tsdbInitMetaFile(char *rootDir, int32_t maxTables, iterFunc iFunc, afterFunc aFunc, void *appH); -int32_t tsdbInsertMetaRecord(SMetaFile *mfh, uint64_t uid, void *cont, int32_t contLen); -int32_t tsdbDeleteMetaRecord(SMetaFile *mfh, uint64_t uid); -int32_t tsdbUpdateMetaRecord(SMetaFile *mfh, uint64_t uid, void *cont, int32_t contLen); -void tsdbCloseMetaFile(SMetaFile *mfh); - -// ------------------------------ TSDB META INTERFACES ------------------------------ -#define IS_CREATE_STABLE(pCfg) ((pCfg)->tagValues != NULL) - -typedef struct { - TSKEY keyFirst; - TSKEY keyLast; - int32_t numOfRows; - void * pData; -} SMemTable; - -// ---------- TSDB TABLE DEFINITION #define TSDB_MAX_TABLE_SCHEMAS 16 -typedef struct STable { - int8_t type; - STableId tableId; - uint64_t superUid; // Super table UID - int16_t numOfSchemas; - STSchema ** schema; - STSchema * tagSchema; - SKVRow tagVal; - SMemTable * mem; - SMemTable * imem; - void * pIndex; // For TSDB_SUPER_TABLE, it is the skiplist index - void * eventHandler; // TODO - void * streamHandler; // TODO - TSKEY lastKey; // lastkey inserted in this table, initialized as 0, TODO: make a structure - struct STable *next; // TODO: remove the next - struct STable *prev; - tstr * name; // NOTE: there a flexible string here - char * sql; - void * cqhandle; -} STable; - -#define TSDB_GET_TABLE_LAST_KEY(tb) ((tb)->lastKey) - -void tsdbEncodeTable(STable *pTable, char *buf, int *contLen); -STable *tsdbDecodeTable(void *cont, int contLen); -void tsdbFreeEncode(void *cont); - -// ---------- TSDB META HANDLE DEFINITION -typedef struct { - int32_t maxTables; // Max number of tables - - int32_t nTables; // Tables created - - STable **tables; // table array - - STable *superList; // super table list TODO: change it to list container - - void *map; // table map of (uid ===> table) - - SMetaFile *mfh; // meta file handle - int maxRowBytes; - int maxCols; - void * pRepo; -} STsdbMeta; - -// element put in skiplist for each table -typedef struct STableIndexElem { - STsdbMeta* pMeta; - STable* pTable; -} STableIndexElem; - -STsdbMeta *tsdbInitMeta(char *rootDir, int32_t maxTables, void *pRepo); -int32_t tsdbFreeMeta(STsdbMeta *pMeta); -STSchema * tsdbGetTableTagSchema(STsdbMeta *pMeta, STable *pTable); - -// ---- Operation on STable -#define TSDB_TABLE_ID(pTable) ((pTable)->tableId) -#define TSDB_TABLE_UID(pTable) ((pTable)->uid) -#define TSDB_TABLE_NAME(pTable) ((pTable)->tableName) -#define TSDB_TABLE_TYPE(pTable) ((pTable)->type) -#define TSDB_TABLE_SUPER_TABLE_UID(pTable) ((pTable)->stableUid) -#define TSDB_TABLE_IS_SUPER_TABLE(pTable) (TSDB_TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) -#define TSDB_TABLE_TAG_VALUE(pTable) ((pTable)->pTagVal) -#define TSDB_TABLE_CACHE_DATA(pTable) ((pTable)->content.pData) -#define TSDB_SUPER_TABLE_INDEX(pTable) ((pTable)->content.pIndex) - -// ---- Operation on SMetaHandle -#define TSDB_NUM_OF_TABLES(pHandle) ((pHandle)->numOfTables) -#define TSDB_NUM_OF_SUPER_TABLES(pHandle) ((pHandle)->numOfSuperTables) -#define TSDB_TABLE_OF_ID(pHandle, id) ((pHandle)->pTables)[id] -#define TSDB_GET_TABLE_OF_NAME(pHandle, name) /* TODO */ - -STsdbMeta *tsdbGetMeta(TSDB_REPO_T *pRepo); - -STable *tsdbIsValidTableToInsert(STsdbMeta *pMeta, STableId tableId); -// int32_t tsdbInsertRowToTableImpl(SSkipListNode *pNode, STable *pTable); -STable *tsdbGetTableByUid(STsdbMeta *pMeta, uint64_t uid); -char *getTSTupleKey(const void * data); - -typedef struct { - int blockId; - int offset; - int remain; - int padding; - char data[]; -} STsdbCacheBlock; - -typedef struct { - int64_t index; - int numOfCacheBlocks; - SList * memPool; -} STsdbBufferPool; - -typedef struct { - TSKEY keyFirst; - TSKEY keyLast; - int64_t numOfRows; - SList * list; -} SCacheMem; - -typedef struct { - int cacheBlockSize; - int totalCacheBlocks; - STsdbBufferPool pool; - STsdbCacheBlock *curBlock; - SCacheMem * mem; - SCacheMem * imem; - TSDB_REPO_T * pRepo; -} STsdbCache; - -STsdbCache *tsdbInitCache(int cacheBlockSize, int totalBlocks, TSDB_REPO_T *pRepo); -void tsdbFreeCache(STsdbCache *pCache); -void * tsdbAllocFromCache(STsdbCache *pCache, int bytes, TSKEY key); - -// ------------------------------ TSDB FILE INTERFACES ------------------------------ #define TSDB_FILE_HEAD_SIZE 512 #define TSDB_FILE_DELIMITER 0xF00AFA0F +#define -#define tsdbGetKeyFileId(key, daysPerFile, precision) ((key) / tsMsPerDay[(precision)] / (daysPerFile)) -#define tsdbGetMaxNumOfFiles(keep, daysPerFile) ((keep) / (daysPerFile) + 3) - -typedef enum { - TSDB_FILE_TYPE_HEAD = 0, // .head file type - TSDB_FILE_TYPE_DATA, // .data file type - TSDB_FILE_TYPE_LAST, // .last file type - TSDB_FILE_TYPE_MAX -} TSDB_FILE_TYPE; - -#define IS_VALID_TSDB_FILE_TYPE(type) ((type) >= TSDB_FILE_TYPE_HEAD && (type) < TSDB_FILE_TYPE_MAX) - -extern const char *tsdbFileSuffix[]; +// Definitions +// ------------------ tsdbMeta.c +typedef struct STable { + ETableType type; + tstr* name; // NOTE: there a flexible string here + STableId tableId; + STable* pSuper; // super table pointer + uint8_t numOfSchemas; + STSchema schema[TSDB_MAX_TABLE_SCHEMAS]; + STSchema* tagSchema; + SKVRow tagVal; + void* pIndex; // For TSDB_SUPER_TABLE, it is the skiplist index + void* eventHandler; // TODO + void* streamHandler; // TODO + TSKEY lastKey; // lastkey inserted in this table, initialized as 0, TODO: make a structure + char* sql; + void* cqhandle; +} STable; + +typedef struct { + pthread_rwlock_t rwLock; + + int32_t nTables; + STable** tables; + SList* superList; + SHashObj* uidMap; + SKVStore* pStore; + int maxRowBytes; + int maxCols; +} STsdbMeta; + +// ------------------ tsdbBuffer.c +typedef struct { + int64_t blockId; + int offset; + int remain; + char data[]; +} STsdbBufBlock; + +typedef struct { + pthread_cond_t poolNotEmpty; + int bufBlockSize; + int tBufBlocks; + int nBufBlocks; + int64_t index; + SList* bufBlockList; +} STsdbBufPool; + +// ------------------ tsdbMemTable.c +typedef struct { + uint64_t uid; + TSKEY keyFirst; + TSKEY keyLast; + int64_t numOfRows; + SSkipList* pData; +} STableData; + +typedef struct { + T_REF_DECLARE(); + TSKEY keyFirst; + TSKEY keyLast; + int64_t numOfRows; + STableData** tData; + SList* actList; + SList* bufBlockList; +} SMemTable; + +// ------------------ tsdbFile.c +typedef enum { TSDB_FILE_TYPE_HEAD = 0, TSDB_FILE_TYPE_DATA, TSDB_FILE_TYPE_LAST, TSDB_FILE_TYPE_MAX } TSDB_FILE_TYPE; typedef struct { - uint32_t offset; - uint32_t len; - uint64_t size; // total size of the file - uint64_t tombSize; // unused file size - uint32_t totalBlocks; - uint32_t totalSubBlocks; } STsdbFileInfo; -void *tsdbEncodeSFileInfo(void *buf, const STsdbFileInfo *pInfo); -void *tsdbDecodeSFileInfo(void *buf, STsdbFileInfo *pInfo); - typedef struct { - int fd; - char fname[128]; - STsdbFileInfo info; + char* fname; + int fd; + uint64_t size; + uint64_t tombSize; + uint64_t totalBlocks; + uint64_t totalSubBlocks; } SFile; -#define TSDB_IS_FILE_OPENED(f) ((f)->fd != -1) - typedef struct { - int32_t fileId; - SFile files[TSDB_FILE_TYPE_MAX]; + int fileId; + SFile headF; + SFile dataF; + SFile lastF; } SFileGroup; -// TSDB file handle typedef struct { - int maxFGroups; - int numOfFGroups; - - SFileGroup *fGroup; + int maxFGroups; + int nFGroups; + SFileGroup* pFGroup; } STsdbFileH; -#define TSDB_MIN_FILE_ID(fh) (fh)->fGroup[0].fileId -#define TSDB_MAX_FILE_ID(fh) (fh)->fGroup[(fh)->numOfFGroups - 1].fileId - -STsdbFileH *tsdbInitFileH(char *dataDir, STsdbCfg *pCfg); -void tsdbCloseFileH(STsdbFileH *pFileH); -int tsdbCreateFile(char *dataDir, int fileId, const char *suffix, SFile *pFile); -SFileGroup *tsdbCreateFGroup(STsdbFileH *pFileH, char *dataDir, int fid, int maxTables); -int tsdbOpenFile(SFile *pFile, int oflag); -int tsdbCloseFile(SFile *pFile); -SFileGroup *tsdbOpenFilesForCommit(STsdbFileH *pFileH, int fid); -int tsdbRemoveFileGroup(STsdbFileH *pFile, int fid); -int tsdbGetFileName(char *dataDir, int fileId, const char *suffix, char *fname); - -#define TSDB_FGROUP_ITER_FORWARD TSDB_ORDER_ASC -#define TSDB_FGROUP_ITER_BACKWARD TSDB_ORDER_DESC - typedef struct { int numOfFGroups; SFileGroup *base; @@ -258,49 +144,30 @@ typedef struct { int direction; } SFileGroupIter; -void tsdbInitFileGroupIter(STsdbFileH *pFileH, SFileGroupIter *pIter, int direction); -void tsdbSeekFileGroupIter(SFileGroupIter *pIter, int fid); -SFileGroup *tsdbGetFileGroupNext(SFileGroupIter *pIter); - +// ------------------ tsdbRWHelper.c typedef struct { uint32_t len; uint32_t offset; - uint32_t padding; // For padding purpose + uint32_t padding; uint32_t hasLast : 2; uint32_t numOfBlocks : 30; uint64_t uid; TSKEY maxKey; -} SCompIdx; /* sizeof(SCompIdx) = 28 */ +} SCompIdx; -void *tsdbEncodeSCompIdx(void *buf, SCompIdx *pIdx); -void *tsdbDecodeSCompIdx(void *buf, SCompIdx *pIdx); - -/** - * if numOfSubBlocks == 0, then the SCompBlock is a sub-block - * if numOfSubBlocks >= 1, then the SCompBlock is a super-block - * - if numOfSubBlocks == 1, then the SCompBlock refers to the data block, and offset/len refer to - * the data block offset and length - * - if numOfSubBlocks > 1, then the offset/len refer to the offset of the first sub-block in the - * binary - */ typedef struct { - int64_t last : 1; // If the block in data file or last file - int64_t offset : 63; // Offset of data block or sub-block index depending on numOfSubBlocks - int32_t algorithm : 8; // Compression algorithm - int32_t numOfRows : 24; // Number of total points - int32_t sversion; // Schema version - int32_t len; // Data block length or nothing - int16_t numOfSubBlocks; // Number of sub-blocks; + int64_t last : 1; + int64_t offset : 63; + int32_t algorithm : 8; + int32_t numOfRows : 24; + int32_t sversion; + int32_t len; + int16_t numOfSubBlocks; int16_t numOfCols; TSKEY keyFirst; TSKEY keyLast; } SCompBlock; -// Maximum number of sub-blocks a super-block can have -#define TSDB_MAX_SUBBLOCKS 8 -#define IS_SUPER_BLOCK(pBlock) ((pBlock)->numOfSubBlocks >= 1) -#define IS_SUB_BLOCK(pBlock) ((pBlock)->numOfSubBlocks == 0) - typedef struct { int32_t delimiter; // For recovery usage int32_t checksum; // TODO: decide if checksum logic in this file or make it one API @@ -308,21 +175,9 @@ typedef struct { SCompBlock blocks[]; } SCompInfo; -#define TSDB_COMPBLOCK_AT(pCompInfo, idx) ((pCompInfo)->blocks + (idx)) -#define TSDB_COMPBLOCK_GET_START_AND_SIZE(pCompInfo, pCompBlock, size) \ - do { \ - if (pCompBlock->numOfSubBlocks > 1) { \ - pCompBlock = pCompInfo->blocks + pCompBlock->offset; \ - size = pCompBlock->numOfSubBlocks; \ - } else { \ - size = 1; \ - } \ - } while (0) - -// TODO: take pre-calculation into account typedef struct { - int16_t colId; // Column ID - int16_t len; // Column length // TODO: int16_t is not enough + int16_t colId; + int16_t len; int32_t type : 8; int32_t offset : 24; int64_t sum; @@ -334,7 +189,6 @@ typedef struct { char padding[2]; } SCompCol; -// TODO: Take recover into account typedef struct { int32_t delimiter; // For recovery usage int32_t numOfCols; // For recovery usage @@ -342,72 +196,6 @@ typedef struct { SCompCol cols[]; } SCompData; -STsdbFileH *tsdbGetFile(TSDB_REPO_T *pRepo); - -int tsdbCopyBlockDataInFile(SFile *pOutFile, SFile *pInFile, SCompInfo *pCompInfo, int idx, int isLast, - SDataCols *pCols); -SFileGroup *tsdbSearchFGroup(STsdbFileH *pFileH, int fid); -void tsdbGetKeyRangeOfFileId(int32_t daysPerFile, int8_t precision, int32_t fileId, TSKEY *minKey, TSKEY *maxKey); - -// TSDB repository definition -typedef struct STsdbRepo { - char *rootDir; - // TSDB configuration - STsdbCfg config; - - STsdbAppH appH; - - STsdbStat stat; - - // The meter meta handle of this TSDB repository - STsdbMeta *tsdbMeta; - - // The cache Handle - STsdbCache *tsdbCache; - - // The TSDB file handle - STsdbFileH *tsdbFileH; - - // Disk tier handle for multi-tier storage - void *diskTier; - - pthread_mutex_t mutex; - - int commit; - pthread_t commitThread; - - // A limiter to monitor the resources used by tsdb - void *limiter; - - int8_t state; - -} STsdbRepo; - -typedef struct { - int32_t totalLen; - int32_t len; - SDataRow row; -} SSubmitBlkIter; - -int tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter); -SDataRow tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter); - -#define TSDB_SUBMIT_MSG_HEAD_SIZE sizeof(SSubmitMsg) - -// SSubmitMsg Iterator -typedef struct { - int32_t totalLen; - int32_t len; - SSubmitBlk *pBlock; -} SSubmitMsgIter; - -int tsdbInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter); -SSubmitBlk *tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter); - -int32_t tsdbTriggerCommit(TSDB_REPO_T *repo); -int32_t tsdbLockRepo(TSDB_REPO_T *repo); -int32_t tsdbUnLockRepo(TSDB_REPO_T *repo); - typedef enum { TSDB_WRITE_HELPER, TSDB_READ_HELPER } tsdb_rw_helper_t; typedef struct { @@ -464,6 +252,87 @@ typedef struct { void *compBuffer; // Buffer for temperary compress/decompress purpose } SRWHelper; +// ------------------ tsdbMain.c +typedef struct { + int8_t state; + + char* rootDir; + STsdbCfg config; + STsdbAppH appH; + STsdbStat stat; + STsdbMeta* tsdbMeta; + STsdbBufPool* pPool; + SMemTable* mem; + SMemTable* imem; + STsdbFileH* tsdbFileH; + pthread_mutex_t mutex; + int commit; + pthread_t commitThread; +} STsdbRepo; + +// Operations +// ------------------ tsdbMeta.c +#define TABLE_TYPE(t) (t)->type +#define TABLE_NAME(t) (t)->name +#define TABLE_CHAR_NAME(t) TABLE_NAME(t)->data +#define TALBE_UID(t) (t)->tableId.uid +#define TABLE_TID(t) (t)->tableId.tid +#define TABLE_SUID(t) (t)->superUid +#define TABLE_LASTKEY(t) (t)->lastKey + +STsdbMeta* tsdbNewMeta(STsdbCfg* pCfg); +void tsdbFreeMeta(STsdbMeta* pMeta); + +// ------------------ tsdbBuffer.c +STsdbBufPool* tsdbNewBufPool(); +void tsdbFreeBufPool(STsdbBufPool* pBufPool); +int tsdbOpenBufPool(STsdbRepo* pRepo); +int tsdbOpenBufPool(STsdbRepo* pRepo); +SListNode* tsdbAllocBufBlockFromPool(STsdbRepo* pRepo); + +// ------------------ tsdbMemTable.c + +// ------------------ tsdbFile.c +#define TSDB_KEY_FILEID(key, daysPerFile, precision) ((key) / tsMsPerDay[(precision)] / (daysPerFile)) +#define TSDB_MAX_FILE(keep, daysPerFile) ((keep) / (daysPerFile) + 3) +#define TSDB_MIN_FILE_ID(fh) (fh)->pFGroup[0].fileId +#define TSDB_MAX_FILE_ID(fh) (fh)->pFGroup[(fh)->nFGroups - 1].fileId +#define TSDB_FGROUP_ITER_FORWARD TSDB_ORDER_ASC +#define TSDB_FGROUP_ITER_BACKWARD TSDB_ORDER_DESC + +STsdbFileH* tsdbNewFileH(STsdbCfg* pCfg); +void tsdbFreeFileH(STsdbFileH* pFileH); + +// ------------------ tsdbRWHelper.c +#define TSDB_MAX_SUBBLOCKS 8 +#define IS_SUB_BLOCK(pBlock) ((pBlock)->numOfSubBlocks == 0) + +// ------------------ tsdbMain.c +#define REPO_ID(r) (r)->config.tsdbId + +char* tsdbGetMetaFileName(char* rootDir); +int tsdbLockRepo(STsdbRepo* pRepo); +int tsdbUnlockRepo(STsdbRepo* pRepo); +void* tsdbCommitData(void* arg); + +#if 0 + + +// TSDB repository definition + +typedef struct { + int32_t totalLen; + int32_t len; + SDataRow row; +} SSubmitBlkIter; + +// SSubmitMsg Iterator +typedef struct { + int32_t totalLen; + int32_t len; + SSubmitBlk *pBlock; +} SSubmitMsgIter; + // --------- Helper state #define TSDB_HELPER_CLEAR_STATE 0x0 // Clear state #define TSDB_HELPER_FILE_SET_AND_OPEN 0x1 // File is set @@ -518,6 +387,7 @@ STSchema *tsdbGetTableSchema(STsdbMeta *pMeta, STable *pTable); #define DEFAULT_TAG_INDEX_COLUMN 0 // skip list built based on the first column of tags int compFGroupKey(const void *key, const void *fgroup); +#endif #ifdef __cplusplus } diff --git a/src/tsdb/src/tsdbBuffer.c b/src/tsdb/src/tsdbBuffer.c new file mode 100644 index 0000000000..013bae69a0 --- /dev/null +++ b/src/tsdb/src/tsdbBuffer.c @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdb.h" +#include "tsdbMain.h" + +#define POOL_IS_EMPTY(b) (listNEles((b)->bufBlockList) == 0) + +static STsdbBufBlock *tsdbNewBufBlock(int bufBlockSize); +static void tsdbFreeBufBlock(STsdbBufBlock *pBufBlock); + +// ---------------- INTERNAL FUNCTIONS ---------------- +STsdbBufPool *tsdbNewBufPool() { + STsdbBufPool *pBufPool = (STsdbBufPool *)calloc(1, sizeof(*pBufPool)); + if (pBufPool == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + goto _err; + } + + int code = pthread_cond_init(&(pBufPool->poolNotEmpty), NULL); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(code); + goto _err; + } + + pBufPool->bufBlockList = tdListNew(sizeof(STsdbBufBlock *)); + if (pBufPool->bufBlockList == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + goto _err; + } + + return pBufPool; + +_err: + tsdbFreeBufPool(pBufPool); + return NULL; +} + +void tsdbFreeBufPool(STsdbBufPool *pBufPool) { + if (pBufPool) { + if (pBufPool->bufBlockList) { + ASSERT(listNEles(pBufPool->bufBlockList) == 0); + tdListFree(pBufPool->bufBlockList); + } + + pthread_cond_destroy(&pBufPool->poolNotEmpty); + + free(pBufPool); + } +} + +int tsdbOpenBufPool(STsdbRepo *pRepo) { + STsdbCfg * pCfg = &(pRepo->config); + STsdbBufPool *pPool = pRepo->pPool; + + ASSERT(pPool != NULL); + + pPool->bufBlockSize = pCfg->cacheBlockSize; + pPool->tBufBlocks = pCfg->totalBlocks; + pPool->nBufBlocks = 0; + pPool->index = 0; + + for (int i = 0; i < pCfg->totalBlocks; i++) { + STsdbBufBlock *pBufBlock = tsdbNewBufBlock(pCfg->cacheBlockSize); + if (pBufBlock == NULL) goto _err; + + if (tdListAppend(pPool->bufBlockList, (void *)(&pBufBlock)) < 0) { + tsdbFreeBufBlock(pBufBlock); + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + goto _err; + } + + pPool->nBufBlocks++; + } + + tsdbTrace("vgId:%d buffer pool is opened! bufBlockSize:%d tBufBlocks:%d nBufBlocks:%d", REPO_ID(pRepo), + pBufPool->bufBlockSize, pBufPool->tBufBlocks, pBufPool->nBufBlocks); + + return 0; + +_err: + tsdbCloseBufPool(pRepo); + return -1; +} + +void tsdbCloseBufPool(STsdbRepo *pRepo) { + if (pRepo == NULL) return; + + STsdbBufPool * pBufPool = pRepo->pPool; + STsdbBufBlock *pBufBlock = NULL; + + if (pBufPool) { + SListNode *pNode = NULL; + while ((pNode = tdListPopHead(pBufPool->bufBlockList)) != NULL) { + tdListNodeGetData(pBufPool->bufBlockList, pNode, (void *)(&pBufBlock)); + tsdbFreeBufBlock(pBufBlock); + } + } + + tsdbTrace("vgId:%d buffer pool is closed", REPO_ID(pRepo)); +} + +SListNode *tsdbAllocBufBlockFromPool(STsdbRepo *pRepo) { + ASSERT(pRepo != NULL && pRepo->pool != NULL); + // ASSERT pRepo is locked + + STsdbCfg * pCfg = &pRepo->config; + STsdbBufPool *pBufPool = pRepo->pool; + + while (POOL_IS_EMPTY(pBufPool)) { + pthread_cond_wait(&(pBufPool->poolNotEmpty), &(pRepo->mutex)); + } + + ASSERT(!POOL_IS_EMPTY(pBufPool)); + + SListNode * pNode = tdListPopHead(pBufPool->bufBlockList); + STsdbBufBlock *pBufBlock = NULL; + tdListNodeGetData(pBufPool->bufBlockList, pNode, (void *)(&pBufBlock)); + + pBufBlock->blockId = pBufPool->index++; + pBufBlock->offset = 0; + pBufBlock->remain = pBufPool->bufBlockSize; + + tsdbTrace("vgId:%d buffer block is allocated, blockId:%" PRId64, REPO_ID(pRepo), pBufBlock->blockId); + return pNode; +} + +// ---------------- LOCAL FUNCTIONS ---------------- +static STsdbBufBlock *tsdbNewBufBlock(int bufBlockSize) { + STsdbBufBlock *pBufBlock = (STsdbBufBlock *)malloc(sizeof(*pBufBlock) + bufBlockSize); + if (pBufBlock == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + goto _err; + } + + pBufBlock->blockId = 0; + pBufBlock->offset = 0; + pBufBlock->remain = bufBlockSize; + + return pBufBlock; + +_err: + tsdbFreeBufBlock(pBufBlock); + return NULL; +} + +static void tsdbFreeBufBlock(STsdbBufBlock *pBufBlock) { tfree(pBufBlock); } \ No newline at end of file diff --git a/src/tsdb/src/tsdbCache.c b/src/tsdb/src/tsdbCache.c deleted file mode 100644 index 56e1b4e52a..0000000000 --- a/src/tsdb/src/tsdbCache.c +++ /dev/null @@ -1,207 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#include - -#include "tsdb.h" -#include "tsdbMain.h" - -static int tsdbAllocBlockFromPool(STsdbCache *pCache); -static void tsdbFreeBlockList(SList *list); -static void tsdbFreeCacheMem(SCacheMem *mem); -static int tsdbAddCacheBlockToPool(STsdbCache *pCache); - -STsdbCache *tsdbInitCache(int cacheBlockSize, int totalBlocks, TSDB_REPO_T *pRepo) { - STsdbCache *pCache = (STsdbCache *)calloc(1, sizeof(STsdbCache)); - if (pCache == NULL) return NULL; - - if (cacheBlockSize < 0) cacheBlockSize = TSDB_DEFAULT_CACHE_BLOCK_SIZE; - cacheBlockSize *= (1024 * 1024); - - if (totalBlocks <= 1) totalBlocks = TSDB_DEFAULT_TOTAL_BLOCKS; - - pCache->cacheBlockSize = cacheBlockSize; - pCache->totalCacheBlocks = totalBlocks; - pCache->pRepo = pRepo; - - STsdbBufferPool *pPool = &(pCache->pool); - pPool->index = 0; - pPool->memPool = tdListNew(sizeof(STsdbCacheBlock *)); - if (pPool->memPool == NULL) goto _err; - - for (int i = 0; i < totalBlocks; i++) { - if (tsdbAddCacheBlockToPool(pCache) < 0) goto _err; - } - - pCache->mem = NULL; - pCache->imem = NULL; - - return pCache; - -_err: - tsdbFreeCache(pCache); - return NULL; -} - -void tsdbFreeCache(STsdbCache *pCache) { - tsdbFreeCacheMem(pCache->imem); - tsdbFreeCacheMem(pCache->mem); - tsdbFreeBlockList(pCache->pool.memPool); - free(pCache); -} - -void *tsdbAllocFromCache(STsdbCache *pCache, int bytes, TSKEY key) { - if (pCache == NULL) return NULL; - if (bytes > pCache->cacheBlockSize) return NULL; - - if (pCache->curBlock == NULL || pCache->curBlock->remain < bytes) { - if (pCache->curBlock !=NULL && listNEles(pCache->mem->list) >= pCache->totalCacheBlocks/2) { - tsdbTriggerCommit(pCache->pRepo); - } - - while (tsdbAllocBlockFromPool(pCache) < 0) { - // TODO: deal with the error - // printf("Failed to allocate from cache pool\n"); - } - } - - void *ptr = (void *)(pCache->curBlock->data + pCache->curBlock->offset); - pCache->curBlock->offset += bytes; - pCache->curBlock->remain -= bytes; - memset(ptr, 0, bytes); - if (key < pCache->mem->keyFirst) pCache->mem->keyFirst = key; - if (key > pCache->mem->keyLast) pCache->mem->keyLast = key; - pCache->mem->numOfRows++; - - return ptr; -} - -static void tsdbFreeBlockList(SList *list) { - SListNode * node = NULL; - STsdbCacheBlock *pBlock = NULL; - while ((node = tdListPopHead(list)) != NULL) { - tdListNodeGetData(list, node, (void *)(&pBlock)); - free(pBlock); - listNodeFree(node); - } - tdListFree(list); -} - -static void tsdbFreeCacheMem(SCacheMem *mem) { - if (mem == NULL) return; - SList *list = mem->list; - tsdbFreeBlockList(list); - free(mem); -} - -static int tsdbAllocBlockFromPool(STsdbCache *pCache) { - STsdbBufferPool *pPool = &(pCache->pool); - - tsdbLockRepo(pCache->pRepo); - if (listNEles(pPool->memPool) == 0) { - tsdbUnLockRepo(pCache->pRepo); - return -1; - } - - SListNode *node = tdListPopHead(pPool->memPool); - - STsdbCacheBlock *pBlock = NULL; - tdListNodeGetData(pPool->memPool, node, (void *)(&pBlock)); - pBlock->blockId = pPool->index++; - pBlock->offset = 0; - pBlock->remain = pCache->cacheBlockSize; - - if (pCache->mem == NULL) { // Create a new one - pCache->mem = (SCacheMem *)malloc(sizeof(SCacheMem)); - if (pCache->mem == NULL) return -1; - pCache->mem->keyFirst = INT64_MAX; - pCache->mem->keyLast = 0; - pCache->mem->numOfRows = 0; - pCache->mem->list = tdListNew(sizeof(STsdbCacheBlock *)); - } - - tdListAppendNode(pCache->mem->list, node); - pCache->curBlock = pBlock; - - tsdbUnLockRepo(pCache->pRepo); - - return 0; -} - -int tsdbAlterCacheTotalBlocks(STsdbRepo *pRepo, int totalBlocks) { - STsdbCache *pCache = pRepo->tsdbCache; - int oldNumOfBlocks = pCache->totalCacheBlocks; - - tsdbLockRepo((TSDB_REPO_T *)pRepo); - - ASSERT(pCache->totalCacheBlocks != totalBlocks); - - if (pCache->totalCacheBlocks < totalBlocks) { - ASSERT(pCache->totalCacheBlocks == pCache->pool.numOfCacheBlocks); - int blocksToAdd = pCache->totalCacheBlocks - totalBlocks; - pCache->totalCacheBlocks = totalBlocks; - for (int i = 0; i < blocksToAdd; i++) { - if (tsdbAddCacheBlockToPool(pCache) < 0) { - tsdbUnLockRepo((TSDB_REPO_T *)pRepo); - tsdbError("tsdbId:%d, failed to add cache block to cache pool", pRepo->config.tsdbId); - return -1; - } - } - } else { - pCache->totalCacheBlocks = totalBlocks; - tsdbAdjustCacheBlocks(pCache); - } - pRepo->config.totalBlocks = totalBlocks; - - tsdbUnLockRepo((TSDB_REPO_T *)pRepo); - tsdbTrace("vgId:%d, tsdb total cache blocks changed from %d to %d", pRepo->config.tsdbId, oldNumOfBlocks, totalBlocks); - return 0; -} - -static int tsdbAddCacheBlockToPool(STsdbCache *pCache) { - STsdbBufferPool *pPool = &pCache->pool; - - STsdbCacheBlock *pBlock = malloc(sizeof(STsdbCacheBlock) + pCache->cacheBlockSize); - if (pBlock == NULL) return -1; - - pBlock->offset = 0; - pBlock->remain = pCache->cacheBlockSize; - tdListAppend(pPool->memPool, (void *)(&pBlock)); - pPool->numOfCacheBlocks++; - - return 0; -} - -static int tsdbRemoveCacheBlockFromPool(STsdbCache *pCache) { - STsdbBufferPool *pPool = &pCache->pool; - STsdbCacheBlock *pBlock = NULL; - - ASSERT(pCache->totalCacheBlocks >= 0); - - SListNode *node = tdListPopHead(pPool->memPool); - if (node == NULL) return -1; - - tdListNodeGetData(pPool->memPool, node, &pBlock); - free(pBlock); - listNodeFree(node); - pPool->numOfCacheBlocks--; - - return 0; -} - -void tsdbAdjustCacheBlocks(STsdbCache *pCache) { - while (pCache->totalCacheBlocks < pCache->pool.numOfCacheBlocks) { - if (tsdbRemoveCacheBlockFromPool(pCache) < 0) break; - } -} \ No newline at end of file diff --git a/src/tsdb/src/tsdbCompactor.c b/src/tsdb/src/tsdbCompactor.c deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index f8c00033b2..e52a72c32a 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -35,8 +35,35 @@ const char *tsdbFileSuffix[] = { ".last" // TSDB_FILE_TYPE_LAST }; -static int compFGroup(const void *arg1, const void *arg2); -static int tsdbOpenFGroup(STsdbFileH *pFileH, char *dataDir, int fid); +// ---------------- INTERNAL FUNCTIONS ---------------- +STsdbFileH* tsdbNewFileH(STsdbCfg* pCfg) { + STsdbFileH *pFileH = (STsdbFileH *)calloc(1, sizeof(*pFileH)); + if (pFileH == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + goto _err; + } + + pFileH->maxFGroups = pCfg->keep / pCfg->daysPerFile + 3; + + pFileH->pFGroup = (SFileGroup *)calloc(pFileH->maxFGroups, sizeof(SFileGroup)); + if (pFileH->pFGroup == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + goto _err; + } + + return pFileH; + + _err: + tsdbFreeFileH(pFileH); + return NULL; +} + +void tsdbFreeFileH(STsdbFileH* pFileH) { + if (pFileH) { + tfree(pFileH->pFGroup); + free(pFileH); + } +} STsdbFileH *tsdbInitFileH(char *dataDir, STsdbCfg *pCfg) { STsdbFileH *pFileH = (STsdbFileH *)calloc(1, sizeof(STsdbFileH)); @@ -80,41 +107,6 @@ void tsdbCloseFileH(STsdbFileH *pFileH) { } } -static int tsdbInitFile(char *dataDir, int fid, const char *suffix, SFile *pFile) { - uint32_t version; - char buf[512] = "\0"; - - tsdbGetFileName(dataDir, fid, suffix, pFile->fname); - if (access(pFile->fname, F_OK|R_OK|W_OK) < 0) return -1; - pFile->fd = -1; - if (tsdbOpenFile(pFile, O_RDONLY) < 0) return -1; - - if (tread(pFile->fd, buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) return -1; - if (!taosCheckChecksumWhole((uint8_t *)buf, TSDB_FILE_HEAD_SIZE)) return -1; - - void *pBuf = buf; - pBuf = taosDecodeFixedU32(pBuf, &version); - pBuf = tsdbDecodeSFileInfo(pBuf, &(pFile->info)); - - tsdbCloseFile(pFile); - - return 0; -} - -static int tsdbOpenFGroup(STsdbFileH *pFileH, char *dataDir, int fid) { - if (tsdbSearchFGroup(pFileH, fid) != NULL) return 0; - - SFileGroup fGroup = {0}; - fGroup.fileId = fid; - - for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { - if (tsdbInitFile(dataDir, fid, tsdbFileSuffix[type], &fGroup.files[type]) < 0) return -1; - } - pFileH->fGroup[pFileH->numOfFGroups++] = fGroup; - qsort((void *)(pFileH->fGroup), pFileH->numOfFGroups, sizeof(SFileGroup), compFGroup); - return 0; -} - /** * Create the file group if the file group not exists. * @@ -228,28 +220,6 @@ SFileGroup *tsdbGetFileGroupNext(SFileGroupIter *pIter) { return ret; } -// int tsdbLoadDataBlock(SFile *pFile, SCompBlock *pStartBlock, int numOfBlocks, SDataCols *pCols, SCompData *pCompData) { -// SCompBlock *pBlock = pStartBlock; -// for (int i = 0; i < numOfBlocks; i++) { -// if (tsdbLoadCompCols(pFile, pBlock, (void *)pCompData) < 0) return -1; -// pCols->numOfRows += (pCompData->cols[0].len / 8); -// for (int iCol = 0; iCol < pBlock->numOfCols; iCol++) { -// SCompCol *pCompCol = &(pCompData->cols[iCol]); -// // pCols->numOfRows += pBlock->numOfRows; -// int k = 0; -// for (; k < pCols->numOfCols; k++) { -// if (pCompCol->colId == pCols->cols[k].colId) break; -// } - -// if (tsdbLoadColData(pFile, pCompCol, pBlock->offset, -// (void *)((char *)(pCols->cols[k].pData) + pCols->cols[k].len)) < 0) -// return -1; -// } -// pStartBlock++; -// } -// return 0; -// } - int tsdbCopyBlockDataInFile(SFile *pOutFile, SFile *pInFile, SCompInfo *pCompInfo, int idx, int isLast, SDataCols *pCols) { SCompBlock *pSuperBlock = TSDB_COMPBLOCK_AT(pCompInfo, idx); SCompBlock *pStartBlock = NULL; @@ -294,10 +264,6 @@ int compFGroupKey(const void *key, const void *fgroup) { } } -static int compFGroup(const void *arg1, const void *arg2) { - return ((SFileGroup *)arg1)->fileId - ((SFileGroup *)arg2)->fileId; -} - int tsdbGetFileName(char *dataDir, int fileId, const char *suffix, char *fname) { if (dataDir == NULL || fname == NULL) return -1; @@ -371,4 +337,44 @@ SFileGroup *tsdbSearchFGroup(STsdbFileH *pFileH, int fid) { void *ptr = bsearch((void *)&fid, (void *)(pFileH->fGroup), pFileH->numOfFGroups, sizeof(SFileGroup), compFGroupKey); if (ptr == NULL) return NULL; return (SFileGroup *)ptr; +} + +// ---------------- LOCAL FUNCTIONS ---------------- +static int tsdbInitFile(char *dataDir, int fid, const char *suffix, SFile *pFile) { + uint32_t version; + char buf[512] = "\0"; + + tsdbGetFileName(dataDir, fid, suffix, pFile->fname); + if (access(pFile->fname, F_OK|R_OK|W_OK) < 0) return -1; + pFile->fd = -1; + if (tsdbOpenFile(pFile, O_RDONLY) < 0) return -1; + + if (tread(pFile->fd, buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) return -1; + if (!taosCheckChecksumWhole((uint8_t *)buf, TSDB_FILE_HEAD_SIZE)) return -1; + + void *pBuf = buf; + pBuf = taosDecodeFixedU32(pBuf, &version); + pBuf = tsdbDecodeSFileInfo(pBuf, &(pFile->info)); + + tsdbCloseFile(pFile); + + return 0; +} + +static int tsdbOpenFGroup(STsdbFileH *pFileH, char *dataDir, int fid) { + if (tsdbSearchFGroup(pFileH, fid) != NULL) return 0; + + SFileGroup fGroup = {0}; + fGroup.fileId = fid; + + for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { + if (tsdbInitFile(dataDir, fid, tsdbFileSuffix[type], &fGroup.files[type]) < 0) return -1; + } + pFileH->fGroup[pFileH->numOfFGroups++] = fGroup; + qsort((void *)(pFileH->fGroup), pFileH->numOfFGroups, sizeof(SFileGroup), compFGroup); + return 0; +} + +static int compFGroup(const void *arg1, const void *arg2) { + return ((SFileGroup *)arg1)->fileId - ((SFileGroup *)arg2)->fileId; } \ No newline at end of file diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index e2af459ae3..36595ab987 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -1,504 +1,172 @@ -#include "os.h" -#include "taosdef.h" -#include "tulog.h" -#include "talgo.h" -#include "tsdb.h" +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ #include "tsdbMain.h" -#include "tscompression.h" +#include "os.h" +#include "talgo.h" +#include "taosdef.h" #include "tchecksum.h" +#include "tscompression.h" +#include "tsdb.h" #include "ttime.h" -#include +#include "tulog.h" -#define IS_VALID_PRECISION(precision) (((precision) >= TSDB_TIME_PRECISION_MILLI) && ((precision) <= TSDB_TIME_PRECISION_NANO)) -#define TSDB_DEFAULT_COMPRESSION TWO_STAGE_COMP -#define IS_VALID_COMPRESSION(compression) (((compression) >= NO_COMPRESSION) && ((compression) <= TWO_STAGE_COMP)) -#define TSDB_MIN_ID 0 -#define TSDB_MAX_ID INT_MAX +#include +#include #define TSDB_CFG_FILE_NAME "config" #define TSDB_DATA_DIR_NAME "data" -#define TSDB_DEFAULT_FILE_BLOCK_ROW_OPTION 0.7 -#define TSDB_MAX_LAST_FILE_SIZE (1024 * 1024 * 10) // 10M +#define TSDB_META_FILE_NAME "meta" +#define TSDB_META_FILE_INDEX 10000000 -enum { TSDB_REPO_STATE_ACTIVE, TSDB_REPO_STATE_CLOSED, TSDB_REPO_STATE_CONFIGURING }; - -static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg); -static int32_t tsdbSetRepoEnv(STsdbRepo *pRepo); -static int32_t tsdbDestroyRepoEnv(STsdbRepo *pRepo); -// static int tsdbOpenMetaFile(char *tsdbDir); -static int32_t tsdbInsertDataToTable(TSDB_REPO_T *repo, SSubmitBlk *pBlock, TSKEY now, int * affectedrows); -static int32_t tsdbRestoreCfg(STsdbRepo *pRepo, STsdbCfg *pCfg); -static int32_t tsdbGetDataDirName(STsdbRepo *pRepo, char *fname); -static void * tsdbCommitData(void *arg); -static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters, SRWHelper *pHelper, - SDataCols *pDataCols); -static TSKEY tsdbNextIterKey(SSkipListIterator *pIter); -static int tsdbHasDataToCommit(SSkipListIterator **iters, int nIters, TSKEY minKey, TSKEY maxKey); -static void tsdbAlterCompression(STsdbRepo *pRepo, int8_t compression); -static void tsdbAlterKeep(STsdbRepo *pRepo, int32_t keep); -static void tsdbAlterMaxTables(STsdbRepo *pRepo, int32_t maxTables); -static int32_t tsdbSaveConfig(STsdbRepo *pRepo); - -#define TSDB_GET_TABLE_BY_ID(pRepo, sid) (((STSDBRepo *)pRepo)->pTableList)[sid] -#define TSDB_GET_TABLE_BY_NAME(pRepo, name) -#define TSDB_IS_REPO_ACTIVE(pRepo) ((pRepo)->state == TSDB_REPO_STATE_ACTIVE) -#define TSDB_IS_REPO_CLOSED(pRepo) ((pRepo)->state == TSDB_REPO_STATE_CLOSED) - -/** - * Set the default TSDB configuration - */ -void tsdbSetDefaultCfg(STsdbCfg *pCfg) { - if (pCfg == NULL) return; - - pCfg->precision = -1; - pCfg->tsdbId = 0; - pCfg->maxTables = -1; - pCfg->daysPerFile = -1; - pCfg->minRowsPerFileBlock = -1; - pCfg->maxRowsPerFileBlock = -1; - pCfg->keep = -1; - pCfg->compression = TWO_STAGE_COMP; -} - -/** - * Create a configuration for TSDB default - * @return a pointer to a configuration. the configuration object - * must call tsdbFreeCfg to free memory after usage - */ -STsdbCfg *tsdbCreateDefaultCfg() { - STsdbCfg *pCfg = (STsdbCfg *)malloc(sizeof(STsdbCfg)); - if (pCfg == NULL) return NULL; - - tsdbSetDefaultCfg(pCfg); - - return pCfg; -} - -void tsdbFreeCfg(STsdbCfg *pCfg) { - if (pCfg != NULL) free(pCfg); -} - -STsdbCfg *tsdbGetCfg(const TSDB_REPO_T *repo) { - assert(repo != NULL); - return &((STsdbRepo*)repo)->config; -} - -/** - * Create a new TSDB repository - * @param rootDir the TSDB repository root directory - * @param pCfg the TSDB repository configuration, upper layer need to free the pointer - * @param limiter the limitation tracker will implement in the future, make it void now - * - * @return a TSDB repository handle on success, NULL for failure - */ +// Function declaration int32_t tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg) { - if (mkdir(rootDir, 0755) != 0) { - tsdbError("vgId:%d, failed to create rootDir! rootDir:%s, reason:%s", pCfg->tsdbId, rootDir, strerror(errno)); - if (errno == EACCES) { - return TSDB_CODE_TDB_NO_DISK_PERMISSIONS; - } else if (errno == ENOSPC) { - return TSDB_CODE_TDB_NO_DISKSPACE; - } else if (errno == EEXIST) { - } else { - return TSDB_CODE_TDB_INIT_FAILED; - } - } - - if (access(rootDir, F_OK | R_OK | W_OK) == -1) return -1; - - if (tsdbCheckAndSetDefaultCfg(pCfg) < 0) { + if (mkdir(rootDir, 0755) < 0) { + tsdbError("vgId:%d failed to create rootDir %s since %s", pCfg->tsdbId, rootDir, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } - STsdbRepo *pRepo = (STsdbRepo *)malloc(sizeof(STsdbRepo)); + if (tsdbCheckAndSetDefaultCfg(pCfg) < 0) return -1; + + if (tsdbSetRepoEnv(rootDir, pCfg) < 0) return -1; + + tsdbTrace( + "vgId%d tsdb env create succeed! cacheBlockSize %d totalBlocks %d maxTables %d daysPerFile %d keep " + "%d minRowsPerFileBlock %d maxRowsPerFileBlock %d precision %d compression %d", + pCfg->tsdbId, pCfg->cacheBlockSize, pCfg->totalBlocks, pCfg->maxTables, pCfg->daysPerFile, pCfg->keep, + pCfg->minRowsPerFileBlock, pCfg->maxRowsPerFileBlock, pCfg->precision, pCfg->compression); + return 0; +} + +int32_t tsdbDropRepo(char *rootDir) { + return tsdbUnsetRepoEnv(rootDir); +} + +TSDB_REPO_T *tsdbOpenRepo(char *rootDir, STsdbAppH *pAppH) { + STsdbCfg config = {0}; + STsdbRepo *pRepo = NULL; + + if (tsdbLoadConfig(rootDir, &config) < 0) { + tsdbError("failed to open repo in rootDir %s since %s", rootDir, tstrerror(terrno)); + return NULL; + } + + pRepo = tsdbNewRepo(rootDir, pAppH, &config); if (pRepo == NULL) { - return -1; + tsdbError("failed to open repo in rootDir %s since %s", rootDir, tstrerror(terrno)); + return NULL; } - pRepo->rootDir = strdup(rootDir); - pRepo->config = *pCfg; - // pRepo->limiter = limiter; - - // Create the environment files and directories - int32_t code = tsdbSetRepoEnv(pRepo); - - free(pRepo->rootDir); - free(pRepo); - return code; -} - -/** - * Close and free all resources taken by the repository - * @param repo the TSDB repository handle. The interface will free the handle too, so upper - * layer do NOT need to free the repo handle again. - * - * @return 0 for success, -1 for failure and the error number is set - */ -int32_t tsdbDropRepo(TSDB_REPO_T *repo) { - STsdbRepo *pRepo = (STsdbRepo *)repo; - int id = pRepo->config.tsdbId; - - pRepo->state = TSDB_REPO_STATE_CLOSED; - - // Free the metaHandle - tsdbFreeMeta(pRepo->tsdbMeta); - - // Free the cache - tsdbFreeCache(pRepo->tsdbCache); - - // Destroy the repository info - tsdbDestroyRepoEnv(pRepo); - - free(pRepo->rootDir); - free(pRepo); - - tsdbTrace("vgId:%d, tsdb repository is dropped!", id); - - return 0; -} - -static int tsdbRestoreInfo(STsdbRepo *pRepo) { - STsdbMeta * pMeta = pRepo->tsdbMeta; - STsdbFileH *pFileH = pRepo->tsdbFileH; - SFileGroup *pFGroup = NULL; - - SFileGroupIter iter; - SRWHelper rhelper = {{0}}; - - if (tsdbInitReadHelper(&rhelper, pRepo) < 0) goto _err; - tsdbInitFileGroupIter(pFileH, &iter, TSDB_ORDER_ASC); - while ((pFGroup = tsdbGetFileGroupNext(&iter)) != NULL) { - if (tsdbSetAndOpenHelperFile(&rhelper, pFGroup) < 0) goto _err; - for (int i = 1; i < pRepo->config.maxTables; i++) { - STable * pTable = pMeta->tables[i]; - if (pTable == NULL) continue; - SCompIdx *pIdx = &rhelper.pCompIdx[i]; - - if (pIdx->offset > 0 && pTable->lastKey < pIdx->maxKey) pTable->lastKey = pIdx->maxKey; - } + if (tsdbOpenMeta(pRepo) < 0) { + tsdbError("vgId:%d failed to open meta since %s", REPO_ID(pRepo), tstrerror(terrno)); + goto _err; } - tsdbDestroyHelper(&rhelper); - return 0; + if (tsdbOpenBufPool(pRepo) < 0) { + tsdbError("vgId:%d failed to open buffer pool since %s", REPO_ID(pRepo), tstrerror(terrno)); + goto _err; + } + + if (tsdbOpenFileH(pRepo) < 0) { + tsdbError("vgId:%d failed to open file handle since %s", REPO_ID(pRepo), tstrerror(terrno)); + goto _err; + } + + // // Restore key from file + // if (tsdbRestoreInfo(pRepo) < 0) { + // tsdbFreeCache(pRepo->tsdbCache); + // tsdbFreeMeta(pRepo->tsdbMeta); + // tsdbCloseFileH(pRepo->tsdbFileH); + // free(pRepo->rootDir); + // free(pRepo); + // return NULL; + // } + + // pRepo->state = TSDB_REPO_STATE_ACTIVE; + + tsdbTrace("vgId:%d open tsdb repository succeed!", REPO_ID(pRepo)); + + return (TSDB_REPO_T *)pRepo; _err: - tsdbDestroyHelper(&rhelper); - return -1; + tsdbCloseRepo(pRepo, false); + tsdbFreeRepo(pRepo); + return NULL; } -/** - * Open an existing TSDB storage repository - * @param tsdbDir the existing TSDB root directory - * - * @return a TSDB repository handle on success, NULL for failure and the error number is set - */ -TSDB_REPO_T *tsdbOpenRepo(char *rootDir, STsdbAppH *pAppH) { - char dataDir[128] = "\0"; - if (access(rootDir, F_OK | W_OK | R_OK) < 0) { - return NULL; - } - - STsdbRepo *pRepo = (STsdbRepo *)calloc(1, sizeof(STsdbRepo)); - if (pRepo == NULL) { - return NULL; - } - - pRepo->rootDir = strdup(rootDir); - - tsdbRestoreCfg(pRepo, &(pRepo->config)); - if (pAppH) pRepo->appH = *pAppH; - - pRepo->tsdbMeta = tsdbInitMeta(rootDir, pRepo->config.maxTables, pRepo); - if (pRepo->tsdbMeta == NULL) { - free(pRepo->rootDir); - free(pRepo); - return NULL; - } - - pRepo->tsdbCache = tsdbInitCache(pRepo->config.cacheBlockSize, pRepo->config.totalBlocks, (TSDB_REPO_T *)pRepo); - if (pRepo->tsdbCache == NULL) { - tsdbFreeMeta(pRepo->tsdbMeta); - free(pRepo->rootDir); - free(pRepo); - return NULL; - } - - tsdbGetDataDirName(pRepo, dataDir); - pRepo->tsdbFileH = tsdbInitFileH(dataDir, &(pRepo->config)); - if (pRepo->tsdbFileH == NULL) { - tsdbFreeCache(pRepo->tsdbCache); - tsdbFreeMeta(pRepo->tsdbMeta); - free(pRepo->rootDir); - free(pRepo); - return NULL; - } - - // Restore key from file - if (tsdbRestoreInfo(pRepo) < 0) { - tsdbFreeCache(pRepo->tsdbCache); - tsdbFreeMeta(pRepo->tsdbMeta); - tsdbCloseFileH(pRepo->tsdbFileH); - free(pRepo->rootDir); - free(pRepo); - return NULL; - } - - pRepo->state = TSDB_REPO_STATE_ACTIVE; - - tsdbTrace("vgId:%d, open tsdb repository successfully!", pRepo->config.tsdbId); - return (TSDB_REPO_T *)pRepo; -} - -// static int32_t tsdbFlushCache(STsdbRepo *pRepo) { -// // TODO -// return 0; -// } - -/** - * Close a TSDB repository. Only free memory resources, and keep the files. - * @param repo the opened TSDB repository handle. The interface will free the handle too, so upper - * layer do NOT need to free the repo handle again. - * - * @return 0 for success, -1 for failure and the error number is set - */ int32_t tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit) { - STsdbRepo *pRepo = (STsdbRepo *)repo; - if (pRepo == NULL) return 0; - int id = pRepo->config.tsdbId; + // TODO + // STsdbRepo *pRepo = (STsdbRepo *)repo; + // if (pRepo == NULL) return 0; + // int id = pRepo->config.tsdbId; - pRepo->state = TSDB_REPO_STATE_CLOSED; - tsdbLockRepo(repo); - if (pRepo->commit) { - tsdbUnLockRepo(repo); - return -1; - } - pRepo->commit = 1; - // Loop to move pData to iData - for (int i = 1; i < pRepo->config.maxTables; i++) { - STable *pTable = pRepo->tsdbMeta->tables[i]; - if (pTable != NULL && pTable->mem != NULL) { - pTable->imem = pTable->mem; - pTable->mem = NULL; - } - } - // TODO: Loop to move mem to imem - pRepo->tsdbCache->imem = pRepo->tsdbCache->mem; - pRepo->tsdbCache->mem = NULL; - pRepo->tsdbCache->curBlock = NULL; - tsdbUnLockRepo(repo); + // pRepo->state = TSDB_REPO_STATE_CLOSED; + // tsdbLockRepo(repo); + // if (pRepo->commit) { + // tsdbUnLockRepo(repo); + // return -1; + // } + // pRepo->commit = 1; + // // Loop to move pData to iData + // for (int i = 1; i < pRepo->config.maxTables; i++) { + // STable *pTable = pRepo->tsdbMeta->tables[i]; + // if (pTable != NULL && pTable->mem != NULL) { + // pTable->imem = pTable->mem; + // pTable->mem = NULL; + // } + // } + // // TODO: Loop to move mem to imem + // pRepo->tsdbCache->imem = pRepo->tsdbCache->mem; + // pRepo->tsdbCache->mem = NULL; + // pRepo->tsdbCache->curBlock = NULL; + // tsdbUnLockRepo(repo); - if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_START); - if (toCommit) tsdbCommitData((void *)repo); + // if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_START); + // if (toCommit) tsdbCommitData((void *)repo); - tsdbCloseFileH(pRepo->tsdbFileH); + // tsdbCloseFileH(pRepo->tsdbFileH); - tsdbFreeMeta(pRepo->tsdbMeta); + // tsdbFreeMeta(pRepo->tsdbMeta); - tsdbFreeCache(pRepo->tsdbCache); + // tsdbFreeCache(pRepo->tsdbCache); - tfree(pRepo->rootDir); - tfree(pRepo); + // tfree(pRepo->rootDir); + // tfree(pRepo); - tsdbTrace("vgId:%d, repository is closed!", id); + // tsdbTrace("vgId:%d repository is closed!", id); return 0; } -/** - * Change the configuration of a repository - * @param pCfg the repository configuration, the upper layer should free the pointer - * - * @return 0 for success, -1 for failure and the error number is set - */ -int32_t tsdbConfigRepo(TSDB_REPO_T *repo, STsdbCfg *pCfg) { - STsdbRepo *pRepo = (STsdbRepo *)repo; - STsdbCfg * pRCfg = &pRepo->config; - - if (tsdbCheckAndSetDefaultCfg(pCfg) < 0) return TSDB_CODE_TDB_INVALID_CONFIG; - - ASSERT(pRCfg->tsdbId == pCfg->tsdbId); - ASSERT(pRCfg->cacheBlockSize == pCfg->cacheBlockSize); - ASSERT(pRCfg->daysPerFile == pCfg->daysPerFile); - ASSERT(pRCfg->minRowsPerFileBlock == pCfg->minRowsPerFileBlock); - ASSERT(pRCfg->maxRowsPerFileBlock == pCfg->maxRowsPerFileBlock); - ASSERT(pRCfg->precision == pCfg->precision); - - bool configChanged = false; - if (pRCfg->compression != pCfg->compression) { - configChanged = true; - tsdbAlterCompression(pRepo, pCfg->compression); - } - if (pRCfg->keep != pCfg->keep) { - configChanged = true; - tsdbAlterKeep(pRepo, pCfg->keep); - } - if (pRCfg->totalBlocks != pCfg->totalBlocks) { - configChanged = true; - tsdbAlterCacheTotalBlocks(pRepo, pCfg->totalBlocks); - } - if (pRCfg->maxTables != pCfg->maxTables) { - configChanged = true; - tsdbAlterMaxTables(pRepo, pCfg->maxTables); - } - - if (configChanged) tsdbSaveConfig(pRepo); - - return TSDB_CODE_SUCCESS; -} - -int32_t tsdbTriggerCommit(TSDB_REPO_T *repo) { - STsdbRepo *pRepo = (STsdbRepo *)repo; - - if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_START); - - tsdbLockRepo(repo); - if (pRepo->commit) { - tsdbUnLockRepo(repo); - return -1; - } - pRepo->commit = 1; - // Loop to move pData to iData - for (int i = 1; i < pRepo->config.maxTables; i++) { - STable *pTable = pRepo->tsdbMeta->tables[i]; - if (pTable != NULL && pTable->mem != NULL) { - pTable->imem = pTable->mem; - pTable->mem = NULL; - } - } - // TODO: Loop to move mem to imem - pRepo->tsdbCache->imem = pRepo->tsdbCache->mem; - pRepo->tsdbCache->mem = NULL; - pRepo->tsdbCache->curBlock = NULL; - tsdbUnLockRepo(repo); - - // TODO: here should set as detached or use join for memory leak - pthread_attr_t thattr; - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_DETACHED); - pthread_create(&(pRepo->commitThread), &thattr, tsdbCommitData, (void *)repo); - tsdbTrace("vgId:%d, start to commit!", pRepo->config.tsdbId); - - return 0; -} - -int32_t tsdbLockRepo(TSDB_REPO_T *repo) { - STsdbRepo *pRepo = (STsdbRepo *)repo; - return pthread_mutex_lock(&(pRepo->mutex)); -} - -int32_t tsdbUnLockRepo(TSDB_REPO_T *repo) { - STsdbRepo *pRepo = (STsdbRepo *)repo; - return pthread_mutex_unlock(&(pRepo->mutex)); -} - -/** - * Get the TSDB repository information, including some statistics - * @param pRepo the TSDB repository handle - * @param error the error number to set when failure occurs - * - * @return a info struct handle on success, NULL for failure and the error number is set. The upper - * layers should free the info handle themselves or memory leak will occur - */ -STsdbRepoInfo *tsdbGetStatus(TSDB_REPO_T *pRepo) { - // TODO - return NULL; -} - -int tsdbUpdateTagValue(TSDB_REPO_T *repo, SUpdateTableTagValMsg *pMsg) { - STsdbRepo *pRepo = (STsdbRepo *)repo; - STsdbMeta *pMeta = pRepo->tsdbMeta; - int16_t tversion = htons(pMsg->tversion); - - STable *pTable = tsdbGetTableByUid(pMeta, htobe64(pMsg->uid)); - if (pTable == NULL) return TSDB_CODE_TDB_INVALID_TABLE_ID; - if (pTable->tableId.tid != htonl(pMsg->tid)) return TSDB_CODE_TDB_INVALID_TABLE_ID; - - if (pTable->type != TSDB_CHILD_TABLE) { - tsdbError("vgId:%d failed to update tag value of table %s since its type is %d", pRepo->config.tsdbId, - varDataVal(pTable->name), pTable->type); - return TSDB_CODE_TDB_INVALID_TABLE_TYPE; - } - - if (schemaVersion(tsdbGetTableTagSchema(pMeta, pTable)) < tversion) { - tsdbTrace("vgId:%d server tag version %d is older than client tag version %d, try to config", pRepo->config.tsdbId, - schemaVersion(tsdbGetTableTagSchema(pMeta, pTable)), tversion); - void *msg = (*pRepo->appH.configFunc)(pRepo->config.tsdbId, htonl(pMsg->tid)); - if (msg == NULL) { - return terrno; - } - // Deal with error her - STableCfg *pTableCfg = tsdbCreateTableCfgFromMsg(msg); - STable *super = tsdbGetTableByUid(pMeta, pTableCfg->superUid); - ASSERT(super != NULL); - - int32_t code = tsdbUpdateTable(pMeta, super, pTableCfg); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - tsdbClearTableCfg(pTableCfg); - rpcFreeCont(msg); - } - - STSchema *pTagSchema = tsdbGetTableTagSchema(pMeta, pTable); - - if (schemaVersion(pTagSchema) > tversion) { - tsdbError( - "vgId:%d failed to update tag value of table %s since version out of date, client tag version:%d server tag " - "version:%d", - pRepo->config.tsdbId, varDataVal(pTable->name), tversion, schemaVersion(pTable->tagSchema)); - return TSDB_CODE_TDB_TAG_VER_OUT_OF_DATE; - } - if (schemaColAt(pTagSchema, DEFAULT_TAG_INDEX_COLUMN)->colId == htons(pMsg->colId)) { - tsdbRemoveTableFromIndex(pMeta, pTable); - } - // TODO: remove table from index if it is the first column of tag - tdSetKVRowDataOfCol(&pTable->tagVal, htons(pMsg->colId), htons(pMsg->type), pMsg->data); - if (schemaColAt(pTagSchema, DEFAULT_TAG_INDEX_COLUMN)->colId == htons(pMsg->colId)) { - tsdbAddTableIntoIndex(pMeta, pTable); - } - return TSDB_CODE_SUCCESS; -} - -TSKEY tsdbGetTableLastKey(TSDB_REPO_T *repo, uint64_t uid) { - STsdbRepo *pRepo = (STsdbRepo *)repo; - - STable *pTable = tsdbGetTableByUid(pRepo->tsdbMeta, uid); - if (pTable == NULL) return -1; - - return TSDB_GET_TABLE_LAST_KEY(pTable); -} - -void tsdbStartStream(TSDB_REPO_T *repo) { - STsdbRepo *pRepo = (STsdbRepo *)repo; - STsdbMeta *pMeta = pRepo->tsdbMeta; - - for (int i = 0; i < pRepo->config.maxTables; i++) { - STable *pTable = pMeta->tables[i]; - if (pTable && pTable->type == TSDB_STREAM_TABLE) { - pTable->cqhandle = (*pRepo->appH.cqCreateFunc)(pRepo->appH.cqH, pTable->tableId.uid, pTable->tableId.tid, pTable->sql, tsdbGetTableSchema(pMeta, pTable)); - } - } -} - -STableInfo *tsdbGetTableInfo(TSDB_REPO_T *pRepo, STableId tableId) { - // TODO - return NULL; -} - -// TODO: need to return the number of data inserted -int32_t tsdbInsertData(TSDB_REPO_T *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg * pRsp) { +int32_t tsdbInsertData(TSDB_REPO_T *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg *pRsp) { + STsdbRepo * pRepo = (STsdbRepo *)repo; SSubmitMsgIter msgIter; - STsdbRepo *pRepo = (STsdbRepo *)repo; - tsdbInitSubmitMsgIter(pMsg, &msgIter); + if (tsdbInitSubmitMsgIter(pMsg, &msgIter) < 0) { + tsdbError("vgId:%d submit message is messed up", REPO_ID(pRepo)); + return terrno; + } + SSubmitBlk *pBlock = NULL; - int32_t code = TSDB_CODE_SUCCESS; - int32_t affectedrows = 0; + int32_t code = TSDB_CODE_SUCCESS; + int32_t affectedrows = 0; TSKEY now = taosGetTimestamp(pRepo->config.precision); while ((pBlock = tsdbGetSubmitMsgNext(&msgIter)) != NULL) { - if ((code = tsdbInsertDataToTable(repo, pBlock, now, &affectedrows)) != TSDB_CODE_SUCCESS) { + if ((code = tsdbInsertDataToTable(pRepo, pBlock, now, &affectedrows)) != TSDB_CODE_SUCCESS) { return code; } } @@ -506,700 +174,6 @@ int32_t tsdbInsertData(TSDB_REPO_T *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg * return code; } -void tsdbClearTableCfg(STableCfg *config) { - if (config) { - if (config->schema) tdFreeSchema(config->schema); - if (config->tagSchema) tdFreeSchema(config->tagSchema); - if (config->tagValues) kvRowFree(config->tagValues); - tfree(config->name); - tfree(config->sname); - tfree(config->sql); - free(config); - } -} - -int tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter) { - if (pBlock->len <= 0) return -1; - pIter->totalLen = pBlock->len; - pIter->len = 0; - pIter->row = (SDataRow)(pBlock->data); - return 0; -} - -SDataRow tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter) { - SDataRow row = pIter->row; - if (row == NULL) return NULL; - - pIter->len += dataRowLen(row); - if (pIter->len >= pIter->totalLen) { - pIter->row = NULL; - } else { - pIter->row = (char *)row + dataRowLen(row); - } - - return row; -} - -int tsdbInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter) { - if (pMsg == NULL || pIter == NULL) return -1; - - pMsg->length = htonl(pMsg->length); - pMsg->numOfBlocks = htonl(pMsg->numOfBlocks); - pMsg->compressed = htonl(pMsg->compressed); - - pIter->totalLen = pMsg->length; - pIter->len = TSDB_SUBMIT_MSG_HEAD_SIZE; - if (pMsg->length <= TSDB_SUBMIT_MSG_HEAD_SIZE) { - pIter->pBlock = NULL; - } else { - pIter->pBlock = pMsg->blocks; - } - - return 0; -} - -SSubmitBlk *tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter) { - SSubmitBlk *pBlock = pIter->pBlock; - if (pBlock == NULL) return NULL; - - pBlock->len = htonl(pBlock->len); - pBlock->numOfRows = htons(pBlock->numOfRows); - pBlock->uid = htobe64(pBlock->uid); - pBlock->tid = htonl(pBlock->tid); - - pBlock->sversion = htonl(pBlock->sversion); - pBlock->padding = htonl(pBlock->padding); - - pIter->len = pIter->len + sizeof(SSubmitBlk) + pBlock->len; - if (pIter->len >= pIter->totalLen) { - pIter->pBlock = NULL; - } else { - pIter->pBlock = (SSubmitBlk *)((char *)pBlock + pBlock->len + sizeof(SSubmitBlk)); - } - - return pBlock; -} - -STsdbMeta* tsdbGetMeta(TSDB_REPO_T* pRepo) { - STsdbRepo *tsdb = (STsdbRepo *)pRepo; - return tsdb->tsdbMeta; -} - -STsdbFileH* tsdbGetFile(TSDB_REPO_T* pRepo) { - STsdbRepo* tsdb = (STsdbRepo*) pRepo; - return tsdb->tsdbFileH; -} - -// Check the configuration and set default options -static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { - // Check precision - if (pCfg->precision == -1) { - pCfg->precision = TSDB_DEFAULT_PRECISION; - } else { - if (!IS_VALID_PRECISION(pCfg->precision)) { - tsdbError("vgId:%d, invalid precision configuration! precision:%d", pCfg->tsdbId, pCfg->precision); - return -1; - } - } - - // Check compression - if (pCfg->compression == -1) { - pCfg->compression = TSDB_DEFAULT_COMPRESSION; - } else { - if (!IS_VALID_COMPRESSION(pCfg->compression)) { - tsdbError("vgId:%d: invalid compression configuration! compression:%d", pCfg->tsdbId, pCfg->precision); - return -1; - } - } - - // Check tsdbId - if (pCfg->tsdbId < 0) return -1; - - // Check maxTables - if (pCfg->maxTables == -1) { - pCfg->maxTables = TSDB_DEFAULT_TABLES; - } else { - if (pCfg->maxTables < TSDB_MIN_TABLES || pCfg->maxTables > TSDB_MAX_TABLES) { - tsdbError("vgId:%d: invalid maxTables configuration! maxTables:%d TSDB_MIN_TABLES:%d TSDB_MAX_TABLES:%d", - pCfg->tsdbId, pCfg->maxTables, TSDB_MIN_TABLES, TSDB_MAX_TABLES); - return -1; - } - } - - // Check daysPerFile - if (pCfg->daysPerFile == -1) { - pCfg->daysPerFile = TSDB_DEFAULT_DAYS_PER_FILE; - } else { - if (pCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) { - tsdbError( - "vgId:%d, invalid daysPerFile configuration! daysPerFile:%d TSDB_MIN_DAYS_PER_FILE:%d TSDB_MAX_DAYS_PER_FILE:" - "%d", - pCfg->tsdbId, pCfg->daysPerFile, TSDB_MIN_DAYS_PER_FILE, TSDB_MAX_DAYS_PER_FILE); - return -1; - } - } - - // Check minRowsPerFileBlock and maxRowsPerFileBlock - if (pCfg->minRowsPerFileBlock == -1) { - pCfg->minRowsPerFileBlock = TSDB_DEFAULT_MIN_ROW_FBLOCK; - } else { - if (pCfg->minRowsPerFileBlock < TSDB_MIN_MIN_ROW_FBLOCK || pCfg->minRowsPerFileBlock > TSDB_MAX_MIN_ROW_FBLOCK) { - tsdbError( - "vgId:%d, invalid minRowsPerFileBlock configuration! minRowsPerFileBlock:%d TSDB_MIN_MIN_ROW_FBLOCK:%d " - "TSDB_MAX_MIN_ROW_FBLOCK:%d", - pCfg->tsdbId, pCfg->minRowsPerFileBlock, TSDB_MIN_MIN_ROW_FBLOCK, TSDB_MAX_MIN_ROW_FBLOCK); - return -1; - } - } - - if (pCfg->maxRowsPerFileBlock == -1) { - pCfg->maxRowsPerFileBlock = TSDB_DEFAULT_MAX_ROW_FBLOCK; - } else { - if (pCfg->maxRowsPerFileBlock < TSDB_MIN_MAX_ROW_FBLOCK || pCfg->maxRowsPerFileBlock > TSDB_MAX_MAX_ROW_FBLOCK) { - tsdbError( - "vgId:%d, invalid maxRowsPerFileBlock configuration! maxRowsPerFileBlock:%d TSDB_MIN_MAX_ROW_FBLOCK:%d " - "TSDB_MAX_MAX_ROW_FBLOCK:%d", - pCfg->tsdbId, pCfg->maxRowsPerFileBlock, TSDB_MIN_MIN_ROW_FBLOCK, TSDB_MAX_MIN_ROW_FBLOCK); - return -1; - } - } - - if (pCfg->minRowsPerFileBlock > pCfg->maxRowsPerFileBlock) return -1; - - // Check keep - if (pCfg->keep == -1) { - pCfg->keep = TSDB_DEFAULT_KEEP; - } else { - if (pCfg->keep < TSDB_MIN_KEEP || pCfg->keep > TSDB_MAX_KEEP) { - tsdbError( - "vgId:%d, invalid keep configuration! keep:%d TSDB_MIN_KEEP:%d " - "TSDB_MAX_KEEP:%d", - pCfg->tsdbId, pCfg->keep, TSDB_MIN_KEEP, TSDB_MAX_KEEP); - return -1; - } - } - - return 0; -} - -static int32_t tsdbGetCfgFname(STsdbRepo *pRepo, char *fname) { - if (pRepo == NULL) return -1; - sprintf(fname, "%s/%s", pRepo->rootDir, TSDB_CFG_FILE_NAME); - return 0; -} - -static int32_t tsdbSaveConfig(STsdbRepo *pRepo) { - char fname[128] = "\0"; // TODO: get rid of the literal 128 - - if (tsdbGetCfgFname(pRepo, fname) < 0) return -1; - - int fd = open(fname, O_WRONLY | O_CREAT, 0755); - if (fd < 0) { - return -1; - } - - if (write(fd, (void *)(&(pRepo->config)), sizeof(STsdbCfg)) < 0) { - return -1; - } - - close(fd); - return 0; -} - -static int32_t tsdbRestoreCfg(STsdbRepo *pRepo, STsdbCfg *pCfg) { - char fname[128] = "\0"; - - if (tsdbGetCfgFname(pRepo, fname) < 0) return -1; - - int fd = open(fname, O_RDONLY); - if (fd < 0) { - return -1; - } - - if (read(fd, (void *)pCfg, sizeof(STsdbCfg)) < sizeof(STsdbCfg)) { - close(fd); - return -1; - } - - close(fd); - - return 0; -} - -static int32_t tsdbGetDataDirName(STsdbRepo *pRepo, char *fname) { - if (pRepo == NULL || pRepo->rootDir == NULL) return -1; - sprintf(fname, "%s/%s", pRepo->rootDir, TSDB_DATA_DIR_NAME); - return 0; -} - -static int32_t tsdbSetRepoEnv(STsdbRepo *pRepo) { - STsdbCfg *pCfg = &pRepo->config; - if (tsdbSaveConfig(pRepo) < 0) return -1; - - char dirName[128] = "\0"; - if (tsdbGetDataDirName(pRepo, dirName) < 0) return -1; - - if (mkdir(dirName, 0755) < 0) { - tsdbError("vgId:%d, failed to create repository directory! reason:%s", pRepo->config.tsdbId, strerror(errno)); - return -1; - } - - tsdbTrace( - "vgId:%d, set up tsdb environment succeed! cacheBlockSize:%d, totalBlocks:%d, maxTables:%d, daysPerFile:%d, keep:" - "%d, minRowsPerFileBlock:%d, maxRowsPerFileBlock:%d, precision:%d, compression:%d", - pRepo->config.tsdbId, pCfg->cacheBlockSize, pCfg->totalBlocks, pCfg->maxTables, pCfg->daysPerFile, pCfg->keep, - pCfg->minRowsPerFileBlock, pCfg->maxRowsPerFileBlock, pCfg->precision, pCfg->compression); - return 0; -} - -static int32_t tsdbDestroyRepoEnv(STsdbRepo *pRepo) { - char fname[260]; - if (pRepo == NULL) return 0; - char *dirName = calloc(1, strlen(pRepo->rootDir) + strlen("tsdb") + 2); - if (dirName == NULL) { - return -1; - } - - sprintf(dirName, "%s/%s", pRepo->rootDir, "tsdb"); - - DIR *dir = opendir(dirName); - if (dir == NULL) return -1; - - struct dirent *dp; - while ((dp = readdir(dir)) != NULL) { - if ((strcmp(dp->d_name, ".") == 0) || (strcmp(dp->d_name, "..") == 0)) continue; - sprintf(fname, "%s/%s", pRepo->rootDir, dp->d_name); - remove(fname); - } - - closedir(dir); - - rmdir(dirName); - - return 0; -} - -// static int tsdbOpenMetaFile(char *tsdbDir) { -// // TODO -// return 0; -// } - -static int32_t tdInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable) { - // TODO - int32_t level = 0; - int32_t headSize = 0; - - if (pTable->mem == NULL) { - pTable->mem = (SMemTable *)calloc(1, sizeof(SMemTable)); - if (pTable->mem == NULL) return -1; - pTable->mem->pData = tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], 0, 0, 0, getTSTupleKey); - pTable->mem->keyFirst = INT64_MAX; - pTable->mem->keyLast = 0; - } - - tSkipListNewNodeInfo(pTable->mem->pData, &level, &headSize); - - TSKEY key = dataRowKey(row); - // printf("insert:%lld, size:%d\n", key, pTable->mem->numOfRows); - - // Copy row into the memory - SSkipListNode *pNode = tsdbAllocFromCache(pRepo->tsdbCache, headSize + dataRowLen(row), key); - if (pNode == NULL) { - // TODO: deal with allocate failure - } - - pNode->level = level; - dataRowCpy(SL_GET_NODE_DATA(pNode), row); - - // Insert the skiplist node into the data - if (pTable->mem == NULL) { - pTable->mem = (SMemTable *)calloc(1, sizeof(SMemTable)); - if (pTable->mem == NULL) return -1; - pTable->mem->pData = tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], 0, 0, 0, getTSTupleKey); - pTable->mem->keyFirst = INT64_MAX; - pTable->mem->keyLast = 0; - } - tSkipListPut(pTable->mem->pData, pNode); - if (key > pTable->mem->keyLast) pTable->mem->keyLast = key; - if (key < pTable->mem->keyFirst) pTable->mem->keyFirst = key; - if (key > pTable->lastKey) pTable->lastKey = key; - - pTable->mem->numOfRows = tSkipListGetSize(pTable->mem->pData); - - tsdbTrace("vgId:%d, tid:%d, uid:%" PRId64 ", table:%s a row is inserted to table! key:%" PRId64, pRepo->config.tsdbId, - pTable->tableId.tid, pTable->tableId.uid, varDataVal(pTable->name), dataRowKey(row)); - - return 0; -} - -static int32_t tsdbInsertDataToTable(TSDB_REPO_T *repo, SSubmitBlk *pBlock, TSKEY now, int32_t *affectedrows) { - STsdbRepo *pRepo = (STsdbRepo *)repo; - STsdbMeta *pMeta = pRepo->tsdbMeta; - int64_t points = 0; - - STableId tableId = {.uid = pBlock->uid, .tid = pBlock->tid}; - STable *pTable = tsdbIsValidTableToInsert(pRepo->tsdbMeta, tableId); - if (pTable == NULL) { - tsdbError("vgId:%d, failed to get table for insert, uid:" PRIu64 ", tid:%d", pRepo->config.tsdbId, pBlock->uid, - pBlock->tid); - return TSDB_CODE_TDB_INVALID_TABLE_ID; - } - - // Check schema version - int32_t tversion = pBlock->sversion; - STSchema * pSchema = tsdbGetTableSchema(pMeta, pTable); - ASSERT(pSchema != NULL); - int16_t nversion = schemaVersion(pSchema); - if (tversion > nversion) { - tsdbTrace("vgId:%d table:%s tid:%d server schema version %d is older than clien version %d, try to config.", - pRepo->config.tsdbId, varDataVal(pTable->name), pTable->tableId.tid, nversion, tversion); - void *msg = (*pRepo->appH.configFunc)(pRepo->config.tsdbId, pTable->tableId.tid); - if (msg == NULL) { - return terrno; - } - // Deal with error her - STableCfg *pTableCfg = tsdbCreateTableCfgFromMsg(msg); - STable *pTableUpdate = NULL; - if (pTable->type == TSDB_CHILD_TABLE) { - pTableUpdate = tsdbGetTableByUid(pMeta, pTableCfg->superUid); - } else { - pTableUpdate = pTable; - } - - int32_t code = tsdbUpdateTable(pMeta, pTableUpdate, pTableCfg); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - tsdbClearTableCfg(pTableCfg); - rpcFreeCont(msg); - } else { - if (tsdbGetTableSchemaByVersion(pMeta, pTable, tversion) == NULL) { - tsdbError("vgId:%d table:%s tid:%d invalid schema version %d from client", pRepo->config.tsdbId, - varDataVal(pTable->name), pTable->tableId.tid, tversion); - return TSDB_CODE_TDB_TABLE_SCHEMA_VERSION; - } - } - - SSubmitBlkIter blkIter = {0}; - SDataRow row = NULL; - - TSKEY minKey = now - tsMsPerDay[pRepo->config.precision] * pRepo->config.keep; - TSKEY maxKey = now + tsMsPerDay[pRepo->config.precision] * pRepo->config.daysPerFile; - - tsdbInitSubmitBlkIter(pBlock, &blkIter); - while ((row = tsdbGetSubmitBlkNext(&blkIter)) != NULL) { - if (dataRowKey(row) < minKey || dataRowKey(row) > maxKey) { - tsdbError("vgId:%d, table:%s, tid:%d, talbe uid:%ld timestamp is out of range. now:" PRId64 ", maxKey:" PRId64 - ", minKey:" PRId64, - pRepo->config.tsdbId, varDataVal(pTable->name), pTable->tableId.tid, pTable->tableId.uid, now, minKey, maxKey); - return TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE; - } - - if (tdInsertRowToTable(pRepo, row, pTable) < 0) { - return -1; - } - (*affectedrows)++; - points++; - } - atomic_fetch_add_64(&(pRepo->stat.pointsWritten), points * (pSchema->numOfCols)); - atomic_fetch_add_64(&(pRepo->stat.totalStorage), points * pSchema->vlen); - - return TSDB_CODE_SUCCESS; -} - -static int tsdbReadRowsFromCache(STsdbMeta *pMeta, STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols) { - ASSERT(maxRowsToRead > 0); - if (pIter == NULL) return 0; - STSchema *pSchema = NULL; - - int numOfRows = 0; - - do { - if (numOfRows >= maxRowsToRead) break; - - SSkipListNode *node = tSkipListIterGet(pIter); - if (node == NULL) break; - - SDataRow row = SL_GET_NODE_DATA(node); - if (dataRowKey(row) > maxKey) break; - - if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) { - pSchema = tsdbGetTableSchemaByVersion(pMeta, pTable, dataRowVersion(row)); - if (pSchema == NULL) { - // TODO: deal with the error here - ASSERT(false); - } - } - - tdAppendDataRowToDataCol(row, pSchema, pCols); - numOfRows++; - } while (tSkipListIterNext(pIter)); - - return numOfRows; -} - -static void tsdbDestroyTableIters(SSkipListIterator **iters, int maxTables) { - if (iters == NULL) return; - - for (int tid = 1; tid < maxTables; tid++) { - if (iters[tid] == NULL) continue; - tSkipListDestroyIter(iters[tid]); - } - - free(iters); -} - -static SSkipListIterator **tsdbCreateTableIters(STsdbMeta *pMeta, int maxTables) { - SSkipListIterator **iters = (SSkipListIterator **)calloc(maxTables, sizeof(SSkipListIterator *)); - if (iters == NULL) return NULL; - - for (int tid = 1; tid < maxTables; tid++) { - STable *pTable = pMeta->tables[tid]; - if (pTable == NULL || pTable->imem == NULL || pTable->imem->numOfRows == 0) continue; - - iters[tid] = tSkipListCreateIter(pTable->imem->pData); - if (iters[tid] == NULL) goto _err; - - if (!tSkipListIterNext(iters[tid])) goto _err; - } - - return iters; - - _err: - tsdbDestroyTableIters(iters, maxTables); - return NULL; -} - -static void tsdbFreeMemTable(SMemTable *pMemTable) { - if (pMemTable) { - tSkipListDestroy(pMemTable->pData); - free(pMemTable); - } -} - -// Commit to file -static void *tsdbCommitData(void *arg) { - STsdbRepo * pRepo = (STsdbRepo *)arg; - STsdbMeta * pMeta = pRepo->tsdbMeta; - STsdbCache *pCache = pRepo->tsdbCache; - STsdbCfg * pCfg = &(pRepo->config); - SDataCols * pDataCols = NULL; - SRWHelper whelper = {{0}}; - if (pCache->imem == NULL) return NULL; - - tsdbPrint("vgId:%d, starting to commit....", pRepo->config.tsdbId); - - // Create the iterator to read from cache - SSkipListIterator **iters = tsdbCreateTableIters(pMeta, pCfg->maxTables); - if (iters == NULL) { - ASSERT(0); - return NULL; - } - - if (tsdbInitWriteHelper(&whelper, pRepo) < 0) goto _exit; - if ((pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pCfg->maxRowsPerFileBlock)) == NULL) goto _exit; - - int sfid = tsdbGetKeyFileId(pCache->imem->keyFirst, pCfg->daysPerFile, pCfg->precision); - int efid = tsdbGetKeyFileId(pCache->imem->keyLast, pCfg->daysPerFile, pCfg->precision); - - // Loop to commit to each file - for (int fid = sfid; fid <= efid; fid++) { - if (tsdbCommitToFile(pRepo, fid, iters, &whelper, pDataCols) < 0) { - ASSERT(false); - goto _exit; - } - } - - // Do retention actions - tsdbFitRetention(pRepo); - if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_OVER); - -_exit: - tdFreeDataCols(pDataCols); - tsdbDestroyTableIters(iters, pCfg->maxTables); - tsdbDestroyHelper(&whelper); - - tsdbLockRepo(arg); - tdListMove(pCache->imem->list, pCache->pool.memPool); - tsdbAdjustCacheBlocks(pCache); - tdListFree(pCache->imem->list); - free(pCache->imem); - pCache->imem = NULL; - pRepo->commit = 0; - for (int i = 1; i < pCfg->maxTables; i++) { - STable *pTable = pMeta->tables[i]; - if (pTable && pTable->imem) { - tsdbFreeMemTable(pTable->imem); - pTable->imem = NULL; - } - } - tsdbUnLockRepo(arg); - tsdbPrint("vgId:%d, commit over....", pRepo->config.tsdbId); - - return NULL; -} - -static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters, SRWHelper *pHelper, SDataCols *pDataCols) { - char dataDir[128] = {0}; - STsdbMeta * pMeta = pRepo->tsdbMeta; - STsdbFileH *pFileH = pRepo->tsdbFileH; - STsdbCfg * pCfg = &pRepo->config; - SFileGroup *pGroup = NULL; - - TSKEY minKey = 0, maxKey = 0; - tsdbGetKeyRangeOfFileId(pCfg->daysPerFile, pCfg->precision, fid, &minKey, &maxKey); - - // Check if there are data to commit to this file - int hasDataToCommit = tsdbHasDataToCommit(iters, pCfg->maxTables, minKey, maxKey); - if (!hasDataToCommit) return 0; // No data to commit, just return - - // Create and open files for commit - tsdbGetDataDirName(pRepo, dataDir); - if ((pGroup = tsdbCreateFGroup(pFileH, dataDir, fid, pCfg->maxTables)) == NULL) { - tsdbError("vgId:%d, failed to create file group %d", pRepo->config.tsdbId, fid); - goto _err; - } - - // Open files for write/read - if (tsdbSetAndOpenHelperFile(pHelper, pGroup) < 0) { - tsdbError("vgId:%d, failed to set helper file", pRepo->config.tsdbId); - goto _err; - } - - // Loop to commit data in each table - for (int tid = 1; tid < pCfg->maxTables; tid++) { - STable * pTable = pMeta->tables[tid]; - if (pTable == NULL) continue; - - SSkipListIterator *pIter = iters[tid]; - - // Set the helper and the buffer dataCols object to help to write this table - tsdbSetHelperTable(pHelper, pTable, pRepo); - tdInitDataCols(pDataCols, tsdbGetTableSchema(pMeta, pTable)); - - // Loop to write the data in the cache to files. If no data to write, just break the loop - int maxRowsToRead = pCfg->maxRowsPerFileBlock * 4 / 5; - int nLoop = 0; - while (true) { - int rowsRead = tsdbReadRowsFromCache(pMeta, pTable, pIter, maxKey, maxRowsToRead, pDataCols); - assert(rowsRead >= 0); - if (pDataCols->numOfRows == 0) break; - nLoop++; - - ASSERT(dataColsKeyFirst(pDataCols) >= minKey && dataColsKeyFirst(pDataCols) <= maxKey); - ASSERT(dataColsKeyLast(pDataCols) >= minKey && dataColsKeyLast(pDataCols) <= maxKey); - - int rowsWritten = tsdbWriteDataBlock(pHelper, pDataCols); - ASSERT(rowsWritten != 0); - if (rowsWritten < 0) goto _err; - ASSERT(rowsWritten <= pDataCols->numOfRows); - - tdPopDataColsPoints(pDataCols, rowsWritten); - maxRowsToRead = pCfg->maxRowsPerFileBlock * 4 / 5 - pDataCols->numOfRows; - } - - ASSERT(pDataCols->numOfRows == 0); - - // Move the last block to the new .l file if neccessary - if (tsdbMoveLastBlockIfNeccessary(pHelper) < 0) { - tsdbError("vgId:%d, failed to move last block", pRepo->config.tsdbId); - goto _err; - } - - // Write the SCompBlock part - if (tsdbWriteCompInfo(pHelper) < 0) { - tsdbError("vgId:%d, failed to write compInfo part", pRepo->config.tsdbId); - goto _err; - } - } - - if (tsdbWriteCompIdx(pHelper) < 0) { - tsdbError("vgId:%d, failed to write compIdx part", pRepo->config.tsdbId); - goto _err; - } - - tsdbCloseHelperFile(pHelper, 0); - // TODO: make it atomic with some methods - pGroup->files[TSDB_FILE_TYPE_HEAD] = pHelper->files.headF; - pGroup->files[TSDB_FILE_TYPE_DATA] = pHelper->files.dataF; - pGroup->files[TSDB_FILE_TYPE_LAST] = pHelper->files.lastF; - - return 0; - - _err: - ASSERT(false); - tsdbCloseHelperFile(pHelper, 1); - return -1; -} - -/** - * Return the next iterator key. - * - * @return the next key if iter has - * -1 if iter not - */ -static TSKEY tsdbNextIterKey(SSkipListIterator *pIter) { - if (pIter == NULL) return -1; - - SSkipListNode *node = tSkipListIterGet(pIter); - if (node == NULL) return -1; - - SDataRow row = SL_GET_NODE_DATA(node); - return dataRowKey(row); -} - -static int tsdbHasDataToCommit(SSkipListIterator **iters, int nIters, TSKEY minKey, TSKEY maxKey) { - TSKEY nextKey; - for (int i = 0; i < nIters; i++) { - SSkipListIterator *pIter = iters[i]; - nextKey = tsdbNextIterKey(pIter); - if (nextKey > 0 && (nextKey >= minKey && nextKey <= maxKey)) return 1; - } - return 0; -} - -static void tsdbAlterCompression(STsdbRepo *pRepo, int8_t compression) { - int8_t oldCompRession = pRepo->config.compression; - pRepo->config.compression = compression; - tsdbTrace("vgId:%d, tsdb compression is changed from %d to %d", oldCompRession, compression); -} - -static void tsdbAlterKeep(STsdbRepo *pRepo, int32_t keep) { - STsdbCfg *pCfg = &pRepo->config; - int oldKeep = pCfg->keep; - - int maxFiles = keep / pCfg->maxTables + 3; - if (pRepo->config.keep > keep) { - pRepo->config.keep = keep; - pRepo->tsdbFileH->maxFGroups = maxFiles; - } else { - pRepo->config.keep = keep; - pRepo->tsdbFileH->fGroup = realloc(pRepo->tsdbFileH->fGroup, sizeof(SFileGroup)); - if (pRepo->tsdbFileH->fGroup == NULL) { - // TODO: deal with the error - } - pRepo->tsdbFileH->maxFGroups = maxFiles; - } - tsdbTrace("vgId:%d, keep is changed from %d to %d", pRepo->config.tsdbId, oldKeep, keep); -} - -static void tsdbAlterMaxTables(STsdbRepo *pRepo, int32_t maxTables) { - int oldMaxTables = pRepo->config.maxTables; - if (oldMaxTables < pRepo->config.maxTables) { - // TODO - } - - STsdbMeta *pMeta = pRepo->tsdbMeta; - - pMeta->maxTables = maxTables; - pMeta->tables = realloc(pMeta->tables, maxTables * sizeof(STable *)); - memset(&pMeta->tables[oldMaxTables], 0, sizeof(STable *) * (maxTables-oldMaxTables)); - pRepo->config.maxTables = maxTables; - - tsdbTrace("vgId:%d, tsdb maxTables is changed from %d to %d!", pRepo->config.tsdbId, oldMaxTables, maxTables); -} - -#define TSDB_META_FILE_INDEX 10000000 uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_t eindex, int32_t *size) { STsdbRepo *pRepo = (STsdbRepo *)repo; // STsdbMeta *pMeta = pRepo->tsdbMeta; @@ -1270,10 +244,1118 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_ return magic; } +int tsdbUpdateTagValue(TSDB_REPO_T *repo, SUpdateTableTagValMsg *pMsg) { + STsdbRepo *pRepo = (STsdbRepo *)repo; + STsdbMeta *pMeta = pRepo->tsdbMeta; + int16_t tversion = htons(pMsg->tversion); + + STable *pTable = tsdbGetTableByUid(pMeta, htobe64(pMsg->uid)); + if (pTable == NULL) return TSDB_CODE_TDB_INVALID_TABLE_ID; + if (pTable->tableId.tid != htonl(pMsg->tid)) return TSDB_CODE_TDB_INVALID_TABLE_ID; + + if (pTable->type != TSDB_CHILD_TABLE) { + tsdbError("vgId:%d failed to update tag value of table %s since its type is %d", pRepo->config.tsdbId, + varDataVal(pTable->name), pTable->type); + return TSDB_CODE_TDB_INVALID_TABLE_TYPE; + } + + if (schemaVersion(tsdbGetTableTagSchema(pMeta, pTable)) < tversion) { + tsdbTrace("vgId:%d server tag version %d is older than client tag version %d, try to config", pRepo->config.tsdbId, + schemaVersion(tsdbGetTableTagSchema(pMeta, pTable)), tversion); + void *msg = (*pRepo->appH.configFunc)(pRepo->config.tsdbId, htonl(pMsg->tid)); + if (msg == NULL) { + return terrno; + } + // Deal with error her + STableCfg *pTableCfg = tsdbCreateTableCfgFromMsg(msg); + STable * super = tsdbGetTableByUid(pMeta, pTableCfg->superUid); + ASSERT(super != NULL); + + int32_t code = tsdbUpdateTable(pMeta, super, pTableCfg); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + tsdbClearTableCfg(pTableCfg); + rpcFreeCont(msg); + } + + STSchema *pTagSchema = tsdbGetTableTagSchema(pMeta, pTable); + + if (schemaVersion(pTagSchema) > tversion) { + tsdbError( + "vgId:%d failed to update tag value of table %s since version out of date, client tag version:%d server tag " + "version:%d", + pRepo->config.tsdbId, varDataVal(pTable->name), tversion, schemaVersion(pTable->tagSchema)); + return TSDB_CODE_TDB_TAG_VER_OUT_OF_DATE; + } + if (schemaColAt(pTagSchema, DEFAULT_TAG_INDEX_COLUMN)->colId == htons(pMsg->colId)) { + tsdbRemoveTableFromIndex(pMeta, pTable); + } + // TODO: remove table from index if it is the first column of tag + tdSetKVRowDataOfCol(&pTable->tagVal, htons(pMsg->colId), htons(pMsg->type), pMsg->data); + if (schemaColAt(pTagSchema, DEFAULT_TAG_INDEX_COLUMN)->colId == htons(pMsg->colId)) { + tsdbAddTableIntoIndex(pMeta, pTable); + } + return TSDB_CODE_SUCCESS; +} + +void tsdbStartStream(TSDB_REPO_T *repo) { + STsdbRepo *pRepo = (STsdbRepo *)repo; + STsdbMeta *pMeta = pRepo->tsdbMeta; + + for (int i = 0; i < pRepo->config.maxTables; i++) { + STable *pTable = pMeta->tables[i]; + if (pTable && pTable->type == TSDB_STREAM_TABLE) { + pTable->cqhandle = (*pRepo->appH.cqCreateFunc)(pRepo->appH.cqH, pTable->tableId.uid, pTable->tableId.tid, + pTable->sql, tsdbGetTableSchema(pMeta, pTable)); + } + } +} + +STsdbCfg *tsdbGetCfg(const TSDB_REPO_T *repo) { + ASSERT(repo != NULL); + return &((STsdbRepo *)repo)->config; +} + +// ----------------- INTERNAL FUNCTIONS ----------------- +char *tsdbGetMetaFileName(char *rootDir) { + int tlen = strlen(rootDir) + strlen(TSDB_META_FILE_NAME) + 2; + char *fname = calloc(1, tlen); + if (fname == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return NULL; + } + + snprintf(fname, tlen, "%s/%s", rootDir, TSDB_META_FILE_NAME); + return fname; +} + +int tsdbLockRepo(STsdbRepo *pRepo) { + int code = pthread_mutex_lock(&pRepo->mutex); + if (code != 0) { + tsdbError("vgId:%d failed to lock tsdb since %s", REPO_ID(pRepo), strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + return 0; +} + +int tsdbUnlockRepo(STsdbRepo *pRepo) { + int code = pthread_mutex_unlock(&pRepo->mutex); + if (code != 0) { + tsdbError("vgId:%d failed to unlock tsdb since %s", REPO_ID(pRepo), strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + return 0; +} + +void *tsdbCommitData(void *arg) { + STsdbRepo *pRepo = (STsdbRepo *)arg; + STsdbMeta *pMeta = pRepo->tsdbMeta; + ASSERT(pRepo->imem != NULL); + ASSERT(pRepo->commit == 1); + + tsdbPrint("vgId:%d start to commit! keyFirst " PRId64 " keyLast " PRId64 " numOfRows " PRId64, REPO_ID(pRepo), + pRepo->imem->keyFirst, pRepo->imem->keyLast, pRepo->imem->numOfRows); + + // STsdbMeta * pMeta = pRepo->tsdbMeta; + // STsdbCache *pCache = pRepo->tsdbCache; + // STsdbCfg * pCfg = &(pRepo->config); + // SDataCols * pDataCols = NULL; + // SRWHelper whelper = {{0}}; + // if (pCache->imem == NULL) return NULL; + + tsdbPrint("vgId:%d, starting to commit....", pRepo->config.tsdbId); + + // Create the iterator to read from cache + SSkipListIterator **iters = tsdbCreateTableIters(pRepo); + if (iters == NULL) { + tsdbError("vgId:%d failed to create table iterators since %s", REPO_ID(pRepo), tstrerror(terrno)); + // TODO: deal with the error here + return NULL; + } + + if (tsdbInitWriteHelper(&whelper, pRepo) < 0) { + tsdbError("vgId:%d failed to init write helper since %s", REPO_ID(pRepo), tstrerror(terrno)); + // TODO + goto _exit; + } + + if ((pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pCfg->maxRowsPerFileBlock)) == NULL) { + tsdbError("vgId:%d failed to init data cols with maxRowBytes %d maxCols %d since %s", REPO_ID(pRepo), + pMeta->maxRowBytes, pMeta->maxCols, tstrerror(terrno)); + // TODO + goto _exit; + } + + int sfid = tsdbGetKeyFileId(pCache->imem->keyFirst, pCfg->daysPerFile, pCfg->precision); + int efid = tsdbGetKeyFileId(pCache->imem->keyLast, pCfg->daysPerFile, pCfg->precision); + + // Loop to commit to each file + for (int fid = sfid; fid <= efid; fid++) { + if (tsdbCommitToFile(pRepo, fid, iters, &whelper, pDataCols) < 0) { + ASSERT(false); + goto _exit; + } + } + + // Do retention actions + tsdbFitRetention(pRepo); + if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_OVER); + +_exit: + tdFreeDataCols(pDataCols); + tsdbDestroyTableIters(iters, pCfg->maxTables); + tsdbDestroyHelper(&whelper); + + tsdbLockRepo(arg); + tdListMove(pCache->imem->list, pCache->pool.memPool); + tsdbAdjustCacheBlocks(pCache); + tdListFree(pCache->imem->list); + free(pCache->imem); + pCache->imem = NULL; + pRepo->commit = 0; + for (int i = 1; i < pCfg->maxTables; i++) { + STable *pTable = pMeta->tables[i]; + if (pTable && pTable->imem) { + tsdbFreeMemTable(pTable->imem); + pTable->imem = NULL; + } + } + tsdbUnLockRepo(arg); + tsdbPrint("vgId:%d, commit over....", pRepo->config.tsdbId); + + return NULL; +} + +// ----------------- LOCAL FUNCTIONS ----------------- +static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { + // Check precision + if (pCfg->precision == -1) { + pCfg->precision = TSDB_DEFAULT_PRECISION; + } else { + if (!IS_VALID_PRECISION(pCfg->precision)) { + tsdbError("vgId:%d invalid precision configuration %d", pCfg->tsdbId, pCfg->precision); + goto _err; + } + } + + // Check compression + if (pCfg->compression == -1) { + pCfg->compression = TSDB_DEFAULT_COMPRESSION; + } else { + if (!IS_VALID_COMPRESSION(pCfg->compression)) { + tsdbError("vgId:%d invalid compression configuration %d", pCfg->tsdbId, pCfg->precision); + goto _err; + } + } + + // Check tsdbId + if (pCfg->tsdbId < 0) { + tsdbError("vgId:%d invalid vgroup ID", pCfg->tsdbId); + goto _err; + } + + // Check maxTables + if (pCfg->maxTables == -1) { + pCfg->maxTables = TSDB_DEFAULT_TABLES; + } else { + if (pCfg->maxTables < TSDB_MIN_TABLES || pCfg->maxTables > TSDB_MAX_TABLES) { + tsdbError("vgId:%d invalid maxTables configuration! maxTables %d TSDB_MIN_TABLES %d TSDB_MAX_TABLES %d", + pCfg->tsdbId, pCfg->maxTables, TSDB_MIN_TABLES, TSDB_MAX_TABLES); + goto _err; + } + } + + // Check daysPerFile + if (pCfg->daysPerFile == -1) { + pCfg->daysPerFile = TSDB_DEFAULT_DAYS_PER_FILE; + } else { + if (pCfg->daysPerFile < TSDB_MIN_DAYS_PER_FILE || pCfg->daysPerFile > TSDB_MAX_DAYS_PER_FILE) { + tsdbError( + "vgId:%d invalid daysPerFile configuration! daysPerFile %d TSDB_MIN_DAYS_PER_FILE %d TSDB_MAX_DAYS_PER_FILE " + "%d", + pCfg->tsdbId, pCfg->daysPerFile, TSDB_MIN_DAYS_PER_FILE, TSDB_MAX_DAYS_PER_FILE); + goto _err; + } + } + + // Check minRowsPerFileBlock and maxRowsPerFileBlock + if (pCfg->minRowsPerFileBlock == -1) { + pCfg->minRowsPerFileBlock = TSDB_DEFAULT_MIN_ROW_FBLOCK; + } else { + if (pCfg->minRowsPerFileBlock < TSDB_MIN_MIN_ROW_FBLOCK || pCfg->minRowsPerFileBlock > TSDB_MAX_MIN_ROW_FBLOCK) { + tsdbError( + "vgId:%d invalid minRowsPerFileBlock configuration! minRowsPerFileBlock %d TSDB_MIN_MIN_ROW_FBLOCK %d " + "TSDB_MAX_MIN_ROW_FBLOCK %d", + pCfg->tsdbId, pCfg->minRowsPerFileBlock, TSDB_MIN_MIN_ROW_FBLOCK, TSDB_MAX_MIN_ROW_FBLOCK); + goto _err; + } + } + + if (pCfg->maxRowsPerFileBlock == -1) { + pCfg->maxRowsPerFileBlock = TSDB_DEFAULT_MAX_ROW_FBLOCK; + } else { + if (pCfg->maxRowsPerFileBlock < TSDB_MIN_MAX_ROW_FBLOCK || pCfg->maxRowsPerFileBlock > TSDB_MAX_MAX_ROW_FBLOCK) { + tsdbError( + "vgId:%d invalid maxRowsPerFileBlock configuration! maxRowsPerFileBlock %d TSDB_MIN_MAX_ROW_FBLOCK %d " + "TSDB_MAX_MAX_ROW_FBLOCK %d", + pCfg->tsdbId, pCfg->maxRowsPerFileBlock, TSDB_MIN_MIN_ROW_FBLOCK, TSDB_MAX_MIN_ROW_FBLOCK); + goto _err; + } + } + + if (pCfg->minRowsPerFileBlock > pCfg->maxRowsPerFileBlock) { + tsdbError("vgId:%d invalid configuration! minRowsPerFileBlock %d maxRowsPerFileBlock %d" pCfg->tsdbId, + pCfg->minRowsPerFileBlock, pCfg->maxRowsPerFileBlock); + goto _err; + } + + // Check keep + if (pCfg->keep == -1) { + pCfg->keep = TSDB_DEFAULT_KEEP; + } else { + if (pCfg->keep < TSDB_MIN_KEEP || pCfg->keep > TSDB_MAX_KEEP) { + tsdbError( + "vgId:%d invalid keep configuration! keep %d TSDB_MIN_KEEP %d " + "TSDB_MAX_KEEP %d", + pCfg->tsdbId, pCfg->keep, TSDB_MIN_KEEP, TSDB_MAX_KEEP); + goto _err; + } + } + + return 0; + +_err: + terrno = TSDB_CODE_TDB_INVALID_CONFIG; + return -1; +} + +static int32_t tsdbSetRepoEnv(char *rootDir, STsdbCfg *pCfg) { + if (tsdbSaveConfig(rootDir, pCfg) < 0) { + tsdbError("vgId:%d failed to set TSDB environment since %s", pCfg->tsdbId, tstrerror(terrno)); + return -1; + } + + char *dirName = tsdbGetDataDirName(rootDir); + if (dirName == NULL) return -1; + + if (mkdir(dirName, 0755) < 0) { + tsdbError("vgId:%d failed to create directory %s since %s", pCfg->tsdbId, dirName, strerror(errno)); + errno = TAOS_SYSTEM_ERROR(errno); + free(dirName); + return -1; + } + + free(dirName); + + char *fname = tsdbGetMetaFileName(rootDir); + if (fname == NULL) return -1; + if (tdCreateKVStore(fname) < 0) { + tsdbError("vgId:%d failed to open KV store since %s", pCfg->tsdbId, tstrerror(terrno)); + free(fname); + return -1; + } + + free(fname); + return 0; +} + +static int32_t tsdbUnsetRepoEnv(char *rootDir) { + taosRemoveDir(rootDir); + tsdbTrace("repository %s is removed", rootDir); + return 0; +} + +static int32_t tsdbSaveConfig(char *rootDir, STsdbCfg *pCfg) { + int fd = -1; + char *fname = NULL; + + fname = tsdbGetCfgFname(rootDir); + if (fname == NULL) { + tsdbError("vgId:%d failed to save configuration since %s", pCfg->tsdbId, tstrerror(terrno)); + goto _err; + } + + fd = open(fname, O_WRONLY | O_CREAT, 0755); + if (fd < 0) { + tsdbError("vgId:%d failed to open file %s since %s", pCfg->tsdbId, fname, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err + } + + if (twrite(fd, (void *)pCfg, sizeof(STsdbCfg)) < sizeof(STsdbCfg)) { + tsdbError("vgId:%d failed to write %d bytes to file %s since %s", pCfg->tsdbId, sizeof(STsdbCfg), fname, + strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + if (fsync(fd) < 0) { + tsdbError("vgId:%d failed to fsync file %s since %s", pCfg->tsdbId, fname, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + free(fname); + close(fd); + return 0; + +_err: + tfree(fname); + if (fd > 0) close(fd); + return -1; +} + +static int tsdbLoadConfig(char *rootDir, STsdbCfg *pCfg) { + char *fname = NULL; + int fd = -1; + + fname = tsdbGetCfgFname(rootDir); + if (fname == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + goto _err; + } + + fd = open(fname, O_RDONLY); + if (fd < 0) { + tsdbError("failed to open file %s since %s", fname, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + if (tread(fd, (void *)pCfg, sizeof(*pCfg)) < sizeof(*pCfg)) { + tsdbError("failed to read %d bytes from file %s since %s", sizeof(*pCfg), fname, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + + tfree(fname); + close(fd); + + return 0; + +_err: + tfree(fname); + if (fd > 0) close(fd); + return -1; +} + +static char *tsdbGetCfgFname(char *rootDir) { + int tlen = strlen(rootDir) + strlen(TSDB_CFG_FILE_NAME) + 2; + char *fname = calloc(1, tlen); + if (fname == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return NULL; + } + + snprintf(fname, tlen, "%s/%s", rootDir, TSDB_CFG_FILE_NAME); + return fname; +} + +static char *tsdbGetDataDirName(char *rootDir) { + int tlen = strlen(rootDir) + strlen(TSDB_DATA_DIR_NAME) + 2; + char *fname = calloc(1, tlen); + if (fname == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return NULL; + } + + snprintf(fname, tlen, "%s/%s", rootDir, TSDB_DATA_DIR_NAME); + return fname; +} + +static STsdbRepo *tsdbNewRepo(char *rootDir, STsdbAppH *pAppH, STsdbCfg *pCfg) { + STsdbRepo *pRepo = (STsdbRepo *)calloc(1, sizeof(STsdbRepo)); + if (pRepo == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + goto _err; + } + + int code = pthread_mutex_init(&pRepo->mutex, NULL); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(code); + goto _err; + } + + pRepo->rootDir = strdup(rootDir); + if (pRepo->rootDir == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + goto _err; + } + + pRepo->config = *pCfg; + pRepo->appH = *pAppH; + + pRepo->tsdbMeta = tsdbNewMeta(pCfg); + if (pRepo->tsdbMeta == NULL) { + tsdbError("vgId:%d failed to create meta since %s", REPO_ID(pRepo), tstrerror(terrno)); + goto _err; + } + + pRepo->pPool = tsdbNewBufPool(pCfg); + if (pRepo->pPool == NULL) { + tsdbError("vgId:%d failed to create buffer pool since %s", REPO_ID(pRepo), tstrerror(terrno)); + goto _err; + } + + pRepo->tsdbFileH = tsdbNewFileH(pRepo); + if (pRepo->tsdbFileH == NULL) { + tsdbError("vgId:%d failed to create file handle since %s", REPO_ID(pRepo), tstrerror(terrno)); + goto _err; + } + + return pRepo; + +_err: + tsdbFreeRepo(pRepo); + return NULL; +} + +static void tsdbFreeRepo(STsdbRepo *pRepo) { + if (pRepo) { + tsdbFreeFileH(pRepo->tsdbFileH); + tsdbFreeBufPool(pRepo->pPool); + tsdbFreeMeta(pRepo->tsdbMeta); + tsdbFreeMemTable(pRepo->mem); + tsdbFreeMemTable(pRepo->imem); + tfree(pRepo->rootDir); + pthread_mutex_destroy(&pRepo->mutex); + free(pRepo); + } +} + +static int tsdbInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter) { + if (pMsg == NULL) { + terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP; + return -1; + } + + pMsg->length = htonl(pMsg->length); + pMsg->numOfBlocks = htonl(pMsg->numOfBlocks); + pMsg->compressed = htonl(pMsg->compressed); + + pIter->totalLen = pMsg->length; + pIter->len = TSDB_SUBMIT_MSG_HEAD_SIZE; + if (pMsg->length <= TSDB_SUBMIT_MSG_HEAD_SIZE) { + terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP; + return -1; + } else { + pIter->pBlock = pMsg->blocks; + } + + return 0; +} + +static int32_t tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, TSKEY now, int32_t *affectedrows) { + STsdbMeta *pMeta = pRepo->tsdbMeta; + int64_t points = 0; + + STable *pTable == tsdbGetTableByUid(pMeta, pBlock->uid); + if (pTable == NULL || TABLE_TID(pTable)) { + tsdbError("vgId:%d failed to get table to insert data, uid " PRIu64 " tid %d", REPO_ID(pRepo), pBlock->uid, + pBlock->tid); + terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; + return -1; + } + + if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) { + tsdbError("vgId:%d invalid action trying to insert a super table %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable)); + terrno = TSDB_CODE_TDB_INVALID_ACTION; + return -1; + } + + // // Check schema version + // int32_t tversion = pBlock->sversion; + // STSchema * pSchema = tsdbGetTableSchema(pMeta, pTable); + // ASSERT(pSchema != NULL); + // int16_t nversion = schemaVersion(pSchema); + // if (tversion > nversion) { + // tsdbTrace("vgId:%d table:%s tid:%d server schema version %d is older than clien version %d, try to config.", + // pRepo->config.tsdbId, varDataVal(pTable->name), pTable->tableId.tid, nversion, tversion); + // void *msg = (*pRepo->appH.configFunc)(pRepo->config.tsdbId, pTable->tableId.tid); + // if (msg == NULL) { + // return terrno; + // } + // // Deal with error her + // STableCfg *pTableCfg = tsdbCreateTableCfgFromMsg(msg); + // STable *pTableUpdate = NULL; + // if (pTable->type == TSDB_CHILD_TABLE) { + // pTableUpdate = tsdbGetTableByUid(pMeta, pTableCfg->superUid); + // } else { + // pTableUpdate = pTable; + // } + + // int32_t code = tsdbUpdateTable(pMeta, pTableUpdate, pTableCfg); + // if (code != TSDB_CODE_SUCCESS) { + // return code; + // } + // tsdbClearTableCfg(pTableCfg); + // rpcFreeCont(msg); + // } else { + // if (tsdbGetTableSchemaByVersion(pMeta, pTable, tversion) == NULL) { + // tsdbError("vgId:%d table:%s tid:%d invalid schema version %d from client", pRepo->config.tsdbId, + // varDataVal(pTable->name), pTable->tableId.tid, tversion); + // return TSDB_CODE_TDB_TABLE_SCHEMA_VERSION; + // } + // } + + SSubmitBlkIter blkIter = {0}; + SDataRow row = NULL; + + TSKEY minKey = now - tsMsPerDay[pRepo->config.precision] * pRepo->config.keep; + TSKEY maxKey = now + tsMsPerDay[pRepo->config.precision] * pRepo->config.daysPerFile; + + tsdbInitSubmitBlkIter(pBlock, &blkIter); + while ((row = tsdbGetSubmitBlkNext(&blkIter)) != NULL) { + if (dataRowKey(row) < minKey || dataRowKey(row) > maxKey) { + tsdbError("vgId:%d table %s tid %d uid %ld timestamp is out of range! now " PRId64 " maxKey " PRId64 + " minKey " PRId64, + REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TALBE_UID(pTable), now, minKey, maxKey); + return TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE; + } + + if (tdInsertRowToTable(pRepo, row, pTable) < 0) { + return -1; + } + (*affectedrows)++; + points++; + } + atomic_fetch_add_64(&(pRepo->stat.pointsWritten), points * (pSchema->numOfCols)); + atomic_fetch_add_64(&(pRepo->stat.totalStorage), points * pSchema->vlen); + + return 0; +} + +static SSubmitBlk *tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter) { + SSubmitBlk *pBlock = pIter->pBlock; + if (pBlock == NULL) return NULL; + + pBlock->len = htonl(pBlock->len); + pBlock->numOfRows = htons(pBlock->numOfRows); + pBlock->uid = htobe64(pBlock->uid); + pBlock->tid = htonl(pBlock->tid); + + pBlock->sversion = htonl(pBlock->sversion); + pBlock->padding = htonl(pBlock->padding); + + pIter->len = pIter->len + sizeof(SSubmitBlk) + pBlock->len; + if (pIter->len >= pIter->totalLen) { + pIter->pBlock = NULL; + } else { + pIter->pBlock = (SSubmitBlk *)((char *)pBlock + pBlock->len + sizeof(SSubmitBlk)); + } + + return pBlock; +} + +static SDataRow tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter) { + SDataRow row = pIter->row; + if (row == NULL) return NULL; + + pIter->len += dataRowLen(row); + if (pIter->len >= pIter->totalLen) { + pIter->row = NULL; + } else { + pIter->row = (char *)row + dataRowLen(row); + } + + return row; +} + +static int32_t tsdbInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable) { + // TODO + int32_t level = 0; + int32_t headSize = 0; + + if (pTable->mem == NULL) { + pTable->mem = (SMemTable *)calloc(1, sizeof(SMemTable)); + if (pTable->mem == NULL) return -1; + pTable->mem->pData = + tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], 0, 0, 0, getTSTupleKey); + pTable->mem->keyFirst = INT64_MAX; + pTable->mem->keyLast = 0; + } + + tSkipListNewNodeInfo(pTable->mem->pData, &level, &headSize); + + TSKEY key = dataRowKey(row); + // printf("insert:%lld, size:%d\n", key, pTable->mem->numOfRows); + + // Copy row into the memory + SSkipListNode *pNode = tsdbAllocFromCache(pRepo->tsdbCache, headSize + dataRowLen(row), key); + if (pNode == NULL) { + // TODO: deal with allocate failure + } + + pNode->level = level; + dataRowCpy(SL_GET_NODE_DATA(pNode), row); + + // Insert the skiplist node into the data + if (pTable->mem == NULL) { + pTable->mem = (SMemTable *)calloc(1, sizeof(SMemTable)); + if (pTable->mem == NULL) return -1; + pTable->mem->pData = + tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], 0, 0, 0, getTSTupleKey); + pTable->mem->keyFirst = INT64_MAX; + pTable->mem->keyLast = 0; + } + tSkipListPut(pTable->mem->pData, pNode); + if (key > pTable->mem->keyLast) pTable->mem->keyLast = key; + if (key < pTable->mem->keyFirst) pTable->mem->keyFirst = key; + if (key > pTable->lastKey) pTable->lastKey = key; + + pTable->mem->numOfRows = tSkipListGetSize(pTable->mem->pData); + + tsdbTrace("vgId:%d, tid:%d, uid:%" PRId64 ", table:%s a row is inserted to table! key:%" PRId64, pRepo->config.tsdbId, + pTable->tableId.tid, pTable->tableId.uid, varDataVal(pTable->name), dataRowKey(row)); + + return 0; +} + +static SSkipListIterator **tsdbCreateTableIters(STsdbRepo *pRepo) { + STsdbCfg *pCfg = &(pRepo->config); + + SSkipListIterator **iters = (SSkipListIterator **)calloc(pCfg->maxTables, sizeof(SSkipListIterator *)); + if (iters == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return NULL; + } + + for (int tid = 1; tid < maxTables; tid++) { + STable *pTable = pMeta->tables[tid]; + if (pTable == NULL || pTable->imem == NULL || pTable->imem->numOfRows == 0) continue; + + iters[tid] = tSkipListCreateIter(pTable->imem->pData); + if (iters[tid] == NULL) goto _err; + + if (!tSkipListIterNext(iters[tid])) goto _err; + } + + return iters; + +_err: + tsdbDestroyTableIters(iters, maxTables); + return NULL; +} + +static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters, SRWHelper *pHelper, + SDataCols *pDataCols) { + char dataDir[128] = {0}; + STsdbMeta * pMeta = pRepo->tsdbMeta; + STsdbFileH *pFileH = pRepo->tsdbFileH; + STsdbCfg * pCfg = &pRepo->config; + SFileGroup *pGroup = NULL; + + TSKEY minKey = 0, maxKey = 0; + tsdbGetKeyRangeOfFileId(pCfg->daysPerFile, pCfg->precision, fid, &minKey, &maxKey); + + // Check if there are data to commit to this file + int hasDataToCommit = tsdbHasDataToCommit(iters, pCfg->maxTables, minKey, maxKey); + if (!hasDataToCommit) return 0; // No data to commit, just return + + // Create and open files for commit + tsdbGetDataDirName(pRepo, dataDir); + if ((pGroup = tsdbCreateFGroup(pFileH, dataDir, fid, pCfg->maxTables)) == NULL) { + tsdbError("vgId:%d, failed to create file group %d", pRepo->config.tsdbId, fid); + goto _err; + } + + // Open files for write/read + if (tsdbSetAndOpenHelperFile(pHelper, pGroup) < 0) { + tsdbError("vgId:%d, failed to set helper file", pRepo->config.tsdbId); + goto _err; + } + + // Loop to commit data in each table + for (int tid = 1; tid < pCfg->maxTables; tid++) { + STable *pTable = pMeta->tables[tid]; + if (pTable == NULL) continue; + + SSkipListIterator *pIter = iters[tid]; + + // Set the helper and the buffer dataCols object to help to write this table + tsdbSetHelperTable(pHelper, pTable, pRepo); + tdInitDataCols(pDataCols, tsdbGetTableSchema(pMeta, pTable)); + + // Loop to write the data in the cache to files. If no data to write, just break the loop + int maxRowsToRead = pCfg->maxRowsPerFileBlock * 4 / 5; + int nLoop = 0; + while (true) { + int rowsRead = tsdbReadRowsFromCache(pMeta, pTable, pIter, maxKey, maxRowsToRead, pDataCols); + assert(rowsRead >= 0); + if (pDataCols->numOfRows == 0) break; + nLoop++; + + ASSERT(dataColsKeyFirst(pDataCols) >= minKey && dataColsKeyFirst(pDataCols) <= maxKey); + ASSERT(dataColsKeyLast(pDataCols) >= minKey && dataColsKeyLast(pDataCols) <= maxKey); + + int rowsWritten = tsdbWriteDataBlock(pHelper, pDataCols); + ASSERT(rowsWritten != 0); + if (rowsWritten < 0) goto _err; + ASSERT(rowsWritten <= pDataCols->numOfRows); + + tdPopDataColsPoints(pDataCols, rowsWritten); + maxRowsToRead = pCfg->maxRowsPerFileBlock * 4 / 5 - pDataCols->numOfRows; + } + + ASSERT(pDataCols->numOfRows == 0); + + // Move the last block to the new .l file if neccessary + if (tsdbMoveLastBlockIfNeccessary(pHelper) < 0) { + tsdbError("vgId:%d, failed to move last block", pRepo->config.tsdbId); + goto _err; + } + + // Write the SCompBlock part + if (tsdbWriteCompInfo(pHelper) < 0) { + tsdbError("vgId:%d, failed to write compInfo part", pRepo->config.tsdbId); + goto _err; + } + } + + if (tsdbWriteCompIdx(pHelper) < 0) { + tsdbError("vgId:%d, failed to write compIdx part", pRepo->config.tsdbId); + goto _err; + } + + tsdbCloseHelperFile(pHelper, 0); + // TODO: make it atomic with some methods + pGroup->files[TSDB_FILE_TYPE_HEAD] = pHelper->files.headF; + pGroup->files[TSDB_FILE_TYPE_DATA] = pHelper->files.dataF; + pGroup->files[TSDB_FILE_TYPE_LAST] = pHelper->files.lastF; + + return 0; + +_err: + ASSERT(false); + tsdbCloseHelperFile(pHelper, 1); + return -1; +} + +#if 0 +** + * Set the default TSDB configuration + */ + +static int tsdbRestoreInfo(STsdbRepo *pRepo) { + STsdbMeta * pMeta = pRepo->tsdbMeta; + STsdbFileH *pFileH = pRepo->tsdbFileH; + SFileGroup *pFGroup = NULL; + + SFileGroupIter iter; + SRWHelper rhelper = {{0}}; + + if (tsdbInitReadHelper(&rhelper, pRepo) < 0) goto _err; + tsdbInitFileGroupIter(pFileH, &iter, TSDB_ORDER_ASC); + while ((pFGroup = tsdbGetFileGroupNext(&iter)) != NULL) { + if (tsdbSetAndOpenHelperFile(&rhelper, pFGroup) < 0) goto _err; + for (int i = 1; i < pRepo->config.maxTables; i++) { + STable * pTable = pMeta->tables[i]; + if (pTable == NULL) continue; + SCompIdx *pIdx = &rhelper.pCompIdx[i]; + + if (pIdx->offset > 0 && pTable->lastKey < pIdx->maxKey) pTable->lastKey = pIdx->maxKey; + } + } + + tsdbDestroyHelper(&rhelper); + return 0; + +_err: + tsdbDestroyHelper(&rhelper); + return -1; +} + +/** + * Change the configuration of a repository + * @param pCfg the repository configuration, the upper layer should free the pointer + * + * @return 0 for success, -1 for failure and the error number is set + */ +int32_t tsdbConfigRepo(TSDB_REPO_T *repo, STsdbCfg *pCfg) { + STsdbRepo *pRepo = (STsdbRepo *)repo; + STsdbCfg * pRCfg = &pRepo->config; + + if (tsdbCheckAndSetDefaultCfg(pCfg) < 0) return TSDB_CODE_TDB_INVALID_CONFIG; + + ASSERT(pRCfg->tsdbId == pCfg->tsdbId); + ASSERT(pRCfg->cacheBlockSize == pCfg->cacheBlockSize); + ASSERT(pRCfg->daysPerFile == pCfg->daysPerFile); + ASSERT(pRCfg->minRowsPerFileBlock == pCfg->minRowsPerFileBlock); + ASSERT(pRCfg->maxRowsPerFileBlock == pCfg->maxRowsPerFileBlock); + ASSERT(pRCfg->precision == pCfg->precision); + + bool configChanged = false; + if (pRCfg->compression != pCfg->compression) { + configChanged = true; + tsdbAlterCompression(pRepo, pCfg->compression); + } + if (pRCfg->keep != pCfg->keep) { + configChanged = true; + tsdbAlterKeep(pRepo, pCfg->keep); + } + if (pRCfg->totalBlocks != pCfg->totalBlocks) { + configChanged = true; + tsdbAlterCacheTotalBlocks(pRepo, pCfg->totalBlocks); + } + if (pRCfg->maxTables != pCfg->maxTables) { + configChanged = true; + tsdbAlterMaxTables(pRepo, pCfg->maxTables); + } + + if (configChanged) tsdbSaveConfig(pRepo); + + return TSDB_CODE_SUCCESS; +} + + +/** + * Get the TSDB repository information, including some statistics + * @param pRepo the TSDB repository handle + * @param error the error number to set when failure occurs + * + * @return a info struct handle on success, NULL for failure and the error number is set. The upper + * layers should free the info handle themselves or memory leak will occur + */ +STsdbRepoInfo *tsdbGetStatus(TSDB_REPO_T *pRepo) { + // TODO + return NULL; +} + + +TSKEY tsdbGetTableLastKey(TSDB_REPO_T *repo, uint64_t uid) { + STsdbRepo *pRepo = (STsdbRepo *)repo; + + STable *pTable = tsdbGetTableByUid(pRepo->tsdbMeta, uid); + if (pTable == NULL) return -1; + + return TSDB_GET_TABLE_LAST_KEY(pTable); +} + + +STableInfo *tsdbGetTableInfo(TSDB_REPO_T *pRepo, STableId tableId) { + // TODO + return NULL; +} + +// TODO: need to return the number of data inserted + +void tsdbClearTableCfg(STableCfg *config) { + if (config) { + if (config->schema) tdFreeSchema(config->schema); + if (config->tagSchema) tdFreeSchema(config->tagSchema); + if (config->tagValues) kvRowFree(config->tagValues); + tfree(config->name); + tfree(config->sname); + tfree(config->sql); + free(config); + } +} + +int tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter) { + if (pBlock->len <= 0) return -1; + pIter->totalLen = pBlock->len; + pIter->len = 0; + pIter->row = (SDataRow)(pBlock->data); + return 0; +} + + + + +STsdbMeta* tsdbGetMeta(TSDB_REPO_T* pRepo) { + STsdbRepo *tsdb = (STsdbRepo *)pRepo; + return tsdb->tsdbMeta; +} + +STsdbFileH* tsdbGetFile(TSDB_REPO_T* pRepo) { + STsdbRepo* tsdb = (STsdbRepo*) pRepo; + return tsdb->tsdbFileH; +} + +// Check the configuration and set default options + + + +static int32_t tsdbRestoreCfg(STsdbRepo *pRepo, STsdbCfg *pCfg) { + char fname[128] = "\0"; + + if (tsdbGetCfgFname(pRepo, fname) < 0) return -1; + + int fd = open(fname, O_RDONLY); + if (fd < 0) { + return -1; + } + + if (read(fd, (void *)pCfg, sizeof(STsdbCfg)) < sizeof(STsdbCfg)) { + close(fd); + return -1; + } + + close(fd); + + return 0; +} + + + +static int32_t tsdbDestroyRepoEnv(STsdbRepo *pRepo) { + char fname[260]; + if (pRepo == NULL) return 0; + char *dirName = calloc(1, strlen(pRepo->rootDir) + strlen("tsdb") + 2); + if (dirName == NULL) { + return -1; + } + + sprintf(dirName, "%s/%s", pRepo->rootDir, "tsdb"); + + DIR *dir = opendir(dirName); + if (dir == NULL) return -1; + + struct dirent *dp; + while ((dp = readdir(dir)) != NULL) { + if ((strcmp(dp->d_name, ".") == 0) || (strcmp(dp->d_name, "..") == 0)) continue; + sprintf(fname, "%s/%s", pRepo->rootDir, dp->d_name); + remove(fname); + } + + closedir(dir); + + rmdir(dirName); + + return 0; +} + + +static int tsdbReadRowsFromCache(STsdbMeta *pMeta, STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols) { + ASSERT(maxRowsToRead > 0); + if (pIter == NULL) return 0; + STSchema *pSchema = NULL; + + int numOfRows = 0; + + do { + if (numOfRows >= maxRowsToRead) break; + + SSkipListNode *node = tSkipListIterGet(pIter); + if (node == NULL) break; + + SDataRow row = SL_GET_NODE_DATA(node); + if (dataRowKey(row) > maxKey) break; + + if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) { + pSchema = tsdbGetTableSchemaByVersion(pMeta, pTable, dataRowVersion(row)); + if (pSchema == NULL) { + // TODO: deal with the error here + ASSERT(false); + } + } + + tdAppendDataRowToDataCol(row, pSchema, pCols); + numOfRows++; + } while (tSkipListIterNext(pIter)); + + return numOfRows; +} + +static void tsdbDestroyTableIters(SSkipListIterator **iters, int maxTables) { + if (iters == NULL) return; + + for (int tid = 1; tid < maxTables; tid++) { + if (iters[tid] == NULL) continue; + tSkipListDestroyIter(iters[tid]); + } + + free(iters); +} + + +static void tsdbFreeMemTable(SMemTable *pMemTable) { + if (pMemTable) { + tSkipListDestroy(pMemTable->pData); + free(pMemTable); + } +} + +// Commit to file + + +/** + * Return the next iterator key. + * + * @return the next key if iter has + * -1 if iter not + */ +static TSKEY tsdbNextIterKey(SSkipListIterator *pIter) { + if (pIter == NULL) return -1; + + SSkipListNode *node = tSkipListIterGet(pIter); + if (node == NULL) return -1; + + SDataRow row = SL_GET_NODE_DATA(node); + return dataRowKey(row); +} + +static int tsdbHasDataToCommit(SSkipListIterator **iters, int nIters, TSKEY minKey, TSKEY maxKey) { + TSKEY nextKey; + for (int i = 0; i < nIters; i++) { + SSkipListIterator *pIter = iters[i]; + nextKey = tsdbNextIterKey(pIter); + if (nextKey > 0 && (nextKey >= minKey && nextKey <= maxKey)) return 1; + } + return 0; +} + +static void tsdbAlterCompression(STsdbRepo *pRepo, int8_t compression) { + int8_t oldCompRession = pRepo->config.compression; + pRepo->config.compression = compression; + tsdbTrace("vgId:%d, tsdb compression is changed from %d to %d", oldCompRession, compression); +} + +static void tsdbAlterKeep(STsdbRepo *pRepo, int32_t keep) { + STsdbCfg *pCfg = &pRepo->config; + int oldKeep = pCfg->keep; + + int maxFiles = keep / pCfg->maxTables + 3; + if (pRepo->config.keep > keep) { + pRepo->config.keep = keep; + pRepo->tsdbFileH->maxFGroups = maxFiles; + } else { + pRepo->config.keep = keep; + pRepo->tsdbFileH->fGroup = realloc(pRepo->tsdbFileH->fGroup, sizeof(SFileGroup)); + if (pRepo->tsdbFileH->fGroup == NULL) { + // TODO: deal with the error + } + pRepo->tsdbFileH->maxFGroups = maxFiles; + } + tsdbTrace("vgId:%d, keep is changed from %d to %d", pRepo->config.tsdbId, oldKeep, keep); +} + +static void tsdbAlterMaxTables(STsdbRepo *pRepo, int32_t maxTables) { + int oldMaxTables = pRepo->config.maxTables; + if (oldMaxTables < pRepo->config.maxTables) { + // TODO + } + + STsdbMeta *pMeta = pRepo->tsdbMeta; + + pMeta->maxTables = maxTables; + pMeta->tables = realloc(pMeta->tables, maxTables * sizeof(STable *)); + memset(&pMeta->tables[oldMaxTables], 0, sizeof(STable *) * (maxTables-oldMaxTables)); + pRepo->config.maxTables = maxTables; + + tsdbTrace("vgId:%d, tsdb maxTables is changed from %d to %d!", pRepo->config.tsdbId, oldMaxTables, maxTables); +} + + void tsdbReportStat(void *repo, int64_t *totalPoints, int64_t *totalStorage, int64_t *compStorage){ ASSERT(repo != NULL); STsdbRepo * pRepo = repo; *totalPoints = pRepo->stat.pointsWritten; *totalStorage = pRepo->stat.totalStorage; *compStorage = pRepo->stat.compStorage; -} \ No newline at end of file +} + +#endif \ No newline at end of file diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c new file mode 100644 index 0000000000..4ac5f5a5f2 --- /dev/null +++ b/src/tsdb/src/tsdbMemTable.c @@ -0,0 +1,282 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdb.h" +#include "tsdbMain.h" + +#define TSDB_DATA_SKIPLIST_LEVEL 5 + +// ---------------- INTERNAL FUNCTIONS ---------------- +int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { + STsdbCfg *pCfg = &pRepo->config; + int32_t level = 0; + int32_t headSize = 0; + TSKEY key = dataRowKey(row); + + // TODO + tSkipListNewNodeInfo(pRepo->mem->tData[TABLE_TID(pTable)]->pData, &level, &headSize); + + // TODO: for duplicate keys, you do not need to allocate memory here + SSkipListNode *pNode = tsdbAllocBytes(pRepo, headSize + dataRowLen(row)); + if (pNode == NULL) { + tsdbError("vgId:%d failed to insert row with key %" PRId64 " to table %s since %s", REPO_ID(pRepo), key, + TABLE_CHAR_NAME(pTable), tstrerror(terrno)); + return -1; + } + + SMemTable *pMemTable = pRepo->mem; + ASSERT(pMemTable != NULL); + + pNode->level = level; + dataRowCpy(SL_GET_NODE_DATA(pNode), row); + + STableData *pTableData = pMemTable->tData[TABLE_TID(pTable)]; + if (pTableData == NULL) { + pTableData = tsdbNewTableData(pCfg); + if (pTableData == NULL) { + tsdbError("vgId:%d failed to insert row with key %" PRId64 " to table %s since %s", REPO_ID(pRepo), key, + TABLE_CHAR_NAME(pTable), tstrerror(terrno)); + return -1; + } + + pRepo->mem->tData[TABLE_TID(pTable)] = pTableData; + } + + ASSERT(pTableData != NULL); + + if (pTableData->uid != TALBE_UID(pTable)) { + // TODO + } + + if (tSkipListPut(pTableData->pData, pNode) == NULL) { + tsdbFreeBytes(pRepo, (void *)pNode, headSize + dataRowLen); + return 0; + } + + if (pMemTable->keyFirst > key) pMemTable->keyFirst = key; + if (pMemTable->keyLast < key) pMemTable->keyLast = key; + pMemTable->numOfRows++; + + if (pTableData->keyFirst > key) pTableData->keyFirst = key; + if (pTableData->keyLast < key) pTableData->keyLast = key; + pTableData->numOfRows++; + + ASSERT(pTableData->numOfRows == tSkipListGetSize(pTableData->pData)); + + tsdbTrace("vgId:%d a row is inserted to table %s tid %d uid %" PRIu64 " key %" PRIu64, REPO_ID(pRepo), + TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TALBE_UID(pTable), key); + + return 0; +} + +int tsdbRefMemTable(STsdbRepo *pRepo, SMemTable *pMemTable) { + ASSERT(pMemTable != NULL); + T_REF_INC(pMemTable); +} + +// Need to lock the repository +int tsdbUnRefMemTable(STsdbRepo *pRepo, SMemTable *pMemTable) { + ASSERT(pMemTable != NULL); + + if (T_REF_DEC(pMemTable) == 0) { + STsdbCfg * pCfg = &pRepo->config; + STsdbBufPool *pBufPool = pRepo->pPool; + + SListNode *pNode = NULL; + while ((pNode = tdListPopHead(pMemTable->bufBlockList)) != NULL) { + tdListAppendNode(pBufPool->bufBlockList, pNode); + if (pthread_cond_signal(&pBufPool->poolNotEmpty) != 0) { + // TODO + } + } + + for (int i = 0; i < pCfg->maxTables; i++) { + if (pMemTable->tData[i] != NULL) { + tsdbFreeTableData(pMemTable->tData[i]); + } + } + + tdListDiscard(pMemTable->actList); + tdListDiscard(pMemTable->bufBlockList); + tsdbFreeMemTable(pMemTable); + } + return 0; +} + +// ---------------- LOCAL FUNCTIONS ---------------- +static FORCE_INLINE STsdbBufBlock *tsdbGetCurrBufBlock(STsdbRepo *pRepo) { + if (pRepo->mem == NULL) return NULL; + + SListNode *pNode = listTail(pRepo->mem); + if (pNode == NULL) return NULL; + + STsdbBufBlock *pBufBlock = NULL; + tdListNodeGetData(pMemTable->bufBlockList, pNode, (void *)(&pBufBlock)); + + return pBufBlock; +} + +static void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) { + STsdbCfg * pCfg = &pRepo->config; + STsdbBufBlock *pBufBlock = tsdbGetCurrBufBlock(pRepo); + + if (pBufBlock != NULL && pBufBlock->remain < bytes) { + if (listNEles(pRepo->mem) >= pCfg->totalBlocks / 2) { // need to trigger commit + if (pRepo->imem != NULL) { + if (pRepo->commit) pthread_join(pRepo->commitThread, NULL); + + ASSERT(pRepo->commit == 0); + + SMemTable *pIMem = pRepo->imem; + if (tsdbLockRepo(pRepo) < 0) { + // TODO + return NULL; + } + + pRepo->imem = pRepo->mem; + pRepo->mem = NULL; + pRepo->commit = 1; + if (pthread_create(&pRepo->commitThread, NULL, tsdbCommitData, (void *)pRepo) != 0) { + // TODO + tsdbUnlockRepo(pRepo); + return NULL; + } + if (tsdbUnlockRepo(pRepo) < 0) { + // TODO + return NULL; + } + + tsdbUnRefMemTable(pRepo, pIMem); + } + } + } + + if (pRepo->mem == NULL) { + SMemTable *pMemTable = tsdbNewMemTable(&pRepo->config); + if (pMemTable == NULL) return NULL; + + if (tsdbLockRepo(pRepo) < 0) { + tsdbFreeMemTable(pMemTable); + return NULL; + } + + SListNode *pNode = tsdbAllocBufBlockFromPool(pRepo); + tdListAppendNode(pMemTable->bufBlockList, pNode); + pRepo->mem = pMemTable; + + if (tsdbUnlockRepo(pRepo) < 0) return NULL; + } + + pBufBlock = tsdbGetCurrBufBlock(pRepo); + ASSERT(pBufBlock->remain >= bytes); + void *ptr = POINTER_SHIFT(pBufBlock->data, pBufBlock->offset); + pBufBlock->offset += bytes; + pBufBlock->remain -= bytes; + + return ptr; +} + +static void tsdbFreeBytes(STsdbRepo *pRepo, void *ptr, int bytes) { + STsdbBufBlock *pBufBlock = tsdbGetCurrBufBlock(pRepo); + ASSERT(pBufBlock != NULL); + pBufBlock->offset -= bytes; + pBufBlock->remain += bytes; + ASSERT(ptr == POINTER_SHIFT(pBufBlock->data, pBufBlock->offset)); +} + +static SMemTable* tsdbNewMemTable(STsdbCfg* pCfg) { + SMemTable *pMemTable = (SMemTable *)calloc(1, sizeof(*pMemTable)); + if (pMemTable == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + goto _err; + } + + pMemTable->keyFirst = INT64_MAX; + pMemTable->keyLast = 0; + pMemTable->numOfRows = 0; + + pMemTable->tData = (STableData**)calloc(pCfg->maxTables, sizeof(STableData*)); + if (pMemTable->tData == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + goto _err; + } + + pMemTable->actList = tdListNew(0); + if (pMemTable->actList == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + goto _err; + } + + pMemTable->bufBlockList = tdListNew(sizeof(STsdbBufBlock*)); + if (pMemTable->bufBlockList == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + goto _err; + } + + T_REF_INC(pMemTable); + + return pMemTable; + +_err: + tsdbFreeMemTable(pMemTable); + return NULL; +} + +static void tsdbFreeMemTable(SMemTable* pMemTable) { + if (pMemTable) { + ASSERT((pMemTable->bufBlockList == NULL) ? true : (listNEles(pMemTable->bufBlockList) == 0)); + ASSERT((pMemTable->actList == NULL) ? true : (listNEles(pMemTable->actList) == 0)); + + tdListFree(pMemTable->bufBlockList); + tdListFree(pMemTable->actList); + tfree(pMemTable->tData); + free(pMemTable); + } +} + +static STableData *tsdbNewTableData(STsdbCfg *pCfg, STable *pTable) { + STableData *pTableData = (STableData *)calloc(1, sizeof(*pTableData)); + if (pTableData == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + goto _err; + } + + pTableData->uid = TALBE_UID(pTable); + pTableData->keyFirst = INT64_MAX; + pTableData->keyLast = 0; + pTableData->numOfRows = 0; + + pTableData->pData = tSkipListCreate(TSDB_DATA_SKIPLIST_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], 0, 0, 0, ); + if (pTableData->pData == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + goto _err; + } + + // TODO + pTableData->pData->level = 1; + + return pTableData; + +_err: + tsdbFreeTableData(pTableData); + return NULL; +} + +static void tsdbFreeTableData(STableData *pTableData) { + if (pTableData) { + tSkipListDestroy(pTableData->pData); + free(pTableData); + } +} \ No newline at end of file diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index d0343999c1..63a6dd9f79 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -1,3 +1,17 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ #include #include "tskiplist.h" #include "tsdb.h" @@ -5,33 +19,337 @@ #include "hash.h" #include "tsdbMain.h" -#define TSDB_SUPER_TABLE_SL_LEVEL 5 // TODO: may change here -// #define TSDB_META_FILE_NAME "META" +// ------------------ OUTER FUNCTIONS ------------------ +int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg) { + STsdbRepo *pRepo = (STsdbRepo *)repo; + STsdbMeta *pMeta = pRepo->tsdbMeta; -static int tsdbFreeTable(STable *pTable); -static int32_t tsdbCheckTableCfg(STableCfg *pCfg); -static int tsdbAddTableToMeta(STsdbMeta *pMeta, STable *pTable, bool addIdx); -static int tsdbRemoveTableFromMeta(STsdbMeta *pMeta, STable *pTable, bool rmFromIdx); -static int tsdbInitTableCfg(STableCfg *config, ETableType type, uint64_t uid, int32_t tid); -static int tsdbTableSetSuperUid(STableCfg *config, uint64_t uid); -static int tsdbTableSetSchema(STableCfg *config, STSchema *pSchema, bool dup); -static int tsdbTableSetTagSchema(STableCfg *config, STSchema *pSchema, bool dup); -static int tsdbTableSetTagValue(STableCfg *config, SKVRow row, bool dup); -static int tsdbTableSetName(STableCfg *config, char *name, bool dup); -static int tsdbTableSetSName(STableCfg *config, char *sname, bool dup); -static int tsdbTableSetStreamSql(STableCfg *config, char *sql, bool dup); + if (tsdbCheckTableCfg(pCfg) < 0) return -1; -/** - * Encode a TSDB table object as a binary content - * ASSUMPTIONS: VALID PARAMETERS - * - * @param pTable table object to encode - * @param contLen the encoded binary content length - * - * @return binary content for success - * NULL fro failure - */ -void tsdbEncodeTable(STable *pTable, char *buf, int *contLen) { + STable *pTable = tsdbGetTableByUid(pMeta, pCfg->tableId.uid); + if (pTable != NULL) { + tsdbError("vgId:%d table %s already exists, tid %d uid %" PRId64, pRepo->config.tsdbId, varDataVal(pTable->name), + pTable->tableId.tid, pTable->tableId.uid); + return TSDB_CODE_TDB_TABLE_ALREADY_EXIST; + } + + STable *super = NULL; + int newSuper = 0; + + if (pCfg->type == TSDB_CHILD_TABLE) { + super = tsdbGetTableByUid(pMeta, pCfg->superUid); + if (super == NULL) { // super table not exists, try to create it + newSuper = 1; + super = tsdbNewTable(pCfg, true); + if (super == NULL) return -1; + } else { + if (super->type != TSDB_SUPER_TABLE) return -1; + if (super->tableId.uid != pCfg->superUid) return -1; + tsdbUpdateTable(pMeta, super, pCfg); + } + } + + STable *table = tsdbNewTable(pCfg, false); + if (table == NULL) { + if (newSuper) { + tsdbFreeTable(super); + return -1; + } + } + + table->lastKey = TSKEY_INITIAL_VAL; + + // Register to meta + if (newSuper) { + tsdbAddTableToMeta(pMeta, super, true); + tsdbTrace("vgId:%d, super table %s is created! uid:%" PRId64, pRepo->config.tsdbId, varDataVal(super->name), + super->tableId.uid); + } + tsdbAddTableToMeta(pMeta, table, true); + tsdbTrace("vgId:%d, table %s is created! tid:%d, uid:%" PRId64, pRepo->config.tsdbId, varDataVal(table->name), + table->tableId.tid, table->tableId.uid); + + // Write to meta file + int bufLen = 0; + char *buf = malloc(1024*1024); + if (newSuper) { + tsdbEncodeTable(super, buf, &bufLen); + tsdbInsertMetaRecord(pMeta->mfh, super->tableId.uid, buf, bufLen); + } + + tsdbEncodeTable(table, buf, &bufLen); + tsdbInsertMetaRecord(pMeta->mfh, table->tableId.uid, buf, bufLen); + tfree(buf); + + return 0; +} + +int tsdbDropTable(TSDB_REPO_T *repo, STableId tableId) { + STsdbRepo *pRepo = (STsdbRepo *)repo; + if (pRepo == NULL) return -1; + + STsdbMeta *pMeta = pRepo->tsdbMeta; + if (pMeta == NULL) return -1; + + STable *pTable = tsdbGetTableByUid(pMeta, tableId.uid); + if (pTable == NULL) { + tsdbError("vgId:%d, failed to drop table since table not exists! tid:%d, uid:" PRId64, pRepo->config.tsdbId, + tableId.tid, tableId.uid); + return -1; + } + + tsdbTrace("vgId:%d, table %s is dropped! tid:%d, uid:%" PRId64, pRepo->config.tsdbId, varDataVal(pTable->name), + tableId.tid, tableId.uid); + if (tsdbRemoveTableFromMeta(pMeta, pTable, true) < 0) return -1; + + return 0; + +} + +void* tsdbGetTableTagVal(TSDB_REPO_T* repo, const STableId* id, int32_t colId, int16_t type, int16_t bytes) { + STsdbMeta* pMeta = tsdbGetMeta(repo); + STable* pTable = tsdbGetTableByUid(pMeta, id->uid); + + STSchema *pSchema = tsdbGetTableTagSchema(pMeta, pTable); + STColumn *pCol = tdGetColOfID(pSchema, colId); + if (pCol == NULL) { + return NULL; // No matched tag volumn + } + + char* val = tdGetKVRowValOfCol(pTable->tagVal, colId); + assert(type == pCol->type && bytes == pCol->bytes); + + if (val != NULL && IS_VAR_DATA_TYPE(type)) { + assert(varDataLen(val) < pCol->bytes); + } + + return val; +} + +char *tsdbGetTableName(TSDB_REPO_T *repo, const STableId *id) { + STsdbMeta *pMeta = tsdbGetMeta(repo); + STable * pTable = tsdbGetTableByUid(pMeta, id->uid); + + if (pTable == NULL) { + return NULL; + } else { + return (char *)pTable->name; + } +} + +STableCfg *tsdbCreateTableCfgFromMsg(SMDCreateTableMsg *pMsg) { + if (pMsg == NULL) return NULL; + SSchema * pSchema = (SSchema *)pMsg->data; + int16_t numOfCols = htons(pMsg->numOfColumns); + int16_t numOfTags = htons(pMsg->numOfTags); + STSchemaBuilder schemaBuilder = {0}; + + STableCfg *pCfg = (STableCfg *)calloc(1, sizeof(STableCfg)); + if (pCfg == NULL) return NULL; + + if (tsdbInitTableCfg(pCfg, pMsg->tableType, htobe64(pMsg->uid), htonl(pMsg->sid)) < 0) goto _err; + if (tdInitTSchemaBuilder(&schemaBuilder, htonl(pMsg->sversion)) < 0) goto _err; + + for (int i = 0; i < numOfCols; i++) { + tdAddColToSchema(&schemaBuilder, pSchema[i].type, htons(pSchema[i].colId), htons(pSchema[i].bytes)); + } + if (tsdbTableSetSchema(pCfg, tdGetSchemaFromBuilder(&schemaBuilder), false) < 0) goto _err; + if (tsdbTableSetName(pCfg, pMsg->tableId, true) < 0) goto _err; + + if (numOfTags > 0) { + // Decode tag schema + tdResetTSchemaBuilder(&schemaBuilder, htonl(pMsg->tversion)); + for (int i = numOfCols; i < numOfCols + numOfTags; i++) { + tdAddColToSchema(&schemaBuilder, pSchema[i].type, htons(pSchema[i].colId), htons(pSchema[i].bytes)); + } + if (tsdbTableSetTagSchema(pCfg, tdGetSchemaFromBuilder(&schemaBuilder), false) < 0) goto _err; + if (tsdbTableSetSName(pCfg, pMsg->superTableId, true) < 0) goto _err; + if (tsdbTableSetSuperUid(pCfg, htobe64(pMsg->superTableUid)) < 0) goto _err; + + // Decode tag values + if (pMsg->tagDataLen) { + int accBytes = 0; + char *pTagData = pMsg->data + (numOfCols + numOfTags) * sizeof(SSchema); + + SKVRowBuilder kvRowBuilder = {0}; + if (tdInitKVRowBuilder(&kvRowBuilder) < 0) goto _err; + for (int i = numOfCols; i < numOfCols + numOfTags; i++) { + tdAddColToKVRow(&kvRowBuilder, htons(pSchema[i].colId), pSchema[i].type, pTagData + accBytes); + accBytes += htons(pSchema[i].bytes); + } + + tsdbTableSetTagValue(pCfg, tdGetKVRowFromBuilder(&kvRowBuilder), false); + tdDestroyKVRowBuilder(&kvRowBuilder); + } + } + + if (pMsg->tableType == TSDB_STREAM_TABLE) { + char *sql = pMsg->data + (numOfCols + numOfTags) * sizeof(SSchema); + tsdbTableSetStreamSql(pCfg, sql, true); + } + + tdDestroyTSchemaBuilder(&schemaBuilder); + + return pCfg; + +_err: + tdDestroyTSchemaBuilder(&schemaBuilder); + tsdbClearTableCfg(pCfg); + tfree(pCfg); + return NULL; +} + +// ------------------ INTERNAL FUNCTIONS ------------------ +STsdbMeta *tsdbNewMeta(STsdbCfg *pCfg) { + STsdbMeta* pMeta = (STsdbMeta*)calloc(1, sizeof(*pMeta)); + if (pMeta == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + goto _err; + } + + int code = pthread_rwlock_init(&pMeta->rwLock, NULL); + if (code != 0) { + terrno = TAOS_SYSTEM_ERROR(code); + goto _err; + } + + pMeta->tables = (STable **)calloc(pCfg->maxTables, sizeof(STable *)); + if (pMeta->tables == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + goto _err; + } + + pMeta->superList = tdListNew(sizeof(STable *)); + if (pMeta->superList == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + goto _err; + } + + pMeta->uidMap = taosHashInit(pCfg->maxTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false); + if (pMeta->uidMap == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + goto _err; + } + + return pMeta; + + _err; + tsdbFreeMeta(pMeta); + return NULL; +} + +void tsdbFreeMeta(STsdbMeta *pMeta) { + if (pMeta) { + taosHashCleanup(pMeta->uidMap); + tdListFree(pMeta->superList); + tfree(pMeta->tables); + pthread_rwlock_destroy(&pMeta->rwLock); + free(pMeta); + } +} + +int tsdbOpenMeta(STsdbRepo *pRepo) { + // TODO +} + +int tsdbCloseMeta(STsdbRepo *pRepo) { + // TODO +} + +STSchema *tsdbGetTableSchema(STsdbMeta *pMeta, STable *pTable) { + if (pTable->type == TSDB_NORMAL_TABLE || pTable->type == TSDB_SUPER_TABLE || pTable->type == TSDB_STREAM_TABLE) { + return pTable->schema[pTable->numOfSchemas - 1]; + } else if (pTable->type == TSDB_CHILD_TABLE) { + STable *pSuper = tsdbGetTableByUid(pMeta, pTable->superUid); + if (pSuper == NULL) return NULL; + return pSuper->schema[pSuper->numOfSchemas-1]; + } else { + return NULL; + } +} + +STable *tsdbGetTableByUid(STsdbMeta *pMeta, uint64_t uid) { + void *ptr = taosHashGet(pMeta->uidMap, (char *)(&uid), sizeof(uid)); + + if (ptr == NULL) return NULL; + + return *(STable **)ptr; +} + +STSchema *tsdbGetTableSchemaByVersion(STsdbMeta *pMeta, STable *pTable, int16_t version) { + STable *pSearchTable = NULL; + if (pTable->type == TSDB_CHILD_TABLE) { + pSearchTable = tsdbGetTableByUid(pMeta, pTable->superUid); + } else { + pSearchTable = pTable; + } + ASSERT(pSearchTable != NULL); + + void *ptr = taosbsearch(&version, pSearchTable->schema, pSearchTable->numOfSchemas, sizeof(STSchema *), + tsdbCompareSchemaVersion, TD_EQ); + if (ptr == NULL) return NULL; + + return *(STSchema **)ptr; +} + +STSchema * tsdbGetTableTagSchema(STsdbMeta *pMeta, STable *pTable) { + if (pTable->type == TSDB_SUPER_TABLE) { + return pTable->tagSchema; + } else if (pTable->type == TSDB_CHILD_TABLE) { + STable *pSuper = tsdbGetTableByUid(pMeta, pTable->superUid); + if (pSuper == NULL) return NULL; + return pSuper->tagSchema; + } else { + return NULL; + } +} + +int tsdbUpdateTable(STsdbMeta *pMeta, STable *pTable, STableCfg *pCfg) { + ASSERT(pTable->type != TSDB_CHILD_TABLE); + bool isChanged = false; + + if (pTable->type == TSDB_SUPER_TABLE) { + if (schemaVersion(pTable->tagSchema) < schemaVersion(pCfg->tagSchema)) { + int32_t code = tsdbUpdateTableTagSchema(pTable, pCfg->tagSchema); + if (code != TSDB_CODE_SUCCESS) return code; + } + isChanged = true; + } + + STSchema *pTSchema = tsdbGetTableSchema(pMeta, pTable); + if (schemaVersion(pTSchema) < schemaVersion(pCfg->schema)) { + if (pTable->numOfSchemas < TSDB_MAX_TABLE_SCHEMAS) { + pTable->schema[pTable->numOfSchemas++] = tdDupSchema(pCfg->schema); + } else { + ASSERT(pTable->numOfSchemas == TSDB_MAX_TABLE_SCHEMAS); + STSchema *tSchema = tdDupSchema(pCfg->schema); + tdFreeSchema(pTable->schema[0]); + memmove(pTable->schema, pTable->schema + 1, sizeof(STSchema *) * (TSDB_MAX_TABLE_SCHEMAS - 1)); + pTable->schema[pTable->numOfSchemas - 1] = tSchema; + } + + isChanged = true; + } + + if (isChanged) { + char *buf = malloc(1024 * 1024); + int bufLen = 0; + tsdbEncodeTable(pTable, buf, &bufLen); + tsdbInsertMetaRecord(pMeta->mfh, pTable->tableId.uid, buf, bufLen); + free(buf); + } + + return TSDB_CODE_SUCCESS; +} + +char *getTSTupleKey(const void * data) { + SDataRow row = (SDataRow)data; + return POINTER_SHIFT(row, TD_DATA_ROW_HEAD_SIZE); +} + + +// ------------------ LOCAL FUNCTIONS ------------------ +static void tsdbEncodeTable(STable *pTable, char *buf, int *contLen) { if (pTable == NULL) return; void *ptr = buf; @@ -68,17 +386,7 @@ void tsdbEncodeTable(STable *pTable, char *buf, int *contLen) { *contLen = (char *)ptr - buf; } -/** - * Decode from an encoded binary - * ASSUMPTIONS: valid parameters - * - * @param cont binary object - * @param contLen binary length - * - * @return TSDB table object for success - * NULL for failure - */ -STable *tsdbDecodeTable(void *cont, int contLen) { +static STable *tsdbDecodeTable(void *cont, int contLen) { STable *pTable = (STable *)calloc(1, sizeof(STable)); if (pTable == NULL) return NULL; @@ -127,20 +435,17 @@ STable *tsdbDecodeTable(void *cont, int contLen) { return pTable; } -void tsdbFreeEncode(void *cont) { - if (cont != NULL) free(cont); +static int tsdbCompareSchemaVersion(const void *key1, const void *key2) { + if (*(int16_t *)key1 < (*(STSchema **)key2)->version) { + return -1; + } else if (*(int16_t *)key1 > (*(STSchema **)key2)->version) { + return 1; + } else { + return 0; + } } -static char* getTagIndexKey(const void* pData) { - STableIndexElem* elem = (STableIndexElem*) pData; - - STSchema* pSchema = tsdbGetTableTagSchema(elem->pMeta, elem->pTable); - STColumn* pCol = &pSchema->columns[DEFAULT_TAG_INDEX_COLUMN]; - void * res = tdGetKVRowValOfCol(elem->pTable->tagVal, pCol->colId); - return res; -} - -int tsdbRestoreTable(void *pHandle, void *cont, int contLen) { +static int tsdbRestoreTable(void *pHandle, void *cont, int contLen) { STsdbMeta *pMeta = (STsdbMeta *)pHandle; STable *pTable = tsdbDecodeTable(cont, contLen); @@ -157,7 +462,7 @@ int tsdbRestoreTable(void *pHandle, void *cont, int contLen) { return 0; } -void tsdbOrgMeta(void *pHandle) { +static void tsdbOrgMeta(void *pHandle) { STsdbMeta *pMeta = (STsdbMeta *)pHandle; for (int i = 1; i < pMeta->maxTables; i++) { @@ -168,154 +473,13 @@ void tsdbOrgMeta(void *pHandle) { } } -/** - * Initialize the meta handle - * ASSUMPTIONS: VALID PARAMETER - */ -STsdbMeta *tsdbInitMeta(char *rootDir, int32_t maxTables, void *pRepo) { - STsdbMeta *pMeta = (STsdbMeta *)malloc(sizeof(STsdbMeta)); - if (pMeta == NULL) return NULL; +static char *getTagIndexKey(const void *pData) { + STableIndexElem *elem = (STableIndexElem *)pData; - pMeta->maxTables = maxTables; - pMeta->nTables = 0; - pMeta->superList = NULL; - pMeta->tables = (STable **)calloc(maxTables, sizeof(STable *)); - pMeta->maxRowBytes = 0; - pMeta->maxCols = 0; - pMeta->pRepo = pRepo; - if (pMeta->tables == NULL) { - free(pMeta); - return NULL; - } - - pMeta->map = taosHashInit(maxTables * TSDB_META_HASH_FRACTION, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false); - if (pMeta->map == NULL) { - free(pMeta->tables); - free(pMeta); - return NULL; - } - - pMeta->mfh = tsdbInitMetaFile(rootDir, maxTables, tsdbRestoreTable, tsdbOrgMeta, pMeta); - if (pMeta->mfh == NULL) { - taosHashCleanup(pMeta->map); - free(pMeta->tables); - free(pMeta); - return NULL; - } - - return pMeta; -} - -int32_t tsdbFreeMeta(STsdbMeta *pMeta) { - STsdbRepo *pRepo = (STsdbRepo *)pMeta->pRepo; - if (pMeta == NULL) return 0; - - tsdbCloseMetaFile(pMeta->mfh); - - for (int i = 1; i < pMeta->maxTables; i++) { - if (pMeta->tables[i] != NULL) { - STable *pTable = pMeta->tables[i]; - if (pTable->type == TSDB_STREAM_TABLE) (*pRepo->appH.cqDropFunc)(pTable->cqhandle); - tsdbFreeTable(pTable); - } - } - - free(pMeta->tables); - - STable *pTable = pMeta->superList; - while (pTable != NULL) { - STable *pTemp = pTable; - pTable = pTemp->next; - tsdbFreeTable(pTemp); - } - - taosHashCleanup(pMeta->map); - - free(pMeta); - - return 0; -} - -// Get the newest table schema -STSchema *tsdbGetTableSchema(STsdbMeta *pMeta, STable *pTable) { - if (pTable->type == TSDB_NORMAL_TABLE || pTable->type == TSDB_SUPER_TABLE || pTable->type == TSDB_STREAM_TABLE) { - return pTable->schema[pTable->numOfSchemas - 1]; - } else if (pTable->type == TSDB_CHILD_TABLE) { - STable *pSuper = tsdbGetTableByUid(pMeta, pTable->superUid); - if (pSuper == NULL) return NULL; - return pSuper->schema[pSuper->numOfSchemas-1]; - } else { - return NULL; - } -} - -static int tsdbCompareSchemaVersion(const void *key1, const void *key2) { - if (*(int16_t *)key1 < (*(STSchema **)key2)->version) { - return -1; - } else if (*(int16_t *)key1 > (*(STSchema **)key2)->version) { - return 1; - } else { - return 0; - } -} - -STSchema *tsdbGetTableSchemaByVersion(STsdbMeta *pMeta, STable *pTable, int16_t version) { - STable *pSearchTable = NULL; - if (pTable->type == TSDB_CHILD_TABLE) { - pSearchTable = tsdbGetTableByUid(pMeta, pTable->superUid); - } else { - pSearchTable = pTable; - } - ASSERT(pSearchTable != NULL); - - void *ptr = taosbsearch(&version, pSearchTable->schema, pSearchTable->numOfSchemas, sizeof(STSchema *), - tsdbCompareSchemaVersion, TD_EQ); - if (ptr == NULL) return NULL; - - return *(STSchema **)ptr; -} - -STSchema * tsdbGetTableTagSchema(STsdbMeta *pMeta, STable *pTable) { - if (pTable->type == TSDB_SUPER_TABLE) { - return pTable->tagSchema; - } else if (pTable->type == TSDB_CHILD_TABLE) { - STable *pSuper = tsdbGetTableByUid(pMeta, pTable->superUid); - if (pSuper == NULL) return NULL; - return pSuper->tagSchema; - } else { - return NULL; - } -} - -void* tsdbGetTableTagVal(TSDB_REPO_T* repo, const STableId* id, int32_t colId, int16_t type, int16_t bytes) { - STsdbMeta* pMeta = tsdbGetMeta(repo); - STable* pTable = tsdbGetTableByUid(pMeta, id->uid); - - STSchema *pSchema = tsdbGetTableTagSchema(pMeta, pTable); - STColumn *pCol = tdGetColOfID(pSchema, colId); - if (pCol == NULL) { - return NULL; // No matched tag volumn - } - - char* val = tdGetKVRowValOfCol(pTable->tagVal, colId); - assert(type == pCol->type && bytes == pCol->bytes); - - if (val != NULL && IS_VAR_DATA_TYPE(type)) { - assert(varDataLen(val) < pCol->bytes); - } - - return val; -} - -char* tsdbGetTableName(TSDB_REPO_T *repo, const STableId* id) { - STsdbMeta* pMeta = tsdbGetMeta(repo); - STable* pTable = tsdbGetTableByUid(pMeta, id->uid); - - if (pTable == NULL) { - return NULL; - } else { - return (char*) pTable->name; - } + STSchema *pSchema = tsdbGetTableTagSchema(elem->pMeta, elem->pTable); + STColumn *pCol = &pSchema->columns[DEFAULT_TAG_INDEX_COLUMN]; + void * res = tdGetKVRowValOfCol(elem->pTable->tagVal, pCol->colId); + return res; } static STable *tsdbNewTable(STableCfg *pCfg, bool isSuper) { @@ -395,234 +559,6 @@ _err: return NULL; } -static int tsdbUpdateTableTagSchema(STable *pTable, STSchema *newSchema) { - ASSERT(pTable->type == TSDB_SUPER_TABLE); - ASSERT(schemaVersion(pTable->tagSchema) < schemaVersion(newSchema)); - STSchema *pOldSchema = pTable->tagSchema; - STSchema *pNewSchema = tdDupSchema(newSchema); - if (pNewSchema == NULL) return TSDB_CODE_TDB_OUT_OF_MEMORY; - pTable->tagSchema = pNewSchema; - tdFreeSchema(pOldSchema); - - return TSDB_CODE_SUCCESS; -} - -int tsdbUpdateTable(STsdbMeta *pMeta, STable *pTable, STableCfg *pCfg) { - ASSERT(pTable->type != TSDB_CHILD_TABLE); - bool isChanged = false; - - if (pTable->type == TSDB_SUPER_TABLE) { - if (schemaVersion(pTable->tagSchema) < schemaVersion(pCfg->tagSchema)) { - int32_t code = tsdbUpdateTableTagSchema(pTable, pCfg->tagSchema); - if (code != TSDB_CODE_SUCCESS) return code; - } - isChanged = true; - } - - STSchema *pTSchema = tsdbGetTableSchema(pMeta, pTable); - if (schemaVersion(pTSchema) < schemaVersion(pCfg->schema)) { - if (pTable->numOfSchemas < TSDB_MAX_TABLE_SCHEMAS) { - pTable->schema[pTable->numOfSchemas++] = tdDupSchema(pCfg->schema); - } else { - ASSERT(pTable->numOfSchemas == TSDB_MAX_TABLE_SCHEMAS); - STSchema *tSchema = tdDupSchema(pCfg->schema); - tdFreeSchema(pTable->schema[0]); - memmove(pTable->schema, pTable->schema+1, sizeof(STSchema *) * (TSDB_MAX_TABLE_SCHEMAS - 1)); - pTable->schema[pTable->numOfSchemas-1] = tSchema; - } - - isChanged = true; - } - - if (isChanged) { - char *buf = malloc(1024 * 1024); - int bufLen = 0; - tsdbEncodeTable(pTable, buf, &bufLen); - tsdbInsertMetaRecord(pMeta->mfh, pTable->tableId.uid, buf, bufLen); - free(buf); - } - - return TSDB_CODE_SUCCESS; -} - -int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg) { - STsdbRepo *pRepo = (STsdbRepo *)repo; - STsdbMeta *pMeta = pRepo->tsdbMeta; - - if (tsdbCheckTableCfg(pCfg) < 0) return -1; - - STable *pTable = tsdbGetTableByUid(pMeta, pCfg->tableId.uid); - if (pTable != NULL) { - tsdbError("vgId:%d table %s already exists, tid %d uid %" PRId64, pRepo->config.tsdbId, varDataVal(pTable->name), - pTable->tableId.tid, pTable->tableId.uid); - return TSDB_CODE_TDB_TABLE_ALREADY_EXIST; - } - - STable *super = NULL; - int newSuper = 0; - - if (pCfg->type == TSDB_CHILD_TABLE) { - super = tsdbGetTableByUid(pMeta, pCfg->superUid); - if (super == NULL) { // super table not exists, try to create it - newSuper = 1; - super = tsdbNewTable(pCfg, true); - if (super == NULL) return -1; - } else { - if (super->type != TSDB_SUPER_TABLE) return -1; - if (super->tableId.uid != pCfg->superUid) return -1; - tsdbUpdateTable(pMeta, super, pCfg); - } - } - - STable *table = tsdbNewTable(pCfg, false); - if (table == NULL) { - if (newSuper) { - tsdbFreeTable(super); - return -1; - } - } - - table->lastKey = TSKEY_INITIAL_VAL; - - // Register to meta - if (newSuper) { - tsdbAddTableToMeta(pMeta, super, true); - tsdbTrace("vgId:%d, super table %s is created! uid:%" PRId64, pRepo->config.tsdbId, varDataVal(super->name), - super->tableId.uid); - } - tsdbAddTableToMeta(pMeta, table, true); - tsdbTrace("vgId:%d, table %s is created! tid:%d, uid:%" PRId64, pRepo->config.tsdbId, varDataVal(table->name), - table->tableId.tid, table->tableId.uid); - - // Write to meta file - int bufLen = 0; - char *buf = malloc(1024*1024); - if (newSuper) { - tsdbEncodeTable(super, buf, &bufLen); - tsdbInsertMetaRecord(pMeta->mfh, super->tableId.uid, buf, bufLen); - } - - tsdbEncodeTable(table, buf, &bufLen); - tsdbInsertMetaRecord(pMeta->mfh, table->tableId.uid, buf, bufLen); - tfree(buf); - - return 0; -} - -/** - * Check if a table is valid to insert. - * @return NULL for invalid and the pointer to the table if valid - */ -STable *tsdbIsValidTableToInsert(STsdbMeta *pMeta, STableId tableId) { - STable *pTable = tsdbGetTableByUid(pMeta, tableId.uid); - if (pTable == NULL) { - return NULL; - } - - if (TSDB_TABLE_IS_SUPER_TABLE(pTable)) return NULL; - if (pTable->tableId.tid != tableId.tid) return NULL; - - return pTable; -} - -STableCfg *tsdbCreateTableCfgFromMsg(SMDCreateTableMsg *pMsg) { - if (pMsg == NULL) return NULL; - SSchema * pSchema = (SSchema *)pMsg->data; - int16_t numOfCols = htons(pMsg->numOfColumns); - int16_t numOfTags = htons(pMsg->numOfTags); - STSchemaBuilder schemaBuilder = {0}; - - STableCfg *pCfg = (STableCfg *)calloc(1, sizeof(STableCfg)); - if (pCfg == NULL) return NULL; - - if (tsdbInitTableCfg(pCfg, pMsg->tableType, htobe64(pMsg->uid), htonl(pMsg->sid)) < 0) goto _err; - if (tdInitTSchemaBuilder(&schemaBuilder, htonl(pMsg->sversion)) < 0) goto _err; - - for (int i = 0; i < numOfCols; i++) { - tdAddColToSchema(&schemaBuilder, pSchema[i].type, htons(pSchema[i].colId), htons(pSchema[i].bytes)); - } - if (tsdbTableSetSchema(pCfg, tdGetSchemaFromBuilder(&schemaBuilder), false) < 0) goto _err; - if (tsdbTableSetName(pCfg, pMsg->tableId, true) < 0) goto _err; - - if (numOfTags > 0) { - // Decode tag schema - tdResetTSchemaBuilder(&schemaBuilder, htonl(pMsg->tversion)); - for (int i = numOfCols; i < numOfCols + numOfTags; i++) { - tdAddColToSchema(&schemaBuilder, pSchema[i].type, htons(pSchema[i].colId), htons(pSchema[i].bytes)); - } - if (tsdbTableSetTagSchema(pCfg, tdGetSchemaFromBuilder(&schemaBuilder), false) < 0) goto _err; - if (tsdbTableSetSName(pCfg, pMsg->superTableId, true) < 0) goto _err; - if (tsdbTableSetSuperUid(pCfg, htobe64(pMsg->superTableUid)) < 0) goto _err; - - // Decode tag values - if (pMsg->tagDataLen) { - int accBytes = 0; - char *pTagData = pMsg->data + (numOfCols + numOfTags) * sizeof(SSchema); - - SKVRowBuilder kvRowBuilder = {0}; - if (tdInitKVRowBuilder(&kvRowBuilder) < 0) goto _err; - for (int i = numOfCols; i < numOfCols + numOfTags; i++) { - tdAddColToKVRow(&kvRowBuilder, htons(pSchema[i].colId), pSchema[i].type, pTagData + accBytes); - accBytes += htons(pSchema[i].bytes); - } - - tsdbTableSetTagValue(pCfg, tdGetKVRowFromBuilder(&kvRowBuilder), false); - tdDestroyKVRowBuilder(&kvRowBuilder); - } - } - - if (pMsg->tableType == TSDB_STREAM_TABLE) { - char *sql = pMsg->data + (numOfCols + numOfTags) * sizeof(SSchema); - tsdbTableSetStreamSql(pCfg, sql, true); - } - - tdDestroyTSchemaBuilder(&schemaBuilder); - - return pCfg; - -_err: - tdDestroyTSchemaBuilder(&schemaBuilder); - tsdbClearTableCfg(pCfg); - tfree(pCfg); - return NULL; -} - -// int32_t tsdbDropTableImpl(STsdbMeta *pMeta, STableId tableId) { -int tsdbDropTable(TSDB_REPO_T *repo, STableId tableId) { - STsdbRepo *pRepo = (STsdbRepo *)repo; - if (pRepo == NULL) return -1; - - STsdbMeta *pMeta = pRepo->tsdbMeta; - if (pMeta == NULL) return -1; - - STable *pTable = tsdbGetTableByUid(pMeta, tableId.uid); - if (pTable == NULL) { - tsdbError("vgId:%d, failed to drop table since table not exists! tid:%d, uid:" PRId64, pRepo->config.tsdbId, - tableId.tid, tableId.uid); - return -1; - } - - tsdbTrace("vgId:%d, table %s is dropped! tid:%d, uid:%" PRId64, pRepo->config.tsdbId, varDataVal(pTable->name), - tableId.tid, tableId.uid); - if (tsdbRemoveTableFromMeta(pMeta, pTable, true) < 0) return -1; - - return 0; - -} - -// int32_t tsdbInsertRowToTableImpl(SSkipListNode *pNode, STable *pTable) { -// tSkipListPut(pTable->mem->pData, pNode); -// return 0; -// } - -static void tsdbFreeMemTable(SMemTable *pMemTable) { - if (pMemTable) { - tSkipListDestroy(pMemTable->pData); - } - - free(pMemTable); -} - static int tsdbFreeTable(STable *pTable) { if (pTable == NULL) return 0; @@ -653,17 +589,16 @@ static int tsdbFreeTable(STable *pTable) { return 0; } -static int32_t tsdbCheckTableCfg(STableCfg *pCfg) { - // TODO - return 0; -} +static int tsdbUpdateTableTagSchema(STable *pTable, STSchema *newSchema) { + ASSERT(pTable->type == TSDB_SUPER_TABLE); + ASSERT(schemaVersion(pTable->tagSchema) < schemaVersion(newSchema)); + STSchema *pOldSchema = pTable->tagSchema; + STSchema *pNewSchema = tdDupSchema(newSchema); + if (pNewSchema == NULL) return TSDB_CODE_TDB_OUT_OF_MEMORY; + pTable->tagSchema = pNewSchema; + tdFreeSchema(pOldSchema); -STable *tsdbGetTableByUid(STsdbMeta *pMeta, uint64_t uid) { - void *ptr = taosHashGet(pMeta->map, (char *)(&uid), sizeof(uid)); - - if (ptr == NULL) return NULL; - - return *(STable **)ptr; + return TSDB_CODE_SUCCESS; } static int tsdbAddTableToMeta(STsdbMeta *pMeta, STable *pTable, bool addIdx) { @@ -746,7 +681,7 @@ static int tsdbRemoveTableFromMeta(STsdbMeta *pMeta, STable *pTable, bool rmFrom return 0; } -int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable) { +static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable) { assert(pTable->type == TSDB_CHILD_TABLE && pTable != NULL); STable* pSTable = tsdbGetTableByUid(pMeta, pTable->superUid); assert(pSTable != NULL); @@ -771,7 +706,7 @@ int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable) { return 0; } -int tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable) { +static int tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable) { assert(pTable->type == TSDB_CHILD_TABLE && pTable != NULL); STable* pSTable = tsdbGetTableByUid(pMeta, pTable->superUid); @@ -799,11 +734,13 @@ int tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable) { return 0; } +#if 0 +#define TSDB_SUPER_TABLE_SL_LEVEL 5 // TODO: may change here +// #define TSDB_META_FILE_NAME "META" + + + -char *getTSTupleKey(const void * data) { - SDataRow row = (SDataRow)data; - return POINTER_SHIFT(row, TD_DATA_ROW_HEAD_SIZE); -} static int tsdbInitTableCfg(STableCfg *config, ETableType type, uint64_t uid, int32_t tid) { if (config == NULL) return -1; @@ -914,3 +851,4 @@ static int tsdbTableSetStreamSql(STableCfg *config, char *sql, bool dup) { return 0; } +#endif \ No newline at end of file diff --git a/src/tsdb/src/tsdbMetaFile.c b/src/tsdb/src/tsdbMetaFile.c deleted file mode 100644 index 921db8674a..0000000000 --- a/src/tsdb/src/tsdbMetaFile.c +++ /dev/null @@ -1,281 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -#include "os.h" - -#include "taosdef.h" -#include "hash.h" -#include "tsdbMain.h" - -#define TSDB_META_FILE_VERSION_MAJOR 1 -#define TSDB_META_FILE_VERSION_MINOR 0 -#define TSDB_META_FILE_HEADER_SIZE 512 - -typedef struct { - int32_t offset; - int32_t size; - uint64_t uid; -} SRecordInfo; - -// static int32_t tsdbGetMetaFileName(char *rootDir, char *fname); -// static int32_t tsdbCheckMetaHeader(int fd); -static int32_t tsdbWriteMetaHeader(int fd); -static int tsdbCreateMetaFile(char *fname); -static int tsdbRestoreFromMetaFile(char *fname, SMetaFile *mfh); - -SMetaFile *tsdbInitMetaFile(char *rootDir, int32_t maxTables, iterFunc iFunc, afterFunc aFunc, void *appH) { - char fname[128] = "\0"; - if (tsdbGetMetaFileName(rootDir, fname) < 0) return NULL; - - SMetaFile *mfh = (SMetaFile *)calloc(1, sizeof(SMetaFile)); - if (mfh == NULL) return NULL; - - mfh->iFunc = iFunc; - mfh->aFunc = aFunc; - mfh->appH = appH; - mfh->nDel = 0; - mfh->tombSize = 0; - mfh->size = 0; - - // OPEN MAP - mfh->map = - taosHashInit(maxTables * TSDB_META_HASH_FRACTION, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false); - if (mfh->map == NULL) { - free(mfh); - return NULL; - } - - // OPEN FILE - if (access(fname, F_OK) < 0) { // file not exists - mfh->fd = tsdbCreateMetaFile(fname); - if (mfh->fd < 0) { - taosHashCleanup(mfh->map); - free(mfh); - return NULL; - } - mfh->size += TSDB_META_FILE_HEADER_SIZE; - } else { // file exists, recover from file - if (tsdbRestoreFromMetaFile(fname, mfh) < 0) { - taosHashCleanup(mfh->map); - free(mfh); - return NULL; - } - } - - return mfh; -} - -int32_t tsdbInsertMetaRecord(SMetaFile *mfh, uint64_t uid, void *cont, int32_t contLen) { - if (taosHashGet(mfh->map, (char *)(&uid), sizeof(uid)) != NULL) { - return -1; - } - - SRecordInfo info; - info.offset = mfh->size; - info.size = contLen; - info.uid = uid; - - mfh->size += (contLen + sizeof(SRecordInfo)); - - if (taosHashPut(mfh->map, (char *)(&uid), sizeof(uid), (void *)(&info), sizeof(SRecordInfo)) < 0) { - return -1; - } - - // TODO: make below a function to implement - if (lseek(mfh->fd, info.offset, SEEK_SET) < 0) { - return -1; - } - - if (write(mfh->fd, (void *)(&info), sizeof(SRecordInfo)) < 0) { - return -1; - } - - if (write(mfh->fd, cont, contLen) < 0) { - return -1; - } - - // fsync(mfh->fd); - - mfh->tombSize++; - - return 0; -} - -int32_t tsdbDeleteMetaRecord(SMetaFile *mfh, uint64_t uid) { - char *ptr = taosHashGet(mfh->map, (char *)(&uid), sizeof(uid)); - if (ptr == NULL) return -1; - - SRecordInfo info = *(SRecordInfo *)ptr; - - // Remove record from hash table - taosHashRemove(mfh->map, (char *)(&uid), sizeof(uid)); - - // Remove record from file - - info.offset = -info.offset; - if (lseek(mfh->fd, -info.offset, SEEK_CUR) < 0) { - return -1; - } - - if (write(mfh->fd, (void *)(&info), sizeof(SRecordInfo)) < 0) { - return -1; - } - - // fsync(mfh->fd); - - mfh->nDel++; - - return 0; -} - -int32_t tsdbUpdateMetaRecord(SMetaFile *mfh, uint64_t uid, void *cont, int32_t contLen) { - char *ptr = taosHashGet(mfh->map, (char *)(&uid), sizeof(uid)); - if (ptr == NULL) return -1; - - SRecordInfo info = *(SRecordInfo *)ptr; - // Update the hash table - if (taosHashPut(mfh->map, (char *)(&uid), sizeof(uid), (void *)(&info), sizeof(SRecordInfo)) < 0) { - return -1; - } - - // Update record in file - if (info.size >= contLen) { // Just update it in place - info.size = contLen; - - } else { // Just append to the end of file - info.offset = mfh->size; - info.size = contLen; - - mfh->size += contLen; - } - if (lseek(mfh->fd, -info.offset, SEEK_CUR) < 0) { - return -1; - } - - if (write(mfh->fd, (void *)(&info), sizeof(SRecordInfo)) < 0) { - return -1; - } - - // fsync(mfh->fd); - - return 0; -} - -void tsdbCloseMetaFile(SMetaFile *mfh) { - if (mfh == NULL) return; - close(mfh->fd); - - taosHashCleanup(mfh->map); - tfree(mfh); -} - -int32_t tsdbGetMetaFileName(char *rootDir, char *fname) { - if (rootDir == NULL) return -1; - sprintf(fname, "%s/%s", rootDir, TSDB_META_FILE_NAME); - return 0; -} - -// static int32_t tsdbCheckMetaHeader(int fd) { -// // TODO: write the meta file header check function -// return 0; -// } - -static int32_t tsdbWriteMetaHeader(int fd) { - // TODO: write the meta file header to file - char head[TSDB_META_FILE_HEADER_SIZE] = "\0"; - sprintf(head, "version: %d.%d", TSDB_META_FILE_VERSION_MAJOR, TSDB_META_FILE_VERSION_MINOR); - - write(fd, (void *)head, TSDB_META_FILE_HEADER_SIZE); - return 0; -} - -// static int32_t tsdbReadMetaHeader(int fd) { -// lseek(fd, TSDB_META_FILE_HEADER_SIZE, SEEK_SET); -// return 0; -// } - -static int tsdbCreateMetaFile(char *fname) { - int fd = open(fname, O_RDWR | O_CREAT, 0755); - if (fd < 0) return -1; - - if (tsdbWriteMetaHeader(fd) < 0) { - close(fd); - return -1; - } - - return fd; -} - -static int tsdbCheckMetaFileIntegrety(int fd) { - // TODO - return 0; -} - -static int tsdbRestoreFromMetaFile(char *fname, SMetaFile *mfh) { - int fd = open(fname, O_RDWR); - if (fd < 0) return -1; - - if (tsdbCheckMetaFileIntegrety(fd) < 0) { - // TODO: decide if to auto-recover the file - close(fd); - return -1; - } - - if (lseek(fd, TSDB_META_FILE_HEADER_SIZE, SEEK_SET) < 0) { - // TODO: deal with the error - close(fd); - return -1; - } - - mfh->size += TSDB_META_FILE_HEADER_SIZE; - - mfh->fd = fd; - - void *buf = NULL; - // int buf_size = 0; - - SRecordInfo info; - while (1) { - if (read(mfh->fd, (void *)(&info), sizeof(SRecordInfo)) == 0) break; - if (info.offset < 0) { - mfh->size += (info.size + sizeof(SRecordInfo)); - mfh->tombSize += (info.size + sizeof(SRecordInfo)); - lseek(mfh->fd, info.size, SEEK_CUR); - mfh->size = mfh->size + sizeof(SRecordInfo) + info.size; - mfh->tombSize = mfh->tombSize + sizeof(SRecordInfo) + info.size; - } else { - if (taosHashPut(mfh->map, (char *)(&info.uid), sizeof(info.uid), (void *)(&info), sizeof(SRecordInfo)) < 0) { - if (buf) free(buf); - return -1; - } - - buf = realloc(buf, info.size); - if (buf == NULL) return -1; - - if (read(mfh->fd, buf, info.size) < 0) { - if (buf) free(buf); - return -1; - } - (*mfh->iFunc)(mfh->appH, buf, info.size); - - mfh->size = mfh->size + sizeof(SRecordInfo) + info.size; - } - - } - (*mfh->aFunc)(mfh->appH); - - if (buf) free(buf); - - return 0; -} \ No newline at end of file diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index add484b5f4..b091544722 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -14,137 +14,13 @@ */ #include "os.h" -#include "tsdbMain.h" -#include "tchecksum.h" -#include "tscompression.h" #include "talgo.h" +#include "tchecksum.h" #include "tcoding.h" +#include "tscompression.h" +#include "tsdbMain.h" -// Local function definitions -// static int tsdbCheckHelperCfg(SHelperCfg *pCfg); -static int tsdbInitHelperFile(SRWHelper *pHelper); -// static void tsdbClearHelperFile(SHelperFile *pHFile); -static bool tsdbShouldCreateNewLast(SRWHelper *pHelper); -static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDataCols, int rowsToWrite, - SCompBlock *pCompBlock, bool isLast, bool isSuperBlock); -static int compareKeyBlock(const void *arg1, const void *arg2); -static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDataCols); -static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx); -static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, int rowsAdded); -static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx); -static int tsdbGetRowsInRange(SDataCols *pDataCols, TSKEY minKey, TSKEY maxKey); -static void tsdbResetHelperBlock(SRWHelper *pHelper); - -// ---------- Operations on Helper File part -static void tsdbResetHelperFileImpl(SRWHelper *pHelper) { - memset((void *)&pHelper->files, 0, sizeof(pHelper->files)); - pHelper->files.fid = -1; - pHelper->files.headF.fd = -1; - pHelper->files.dataF.fd = -1; - pHelper->files.lastF.fd = -1; - pHelper->files.nHeadF.fd = -1; - pHelper->files.nLastF.fd = -1; -} - -static int tsdbInitHelperFile(SRWHelper *pHelper) { - // pHelper->compIdxSize = sizeof(SCompIdx) * pHelper->config.maxTables + sizeof(TSCKSUM); - size_t tsize = sizeof(SCompIdx) * pHelper->config.maxTables + sizeof(TSCKSUM); - pHelper->pCompIdx = (SCompIdx *)tmalloc(tsize); - if (pHelper->pCompIdx == NULL) return -1; - - tsdbResetHelperFileImpl(pHelper); - return 0; -} - -static void tsdbDestroyHelperFile(SRWHelper *pHelper) { - tsdbCloseHelperFile(pHelper, false); - tzfree(pHelper->pCompIdx); -} - -// ---------- Operations on Helper Table part -static void tsdbResetHelperTableImpl(SRWHelper *pHelper) { - memset((void *)&pHelper->tableInfo, 0, sizeof(SHelperTable)); - pHelper->hasOldLastBlock = false; -} - -static void tsdbResetHelperTable(SRWHelper *pHelper) { - tsdbResetHelperBlock(pHelper); - tsdbResetHelperTableImpl(pHelper); - helperClearState(pHelper, (TSDB_HELPER_TABLE_SET|TSDB_HELPER_INFO_LOAD)); -} - -static void tsdbInitHelperTable(SRWHelper *pHelper) { - tsdbResetHelperTableImpl(pHelper); -} - -static void tsdbDestroyHelperTable(SRWHelper *pHelper) { tzfree((void *)pHelper->pCompInfo); } - -// ---------- Operations on Helper Block part -static void tsdbResetHelperBlockImpl(SRWHelper *pHelper) { - tdResetDataCols(pHelper->pDataCols[0]); - tdResetDataCols(pHelper->pDataCols[1]); -} - -static void tsdbResetHelperBlock(SRWHelper *pHelper) { - tsdbResetHelperBlockImpl(pHelper); - // helperClearState(pHelper, TSDB_HELPER_) -} - -static int tsdbInitHelperBlock(SRWHelper *pHelper) { - pHelper->pDataCols[0] = tdNewDataCols(pHelper->config.maxRowSize, pHelper->config.maxCols, pHelper->config.maxRows); - pHelper->pDataCols[1] = tdNewDataCols(pHelper->config.maxRowSize, pHelper->config.maxCols, pHelper->config.maxRows); - if (pHelper->pDataCols[0] == NULL || pHelper->pDataCols[1] == NULL) return -1; - - tsdbResetHelperBlockImpl(pHelper); - - return 0; -} - -static void tsdbDestroyHelperBlock(SRWHelper *pHelper) { - tzfree(pHelper->pCompData); - tdFreeDataCols(pHelper->pDataCols[0]); - tdFreeDataCols(pHelper->pDataCols[1]); -} - -static int tsdbInitHelper(SRWHelper *pHelper, STsdbRepo *pRepo, tsdb_rw_helper_t type) { - if (pHelper == NULL || pRepo == NULL) return -1; - - memset((void *)pHelper, 0, sizeof(*pHelper)); - - // Init global configuration - pHelper->config.type = type; - pHelper->config.maxTables = pRepo->config.maxTables; - pHelper->config.maxRowSize = pRepo->tsdbMeta->maxRowBytes; - pHelper->config.maxRows = pRepo->config.maxRowsPerFileBlock; - pHelper->config.maxCols = pRepo->tsdbMeta->maxCols; - pHelper->config.minRowsPerFileBlock = pRepo->config.minRowsPerFileBlock; - pHelper->config.maxRowsPerFileBlock = pRepo->config.maxRowsPerFileBlock; - pHelper->config.compress = pRepo->config.compression; - - pHelper->state = TSDB_HELPER_CLEAR_STATE; - - // Init file part - if (tsdbInitHelperFile(pHelper) < 0) goto _err; - - // Init table part - tsdbInitHelperTable(pHelper); - - // Init block part - if (tsdbInitHelperBlock(pHelper) < 0) goto _err; - - pHelper->pBuffer = - tmalloc(sizeof(SCompData) + (sizeof(SCompCol) + sizeof(TSCKSUM) + COMP_OVERFLOW_BYTES) * pHelper->config.maxCols + - pHelper->config.maxRowSize * pHelper->config.maxRowsPerFileBlock + sizeof(TSCKSUM)); - if (pHelper->pBuffer == NULL) goto _err; - - return 0; - -_err: - tsdbDestroyHelper(pHelper); - return -1; -} - -// ------------------------------------------ OPERATIONS FOR OUTSIDE ------------------------------------------ +// ---------------------- INTERNAL FUNCTIONS ---------------------- int tsdbInitReadHelper(SRWHelper *pHelper, STsdbRepo *pRepo) { return tsdbInitHelper(pHelper, pRepo, TSDB_READ_HELPER); } @@ -180,7 +56,6 @@ void tsdbResetHelper(SRWHelper *pHelper) { } } -// ------------ Operations for read/write purpose int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { ASSERT(pHelper != NULL && pGroup != NULL); @@ -231,7 +106,7 @@ int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { return tsdbLoadCompIdx(pHelper, NULL); - _err: +_err: return -1; } @@ -264,7 +139,7 @@ int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) { pHelper->files.headF.info = pHelper->files.nHeadF.info; } } - + if (pHelper->files.nLastF.fd > 0) { if (!hasError) tsdbUpdateFileHeader(&(pHelper->files.nLastF), 0); fsync(pHelper->files.nLastF.fd); @@ -306,7 +181,7 @@ void tsdbSetHelperTable(SRWHelper *pHelper, STable *pTable, STsdbRepo *pRepo) { /** * Write part of of points from pDataCols to file - * + * * @return: number of points written to file successfully * -1 for failure */ @@ -342,8 +217,8 @@ int tsdbWriteDataBlock(SRWHelper *pHelper, SDataCols *pDataCols) { if (tsdbInsertSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks) < 0) goto _err; } else { // (Has old data) AND ((has last block) OR (key overlap)), need to merge the block - SCompBlock *pCompBlock = taosbsearch((void *)(&keyFirst), (void *)(pHelper->pCompInfo->blocks), - pIdx->numOfBlocks, sizeof(SCompBlock), compareKeyBlock, TD_GE); + SCompBlock *pCompBlock = taosbsearch((void *)(&keyFirst), (void *)(pHelper->pCompInfo->blocks), pIdx->numOfBlocks, + sizeof(SCompBlock), compareKeyBlock, TD_GE); int blkIdx = (pCompBlock == NULL) ? (pIdx->numOfBlocks - 1) : (pCompBlock - pHelper->pCompInfo->blocks); @@ -358,10 +233,11 @@ int tsdbWriteDataBlock(SRWHelper *pHelper, SDataCols *pDataCols) { rowsToWrite = tsdbMergeDataWithBlock(pHelper, blkIdx, pDataCols); if (rowsToWrite < 0) goto _err; - } else { // Save as a super block in the middle - rowsToWrite = tsdbGetRowsInRange(pDataCols, 0, pCompBlock->keyFirst-1); + } else { // Save as a super block in the middle + rowsToWrite = tsdbGetRowsInRange(pDataCols, 0, pCompBlock->keyFirst - 1); ASSERT(rowsToWrite > 0); - if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.dataF), pDataCols, rowsToWrite, &compBlock, false, true) < 0) goto _err; + if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.dataF), pDataCols, rowsToWrite, &compBlock, false, true) < 0) + goto _err; if (tsdbInsertSuperBlock(pHelper, &compBlock, blkIdx) < 0) goto _err; } } @@ -375,7 +251,7 @@ _err: int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper) { ASSERT(TSDB_HELPER_TYPE(pHelper) == TSDB_WRITE_HELPER); - SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; + SCompIdx * pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; SCompBlock compBlock; if ((pHelper->files.nLastF.fd > 0) && (pHelper->hasOldLastBlock)) { if (tsdbLoadCompInfo(pHelper, NULL) < 0) return -1; @@ -450,7 +326,7 @@ int tsdbWriteCompIdx(SRWHelper *pHelper) { if (pCompIdx->offset > 0) { int drift = POINTER_DISTANCE(buf, pHelper->pBuffer); if (tsizeof(pHelper->pBuffer) - drift < 128) { - pHelper->pBuffer = trealloc(pHelper->pBuffer, tsizeof(pHelper->pBuffer)*2); + pHelper->pBuffer = trealloc(pHelper->pBuffer, tsizeof(pHelper->pBuffer) * 2); } buf = POINTER_SHIFT(pHelper->pBuffer, drift); buf = taosEncodeVariantU32(buf, i); @@ -472,7 +348,7 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { if (!helperHasState(pHelper, TSDB_HELPER_IDX_LOAD)) { // If not load from file, just load it in object SFile *pFile = &(pHelper->files.headF); - int fd = pFile->fd; + int fd = pFile->fd; memset(pHelper->pCompIdx, 0, tsizeof(pHelper->pCompIdx)); if (pFile->info.offset > 0) { @@ -480,8 +356,7 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { if (lseek(fd, pFile->info.offset, SEEK_SET) < 0) return -1; if ((pHelper->pBuffer = trealloc(pHelper->pBuffer, pFile->info.len)) == NULL) return -1; - if (tread(fd, (void *)(pHelper->pBuffer), pFile->info.len) < pFile->info.len) - return -1; + if (tread(fd, (void *)(pHelper->pBuffer), pFile->info.len) < pFile->info.len) return -1; if (!taosCheckChecksumWhole((uint8_t *)(pHelper->pBuffer), pFile->info.len)) { // TODO: File is broken, try to deal with it return -1; @@ -502,7 +377,6 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { ASSERT(((char *)ptr - (char *)pHelper->pBuffer) == (pFile->info.len - sizeof(TSCKSUM))); if (lseek(fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) return -1; } - } helperSetState(pHelper, TSDB_HELPER_IDX_LOAD); @@ -582,53 +456,12 @@ void tsdbGetDataStatis(SRWHelper *pHelper, SDataStatis *pStatis, int numOfCols) } } -static int comparColIdCompCol(const void *arg1, const void *arg2) { - return (*(int16_t *)arg1) - ((SCompCol *)arg2)->colId; -} - -static int comparColIdDataCol(const void *arg1, const void *arg2) { - return (*(int16_t *)arg1) - ((SDataCol *)arg2)->colId; -} - -static int tsdbLoadSingleColumnData(int fd, SCompBlock *pCompBlock, SCompCol *pCompCol, void *buf) { - size_t tsize = sizeof(SCompData) + sizeof(SCompCol) * pCompBlock->numOfCols; - if (lseek(fd, pCompBlock->offset + tsize + pCompCol->offset, SEEK_SET) < 0) return -1; - if (tread(fd, buf, pCompCol->len) < pCompCol->len) return -1; - - return 0; -} - -static int tsdbLoadSingleBlockDataCols(SRWHelper *pHelper, SCompBlock *pCompBlock, int16_t *colIds, int numOfColIds, - SDataCols *pDataCols) { - if (tsdbLoadCompData(pHelper, pCompBlock, NULL) < 0) return -1; - int fd = (pCompBlock->last) ? pHelper->files.lastF.fd : pHelper->files.dataF.fd; - - void *ptr = NULL; - for (int i = 0; i < numOfColIds; i++) { - int16_t colId = colIds[i]; - - ptr = bsearch((void *)&colId, (void *)pHelper->pCompData->cols, pHelper->pCompData->numOfCols, sizeof(SCompCol), comparColIdCompCol); - if (ptr == NULL) continue; - SCompCol *pCompCol = (SCompCol *)ptr; - - ptr = bsearch((void *)&colId, (void *)(pDataCols->cols), pDataCols->numOfCols, sizeof(SDataCol), comparColIdDataCol); - ASSERT(ptr != NULL); - SDataCol *pDataCol = (SDataCol *)ptr; - - pDataCol->len = pCompCol->len; - if (tsdbLoadSingleColumnData(fd, pCompBlock, pCompCol, pDataCol->pData) < 0) return -1; - } - - return 0; -} - -// Load specific column data from file int tsdbLoadBlockDataCols(SRWHelper *pHelper, SDataCols *pDataCols, int blkIdx, int16_t *colIds, int numOfColIds) { SCompBlock *pCompBlock = pHelper->pCompInfo->blocks + blkIdx; - ASSERT(pCompBlock->numOfSubBlocks >= 1); // Must be super block + ASSERT(pCompBlock->numOfSubBlocks >= 1); // Must be super block - int numOfSubBlocks = pCompBlock->numOfSubBlocks; + int numOfSubBlocks = pCompBlock->numOfSubBlocks; SCompBlock *pStartBlock = (numOfSubBlocks == 1) ? pCompBlock : (SCompBlock *)((char *)pHelper->pCompInfo->blocks + pCompBlock->offset); @@ -642,95 +475,6 @@ int tsdbLoadBlockDataCols(SRWHelper *pHelper, SDataCols *pDataCols, int blkIdx, return 0; } -static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, char *content, int32_t len, int8_t comp, int numOfRows, - int maxPoints, char *buffer, int bufferSize) { - // Verify by checksum - if (!taosCheckChecksumWhole((uint8_t *)content, len)) return -1; - - // Decode the data - if (comp) { - // // Need to decompress - pDataCol->len = (*(tDataTypeDesc[pDataCol->type].decompFunc))( - content, len - sizeof(TSCKSUM), numOfRows, pDataCol->pData, pDataCol->spaceSize, comp, buffer, bufferSize); - if (pDataCol->type == TSDB_DATA_TYPE_BINARY || pDataCol->type == TSDB_DATA_TYPE_NCHAR) { - dataColSetOffset(pDataCol, numOfRows); - } - } else { - // No need to decompress, just memcpy it - pDataCol->len = len - sizeof(TSCKSUM); - memcpy(pDataCol->pData, content, pDataCol->len); - if (pDataCol->type == TSDB_DATA_TYPE_BINARY || pDataCol->type == TSDB_DATA_TYPE_NCHAR) { - dataColSetOffset(pDataCol, numOfRows); - } - } - return 0; -} - -/** - * Interface to read the data of a sub-block OR the data of a super-block of which (numOfSubBlocks == 1) - */ -static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *pDataCols) { - ASSERT(pCompBlock->numOfSubBlocks <= 1); - - ASSERT(tsizeof(pHelper->pBuffer) >= pCompBlock->len); - - SCompData *pCompData = (SCompData *)pHelper->pBuffer; - - int fd = (pCompBlock->last) ? pHelper->files.lastF.fd : pHelper->files.dataF.fd; - if (lseek(fd, pCompBlock->offset, SEEK_SET) < 0) goto _err; - if (tread(fd, (void *)pCompData, pCompBlock->len) < pCompBlock->len) goto _err; - ASSERT(pCompData->numOfCols == pCompBlock->numOfCols); - - int32_t tsize = sizeof(SCompData) + sizeof(SCompCol) * pCompBlock->numOfCols + sizeof(TSCKSUM); - if (!taosCheckChecksumWhole((uint8_t *)pCompData, tsize)) goto _err; - - pDataCols->numOfRows = pCompBlock->numOfRows; - - // Recover the data - int ccol = 0; - int dcol = 0; - while (dcol < pDataCols->numOfCols) { - SDataCol *pDataCol = &(pDataCols->cols[dcol]); - if (ccol >= pCompData->numOfCols) { - // Set current column as NULL and forward - dataColSetNEleNull(pDataCol, pCompBlock->numOfRows, pDataCols->maxPoints); - dcol++; - continue; - } - - SCompCol *pCompCol = &(pCompData->cols[ccol]); - - if (pCompCol->colId == pDataCol->colId) { - if (pCompBlock->algorithm == TWO_STAGE_COMP) { - int zsize = pDataCol->bytes * pCompBlock->numOfRows + COMP_OVERFLOW_BYTES; - if (pCompCol->type == TSDB_DATA_TYPE_BINARY || pCompCol->type == TSDB_DATA_TYPE_NCHAR) { - zsize += (sizeof(VarDataLenT) * pCompBlock->numOfRows); - } - pHelper->compBuffer = trealloc(pHelper->compBuffer, zsize); - if (pHelper->compBuffer == NULL) goto _err; - } - if (tsdbCheckAndDecodeColumnData(pDataCol, (char *)pCompData + tsize + pCompCol->offset, pCompCol->len, - pCompBlock->algorithm, pCompBlock->numOfRows, pDataCols->maxPoints, - pHelper->compBuffer, tsizeof(pHelper->compBuffer)) < 0) - goto _err; - dcol++; - ccol++; - } else if (pCompCol->colId < pDataCol->colId) { - ccol++; - } else { - // Set current column as NULL and forward - dataColSetNEleNull(pDataCol, pCompBlock->numOfRows, pDataCols->maxPoints); - dcol++; - } - } - - return 0; - -_err: - return -1; -} - -// Load the whole block data int tsdbLoadBlockData(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *target) { // SCompBlock *pCompBlock = pHelper->pCompInfo->blocks + blkIdx; @@ -754,6 +498,74 @@ _err: return -1; } +void *tsdbEncodeSCompIdx(void *buf, SCompIdx *pIdx) { + buf = taosEncodeVariantU32(buf, pIdx->len); + buf = taosEncodeVariantU32(buf, pIdx->offset); + buf = taosEncodeFixedU8(buf, pIdx->hasLast); + buf = taosEncodeVariantU32(buf, pIdx->numOfBlocks); + buf = taosEncodeFixedU64(buf, pIdx->uid); + buf = taosEncodeFixedU64(buf, pIdx->maxKey); + + return buf; +} + +void *tsdbDecodeSCompIdx(void *buf, SCompIdx *pIdx) { + uint8_t hasLast = 0; + uint32_t numOfBlocks = 0; + uint64_t value = 0; + + if ((buf = taosDecodeVariantU32(buf, &(pIdx->len))) == NULL) return NULL; + if ((buf = taosDecodeVariantU32(buf, &(pIdx->offset))) == NULL) return NULL; + if ((buf = taosDecodeFixedU8(buf, &(hasLast))) == NULL) return NULL; + pIdx->hasLast = hasLast; + if ((buf = taosDecodeVariantU32(buf, &(numOfBlocks))) == NULL) return NULL; + pIdx->numOfBlocks = numOfBlocks; + if ((buf = taosDecodeFixedU64(buf, &value)) == NULL) return NULL; + pIdx->uid = (int64_t)value; + if ((buf = taosDecodeFixedU64(buf, &value)) == NULL) return NULL; + pIdx->maxKey = (TSKEY)value; + + return buf; +} + +int tsdbUpdateFileHeader(SFile *pFile, uint32_t version) { + char buf[TSDB_FILE_HEAD_SIZE] = "\0"; + + void *pBuf = (void *)buf; + pBuf = taosEncodeFixedU32(pBuf, version); + pBuf = tsdbEncodeSFileInfo(pBuf, &(pFile->info)); + + taosCalcChecksumAppend(0, (uint8_t *)buf, TSDB_FILE_HEAD_SIZE); + + if (lseek(pFile->fd, 0, SEEK_SET) < 0) return -1; + if (twrite(pFile->fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) return -1; + + return 0; +} + +void *tsdbEncodeSFileInfo(void *buf, const STsdbFileInfo *pInfo) { + buf = taosEncodeFixedU32(buf, pInfo->offset); + buf = taosEncodeFixedU32(buf, pInfo->len); + buf = taosEncodeFixedU64(buf, pInfo->size); + buf = taosEncodeFixedU64(buf, pInfo->tombSize); + buf = taosEncodeFixedU32(buf, pInfo->totalBlocks); + buf = taosEncodeFixedU32(buf, pInfo->totalSubBlocks); + + return buf; +} + +void *tsdbDecodeSFileInfo(void *buf, STsdbFileInfo *pInfo) { + buf = taosDecodeFixedU32(buf, &(pInfo->offset)); + buf = taosDecodeFixedU32(buf, &(pInfo->len)); + buf = taosDecodeFixedU64(buf, &(pInfo->size)); + buf = taosDecodeFixedU64(buf, &(pInfo->tombSize)); + buf = taosDecodeFixedU32(buf, &(pInfo->totalBlocks)); + buf = taosDecodeFixedU32(buf, &(pInfo->totalSubBlocks)); + + return buf; +} + +// ---------------------- INTERNAL FUNCTIONS ---------------------- static bool tsdbShouldCreateNewLast(SRWHelper *pHelper) { ASSERT(pHelper->files.lastF.fd > 0); struct stat st; @@ -762,13 +574,13 @@ static bool tsdbShouldCreateNewLast(SRWHelper *pHelper) { return false; } -static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDataCols, int rowsToWrite, SCompBlock *pCompBlock, - bool isLast, bool isSuperBlock) { +static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDataCols, int rowsToWrite, + SCompBlock *pCompBlock, bool isLast, bool isSuperBlock) { ASSERT(rowsToWrite > 0 && rowsToWrite <= pDataCols->numOfRows && rowsToWrite <= pHelper->config.maxRowsPerFileBlock); ASSERT(isLast ? rowsToWrite < pHelper->config.minRowsPerFileBlock : true); SCompData *pCompData = (SCompData *)(pHelper->pBuffer); - int64_t offset = 0; + int64_t offset = 0; offset = lseek(pFile->fd, 0, SEEK_END); if (offset < 0) goto _err; @@ -822,8 +634,8 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa } pCompCol->len = (*(tDataTypeDesc[pDataCol->type].compFunc))( - (char *)pDataCol->pData, tlen, rowsToWrite, tptr, tsizeof(pHelper->pBuffer) - lsize, - pHelper->config.compress, pHelper->compBuffer, tsizeof(pHelper->compBuffer)); + (char *)pDataCol->pData, tlen, rowsToWrite, tptr, tsizeof(pHelper->pBuffer) - lsize, pHelper->config.compress, + pHelper->compBuffer, tsizeof(pHelper->compBuffer)); } else { pCompCol->len = tlen; memcpy(tptr, pDataCol->pData, pCompCol->len); @@ -861,7 +673,7 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa return 0; - _err: +_err: return -1; } @@ -878,11 +690,6 @@ static int compareKeyBlock(const void *arg1, const void *arg2) { return 0; } -// static FORCE_INLINE int compKeyFunc(const void *arg1, const void *arg2) { -// return ((*(TSKEY *)arg1) - (*(TSKEY *)arg2)); -// } - -// Merge the data with a block in file static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDataCols) { // TODO: set pHelper->hasOldBlock int rowsWritten = 0; @@ -899,8 +706,9 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa ASSERT(keyFirst >= blockAtIdx(pHelper, blkIdx)->keyFirst); // ASSERT(compareKeyBlock((void *)&keyFirst, (void *)pCompBlock) == 0); - if (keyFirst > blockAtIdx(pHelper, blkIdx)->keyLast) { // Merge with the last block by append - ASSERT(blockAtIdx(pHelper, blkIdx)->numOfRows < pHelper->config.minRowsPerFileBlock && blkIdx == pIdx->numOfBlocks-1); + if (keyFirst > blockAtIdx(pHelper, blkIdx)->keyLast) { // Merge with the last block by append + ASSERT(blockAtIdx(pHelper, blkIdx)->numOfRows < pHelper->config.minRowsPerFileBlock && + blkIdx == pIdx->numOfBlocks - 1); int defaultRowsToWrite = pHelper->config.maxRowsPerFileBlock * 4 / 5; // TODO: make a interface rowsWritten = MIN((defaultRowsToWrite - blockAtIdx(pHelper, blkIdx)->numOfRows), pDataCols->numOfRows); @@ -918,15 +726,15 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa if (tdMergeDataCols(pHelper->pDataCols[0], pDataCols, rowsWritten) < 0) goto _err; // Write SFile *pWFile = NULL; - bool isLast = false; + bool isLast = false; if (pHelper->pDataCols[0]->numOfRows >= pHelper->config.minRowsPerFileBlock) { pWFile = &(pHelper->files.dataF); } else { isLast = true; pWFile = (pHelper->files.nLastF.fd > 0) ? &(pHelper->files.nLastF) : &(pHelper->files.lastF); } - if (tsdbWriteBlockToFile(pHelper, pWFile, pHelper->pDataCols[0], - pHelper->pDataCols[0]->numOfRows, &compBlock, isLast, true) < 0) + if (tsdbWriteBlockToFile(pHelper, pWFile, pHelper->pDataCols[0], pHelper->pDataCols[0]->numOfRows, &compBlock, + isLast, true) < 0) goto _err; if (tsdbUpdateSuperBlock(pHelper, &compBlock, blkIdx) < 0) goto _err; } @@ -940,7 +748,8 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa TSKEY keyLimit = (blkIdx == pIdx->numOfBlocks - 1) ? INT64_MAX : blockAtIdx(pHelper, blkIdx + 1)->keyFirst - 1; // rows1: number of rows must merge in this block - int rows1 = tsdbGetRowsInRange(pDataCols, blockAtIdx(pHelper, blkIdx)->keyFirst, blockAtIdx(pHelper, blkIdx)->keyLast); + int rows1 = + tsdbGetRowsInRange(pDataCols, blockAtIdx(pHelper, blkIdx)->keyFirst, blockAtIdx(pHelper, blkIdx)->keyLast); // rows2: max number of rows the block can have more int rows2 = pHelper->config.maxRowsPerFileBlock - blockAtIdx(pHelper, blkIdx)->numOfRows; // rows3: number of rows between this block and the next block @@ -972,8 +781,8 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa rowsWritten = rows3; - int iter1 = 0; // iter over pHelper->pDataCols[0] - int iter2 = 0; // iter over pDataCols + int iter1 = 0; // iter over pHelper->pDataCols[0] + int iter2 = 0; // iter over pDataCols int round = 0; // tdResetDataCols(pHelper->pDataCols[1]); while (true) { @@ -997,7 +806,7 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa return rowsWritten; - _err: +_err: return -1; } @@ -1012,7 +821,6 @@ static int compTSKEY(const void *key1, const void *key2) { } static int tsdbAdjustInfoSizeIfNeeded(SRWHelper *pHelper, size_t esize) { - if (tsizeof((void *)pHelper->pCompInfo) <= esize) { size_t tsize = esize + sizeof(SCompBlock) * 16; pHelper->pCompInfo = (SCompInfo *)trealloc(pHelper->pCompInfo, tsize); @@ -1075,7 +883,7 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId size_t spaceNeeded = (pSCompBlock->numOfSubBlocks == 1) ? pIdx->len + sizeof(SCompBlock) * 2 : pIdx->len + sizeof(SCompBlock); - if (tsdbAdjustInfoSizeIfNeeded(pHelper, spaceNeeded) < 0) goto _err; + if (tsdbAdjustInfoSizeIfNeeded(pHelper, spaceNeeded) < 0) goto _err; pSCompBlock = pHelper->pCompInfo->blocks + blkIdx; @@ -1092,7 +900,6 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId } } - *(SCompBlock *)((char *)(pHelper->pCompInfo) + pSCompBlock->offset + pSCompBlock->len) = *pCompBlock; pSCompBlock->numOfSubBlocks++; @@ -1112,7 +919,7 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId } } - if (ptr == NULL) ptr = POINTER_SHIFT(pHelper->pCompInfo, pIdx->len-sizeof(TSCKSUM)); + if (ptr == NULL) ptr = POINTER_SHIFT(pHelper->pCompInfo, pIdx->len - sizeof(TSCKSUM)); size_t tsize = pIdx->len - ((char *)ptr - (char *)(pHelper->pCompInfo)); if (tsize > 0) { @@ -1206,71 +1013,235 @@ static int tsdbGetRowsInRange(SDataCols *pDataCols, TSKEY minKey, TSKEY maxKey) return ((TSKEY *)ptr2 - (TSKEY *)ptr1) + 1; } -void *tsdbEncodeSCompIdx(void *buf, SCompIdx *pIdx) { - buf = taosEncodeVariantU32(buf, pIdx->len); - buf = taosEncodeVariantU32(buf, pIdx->offset); - buf = taosEncodeFixedU8(buf, pIdx->hasLast); - buf = taosEncodeVariantU32(buf, pIdx->numOfBlocks); - buf = taosEncodeFixedU64(buf, pIdx->uid); - buf = taosEncodeFixedU64(buf, pIdx->maxKey); - - return buf; +static void tsdbResetHelperFileImpl(SRWHelper *pHelper) { + memset((void *)&pHelper->files, 0, sizeof(pHelper->files)); + pHelper->files.fid = -1; + pHelper->files.headF.fd = -1; + pHelper->files.dataF.fd = -1; + pHelper->files.lastF.fd = -1; + pHelper->files.nHeadF.fd = -1; + pHelper->files.nLastF.fd = -1; } -void *tsdbDecodeSCompIdx(void *buf, SCompIdx *pIdx) { - uint8_t hasLast = 0; - uint32_t numOfBlocks = 0; - uint64_t value = 0; +static int tsdbInitHelperFile(SRWHelper *pHelper) { + // pHelper->compIdxSize = sizeof(SCompIdx) * pHelper->config.maxTables + sizeof(TSCKSUM); + size_t tsize = sizeof(SCompIdx) * pHelper->config.maxTables + sizeof(TSCKSUM); + pHelper->pCompIdx = (SCompIdx *)tmalloc(tsize); + if (pHelper->pCompIdx == NULL) return -1; - if ((buf = taosDecodeVariantU32(buf, &(pIdx->len))) == NULL) return NULL; - if ((buf = taosDecodeVariantU32(buf, &(pIdx->offset))) == NULL) return NULL; - if ((buf = taosDecodeFixedU8(buf, &(hasLast))) == NULL) return NULL; - pIdx->hasLast = hasLast; - if ((buf = taosDecodeVariantU32(buf, &(numOfBlocks))) == NULL) return NULL; - pIdx->numOfBlocks = numOfBlocks; - if ((buf = taosDecodeFixedU64(buf, &value)) == NULL) return NULL; - pIdx->uid = (int64_t)value; - if ((buf = taosDecodeFixedU64(buf, &value)) == NULL) return NULL; - pIdx->maxKey = (TSKEY)value; - - return buf; + tsdbResetHelperFileImpl(pHelper); + return 0; } -int tsdbUpdateFileHeader(SFile *pFile, uint32_t version) { - char buf[TSDB_FILE_HEAD_SIZE] = "\0"; +static void tsdbDestroyHelperFile(SRWHelper *pHelper) { + tsdbCloseHelperFile(pHelper, false); + tzfree(pHelper->pCompIdx); +} - void *pBuf = (void *)buf; - pBuf = taosEncodeFixedU32(pBuf, version); - pBuf = tsdbEncodeSFileInfo(pBuf, &(pFile->info)); +// ---------- Operations on Helper Table part +static void tsdbResetHelperTableImpl(SRWHelper *pHelper) { + memset((void *)&pHelper->tableInfo, 0, sizeof(SHelperTable)); + pHelper->hasOldLastBlock = false; +} - taosCalcChecksumAppend(0, (uint8_t *)buf, TSDB_FILE_HEAD_SIZE); +static void tsdbResetHelperTable(SRWHelper *pHelper) { + tsdbResetHelperBlock(pHelper); + tsdbResetHelperTableImpl(pHelper); + helperClearState(pHelper, (TSDB_HELPER_TABLE_SET | TSDB_HELPER_INFO_LOAD)); +} - if (lseek(pFile->fd, 0, SEEK_SET) < 0) return -1; - if (twrite(pFile->fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) return -1; +static void tsdbInitHelperTable(SRWHelper *pHelper) { tsdbResetHelperTableImpl(pHelper); } + +static void tsdbDestroyHelperTable(SRWHelper *pHelper) { tzfree((void *)pHelper->pCompInfo); } + +// ---------- Operations on Helper Block part +static void tsdbResetHelperBlockImpl(SRWHelper *pHelper) { + tdResetDataCols(pHelper->pDataCols[0]); + tdResetDataCols(pHelper->pDataCols[1]); +} + +static void tsdbResetHelperBlock(SRWHelper *pHelper) { + tsdbResetHelperBlockImpl(pHelper); + // helperClearState(pHelper, TSDB_HELPER_) +} + +static int tsdbInitHelperBlock(SRWHelper *pHelper) { + pHelper->pDataCols[0] = tdNewDataCols(pHelper->config.maxRowSize, pHelper->config.maxCols, pHelper->config.maxRows); + pHelper->pDataCols[1] = tdNewDataCols(pHelper->config.maxRowSize, pHelper->config.maxCols, pHelper->config.maxRows); + if (pHelper->pDataCols[0] == NULL || pHelper->pDataCols[1] == NULL) return -1; + + tsdbResetHelperBlockImpl(pHelper); return 0; } - - -void *tsdbEncodeSFileInfo(void *buf, const STsdbFileInfo *pInfo) { - buf = taosEncodeFixedU32(buf, pInfo->offset); - buf = taosEncodeFixedU32(buf, pInfo->len); - buf = taosEncodeFixedU64(buf, pInfo->size); - buf = taosEncodeFixedU64(buf, pInfo->tombSize); - buf = taosEncodeFixedU32(buf, pInfo->totalBlocks); - buf = taosEncodeFixedU32(buf, pInfo->totalSubBlocks); - - return buf; +static void tsdbDestroyHelperBlock(SRWHelper *pHelper) { + tzfree(pHelper->pCompData); + tdFreeDataCols(pHelper->pDataCols[0]); + tdFreeDataCols(pHelper->pDataCols[1]); } -void *tsdbDecodeSFileInfo(void *buf, STsdbFileInfo *pInfo) { - buf = taosDecodeFixedU32(buf, &(pInfo->offset)); - buf = taosDecodeFixedU32(buf, &(pInfo->len)); - buf = taosDecodeFixedU64(buf, &(pInfo->size)); - buf = taosDecodeFixedU64(buf, &(pInfo->tombSize)); - buf = taosDecodeFixedU32(buf, &(pInfo->totalBlocks)); - buf = taosDecodeFixedU32(buf, &(pInfo->totalSubBlocks)); +static int tsdbInitHelper(SRWHelper *pHelper, STsdbRepo *pRepo, tsdb_rw_helper_t type) { + if (pHelper == NULL || pRepo == NULL) return -1; - return buf; -} \ No newline at end of file + memset((void *)pHelper, 0, sizeof(*pHelper)); + + // Init global configuration + pHelper->config.type = type; + pHelper->config.maxTables = pRepo->config.maxTables; + pHelper->config.maxRowSize = pRepo->tsdbMeta->maxRowBytes; + pHelper->config.maxRows = pRepo->config.maxRowsPerFileBlock; + pHelper->config.maxCols = pRepo->tsdbMeta->maxCols; + pHelper->config.minRowsPerFileBlock = pRepo->config.minRowsPerFileBlock; + pHelper->config.maxRowsPerFileBlock = pRepo->config.maxRowsPerFileBlock; + pHelper->config.compress = pRepo->config.compression; + + pHelper->state = TSDB_HELPER_CLEAR_STATE; + + // Init file part + if (tsdbInitHelperFile(pHelper) < 0) goto _err; + + // Init table part + tsdbInitHelperTable(pHelper); + + // Init block part + if (tsdbInitHelperBlock(pHelper) < 0) goto _err; + + pHelper->pBuffer = + tmalloc(sizeof(SCompData) + (sizeof(SCompCol) + sizeof(TSCKSUM) + COMP_OVERFLOW_BYTES) * pHelper->config.maxCols + + pHelper->config.maxRowSize * pHelper->config.maxRowsPerFileBlock + sizeof(TSCKSUM)); + if (pHelper->pBuffer == NULL) goto _err; + + return 0; + +_err: + tsdbDestroyHelper(pHelper); + return -1; +} + +static int comparColIdCompCol(const void *arg1, const void *arg2) { + return (*(int16_t *)arg1) - ((SCompCol *)arg2)->colId; +} + +static int comparColIdDataCol(const void *arg1, const void *arg2) { + return (*(int16_t *)arg1) - ((SDataCol *)arg2)->colId; +} + +static int tsdbLoadSingleColumnData(int fd, SCompBlock *pCompBlock, SCompCol *pCompCol, void *buf) { + size_t tsize = sizeof(SCompData) + sizeof(SCompCol) * pCompBlock->numOfCols; + if (lseek(fd, pCompBlock->offset + tsize + pCompCol->offset, SEEK_SET) < 0) return -1; + if (tread(fd, buf, pCompCol->len) < pCompCol->len) return -1; + + return 0; +} + +static int tsdbLoadSingleBlockDataCols(SRWHelper *pHelper, SCompBlock *pCompBlock, int16_t *colIds, int numOfColIds, + SDataCols *pDataCols) { + if (tsdbLoadCompData(pHelper, pCompBlock, NULL) < 0) return -1; + int fd = (pCompBlock->last) ? pHelper->files.lastF.fd : pHelper->files.dataF.fd; + + void *ptr = NULL; + for (int i = 0; i < numOfColIds; i++) { + int16_t colId = colIds[i]; + + ptr = bsearch((void *)&colId, (void *)pHelper->pCompData->cols, pHelper->pCompData->numOfCols, sizeof(SCompCol), + comparColIdCompCol); + if (ptr == NULL) continue; + SCompCol *pCompCol = (SCompCol *)ptr; + + ptr = + bsearch((void *)&colId, (void *)(pDataCols->cols), pDataCols->numOfCols, sizeof(SDataCol), comparColIdDataCol); + ASSERT(ptr != NULL); + SDataCol *pDataCol = (SDataCol *)ptr; + + pDataCol->len = pCompCol->len; + if (tsdbLoadSingleColumnData(fd, pCompBlock, pCompCol, pDataCol->pData) < 0) return -1; + } + + return 0; +} + +static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, char *content, int32_t len, int8_t comp, int numOfRows, + int maxPoints, char *buffer, int bufferSize) { + // Verify by checksum + if (!taosCheckChecksumWhole((uint8_t *)content, len)) return -1; + + // Decode the data + if (comp) { + // // Need to decompress + pDataCol->len = (*(tDataTypeDesc[pDataCol->type].decompFunc))( + content, len - sizeof(TSCKSUM), numOfRows, pDataCol->pData, pDataCol->spaceSize, comp, buffer, bufferSize); + if (pDataCol->type == TSDB_DATA_TYPE_BINARY || pDataCol->type == TSDB_DATA_TYPE_NCHAR) { + dataColSetOffset(pDataCol, numOfRows); + } + } else { + // No need to decompress, just memcpy it + pDataCol->len = len - sizeof(TSCKSUM); + memcpy(pDataCol->pData, content, pDataCol->len); + if (pDataCol->type == TSDB_DATA_TYPE_BINARY || pDataCol->type == TSDB_DATA_TYPE_NCHAR) { + dataColSetOffset(pDataCol, numOfRows); + } + } + return 0; +} + +static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *pDataCols) { + ASSERT(pCompBlock->numOfSubBlocks <= 1); + + ASSERT(tsizeof(pHelper->pBuffer) >= pCompBlock->len); + + SCompData *pCompData = (SCompData *)pHelper->pBuffer; + + int fd = (pCompBlock->last) ? pHelper->files.lastF.fd : pHelper->files.dataF.fd; + if (lseek(fd, pCompBlock->offset, SEEK_SET) < 0) goto _err; + if (tread(fd, (void *)pCompData, pCompBlock->len) < pCompBlock->len) goto _err; + ASSERT(pCompData->numOfCols == pCompBlock->numOfCols); + + int32_t tsize = sizeof(SCompData) + sizeof(SCompCol) * pCompBlock->numOfCols + sizeof(TSCKSUM); + if (!taosCheckChecksumWhole((uint8_t *)pCompData, tsize)) goto _err; + + pDataCols->numOfRows = pCompBlock->numOfRows; + + // Recover the data + int ccol = 0; + int dcol = 0; + while (dcol < pDataCols->numOfCols) { + SDataCol *pDataCol = &(pDataCols->cols[dcol]); + if (ccol >= pCompData->numOfCols) { + // Set current column as NULL and forward + dataColSetNEleNull(pDataCol, pCompBlock->numOfRows, pDataCols->maxPoints); + dcol++; + continue; + } + + SCompCol *pCompCol = &(pCompData->cols[ccol]); + + if (pCompCol->colId == pDataCol->colId) { + if (pCompBlock->algorithm == TWO_STAGE_COMP) { + int zsize = pDataCol->bytes * pCompBlock->numOfRows + COMP_OVERFLOW_BYTES; + if (pCompCol->type == TSDB_DATA_TYPE_BINARY || pCompCol->type == TSDB_DATA_TYPE_NCHAR) { + zsize += (sizeof(VarDataLenT) * pCompBlock->numOfRows); + } + pHelper->compBuffer = trealloc(pHelper->compBuffer, zsize); + if (pHelper->compBuffer == NULL) goto _err; + } + if (tsdbCheckAndDecodeColumnData(pDataCol, (char *)pCompData + tsize + pCompCol->offset, pCompCol->len, + pCompBlock->algorithm, pCompBlock->numOfRows, pDataCols->maxPoints, + pHelper->compBuffer, tsizeof(pHelper->compBuffer)) < 0) + goto _err; + dcol++; + ccol++; + } else if (pCompCol->colId < pDataCol->colId) { + ccol++; + } else { + // Set current column as NULL and forward + dataColSetNEleNull(pDataCol, pCompBlock->numOfRows, pDataCols->maxPoints); + dcol++; + } + } + + return 0; + +_err: + return -1; +} diff --git a/src/util/inc/tlist.h b/src/util/inc/tlist.h index 9e4dfe4580..a4ed9311e2 100644 --- a/src/util/inc/tlist.h +++ b/src/util/inc/tlist.h @@ -57,6 +57,7 @@ SListNode *tdListPopHead(SList *list); SListNode *tdListPopTail(SList *list); SListNode *tdListPopNode(SList *list, SListNode *node); void tdListMove(SList *src, SList *dst); +void tdListDiscard(SList *list); void tdListNodeGetData(SList *list, SListNode *node, void *target); void tdListInitIter(SList *list, SListIter *pIter, TD_LIST_DIRECTION_T direction); diff --git a/src/util/src/tlist.c b/src/util/src/tlist.c index bdb12a59f9..0c1b8bd59a 100644 --- a/src/util/src/tlist.c +++ b/src/util/src/tlist.c @@ -148,6 +148,13 @@ void tdListMove(SList *src, SList *dst) { } } +void tdListDiscard(SList *list) { + if (list) { + list->head = list->tail = NULL; + list->numOfEles = 0; + } +} + void tdListNodeGetData(SList *list, SListNode *node, void *target) { memcpy(target, node->data, list->eleSize); } void tdListInitIter(SList *list, SListIter *pIter, TD_LIST_DIRECTION_T direction) { diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c index f1f3481865..0532b442bb 100644 --- a/src/util/src/tskiplist.c +++ b/src/util/src/tskiplist.c @@ -219,6 +219,8 @@ void *tSkipListDestroy(SSkipList *pSkipList) { void tSkipListNewNodeInfo(SSkipList *pSkipList, int32_t *level, int32_t *headSize) { if (pSkipList == NULL) { + *level = 1; + *headSize = SL_NODE_HEADER_SIZE(*level); return; } From c2d352b95aae3efbd60a006a897a52af4f7e3261 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 12 Jun 2020 15:49:34 +0000 Subject: [PATCH 05/65] TD-353 --- src/util/inc/tkvstore.h | 2 +- src/util/src/tkvstore.c | 82 +++++++++++++++++++++-------------------- 2 files changed, 44 insertions(+), 40 deletions(-) diff --git a/src/util/inc/tkvstore.h b/src/util/inc/tkvstore.h index a57d0e95cf..346e567c41 100644 --- a/src/util/inc/tkvstore.h +++ b/src/util/inc/tkvstore.h @@ -25,7 +25,7 @@ typedef int (*iterFunc)(void *, void *cont, int contLen); typedef void (*afterFunc)(void *); typedef struct { - int64_t size; + int64_t size; // including 512 bytes of header size int64_t tombSize; int64_t nRecords; int64_t nDels; diff --git a/src/util/src/tkvstore.c b/src/util/src/tkvstore.c index 88cd446349..abd9d039a9 100644 --- a/src/util/src/tkvstore.c +++ b/src/util/src/tkvstore.c @@ -58,7 +58,7 @@ int tdCreateKVStore(char *fname) { if (fd < 0) { uError("failed to open file %s since %s", fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + goto _err; } if (tdInitKVStoreHeader(fd, fname) < 0) { @@ -80,6 +80,11 @@ int tdCreateKVStore(char *fname) { } return 0; + +_err: + if (fd > 0) close(fd); + remove(fname); + return -1; } int tdDestroyKVStore(char *fname) { @@ -105,7 +110,7 @@ SKVStore *tdOpenKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void *appH goto _err; } - if (access(pStore->fsnap, F_OK) == 0) { + if (access(pStore->fsnap, F_OK) == 0) { // .snap file exists uTrace("file %s exists, try to recover the KV store", pStore->fsnap); pStore->sfd = open(pStore->fsnap, O_RDONLY); if (pStore->sfd < 0) { @@ -114,16 +119,22 @@ SKVStore *tdOpenKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void *appH goto _err; } - if (tdLoadKVStoreHeader(pStore->sfd, pStore->fsnap, &info) < 0) goto _err; + if (tdLoadKVStoreHeader(pStore->sfd, pStore->fsnap, &info) < 0) { + if (terrno != TSDB_CODE_COM_FILE_CORRUPTED) goto _err; + } else { + if (ftruncate(pStore->fd, info.size) < 0) { + uError("failed to truncate %s to " PRId64 " size since %s", pStore->fname, info.size, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } - if (ftruncate(pStore->fd, info.size) < 0) { - uError("failed to truncate %s to " PRId64 " size since %s", pStore->fname, info.size, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; + if (tdUpdateKVStoreHeader(pStore->fd, pStore->fname, &info) < 0) goto _err; + if (fsync(pStore->fd) < 0) { + uError("failed to fsync file %s since %s", pStore->fname, strerror(errno)); + goto _err; + } } - if (tdUpdateKVStoreHeader(pStore->fd, pStore->fname, &info) < 0) goto _err; - close(pStore->sfd); pStore->sfd = -1; remove(pStore->fsnap); @@ -131,22 +142,7 @@ SKVStore *tdOpenKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void *appH if (tdLoadKVStoreHeader(pStore->fd, pStore->fname, &info) < 0) goto _err; - struct stat tfstat; - if (fstat(pStore->fd, &tfstat) < 0) { - uError("failed to fstat file %s since %s", pStore->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - ASSERT(info.size == tfstat.st_size); - - if (lseek(pStore->fd, TD_KVSTORE_HEADER_SIZE, SEEK_SET) < 0) { - uError("failed to lseek file %s since %s", pStore->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - pStore->info.size += TD_KVSTORE_HEADER_SIZE; + pStore->info.size = TD_KVSTORE_HEADER_SIZE; if (tdRestoreKVStore(pStore) < 0) goto _err; @@ -326,7 +322,9 @@ static int tdUpdateKVStoreHeader(int fd, char *fname, SStoreInfo *pInfo) { return -1; } - tdEncodeStoreInfo(buf, pInfo); + void *pBuf = tdEncodeStoreInfo(buf, pInfo); + ASSERT(POINTER_DISTANCE(pBuf, buf) + sizeof(TSCKSUM) <= TD_KVSTORE_HEADER_SIZE); + taosCalcChecksumAppend(0, (uint8_t *)buf, TD_KVSTORE_HEADER_SIZE); if (twrite(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) { uError("failed to write %d bytes to file %s since %s", TD_KVSTORE_HEADER_SIZE, fname, strerror(errno)); @@ -445,34 +443,36 @@ static void *tdDecodeKVRecord(void *buf, SKVRecord *pRecord) { } static int tdRestoreKVStore(SKVStore *pStore) { - char tbuf[128] = "\0"; - void * buf = NULL; - int maxBufSize = 0; - SKVRecord rInfo = {0}; + char tbuf[128] = "\0"; + void * buf = NULL; + int maxBufSize = 0; + SKVRecord rInfo = {0}; + SHashMutableIterator *pIter = NULL; ASSERT(TD_KVSTORE_HEADER_SIZE == lseek(pStore->fd, 0, SEEK_CUR)); + ASSERT(pStore->info.size == TD_KVSTORE_HEADER_SIZE); while (true) { ssize_t tsize = tread(pStore->fd, tbuf, sizeof(SKVRecord)); if (tsize == 0) break; if (tsize < sizeof(SKVRecord)) { - uError("failed to read %d bytes from file %s since %s", sizeof(SKVRecord), pStore->fname, strerror(errno)); + uError("failed to read %d bytes from file %s at offset %" PRId64 "since %s", sizeof(SKVRecord), pStore->fname, + pStore->info.size, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); goto _err; } char *pBuf = tdDecodeKVRecord(tbuf, &rInfo); ASSERT(POINTER_DISTANCE(pBuf, tbuf) == sizeof(SKVRecord)); - ASSERT(rInfo.offset > 0 ? pStore->info.size == rInfo.offset : true); + ASSERT(pStore->info.size == rInfo.offset); if (rInfo.offset < 0) { taosHashRemove(pStore->map, (void *)(&rInfo.uid), sizeof(rInfo.uid)); pStore->info.size += sizeof(SKVRecord); pStore->info.nRecords--; pStore->info.nDels++; - pStore->info.tombSize += (rInfo.size + sizeof(SKVRecord) + sizeof(SKVRecord)); + pStore->info.tombSize += (rInfo.size + sizeof(SKVRecord) * 2); } else { - // TODO: add statistics ASSERT(rInfo.offset > 0 && rInfo.size > 0); if (taosHashPut(pStore->map, (void *)(&rInfo.uid), sizeof(rInfo.uid), &rInfo, sizeof(rInfo)) < 0) { uError("failed to put record in KV store %s", pStore->fname); @@ -487,6 +487,9 @@ static int tdRestoreKVStore(SKVStore *pStore) { terrno = TAOS_SYSTEM_ERROR(errno); goto _err; } + + pStore->info.size += (sizeof(SKVRecord) + rInfo.size); + pStore->info.nRecords++; } } @@ -497,7 +500,7 @@ static int tdRestoreKVStore(SKVStore *pStore) { goto _err; } - SHashMutableIterator *pIter = taosHashCreateIter(pStore->map); + pIter = taosHashCreateIter(pStore->map); if (pIter == NULL) { uError("failed to create hash iter while opening KV store %s", pStore->fname); terrno = TSDB_CODE_COM_OUT_OF_MEMORY; @@ -508,13 +511,14 @@ static int tdRestoreKVStore(SKVStore *pStore) { SKVRecord *pRecord = taosHashIterGet(pIter); if (lseek(pStore->fd, pRecord->offset, SEEK_SET) < 0) { - uError("failed to lseek file %s since %s", pStore->fname, strerror(errno)); + uError("failed to lseek file %s since %s, offset %" PRId64, pStore->fname, strerror(errno), pRecord->offset); terrno = TAOS_SYSTEM_ERROR(errno); goto _err; } if (tread(pStore->fd, buf, pRecord->size) < pRecord->size) { - uError("failed to read %d bytes from file %s since %s", pRecord->size, pStore->fname, strerror(errno)); + uError("failed to read %" PRId64 " bytes from file %s since %s, offset %" PRId64, pRecord->size, pStore->fname, + strerror(errno), pRecord->offset); terrno = TAOS_SYSTEM_ERROR(errno); goto _err; } @@ -528,14 +532,14 @@ static int tdRestoreKVStore(SKVStore *pStore) { if (pStore->iFunc) (*pStore->iFunc)(pStore->appH, buf, pRecord->size); } - taosHashDestroyIter(pIter); - if (pStore->aFunc) (*pStore->aFunc)(pStore->appH); + taosHashDestroyIter(pIter); tfree(buf); return 0; _err: + taosHashDestroyIter(pIter); tfree(buf); return -1; } \ No newline at end of file From 6413285bb9c14668ba1299e4cb643132636a5dd8 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 13 Jun 2020 02:43:15 +0000 Subject: [PATCH 06/65] TD-353 --- src/tsdb/inc/tsdbMain.h | 6 ++- src/tsdb/src/tsdbMain.c | 4 ++ src/tsdb/src/tsdbMemTable.c | 95 +++++++++++++++++++++---------------- 3 files changed, 62 insertions(+), 43 deletions(-) diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 6055f30aad..40c5ddbab5 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -254,7 +254,7 @@ typedef struct { // ------------------ tsdbMain.c typedef struct { - int8_t state; + int8_t state; char* rootDir; STsdbCfg config; @@ -265,9 +265,10 @@ typedef struct { SMemTable* mem; SMemTable* imem; STsdbFileH* tsdbFileH; - pthread_mutex_t mutex; int commit; pthread_t commitThread; + pthread_mutex_t mutex; + bool repoLocked; } STsdbRepo; // Operations @@ -309,6 +310,7 @@ void tsdbFreeFileH(STsdbFileH* pFileH); // ------------------ tsdbMain.c #define REPO_ID(r) (r)->config.tsdbId +#define IS_REPO_LOCKED(r) (r)->repoLocked char* tsdbGetMetaFileName(char* rootDir); int tsdbLockRepo(STsdbRepo* pRepo); diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 36595ab987..ef1a4af064 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -337,10 +337,12 @@ int tsdbLockRepo(STsdbRepo *pRepo) { terrno = TAOS_SYSTEM_ERROR(code); return -1; } + pRepo->repoLocked = true; return 0; } int tsdbUnlockRepo(STsdbRepo *pRepo) { + pRepo->repoLocked = false; int code = pthread_mutex_unlock(&pRepo->mutex); if (code != 0) { tsdbError("vgId:%d failed to unlock tsdb since %s", REPO_ID(pRepo), strerror(errno)); @@ -679,6 +681,8 @@ static STsdbRepo *tsdbNewRepo(char *rootDir, STsdbAppH *pAppH, STsdbCfg *pCfg) { goto _err; } + pRepo->repoLocked = false; + pRepo->rootDir = strdup(rootDir); if (pRepo->rootDir == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 4ac5f5a5f2..c848b7644b 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -20,61 +20,69 @@ // ---------------- INTERNAL FUNCTIONS ---------------- int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { - STsdbCfg *pCfg = &pRepo->config; - int32_t level = 0; - int32_t headSize = 0; - TSKEY key = dataRowKey(row); + STsdbCfg * pCfg = &pRepo->config; + int32_t level = 0; + int32_t headSize = 0; + TSKEY key = dataRowKey(row); + SMemTable * pMemTable = pRepo->mem; + STableData *pTableData = NULL; + int bytes = 0; - // TODO - tSkipListNewNodeInfo(pRepo->mem->tData[TABLE_TID(pTable)]->pData, &level, &headSize); - - // TODO: for duplicate keys, you do not need to allocate memory here - SSkipListNode *pNode = tsdbAllocBytes(pRepo, headSize + dataRowLen(row)); - if (pNode == NULL) { - tsdbError("vgId:%d failed to insert row with key %" PRId64 " to table %s since %s", REPO_ID(pRepo), key, - TABLE_CHAR_NAME(pTable), tstrerror(terrno)); - return -1; + if (pMemTable != NULL && pMemTable->tData[TABLE_TID(pTable)] != NULL && + pMemTable->tData[TABLE_TID(pTable)]->uid == TALBE_UID(pTable)) { + pTableData = pMemTable->tData[TABLE_TID(pTable)]; } - SMemTable *pMemTable = pRepo->mem; - ASSERT(pMemTable != NULL); + tSkipListNewNodeInfo(pTableData, &level, &headSize); + bytes = headSize + dataRowLen(row); + SSkipListNode *pNode = tsdbAllocBytes(pRepo, bytes); + if (pNode == NULL) { + tsdbError("vgId:%d failed to insert row with key %" PRId64 " to table %s while allocate %d bytes since %s", + REPO_ID(pRepo), key, TABLE_CHAR_NAME(pTable), bytes, tstrerror(terrno)); + return -1; + } pNode->level = level; dataRowCpy(SL_GET_NODE_DATA(pNode), row); - STableData *pTableData = pMemTable->tData[TABLE_TID(pTable)]; - if (pTableData == NULL) { - pTableData = tsdbNewTableData(pCfg); + // Operations above may change pRepo->mem, retake those values + ASSERT(pRepo->mem != NULL); + pMemTable = pRepo->mem; + pTableData = pMemTable->tData[TABLE_TID(pTable)]; + + if (pTableData == NULL || pTableData->uid != TALBE_UID(pTable)) { + if (pTableData != NULL) { // destroy the table skiplist (may have race condition problem) + pMemTable->tData[TABLE_TID(pTable)] = NULL; + tsdbFreeTableData(pTableData); + } + pTableData = tsdbNewTableData(pCfg, pTable); if (pTableData == NULL) { - tsdbError("vgId:%d failed to insert row with key %" PRId64 " to table %s since %s", REPO_ID(pRepo), key, - TABLE_CHAR_NAME(pTable), tstrerror(terrno)); + tsdbError("vgId:%d failed to insert row with key %" PRId64 + " to table %s while create new table data object since %s", + REPO_ID(pRepo), key, TABLE_CHAR_NAME(pTable), tstrerror(terrno)); + tsdbFreeBytes(pRepo, (void *)pNode, bytes); return -1; } pRepo->mem->tData[TABLE_TID(pTable)] = pTableData; } - ASSERT(pTableData != NULL); - - if (pTableData->uid != TALBE_UID(pTable)) { - // TODO - } + ASSERT(pTableData != NULL) && pTableData->uid == TALBE_UID(pTable); if (tSkipListPut(pTableData->pData, pNode) == NULL) { - tsdbFreeBytes(pRepo, (void *)pNode, headSize + dataRowLen); - return 0; + tsdbFreeBytes(pRepo, (void *)pNode, bytes); + } else { + if (pMemTable->keyFirst > key) pMemTable->keyFirst = key; + if (pMemTable->keyLast < key) pMemTable->keyLast = key; + pMemTable->numOfRows++; + + if (pTableData->keyFirst > key) pTableData->keyFirst = key; + if (pTableData->keyLast < key) pTableData->keyLast = key; + pTableData->numOfRows++; + + ASSERT(pTableData->numOfRows == tSkipListGetSize(pTableData->pData)); } - if (pMemTable->keyFirst > key) pMemTable->keyFirst = key; - if (pMemTable->keyLast < key) pMemTable->keyLast = key; - pMemTable->numOfRows++; - - if (pTableData->keyFirst > key) pTableData->keyFirst = key; - if (pTableData->keyLast < key) pTableData->keyLast = key; - pTableData->numOfRows++; - - ASSERT(pTableData->numOfRows == tSkipListGetSize(pTableData->pData)); - tsdbTrace("vgId:%d a row is inserted to table %s tid %d uid %" PRIu64 " key %" PRIu64, REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TALBE_UID(pTable), key); @@ -82,12 +90,14 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { } int tsdbRefMemTable(STsdbRepo *pRepo, SMemTable *pMemTable) { + ASSERT(IS_REPO_LOCKED(pRepo)); ASSERT(pMemTable != NULL); T_REF_INC(pMemTable); } // Need to lock the repository int tsdbUnRefMemTable(STsdbRepo *pRepo, SMemTable *pMemTable) { + ASSERT(IS_REPO_LOCKED(pRepo)); ASSERT(pMemTable != NULL); if (T_REF_DEC(pMemTable) == 0) { @@ -117,7 +127,7 @@ int tsdbUnRefMemTable(STsdbRepo *pRepo, SMemTable *pMemTable) { // ---------------- LOCAL FUNCTIONS ---------------- static FORCE_INLINE STsdbBufBlock *tsdbGetCurrBufBlock(STsdbRepo *pRepo) { - if (pRepo->mem == NULL) return NULL; + if (pRepo == NULL || pRepo->mem == NULL) return NULL; SListNode *pNode = listTail(pRepo->mem); if (pNode == NULL) return NULL; @@ -258,13 +268,14 @@ static STableData *tsdbNewTableData(STsdbCfg *pCfg, STable *pTable) { pTableData->keyLast = 0; pTableData->numOfRows = 0; - pTableData->pData = tSkipListCreate(TSDB_DATA_SKIPLIST_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], 0, 0, 0, ); + pTableData->pData = tSkipListCreate(TSDB_DATA_SKIPLIST_LEVEL, TSDB_DATA_TYPE_TIMESTAMP, + TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], 0, 0, 0, tsdbGetTsTupleKey); if (pTableData->pData == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _err; } - // TODO + // TODO: operation here should not be here, remove it pTableData->pData->level = 1; return pTableData; @@ -279,4 +290,6 @@ static void tsdbFreeTableData(STableData *pTableData) { tSkipListDestroy(pTableData->pData); free(pTableData); } -} \ No newline at end of file +} + +static char *tsdbGetTsTupleKey(const void *data) { return dataRowTuple(data); } \ No newline at end of file From 21a5beabb94ad3d296f10a42c4f5a59264bb6f32 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 13 Jun 2020 02:47:44 +0000 Subject: [PATCH 07/65] TD-353 --- src/tsdb/src/tsdbBuffer.c | 2 +- src/tsdb/src/tsdbMemTable.c | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/tsdb/src/tsdbBuffer.c b/src/tsdb/src/tsdbBuffer.c index 013bae69a0..f27af3b722 100644 --- a/src/tsdb/src/tsdbBuffer.c +++ b/src/tsdb/src/tsdbBuffer.c @@ -114,7 +114,7 @@ void tsdbCloseBufPool(STsdbRepo *pRepo) { SListNode *tsdbAllocBufBlockFromPool(STsdbRepo *pRepo) { ASSERT(pRepo != NULL && pRepo->pool != NULL); - // ASSERT pRepo is locked + ASSERT(IS_REPO_LOCKED(pRepo)); STsdbCfg * pCfg = &pRepo->config; STsdbBufPool *pBufPool = pRepo->pool; diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index c848b7644b..55fda4e116 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -170,6 +170,17 @@ static void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) { tsdbUnRefMemTable(pRepo, pIMem); } + } else { + if (tsdbLockRepo(pRepo) < 0) { + tsdbFreeMemTable(pMemTable); + return NULL; + } + + SListNode *pNode = tsdbAllocBufBlockFromPool(pRepo); + tdListAppendNode(pMemTable->bufBlockList, pNode); + pRepo->mem = pMemTable; + + if (tsdbUnlockRepo(pRepo) < 0) return NULL; } } From 7aa1ea29cff8f17de66f43534d06239f69ef646a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 13 Jun 2020 05:28:26 +0000 Subject: [PATCH 08/65] TD-353 --- src/tsdb/src/tsdbBuffer.c | 2 -- src/tsdb/src/tsdbMemTable.c | 62 ++++++++++++++++++------------------- 2 files changed, 30 insertions(+), 34 deletions(-) diff --git a/src/tsdb/src/tsdbBuffer.c b/src/tsdb/src/tsdbBuffer.c index f27af3b722..70f8e054f8 100644 --- a/src/tsdb/src/tsdbBuffer.c +++ b/src/tsdb/src/tsdbBuffer.c @@ -123,8 +123,6 @@ SListNode *tsdbAllocBufBlockFromPool(STsdbRepo *pRepo) { pthread_cond_wait(&(pBufPool->poolNotEmpty), &(pRepo->mutex)); } - ASSERT(!POOL_IS_EMPTY(pBufPool)); - SListNode * pNode = tdListPopHead(pBufPool->bufBlockList); STsdbBufBlock *pBufBlock = NULL; tdListNodeGetData(pBufPool->bufBlockList, pNode, (void *)(&pBufBlock)); diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 55fda4e116..cfb405f518 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -97,7 +97,6 @@ int tsdbRefMemTable(STsdbRepo *pRepo, SMemTable *pMemTable) { // Need to lock the repository int tsdbUnRefMemTable(STsdbRepo *pRepo, SMemTable *pMemTable) { - ASSERT(IS_REPO_LOCKED(pRepo)); ASSERT(pMemTable != NULL); if (T_REF_DEC(pMemTable) == 0) { @@ -105,12 +104,15 @@ int tsdbUnRefMemTable(STsdbRepo *pRepo, SMemTable *pMemTable) { STsdbBufPool *pBufPool = pRepo->pPool; SListNode *pNode = NULL; + // TODO: check the correctness of this code part + if (tsdbLockRepo(pRepo) < 0) return -1; while ((pNode = tdListPopHead(pMemTable->bufBlockList)) != NULL) { tdListAppendNode(pBufPool->bufBlockList, pNode); if (pthread_cond_signal(&pBufPool->poolNotEmpty) != 0) { // TODO } } + if (tsdbUnlockRepo(pRepo) < 0) return -1; for (int i = 0; i < pCfg->maxTables; i++) { if (pMemTable->tData[i] != NULL) { @@ -127,7 +129,8 @@ int tsdbUnRefMemTable(STsdbRepo *pRepo, SMemTable *pMemTable) { // ---------------- LOCAL FUNCTIONS ---------------- static FORCE_INLINE STsdbBufBlock *tsdbGetCurrBufBlock(STsdbRepo *pRepo) { - if (pRepo == NULL || pRepo->mem == NULL) return NULL; + ASSERT(pRepo != NULL); + if (pRepo->mem == NULL) return NULL; SListNode *pNode = listTail(pRepo->mem); if (pNode == NULL) return NULL; @@ -141,45 +144,40 @@ static FORCE_INLINE STsdbBufBlock *tsdbGetCurrBufBlock(STsdbRepo *pRepo) { static void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) { STsdbCfg * pCfg = &pRepo->config; STsdbBufBlock *pBufBlock = tsdbGetCurrBufBlock(pRepo); + int code = 0; if (pBufBlock != NULL && pBufBlock->remain < bytes) { - if (listNEles(pRepo->mem) >= pCfg->totalBlocks / 2) { // need to trigger commit - if (pRepo->imem != NULL) { - if (pRepo->commit) pthread_join(pRepo->commitThread, NULL); - - ASSERT(pRepo->commit == 0); - - SMemTable *pIMem = pRepo->imem; - if (tsdbLockRepo(pRepo) < 0) { - // TODO + if (listNEles(pRepo->mem) >= pCfg->totalBlocks / 2) { // need to commit mem + if (pRepo->imem) { + code = pthread_join(pRepo->commitThread, NULL); + if (code != 0) { + tsdbError("vgId:%d failed to thread join since %s", REPO_ID(pRepo), strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); return NULL; } - - pRepo->imem = pRepo->mem; - pRepo->mem = NULL; - pRepo->commit = 1; - if (pthread_create(&pRepo->commitThread, NULL, tsdbCommitData, (void *)pRepo) != 0) { - // TODO - tsdbUnlockRepo(pRepo); - return NULL; - } - if (tsdbUnlockRepo(pRepo) < 0) { - // TODO - return NULL; - } - - tsdbUnRefMemTable(pRepo, pIMem); } - } else { - if (tsdbLockRepo(pRepo) < 0) { - tsdbFreeMemTable(pMemTable); + + ASSERT(pRepo->commit == 0); + SMemTable *pImem = pRepo->imem; + + if (tsdbLockRepo(pRepo) < 0) return NULL; + pRepo->imem = pRepo->mem; + pRepo->mem = NULL; + pRepo->commit = 1; + code = pthread_create(&pRepo->commitThread, NULL, tsdbCommitData, (void *)pRepo); + if (code != 0) { + tsdbError("vgId:%d failed to create commit thread since %s", REPO_ID(pRepo), strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(code); + tsdbUnlockRepo(pRepo); return NULL; } + if (tsdbUnlockRepo(pRepo) < 0) return NULL; + if (pImem && tsdbUnRefMemTable(pRepo, pImem) < 0) return NULL; + } else { + if (tsdbLockRepo(pRepo) < 0) return NULL; SListNode *pNode = tsdbAllocBufBlockFromPool(pRepo); - tdListAppendNode(pMemTable->bufBlockList, pNode); - pRepo->mem = pMemTable; - + tdListAppendNode(pRepo->mem->bufBlockList, pNode); if (tsdbUnlockRepo(pRepo) < 0) return NULL; } } From 28fd0574dbda478f6963d381715d390aae9fbc8e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 13 Jun 2020 05:38:21 +0000 Subject: [PATCH 09/65] TD-353 --- src/tsdb/src/tsdbMemTable.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index cfb405f518..ab8cccb293 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -108,9 +108,13 @@ int tsdbUnRefMemTable(STsdbRepo *pRepo, SMemTable *pMemTable) { if (tsdbLockRepo(pRepo) < 0) return -1; while ((pNode = tdListPopHead(pMemTable->bufBlockList)) != NULL) { tdListAppendNode(pBufPool->bufBlockList, pNode); - if (pthread_cond_signal(&pBufPool->poolNotEmpty) != 0) { - // TODO - } + } + int code = pthread_cond_signal(&pBufPool->poolNotEmpty); + if (code != 0) { + tsdbUnlockRepo(pRepo); + tsdbError("vgId:%d failed to signal pool not empty since %s", REPO_ID(pRepo), strerror(code)); + terrno = TAOS_SYSTEM_ERROR(code); + return -1; } if (tsdbUnlockRepo(pRepo) < 0) return -1; From d5c20d7b539ce63f8a6c63c392350331c94d1e15 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 13 Jun 2020 05:42:52 +0000 Subject: [PATCH 10/65] TD-353 --- src/tsdb/inc/tsdbMain.h | 3 +++ src/tsdb/src/tsdbMemTable.c | 11 ++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 40c5ddbab5..25e8b9ce40 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -292,6 +292,9 @@ int tsdbOpenBufPool(STsdbRepo* pRepo); SListNode* tsdbAllocBufBlockFromPool(STsdbRepo* pRepo); // ------------------ tsdbMemTable.c +int tsdbInsertRowToMem(STsdbRepo* pRepo, SDataRow row, STable* pTable); +int tsdbRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable); +int tsdbUnRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable); // ------------------ tsdbFile.c #define TSDB_KEY_FILEID(key, daysPerFile, precision) ((key) / tsMsPerDay[(precision)] / (daysPerFile)) diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index ab8cccb293..074007af5a 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -18,6 +18,16 @@ #define TSDB_DATA_SKIPLIST_LEVEL 5 +static FORCE_INLINE STsdbBufBlock *tsdbGetCurrBufBlock(STsdbRepo *pRepo); + +static void * tsdbAllocBytes(STsdbRepo *pRepo, int bytes); +static void tsdbFreeBytes(STsdbRepo *pRepo, void *ptr, int bytes); +static SMemTable * tsdbNewMemTable(STsdbCfg *pCfg); +static void tsdbFreeMemTable(SMemTable *pMemTable); +static STableData *tsdbNewTableData(STsdbCfg *pCfg, STable *pTable); +static void tsdbFreeTableData(STableData *pTableData); +static char * tsdbGetTsTupleKey(const void *data); + // ---------------- INTERNAL FUNCTIONS ---------------- int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { STsdbCfg * pCfg = &pRepo->config; @@ -104,7 +114,6 @@ int tsdbUnRefMemTable(STsdbRepo *pRepo, SMemTable *pMemTable) { STsdbBufPool *pBufPool = pRepo->pPool; SListNode *pNode = NULL; - // TODO: check the correctness of this code part if (tsdbLockRepo(pRepo) < 0) return -1; while ((pNode = tdListPopHead(pMemTable->bufBlockList)) != NULL) { tdListAppendNode(pBufPool->bufBlockList, pNode); From 112ce3b1a4c4e51eea1ab2849ea09984046fe64b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 13 Jun 2020 06:40:57 +0000 Subject: [PATCH 11/65] TD-353 --- src/tsdb/src/tsdbMeta.c | 84 +++++++++++++++++++++++++++++++++-------- src/util/src/tkvstore.c | 13 ++++--- 2 files changed, 75 insertions(+), 22 deletions(-) diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 63a6dd9f79..9bbbce0f7b 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -13,11 +13,12 @@ * along with this program. If not, see . */ #include -#include "tskiplist.h" -#include "tsdb.h" -#include "taosdef.h" #include "hash.h" +#include "taosdef.h" +#include "tchecksum.h" +#include "tsdb.h" #include "tsdbMain.h" +#include "tskiplist.h" // ------------------ OUTER FUNCTIONS ------------------ int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg) { @@ -201,7 +202,7 @@ _err: // ------------------ INTERNAL FUNCTIONS ------------------ STsdbMeta *tsdbNewMeta(STsdbCfg *pCfg) { - STsdbMeta* pMeta = (STsdbMeta*)calloc(1, sizeof(*pMeta)); + STsdbMeta *pMeta = (STsdbMeta *)calloc(1, sizeof(*pMeta)); if (pMeta == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _err; @@ -209,6 +210,7 @@ STsdbMeta *tsdbNewMeta(STsdbCfg *pCfg) { int code = pthread_rwlock_init(&pMeta->rwLock, NULL); if (code != 0) { + tsdbError("vgId:%d failed to init TSDB meta r/w lock since %s", pCfg->tsdbId, strerror(code)); terrno = TAOS_SYSTEM_ERROR(code); goto _err; } @@ -233,7 +235,7 @@ STsdbMeta *tsdbNewMeta(STsdbCfg *pCfg) { return pMeta; - _err; +_err: tsdbFreeMeta(pMeta); return NULL; } @@ -249,11 +251,50 @@ void tsdbFreeMeta(STsdbMeta *pMeta) { } int tsdbOpenMeta(STsdbRepo *pRepo) { - // TODO + char * fname = NULL; + STsdbMeta *pMeta = pRepo->tsdbMeta; + ASSERT(pMeta != NULL); + + fname = tsdbGetMetaFileName(pRepo->rootDir); + if (fname == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + goto _err; + } + + pMeta->pStore = tdOpenKVStore(fname, tsdbRestoreTable, tsdbOrgMeta, (void *)pRepo); + if (pMeta->pStore == NULL) { + tsdbError("vgId:%d failed to open TSDB meta while open the kv store since %s", REPO_ID(pRepo), tstrerror(terrno)); + goto _err; + } + + tsdbTrace("vgId:%d open TSDB meta succeed", REPO_ID(pRepo)); + tfree(fname); + return 0; + +_err: + tfree(fname); + return -1; } int tsdbCloseMeta(STsdbRepo *pRepo) { - // TODO + STsdbCfg * pCfg = &pRepo->config; + STsdbMeta *pMeta = pRepo->tsdbMeta; + SListNode *pNode = NULL; + STable * pTable = NULL; + + if (pMeta == NULL) return 0; + tdCloseKVStore(pMeta->pStore); + for (int i = 1; i < pCfg->maxTables; i++) { + tsdbFreeTable(pMeta->tables[i]); + } + + while ((pNode = tdListPopHead(pMeta->superList)) != NULL) { + tdListNodeGetData(pMeta->superList, pNode, (void *)(&pTable)); + tsdbFreeTable(pTable); + } + + tsdbTrace("vgId:%d TSDB meta is closed", REPO_ID(pRepo)); + return 0; } STSchema *tsdbGetTableSchema(STsdbMeta *pMeta, STable *pTable) { @@ -387,6 +428,7 @@ static void tsdbEncodeTable(STable *pTable, char *buf, int *contLen) { } static STable *tsdbDecodeTable(void *cont, int contLen) { + // TODO STable *pTable = (STable *)calloc(1, sizeof(STable)); if (pTable == NULL) return NULL; @@ -432,6 +474,13 @@ static STable *tsdbDecodeTable(void *cont, int contLen) { } pTable->lastKey = TSKEY_INITIAL_VAL; + + if (pTable->type == TSDB_SUPER_TABLE) { + STColumn *pColSchema = schemaColAt(pTable->tagSchema, 0); + pTable->pIndex = + tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, pColSchema->type, pColSchema->bytes, 1, 0, 1, getTagIndexKey); + } + return pTable; } @@ -446,24 +495,27 @@ static int tsdbCompareSchemaVersion(const void *key1, const void *key2) { } static int tsdbRestoreTable(void *pHandle, void *cont, int contLen) { - STsdbMeta *pMeta = (STsdbMeta *)pHandle; + STsdbRepo *pRepo = (STsdbRepo *)pHandle; + STsdbMeta *pMeta = pRepo->tsdbMeta; + + if (!taosCheckChecksumWhole((uint8_t *)cont, contLen)) { + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + return -1; + } STable *pTable = tsdbDecodeTable(cont, contLen); if (pTable == NULL) return -1; - - if (pTable->type == TSDB_SUPER_TABLE) { - STColumn* pColSchema = schemaColAt(pTable->tagSchema, 0); - pTable->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, pColSchema->type, pColSchema->bytes, - 1, 0, 1, getTagIndexKey); - } - tsdbAddTableToMeta(pMeta, pTable, false); + if (tsdbAddTableToMeta(pMeta, pTable, false) < 0) return -1; + tsdbTrace("vgId:%d table %s tid %d uid %" PRIu64 " is restored from file", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), + TABLE_TID(pTable), TALBE_UID(pTable)); return 0; } static void tsdbOrgMeta(void *pHandle) { - STsdbMeta *pMeta = (STsdbMeta *)pHandle; + STsdbRepo *pRepo = (STsdbRepo *)pHandle; + STsdbCfg * pCfg = &pRepo->config; for (int i = 1; i < pMeta->maxTables; i++) { STable *pTable = pMeta->tables[i]; diff --git a/src/util/src/tkvstore.c b/src/util/src/tkvstore.c index abd9d039a9..3f27151aca 100644 --- a/src/util/src/tkvstore.c +++ b/src/util/src/tkvstore.c @@ -523,13 +523,14 @@ static int tdRestoreKVStore(SKVStore *pStore) { goto _err; } - if (!taosCheckChecksumWhole((uint8_t *)buf, pRecord->size)) { - uError("file %s has checksum error, offset " PRId64 " size %d", pStore->fname, pRecord->offset, pRecord->size); - terrno = TSDB_CODE_COM_FILE_CORRUPTED; - goto _err; + if (pStore->iFunc) { + if ((*pStore->iFunc)(pStore->appH, buf, pRecord->size) < 0) { + uError("failed to restore record uid %" PRIu64 " in kv store %s at offset %" PRId64 " size %" PRId64 + " since %s", + pStore->fname, pRecord->uid, pRecord->offset, pRecord->size, tstrerror(terrno)); + goto _err; + } } - - if (pStore->iFunc) (*pStore->iFunc)(pStore->appH, buf, pRecord->size); } if (pStore->aFunc) (*pStore->aFunc)(pStore->appH); From adc6a190be3c8429f1c01357f8b1e9836095dd0c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 13 Jun 2020 07:31:21 +0000 Subject: [PATCH 12/65] TD-353 --- src/common/inc/tdataformat.h | 4 +- src/inc/taoserror.h | 3 +- src/tsdb/src/tsdbMeta.c | 193 ++++++++++++++++++++--------------- 3 files changed, 116 insertions(+), 84 deletions(-) diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index ea0eb9ff29..47753b35a7 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -107,8 +107,8 @@ typedef struct { int tCols; int nCols; int tlen; - uint16_t flen; - uint16_t vlen; + uint16_t flen; + uint16_t vlen; int version; STColumn *columns; } STSchemaBuilder; diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 715a45d8e3..9a6045396a 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -177,7 +177,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_VND_APP_ERROR, 0, 0x0509, "vnode app // tsdb TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_ID, 0, 0x0600, "tsdb invalid table id") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_TYPE, 0, 0x0601, "tsdb invalid table schema version") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_TYPE, 0, 0x0601, "tsdb invalid table type") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_SCHEMA_VERSION, 0, 0x0602, "tsdb invalid table schema version") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_ALREADY_EXIST, 0, 0x0603, "tsdb table already exist") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_CONFIG, 0, 0x0604, "tsdb invalid configuration") @@ -190,6 +190,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TAG_VER_OUT_OF_DATE, 0, 0x060A, "tsdb tag v TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE, 0, 0x060B, "tsdb timestamp is out of range") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP, 0, 0x060C, "tsdb submit message is messed up") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_ACTION, 0, 0x060D, "tsdb invalid action") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_CREATE_TB_MSG, 0, 0x060E, "tsdb invalid create table message") // query TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_QHANDLE, 0, 0x0700, "query invalid handle") diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 9bbbce0f7b..57ab738444 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -108,6 +108,7 @@ int tsdbDropTable(TSDB_REPO_T *repo, STableId tableId) { } void* tsdbGetTableTagVal(TSDB_REPO_T* repo, const STableId* id, int32_t colId, int16_t type, int16_t bytes) { + // TODO: this function should be changed also STsdbMeta* pMeta = tsdbGetMeta(repo); STable* pTable = tsdbGetTableByUid(pMeta, id->uid); @@ -128,7 +129,10 @@ void* tsdbGetTableTagVal(TSDB_REPO_T* repo, const STableId* id, int32_t colId, i } char *tsdbGetTableName(TSDB_REPO_T *repo, const STableId *id) { - STsdbMeta *pMeta = tsdbGetMeta(repo); + // TODO: need to change as thread-safe + STsdbRepo *pRepo = (STsdbRepo *)repo; + STsdbMeta *pMeta = pRepo->tsdbMeta; + STable * pTable = tsdbGetTableByUid(pMeta, id->uid); if (pTable == NULL) { @@ -140,19 +144,30 @@ char *tsdbGetTableName(TSDB_REPO_T *repo, const STableId *id) { STableCfg *tsdbCreateTableCfgFromMsg(SMDCreateTableMsg *pMsg) { if (pMsg == NULL) return NULL; - SSchema * pSchema = (SSchema *)pMsg->data; - int16_t numOfCols = htons(pMsg->numOfColumns); - int16_t numOfTags = htons(pMsg->numOfTags); + + SSchema *pSchema = (SSchema *)pMsg->data; + int16_t numOfCols = htons(pMsg->numOfColumns); + int16_t numOfTags = htons(pMsg->numOfTags); + STSchemaBuilder schemaBuilder = {0}; STableCfg *pCfg = (STableCfg *)calloc(1, sizeof(STableCfg)); - if (pCfg == NULL) return NULL; + if (pCfg == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return NULL; + } if (tsdbInitTableCfg(pCfg, pMsg->tableType, htobe64(pMsg->uid), htonl(pMsg->sid)) < 0) goto _err; - if (tdInitTSchemaBuilder(&schemaBuilder, htonl(pMsg->sversion)) < 0) goto _err; + if (tdInitTSchemaBuilder(&schemaBuilder, htonl(pMsg->sversion)) < 0) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + goto _err; + } for (int i = 0; i < numOfCols; i++) { - tdAddColToSchema(&schemaBuilder, pSchema[i].type, htons(pSchema[i].colId), htons(pSchema[i].bytes)); + if (tdAddColToSchema(&schemaBuilder, pSchema[i].type, htons(pSchema[i].colId), htons(pSchema[i].bytes)) < 0) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + goto _err; + } } if (tsdbTableSetSchema(pCfg, tdGetSchemaFromBuilder(&schemaBuilder), false) < 0) goto _err; if (tsdbTableSetName(pCfg, pMsg->tableId, true) < 0) goto _err; @@ -161,7 +176,10 @@ STableCfg *tsdbCreateTableCfgFromMsg(SMDCreateTableMsg *pMsg) { // Decode tag schema tdResetTSchemaBuilder(&schemaBuilder, htonl(pMsg->tversion)); for (int i = numOfCols; i < numOfCols + numOfTags; i++) { - tdAddColToSchema(&schemaBuilder, pSchema[i].type, htons(pSchema[i].colId), htons(pSchema[i].bytes)); + if (tdAddColToSchema(&schemaBuilder, pSchema[i].type, htons(pSchema[i].colId), htons(pSchema[i].bytes)) < 0) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + goto _err; + } } if (tsdbTableSetTagSchema(pCfg, tdGetSchemaFromBuilder(&schemaBuilder), false) < 0) goto _err; if (tsdbTableSetSName(pCfg, pMsg->superTableId, true) < 0) goto _err; @@ -173,9 +191,15 @@ STableCfg *tsdbCreateTableCfgFromMsg(SMDCreateTableMsg *pMsg) { char *pTagData = pMsg->data + (numOfCols + numOfTags) * sizeof(SSchema); SKVRowBuilder kvRowBuilder = {0}; - if (tdInitKVRowBuilder(&kvRowBuilder) < 0) goto _err; + if (tdInitKVRowBuilder(&kvRowBuilder) < 0) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + goto _err; + } for (int i = numOfCols; i < numOfCols + numOfTags; i++) { - tdAddColToKVRow(&kvRowBuilder, htons(pSchema[i].colId), pSchema[i].type, pTagData + accBytes); + if (tdAddColToKVRow(&kvRowBuilder, htons(pSchema[i].colId), pSchema[i].type, pTagData + accBytes) < 0) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + goto _err; + } accBytes += htons(pSchema[i].bytes); } @@ -786,92 +810,41 @@ static int tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable) { return 0; } -#if 0 -#define TSDB_SUPER_TABLE_SL_LEVEL 5 // TODO: may change here -// #define TSDB_META_FILE_NAME "META" - - - - - static int tsdbInitTableCfg(STableCfg *config, ETableType type, uint64_t uid, int32_t tid) { - if (config == NULL) return -1; - if (type != TSDB_CHILD_TABLE && type != TSDB_NORMAL_TABLE && type != TSDB_STREAM_TABLE) return -1; + if (type != TSDB_CHILD_TABLE && type != TSDB_NORMAL_TABLE && type != TSDB_STREAM_TABLE) { + terrno = TSDB_CODE_TDB_INVALID_TABLE_TYPE; + return -1; + } - memset((void *)config, 0, sizeof(STableCfg)); + memset((void *)config, 0, sizeof(*config)); config->type = type; config->superUid = TSDB_INVALID_SUPER_TABLE_ID; config->tableId.uid = uid; config->tableId.tid = tid; - config->name = NULL; - config->sql = NULL; return 0; } -/** - * Set the super table UID of the created table - */ -static int tsdbTableSetSuperUid(STableCfg *config, uint64_t uid) { - if (config->type != TSDB_CHILD_TABLE) return -1; - if (uid == TSDB_INVALID_SUPER_TABLE_ID) return -1; - - config->superUid = uid; - return 0; -} - -/** - * Set the table schema in the configuration - * @param config the configuration to set - * @param pSchema the schema to set - * @param dup use the schema directly or duplicate one for use - * - * @return 0 for success and -1 for failure - */ static int tsdbTableSetSchema(STableCfg *config, STSchema *pSchema, bool dup) { if (dup) { config->schema = tdDupSchema(pSchema); + if (config->schema == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } } else { config->schema = pSchema; } return 0; } -/** - * Set the table schema in the configuration - * @param config the configuration to set - * @param pSchema the schema to set - * @param dup use the schema directly or duplicate one for use - * - * @return 0 for success and -1 for failure - */ -static int tsdbTableSetTagSchema(STableCfg *config, STSchema *pSchema, bool dup) { - if (config->type != TSDB_CHILD_TABLE) return -1; - - if (dup) { - config->tagSchema = tdDupSchema(pSchema); - } else { - config->tagSchema = pSchema; - } - return 0; -} - -static int tsdbTableSetTagValue(STableCfg *config, SKVRow row, bool dup) { - if (config->type != TSDB_CHILD_TABLE) return -1; - - if (dup) { - config->tagValues = tdKVRowDup(row); - } else { - config->tagValues = row; - } - - return 0; -} - static int tsdbTableSetName(STableCfg *config, char *name, bool dup) { if (dup) { config->name = strdup(name); - if (config->name == NULL) return -1; + if (config->name == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } } else { config->name = name; } @@ -879,28 +852,86 @@ static int tsdbTableSetName(STableCfg *config, char *name, bool dup) { return 0; } +static int tsdbTableSetTagSchema(STableCfg *config, STSchema *pSchema, bool dup) { + if (config->type != TSDB_CHILD_TABLE) { + terrno = TSDB_CODE_TDB_INVALID_CREATE_TB_MSG; + return -1; + } + + if (dup) { + config->tagSchema = tdDupSchema(pSchema); + if (config->tagSchema == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + } else { + config->tagSchema = pSchema; + } + return 0; +} + static int tsdbTableSetSName(STableCfg *config, char *sname, bool dup) { - if (config->type != TSDB_CHILD_TABLE) return -1; + if (config->type != TSDB_CHILD_TABLE) { + terrno = TSDB_CODE_TDB_INVALID_CREATE_TB_MSG; + return -1; + } if (dup) { config->sname = strdup(sname); - if (config->sname == NULL) return -1; + if (config->sname == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } } else { config->sname = sname; } return 0; } +static int tsdbTableSetSuperUid(STableCfg *config, uint64_t uid) { + if (config->type != TSDB_CHILD_TABLE || uid == TSDB_INVALID_SUPER_TABLE_ID) { + terrno = TSDB_CODE_TDB_INVALID_CREATE_TB_MSG; + return -1; + } + + config->superUid = uid; + return 0; +} + +static int tsdbTableSetTagValue(STableCfg *config, SKVRow row, bool dup) { + if (config->type != TSDB_CHILD_TABLE) { + terrno = TSDB_CODE_TDB_INVALID_CREATE_TB_MSG; + return -1; + } + + if (dup) { + config->tagValues = tdKVRowDup(row); + if (config->tagValues == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + } else { + config->tagValues = row; + } + + return 0; +} + static int tsdbTableSetStreamSql(STableCfg *config, char *sql, bool dup) { - if (config->type != TSDB_STREAM_TABLE) return -1; - + if (config->type != TSDB_STREAM_TABLE) { + terrno = TSDB_CODE_TDB_INVALID_CREATE_TB_MSG; + return -1; + } + if (dup) { config->sql = strdup(sql); - if (config->sql == NULL) return -1; + if (config->sql == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } } else { config->sql = sql; } return 0; -} -#endif \ No newline at end of file +} \ No newline at end of file From b7b4148b6743d37486bcf0c87f83ebd5d6559c52 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 14 Jun 2020 06:00:24 +0000 Subject: [PATCH 13/65] TD-353 --- src/tsdb/inc/tsdbMain.h | 8 +- src/tsdb/src/tsdbMeta.c | 276 +++++++++++++++++++++++----------------- 2 files changed, 163 insertions(+), 121 deletions(-) diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 25e8b9ce40..8d66c6325a 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -48,6 +48,7 @@ typedef struct STable { ETableType type; tstr* name; // NOTE: there a flexible string here STableId tableId; + uint64_t suid; STable* pSuper; // super table pointer uint8_t numOfSchemas; STSchema schema[TSDB_MAX_TABLE_SCHEMAS]; @@ -59,6 +60,7 @@ typedef struct STable { TSKEY lastKey; // lastkey inserted in this table, initialized as 0, TODO: make a structure char* sql; void* cqhandle; + T_REF_DECLARE(); } STable; typedef struct { @@ -69,8 +71,6 @@ typedef struct { SList* superList; SHashObj* uidMap; SKVStore* pStore; - int maxRowBytes; - int maxCols; } STsdbMeta; // ------------------ tsdbBuffer.c @@ -107,6 +107,8 @@ typedef struct { STableData** tData; SList* actList; SList* bufBlockList; + int maxCols; + int maxRowBytes; } SMemTable; // ------------------ tsdbFile.c @@ -278,7 +280,7 @@ typedef struct { #define TABLE_CHAR_NAME(t) TABLE_NAME(t)->data #define TALBE_UID(t) (t)->tableId.uid #define TABLE_TID(t) (t)->tableId.tid -#define TABLE_SUID(t) (t)->superUid +#define TABLE_SUID(t) (t)->suid #define TABLE_LASTKEY(t) (t)->lastKey STsdbMeta* tsdbNewMeta(STsdbCfg* pCfg); diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 57ab738444..47f803f915 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -20,69 +20,64 @@ #include "tsdbMain.h" #include "tskiplist.h" +#define TSDB_SUPER_TABLE_SL_LEVEL 5 + // ------------------ OUTER FUNCTIONS ------------------ int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg) { STsdbRepo *pRepo = (STsdbRepo *)repo; STsdbMeta *pMeta = pRepo->tsdbMeta; - - if (tsdbCheckTableCfg(pCfg) < 0) return -1; + STable * super = NULL; + STable * table = NULL; + int newSuper = 0; STable *pTable = tsdbGetTableByUid(pMeta, pCfg->tableId.uid); if (pTable != NULL) { - tsdbError("vgId:%d table %s already exists, tid %d uid %" PRId64, pRepo->config.tsdbId, varDataVal(pTable->name), - pTable->tableId.tid, pTable->tableId.uid); + tsdbError("vgId:%d table %s already exists, tid %d uid %" PRId64, REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), + TABLE_TID(pTable), TALBE_UID(pTable)); return TSDB_CODE_TDB_TABLE_ALREADY_EXIST; } - STable *super = NULL; - int newSuper = 0; - if (pCfg->type == TSDB_CHILD_TABLE) { super = tsdbGetTableByUid(pMeta, pCfg->superUid); if (super == NULL) { // super table not exists, try to create it newSuper = 1; super = tsdbNewTable(pCfg, true); - if (super == NULL) return -1; + if (super == NULL) goto _err; } else { + // TODO if (super->type != TSDB_SUPER_TABLE) return -1; if (super->tableId.uid != pCfg->superUid) return -1; tsdbUpdateTable(pMeta, super, pCfg); } } - STable *table = tsdbNewTable(pCfg, false); - if (table == NULL) { - if (newSuper) { - tsdbFreeTable(super); - return -1; - } - } - - table->lastKey = TSKEY_INITIAL_VAL; - + table = tsdbNewTable(pCfg, false); + if (table == NULL) goto _err; + // Register to meta if (newSuper) { - tsdbAddTableToMeta(pMeta, super, true); - tsdbTrace("vgId:%d, super table %s is created! uid:%" PRId64, pRepo->config.tsdbId, varDataVal(super->name), - super->tableId.uid); + if (tsdbAddTableToMeta(pRepo, super, true) < 0) goto _err; } - tsdbAddTableToMeta(pMeta, table, true); - tsdbTrace("vgId:%d, table %s is created! tid:%d, uid:%" PRId64, pRepo->config.tsdbId, varDataVal(table->name), - table->tableId.tid, table->tableId.uid); + if (tsdbAddTableToMeta(pRepo, table, true) < 0) goto _err; - // Write to meta file - int bufLen = 0; - char *buf = malloc(1024*1024); - if (newSuper) { - tsdbEncodeTable(super, buf, &bufLen); - tsdbInsertMetaRecord(pMeta->mfh, super->tableId.uid, buf, bufLen); - } + // // Write to meta file + // int bufLen = 0; + // char *buf = malloc(1024 * 1024); + // if (newSuper) { + // tsdbEncodeTable(super, buf, &bufLen); + // tsdbInsertMetaRecord(pMeta->mfh, super->tableId.uid, buf, bufLen); + // } - tsdbEncodeTable(table, buf, &bufLen); - tsdbInsertMetaRecord(pMeta->mfh, table->tableId.uid, buf, bufLen); - tfree(buf); + // tsdbEncodeTable(table, buf, &bufLen); + // tsdbInsertMetaRecord(pMeta->mfh, table->tableId.uid, buf, bufLen); + // tfree(buf); return 0; + +_err: + tsdbFreeTable(super); + tsdbFreeTable(table); + return -1; } int tsdbDropTable(TSDB_REPO_T *repo, STableId tableId) { @@ -412,6 +407,40 @@ char *getTSTupleKey(const void * data) { return POINTER_SHIFT(row, TD_DATA_ROW_HEAD_SIZE); } +int tsdbWLockRepoMeta(STsdbRepo *pRepo) { + int code = pthread_rwlock_wrlock(&(pRepo->tsdbMeta->rwLock)); + if (code != 0) { + tsdbError("vgId:%d failed to write lock TSDB meta since %s", REPO_ID(pRepo), strerror(code)); + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + + return 0; +} + +int tsdbRLockRepoMeta(STsdbRepo *pRepo) { + int code = pthread_rwlock_rdlock(&(pRepo->tsdbMeta->rwLock)); + if (code != 0) { + tsdbError("vgId:%d failed to read lock TSDB meta since %s", REPO_ID(pRepo), strerror(code)); + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + + return 0; +} + +int tsdbUnlockRepoMeta(STsdbRepo *pRepo) { + int code = pthread_rwlock_unlock(&(pRepo->tsdbMeta->rwLock)); + if (code != 0) { + tsdbError("vgId:%d failed to unlock TSDB meta since %s", REPO_ID(pRepo), strerror(code)); + terrno = TAOS_SYSTEM_ERROR(code); + return -1; + } + + return 0; +} + + // ------------------ LOCAL FUNCTIONS ------------------ static void tsdbEncodeTable(STable *pTable, char *buf, int *contLen) { @@ -568,19 +597,8 @@ static STable *tsdbNewTable(STableCfg *pCfg, bool isSuper) { goto _err; } - pTable->type = pCfg->type; - pTable->numOfSchemas = 0; - if (isSuper) { pTable->type = TSDB_SUPER_TABLE; - pTable->tableId.uid = pCfg->superUid; - pTable->tableId.tid = -1; - pTable->superUid = TSDB_INVALID_SUPER_TABLE_ID; - pTable->schema = (STSchema **)malloc(sizeof(STSchema *) * TSDB_MAX_TABLE_SCHEMAS); - pTable->numOfSchemas = 1; - pTable->schema[0] = tdDupSchema(pCfg->schema); - pTable->tagSchema = tdDupSchema(pCfg->tagSchema); - tsize = strnlen(pCfg->sname, TSDB_TABLE_NAME_LEN - 1); pTable->name = calloc(1, tsize + VARSTR_HEADER_SIZE + 1); if (pTable->name == NULL) { @@ -588,20 +606,31 @@ static STable *tsdbNewTable(STableCfg *pCfg, bool isSuper) { goto _err; } STR_WITH_SIZE_TO_VARSTR(pTable->name, pCfg->sname, tsize); - + TALBE_UID(pTable) = pCfg->superUid; + TABLE_TID(pTable) = -1; + TABLE_SUID(pTable) = -1; + pTable->pSuper = NULL; + pTable->numOfSchemas = 1; + pTable->schema[0] = tdDupSchema(pCfg->schema); + if (pTable->schema[0] == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + goto _err; + } + pTable->tagSchema = tdDupSchema(pCfg->tagSchema); + if (pTable->tagSchema == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + goto _err; + } + pTable->tagVal = NULL; STColumn *pColSchema = schemaColAt(pTable->tagSchema, 0); - pTable->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, pColSchema->type, pColSchema->bytes, 1, 0, 0, - getTagIndexKey); // Allow duplicate key, no lock + pTable->pIndex = + tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, pColSchema->type, pColSchema->bytes, 1, 0, 1, getTagIndexKey); if (pTable->pIndex == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _err; } } else { pTable->type = pCfg->type; - pTable->tableId.uid = pCfg->tableId.uid; - pTable->tableId.tid = pCfg->tableId.tid; - pTable->lastKey = TSKEY_INITIAL_VAL; - tsize = strnlen(pCfg->name, TSDB_TABLE_NAME_LEN - 1); pTable->name = calloc(1, tsize + VARSTR_HEADER_SIZE + 1); if (pTable->name == NULL) { @@ -609,25 +638,39 @@ static STable *tsdbNewTable(STableCfg *pCfg, bool isSuper) { goto _err; } STR_WITH_SIZE_TO_VARSTR(pTable->name, pCfg->name, tsize); + TALBE_UID(pTable) = pCfg->tableId.uid; + TABLE_TID(pTable) = pCfg->tableId.tid; if (pCfg->type == TSDB_CHILD_TABLE) { - pTable->superUid = pCfg->superUid; + TABLE_SUID(pTable) = pCfg->superUid; pTable->tagVal = tdKVRowDup(pCfg->tagValues); + if (pTable->tagVal == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + goto _err; + } } else { - pTable->schema = (STSchema **)malloc(sizeof(STSchema *) * TSDB_MAX_TABLE_SCHEMAS); + TABLE_SUID(pTable) = -1; pTable->numOfSchemas = 1; pTable->schema[0] = tdDupSchema(pCfg->schema); + if (pTable->schema[0] == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + goto _err; + } - if (pCfg->type == TSDB_NORMAL_TABLE) { - pTable->superUid = -1; - } else { - ASSERT(pCfg->type == TSDB_STREAM_TABLE); - pTable->superUid = -1; + if (TABLE_TYPE(pTable) == TSDB_STREAM_TABLE) { pTable->sql = strdup(pCfg->sql); + if (pTable->sql == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + goto _err; + } } } + + pTable->lastKey = TSKEY_INITIAL_VAL; } + T_REF_INC(pTable); + return pTable; _err: @@ -635,34 +678,24 @@ _err: return NULL; } -static int tsdbFreeTable(STable *pTable) { - if (pTable == NULL) return 0; +static void tsdbFreeTable(STable *pTable) { + if (pTable) { + tfree(TABLE_NAME(pTable)); + if (TABLE_TYPE(pTable) != TSDB_CHILD_TABLE) { + for (int i = 0; i < TSDB_MAX_TABLE_SCHEMAS; i++) { + tdFreeSchema(pTable->schema[i]); + } - if (pTable->type == TSDB_CHILD_TABLE) { - kvRowFree(pTable->tagVal); - } else { - if (pTable->schema) { - for (int i = 0; i < pTable->numOfSchemas; i++) tdFreeSchema(pTable->schema[i]); - free(pTable->schema); + if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) { + tdFreeSchema(pTable->tagSchema); + } } - } - if (pTable->type == TSDB_STREAM_TABLE) { + kvRowFree(pTable->tagVal); + + tSkipListDestroy(pTable->pIndex); tfree(pTable->sql); } - - // Free content - if (TSDB_TABLE_IS_SUPER_TABLE(pTable)) { - tdFreeSchema(pTable->tagSchema); - tSkipListDestroy(pTable->pIndex); - } - - tsdbFreeMemTable(pTable->mem); - tsdbFreeMemTable(pTable->imem); - - tfree(pTable->name); - free(pTable); - return 0; } static int tsdbUpdateTableTagSchema(STable *pTable, STSchema *newSchema) { @@ -677,48 +710,55 @@ static int tsdbUpdateTableTagSchema(STable *pTable, STSchema *newSchema) { return TSDB_CODE_SUCCESS; } -static int tsdbAddTableToMeta(STsdbMeta *pMeta, STable *pTable, bool addIdx) { - STsdbRepo *pRepo = (STsdbRepo *)pMeta->pRepo; - if (pTable->type == TSDB_SUPER_TABLE) { - // add super table to the linked list - if (pMeta->superList == NULL) { - pMeta->superList = pTable; - pTable->next = NULL; - pTable->prev = NULL; - } else { - pTable->next = pMeta->superList; - pTable->prev = NULL; - pTable->next->prev = pTable; - pMeta->superList = pTable; - } - } else { - // add non-super table to the array - pMeta->tables[pTable->tableId.tid] = pTable; - if (pTable->type == TSDB_CHILD_TABLE && addIdx) { // add STABLE to the index - tsdbAddTableIntoIndex(pMeta, pTable); - } - if (pTable->type == TSDB_STREAM_TABLE && addIdx) { - pTable->cqhandle = (*pRepo->appH.cqCreateFunc)(pRepo->appH.cqH, pTable->tableId.uid, pTable->tableId.tid, pTable->sql, tsdbGetTableSchema(pMeta, pTable)); - } - - pMeta->nTables++; - } +static int tsdbAddTableToMeta(STsdbRepo *pRepo, STable *pTable, bool addIdx) { + STsdbMeta *pMeta = pRepo->tsdbMeta; - // Update the pMeta->maxCols and pMeta->maxRowBytes - if (pTable->type == TSDB_SUPER_TABLE || pTable->type == TSDB_NORMAL_TABLE || pTable->type == TSDB_STREAM_TABLE) { - if (schemaNCols(pTable->schema[pTable->numOfSchemas - 1]) > pMeta->maxCols) - pMeta->maxCols = schemaNCols(pTable->schema[pTable->numOfSchemas - 1]); - int bytes = dataRowMaxBytesFromSchema(pTable->schema[pTable->numOfSchemas - 1]); - if (bytes > pMeta->maxRowBytes) pMeta->maxRowBytes = bytes; - } - - if (taosHashPut(pMeta->map, (char *)(&pTable->tableId.uid), sizeof(pTable->tableId.uid), (void *)(&pTable), sizeof(pTable)) < 0) { + if (addIdx && tsdbWLockRepoMeta(pRepo) < 0) { + tsdbError("vgId:%d failed to add table %s to meta since %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), + tstrerror(terrno)); return -1; } + + if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) { + if (tdListAppend(pMeta->superList, (void *)(&pTable)) < 0) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tsdbError("vgId:%d failed to add table %s to meta since %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), + tstrerror(terrno)); + goto _err; + } + } else { + pMeta->tables[TABLE_TID(pTable)] = pTable; + if (TABLE_TID(pTable) == TSDB_CHILD_TABLE && addIdx) { // add STABLE to the index + if (tsdbAddTableIntoIndex(pMeta, pTable) < 0) { + tsdbTrace("vgId:%d failed to add table %s to meta while add table to index since %s", REPO_ID(pRepo), + TABLE_CHAR_NAME(pTable), tstrerror(terrno)); + goto _err; + } + } + + pMeta->nTables++; + + return 0; + } + + if (taosHashPut(pMeta->map, (char *)(&pTable->tableId.uid), sizeof(pTable->tableId.uid), (void *)(&pTable), + sizeof(pTable)) < 0) { + return -1; + } + + if (addIdx && tsdbUnlockRepoMeta(pRepo) < 0) return -1; + + tsdbTrace("vgId:%d table %s tid %d uid %" PRIu64 " is added to meta", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), + TABLE_TID(pTable), TALBE_UID(pTable)); return 0; + +_err: + tsdbRemoveTableFromIndex() + if (addIdx) tsdbUnlockRepoMeta(pRepo); + return -1; } -static int tsdbRemoveTableFromMeta(STsdbMeta *pMeta, STable *pTable, bool rmFromIdx) { +static int tsdbRemoveTableFromMeta(STsdbRepo *pRepo, STable *pTable, bool rmFromIdx) { if (pTable->type == TSDB_SUPER_TABLE) { SSkipListIterator *pIter = tSkipListCreateIter(pTable->pIndex); while (tSkipListIterNext(pIter)) { From 472b194a1ecada0f86290fb1c74ecccc41ff3125 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 14 Jun 2020 08:51:58 +0000 Subject: [PATCH 14/65] TD-353 --- src/common/inc/tdataformat.h | 8 +- src/common/src/tdataformat.c | 38 ++-- src/inc/taosdef.h | 2 +- src/tsdb/inc/tsdbMain.h | 1 - src/tsdb/src/tsdbMeta.c | 341 +++++++++++++++++------------------ 5 files changed, 191 insertions(+), 199 deletions(-) diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index 47753b35a7..ca9fd3ee33 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -69,8 +69,8 @@ typedef struct { int version; // version int numOfCols; // Number of columns appended int tlen; // maximum length of a SDataRow without the header part - uint16_t flen; // First part length in a SDataRow after the header part - uint16_t vlen; // pure value part length, excluded the overhead + uint16_t flen; // First part length in a SDataRow after the header part + uint16_t vlen; // pure value part length, excluded the overhead STColumn columns[]; } STSchema; @@ -83,8 +83,8 @@ typedef struct { #define tdFreeSchema(s) tfree((s)) STSchema *tdDupSchema(STSchema *pSchema); -void * tdEncodeSchema(void *dst, STSchema *pSchema); -STSchema *tdDecodeSchema(void **psrc); +void * tdEncodeSchema(void *buf, STSchema *pSchema); +void * tdDecodeSchema(void *buf, STSchema **pRSchema); static FORCE_INLINE int comparColId(const void *key1, const void *key2) { if (*(int16_t *)key1 > ((STColumn *)key2)->colId) { diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index 77e91acc14..d8b67e1105 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -14,6 +14,7 @@ */ #include "tdataformat.h" #include "talgo.h" +#include "tcoding.h" #include "wchar.h" /** @@ -33,50 +34,48 @@ STSchema *tdDupSchema(STSchema *pSchema) { /** * Encode a schema to dst, and return the next pointer */ -void *tdEncodeSchema(void *dst, STSchema *pSchema) { +void *tdEncodeSchema(void *buf, STSchema *pSchema) { + buf = taosEncodeFixedI32(buf, schemaVersion(pSchema)); + buf = taosEncodeFixedI32(buf, schemaNCols(pSchema)); - T_APPEND_MEMBER(dst, pSchema, STSchema, version); - T_APPEND_MEMBER(dst, pSchema, STSchema, numOfCols); for (int i = 0; i < schemaNCols(pSchema); i++) { STColumn *pCol = schemaColAt(pSchema, i); - T_APPEND_MEMBER(dst, pCol, STColumn, type); - T_APPEND_MEMBER(dst, pCol, STColumn, colId); - T_APPEND_MEMBER(dst, pCol, STColumn, bytes); + buf = taosEncodeFixedI8(buf, colType(pCol)); + buf = taosEncodeFixedI16(buf, colColId(pCol)); + buf = taosEncodeFixedI32(buf, colBytes(pCol)) : } - return dst; + return buf; } /** * Decode a schema from a binary. */ -STSchema *tdDecodeSchema(void **psrc) { - int totalCols = 0; +void *tdDecodeSchema(void *buf, STSchema **pRSchema) { int version = 0; - STSchemaBuilder schemaBuilder = {0}; + int numOfCols = 0; - T_READ_MEMBER(*psrc, int, version); - T_READ_MEMBER(*psrc, int, totalCols); + buf = taosDecodeFixedI32(buf, &version); + buf = taosDecodeFixedI32(buf, &numOfCols); if (tdInitTSchemaBuilder(&schemaBuilder, version) < 0) return NULL; - for (int i = 0; i < totalCols; i++) { + for (int i = 0; i < numOfCols; i++) { int8_t type = 0; int16_t colId = 0; int32_t bytes = 0; - T_READ_MEMBER(*psrc, int8_t, type); - T_READ_MEMBER(*psrc, int16_t, colId); - T_READ_MEMBER(*psrc, int32_t, bytes); - + buf = taosDecodeFixedI8(buf, &type); + buf = taosDecodeFixedI16(buf, &colId); + buf = taosDecodeFixedI32(buf, &bytes); if (tdAddColToSchema(&schemaBuilder, type, colId, bytes) < 0) { tdDestroyTSchemaBuilder(&schemaBuilder); return NULL; } } - STSchema *pSchema = tdGetSchemaFromBuilder(&schemaBuilder); + *pRSchema = tdGetSchemaFromBuilder(&schemaBuilder); tdDestroyTSchemaBuilder(&schemaBuilder); - return pSchema; + return buf; } int tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version) { @@ -613,6 +612,7 @@ void *tdEncodeKVRow(void *buf, SKVRow row) { void *tdDecodeKVRow(void *buf, SKVRow *row) { *row = tdKVRowDup(buf); + if (*row == NULL) return NULL; return POINTER_SHIFT(buf, kvRowLen(*row)); } diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index 7490de90d0..f84a7581e4 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -362,7 +362,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TAOS_QTYPE_WAL 2 #define TAOS_QTYPE_CQ 3 -typedef enum { +typedef enum : uint8_t{ TSDB_SUPER_TABLE = 0, // super table TSDB_CHILD_TABLE = 1, // table created from super table TSDB_NORMAL_TABLE = 2, // ordinary table diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 8d66c6325a..5f6d72128d 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -391,7 +391,6 @@ int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable); STSchema *tsdbGetTableSchemaByVersion(STsdbMeta *pMeta, STable *pTable, int16_t version); STSchema *tsdbGetTableSchema(STsdbMeta *pMeta, STable *pTable); -#define DEFAULT_TAG_INDEX_COLUMN 0 // skip list built based on the first column of tags int compFGroupKey(const void *key, const void *fgroup); #endif diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 47f803f915..3b997ddbbe 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -21,6 +21,7 @@ #include "tskiplist.h" #define TSDB_SUPER_TABLE_SL_LEVEL 5 +#define DEFAULT_TAG_INDEX_COLUMN 0 // ------------------ OUTER FUNCTIONS ------------------ int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg) { @@ -316,13 +317,13 @@ int tsdbCloseMeta(STsdbRepo *pRepo) { return 0; } -STSchema *tsdbGetTableSchema(STsdbMeta *pMeta, STable *pTable) { +STSchema *tsdbGetTableSchema(STable *pTable) { if (pTable->type == TSDB_NORMAL_TABLE || pTable->type == TSDB_SUPER_TABLE || pTable->type == TSDB_STREAM_TABLE) { return pTable->schema[pTable->numOfSchemas - 1]; } else if (pTable->type == TSDB_CHILD_TABLE) { - STable *pSuper = tsdbGetTableByUid(pMeta, pTable->superUid); + STable *pSuper = pTable->pSuper; if (pSuper == NULL) return NULL; - return pSuper->schema[pSuper->numOfSchemas-1]; + return pSuper->schema[pSuper->numOfSchemas - 1]; } else { return NULL; } @@ -336,14 +337,9 @@ STable *tsdbGetTableByUid(STsdbMeta *pMeta, uint64_t uid) { return *(STable **)ptr; } -STSchema *tsdbGetTableSchemaByVersion(STsdbMeta *pMeta, STable *pTable, int16_t version) { - STable *pSearchTable = NULL; - if (pTable->type == TSDB_CHILD_TABLE) { - pSearchTable = tsdbGetTableByUid(pMeta, pTable->superUid); - } else { - pSearchTable = pTable; - } - ASSERT(pSearchTable != NULL); +STSchema *tsdbGetTableSchemaByVersion(STable *pTable, int16_t version) { + STable *pSearchTable = (pTable->type == TSDB_CHILD_TABLE) ? pTable->pSuper : pTable; + if (pSearchTable == NULL) return NULL; void *ptr = taosbsearch(&version, pSearchTable->schema, pSearchTable->numOfSchemas, sizeof(STSchema *), tsdbCompareSchemaVersion, TD_EQ); @@ -352,11 +348,11 @@ STSchema *tsdbGetTableSchemaByVersion(STsdbMeta *pMeta, STable *pTable, int16_t return *(STSchema **)ptr; } -STSchema * tsdbGetTableTagSchema(STsdbMeta *pMeta, STable *pTable) { +STSchema *tsdbGetTableTagSchema(STable *pTable) { if (pTable->type == TSDB_SUPER_TABLE) { return pTable->tagSchema; } else if (pTable->type == TSDB_CHILD_TABLE) { - STable *pSuper = tsdbGetTableByUid(pMeta, pTable->superUid); + STable *pSuper = pTable->pSuper; if (pSuper == NULL) return NULL; return pSuper->tagSchema; } else { @@ -440,107 +436,11 @@ int tsdbUnlockRepoMeta(STsdbRepo *pRepo) { return 0; } - - // ------------------ LOCAL FUNCTIONS ------------------ -static void tsdbEncodeTable(STable *pTable, char *buf, int *contLen) { - if (pTable == NULL) return; - - void *ptr = buf; - T_APPEND_MEMBER(ptr, pTable, STable, type); - // Encode name, todo refactor - *(int *)ptr = varDataLen(pTable->name); - ptr = (char *)ptr + sizeof(int); - memcpy(ptr, varDataVal(pTable->name), varDataLen(pTable->name)); - ptr = (char *)ptr + varDataLen(pTable->name); - - T_APPEND_MEMBER(ptr, &(pTable->tableId), STableId, uid); - T_APPEND_MEMBER(ptr, &(pTable->tableId), STableId, tid); - T_APPEND_MEMBER(ptr, pTable, STable, superUid); - - if (pTable->type == TSDB_SUPER_TABLE) { - T_APPEND_MEMBER(ptr, pTable, STable, numOfSchemas); - for (int i = 0; i < pTable->numOfSchemas; i++) { - ptr = tdEncodeSchema(ptr, pTable->schema[i]); - } - ptr = tdEncodeSchema(ptr, pTable->tagSchema); - } else if (pTable->type == TSDB_CHILD_TABLE) { - ptr = tdEncodeKVRow(ptr, pTable->tagVal); - } else { - T_APPEND_MEMBER(ptr, pTable, STable, numOfSchemas); - for (int i = 0; i < pTable->numOfSchemas; i++) { - ptr = tdEncodeSchema(ptr, pTable->schema[i]); - } - } - - if (pTable->type == TSDB_STREAM_TABLE) { - ptr = taosEncodeString(ptr, pTable->sql); - } - - *contLen = (char *)ptr - buf; -} - -static STable *tsdbDecodeTable(void *cont, int contLen) { - // TODO - STable *pTable = (STable *)calloc(1, sizeof(STable)); - if (pTable == NULL) return NULL; - - void *ptr = cont; - T_READ_MEMBER(ptr, int8_t, pTable->type); - if (pTable->type != TSDB_CHILD_TABLE) { - pTable->schema = (STSchema **)malloc(sizeof(STSchema *) * TSDB_MAX_TABLE_SCHEMAS); - if (pTable->schema == NULL) { - free(pTable); - return NULL; - } - } - int len = *(int *)ptr; - ptr = (char *)ptr + sizeof(int); - pTable->name = calloc(1, len + VARSTR_HEADER_SIZE + 1); - if (pTable->name == NULL) return NULL; - - varDataSetLen(pTable->name, len); - memcpy(pTable->name->data, ptr, len); - - ptr = (char *)ptr + len; - T_READ_MEMBER(ptr, uint64_t, pTable->tableId.uid); - T_READ_MEMBER(ptr, int32_t, pTable->tableId.tid); - T_READ_MEMBER(ptr, uint64_t, pTable->superUid); - - if (pTable->type == TSDB_SUPER_TABLE) { - T_READ_MEMBER(ptr, int16_t, pTable->numOfSchemas); - for (int i = 0; i < pTable->numOfSchemas; i++) { - pTable->schema[i] = tdDecodeSchema(&ptr); - } - pTable->tagSchema = tdDecodeSchema(&ptr); - } else if (pTable->type == TSDB_CHILD_TABLE) { - ptr = tdDecodeKVRow(ptr, &pTable->tagVal); - } else { - T_READ_MEMBER(ptr, int16_t, pTable->numOfSchemas); - for (int i = 0; i < pTable->numOfSchemas; i++) { - pTable->schema[i] = tdDecodeSchema(&ptr); - } - } - - if (pTable->type == TSDB_STREAM_TABLE) { - ptr = taosDecodeString(ptr, &(pTable->sql)); - } - - pTable->lastKey = TSKEY_INITIAL_VAL; - - if (pTable->type == TSDB_SUPER_TABLE) { - STColumn *pColSchema = schemaColAt(pTable->tagSchema, 0); - pTable->pIndex = - tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, pColSchema->type, pColSchema->bytes, 1, 0, 1, getTagIndexKey); - } - - return pTable; -} - static int tsdbCompareSchemaVersion(const void *key1, const void *key2) { - if (*(int16_t *)key1 < (*(STSchema **)key2)->version) { + if (*(int16_t *)key1 < schemaVersion(*(STSchema **)key2)) { return -1; - } else if (*(int16_t *)key1 > (*(STSchema **)key2)->version) { + } else if (*(int16_t *)key1 > schemaVersion(*(STSchema **)key2)) { return 1; } else { return 0; @@ -550,16 +450,20 @@ static int tsdbCompareSchemaVersion(const void *key1, const void *key2) { static int tsdbRestoreTable(void *pHandle, void *cont, int contLen) { STsdbRepo *pRepo = (STsdbRepo *)pHandle; STsdbMeta *pMeta = pRepo->tsdbMeta; + STable * pTable = NULL; if (!taosCheckChecksumWhole((uint8_t *)cont, contLen)) { terrno = TSDB_CODE_TDB_FILE_CORRUPTED; return -1; } - STable *pTable = tsdbDecodeTable(cont, contLen); + pTable = tsdbDecodeTable(cont, contLen); if (pTable == NULL) return -1; - if (tsdbAddTableToMeta(pMeta, pTable, false) < 0) return -1; + if (tsdbAddTableToMeta(pMeta, pTable, false) < 0) { + tsdbFreeTable(pTable); + return -1; + } tsdbTrace("vgId:%d table %s tid %d uid %" PRIu64 " is restored from file", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TALBE_UID(pTable)); @@ -579,11 +483,11 @@ static void tsdbOrgMeta(void *pHandle) { } static char *getTagIndexKey(const void *pData) { - STableIndexElem *elem = (STableIndexElem *)pData; + STable *pTable = *(STable **)pData; - STSchema *pSchema = tsdbGetTableTagSchema(elem->pMeta, elem->pTable); - STColumn *pCol = &pSchema->columns[DEFAULT_TAG_INDEX_COLUMN]; - void * res = tdGetKVRowValOfCol(elem->pTable->tagVal, pCol->colId); + STSchema *pSchema = tsdbGetTableTagSchema(pTable); + STColumn *pCol = schemaColAt(DEFAULT_TAG_INDEX_COLUMN); + void * res = tdGetKVRowValOfCol(pTable->tagVal, pCol->colId); return res; } @@ -703,11 +607,14 @@ static int tsdbUpdateTableTagSchema(STable *pTable, STSchema *newSchema) { ASSERT(schemaVersion(pTable->tagSchema) < schemaVersion(newSchema)); STSchema *pOldSchema = pTable->tagSchema; STSchema *pNewSchema = tdDupSchema(newSchema); - if (pNewSchema == NULL) return TSDB_CODE_TDB_OUT_OF_MEMORY; + if (pNewSchema == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } pTable->tagSchema = pNewSchema; tdFreeSchema(pOldSchema); - return TSDB_CODE_SUCCESS; + return 0; } static int tsdbAddTableToMeta(STsdbRepo *pRepo, STable *pTable, bool addIdx) { @@ -727,7 +634,6 @@ static int tsdbAddTableToMeta(STsdbRepo *pRepo, STable *pTable, bool addIdx) { goto _err; } } else { - pMeta->tables[TABLE_TID(pTable)] = pTable; if (TABLE_TID(pTable) == TSDB_CHILD_TABLE && addIdx) { // add STABLE to the index if (tsdbAddTableIntoIndex(pMeta, pTable) < 0) { tsdbTrace("vgId:%d failed to add table %s to meta while add table to index since %s", REPO_ID(pRepo), @@ -735,15 +641,15 @@ static int tsdbAddTableToMeta(STsdbRepo *pRepo, STable *pTable, bool addIdx) { goto _err; } } - + pMeta->tables[TABLE_TID(pTable)] = pTable; pMeta->nTables++; - - return 0; } - if (taosHashPut(pMeta->map, (char *)(&pTable->tableId.uid), sizeof(pTable->tableId.uid), (void *)(&pTable), + if (taosHashPut(pMeta->uidMap, (char *)(&pTable->tableId.uid), sizeof(pTable->tableId.uid), (void *)(&pTable), sizeof(pTable)) < 0) { - return -1; + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tsdbError("vgId:%d failed to add table %s to meta while put into uid map since %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), tstrerror(terrno)); + goto _err; } if (addIdx && tsdbUnlockRepoMeta(pRepo) < 0) return -1; @@ -753,99 +659,96 @@ static int tsdbAddTableToMeta(STsdbRepo *pRepo, STable *pTable, bool addIdx) { return 0; _err: - tsdbRemoveTableFromIndex() + tsdbRemoveTableFromMeta(pRepo, pTable, false); if (addIdx) tsdbUnlockRepoMeta(pRepo); return -1; } -static int tsdbRemoveTableFromMeta(STsdbRepo *pRepo, STable *pTable, bool rmFromIdx) { - if (pTable->type == TSDB_SUPER_TABLE) { - SSkipListIterator *pIter = tSkipListCreateIter(pTable->pIndex); - while (tSkipListIterNext(pIter)) { - STableIndexElem *pEle = (STableIndexElem *)SL_GET_NODE_DATA(tSkipListIterGet(pIter)); - STable *tTable = pEle->pTable; +static void tsdbRemoveTableFromMeta(STsdbRepo *pRepo, STable *pTable, bool rmFromIdx) { + STsdbMeta *pMeta = pRepo->tsdbMeta; + SListIter lIter = {0}; + SListNode *pNode = NULL; + STable * tTable = NULL; - ASSERT(tTable != NULL && tTable->type == TSDB_CHILD_TABLE); + if (rmFromIdx) tsdbWLockRepoMeta(pRepo); - tsdbRemoveTableFromMeta(pMeta, tTable, false); - } + if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) { + tdListInitIter(pMeta->superList, &lIter, TD_LIST_BACKWARD); - tSkipListDestroyIter(pIter); - - if (pTable->prev != NULL) { - pTable->prev->next = pTable->next; - if (pTable->next != NULL) { - pTable->next->prev = pTable->prev; + while ((pNode = tdListNext(&lIter)) != NULL) { + tdListNodeGetData(pMeta->superList, pNode, (void *)(tTable)); + if (pTable == tTable) { + break; + tdListPopNode(pMeta->superList, pNode); + free(pNode); } - } else { - pMeta->superList = pTable->next; } } else { pMeta->tables[pTable->tableId.tid] = NULL; - if (pTable->type == TSDB_CHILD_TABLE && rmFromIdx) { + if (TABLE_TYPE(pTable) == TSDB_CHILD_TABLE && rmFromIdx) { tsdbRemoveTableFromIndex(pMeta, pTable); } - if (pTable->type == TSDB_STREAM_TABLE && rmFromIdx) { - // TODO - } pMeta->nTables--; } - taosHashRemove(pMeta->map, (char *)(&(pTable->tableId.uid)), sizeof(pTable->tableId.uid)); - tsdbFreeTable(pTable); - return 0; + taosHashRemove(pMeta->uid, (char *)(&(TALBE_UID(pTable))), sizeof(TALBE_UID(pTable))); + + if (rmFromIdx) tsdbUnlockRepoMeta(pRepo); } static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable) { - assert(pTable->type == TSDB_CHILD_TABLE && pTable != NULL); - STable* pSTable = tsdbGetTableByUid(pMeta, pTable->superUid); - assert(pSTable != NULL); - + ASSERT(pTable->type == TSDB_CHILD_TABLE && pTable != NULL); + STable *pSTable = tsdbGetTableByUid(pMeta, TABLE_SUID(pTable)); + ASSERT(pSTable != NULL); + + pTable->pSuper = pSTable; + int32_t level = 0; int32_t headSize = 0; - + tSkipListNewNodeInfo(pSTable->pIndex, &level, &headSize); - + // NOTE: do not allocate the space for key, since in each skip list node, only keep the pointer to pTable, not the // actual key value, and the key value will be retrieved during query through the pTable and getTagIndexKey function - SSkipListNode* pNode = calloc(1, headSize + sizeof(STableIndexElem)); + SSkipListNode *pNode = calloc(1, headSize + sizeof(STable *)); + if (pNode == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } pNode->level = level; - - SSkipList* list = pSTable->pIndex; - STableIndexElem* elem = (STableIndexElem*) (SL_GET_NODE_DATA(pNode)); - - elem->pTable = pTable; - elem->pMeta = pMeta; - - tSkipListPut(list, pNode); + + SSkipList *list = pSTable->pIndex; + (STable *)(SL_GET_NODE_DATA(pNode)) = pTable; + + tSkipListPut(pSTable->pIndex, pNode); return 0; } static int tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable) { - assert(pTable->type == TSDB_CHILD_TABLE && pTable != NULL); + ASSERT(pTable->type == TSDB_CHILD_TABLE && pTable != NULL); STable* pSTable = tsdbGetTableByUid(pMeta, pTable->superUid); - assert(pSTable != NULL); - - STSchema* pSchema = tsdbGetTableTagSchema(pMeta, pTable); - STColumn* pCol = &pSchema->columns[DEFAULT_TAG_INDEX_COLUMN]; + ASSERT(pSTable != NULL); + + STSchema *pSchema = tsdbGetTableTagSchema(pTable); + STColumn *pCol = schemaColAt(pSchema, DEFAULT_TAG_INDEX_COLUMN); char* key = tdGetKVRowValOfCol(pTable->tagVal, pCol->colId); SArray* res = tSkipListGet(pSTable->pIndex, key); size_t size = taosArrayGetSize(res); - assert(size > 0); + ASSERT(size > 0); for(int32_t i = 0; i < size; ++i) { SSkipListNode* pNode = taosArrayGetP(res, i); - STableIndexElem* pElem = (STableIndexElem*) SL_GET_NODE_DATA(pNode); - if (pElem->pTable == pTable) { // this is the exact what we need + // STableIndexElem* pElem = (STableIndexElem*) SL_GET_NODE_DATA(pNode); + if ((STable *)SL_GET_NODE_DATA(pNode) == pTable) { // this is the exact what we need tSkipListRemoveNode(pSTable->pIndex, pNode); } } - + taosArrayDestroy(res); return 0; } @@ -974,4 +877,94 @@ static int tsdbTableSetStreamSql(STableCfg *config, char *sql, bool dup) { } return 0; +} + +static void *tsdbEncodeTableName(void *buf, tstr *name) { + void *pBuf = buf; + + pBuf = taosEncodeFixedI16(pBuf, name->len); + memcpy(pBuf, name->data, name->len); + pBuf = POINTER_SHIFT(pBuf, name->len); + + return POINTER_DISTANCE(pBuf, buf); +} + +static void *tsdbDecodeTableName(void *buf, tstr **name) { + VarDataLenT len = 0; + + buf = taosDecodeFixedI16(buf, &len); + *name = calloc(1, sizeof(tstr) + len + 1); + if (*name == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return NULL; + } + (*name)->len = len; + memcpy((*name)->data, buf, len); + + buf = POINTER_SHIFT(buf, len); + return buf; +} + +static void *tsdbEncodeTable(void *buf, STable *pTable) { + ASSERT(pTable != NULL); + + buf = taosEncodeFixedU8(buf, pTable->type); + buf = tsdbEncodeTableName(buf, pTable->name); + buf = taosEncodeFixedU64(buf, TALBE_UID(pTable)); + buf = taosEncodeFixedI32(buf, TABLE_TID(pTable)); + + if (TABLE_TYPE(pTable) == TSDB_CHILD_TABLE) { + buf = taosEncodeFixedU64(buf, TABLE_SUID(pTable)); + buf = tdEncodeKVRow(buf, pTable->tagVal); + } else { + buf = taosEncodeFixedU8(buf, pTable->numOfSchemas); + for (int i = 0; i < pTable->numOfSchemas; i++) { + buf = tdEncodeSchema(buf, pTable->schema[i]); + } + + if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) { + buf = tdEncodeSchema(buf, pTable->tagSchema); + } + + if (TABLE_TYPE(pTable) == TSDB_STREAM_TABLE) { + buf = taosEncodeString(buf, pTable->sql); + } + } + + return buf; +} + +static void *tsdbDecodeTable(void *buf, STable **pRTable) { + STable *pTable = (STable *)calloc(1, sizeof(STable)); + if (pTable == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return NULL; + } + + buf = taosDecodeFixedU8(buf, &(pTable->type)); + buf = tsdbDecodeTableName(buf, &(pTable->name)); + buf = taosDecodeFixedU64(buf, &TALBE_UID(pTable)); + buf = taosDecodeFixedI32(buf, &TABLE_TID(pTable)); + + if (TABLE_TYPE(pTable) == TSDB_CHILD_TABLE) { + buf = taosDecodeFixedU64(buf, &TABLE_SUID(pTable)); + buf = tdDecodeKVRow(buf, &(pTable->tagVal)); + } else { + buf = taosDecodeFixedU8(buf, &(pTable->numOfSchemas)); + for (int i = 0; i < pTable->numOfSchemas; i++) { + buf = tdDecodeSchema(buf, &(pTable->schema[i])); + } + + if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) { + buf = tdDecodeSchema(buf, &(pTable->tagSchema)); + } + + if (TABLE_TYPE(pTable) == TSDB_STREAM_TABLE) { + buf = taosDecodeString(buf, &(pTable->sql)); + } + } + + *pRTable = pTable; + + return buf; } \ No newline at end of file From ec4b6b085ead2e17b9e228ac734b07a9be04d937 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 14 Jun 2020 10:13:02 +0000 Subject: [PATCH 15/65] TD-353 --- src/tsdb/src/tsdbMeta.c | 60 +++++++++++++++++++++++------------------ 1 file changed, 34 insertions(+), 26 deletions(-) diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 3b997ddbbe..3d11383e5b 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -100,27 +100,26 @@ int tsdbDropTable(TSDB_REPO_T *repo, STableId tableId) { if (tsdbRemoveTableFromMeta(pMeta, pTable, true) < 0) return -1; return 0; - } -void* tsdbGetTableTagVal(TSDB_REPO_T* repo, const STableId* id, int32_t colId, int16_t type, int16_t bytes) { +void *tsdbGetTableTagVal(TSDB_REPO_T *repo, const STableId *id, int32_t colId, int16_t type, int16_t bytes) { // TODO: this function should be changed also - STsdbMeta* pMeta = tsdbGetMeta(repo); - STable* pTable = tsdbGetTableByUid(pMeta, id->uid); + STsdbMeta *pMeta = tsdbGetMeta(repo); + STable * pTable = tsdbGetTableByUid(pMeta, id->uid); STSchema *pSchema = tsdbGetTableTagSchema(pMeta, pTable); STColumn *pCol = tdGetColOfID(pSchema, colId); if (pCol == NULL) { return NULL; // No matched tag volumn } - - char* val = tdGetKVRowValOfCol(pTable->tagVal, colId); + + char *val = tdGetKVRowValOfCol(pTable->tagVal, colId); assert(type == pCol->type && bytes == pCol->bytes); - + if (val != NULL && IS_VAR_DATA_TYPE(type)) { assert(varDataLen(val) < pCol->bytes); } - + return val; } @@ -129,7 +128,7 @@ char *tsdbGetTableName(TSDB_REPO_T *repo, const STableId *id) { STsdbRepo *pRepo = (STsdbRepo *)repo; STsdbMeta *pMeta = pRepo->tsdbMeta; - STable * pTable = tsdbGetTableByUid(pMeta, id->uid); + STable *pTable = tsdbGetTableByUid(pMeta, id->uid); if (pTable == NULL) { return NULL; @@ -398,7 +397,7 @@ int tsdbUpdateTable(STsdbMeta *pMeta, STable *pTable, STableCfg *pCfg) { return TSDB_CODE_SUCCESS; } -char *getTSTupleKey(const void * data) { +char *getTSTupleKey(const void *data) { SDataRow row = (SDataRow)data; return POINTER_SHIFT(row, TD_DATA_ROW_HEAD_SIZE); } @@ -457,8 +456,7 @@ static int tsdbRestoreTable(void *pHandle, void *cont, int contLen) { return -1; } - pTable = tsdbDecodeTable(cont, contLen); - if (pTable == NULL) return -1; + tsdbDecodeTable(cont, &pTable); if (tsdbAddTableToMeta(pMeta, pTable, false) < 0) { tsdbFreeTable(pTable); @@ -526,9 +524,8 @@ static STable *tsdbNewTable(STableCfg *pCfg, bool isSuper) { goto _err; } pTable->tagVal = NULL; - STColumn *pColSchema = schemaColAt(pTable->tagSchema, 0); - pTable->pIndex = - tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, pColSchema->type, pColSchema->bytes, 1, 0, 1, getTagIndexKey); + STColumn *pCol = schemaColAt(pTable->tagSchema, DEFAULT_TAG_INDEX_COLUMN); + pTable->pIndex = tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), colBytes(pCol), 1, 0, 1, getTagIndexKey); if (pTable->pIndex == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _err; @@ -648,7 +645,8 @@ static int tsdbAddTableToMeta(STsdbRepo *pRepo, STable *pTable, bool addIdx) { if (taosHashPut(pMeta->uidMap, (char *)(&pTable->tableId.uid), sizeof(pTable->tableId.uid), (void *)(&pTable), sizeof(pTable)) < 0) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - tsdbError("vgId:%d failed to add table %s to meta while put into uid map since %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), tstrerror(terrno)); + tsdbError("vgId:%d failed to add table %s to meta while put into uid map since %s", REPO_ID(pRepo), + TABLE_CHAR_NAME(pTable), tstrerror(terrno)); goto _err; } @@ -727,22 +725,22 @@ static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable) { static int tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable) { ASSERT(pTable->type == TSDB_CHILD_TABLE && pTable != NULL); - - STable* pSTable = tsdbGetTableByUid(pMeta, pTable->superUid); + + STable *pSTable = tsdbGetTableByUid(pMeta, pTable->superUid); ASSERT(pSTable != NULL); STSchema *pSchema = tsdbGetTableTagSchema(pTable); STColumn *pCol = schemaColAt(pSchema, DEFAULT_TAG_INDEX_COLUMN); - - char* key = tdGetKVRowValOfCol(pTable->tagVal, pCol->colId); - SArray* res = tSkipListGet(pSTable->pIndex, key); - + + char * key = tdGetKVRowValOfCol(pTable->tagVal, pCol->colId); + SArray *res = tSkipListGet(pSTable->pIndex, key); + size_t size = taosArrayGetSize(res); ASSERT(size > 0); - - for(int32_t i = 0; i < size; ++i) { - SSkipListNode* pNode = taosArrayGetP(res, i); - + + for (int32_t i = 0; i < size; ++i) { + SSkipListNode *pNode = taosArrayGetP(res, i); + // STableIndexElem* pElem = (STableIndexElem*) SL_GET_NODE_DATA(pNode); if ((STable *)SL_GET_NODE_DATA(pNode) == pTable) { // this is the exact what we need tSkipListRemoveNode(pSTable->pIndex, pNode); @@ -957,6 +955,14 @@ static void *tsdbDecodeTable(void *buf, STable **pRTable) { if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) { buf = tdDecodeSchema(buf, &(pTable->tagSchema)); + STColumn *pCol = schemaColAt(pTable->tagSchema, DEFAULT_TAG_INDEX_COLUMN); + pTable->pIndex = + tSkipListCreate(TSDB_SUPER_TABLE_SL_LEVEL, colType(pCol), colBytes(pCol), 1, 0, 1, getTagIndexKey); + if (pTable->pIndex == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tsdbFreeTable(pTable); + return NULL; + } } if (TABLE_TYPE(pTable) == TSDB_STREAM_TABLE) { @@ -964,6 +970,8 @@ static void *tsdbDecodeTable(void *buf, STable **pRTable) { } } + T_REF_INC(pTable); + *pRTable = pTable; return buf; From 0f8b24ae2eb16c2a29eb146731ed852538eb6ead Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 14 Jun 2020 10:33:51 +0000 Subject: [PATCH 16/65] TD-353 --- src/tsdb/inc/tsdbMain.h | 10 ++++++++++ src/tsdb/src/tsdbMain.c | 5 +++++ src/tsdb/src/tsdbMeta.c | 29 ++++++++++++++++++++++++----- 3 files changed, 39 insertions(+), 5 deletions(-) diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 5f6d72128d..6187146dd0 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -285,6 +285,16 @@ typedef struct { STsdbMeta* tsdbNewMeta(STsdbCfg* pCfg); void tsdbFreeMeta(STsdbMeta* pMeta); +int tsdbOpenMeta(STsdbRepo* pRepo); +int tsdbCloseMeta(STsdbRepo* pRepo); +STSchema* tsdbGetTableSchema(STable* pTable); +STable* tsdbGetTableByUid(STsdbMeta* pMeta, uint64_t uid); +STSchema* tsdbGetTableSchemaByVersion(STable* pTable, int16_t version); +STSchema* tsdbGetTableTagSchema(STable* pTable); +int tsdbUpdateTable(STsdbMeta* pMeta, STable* pTable, STableCfg* pCfg); +int tsdbWLockRepoMeta(STsdbRepo* pRepo); +int tsdbRLockRepoMeta(STsdbRepo* pRepo); +int tsdbUnlockRepoMeta(STsdbRepo* pRepo); // ------------------ tsdbBuffer.c STsdbBufPool* tsdbNewBufPool(); diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index ef1a4af064..022923e4b2 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -1038,6 +1038,11 @@ _err: return -1; } +static char *getTSTupleKey(const void *data) { + SDataRow row = (SDataRow)data; + return POINTER_SHIFT(row, TD_DATA_ROW_HEAD_SIZE); +} + #if 0 ** * Set the default TSDB configuration diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 3d11383e5b..d9b054ecbb 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -23,6 +23,30 @@ #define TSDB_SUPER_TABLE_SL_LEVEL 5 #define DEFAULT_TAG_INDEX_COLUMN 0 +static int tsdbCompareSchemaVersion(const void *key1, const void *key2); +static int tsdbRestoreTable(void *pHandle, void *cont, int contLen); +static void tsdbOrgMeta(void *pHandle); +static char * getTagIndexKey(const void *pData); +static STable *tsdbNewTable(STableCfg *pCfg, bool isSuper); +static void tsdbFreeTable(STable *pTable); +static int tsdbUpdateTableTagSchema(STable *pTable, STSchema *newSchema); +static int tsdbAddTableToMeta(STsdbRepo *pRepo, STable *pTable, bool addIdx); +static void tsdbRemoveTableFromMeta(STsdbRepo *pRepo, STable *pTable, bool rmFromIdx); +static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable); +static int tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable); +static int tsdbInitTableCfg(STableCfg *config, ETableType type, uint64_t uid, int32_t tid); +static int tsdbTableSetSchema(STableCfg *config, STSchema *pSchema, bool dup); +static int tsdbTableSetName(STableCfg *config, char *name, bool dup); +static int tsdbTableSetTagSchema(STableCfg *config, STSchema *pSchema, bool dup); +static int tsdbTableSetSName(STableCfg *config, char *sname, bool dup); +static int tsdbTableSetSuperUid(STableCfg *config, uint64_t uid); +static int tsdbTableSetTagValue(STableCfg *config, SKVRow row, bool dup); +static int tsdbTableSetStreamSql(STableCfg *config, char *sql, bool dup); +static void * tsdbEncodeTableName(void *buf, tstr *name); +static void * tsdbDecodeTableName(void *buf, tstr **name); +static void * tsdbEncodeTable(void *buf, STable *pTable); +static void * tsdbDecodeTable(void *buf, STable **pRTable); + // ------------------ OUTER FUNCTIONS ------------------ int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg) { STsdbRepo *pRepo = (STsdbRepo *)repo; @@ -397,11 +421,6 @@ int tsdbUpdateTable(STsdbMeta *pMeta, STable *pTable, STableCfg *pCfg) { return TSDB_CODE_SUCCESS; } -char *getTSTupleKey(const void *data) { - SDataRow row = (SDataRow)data; - return POINTER_SHIFT(row, TD_DATA_ROW_HEAD_SIZE); -} - int tsdbWLockRepoMeta(STsdbRepo *pRepo) { int code = pthread_rwlock_wrlock(&(pRepo->tsdbMeta->rwLock)); if (code != 0) { From 618b3d22ff737fc747b543caeaf784ee6756466a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 14 Jun 2020 11:13:20 +0000 Subject: [PATCH 17/65] TD-353 --- src/tsdb/src/tsdbMemTable.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 074007af5a..0a693cedc6 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -91,6 +91,9 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { pTableData->numOfRows++; ASSERT(pTableData->numOfRows == tSkipListGetSize(pTableData->pData)); + STSchema *pSchema = tsdbGetTableSchema(pTable); + if (schemaNCols(pSchema) > pMemTable->maxCols) pMemTable->maxCols = schemaNCols; + if (schemaTLen(pSchema) > pMemTable->maxRowBytes) pMemTable->maxRowBytes = schemaTLen(pSchema); } tsdbTrace("vgId:%d a row is inserted to table %s tid %d uid %" PRIu64 " key %" PRIu64, REPO_ID(pRepo), From 77b4dcf8600535a95d40eb2563d5ecf87fe65531 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sun, 14 Jun 2020 13:04:51 +0000 Subject: [PATCH 18/65] TD-353 --- src/inc/taoserror.h | 2 +- src/tsdb/src/tsdbMain.c | 814 +++++++----------------------------- src/tsdb/src/tsdbMemTable.c | 243 ++++++++++- src/tsdb/src/tsdbMeta.c | 74 ++++ src/vnode/src/vnodeWrite.c | 4 +- 5 files changed, 480 insertions(+), 657 deletions(-) diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index 9a6045396a..eb457df624 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -178,7 +178,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_VND_APP_ERROR, 0, 0x0509, "vnode app // tsdb TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_ID, 0, 0x0600, "tsdb invalid table id") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_TABLE_TYPE, 0, 0x0601, "tsdb invalid table type") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_SCHEMA_VERSION, 0, 0x0602, "tsdb invalid table schema version") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION, 0, 0x0602, "tsdb invalid table schema version") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_ALREADY_EXIST, 0, 0x0603, "tsdb table already exist") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_CONFIG, 0, 0x0604, "tsdb invalid configuration") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INIT_FAILED, 0, 0x0605, "tsdb init failed") diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 022923e4b2..c783b91ec1 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -84,15 +84,10 @@ TSDB_REPO_T *tsdbOpenRepo(char *rootDir, STsdbAppH *pAppH) { goto _err; } - // // Restore key from file - // if (tsdbRestoreInfo(pRepo) < 0) { - // tsdbFreeCache(pRepo->tsdbCache); - // tsdbFreeMeta(pRepo->tsdbMeta); - // tsdbCloseFileH(pRepo->tsdbFileH); - // free(pRepo->rootDir); - // free(pRepo); - // return NULL; - // } + if (tsdbRestoreInfo(pRepo) < 0) { + tsdbError("vgId:%d failed to restore info from file since %s", REPO_ID(pRepo), tstrerror(terrno)); + goto _err; + } // pRepo->state = TSDB_REPO_STATE_ACTIVE; @@ -106,57 +101,28 @@ _err: return NULL; } -int32_t tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit) { - // TODO - // STsdbRepo *pRepo = (STsdbRepo *)repo; - // if (pRepo == NULL) return 0; - // int id = pRepo->config.tsdbId; +void tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit) { + if (repo == NULL) return; - // pRepo->state = TSDB_REPO_STATE_CLOSED; - // tsdbLockRepo(repo); - // if (pRepo->commit) { - // tsdbUnLockRepo(repo); - // return -1; - // } - // pRepo->commit = 1; - // // Loop to move pData to iData - // for (int i = 1; i < pRepo->config.maxTables; i++) { - // STable *pTable = pRepo->tsdbMeta->tables[i]; - // if (pTable != NULL && pTable->mem != NULL) { - // pTable->imem = pTable->mem; - // pTable->mem = NULL; - // } - // } - // // TODO: Loop to move mem to imem - // pRepo->tsdbCache->imem = pRepo->tsdbCache->mem; - // pRepo->tsdbCache->mem = NULL; - // pRepo->tsdbCache->curBlock = NULL; - // tsdbUnLockRepo(repo); + STsdbRepo *pRepo = (STsdbRepo *)repo; - // if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_START); - // if (toCommit) tsdbCommitData((void *)repo); + // TODO: wait for commit over - // tsdbCloseFileH(pRepo->tsdbFileH); - - // tsdbFreeMeta(pRepo->tsdbMeta); - - // tsdbFreeCache(pRepo->tsdbCache); - - // tfree(pRepo->rootDir); - // tfree(pRepo); - - // tsdbTrace("vgId:%d repository is closed!", id); + tsdbCloseFileH(pRepo); + tsdbCloseBufPool(pRepo); + tsdbCloseMeta(pRepo); + tsdbTrace("vgId:%d repository is closed", REPO_ID(pRepo)); return 0; } int32_t tsdbInsertData(TSDB_REPO_T *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg *pRsp) { STsdbRepo * pRepo = (STsdbRepo *)repo; - SSubmitMsgIter msgIter; + SSubmitMsgIter msgIter = {0}; if (tsdbInitSubmitMsgIter(pMsg, &msgIter) < 0) { - tsdbError("vgId:%d submit message is messed up", REPO_ID(pRepo)); - return terrno; + tsdbError("vgId:%d failed to insert data since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; } SSubmitBlk *pBlock = NULL; @@ -166,12 +132,13 @@ int32_t tsdbInsertData(TSDB_REPO_T *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg * TSKEY now = taosGetTimestamp(pRepo->config.precision); while ((pBlock = tsdbGetSubmitMsgNext(&msgIter)) != NULL) { - if ((code = tsdbInsertDataToTable(pRepo, pBlock, now, &affectedrows)) != TSDB_CODE_SUCCESS) { - return code; + if (tsdbInsertDataToTable(pRepo, pBlock, now, &affectedrows) < 0) { + pRsp->affectedRows = htonl(affectedrows); + return -1; } } pRsp->affectedRows = htonl(affectedrows); - return code; + return 0; } uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_t eindex, int32_t *size) { @@ -244,60 +211,6 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_ return magic; } -int tsdbUpdateTagValue(TSDB_REPO_T *repo, SUpdateTableTagValMsg *pMsg) { - STsdbRepo *pRepo = (STsdbRepo *)repo; - STsdbMeta *pMeta = pRepo->tsdbMeta; - int16_t tversion = htons(pMsg->tversion); - - STable *pTable = tsdbGetTableByUid(pMeta, htobe64(pMsg->uid)); - if (pTable == NULL) return TSDB_CODE_TDB_INVALID_TABLE_ID; - if (pTable->tableId.tid != htonl(pMsg->tid)) return TSDB_CODE_TDB_INVALID_TABLE_ID; - - if (pTable->type != TSDB_CHILD_TABLE) { - tsdbError("vgId:%d failed to update tag value of table %s since its type is %d", pRepo->config.tsdbId, - varDataVal(pTable->name), pTable->type); - return TSDB_CODE_TDB_INVALID_TABLE_TYPE; - } - - if (schemaVersion(tsdbGetTableTagSchema(pMeta, pTable)) < tversion) { - tsdbTrace("vgId:%d server tag version %d is older than client tag version %d, try to config", pRepo->config.tsdbId, - schemaVersion(tsdbGetTableTagSchema(pMeta, pTable)), tversion); - void *msg = (*pRepo->appH.configFunc)(pRepo->config.tsdbId, htonl(pMsg->tid)); - if (msg == NULL) { - return terrno; - } - // Deal with error her - STableCfg *pTableCfg = tsdbCreateTableCfgFromMsg(msg); - STable * super = tsdbGetTableByUid(pMeta, pTableCfg->superUid); - ASSERT(super != NULL); - - int32_t code = tsdbUpdateTable(pMeta, super, pTableCfg); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - tsdbClearTableCfg(pTableCfg); - rpcFreeCont(msg); - } - - STSchema *pTagSchema = tsdbGetTableTagSchema(pMeta, pTable); - - if (schemaVersion(pTagSchema) > tversion) { - tsdbError( - "vgId:%d failed to update tag value of table %s since version out of date, client tag version:%d server tag " - "version:%d", - pRepo->config.tsdbId, varDataVal(pTable->name), tversion, schemaVersion(pTable->tagSchema)); - return TSDB_CODE_TDB_TAG_VER_OUT_OF_DATE; - } - if (schemaColAt(pTagSchema, DEFAULT_TAG_INDEX_COLUMN)->colId == htons(pMsg->colId)) { - tsdbRemoveTableFromIndex(pMeta, pTable); - } - // TODO: remove table from index if it is the first column of tag - tdSetKVRowDataOfCol(&pTable->tagVal, htons(pMsg->colId), htons(pMsg->type), pMsg->data); - if (schemaColAt(pTagSchema, DEFAULT_TAG_INDEX_COLUMN)->colId == htons(pMsg->colId)) { - tsdbAddTableIntoIndex(pMeta, pTable); - } - return TSDB_CODE_SUCCESS; -} void tsdbStartStream(TSDB_REPO_T *repo) { STsdbRepo *pRepo = (STsdbRepo *)repo; @@ -306,8 +219,8 @@ void tsdbStartStream(TSDB_REPO_T *repo) { for (int i = 0; i < pRepo->config.maxTables; i++) { STable *pTable = pMeta->tables[i]; if (pTable && pTable->type == TSDB_STREAM_TABLE) { - pTable->cqhandle = (*pRepo->appH.cqCreateFunc)(pRepo->appH.cqH, pTable->tableId.uid, pTable->tableId.tid, - pTable->sql, tsdbGetTableSchema(pMeta, pTable)); + pTable->cqhandle = (*pRepo->appH.cqCreateFunc)(pRepo->appH.cqH, TALBE_UID(pTable), TABLE_TID(pTable), pTable->sql, + tsdbGetTableSchema(pMeta, pTable)); } } } @@ -317,6 +230,50 @@ STsdbCfg *tsdbGetCfg(const TSDB_REPO_T *repo) { return &((STsdbRepo *)repo)->config; } +int32_t tsdbConfigRepo(TSDB_REPO_T *repo, STsdbCfg *pCfg) { + STsdbRepo *pRepo = (STsdbRepo *)repo; + STsdbCfg * pRCfg = &pRepo->config; + + if (tsdbCheckAndSetDefaultCfg(pCfg) < 0) return TSDB_CODE_TDB_INVALID_CONFIG; + + ASSERT(pRCfg->tsdbId == pCfg->tsdbId); + ASSERT(pRCfg->cacheBlockSize == pCfg->cacheBlockSize); + ASSERT(pRCfg->daysPerFile == pCfg->daysPerFile); + ASSERT(pRCfg->minRowsPerFileBlock == pCfg->minRowsPerFileBlock); + ASSERT(pRCfg->maxRowsPerFileBlock == pCfg->maxRowsPerFileBlock); + ASSERT(pRCfg->precision == pCfg->precision); + + bool configChanged = false; + if (pRCfg->compression != pCfg->compression) { + configChanged = true; + tsdbAlterCompression(pRepo, pCfg->compression); + } + if (pRCfg->keep != pCfg->keep) { + configChanged = true; + tsdbAlterKeep(pRepo, pCfg->keep); + } + if (pRCfg->totalBlocks != pCfg->totalBlocks) { + configChanged = true; + tsdbAlterCacheTotalBlocks(pRepo, pCfg->totalBlocks); + } + if (pRCfg->maxTables != pCfg->maxTables) { + configChanged = true; + tsdbAlterMaxTables(pRepo, pCfg->maxTables); + } + + if (configChanged) tsdbSaveConfig(pRepo); + + return TSDB_CODE_SUCCESS; +} + +void tsdbReportStat(void *repo, int64_t *totalPoints, int64_t *totalStorage, int64_t *compStorage) { + ASSERT(repo != NULL); + STsdbRepo *pRepo = repo; + *totalPoints = pRepo->stat.pointsWritten; + *totalStorage = pRepo->stat.totalStorage; + *compStorage = pRepo->stat.compStorage; +} + // ----------------- INTERNAL FUNCTIONS ----------------- char *tsdbGetMetaFileName(char *rootDir) { int tlen = strlen(rootDir) + strlen(TSDB_META_FILE_NAME) + 2; @@ -352,84 +309,9 @@ int tsdbUnlockRepo(STsdbRepo *pRepo) { return 0; } -void *tsdbCommitData(void *arg) { - STsdbRepo *pRepo = (STsdbRepo *)arg; - STsdbMeta *pMeta = pRepo->tsdbMeta; - ASSERT(pRepo->imem != NULL); - ASSERT(pRepo->commit == 1); - - tsdbPrint("vgId:%d start to commit! keyFirst " PRId64 " keyLast " PRId64 " numOfRows " PRId64, REPO_ID(pRepo), - pRepo->imem->keyFirst, pRepo->imem->keyLast, pRepo->imem->numOfRows); - - // STsdbMeta * pMeta = pRepo->tsdbMeta; - // STsdbCache *pCache = pRepo->tsdbCache; - // STsdbCfg * pCfg = &(pRepo->config); - // SDataCols * pDataCols = NULL; - // SRWHelper whelper = {{0}}; - // if (pCache->imem == NULL) return NULL; - - tsdbPrint("vgId:%d, starting to commit....", pRepo->config.tsdbId); - - // Create the iterator to read from cache - SSkipListIterator **iters = tsdbCreateTableIters(pRepo); - if (iters == NULL) { - tsdbError("vgId:%d failed to create table iterators since %s", REPO_ID(pRepo), tstrerror(terrno)); - // TODO: deal with the error here - return NULL; - } - - if (tsdbInitWriteHelper(&whelper, pRepo) < 0) { - tsdbError("vgId:%d failed to init write helper since %s", REPO_ID(pRepo), tstrerror(terrno)); - // TODO - goto _exit; - } - - if ((pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pCfg->maxRowsPerFileBlock)) == NULL) { - tsdbError("vgId:%d failed to init data cols with maxRowBytes %d maxCols %d since %s", REPO_ID(pRepo), - pMeta->maxRowBytes, pMeta->maxCols, tstrerror(terrno)); - // TODO - goto _exit; - } - - int sfid = tsdbGetKeyFileId(pCache->imem->keyFirst, pCfg->daysPerFile, pCfg->precision); - int efid = tsdbGetKeyFileId(pCache->imem->keyLast, pCfg->daysPerFile, pCfg->precision); - - // Loop to commit to each file - for (int fid = sfid; fid <= efid; fid++) { - if (tsdbCommitToFile(pRepo, fid, iters, &whelper, pDataCols) < 0) { - ASSERT(false); - goto _exit; - } - } - - // Do retention actions - tsdbFitRetention(pRepo); - if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_OVER); - -_exit: - tdFreeDataCols(pDataCols); - tsdbDestroyTableIters(iters, pCfg->maxTables); - tsdbDestroyHelper(&whelper); - - tsdbLockRepo(arg); - tdListMove(pCache->imem->list, pCache->pool.memPool); - tsdbAdjustCacheBlocks(pCache); - tdListFree(pCache->imem->list); - free(pCache->imem); - pCache->imem = NULL; - pRepo->commit = 0; - for (int i = 1; i < pCfg->maxTables; i++) { - STable *pTable = pMeta->tables[i]; - if (pTable && pTable->imem) { - tsdbFreeMemTable(pTable->imem); - pTable->imem = NULL; - } - } - tsdbUnLockRepo(arg); - tsdbPrint("vgId:%d, commit over....", pRepo->config.tsdbId); - - return NULL; -} +STsdbMeta * tsdbGetMeta(TSDB_REPO_T *pRepo) { return ((STsdbRepo *)pRepo)->tsdbMeta; } +STsdbFileH * tsdbGetFile(TSDB_REPO_T *pRepo) { return ((STsdbRepo *)pRepo)->tsdbFileH; } +STsdbRepoInfo *tsdbGetStatus(TSDB_REPO_T *pRepo) { return NULL; } // ----------------- LOCAL FUNCTIONS ----------------- static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { @@ -757,7 +639,7 @@ static int32_t tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, TSKEY int64_t points = 0; STable *pTable == tsdbGetTableByUid(pMeta, pBlock->uid); - if (pTable == NULL || TABLE_TID(pTable)) { + if (pTable == NULL || TABLE_TID(pTable) != pBlock->tid) { tsdbError("vgId:%d failed to get table to insert data, uid " PRIu64 " tid %d", REPO_ID(pRepo), pBlock->uid, pBlock->tid); terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; @@ -770,40 +652,43 @@ static int32_t tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, TSKEY return -1; } - // // Check schema version - // int32_t tversion = pBlock->sversion; - // STSchema * pSchema = tsdbGetTableSchema(pMeta, pTable); - // ASSERT(pSchema != NULL); - // int16_t nversion = schemaVersion(pSchema); - // if (tversion > nversion) { - // tsdbTrace("vgId:%d table:%s tid:%d server schema version %d is older than clien version %d, try to config.", - // pRepo->config.tsdbId, varDataVal(pTable->name), pTable->tableId.tid, nversion, tversion); - // void *msg = (*pRepo->appH.configFunc)(pRepo->config.tsdbId, pTable->tableId.tid); - // if (msg == NULL) { - // return terrno; - // } - // // Deal with error her - // STableCfg *pTableCfg = tsdbCreateTableCfgFromMsg(msg); - // STable *pTableUpdate = NULL; - // if (pTable->type == TSDB_CHILD_TABLE) { - // pTableUpdate = tsdbGetTableByUid(pMeta, pTableCfg->superUid); - // } else { - // pTableUpdate = pTable; - // } + // Check schema version + int32_t tversion = pBlock->sversion; + STSchema * pSchema = tsdbGetTableSchema(pMeta, pTable); + ASSERT(pSchema != NULL); + int16_t nversion = schemaVersion(pSchema); + if (tversion > nversion) { + tsdbTrace("vgId:%d table %s tid %d server schema version %d is older than clien version %d, try to config.", + REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), nversion, tversion); + void *msg = (*pRepo->appH.configFunc)(REPO_ID(pRepo), TABLE_TID(pTable)); + if (msg == NULL) return -1; - // int32_t code = tsdbUpdateTable(pMeta, pTableUpdate, pTableCfg); - // if (code != TSDB_CODE_SUCCESS) { - // return code; - // } - // tsdbClearTableCfg(pTableCfg); - // rpcFreeCont(msg); - // } else { - // if (tsdbGetTableSchemaByVersion(pMeta, pTable, tversion) == NULL) { - // tsdbError("vgId:%d table:%s tid:%d invalid schema version %d from client", pRepo->config.tsdbId, - // varDataVal(pTable->name), pTable->tableId.tid, tversion); - // return TSDB_CODE_TDB_TABLE_SCHEMA_VERSION; - // } - // } + // TODO: Deal with error her + STableCfg *pTableCfg = tsdbCreateTableCfgFromMsg(msg); + STable * pTableUpdate = NULL; + if (pTable->type == TSDB_CHILD_TABLE) { + pTableUpdate = tsdbGetTableByUid(pMeta, pTableCfg->superUid); + } else { + pTableUpdate = pTable; + } + + int32_t code = tsdbUpdateTable(pMeta, pTableUpdate, pTableCfg); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + tsdbClearTableCfg(pTableCfg); + rpcFreeCont(msg); + + pSchema = tsdbGetTableSchemaByVersion(pMeta, pTable, tversion); + } else if (tversion < nversion) { + pSchema = tsdbGetTableSchemaByVersion(pMeta, pTable, tversion); + if (pSchema == NULL) { + tsdbError("vgId:%d table %s tid %d invalid schema version %d from client", REPO_ID(pRepo), + TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), tversion); + terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION; + return -1; + } + } SSubmitBlkIter blkIter = {0}; SDataRow row = NULL; @@ -817,17 +702,17 @@ static int32_t tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, TSKEY tsdbError("vgId:%d table %s tid %d uid %ld timestamp is out of range! now " PRId64 " maxKey " PRId64 " minKey " PRId64, REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TALBE_UID(pTable), now, minKey, maxKey); - return TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE; - } - - if (tdInsertRowToTable(pRepo, row, pTable) < 0) { + terrno = TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE; return -1; } + + if (tsdbInsertRowToMem(pRepo, row, pTable) < 0) return -1; + (*affectedrows)++; points++; } - atomic_fetch_add_64(&(pRepo->stat.pointsWritten), points * (pSchema->numOfCols)); - atomic_fetch_add_64(&(pRepo->stat.totalStorage), points * pSchema->vlen); + pRepo->stat.pointsWritten += points * schemaNCols(pSchema); + pRepo->stat.totalStorage += points * schemaVLen(pSchema); return 0; } @@ -868,185 +753,6 @@ static SDataRow tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter) { return row; } -static int32_t tsdbInsertRowToTable(STsdbRepo *pRepo, SDataRow row, STable *pTable) { - // TODO - int32_t level = 0; - int32_t headSize = 0; - - if (pTable->mem == NULL) { - pTable->mem = (SMemTable *)calloc(1, sizeof(SMemTable)); - if (pTable->mem == NULL) return -1; - pTable->mem->pData = - tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], 0, 0, 0, getTSTupleKey); - pTable->mem->keyFirst = INT64_MAX; - pTable->mem->keyLast = 0; - } - - tSkipListNewNodeInfo(pTable->mem->pData, &level, &headSize); - - TSKEY key = dataRowKey(row); - // printf("insert:%lld, size:%d\n", key, pTable->mem->numOfRows); - - // Copy row into the memory - SSkipListNode *pNode = tsdbAllocFromCache(pRepo->tsdbCache, headSize + dataRowLen(row), key); - if (pNode == NULL) { - // TODO: deal with allocate failure - } - - pNode->level = level; - dataRowCpy(SL_GET_NODE_DATA(pNode), row); - - // Insert the skiplist node into the data - if (pTable->mem == NULL) { - pTable->mem = (SMemTable *)calloc(1, sizeof(SMemTable)); - if (pTable->mem == NULL) return -1; - pTable->mem->pData = - tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, TYPE_BYTES[TSDB_DATA_TYPE_TIMESTAMP], 0, 0, 0, getTSTupleKey); - pTable->mem->keyFirst = INT64_MAX; - pTable->mem->keyLast = 0; - } - tSkipListPut(pTable->mem->pData, pNode); - if (key > pTable->mem->keyLast) pTable->mem->keyLast = key; - if (key < pTable->mem->keyFirst) pTable->mem->keyFirst = key; - if (key > pTable->lastKey) pTable->lastKey = key; - - pTable->mem->numOfRows = tSkipListGetSize(pTable->mem->pData); - - tsdbTrace("vgId:%d, tid:%d, uid:%" PRId64 ", table:%s a row is inserted to table! key:%" PRId64, pRepo->config.tsdbId, - pTable->tableId.tid, pTable->tableId.uid, varDataVal(pTable->name), dataRowKey(row)); - - return 0; -} - -static SSkipListIterator **tsdbCreateTableIters(STsdbRepo *pRepo) { - STsdbCfg *pCfg = &(pRepo->config); - - SSkipListIterator **iters = (SSkipListIterator **)calloc(pCfg->maxTables, sizeof(SSkipListIterator *)); - if (iters == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return NULL; - } - - for (int tid = 1; tid < maxTables; tid++) { - STable *pTable = pMeta->tables[tid]; - if (pTable == NULL || pTable->imem == NULL || pTable->imem->numOfRows == 0) continue; - - iters[tid] = tSkipListCreateIter(pTable->imem->pData); - if (iters[tid] == NULL) goto _err; - - if (!tSkipListIterNext(iters[tid])) goto _err; - } - - return iters; - -_err: - tsdbDestroyTableIters(iters, maxTables); - return NULL; -} - -static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters, SRWHelper *pHelper, - SDataCols *pDataCols) { - char dataDir[128] = {0}; - STsdbMeta * pMeta = pRepo->tsdbMeta; - STsdbFileH *pFileH = pRepo->tsdbFileH; - STsdbCfg * pCfg = &pRepo->config; - SFileGroup *pGroup = NULL; - - TSKEY minKey = 0, maxKey = 0; - tsdbGetKeyRangeOfFileId(pCfg->daysPerFile, pCfg->precision, fid, &minKey, &maxKey); - - // Check if there are data to commit to this file - int hasDataToCommit = tsdbHasDataToCommit(iters, pCfg->maxTables, minKey, maxKey); - if (!hasDataToCommit) return 0; // No data to commit, just return - - // Create and open files for commit - tsdbGetDataDirName(pRepo, dataDir); - if ((pGroup = tsdbCreateFGroup(pFileH, dataDir, fid, pCfg->maxTables)) == NULL) { - tsdbError("vgId:%d, failed to create file group %d", pRepo->config.tsdbId, fid); - goto _err; - } - - // Open files for write/read - if (tsdbSetAndOpenHelperFile(pHelper, pGroup) < 0) { - tsdbError("vgId:%d, failed to set helper file", pRepo->config.tsdbId); - goto _err; - } - - // Loop to commit data in each table - for (int tid = 1; tid < pCfg->maxTables; tid++) { - STable *pTable = pMeta->tables[tid]; - if (pTable == NULL) continue; - - SSkipListIterator *pIter = iters[tid]; - - // Set the helper and the buffer dataCols object to help to write this table - tsdbSetHelperTable(pHelper, pTable, pRepo); - tdInitDataCols(pDataCols, tsdbGetTableSchema(pMeta, pTable)); - - // Loop to write the data in the cache to files. If no data to write, just break the loop - int maxRowsToRead = pCfg->maxRowsPerFileBlock * 4 / 5; - int nLoop = 0; - while (true) { - int rowsRead = tsdbReadRowsFromCache(pMeta, pTable, pIter, maxKey, maxRowsToRead, pDataCols); - assert(rowsRead >= 0); - if (pDataCols->numOfRows == 0) break; - nLoop++; - - ASSERT(dataColsKeyFirst(pDataCols) >= minKey && dataColsKeyFirst(pDataCols) <= maxKey); - ASSERT(dataColsKeyLast(pDataCols) >= minKey && dataColsKeyLast(pDataCols) <= maxKey); - - int rowsWritten = tsdbWriteDataBlock(pHelper, pDataCols); - ASSERT(rowsWritten != 0); - if (rowsWritten < 0) goto _err; - ASSERT(rowsWritten <= pDataCols->numOfRows); - - tdPopDataColsPoints(pDataCols, rowsWritten); - maxRowsToRead = pCfg->maxRowsPerFileBlock * 4 / 5 - pDataCols->numOfRows; - } - - ASSERT(pDataCols->numOfRows == 0); - - // Move the last block to the new .l file if neccessary - if (tsdbMoveLastBlockIfNeccessary(pHelper) < 0) { - tsdbError("vgId:%d, failed to move last block", pRepo->config.tsdbId); - goto _err; - } - - // Write the SCompBlock part - if (tsdbWriteCompInfo(pHelper) < 0) { - tsdbError("vgId:%d, failed to write compInfo part", pRepo->config.tsdbId); - goto _err; - } - } - - if (tsdbWriteCompIdx(pHelper) < 0) { - tsdbError("vgId:%d, failed to write compIdx part", pRepo->config.tsdbId); - goto _err; - } - - tsdbCloseHelperFile(pHelper, 0); - // TODO: make it atomic with some methods - pGroup->files[TSDB_FILE_TYPE_HEAD] = pHelper->files.headF; - pGroup->files[TSDB_FILE_TYPE_DATA] = pHelper->files.dataF; - pGroup->files[TSDB_FILE_TYPE_LAST] = pHelper->files.lastF; - - return 0; - -_err: - ASSERT(false); - tsdbCloseHelperFile(pHelper, 1); - return -1; -} - -static char *getTSTupleKey(const void *data) { - SDataRow row = (SDataRow)data; - return POINTER_SHIFT(row, TD_DATA_ROW_HEAD_SIZE); -} - -#if 0 -** - * Set the default TSDB configuration - */ static int tsdbRestoreInfo(STsdbRepo *pRepo) { STsdbMeta * pMeta = pRepo->tsdbMeta; @@ -1057,11 +763,12 @@ static int tsdbRestoreInfo(STsdbRepo *pRepo) { SRWHelper rhelper = {{0}}; if (tsdbInitReadHelper(&rhelper, pRepo) < 0) goto _err; - tsdbInitFileGroupIter(pFileH, &iter, TSDB_ORDER_ASC); + + tsdbInitFileGroupIter(pFileH, &iter, TSDB_ORDER_DESC); while ((pFGroup = tsdbGetFileGroupNext(&iter)) != NULL) { if (tsdbSetAndOpenHelperFile(&rhelper, pFGroup) < 0) goto _err; for (int i = 1; i < pRepo->config.maxTables; i++) { - STable * pTable = pMeta->tables[i]; + STable *pTable = pMeta->tables[i]; if (pTable == NULL) continue; SCompIdx *pIdx = &rhelper.pCompIdx[i]; @@ -1077,62 +784,56 @@ _err: return -1; } -/** - * Change the configuration of a repository - * @param pCfg the repository configuration, the upper layer should free the pointer - * - * @return 0 for success, -1 for failure and the error number is set - */ -int32_t tsdbConfigRepo(TSDB_REPO_T *repo, STsdbCfg *pCfg) { - STsdbRepo *pRepo = (STsdbRepo *)repo; - STsdbCfg * pRCfg = &pRepo->config; - - if (tsdbCheckAndSetDefaultCfg(pCfg) < 0) return TSDB_CODE_TDB_INVALID_CONFIG; - - ASSERT(pRCfg->tsdbId == pCfg->tsdbId); - ASSERT(pRCfg->cacheBlockSize == pCfg->cacheBlockSize); - ASSERT(pRCfg->daysPerFile == pCfg->daysPerFile); - ASSERT(pRCfg->minRowsPerFileBlock == pCfg->minRowsPerFileBlock); - ASSERT(pRCfg->maxRowsPerFileBlock == pCfg->maxRowsPerFileBlock); - ASSERT(pRCfg->precision == pCfg->precision); - - bool configChanged = false; - if (pRCfg->compression != pCfg->compression) { - configChanged = true; - tsdbAlterCompression(pRepo, pCfg->compression); - } - if (pRCfg->keep != pCfg->keep) { - configChanged = true; - tsdbAlterKeep(pRepo, pCfg->keep); - } - if (pRCfg->totalBlocks != pCfg->totalBlocks) { - configChanged = true; - tsdbAlterCacheTotalBlocks(pRepo, pCfg->totalBlocks); - } - if (pRCfg->maxTables != pCfg->maxTables) { - configChanged = true; - tsdbAlterMaxTables(pRepo, pCfg->maxTables); - } - - if (configChanged) tsdbSaveConfig(pRepo); - - return TSDB_CODE_SUCCESS; +static int tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter) { + if (pBlock->len <= 0) return -1; + pIter->totalLen = pBlock->len; + pIter->len = 0; + pIter->row = (SDataRow)(pBlock->data); + return 0; } - -/** - * Get the TSDB repository information, including some statistics - * @param pRepo the TSDB repository handle - * @param error the error number to set when failure occurs - * - * @return a info struct handle on success, NULL for failure and the error number is set. The upper - * layers should free the info handle themselves or memory leak will occur - */ -STsdbRepoInfo *tsdbGetStatus(TSDB_REPO_T *pRepo) { - // TODO - return NULL; +static void tsdbAlterCompression(STsdbRepo *pRepo, int8_t compression) { + int8_t oldCompRession = pRepo->config.compression; + pRepo->config.compression = compression; + tsdbTrace("vgId:%d tsdb compression is changed from %d to %d", oldCompRession, compression); } +static void tsdbAlterKeep(STsdbRepo *pRepo, int32_t keep) { + STsdbCfg *pCfg = &pRepo->config; + int oldKeep = pCfg->keep; + + int maxFiles = keep / pCfg->maxTables + 3; + if (pRepo->config.keep > keep) { + pRepo->config.keep = keep; + pRepo->tsdbFileH->maxFGroups = maxFiles; + } else { + pRepo->config.keep = keep; + pRepo->tsdbFileH->fGroup = realloc(pRepo->tsdbFileH->fGroup, sizeof(SFileGroup)); + if (pRepo->tsdbFileH->fGroup == NULL) { + // TODO: deal with the error + } + pRepo->tsdbFileH->maxFGroups = maxFiles; + } + tsdbTrace("vgId:%d, keep is changed from %d to %d", pRepo->config.tsdbId, oldKeep, keep); +} + +static void tsdbAlterMaxTables(STsdbRepo *pRepo, int32_t maxTables) { + int oldMaxTables = pRepo->config.maxTables; + if (oldMaxTables < pRepo->config.maxTables) { + // TODO + } + + STsdbMeta *pMeta = pRepo->tsdbMeta; + + pMeta->maxTables = maxTables; + pMeta->tables = realloc(pMeta->tables, maxTables * sizeof(STable *)); + memset(&pMeta->tables[oldMaxTables], 0, sizeof(STable *) * (maxTables - oldMaxTables)); + pRepo->config.maxTables = maxTables; + + tsdbTrace("vgId:%d, tsdb maxTables is changed from %d to %d!", pRepo->config.tsdbId, oldMaxTables, maxTables); +} + +#if 0 TSKEY tsdbGetTableLastKey(TSDB_REPO_T *repo, uint64_t uid) { STsdbRepo *pRepo = (STsdbRepo *)repo; @@ -1144,153 +845,6 @@ TSKEY tsdbGetTableLastKey(TSDB_REPO_T *repo, uint64_t uid) { } -STableInfo *tsdbGetTableInfo(TSDB_REPO_T *pRepo, STableId tableId) { - // TODO - return NULL; -} - -// TODO: need to return the number of data inserted - -void tsdbClearTableCfg(STableCfg *config) { - if (config) { - if (config->schema) tdFreeSchema(config->schema); - if (config->tagSchema) tdFreeSchema(config->tagSchema); - if (config->tagValues) kvRowFree(config->tagValues); - tfree(config->name); - tfree(config->sname); - tfree(config->sql); - free(config); - } -} - -int tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter) { - if (pBlock->len <= 0) return -1; - pIter->totalLen = pBlock->len; - pIter->len = 0; - pIter->row = (SDataRow)(pBlock->data); - return 0; -} - - - - -STsdbMeta* tsdbGetMeta(TSDB_REPO_T* pRepo) { - STsdbRepo *tsdb = (STsdbRepo *)pRepo; - return tsdb->tsdbMeta; -} - -STsdbFileH* tsdbGetFile(TSDB_REPO_T* pRepo) { - STsdbRepo* tsdb = (STsdbRepo*) pRepo; - return tsdb->tsdbFileH; -} - -// Check the configuration and set default options - - - -static int32_t tsdbRestoreCfg(STsdbRepo *pRepo, STsdbCfg *pCfg) { - char fname[128] = "\0"; - - if (tsdbGetCfgFname(pRepo, fname) < 0) return -1; - - int fd = open(fname, O_RDONLY); - if (fd < 0) { - return -1; - } - - if (read(fd, (void *)pCfg, sizeof(STsdbCfg)) < sizeof(STsdbCfg)) { - close(fd); - return -1; - } - - close(fd); - - return 0; -} - - - -static int32_t tsdbDestroyRepoEnv(STsdbRepo *pRepo) { - char fname[260]; - if (pRepo == NULL) return 0; - char *dirName = calloc(1, strlen(pRepo->rootDir) + strlen("tsdb") + 2); - if (dirName == NULL) { - return -1; - } - - sprintf(dirName, "%s/%s", pRepo->rootDir, "tsdb"); - - DIR *dir = opendir(dirName); - if (dir == NULL) return -1; - - struct dirent *dp; - while ((dp = readdir(dir)) != NULL) { - if ((strcmp(dp->d_name, ".") == 0) || (strcmp(dp->d_name, "..") == 0)) continue; - sprintf(fname, "%s/%s", pRepo->rootDir, dp->d_name); - remove(fname); - } - - closedir(dir); - - rmdir(dirName); - - return 0; -} - - -static int tsdbReadRowsFromCache(STsdbMeta *pMeta, STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols) { - ASSERT(maxRowsToRead > 0); - if (pIter == NULL) return 0; - STSchema *pSchema = NULL; - - int numOfRows = 0; - - do { - if (numOfRows >= maxRowsToRead) break; - - SSkipListNode *node = tSkipListIterGet(pIter); - if (node == NULL) break; - - SDataRow row = SL_GET_NODE_DATA(node); - if (dataRowKey(row) > maxKey) break; - - if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) { - pSchema = tsdbGetTableSchemaByVersion(pMeta, pTable, dataRowVersion(row)); - if (pSchema == NULL) { - // TODO: deal with the error here - ASSERT(false); - } - } - - tdAppendDataRowToDataCol(row, pSchema, pCols); - numOfRows++; - } while (tSkipListIterNext(pIter)); - - return numOfRows; -} - -static void tsdbDestroyTableIters(SSkipListIterator **iters, int maxTables) { - if (iters == NULL) return; - - for (int tid = 1; tid < maxTables; tid++) { - if (iters[tid] == NULL) continue; - tSkipListDestroyIter(iters[tid]); - } - - free(iters); -} - - -static void tsdbFreeMemTable(SMemTable *pMemTable) { - if (pMemTable) { - tSkipListDestroy(pMemTable->pData); - free(pMemTable); - } -} - -// Commit to file - - /** * Return the next iterator key. * @@ -1317,54 +871,8 @@ static int tsdbHasDataToCommit(SSkipListIterator **iters, int nIters, TSKEY minK return 0; } -static void tsdbAlterCompression(STsdbRepo *pRepo, int8_t compression) { - int8_t oldCompRession = pRepo->config.compression; - pRepo->config.compression = compression; - tsdbTrace("vgId:%d, tsdb compression is changed from %d to %d", oldCompRession, compression); -} - -static void tsdbAlterKeep(STsdbRepo *pRepo, int32_t keep) { - STsdbCfg *pCfg = &pRepo->config; - int oldKeep = pCfg->keep; - - int maxFiles = keep / pCfg->maxTables + 3; - if (pRepo->config.keep > keep) { - pRepo->config.keep = keep; - pRepo->tsdbFileH->maxFGroups = maxFiles; - } else { - pRepo->config.keep = keep; - pRepo->tsdbFileH->fGroup = realloc(pRepo->tsdbFileH->fGroup, sizeof(SFileGroup)); - if (pRepo->tsdbFileH->fGroup == NULL) { - // TODO: deal with the error - } - pRepo->tsdbFileH->maxFGroups = maxFiles; - } - tsdbTrace("vgId:%d, keep is changed from %d to %d", pRepo->config.tsdbId, oldKeep, keep); -} - -static void tsdbAlterMaxTables(STsdbRepo *pRepo, int32_t maxTables) { - int oldMaxTables = pRepo->config.maxTables; - if (oldMaxTables < pRepo->config.maxTables) { - // TODO - } - - STsdbMeta *pMeta = pRepo->tsdbMeta; - - pMeta->maxTables = maxTables; - pMeta->tables = realloc(pMeta->tables, maxTables * sizeof(STable *)); - memset(&pMeta->tables[oldMaxTables], 0, sizeof(STable *) * (maxTables-oldMaxTables)); - pRepo->config.maxTables = maxTables; - - tsdbTrace("vgId:%d, tsdb maxTables is changed from %d to %d!", pRepo->config.tsdbId, oldMaxTables, maxTables); -} -void tsdbReportStat(void *repo, int64_t *totalPoints, int64_t *totalStorage, int64_t *compStorage){ - ASSERT(repo != NULL); - STsdbRepo * pRepo = repo; - *totalPoints = pRepo->stat.pointsWritten; - *totalStorage = pRepo->stat.totalStorage; - *compStorage = pRepo->stat.compStorage; -} + #endif \ No newline at end of file diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 0a693cedc6..593454243d 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -317,4 +317,245 @@ static void tsdbFreeTableData(STableData *pTableData) { } } -static char *tsdbGetTsTupleKey(const void *data) { return dataRowTuple(data); } \ No newline at end of file +static char *tsdbGetTsTupleKey(const void *data) { return dataRowTuple(data); } + +static void *tsdbCommitData(void *arg) { + STsdbRepo *pRepo = (STsdbRepo *)arg; + STsdbMeta *pMeta = pRepo->tsdbMeta; + ASSERT(pRepo->imem != NULL); + ASSERT(pRepo->commit == 1); + + tsdbPrint("vgId:%d start to commit! keyFirst " PRId64 " keyLast " PRId64 " numOfRows " PRId64, REPO_ID(pRepo), + pRepo->imem->keyFirst, pRepo->imem->keyLast, pRepo->imem->numOfRows); + + // STsdbMeta * pMeta = pRepo->tsdbMeta; + // STsdbCache *pCache = pRepo->tsdbCache; + // STsdbCfg * pCfg = &(pRepo->config); + // SDataCols * pDataCols = NULL; + // SRWHelper whelper = {{0}}; + // if (pCache->imem == NULL) return NULL; + + tsdbPrint("vgId:%d, starting to commit....", pRepo->config.tsdbId); + + // Create the iterator to read from cache + SSkipListIterator **iters = tsdbCreateTableIters(pRepo); + if (iters == NULL) { + tsdbError("vgId:%d failed to create table iterators since %s", REPO_ID(pRepo), tstrerror(terrno)); + // TODO: deal with the error here + return NULL; + } + + if (tsdbInitWriteHelper(&whelper, pRepo) < 0) { + tsdbError("vgId:%d failed to init write helper since %s", REPO_ID(pRepo), tstrerror(terrno)); + // TODO + goto _exit; + } + + if ((pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pCfg->maxRowsPerFileBlock)) == NULL) { + tsdbError("vgId:%d failed to init data cols with maxRowBytes %d maxCols %d since %s", REPO_ID(pRepo), + pMeta->maxRowBytes, pMeta->maxCols, tstrerror(terrno)); + // TODO + goto _exit; + } + + int sfid = tsdbGetKeyFileId(pCache->imem->keyFirst, pCfg->daysPerFile, pCfg->precision); + int efid = tsdbGetKeyFileId(pCache->imem->keyLast, pCfg->daysPerFile, pCfg->precision); + + // Loop to commit to each file + for (int fid = sfid; fid <= efid; fid++) { + if (tsdbCommitToFile(pRepo, fid, iters, &whelper, pDataCols) < 0) { + ASSERT(false); + goto _exit; + } + } + + // Do retention actions + tsdbFitRetention(pRepo); + if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_OVER); + +_exit: + tdFreeDataCols(pDataCols); + tsdbDestroyTableIters(iters, pCfg->maxTables); + tsdbDestroyHelper(&whelper); + + tsdbLockRepo(arg); + tdListMove(pCache->imem->list, pCache->pool.memPool); + tsdbAdjustCacheBlocks(pCache); + tdListFree(pCache->imem->list); + free(pCache->imem); + pCache->imem = NULL; + pRepo->commit = 0; + for (int i = 1; i < pCfg->maxTables; i++) { + STable *pTable = pMeta->tables[i]; + if (pTable && pTable->imem) { + tsdbFreeMemTable(pTable->imem); + pTable->imem = NULL; + } + } + tsdbUnLockRepo(arg); + tsdbPrint("vgId:%d, commit over....", pRepo->config.tsdbId); + + return NULL; +} + +static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters, SRWHelper *pHelper, + SDataCols *pDataCols) { + char dataDir[128] = {0}; + STsdbMeta * pMeta = pRepo->tsdbMeta; + STsdbFileH *pFileH = pRepo->tsdbFileH; + STsdbCfg * pCfg = &pRepo->config; + SFileGroup *pGroup = NULL; + + TSKEY minKey = 0, maxKey = 0; + tsdbGetKeyRangeOfFileId(pCfg->daysPerFile, pCfg->precision, fid, &minKey, &maxKey); + + // Check if there are data to commit to this file + int hasDataToCommit = tsdbHasDataToCommit(iters, pCfg->maxTables, minKey, maxKey); + if (!hasDataToCommit) return 0; // No data to commit, just return + + // Create and open files for commit + tsdbGetDataDirName(pRepo, dataDir); + if ((pGroup = tsdbCreateFGroup(pFileH, dataDir, fid, pCfg->maxTables)) == NULL) { + tsdbError("vgId:%d, failed to create file group %d", pRepo->config.tsdbId, fid); + goto _err; + } + + // Open files for write/read + if (tsdbSetAndOpenHelperFile(pHelper, pGroup) < 0) { + tsdbError("vgId:%d, failed to set helper file", pRepo->config.tsdbId); + goto _err; + } + + // Loop to commit data in each table + for (int tid = 1; tid < pCfg->maxTables; tid++) { + STable *pTable = pMeta->tables[tid]; + if (pTable == NULL) continue; + + SSkipListIterator *pIter = iters[tid]; + + // Set the helper and the buffer dataCols object to help to write this table + tsdbSetHelperTable(pHelper, pTable, pRepo); + tdInitDataCols(pDataCols, tsdbGetTableSchema(pMeta, pTable)); + + // Loop to write the data in the cache to files. If no data to write, just break the loop + int maxRowsToRead = pCfg->maxRowsPerFileBlock * 4 / 5; + int nLoop = 0; + while (true) { + int rowsRead = tsdbReadRowsFromCache(pMeta, pTable, pIter, maxKey, maxRowsToRead, pDataCols); + assert(rowsRead >= 0); + if (pDataCols->numOfRows == 0) break; + nLoop++; + + ASSERT(dataColsKeyFirst(pDataCols) >= minKey && dataColsKeyFirst(pDataCols) <= maxKey); + ASSERT(dataColsKeyLast(pDataCols) >= minKey && dataColsKeyLast(pDataCols) <= maxKey); + + int rowsWritten = tsdbWriteDataBlock(pHelper, pDataCols); + ASSERT(rowsWritten != 0); + if (rowsWritten < 0) goto _err; + ASSERT(rowsWritten <= pDataCols->numOfRows); + + tdPopDataColsPoints(pDataCols, rowsWritten); + maxRowsToRead = pCfg->maxRowsPerFileBlock * 4 / 5 - pDataCols->numOfRows; + } + + ASSERT(pDataCols->numOfRows == 0); + + // Move the last block to the new .l file if neccessary + if (tsdbMoveLastBlockIfNeccessary(pHelper) < 0) { + tsdbError("vgId:%d, failed to move last block", pRepo->config.tsdbId); + goto _err; + } + + // Write the SCompBlock part + if (tsdbWriteCompInfo(pHelper) < 0) { + tsdbError("vgId:%d, failed to write compInfo part", pRepo->config.tsdbId); + goto _err; + } + } + + if (tsdbWriteCompIdx(pHelper) < 0) { + tsdbError("vgId:%d, failed to write compIdx part", pRepo->config.tsdbId); + goto _err; + } + + tsdbCloseHelperFile(pHelper, 0); + // TODO: make it atomic with some methods + pGroup->files[TSDB_FILE_TYPE_HEAD] = pHelper->files.headF; + pGroup->files[TSDB_FILE_TYPE_DATA] = pHelper->files.dataF; + pGroup->files[TSDB_FILE_TYPE_LAST] = pHelper->files.lastF; + + return 0; + +_err: + ASSERT(false); + tsdbCloseHelperFile(pHelper, 1); + return -1; +} + +static SSkipListIterator **tsdbCreateTableIters(STsdbRepo *pRepo) { + STsdbCfg *pCfg = &(pRepo->config); + + SSkipListIterator **iters = (SSkipListIterator **)calloc(pCfg->maxTables, sizeof(SSkipListIterator *)); + if (iters == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return NULL; + } + + for (int tid = 1; tid < maxTables; tid++) { + STable *pTable = pMeta->tables[tid]; + if (pTable == NULL || pTable->imem == NULL || pTable->imem->numOfRows == 0) continue; + + iters[tid] = tSkipListCreateIter(pTable->imem->pData); + if (iters[tid] == NULL) goto _err; + + if (!tSkipListIterNext(iters[tid])) goto _err; + } + + return iters; + +_err: + tsdbDestroyTableIters(iters, maxTables); + return NULL; +} + +static void tsdbDestroyTableIters(SSkipListIterator **iters, int maxTables) { + if (iters == NULL) return; + + for (int tid = 1; tid < maxTables; tid++) { + if (iters[tid] == NULL) continue; + tSkipListDestroyIter(iters[tid]); + } + + free(iters); +} + +static int tsdbReadRowsFromCache(STsdbMeta *pMeta, STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols) { + ASSERT(maxRowsToRead > 0); + if (pIter == NULL) return 0; + STSchema *pSchema = NULL; + + int numOfRows = 0; + + do { + if (numOfRows >= maxRowsToRead) break; + + SSkipListNode *node = tSkipListIterGet(pIter); + if (node == NULL) break; + + SDataRow row = SL_GET_NODE_DATA(node); + if (dataRowKey(row) > maxKey) break; + + if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) { + pSchema = tsdbGetTableSchemaByVersion(pMeta, pTable, dataRowVersion(row)); + if (pSchema == NULL) { + // TODO: deal with the error here + ASSERT(false); + } + } + + tdAppendDataRowToDataCol(row, pSchema, pCols); + numOfRows++; + } while (tSkipListIterNext(pIter)); + + return numOfRows; +} \ No newline at end of file diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index d9b054ecbb..10866bbae5 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -42,6 +42,7 @@ static int tsdbTableSetSName(STableCfg *config, char *sname, bool dup); static int tsdbTableSetSuperUid(STableCfg *config, uint64_t uid); static int tsdbTableSetTagValue(STableCfg *config, SKVRow row, bool dup); static int tsdbTableSetStreamSql(STableCfg *config, char *sql, bool dup); +static void tsdbClearTableCfg(STableCfg *config); static void * tsdbEncodeTableName(void *buf, tstr *name); static void * tsdbDecodeTableName(void *buf, tstr **name); static void * tsdbEncodeTable(void *buf, STable *pTable); @@ -243,6 +244,67 @@ _err: return NULL; } +int tsdbUpdateTagValue(TSDB_REPO_T *repo, SUpdateTableTagValMsg *pMsg) { + STsdbRepo *pRepo = (STsdbRepo *)repo; + STsdbMeta *pMeta = pRepo->tsdbMeta; + int16_t tversion = htons(pMsg->tversion); + + STable *pTable = tsdbGetTableByUid(pMeta, htobe64(pMsg->uid)); + if (pTable == NULL) { + terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; + return -1; + } + if (TABLE_TID(pTable) != htonl(pMsg->tid)) { + terrno = TSDB_CODE_TDB_INVALID_TABLE_ID + return -1; + } + + if (TABLE_TYPE(pTable) != TSDB_CHILD_TABLE) { + tsdbError("vgId:%d failed to update tag value of table %s since its type is %d", REPO_ID(pRepo), + TABLE_CHAR_NAME(pTable), TABLE_TYPE(pTable)); + terrno = TSDB_CODE_TDB_INVALID_ACTION; + return -1; + } + + if (schemaVersion(tsdbGetTableTagSchema(pTable)) < tversion) { + tsdbTrace("vgId:%d server tag version %d is older than client tag version %d, try to config", REPO_ID(pRepo), + schemaVersion(tsdbGetTableTagSchema(pTable)), tversion); + void *msg = (*pRepo->appH.configFunc)(pRepo->config.tsdbId, htonl(pMsg->tid)); + if (msg == NULL) return -1; + + // Deal with error her + STableCfg *pTableCfg = tsdbCreateTableCfgFromMsg(msg); + STable * super = tsdbGetTableByUid(pMeta, pTableCfg->superUid); + ASSERT(super != NULL); + + int32_t code = tsdbUpdateTable(pMeta, super, pTableCfg); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + tsdbClearTableCfg(pTableCfg); + rpcFreeCont(msg); + } + + STSchema *pTagSchema = tsdbGetTableTagSchema(pTable); + + if (schemaVersion(pTagSchema) > tversion) { + tsdbError( + "vgId:%d failed to update tag value of table %s since version out of date, client tag version:%d server tag " + "version:%d", + pRepo->config.tsdbId, varDataVal(pTable->name), tversion, schemaVersion(pTable->tagSchema)); + return TSDB_CODE_TDB_TAG_VER_OUT_OF_DATE; + } + if (schemaColAt(pTagSchema, DEFAULT_TAG_INDEX_COLUMN)->colId == htons(pMsg->colId)) { + tsdbRemoveTableFromIndex(pMeta, pTable); + } + // TODO: remove table from index if it is the first column of tag + tdSetKVRowDataOfCol(&pTable->tagVal, htons(pMsg->colId), htons(pMsg->type), pMsg->data); + if (schemaColAt(pTagSchema, DEFAULT_TAG_INDEX_COLUMN)->colId == htons(pMsg->colId)) { + tsdbAddTableIntoIndex(pMeta, pTable); + } + return TSDB_CODE_SUCCESS; +} + // ------------------ INTERNAL FUNCTIONS ------------------ STsdbMeta *tsdbNewMeta(STsdbCfg *pCfg) { STsdbMeta *pMeta = (STsdbMeta *)calloc(1, sizeof(*pMeta)); @@ -896,6 +958,18 @@ static int tsdbTableSetStreamSql(STableCfg *config, char *sql, bool dup) { return 0; } +static void tsdbClearTableCfg(STableCfg *config) { + if (config) { + if (config->schema) tdFreeSchema(config->schema); + if (config->tagSchema) tdFreeSchema(config->tagSchema); + if (config->tagValues) kvRowFree(config->tagValues); + tfree(config->name); + tfree(config->sname); + tfree(config->sql); + free(config); + } +} + static void *tsdbEncodeTableName(void *buf, tstr *name) { void *pBuf = buf; diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index e57188b479..032dbde9ea 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -87,7 +87,7 @@ int32_t vnodeProcessWrite(void *param1, int qtype, void *param2, void *item) { } static int32_t vnodeProcessSubmitMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pRet) { - int32_t code = 0; + int32_t code = TSDB_CODE_SUCCESS; // save insert result into item @@ -96,7 +96,7 @@ static int32_t vnodeProcessSubmitMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pR pRet->len = sizeof(SShellSubmitRspMsg); pRet->rsp = rpcMallocCont(pRet->len); SShellSubmitRspMsg *pRsp = pRet->rsp; - code = tsdbInsertData(pVnode->tsdb, pCont, pRsp); + if (tsdbInsertData(pVnode->tsdb, pCont, pRsp) < 0) code = terrno; pRsp->numOfFailedBlocks = 0; //TODO //pRet->len += pRsp->numOfFailedBlocks * sizeof(SShellSubmitRspBlock); //TODO pRsp->code = 0; From f09d73cfc9837e520cb2437d10175e14fead28f4 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 15 Jun 2020 01:32:00 +0000 Subject: [PATCH 19/65] TD-353 --- src/tsdb/inc/tsdbMain.h | 1 + src/tsdb/src/tsdbMemTable.c | 39 +++++++++++++++++++++++-------------- 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 6187146dd0..03b2bc255b 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -307,6 +307,7 @@ SListNode* tsdbAllocBufBlockFromPool(STsdbRepo* pRepo); int tsdbInsertRowToMem(STsdbRepo* pRepo, SDataRow row, STable* pTable); int tsdbRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable); int tsdbUnRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable); +int tsdbTakeMemSnapshot(STsdbRepo* pRepo, SMemTable** pMem, SMemTable** pIMem); // ------------------ tsdbFile.c #define TSDB_KEY_FILEID(key, daysPerFile, precision) ((key) / tsMsPerDay[(precision)] / (daysPerFile)) diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 593454243d..57e7961049 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -103,14 +103,14 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { } int tsdbRefMemTable(STsdbRepo *pRepo, SMemTable *pMemTable) { - ASSERT(IS_REPO_LOCKED(pRepo)); - ASSERT(pMemTable != NULL); + if (pMemTable == NULL) return 0; T_REF_INC(pMemTable); + return 0; } // Need to lock the repository int tsdbUnRefMemTable(STsdbRepo *pRepo, SMemTable *pMemTable) { - ASSERT(pMemTable != NULL); + if (pMemTable == NULL) return 0; if (T_REF_DEC(pMemTable) == 0) { STsdbCfg * pCfg = &pRepo->config; @@ -143,6 +143,17 @@ int tsdbUnRefMemTable(STsdbRepo *pRepo, SMemTable *pMemTable) { return 0; } +int tsdbTakeMemSnapshot(STsdbRepo *pRepo, SMemTable **pMem, SMemTable **pIMem) { + if (tsdbLockRepo(pRepo) < 0) return -1; + + *pMem = pRepo->mem; + *pIMem = pRepo->mem; + tsdbRefMemTable(pRepo, *pMem); + tsdbRefMemTable(pRepo, *pIMem); + + if (tsdbUnlockRepo(pRepo) < 0) return -1; +} + // ---------------- LOCAL FUNCTIONS ---------------- static FORCE_INLINE STsdbBufBlock *tsdbGetCurrBufBlock(STsdbRepo *pRepo) { ASSERT(pRepo != NULL); @@ -171,10 +182,16 @@ static void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) { terrno = TAOS_SYSTEM_ERROR(errno); return NULL; } + + if (tsdbUnRefMemTable(pRepo, pRepo->imem) < 0) { + tsdbError("vgId:%d failed to unref memtable since %s", REPO_ID(pRepo), tstrerror(terrno)) + return NULL; + } } ASSERT(pRepo->commit == 0); SMemTable *pImem = pRepo->imem; + if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_START); if (tsdbLockRepo(pRepo) < 0) return NULL; pRepo->imem = pRepo->mem; @@ -322,20 +339,12 @@ static char *tsdbGetTsTupleKey(const void *data) { return dataRowTuple(data); } static void *tsdbCommitData(void *arg) { STsdbRepo *pRepo = (STsdbRepo *)arg; STsdbMeta *pMeta = pRepo->tsdbMeta; - ASSERT(pRepo->imem != NULL); + SMemTable *pMem = pRepo->imem; ASSERT(pRepo->commit == 1); + ASSERT(pMem != NULL); - tsdbPrint("vgId:%d start to commit! keyFirst " PRId64 " keyLast " PRId64 " numOfRows " PRId64, REPO_ID(pRepo), - pRepo->imem->keyFirst, pRepo->imem->keyLast, pRepo->imem->numOfRows); - - // STsdbMeta * pMeta = pRepo->tsdbMeta; - // STsdbCache *pCache = pRepo->tsdbCache; - // STsdbCfg * pCfg = &(pRepo->config); - // SDataCols * pDataCols = NULL; - // SRWHelper whelper = {{0}}; - // if (pCache->imem == NULL) return NULL; - - tsdbPrint("vgId:%d, starting to commit....", pRepo->config.tsdbId); + tsdbPrint("vgId:%d start to commit! keyFirst %" PRId64 " keyLast %" PRId64 " numOfRows %" PRId64, REPO_ID(pRepo), + pMem->keyFirst, pMem->keyLast, pMem->numOfRows); // Create the iterator to read from cache SSkipListIterator **iters = tsdbCreateTableIters(pRepo); From 1a139a077268923497f598efa26e1fddb3f7f6c7 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 15 Jun 2020 07:38:15 +0000 Subject: [PATCH 20/65] TD-353 --- src/inc/taoserror.h | 1 + src/tsdb/inc/tsdbMain.h | 6 +- src/tsdb/src/tsdbFile.c | 5 - src/tsdb/src/tsdbMain.c | 31 ----- src/tsdb/src/tsdbMemTable.c | 223 ++++++++++++++++++++++-------------- src/tsdb/src/tsdbMeta.c | 8 ++ 6 files changed, 153 insertions(+), 121 deletions(-) diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index eb457df624..a5bfb0fa1e 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -191,6 +191,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE, 0, 0x060B, "tsdb times TAOS_DEFINE_ERROR(TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP, 0, 0x060C, "tsdb submit message is messed up") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_ACTION, 0, 0x060D, "tsdb invalid action") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_CREATE_TB_MSG, 0, 0x060E, "tsdb invalid create table message") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_TABLE_DATA_IN_MEM, 0, 0x060F, "tsdb no table data in memory skiplist") // query TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_QHANDLE, 0, 0x0700, "query invalid handle") diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 03b2bc255b..70cd19d55a 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -295,6 +295,8 @@ int tsdbUpdateTable(STsdbMeta* pMeta, STable* pTable, STableCfg* pCfg); int tsdbWLockRepoMeta(STsdbRepo* pRepo); int tsdbRLockRepoMeta(STsdbRepo* pRepo); int tsdbUnlockRepoMeta(STsdbRepo* pRepo); +void tsdbRefTable(STable* pTable); +void tsdbUnRefTable(STable* pTable); // ------------------ tsdbBuffer.c STsdbBufPool* tsdbNewBufPool(); @@ -317,8 +319,8 @@ int tsdbTakeMemSnapshot(STsdbRepo* pRepo, SMemTable** pMem, SMemTable** pIMem); #define TSDB_FGROUP_ITER_FORWARD TSDB_ORDER_ASC #define TSDB_FGROUP_ITER_BACKWARD TSDB_ORDER_DESC -STsdbFileH* tsdbNewFileH(STsdbCfg* pCfg); -void tsdbFreeFileH(STsdbFileH* pFileH); +STsdbFileH* tsdbNewFileH(STsdbCfg* pCfg); +void tsdbFreeFileH(STsdbFileH* pFileH); // ------------------ tsdbRWHelper.c #define TSDB_MAX_SUBBLOCKS 8 diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index e52a72c32a..77e07b54c4 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -325,11 +325,6 @@ int tsdbCreateFile(char *dataDir, int fileId, const char *suffix, SFile *pFile) return 0; } -void tsdbGetKeyRangeOfFileId(int32_t daysPerFile, int8_t precision, int32_t fileId, TSKEY *minKey, - TSKEY *maxKey) { - *minKey = fileId * daysPerFile * tsMsPerDay[precision]; - *maxKey = *minKey + daysPerFile * tsMsPerDay[precision] - 1; -} SFileGroup *tsdbSearchFGroup(STsdbFileH *pFileH, int fid) { if (pFileH->numOfFGroups == 0 || fid < pFileH->fGroup[0].fileId || fid > pFileH->fGroup[pFileH->numOfFGroups - 1].fileId) diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index c783b91ec1..298caff0f6 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -844,35 +844,4 @@ TSKEY tsdbGetTableLastKey(TSDB_REPO_T *repo, uint64_t uid) { return TSDB_GET_TABLE_LAST_KEY(pTable); } - -/** - * Return the next iterator key. - * - * @return the next key if iter has - * -1 if iter not - */ -static TSKEY tsdbNextIterKey(SSkipListIterator *pIter) { - if (pIter == NULL) return -1; - - SSkipListNode *node = tSkipListIterGet(pIter); - if (node == NULL) return -1; - - SDataRow row = SL_GET_NODE_DATA(node); - return dataRowKey(row); -} - -static int tsdbHasDataToCommit(SSkipListIterator **iters, int nIters, TSKEY minKey, TSKEY maxKey) { - TSKEY nextKey; - for (int i = 0; i < nIters; i++) { - SSkipListIterator *pIter = iters[i]; - nextKey = tsdbNextIterKey(pIter); - if (nextKey > 0 && (nextKey >= minKey && nextKey <= maxKey)) return 1; - } - return 0; -} - - - - - #endif \ No newline at end of file diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 57e7961049..78ce9be30a 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -18,6 +18,11 @@ #define TSDB_DATA_SKIPLIST_LEVEL 5 +typedef struct { + STable * pTable; + SSkipListIterator *pIter; +} SCommitIter; + static FORCE_INLINE STsdbBufBlock *tsdbGetCurrBufBlock(STsdbRepo *pRepo); static void * tsdbAllocBytes(STsdbRepo *pRepo, int bytes); @@ -337,9 +342,12 @@ static void tsdbFreeTableData(STableData *pTableData) { static char *tsdbGetTsTupleKey(const void *data) { return dataRowTuple(data); } static void *tsdbCommitData(void *arg) { - STsdbRepo *pRepo = (STsdbRepo *)arg; - STsdbMeta *pMeta = pRepo->tsdbMeta; - SMemTable *pMem = pRepo->imem; + STsdbRepo * pRepo = (STsdbRepo *)arg; + STsdbMeta * pMeta = pRepo->tsdbMeta; + SMemTable * pMem = pRepo->imem; + STsdbCfg * pCfg = &pRepo->config; + SDataCols * pDataCols = NULL; + SCommitIter *iters = NULL; ASSERT(pRepo->commit == 1); ASSERT(pMem != NULL); @@ -347,69 +355,84 @@ static void *tsdbCommitData(void *arg) { pMem->keyFirst, pMem->keyLast, pMem->numOfRows); // Create the iterator to read from cache - SSkipListIterator **iters = tsdbCreateTableIters(pRepo); + iters = tsdbCreateTableIters(pRepo); if (iters == NULL) { - tsdbError("vgId:%d failed to create table iterators since %s", REPO_ID(pRepo), tstrerror(terrno)); - // TODO: deal with the error here - return NULL; + tsdbError("vgId:%d failed to create commit iterator since %s", REPO_ID(pRepo), tstrerror(terrno)); + goto _exit; } if (tsdbInitWriteHelper(&whelper, pRepo) < 0) { tsdbError("vgId:%d failed to init write helper since %s", REPO_ID(pRepo), tstrerror(terrno)); - // TODO goto _exit; } - if ((pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pCfg->maxRowsPerFileBlock)) == NULL) { - tsdbError("vgId:%d failed to init data cols with maxRowBytes %d maxCols %d since %s", REPO_ID(pRepo), - pMeta->maxRowBytes, pMeta->maxCols, tstrerror(terrno)); - // TODO + if ((pDataCols = tdNewDataCols(pMem->maxRowBytes, pMem->maxCols, pCfg->maxRowsPerFileBlock)) == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tsdbError("vgId:%d failed to init data cols with maxRowBytes %d maxCols %d maxRowsPerFileBlock %d since %s", + REPO_ID(pRepo), pMeta->maxRowBytes, pMeta->maxCols, pCfg->maxRowsPerFileBlock, tstrerror(terrno)); goto _exit; } - int sfid = tsdbGetKeyFileId(pCache->imem->keyFirst, pCfg->daysPerFile, pCfg->precision); - int efid = tsdbGetKeyFileId(pCache->imem->keyLast, pCfg->daysPerFile, pCfg->precision); + int sfid = TSDB_KEY_FILEID(pMem->keyFirst, pCfg->daysPerFile, pCfg->precision); + int efid = TSDB_KEY_FILEID(pMem->keyLast, pCfg->daysPerFile, pCfg->precision); // Loop to commit to each file for (int fid = sfid; fid <= efid; fid++) { if (tsdbCommitToFile(pRepo, fid, iters, &whelper, pDataCols) < 0) { - ASSERT(false); + tsdbError("vgId:%d failed to commit to file %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno)); goto _exit; } } - // Do retention actions + // TODO: Do retention actions tsdbFitRetention(pRepo); - if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_OVER); + + // TODO: Commit action meta data _exit: tdFreeDataCols(pDataCols); tsdbDestroyTableIters(iters, pCfg->maxTables); tsdbDestroyHelper(&whelper); - - tsdbLockRepo(arg); - tdListMove(pCache->imem->list, pCache->pool.memPool); - tsdbAdjustCacheBlocks(pCache); - tdListFree(pCache->imem->list); - free(pCache->imem); - pCache->imem = NULL; - pRepo->commit = 0; - for (int i = 1; i < pCfg->maxTables; i++) { - STable *pTable = pMeta->tables[i]; - if (pTable && pTable->imem) { - tsdbFreeMemTable(pTable->imem); - pTable->imem = NULL; - } - } - tsdbUnLockRepo(arg); - tsdbPrint("vgId:%d, commit over....", pRepo->config.tsdbId); + tsdbEndCommit(pRepo); + tsdbPrint("vgId:%d commit over", pRepo->config.tsdbId); return NULL; } -static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters, SRWHelper *pHelper, - SDataCols *pDataCols) { - char dataDir[128] = {0}; +static void tsdbEndCommit(STsdbRepo *pRepo) { + ASSERT(pRepo->commit == 1); + tsdbLockRepo(pRepo); + pRepo->commit = 0; + tsdbUnlockRepo(pRepo); + if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_OVER); +} + +static TSKEY tsdbNextIterKey(SCommitIter *pIter) { + if (pIter == NULL) return -1; + + SSkipListNode *node = tSkipListIterGet(pIter->pIter); + if (node == NULL) return -1; + + SDataRow row = SL_GET_NODE_DATA(node); + return dataRowKey(row); +} + +static int tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey) { + for (int i = 0; i < nIters; i++) { + TSKEY nextKey = tsdbNextIterKey(iters + i); + if (nextKey > 0 && (nextKey >= minKey && nextKey <= maxKey)) return 1; + } + return 0; +} + +static void tsdbGetKeyRangeOfFileId(int32_t daysPerFile, int8_t precision, int32_t fileId, TSKEY *minKey, + TSKEY *maxKey) { + *minKey = fileId * daysPerFile * tsMsPerDay[precision]; + *maxKey = *minKey + daysPerFile * tsMsPerDay[precision] - 1; +} + +static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHelper *pHelper, SDataCols *pDataCols) { + char * dataDir = NULL; STsdbMeta * pMeta = pRepo->tsdbMeta; STsdbFileH *pFileH = pRepo->tsdbFileH; STsdbCfg * pCfg = &pRepo->config; @@ -420,70 +443,84 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters // Check if there are data to commit to this file int hasDataToCommit = tsdbHasDataToCommit(iters, pCfg->maxTables, minKey, maxKey); - if (!hasDataToCommit) return 0; // No data to commit, just return + if (!hasDataToCommit) { + tsdbTrace("vgId:%d no data to commit to file %d", REPO_ID(pRepo), fid); + return 0; + } // Create and open files for commit - tsdbGetDataDirName(pRepo, dataDir); + dataDir = tsdbGetDataDirName(pRepo->rootDir); + if (dataDir == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + if ((pGroup = tsdbCreateFGroup(pFileH, dataDir, fid, pCfg->maxTables)) == NULL) { - tsdbError("vgId:%d, failed to create file group %d", pRepo->config.tsdbId, fid); + tsdbError("vgId:%d failed to create file group %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno)); goto _err; } + free(dataDir); + // Open files for write/read if (tsdbSetAndOpenHelperFile(pHelper, pGroup) < 0) { - tsdbError("vgId:%d, failed to set helper file", pRepo->config.tsdbId); + tsdbError("vgId:%d failed to set helper file since %s", REPO_ID(pRepo), tstrerror(terrno)); goto _err; } // Loop to commit data in each table for (int tid = 1; tid < pCfg->maxTables; tid++) { - STable *pTable = pMeta->tables[tid]; - if (pTable == NULL) continue; + SCommitIter *pIter = iters + tid; + if (pIter->pTable == NULL) continue; - SSkipListIterator *pIter = iters[tid]; - - // Set the helper and the buffer dataCols object to help to write this table tsdbSetHelperTable(pHelper, pTable, pRepo); - tdInitDataCols(pDataCols, tsdbGetTableSchema(pMeta, pTable)); - // Loop to write the data in the cache to files. If no data to write, just break the loop - int maxRowsToRead = pCfg->maxRowsPerFileBlock * 4 / 5; - int nLoop = 0; - while (true) { - int rowsRead = tsdbReadRowsFromCache(pMeta, pTable, pIter, maxKey, maxRowsToRead, pDataCols); - assert(rowsRead >= 0); - if (pDataCols->numOfRows == 0) break; - nLoop++; + if (pIter->pIter != NULL) { + tdInitDataCols(pDataCols, tsdbGetTableSchema(pIter->pTable)); - ASSERT(dataColsKeyFirst(pDataCols) >= minKey && dataColsKeyFirst(pDataCols) <= maxKey); - ASSERT(dataColsKeyLast(pDataCols) >= minKey && dataColsKeyLast(pDataCols) <= maxKey); + int maxRowsToRead = pCfg->maxRowsPerFileBlock * 4 / 5; + int nLoop = 0; + while (true) { + int rowsRead = tsdbReadRowsFromCache(pMeta, pIter->pTable, pIter->pIter, maxKey, maxRowsToRead, pDataCols); + ASSERT(rowsRead >= 0); + if (pDataCols->numOfRows == 0) break; + nLoop++; - int rowsWritten = tsdbWriteDataBlock(pHelper, pDataCols); - ASSERT(rowsWritten != 0); - if (rowsWritten < 0) goto _err; - ASSERT(rowsWritten <= pDataCols->numOfRows); + ASSERT(dataColsKeyFirst(pDataCols) >= minKey && dataColsKeyFirst(pDataCols) <= maxKey); + ASSERT(dataColsKeyLast(pDataCols) >= minKey && dataColsKeyLast(pDataCols) <= maxKey); - tdPopDataColsPoints(pDataCols, rowsWritten); - maxRowsToRead = pCfg->maxRowsPerFileBlock * 4 / 5 - pDataCols->numOfRows; + int rowsWritten = tsdbWriteDataBlock(pHelper, pDataCols); + ASSERT(rowsWritten != 0); + if (rowsWritten < 0) { + tsdbError("vgId:%d failed to write data block to table %s tid %d uid %" PRIu64 " since %s", REPO_ID(pRepo), + TABLE_CHAR_NAME(pIter->pTable), TABLE_TID(pIter->pTable), TABLE_UID(pIter->pTable), + tstrerror(terrno)); + goto _err; + } + ASSERT(rowsWritten <= pDataCols->numOfRows); + + tdPopDataColsPoints(pDataCols, rowsWritten); + maxRowsToRead = pCfg->maxRowsPerFileBlock * 4 / 5 - pDataCols->numOfRows; + } + + ASSERT(pDataCols->numOfRows == 0); } - ASSERT(pDataCols->numOfRows == 0); - // Move the last block to the new .l file if neccessary if (tsdbMoveLastBlockIfNeccessary(pHelper) < 0) { - tsdbError("vgId:%d, failed to move last block", pRepo->config.tsdbId); + tsdbError("vgId:%d, failed to move last block, since %s", REPO_ID(pRepo), tstrerror(terrno)); goto _err; } // Write the SCompBlock part if (tsdbWriteCompInfo(pHelper) < 0) { - tsdbError("vgId:%d, failed to write compInfo part", pRepo->config.tsdbId); + tsdbError("vgId:%d, failed to write compInfo part since %s", REPO_ID(pRepo), tstrerror(terrno)); goto _err; } } if (tsdbWriteCompIdx(pHelper) < 0) { - tsdbError("vgId:%d, failed to write compIdx part", pRepo->config.tsdbId); + tsdbError("vgId:%d failed to write compIdx part to file %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno)); goto _err; } @@ -496,43 +533,63 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SSkipListIterator **iters return 0; _err: - ASSERT(false); + // ASSERT(false); tsdbCloseHelperFile(pHelper, 1); return -1; } -static SSkipListIterator **tsdbCreateTableIters(STsdbRepo *pRepo) { - STsdbCfg *pCfg = &(pRepo->config); +static SCommitIter *tsdbCreateTableIters(STsdbRepo *pRepo) { + STsdbCfg * pCfg = &(pRepo->config); + SMemTable *pMem = pRepo->imem; + STsdbMeta *pMeta = pRepo->tsdbMeta; - SSkipListIterator **iters = (SSkipListIterator **)calloc(pCfg->maxTables, sizeof(SSkipListIterator *)); + SCommitIter *iters = (SCommitIter *)calloc(pCfg->maxTables, sizeof(SCommitIter)); if (iters == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return NULL; } - for (int tid = 1; tid < maxTables; tid++) { - STable *pTable = pMeta->tables[tid]; - if (pTable == NULL || pTable->imem == NULL || pTable->imem->numOfRows == 0) continue; + if (tsdbRLockRepoMeta(pRepo) < 0) goto _err; - iters[tid] = tSkipListCreateIter(pTable->imem->pData); - if (iters[tid] == NULL) goto _err; + // reference all tables + for (int i = 0; i < pCfg->maxTables; i++) { + if (pMeta->tables[i] != NULL) { + tsdbRefTable(pMeta->tables[i]); + iters[i].pTable = pMeta->tables[i]; + } + } - if (!tSkipListIterNext(iters[tid])) goto _err; + if (tsdbUnlockRepoMeta(pRepo) < 0) goto _err; + + for (int i = 0; i < pCfg->maxTables; i++) { + if ((iters[i].pTable != NULL) && (pMem->tData[i] != NULL) && (TALBE_UID(iters[i].pTable) == pMem->tData[i]->uid)) { + if ((iters[i].pIter = tSkipListCreateIter(pMem->tData[i]->pData)) == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + goto _err; + } + + if (!tSkipListIterNext(iters[i].pIter)) { + terrno = TSDB_CODE_TDB_NO_TABLE_DATA_IN_MEM; + goto _err; + } + } } return iters; _err: - tsdbDestroyTableIters(iters, maxTables); + tsdbDestroyTableIters(iters, pCfg->maxTables); return NULL; } -static void tsdbDestroyTableIters(SSkipListIterator **iters, int maxTables) { +static void tsdbDestroyTableIters(SCommitIter *iters, int maxTables) { if (iters == NULL) return; - for (int tid = 1; tid < maxTables; tid++) { - if (iters[tid] == NULL) continue; - tSkipListDestroyIter(iters[tid]); + for (int i = 1; i < maxTables; i++) { + if (iters[i].pTable != NULL) { + tsdbUnRefTable(iters[i].pTable); + tSkipListDestroyIter(iters[i].pIter); + } } free(iters); diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 10866bbae5..f96b0ae1da 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -516,6 +516,14 @@ int tsdbUnlockRepoMeta(STsdbRepo *pRepo) { return 0; } +void tsdbRefTable(STable *pTable) { T_REF_INC(pTable); } + +void tsdbUnRefTable(STable *pTable) { + if (T_REF_DEC(pTable) == 0) { + tsdbFreeTable(pTable); + } +} + // ------------------ LOCAL FUNCTIONS ------------------ static int tsdbCompareSchemaVersion(const void *key1, const void *key2) { if (*(int16_t *)key1 < schemaVersion(*(STSchema **)key2)) { From 209ebe5919f850440b452bef66da0000fc21a381 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 15 Jun 2020 08:38:47 +0000 Subject: [PATCH 21/65] TD-353 --- src/tsdb/inc/tsdbMain.h | 71 ++++++++++--------------------------- src/tsdb/src/tsdbMain.c | 24 ++++++++----- src/tsdb/src/tsdbMemTable.c | 15 ++++++-- src/tsdb/src/tsdbRWHelper.c | 16 ++------- 4 files changed, 50 insertions(+), 76 deletions(-) diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 70cd19d55a..3ebd892fe7 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -200,18 +200,6 @@ typedef struct { typedef enum { TSDB_WRITE_HELPER, TSDB_READ_HELPER } tsdb_rw_helper_t; -typedef struct { - tsdb_rw_helper_t type; // helper type - - int maxTables; - int maxRowSize; - int maxRows; - int maxCols; - int minRowsPerFileBlock; - int maxRowsPerFileBlock; - int8_t compress; -} SHelperCfg; - typedef struct { int fid; TSKEY minKey; @@ -232,26 +220,22 @@ typedef struct { } SHelperTable; typedef struct { - // Global configuration - SHelperCfg config; - - int8_t state; + tsdb_rw_helper_t type; + STsdbRepo* pRepo; + int8_t state; // For file set usage SHelperFile files; - SCompIdx * pCompIdx; - + SCompIdx* pCompIdx; // For table set usage SHelperTable tableInfo; - SCompInfo * pCompInfo; + SCompInfo* pCompInfo; bool hasOldLastBlock; - // For block set usage - SCompData *pCompData; - SDataCols *pDataCols[2]; - - void *pBuffer; // Buffer to hold the whole data block - void *compBuffer; // Buffer for temperary compress/decompress purpose + SCompData* pCompData[TSDB_MAX_SUBBLOCKS]; + SDataCols* pDataCols[2]; + void* pBuffer; // Buffer to hold the whole data block + void* compBuffer; // Buffer for temperary compress/decompress purpose } SRWHelper; // ------------------ tsdbMain.c @@ -323,9 +307,19 @@ STsdbFileH* tsdbNewFileH(STsdbCfg* pCfg); void tsdbFreeFileH(STsdbFileH* pFileH); // ------------------ tsdbRWHelper.c +#define TSDB_HELPER_CLEAR_STATE 0x0 // Clear state +#define TSDB_HELPER_FILE_SET_AND_OPEN 0x1 // File is set +#define TSDB_HELPER_IDX_LOAD 0x2 // SCompIdx part is loaded +#define TSDB_HELPER_TABLE_SET 0x4 // Table is set +#define TSDB_HELPER_INFO_LOAD 0x8 // SCompInfo part is loaded +#define TSDB_HELPER_FILE_DATA_LOAD 0x10 // SCompData part is loaded #define TSDB_MAX_SUBBLOCKS 8 #define IS_SUB_BLOCK(pBlock) ((pBlock)->numOfSubBlocks == 0) +#define helperType(h) (h)->type +#define helperRepo(h) (h)->pRepo +#define helperState(h) (h)->state + // ------------------ tsdbMain.c #define REPO_ID(r) (r)->config.tsdbId #define IS_REPO_LOCKED(r) (r)->repoLocked @@ -337,29 +331,7 @@ void* tsdbCommitData(void* arg); #if 0 - -// TSDB repository definition - -typedef struct { - int32_t totalLen; - int32_t len; - SDataRow row; -} SSubmitBlkIter; - -// SSubmitMsg Iterator -typedef struct { - int32_t totalLen; - int32_t len; - SSubmitBlk *pBlock; -} SSubmitMsgIter; - // --------- Helper state -#define TSDB_HELPER_CLEAR_STATE 0x0 // Clear state -#define TSDB_HELPER_FILE_SET_AND_OPEN 0x1 // File is set -#define TSDB_HELPER_IDX_LOAD 0x2 // SCompIdx part is loaded -#define TSDB_HELPER_TABLE_SET 0x4 // Table is set -#define TSDB_HELPER_INFO_LOAD 0x8 // SCompInfo part is loaded -#define TSDB_HELPER_FILE_DATA_LOAD 0x10 // SCompData part is loaded #define TSDB_HELPER_TYPE(h) ((h)->config.type) @@ -398,11 +370,6 @@ int tsdbAlterCacheTotalBlocks(STsdbRepo *pRepo, int totalBlocks); void tsdbAdjustCacheBlocks(STsdbCache *pCache); int32_t tsdbGetMetaFileName(char *rootDir, char *fname); int tsdbUpdateFileHeader(SFile *pFile, uint32_t version); -int tsdbUpdateTable(STsdbMeta *pMeta, STable *pTable, STableCfg *pCfg); -int tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable); -int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable); -STSchema *tsdbGetTableSchemaByVersion(STsdbMeta *pMeta, STable *pTable, int16_t version); -STSchema *tsdbGetTableSchema(STsdbMeta *pMeta, STable *pTable); int compFGroupKey(const void *key, const void *fgroup); diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 298caff0f6..e44f4838b0 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -30,6 +30,18 @@ #define TSDB_META_FILE_NAME "meta" #define TSDB_META_FILE_INDEX 10000000 +typedef struct { + int32_t totalLen; + int32_t len; + SDataRow row; +} SSubmitBlkIter; + +typedef struct { + int32_t totalLen; + int32_t len; + SSubmitBlk *pBlock; +} SSubmitMsgIter; + // Function declaration int32_t tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg) { if (mkdir(rootDir, 0755) < 0) { @@ -50,9 +62,7 @@ int32_t tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg) { return 0; } -int32_t tsdbDropRepo(char *rootDir) { - return tsdbUnsetRepoEnv(rootDir); -} +int32_t tsdbDropRepo(char *rootDir) { return tsdbUnsetRepoEnv(rootDir); } TSDB_REPO_T *tsdbOpenRepo(char *rootDir, STsdbAppH *pAppH) { STsdbCfg config = {0}; @@ -211,7 +221,6 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_ return magic; } - void tsdbStartStream(TSDB_REPO_T *repo) { STsdbRepo *pRepo = (STsdbRepo *)repo; STsdbMeta *pMeta = pRepo->tsdbMeta; @@ -653,8 +662,8 @@ static int32_t tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, TSKEY } // Check schema version - int32_t tversion = pBlock->sversion; - STSchema * pSchema = tsdbGetTableSchema(pMeta, pTable); + int32_t tversion = pBlock->sversion; + STSchema *pSchema = tsdbGetTableSchema(pMeta, pTable); ASSERT(pSchema != NULL); int16_t nversion = schemaVersion(pSchema); if (tversion > nversion) { @@ -688,7 +697,7 @@ static int32_t tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, TSKEY terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION; return -1; } - } + } SSubmitBlkIter blkIter = {0}; SDataRow row = NULL; @@ -753,7 +762,6 @@ static SDataRow tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter) { return row; } - static int tsdbRestoreInfo(STsdbRepo *pRepo) { STsdbMeta * pMeta = pRepo->tsdbMeta; STsdbFileH *pFileH = pRepo->tsdbFileH; diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 78ce9be30a..75bfe0d7ce 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -32,6 +32,16 @@ static void tsdbFreeMemTable(SMemTable *pMemTable); static STableData *tsdbNewTableData(STsdbCfg *pCfg, STable *pTable); static void tsdbFreeTableData(STableData *pTableData); static char * tsdbGetTsTupleKey(const void *data); +static void * tsdbCommitData(void *arg); +static void tsdbEndCommit(STsdbRepo *pRepo); +static TSKEY tsdbNextIterKey(SCommitIter *pIter); +static int tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey); +static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHelper *pHelper, SDataCols *pDataCols); +static void tsdbGetFidKeyRange(int daysPerFile, int8_t precision, int fileId, TSKEY *minKey, TSKEY *maxKey); +static SCommitIter *tsdbCreateTableIters(STsdbRepo *pRepo); +static void tsdbDestroyTableIters(SCommitIter *iters, int maxTables); +static int tsdbReadRowsFromCache(STsdbMeta *pMeta, STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, + int maxRowsToRead, SDataCols *pCols); // ---------------- INTERNAL FUNCTIONS ---------------- int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { @@ -425,8 +435,7 @@ static int tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSK return 0; } -static void tsdbGetKeyRangeOfFileId(int32_t daysPerFile, int8_t precision, int32_t fileId, TSKEY *minKey, - TSKEY *maxKey) { +static void tsdbGetFidKeyRange(int daysPerFile, int8_t precision, int fileId, TSKEY *minKey, TSKEY *maxKey) { *minKey = fileId * daysPerFile * tsMsPerDay[precision]; *maxKey = *minKey + daysPerFile * tsMsPerDay[precision] - 1; } @@ -439,7 +448,7 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe SFileGroup *pGroup = NULL; TSKEY minKey = 0, maxKey = 0; - tsdbGetKeyRangeOfFileId(pCfg->daysPerFile, pCfg->precision, fid, &minKey, &maxKey); + tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, fid, &minKey, &maxKey); // Check if there are data to commit to this file int hasDataToCommit = tsdbHasDataToCommit(iters, pCfg->maxTables, minKey, maxKey); diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index b091544722..83d9b99f55 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -1082,21 +1082,11 @@ static void tsdbDestroyHelperBlock(SRWHelper *pHelper) { } static int tsdbInitHelper(SRWHelper *pHelper, STsdbRepo *pRepo, tsdb_rw_helper_t type) { - if (pHelper == NULL || pRepo == NULL) return -1; - memset((void *)pHelper, 0, sizeof(*pHelper)); - // Init global configuration - pHelper->config.type = type; - pHelper->config.maxTables = pRepo->config.maxTables; - pHelper->config.maxRowSize = pRepo->tsdbMeta->maxRowBytes; - pHelper->config.maxRows = pRepo->config.maxRowsPerFileBlock; - pHelper->config.maxCols = pRepo->tsdbMeta->maxCols; - pHelper->config.minRowsPerFileBlock = pRepo->config.minRowsPerFileBlock; - pHelper->config.maxRowsPerFileBlock = pRepo->config.maxRowsPerFileBlock; - pHelper->config.compress = pRepo->config.compression; - - pHelper->state = TSDB_HELPER_CLEAR_STATE; + helperType(pHelper) = type; + helperRepo(pHelper) = pRepo; + helperState(pHelper) = TSDB_HELPER_CLEAR_STATE; // Init file part if (tsdbInitHelperFile(pHelper) < 0) goto _err; From 7ca58999eda45602be3b619e6dc73ef49b216f0a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 15 Jun 2020 09:35:07 +0000 Subject: [PATCH 22/65] TD-353 --- src/tsdb/inc/tsdbMain.h | 2 +- src/tsdb/src/tsdbRWHelper.c | 85 ++++++++++++++++++++++--------------- 2 files changed, 51 insertions(+), 36 deletions(-) diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 3ebd892fe7..78492eb832 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -150,7 +150,7 @@ typedef struct { typedef struct { uint32_t len; uint32_t offset; - uint32_t padding; + // uint32_t padding; uint32_t hasLast : 2; uint32_t numOfBlocks : 30; uint64_t uid; diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 83d9b99f55..0be6ae530a 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -498,36 +498,6 @@ _err: return -1; } -void *tsdbEncodeSCompIdx(void *buf, SCompIdx *pIdx) { - buf = taosEncodeVariantU32(buf, pIdx->len); - buf = taosEncodeVariantU32(buf, pIdx->offset); - buf = taosEncodeFixedU8(buf, pIdx->hasLast); - buf = taosEncodeVariantU32(buf, pIdx->numOfBlocks); - buf = taosEncodeFixedU64(buf, pIdx->uid); - buf = taosEncodeFixedU64(buf, pIdx->maxKey); - - return buf; -} - -void *tsdbDecodeSCompIdx(void *buf, SCompIdx *pIdx) { - uint8_t hasLast = 0; - uint32_t numOfBlocks = 0; - uint64_t value = 0; - - if ((buf = taosDecodeVariantU32(buf, &(pIdx->len))) == NULL) return NULL; - if ((buf = taosDecodeVariantU32(buf, &(pIdx->offset))) == NULL) return NULL; - if ((buf = taosDecodeFixedU8(buf, &(hasLast))) == NULL) return NULL; - pIdx->hasLast = hasLast; - if ((buf = taosDecodeVariantU32(buf, &(numOfBlocks))) == NULL) return NULL; - pIdx->numOfBlocks = numOfBlocks; - if ((buf = taosDecodeFixedU64(buf, &value)) == NULL) return NULL; - pIdx->uid = (int64_t)value; - if ((buf = taosDecodeFixedU64(buf, &value)) == NULL) return NULL; - pIdx->maxKey = (TSKEY)value; - - return buf; -} - int tsdbUpdateFileHeader(SFile *pFile, uint32_t version) { char buf[TSDB_FILE_HEAD_SIZE] = "\0"; @@ -583,15 +553,18 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa int64_t offset = 0; offset = lseek(pFile->fd, 0, SEEK_END); - if (offset < 0) goto _err; + if (offset < 0) { + tsdbError("vgId:%d failed to write block to file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } int nColsNotAllNull = 0; for (int ncol = 0; ncol < pDataCols->numOfCols; ncol++) { SDataCol *pDataCol = pDataCols->cols + ncol; SCompCol *pCompCol = pCompData->cols + nColsNotAllNull; - if (isNEleNull(pDataCol, rowsToWrite)) { - // all data to commit are NULL, just ignore it + if (isNEleNull(pDataCol, rowsToWrite)) { // all data to commit are NULL, just ignore it continue; } @@ -630,7 +603,10 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa if (pHelper->config.compress) { if (pHelper->config.compress == TWO_STAGE_COMP) { pHelper->compBuffer = trealloc(pHelper->compBuffer, tlen + COMP_OVERFLOW_BYTES); - if (pHelper->compBuffer == NULL) goto _err; + if (pHelper->compBuffer == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + goto _err; + } } pCompCol->len = (*(tDataTypeDesc[pDataCol->type].compFunc))( @@ -657,7 +633,11 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa taosCalcChecksumAppend(0, (uint8_t *)pCompData, tsize); // Write the whole block to file - if (twrite(pFile->fd, (void *)pCompData, lsize) < lsize) goto _err; + if (twrite(pFile->fd, (void *)pCompData, lsize) < lsize) { + tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(helperRepo(pHelper)), lsize, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } // Update pCompBlock membership vairables pCompBlock->last = isLast; @@ -671,6 +651,11 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa pCompBlock->keyFirst = dataColsKeyFirst(pDataCols); pCompBlock->keyLast = dataColsKeyAt(pDataCols, rowsToWrite - 1); + tsdbTrace("vgId:%d tid:%d a block of data is written to file %s, offset %" PRId64 + " numOfRows %d len %d numOfCols %" PRId16 " keyFirst %" PRId64 " keyLast %" PRId64, + REPO_ID(helperRepo(pHelper)), pHelper->tableInfo.tid, pFile->fname, pCompBlock->offset, + pCompBlock->numOfRows, pCompBlock->len, pCompBlock->numOfCols, pCompBlock->keyFirst, pCompBlock->keyLast); + return 0; _err: @@ -1235,3 +1220,33 @@ static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDa _err: return -1; } + +static void *tsdbEncodeSCompIdx(void *buf, SCompIdx *pIdx) { + buf = taosEncodeVariantU32(buf, pIdx->len); + buf = taosEncodeVariantU32(buf, pIdx->offset); + buf = taosEncodeFixedU8(buf, pIdx->hasLast); + buf = taosEncodeVariantU32(buf, pIdx->numOfBlocks); + buf = taosEncodeFixedU64(buf, pIdx->uid); + buf = taosEncodeFixedU64(buf, pIdx->maxKey); + + return buf; +} + +static void *tsdbDecodeSCompIdx(void *buf, SCompIdx *pIdx) { + uint8_t hasLast = 0; + uint32_t numOfBlocks = 0; + uint64_t value = 0; + + if ((buf = taosDecodeVariantU32(buf, &(pIdx->len))) == NULL) return NULL; + if ((buf = taosDecodeVariantU32(buf, &(pIdx->offset))) == NULL) return NULL; + if ((buf = taosDecodeFixedU8(buf, &(hasLast))) == NULL) return NULL; + pIdx->hasLast = hasLast; + if ((buf = taosDecodeVariantU32(buf, &(numOfBlocks))) == NULL) return NULL; + pIdx->numOfBlocks = numOfBlocks; + if ((buf = taosDecodeFixedU64(buf, &value)) == NULL) return NULL; + pIdx->uid = (int64_t)value; + if ((buf = taosDecodeFixedU64(buf, &value)) == NULL) return NULL; + pIdx->maxKey = (TSKEY)value; + + return buf; +} From d4e84b4fb29d7788439285ec6f2d7f1427a05a69 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 15 Jun 2020 11:08:17 +0000 Subject: [PATCH 23/65] TD-353 --- src/tsdb/inc/tsdbMain.h | 30 ++++---- src/tsdb/src/tsdbRWHelper.c | 141 ++++++++++++++++++++++++++++-------- 2 files changed, 127 insertions(+), 44 deletions(-) diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 78492eb832..34b8b99b5b 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -115,15 +115,18 @@ typedef struct { typedef enum { TSDB_FILE_TYPE_HEAD = 0, TSDB_FILE_TYPE_DATA, TSDB_FILE_TYPE_LAST, TSDB_FILE_TYPE_MAX } TSDB_FILE_TYPE; typedef struct { + uint32_t offset; + uint32_t len; + uint64_t size; // total size of the file + uint64_t tombSize; // unused file size + uint32_t totalBlocks; + uint32_t totalSubBlocks; } STsdbFileInfo; typedef struct { - char* fname; - int fd; - uint64_t size; - uint64_t tombSize; - uint64_t totalBlocks; - uint64_t totalSubBlocks; + char* fname; + int fd; + STsdbFileInfo info; } SFile; typedef struct { @@ -201,7 +204,7 @@ typedef struct { typedef enum { TSDB_WRITE_HELPER, TSDB_READ_HELPER } tsdb_rw_helper_t; typedef struct { - int fid; + int fid; TSKEY minKey; TSKEY maxKey; // For read/write purpose @@ -232,7 +235,7 @@ typedef struct { SCompInfo* pCompInfo; bool hasOldLastBlock; // For block set usage - SCompData* pCompData[TSDB_MAX_SUBBLOCKS]; + SCompData* pCompData; SDataCols* pDataCols[2]; void* pBuffer; // Buffer to hold the whole data block void* compBuffer; // Buffer for temperary compress/decompress purpose @@ -313,9 +316,12 @@ void tsdbFreeFileH(STsdbFileH* pFileH); #define TSDB_HELPER_TABLE_SET 0x4 // Table is set #define TSDB_HELPER_INFO_LOAD 0x8 // SCompInfo part is loaded #define TSDB_HELPER_FILE_DATA_LOAD 0x10 // SCompData part is loaded +#define helperSetState(h, s) (((h)->state) |= (s)) +#define helperClearState(h, s) ((h)->state &= (~(s))) +#define helperHasState(h, s) ((((h)->state) & (s)) == (s)) +#define blockAtIdx(h, idx) ((h)->pCompInfo->blocks + idx) #define TSDB_MAX_SUBBLOCKS 8 #define IS_SUB_BLOCK(pBlock) ((pBlock)->numOfSubBlocks == 0) - #define helperType(h) (h)->type #define helperRepo(h) (h)->pRepo #define helperState(h) (h)->state @@ -333,12 +339,6 @@ void* tsdbCommitData(void* arg); // --------- Helper state -#define TSDB_HELPER_TYPE(h) ((h)->config.type) - -#define helperSetState(h, s) (((h)->state) |= (s)) -#define helperClearState(h, s) ((h)->state &= (~(s))) -#define helperHasState(h, s) ((((h)->state) & (s)) == (s)) -#define blockAtIdx(h, idx) ((h)->pCompInfo->blocks + idx) int tsdbInitReadHelper(SRWHelper *pHelper, STsdbRepo *pRepo); int tsdbInitWriteHelper(SRWHelper *pHelper, STsdbRepo *pRepo); diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 0be6ae530a..d1a41ce550 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -71,7 +71,10 @@ int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { pHelper->files.lastF = pGroup->files[TSDB_FILE_TYPE_LAST]; if (TSDB_HELPER_TYPE(pHelper) == TSDB_WRITE_HELPER) { char *fnameDup = strdup(pHelper->files.headF.fname); - if (fnameDup == NULL) return -1; + if (fnameDup == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } char *dataDir = dirname(fnameDup); tsdbGetFileName(dataDir, pHelper->files.fid, ".h", pHelper->files.nHeadF.fname); @@ -81,21 +84,28 @@ int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { // Open the files if (tsdbOpenFile(&(pHelper->files.headF), O_RDONLY) < 0) goto _err; - if (TSDB_HELPER_TYPE(pHelper) == TSDB_WRITE_HELPER) { + if (helperType(pHelper) == TSDB_WRITE_HELPER) { if (tsdbOpenFile(&(pHelper->files.dataF), O_RDWR) < 0) goto _err; if (tsdbOpenFile(&(pHelper->files.lastF), O_RDWR) < 0) goto _err; // Create and open .h if (tsdbOpenFile(&(pHelper->files.nHeadF), O_WRONLY | O_CREAT) < 0) return -1; // size_t tsize = TSDB_FILE_HEAD_SIZE + sizeof(SCompIdx) * pHelper->config.maxTables + sizeof(TSCKSUM); - if (tsendfile(pHelper->files.nHeadF.fd, pHelper->files.headF.fd, NULL, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) + if (tsendfile(pHelper->files.nHeadF.fd, pHelper->files.headF.fd, NULL, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { + tsdbError("vgId:%d failed to sendfile %d bytes from file %s to %s since %s", REPO_ID(pHelper->pRepo), + TSDB_FILE_HEAD_SIZE, hpHelper->files.headF.fname, pHelper->files.nHeadF.fname, strerror(errno)); + errno = TAOS_SYSTEM_ERROR(errno); goto _err; + } // Create and open .l file if should if (tsdbShouldCreateNewLast(pHelper)) { if (tsdbOpenFile(&(pHelper->files.nLastF), O_WRONLY | O_CREAT) < 0) goto _err; if (tsendfile(pHelper->files.nLastF.fd, pHelper->files.lastF.fd, NULL, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) - goto _err; + tsdbError("vgId:%d failed to sendfile %d bytes from file %s to %s since %s", REPO_ID(pHelper->pRepo), + TSDB_FILE_HEAD_SIZE, hpHelper->files.lastF.fname, pHelper->files.nLastF.fname, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; } } else { if (tsdbOpenFile(&(pHelper->files.dataF), O_RDONLY) < 0) goto _err; @@ -164,7 +174,7 @@ void tsdbSetHelperTable(SRWHelper *pHelper, STable *pTable, STsdbRepo *pRepo) { pHelper->tableInfo.tid = pTable->tableId.tid; pHelper->tableInfo.uid = pTable->tableId.uid; - STSchema *pSchema = tsdbGetTableSchema(pRepo->tsdbMeta, pTable); + STSchema *pSchema = tsdbGetTableSchema(pTable); pHelper->tableInfo.sversion = schemaVersion(pSchema); tdInitDataCols(pHelper->pDataCols[0], pSchema); @@ -354,28 +364,45 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { if (pFile->info.offset > 0) { ASSERT(pFile->info.offset > TSDB_FILE_HEAD_SIZE); - if (lseek(fd, pFile->info.offset, SEEK_SET) < 0) return -1; - if ((pHelper->pBuffer = trealloc(pHelper->pBuffer, pFile->info.len)) == NULL) return -1; - if (tread(fd, (void *)(pHelper->pBuffer), pFile->info.len) < pFile->info.len) return -1; + if (lseek(fd, pFile->info.offset, SEEK_SET) < 0) { + tsdbError("vgId:%d failed to lseek file %s to %u since %s", REPO_ID(pHelper->pRepo), pFile->fname, + pFile->info.offset, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + if ((pHelper->pBuffer = trealloc(pHelper->pBuffer, pFile->info.len)) == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + if (tread(fd, (void *)(pHelper->pBuffer), pFile->info.len) < pFile->info.len) { + tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pHelper->pRepo), pFile->info.len, + pFile->fname, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } if (!taosCheckChecksumWhole((uint8_t *)(pHelper->pBuffer), pFile->info.len)) { - // TODO: File is broken, try to deal with it + tsdbError("vgId:%d file %s SCompIdx part is corrupted. offset %u len %u", REPO_ID(pHelper->pRepo), pFile->fname, + pFile->info.offset, pFile->info.len); + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; return -1; } // Decode it void *ptr = pHelper->pBuffer; - while (((char *)ptr - (char *)pHelper->pBuffer) < (pFile->info.len - sizeof(TSCKSUM))) { + while (POINTER_DISTANCE(ptr, pHelper->pBuffer) < (pFile->info.len - sizeof(TSCKSUM))) { uint32_t tid = 0; if ((ptr = taosDecodeVariantU32(ptr, &tid)) == NULL) return -1; ASSERT(tid > 0 && tid < pHelper->config.maxTables); if ((ptr = tsdbDecodeSCompIdx(ptr, pHelper->pCompIdx + tid)) == NULL) return -1; - ASSERT((char *)ptr - (char *)pHelper->pBuffer <= pFile->info.len - sizeof(TSCKSUM)); + ASSERT(POINTER_DISTANCE(ptr, pHelper->pBuffer) <= pFile->info.len - sizeof(TSCKSUM)); } - ASSERT(((char *)ptr - (char *)pHelper->pBuffer) == (pFile->info.len - sizeof(TSCKSUM))); - if (lseek(fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) return -1; + if (lseek(fd, TSDB_FILE_HEAD_SIZE, SEEK_SET) < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } } } helperSetState(pHelper, TSDB_HELPER_IDX_LOAD); @@ -507,8 +534,16 @@ int tsdbUpdateFileHeader(SFile *pFile, uint32_t version) { taosCalcChecksumAppend(0, (uint8_t *)buf, TSDB_FILE_HEAD_SIZE); - if (lseek(pFile->fd, 0, SEEK_SET) < 0) return -1; - if (twrite(pFile->fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) return -1; + if (lseek(pFile->fd, 0, SEEK_SET) < 0) { + tsdbError("failed to lseek file %s since %s", pFile->fname, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + if (twrite(pFile->fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { + tsdbError("failed to write %d bytes to file %s since %s", TSDB_FILE_HEAD_SIZE, pFile->fname, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } return 0; } @@ -851,6 +886,9 @@ static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int ASSERT(pHelper->pCompInfo->blocks[0].keyLast < pHelper->pCompInfo->blocks[1].keyFirst); } + tsdbTrace("vgId:%d tid:%d a super block is inserted at index %d", REPO_ID(pHelper->pRepo), pHelper->tableInfo.tid, + blkIdx); + return 0; _err: @@ -933,6 +971,8 @@ static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkId pIdx->maxKey = pHelper->pCompInfo->blocks[pIdx->numOfBlocks - 1].keyLast; pIdx->hasLast = pHelper->pCompInfo->blocks[pIdx->numOfBlocks - 1].last; + tsdbTrace("vgId:%d tid:%d a subblock is added at index %d", REPO_ID(pHelper->pRepo), pHelper->tableInfo.tid, blkIdx); + return 0; _err: @@ -971,6 +1011,9 @@ static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int pIdx->maxKey = pHelper->pCompInfo->blocks[pIdx->numOfBlocks - 1].keyLast; pIdx->hasLast = pHelper->pCompInfo->blocks[pIdx->numOfBlocks - 1].last; + tsdbTrace("vgId:%d tid:%d a super block is updated at index %d", REPO_ID(pHelper->pRepo), pHelper->tableInfo.tid, + blkIdx); + return 0; } @@ -1001,18 +1044,26 @@ static int tsdbGetRowsInRange(SDataCols *pDataCols, TSKEY minKey, TSKEY maxKey) static void tsdbResetHelperFileImpl(SRWHelper *pHelper) { memset((void *)&pHelper->files, 0, sizeof(pHelper->files)); pHelper->files.fid = -1; + tfree(pHelper->files.headF.fname); pHelper->files.headF.fd = -1; + tfree(pHelper->files.dataF.fname); pHelper->files.dataF.fd = -1; + tfree(pHelper->files.lastF.fname); pHelper->files.lastF.fd = -1; + tfree(pHelper->files.nHeadF.fname); pHelper->files.nHeadF.fd = -1; + tfree(pHelper->files.nLastF.fname); pHelper->files.nLastF.fd = -1; } static int tsdbInitHelperFile(SRWHelper *pHelper) { - // pHelper->compIdxSize = sizeof(SCompIdx) * pHelper->config.maxTables + sizeof(TSCKSUM); - size_t tsize = sizeof(SCompIdx) * pHelper->config.maxTables + sizeof(TSCKSUM); + STsdbCfg *pCfg = &pHelper->pRepo->config; + size_t tsize = sizeof(SCompIdx) * pCfg->maxTables + sizeof(TSCKSUM); pHelper->pCompIdx = (SCompIdx *)tmalloc(tsize); - if (pHelper->pCompIdx == NULL) return -1; + if (pHelper->pCompIdx == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } tsdbResetHelperFileImpl(pHelper); return 0; @@ -1020,6 +1071,7 @@ static int tsdbInitHelperFile(SRWHelper *pHelper) { static void tsdbDestroyHelperFile(SRWHelper *pHelper) { tsdbCloseHelperFile(pHelper, false); + tsdbResetHelperFileImpl(pHelper); tzfree(pHelper->pCompIdx); } @@ -1051,9 +1103,16 @@ static void tsdbResetHelperBlock(SRWHelper *pHelper) { } static int tsdbInitHelperBlock(SRWHelper *pHelper) { - pHelper->pDataCols[0] = tdNewDataCols(pHelper->config.maxRowSize, pHelper->config.maxCols, pHelper->config.maxRows); - pHelper->pDataCols[1] = tdNewDataCols(pHelper->config.maxRowSize, pHelper->config.maxCols, pHelper->config.maxRows); - if (pHelper->pDataCols[0] == NULL || pHelper->pDataCols[1] == NULL) return -1; + STsdbRepo *pRepo = helperRepo(pHelper); + + pHelper->pDataCols[0] = + tdNewDataCols(pRepo->imem->maxRowBytes, pRepo->imem->maxCols, pRepo->config.maxRowsPerFileBlock); + pHelper->pDataCols[1] = + tdNewDataCols(pRepo->imem->maxRowBytes, pRepo->imem->maxCols, pRepo->config.maxRowsPerFileBlock); + if (pHelper->pDataCols[0] == NULL || pHelper->pDataCols[1] == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } tsdbResetHelperBlockImpl(pHelper); @@ -1067,6 +1126,7 @@ static void tsdbDestroyHelperBlock(SRWHelper *pHelper) { } static int tsdbInitHelper(SRWHelper *pHelper, STsdbRepo *pRepo, tsdb_rw_helper_t type) { + STsdbCfg *pCfg = &pRepo->config; memset((void *)pHelper, 0, sizeof(*pHelper)); helperType(pHelper) = type; @@ -1083,9 +1143,12 @@ static int tsdbInitHelper(SRWHelper *pHelper, STsdbRepo *pRepo, tsdb_rw_helper_t if (tsdbInitHelperBlock(pHelper) < 0) goto _err; pHelper->pBuffer = - tmalloc(sizeof(SCompData) + (sizeof(SCompCol) + sizeof(TSCKSUM) + COMP_OVERFLOW_BYTES) * pHelper->config.maxCols + - pHelper->config.maxRowSize * pHelper->config.maxRowsPerFileBlock + sizeof(TSCKSUM)); - if (pHelper->pBuffer == NULL) goto _err; + tmalloc(sizeof(SCompData) + (sizeof(SCompCol) + sizeof(TSCKSUM) + COMP_OVERFLOW_BYTES) * pRepo->imem->maxCols + + pRepo->imem->maxRowBytes * pCfg->maxRowsPerFileBlock + sizeof(TSCKSUM)); + if (pHelper->pBuffer == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + goto _err; + } return 0; @@ -1167,13 +1230,30 @@ static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDa SCompData *pCompData = (SCompData *)pHelper->pBuffer; - int fd = (pCompBlock->last) ? pHelper->files.lastF.fd : pHelper->files.dataF.fd; - if (lseek(fd, pCompBlock->offset, SEEK_SET) < 0) goto _err; - if (tread(fd, (void *)pCompData, pCompBlock->len) < pCompBlock->len) goto _err; + SFile *pFile = (pCompBlock->last) ? &(pHelper->files.lastF) : &(pHelper->files.dataF); + + int fd = pFile->fd; + if (lseek(fd, pCompBlock->offset, SEEK_SET) < 0) { + tsdbError("vgId:%d tid:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pHelper->tableInfo.tid, + pFile->fname, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + if (tread(fd, (void *)pCompData, pCompBlock->len) < pCompBlock->len) { + tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pHelper->pRepo), pCompBlock->len, + pFile->fname, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } ASSERT(pCompData->numOfCols == pCompBlock->numOfCols); int32_t tsize = sizeof(SCompData) + sizeof(SCompCol) * pCompBlock->numOfCols + sizeof(TSCKSUM); - if (!taosCheckChecksumWhole((uint8_t *)pCompData, tsize)) goto _err; + if (!taosCheckChecksumWhole((uint8_t *)pCompData, tsize)) { + tsdbError("vgId:%d file %s block data is corrupted offset %" PRId64 " len %d", REPO_ID(pHelper->pRepo), + pFile->fname, pCompBlock->offset, pCompBlock->len); + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + goto _err; + } pDataCols->numOfRows = pCompBlock->numOfRows; @@ -1198,7 +1278,10 @@ static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDa zsize += (sizeof(VarDataLenT) * pCompBlock->numOfRows); } pHelper->compBuffer = trealloc(pHelper->compBuffer, zsize); - if (pHelper->compBuffer == NULL) goto _err; + if (pHelper->compBuffer == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + goto _err; + } } if (tsdbCheckAndDecodeColumnData(pDataCol, (char *)pCompData + tsize + pCompCol->offset, pCompCol->len, pCompBlock->algorithm, pCompBlock->numOfRows, pDataCols->maxPoints, From 44d5644c358f4db2d1433c757478506f6746dabc Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 15 Jun 2020 11:16:32 +0000 Subject: [PATCH 24/65] TD-353 --- src/common/src/tdataformat.c | 3 +- src/inc/taosdef.h | 2 +- src/tsdb/inc/tsdbMain.h | 71 ++++++++++++++++++------------------ src/tsdb/src/tsdbBuffer.c | 7 ++-- 4 files changed, 41 insertions(+), 42 deletions(-) diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index d8b67e1105..77af404eda 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -42,7 +42,7 @@ void *tdEncodeSchema(void *buf, STSchema *pSchema) { STColumn *pCol = schemaColAt(pSchema, i); buf = taosEncodeFixedI8(buf, colType(pCol)); buf = taosEncodeFixedI16(buf, colColId(pCol)); - buf = taosEncodeFixedI32(buf, colBytes(pCol)) : + buf = taosEncodeFixedI32(buf, colBytes(pCol)); } return buf; @@ -54,6 +54,7 @@ void *tdEncodeSchema(void *buf, STSchema *pSchema) { void *tdDecodeSchema(void *buf, STSchema **pRSchema) { int version = 0; int numOfCols = 0; + STSchemaBuilder schemaBuilder; buf = taosDecodeFixedI32(buf, &version); buf = taosDecodeFixedI32(buf, &numOfCols); diff --git a/src/inc/taosdef.h b/src/inc/taosdef.h index f84a7581e4..7490de90d0 100644 --- a/src/inc/taosdef.h +++ b/src/inc/taosdef.h @@ -362,7 +362,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size); #define TAOS_QTYPE_WAL 2 #define TAOS_QTYPE_CQ 3 -typedef enum : uint8_t{ +typedef enum { TSDB_SUPER_TABLE = 0, // super table TSDB_CHILD_TABLE = 1, // table created from super table TSDB_NORMAL_TABLE = 2, // ordinary table diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 34b8b99b5b..c000ac7e79 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -40,26 +40,25 @@ extern int tsdbDebugFlag; #define TSDB_MAX_TABLE_SCHEMAS 16 #define TSDB_FILE_HEAD_SIZE 512 #define TSDB_FILE_DELIMITER 0xF00AFA0F -#define // Definitions // ------------------ tsdbMeta.c typedef struct STable { - ETableType type; - tstr* name; // NOTE: there a flexible string here - STableId tableId; - uint64_t suid; - STable* pSuper; // super table pointer - uint8_t numOfSchemas; - STSchema schema[TSDB_MAX_TABLE_SCHEMAS]; - STSchema* tagSchema; - SKVRow tagVal; - void* pIndex; // For TSDB_SUPER_TABLE, it is the skiplist index - void* eventHandler; // TODO - void* streamHandler; // TODO - TSKEY lastKey; // lastkey inserted in this table, initialized as 0, TODO: make a structure - char* sql; - void* cqhandle; + ETableType type; + tstr* name; // NOTE: there a flexible string here + STableId tableId; + uint64_t suid; + struct STable* pSuper; // super table pointer + uint8_t numOfSchemas; + STSchema schema[TSDB_MAX_TABLE_SCHEMAS]; + STSchema* tagSchema; + SKVRow tagVal; + void* pIndex; // For TSDB_SUPER_TABLE, it is the skiplist index + void* eventHandler; // TODO + void* streamHandler; // TODO + TSKEY lastKey; // lastkey inserted in this table, initialized as 0, TODO: make a structure + char* sql; + void* cqhandle; T_REF_DECLARE(); } STable; @@ -149,6 +148,25 @@ typedef struct { int direction; } SFileGroupIter; +// ------------------ tsdbMain.c +typedef struct { + int8_t state; + + char* rootDir; + STsdbCfg config; + STsdbAppH appH; + STsdbStat stat; + STsdbMeta* tsdbMeta; + STsdbBufPool* pPool; + SMemTable* mem; + SMemTable* imem; + STsdbFileH* tsdbFileH; + int commit; + pthread_t commitThread; + pthread_mutex_t mutex; + bool repoLocked; +} STsdbRepo; + // ------------------ tsdbRWHelper.c typedef struct { uint32_t len; @@ -241,24 +259,6 @@ typedef struct { void* compBuffer; // Buffer for temperary compress/decompress purpose } SRWHelper; -// ------------------ tsdbMain.c -typedef struct { - int8_t state; - - char* rootDir; - STsdbCfg config; - STsdbAppH appH; - STsdbStat stat; - STsdbMeta* tsdbMeta; - STsdbBufPool* pPool; - SMemTable* mem; - SMemTable* imem; - STsdbFileH* tsdbFileH; - int commit; - pthread_t commitThread; - pthread_mutex_t mutex; - bool repoLocked; -} STsdbRepo; // Operations // ------------------ tsdbMeta.c @@ -289,7 +289,7 @@ void tsdbUnRefTable(STable* pTable); STsdbBufPool* tsdbNewBufPool(); void tsdbFreeBufPool(STsdbBufPool* pBufPool); int tsdbOpenBufPool(STsdbRepo* pRepo); -int tsdbOpenBufPool(STsdbRepo* pRepo); +void tsdbCloseBufPool(STsdbRepo* pRepo); SListNode* tsdbAllocBufBlockFromPool(STsdbRepo* pRepo); // ------------------ tsdbMemTable.c @@ -333,7 +333,6 @@ void tsdbFreeFileH(STsdbFileH* pFileH); char* tsdbGetMetaFileName(char* rootDir); int tsdbLockRepo(STsdbRepo* pRepo); int tsdbUnlockRepo(STsdbRepo* pRepo); -void* tsdbCommitData(void* arg); #if 0 diff --git a/src/tsdb/src/tsdbBuffer.c b/src/tsdb/src/tsdbBuffer.c index 70f8e054f8..95136454a0 100644 --- a/src/tsdb/src/tsdbBuffer.c +++ b/src/tsdb/src/tsdbBuffer.c @@ -86,7 +86,7 @@ int tsdbOpenBufPool(STsdbRepo *pRepo) { } tsdbTrace("vgId:%d buffer pool is opened! bufBlockSize:%d tBufBlocks:%d nBufBlocks:%d", REPO_ID(pRepo), - pBufPool->bufBlockSize, pBufPool->tBufBlocks, pBufPool->nBufBlocks); + pPool->bufBlockSize, pPool->tBufBlocks, pPool->nBufBlocks); return 0; @@ -113,11 +113,10 @@ void tsdbCloseBufPool(STsdbRepo *pRepo) { } SListNode *tsdbAllocBufBlockFromPool(STsdbRepo *pRepo) { - ASSERT(pRepo != NULL && pRepo->pool != NULL); + ASSERT(pRepo != NULL && pRepo->pPool != NULL); ASSERT(IS_REPO_LOCKED(pRepo)); - STsdbCfg * pCfg = &pRepo->config; - STsdbBufPool *pBufPool = pRepo->pool; + STsdbBufPool *pBufPool = pRepo->pPool; while (POOL_IS_EMPTY(pBufPool)) { pthread_cond_wait(&(pBufPool->poolNotEmpty), &(pRepo->mutex)); From 32dcbfe52ad665a232d19f141928e76251a16994 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 16 Jun 2020 02:59:18 +0000 Subject: [PATCH 25/65] TD-353 --- src/tsdb/src/tsdbFile.c | 66 ++++++++++++++++++++++------------------- src/tsdb/src/tsdbMain.c | 2 +- 2 files changed, 37 insertions(+), 31 deletions(-) diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 77e07b54c4..ce34cbc08c 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -30,20 +30,22 @@ #include "ttime.h" const char *tsdbFileSuffix[] = { - ".head", // TSDB_FILE_TYPE_HEAD - ".data", // TSDB_FILE_TYPE_DATA - ".last" // TSDB_FILE_TYPE_LAST + ".head", + ".data", + ".last", + ".h", + ".l" }; // ---------------- INTERNAL FUNCTIONS ---------------- -STsdbFileH* tsdbNewFileH(STsdbCfg* pCfg) { +STsdbFileH *tsdbNewFileH(STsdbCfg *pCfg) { STsdbFileH *pFileH = (STsdbFileH *)calloc(1, sizeof(*pFileH)); if (pFileH == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _err; } - pFileH->maxFGroups = pCfg->keep / pCfg->daysPerFile + 3; + pFileH->maxFGroups = TSDB_MAX_FILE(pCfg->keep, pCfg->daysPerFile); pFileH->pFGroup = (SFileGroup *)calloc(pFileH->maxFGroups, sizeof(SFileGroup)); if (pFileH->pFGroup == NULL) { @@ -53,51 +55,55 @@ STsdbFileH* tsdbNewFileH(STsdbCfg* pCfg) { return pFileH; - _err: +_err: tsdbFreeFileH(pFileH); return NULL; } -void tsdbFreeFileH(STsdbFileH* pFileH) { +void tsdbFreeFileH(STsdbFileH *pFileH) { if (pFileH) { tfree(pFileH->pFGroup); free(pFileH); } } -STsdbFileH *tsdbInitFileH(char *dataDir, STsdbCfg *pCfg) { - STsdbFileH *pFileH = (STsdbFileH *)calloc(1, sizeof(STsdbFileH)); - if (pFileH == NULL) { // TODO: deal with ERROR here - return NULL; +int *tsdbOpenFileH(STsdbRepo *pRepo) { + ASSERT(pRepo != NULL && pRepo->tsdbFileH != NULL); + + char *tDataDir = NULL; + DIR * dir = NULL; + int fid = 0; + + tsdbGetDataDirName(pRepo->rootDir); + if (tDataDir == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + goto _err; } - pFileH->maxFGroups = pCfg->keep / pCfg->daysPerFile + 3; - - pFileH->fGroup = (SFileGroup *)calloc(pFileH->maxFGroups, sizeof(SFileGroup)); - if (pFileH->fGroup == NULL) { - free(pFileH); - return NULL; - } - - DIR *dir = opendir(dataDir); + DIR *dir = opendir(tDataDir); if (dir == NULL) { - free(pFileH); - return NULL; + tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), tDataDir, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; } struct dirent *dp = NULL; while ((dp = readdir(dir)) != NULL) { - if (strncmp(dp->d_name, ".", 1) == 0 || strncmp(dp->d_name, "..", 1) == 0) continue; - int fid = 0; + if (strncmp(dp->d_name, ".", 1) == 0 || strncmp(dp->d_name, "..", 2) == 0) continue; sscanf(dp->d_name, "f%d", &fid); - if (tsdbOpenFGroup(pFileH, dataDir, fid) < 0) { - break; - // TODO - } + // if (tsdbOpenFGroup(pFileH, dataDir, fid) < 0) { + // break; + // } } - closedir(dir); - return pFileH; + tfree(tDataDir); + closedir(dir); + return 0; + +_err: + tfree(tDataDir); + if (dir != NULL) closedir(tDataDir); + return -1; } void tsdbCloseFileH(STsdbFileH *pFileH) { diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index e44f4838b0..c459f766d9 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -595,7 +595,7 @@ static STsdbRepo *tsdbNewRepo(char *rootDir, STsdbAppH *pAppH, STsdbCfg *pCfg) { goto _err; } - pRepo->tsdbFileH = tsdbNewFileH(pRepo); + pRepo->tsdbFileH = tsdbNewFileH(pCfg); if (pRepo->tsdbFileH == NULL) { tsdbError("vgId:%d failed to create file handle since %s", REPO_ID(pRepo), tstrerror(terrno)); goto _err; From ab64511bea9d93911f72bfeb1bec7d12f85fb9df Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 16 Jun 2020 09:12:21 +0000 Subject: [PATCH 26/65] TD-353 --- src/tsdb/inc/tsdbMain.h | 15 +++- src/tsdb/src/tsdbFile.c | 176 ++++++++++++++++++---------------------- 2 files changed, 93 insertions(+), 98 deletions(-) diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index c000ac7e79..1deb66c0a3 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -111,7 +111,13 @@ typedef struct { } SMemTable; // ------------------ tsdbFile.c -typedef enum { TSDB_FILE_TYPE_HEAD = 0, TSDB_FILE_TYPE_DATA, TSDB_FILE_TYPE_LAST, TSDB_FILE_TYPE_MAX } TSDB_FILE_TYPE; +typedef enum { + TSDB_FILE_TYPE_HEAD = 0, + TSDB_FILE_TYPE_DATA, + TSDB_FILE_TYPE_LAST, + TSDB_FILE_TYPE_NHEAD, + TSDB_FILE_TYPE_NLAST +} TSDB_FILE_TYPE; typedef struct { uint32_t offset; @@ -123,8 +129,9 @@ typedef struct { } STsdbFileInfo; typedef struct { - char* fname; - int fd; + char* fname; + int fd; + STsdbFileInfo info; } SFile; @@ -136,6 +143,8 @@ typedef struct { } SFileGroup; typedef struct { + pthread_rwlock_t fhlock; + int maxFGroups; int nFGroups; SFileGroup* pFGroup; diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index ce34cbc08c..f48c72dd8f 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -29,13 +29,7 @@ #include "tutil.h" #include "ttime.h" -const char *tsdbFileSuffix[] = { - ".head", - ".data", - ".last", - ".h", - ".l" -}; +const char *tsdbFileSuffix[] = {".head", ".data", ".last", ".h", ".l"}; // ---------------- INTERNAL FUNCTIONS ---------------- STsdbFileH *tsdbNewFileH(STsdbCfg *pCfg) { @@ -45,6 +39,13 @@ STsdbFileH *tsdbNewFileH(STsdbCfg *pCfg) { goto _err; } + int code = pthread_rwlock_init(&(pFileH->fhlock)); + if (code != 0) { + tsdbError("vgId:%d failed to init file handle lock since %s", pCfg->tsdbId, strerror(code)); + terrno = TAOS_SYSTEM_ERROR(code); + goto _err; + } + pFileH->maxFGroups = TSDB_MAX_FILE(pCfg->keep, pCfg->daysPerFile); pFileH->pFGroup = (SFileGroup *)calloc(pFileH->maxFGroups, sizeof(SFileGroup)); @@ -62,6 +63,7 @@ _err: void tsdbFreeFileH(STsdbFileH *pFileH) { if (pFileH) { + pthread_rwlock_destroy(&pFileH->fhlock); tfree(pFileH->pFGroup); free(pFileH); } @@ -74,7 +76,9 @@ int *tsdbOpenFileH(STsdbRepo *pRepo) { DIR * dir = NULL; int fid = 0; - tsdbGetDataDirName(pRepo->rootDir); + STsdbFileH pFileH = pRepo->tsdbFileH; + + tDataDir = tsdbGetDataDirName(pRepo->rootDir); if (tDataDir == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _err; @@ -91,9 +95,15 @@ int *tsdbOpenFileH(STsdbRepo *pRepo) { while ((dp = readdir(dir)) != NULL) { if (strncmp(dp->d_name, ".", 1) == 0 || strncmp(dp->d_name, "..", 2) == 0) continue; sscanf(dp->d_name, "f%d", &fid); - // if (tsdbOpenFGroup(pFileH, dataDir, fid) < 0) { - // break; - // } + + SFileGroup fileGroup = {0}; + + if (tsdbSearchFGroup(pFileH, fid, TD_EQ) != NULL) continue; + for (int type = TSDB_FILE_TYPE_HEAD; type <= TSDB_FILE_TYPE_LAST; type++) { + } + for (int type = TSDB_FILE_TYPE_NHEAD; type <= TSDB_FILE_TYPE_NLAST; type++) { + + } } tfree(tDataDir); @@ -106,10 +116,12 @@ _err: return -1; } -void tsdbCloseFileH(STsdbFileH *pFileH) { - if (pFileH) { - tfree(pFileH->fGroup); - free(pFileH); +void tsdbCloseFileH(STsdbRepo *pRepo) { + STsdbFileH *pFileH = pRepo->tsdbFileH; + + for (int i = 0; i < pFileH->nFGroups; i++) { + // TODO + } } @@ -146,7 +158,7 @@ _err: int tsdbRemoveFileGroup(STsdbFileH *pFileH, int fid) { SFileGroup *pGroup = - bsearch((void *)&fid, (void *)(pFileH->fGroup), pFileH->numOfFGroups, sizeof(SFileGroup), compFGroupKey); + bsearch((void *)&fid, (void *)(pFileH->fGroup), pFileH->numOfFGroups, sizeof(SFileGroup), keyFGroupCompFunc); if (pGroup == NULL) return -1; // Remove from disk @@ -198,7 +210,7 @@ void tsdbSeekFileGroupIter(SFileGroupIter *pIter, int fid) { } int flags = (pIter->direction == TSDB_FGROUP_ITER_FORWARD) ? TD_GE : TD_LE; - void *ptr = taosbsearch(&fid, pIter->base, pIter->numOfFGroups, sizeof(SFileGroup), compFGroupKey, flags); + void *ptr = taosbsearch(&fid, pIter->base, pIter->numOfFGroups, sizeof(SFileGroup), keyFGroupCompFunc, flags); if (ptr == NULL) { pIter->pFileGroup = NULL; } else { @@ -226,81 +238,38 @@ SFileGroup *tsdbGetFileGroupNext(SFileGroupIter *pIter) { return ret; } -int tsdbCopyBlockDataInFile(SFile *pOutFile, SFile *pInFile, SCompInfo *pCompInfo, int idx, int isLast, SDataCols *pCols) { - SCompBlock *pSuperBlock = TSDB_COMPBLOCK_AT(pCompInfo, idx); - SCompBlock *pStartBlock = NULL; - SCompBlock *pBlock = NULL; - int numOfBlocks = pSuperBlock->numOfSubBlocks; +char *tsdbGetFileName(char *dataDir, int fileId, int type) { + int tlen = strlen(dataDir) + strlen(tsdbFileSuffix[type]) + 24; - if (numOfBlocks == 1) - pStartBlock = pSuperBlock; - else - pStartBlock = TSDB_COMPBLOCK_AT(pCompInfo, pSuperBlock->offset); - - int maxNumOfCols = 0; - pBlock = pStartBlock; - for (int i = 0; i < numOfBlocks; i++) { - if (pBlock->numOfCols > maxNumOfCols) maxNumOfCols = pBlock->numOfCols; - pBlock++; + char *fname = (char *)malloc(tlen); + if (fname == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; } - SCompData *pCompData = (SCompData *)malloc(sizeof(SCompData) + sizeof(SCompCol) * maxNumOfCols); - if (pCompData == NULL) return -1; - - // Load data from the block - // if (tsdbLoadDataBlock(pOutFile, pStartBlock, numOfBlocks, pCols, pCompData)); - - // Write data block to the file - { - // TODO - } - - - if (pCompData) free(pCompData); - return 0; -} - -int compFGroupKey(const void *key, const void *fgroup) { - int fid = *(int *)key; - SFileGroup *pFGroup = (SFileGroup *)fgroup; - if (fid == pFGroup->fileId) { - return 0; - } else { - return fid > pFGroup->fileId? 1:-1; - } -} - -int tsdbGetFileName(char *dataDir, int fileId, const char *suffix, char *fname) { - if (dataDir == NULL || fname == NULL) return -1; - - sprintf(fname, "%s/f%d%s", dataDir, fileId, suffix); + sprintf(fname, "%s/v%df%d%s", dataDir, fileId, tsdbFileSuffix[type]); return 0; } -int tsdbOpenFile(SFile *pFile, int oflag) { // TODO: change the function - if (TSDB_IS_FILE_OPENED(pFile)) return -1; +int tsdbOpenFile(SFile *pFile, int oflag) { + ASSERT(!TSDB_IS_FILE_OPENED(pFile)); pFile->fd = open(pFile->fname, oflag, 0755); - if (pFile->fd < 0) return -1; + if (pFile->fd < 0) { + tsdbError("failed to open file %s since %s", pFile->fname, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } return 0; } -int tsdbCloseFile(SFile *pFile) { - int ret = close(pFile->fd); - pFile->fd = -1; - return ret; -} - -SFileGroup * tsdbOpenFilesForCommit(STsdbFileH *pFileH, int fid) { - SFileGroup *pGroup = tsdbSearchFGroup(pFileH, fid); - if (pGroup == NULL) return NULL; - - for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { - tsdbOpenFile(&(pGroup->files[type]), O_RDWR); +void tsdbCloseFile(SFile *pFile) { + if (TSDB_IS_FILE_OPENED(pFile)) { + close(pFile->fd); + pFile->fd = -1; } - return pGroup; } int tsdbCreateFile(char *dataDir, int fileId, const char *suffix, SFile *pFile) { @@ -331,11 +300,9 @@ int tsdbCreateFile(char *dataDir, int fileId, const char *suffix, SFile *pFile) return 0; } - -SFileGroup *tsdbSearchFGroup(STsdbFileH *pFileH, int fid) { - if (pFileH->numOfFGroups == 0 || fid < pFileH->fGroup[0].fileId || fid > pFileH->fGroup[pFileH->numOfFGroups - 1].fileId) - return NULL; - void *ptr = bsearch((void *)&fid, (void *)(pFileH->fGroup), pFileH->numOfFGroups, sizeof(SFileGroup), compFGroupKey); +SFileGroup *tsdbSearchFGroup(STsdbFileH *pFileH, int fid, int flags) { + void *ptr = + taosbsearch((void *)(&fid), (void *)(pFileH->pFGroup), pFileH->nFGroups, sizeof(SFileGroup), keyFGroupCompFunc); if (ptr == NULL) return NULL; return (SFileGroup *)ptr; } @@ -362,20 +329,39 @@ static int tsdbInitFile(char *dataDir, int fid, const char *suffix, SFile *pFile return 0; } -static int tsdbOpenFGroup(STsdbFileH *pFileH, char *dataDir, int fid) { - if (tsdbSearchFGroup(pFileH, fid) != NULL) return 0; +// static int tsdbOpenFGroup(STsdbFileH *pFileH, char *dataDir, int fid) { +// if (tsdbSearchFGroup(pFileH, fid) != NULL) return 0; - SFileGroup fGroup = {0}; - fGroup.fileId = fid; +// SFileGroup fGroup = {0}; +// fGroup.fileId = fid; - for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { - if (tsdbInitFile(dataDir, fid, tsdbFileSuffix[type], &fGroup.files[type]) < 0) return -1; - } - pFileH->fGroup[pFileH->numOfFGroups++] = fGroup; - qsort((void *)(pFileH->fGroup), pFileH->numOfFGroups, sizeof(SFileGroup), compFGroup); - return 0; -} +// for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { +// if (tsdbInitFile(dataDir, fid, tsdbFileSuffix[type], &fGroup.files[type]) < 0) return -1; +// } +// pFileH->fGroup[pFileH->numOfFGroups++] = fGroup; +// qsort((void *)(pFileH->fGroup), pFileH->numOfFGroups, sizeof(SFileGroup), compFGroup); +// return 0; +// } static int compFGroup(const void *arg1, const void *arg2) { - return ((SFileGroup *)arg1)->fileId - ((SFileGroup *)arg2)->fileId; + int val1 = ((SFileGroup *)arg1)->fileId; + int val2 = ((SFileGroup *)arg2)->fileId; + + if (val1 < val2) { + return -1; + } else if (val1 > val2) { + return 1; + } else { + return 0; + } +} + +static int keyFGroupCompFunc(const void *key, const void *fgroup) { + int fid = *(int *)key; + SFileGroup *pFGroup = (SFileGroup *)fgroup; + if (fid == pFGroup->fileId) { + return 0; + } else { + return fid > pFGroup->fileId ? 1 : -1; + } } \ No newline at end of file From 3041354e8d9d835e0b4cd07bab22ad2a20e11361 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 16 Jun 2020 09:33:19 +0000 Subject: [PATCH 27/65] TD-353 --- src/tsdb/inc/tsdbMain.h | 2 ++ src/tsdb/src/tsdbFile.c | 20 ++++++-------------- src/tsdb/src/tsdbMain.c | 13 +++++++++++++ 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 1deb66c0a3..aaa84884a1 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -111,6 +111,7 @@ typedef struct { } SMemTable; // ------------------ tsdbFile.c +extern const char* tsdbFileSuffix[]; typedef enum { TSDB_FILE_TYPE_HEAD = 0, TSDB_FILE_TYPE_DATA, @@ -340,6 +341,7 @@ void tsdbFreeFileH(STsdbFileH* pFileH); #define IS_REPO_LOCKED(r) (r)->repoLocked char* tsdbGetMetaFileName(char* rootDir); +char* tsdbGetDataFileName(STsdbRepo* pRepo, int fid, int type); int tsdbLockRepo(STsdbRepo* pRepo); int tsdbUnlockRepo(STsdbRepo* pRepo); diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index f48c72dd8f..59bc2f53bb 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -99,7 +99,13 @@ int *tsdbOpenFileH(STsdbRepo *pRepo) { SFileGroup fileGroup = {0}; if (tsdbSearchFGroup(pFileH, fid, TD_EQ) != NULL) continue; + fileGroup.fileId = fid; for (int type = TSDB_FILE_TYPE_HEAD; type <= TSDB_FILE_TYPE_LAST; type++) { + fileGroup.headF.fname = tsdbGetDataFileName(pRepo, fid, type); + if (fileGroup.headF.fname == NULL) goto _err; + if (tsdbInitFile(fileGroup.headF)) + + } for (int type = TSDB_FILE_TYPE_NHEAD; type <= TSDB_FILE_TYPE_NLAST; type++) { @@ -238,20 +244,6 @@ SFileGroup *tsdbGetFileGroupNext(SFileGroupIter *pIter) { return ret; } -char *tsdbGetFileName(char *dataDir, int fileId, int type) { - int tlen = strlen(dataDir) + strlen(tsdbFileSuffix[type]) + 24; - - char *fname = (char *)malloc(tlen); - if (fname == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - - sprintf(fname, "%s/v%df%d%s", dataDir, fileId, tsdbFileSuffix[type]); - - return 0; -} - int tsdbOpenFile(SFile *pFile, int oflag) { ASSERT(!TSDB_IS_FILE_OPENED(pFile)); diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index c459f766d9..4ecd87b250 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -296,6 +296,19 @@ char *tsdbGetMetaFileName(char *rootDir) { return fname; } +char *tsdbGetDataFileName(STsdbRepo *pRepo, int fid, int type) { + int tlen = strlen(pRepo->rootDir) + strlen(tsdbFileSuffix[type]) + 24; + + char *fname = malloc(tlen); + if (fname == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + + sprintf(fname, "%s/%s/v%df%d.%s", pRepo->rootDir, TSDB_DATA_DIR_NAME, REPO_ID(pRepo), fid, tsdbFileSuffix[type]); + return fname; +} + int tsdbLockRepo(STsdbRepo *pRepo) { int code = pthread_mutex_lock(&pRepo->mutex); if (code != 0) { From ea86b6e80048fd5b401f85e03088a8e7f8b9887e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 17 Jun 2020 02:45:36 +0000 Subject: [PATCH 28/65] TD-353 --- src/inc/taoserror.h | 1 + src/tsdb/inc/tsdbMain.h | 22 ++-- src/tsdb/src/tsdbFile.c | 196 ++++++++++++++++++++---------------- src/tsdb/src/tsdbMemTable.c | 2 +- 4 files changed, 126 insertions(+), 95 deletions(-) diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index a5bfb0fa1e..b7894fd386 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -192,6 +192,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP, 0, 0x060C, "tsdb submi TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_ACTION, 0, 0x060D, "tsdb invalid action") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_INVALID_CREATE_TB_MSG, 0, 0x060E, "tsdb invalid create table message") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_TABLE_DATA_IN_MEM, 0, 0x060F, "tsdb no table data in memory skiplist") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_FILE_ALREADY_EXISTS, 0, 0x0610, "tsdb file already exists") // query TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_QHANDLE, 0, 0x0700, "query invalid handle") diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index aaa84884a1..c26beba21c 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -116,8 +116,7 @@ typedef enum { TSDB_FILE_TYPE_HEAD = 0, TSDB_FILE_TYPE_DATA, TSDB_FILE_TYPE_LAST, - TSDB_FILE_TYPE_NHEAD, - TSDB_FILE_TYPE_NLAST + TSDB_FILE_TYPE_MAX, } TSDB_FILE_TYPE; typedef struct { @@ -137,10 +136,8 @@ typedef struct { } SFile; typedef struct { - int fileId; - SFile headF; - SFile dataF; - SFile lastF; + int fileId; + SFile files[TSDB_FILE_TYPE_MAX]; } SFileGroup; typedef struct { @@ -313,11 +310,23 @@ int tsdbTakeMemSnapshot(STsdbRepo* pRepo, SMemTable** pMem, SMemTable** pIMem); #define TSDB_MAX_FILE(keep, daysPerFile) ((keep) / (daysPerFile) + 3) #define TSDB_MIN_FILE_ID(fh) (fh)->pFGroup[0].fileId #define TSDB_MAX_FILE_ID(fh) (fh)->pFGroup[(fh)->nFGroups - 1].fileId +#define TSDB_IS_FILE_OPENED(f) ((f)->fd > 0) #define TSDB_FGROUP_ITER_FORWARD TSDB_ORDER_ASC #define TSDB_FGROUP_ITER_BACKWARD TSDB_ORDER_DESC STsdbFileH* tsdbNewFileH(STsdbCfg* pCfg); void tsdbFreeFileH(STsdbFileH* pFileH); +int* tsdbOpenFileH(STsdbRepo* pRepo); +void tsdbCloseFileH(STsdbRepo* pRepo); +SFileGroup* tsdbCreateFGroupIfNeed(STsdbFileH* pFileH, char* dataDir, int fid, int maxTables); +void tsdbInitFileGroupIter(STsdbFileH* pFileH, SFileGroupIter* pIter, int direction); +void tsdbSeekFileGroupIter(SFileGroupIter* pIter, int fid); +SFileGroup* tsdbGetFileGroupNext(SFileGroupIter* pIter); +int tsdbOpenFile(SFile* pFile, int oflag); +void tsdbCloseFile(SFile* pFile); +int tsdbCreateFile(SFile* pFile, STsdbRepo* pRepo, int fid, int type); +SFileGroup* tsdbSearchFGroup(STsdbFileH* pFileH, int fid, int flags); +void tsdbFitRetention(STsdbRepo* pRepo); // ------------------ tsdbRWHelper.c #define TSDB_HELPER_CLEAR_STATE 0x0 // Clear state @@ -349,7 +358,6 @@ int tsdbUnlockRepo(STsdbRepo* pRepo); // --------- Helper state - int tsdbInitReadHelper(SRWHelper *pHelper, STsdbRepo *pRepo); int tsdbInitWriteHelper(SRWHelper *pHelper, STsdbRepo *pRepo); void tsdbDestroyHelper(SRWHelper *pHelper); diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 59bc2f53bb..25f8e032ed 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -29,7 +29,13 @@ #include "tutil.h" #include "ttime.h" -const char *tsdbFileSuffix[] = {".head", ".data", ".last", ".h", ".l"}; +const char *tsdbFileSuffix[] = {".head", ".data", ".last"}; + +static int tsdbInitFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type); +static void tsdbDestroyFile(SFile *pFile); +static int compFGroup(const void *arg1, const void *arg2); +static int keyFGroupCompFunc(const void *key, const void *fgroup); +static void tsdbRemoveFileGroup(STsdbRepo *pRepo, SFileGroup *pFGroup); // ---------------- INTERNAL FUNCTIONS ---------------- STsdbFileH *tsdbNewFileH(STsdbCfg *pCfg) { @@ -39,7 +45,7 @@ STsdbFileH *tsdbNewFileH(STsdbCfg *pCfg) { goto _err; } - int code = pthread_rwlock_init(&(pFileH->fhlock)); + int code = pthread_rwlock_init(&(pFileH->fhlock), NULL); if (code != 0) { tsdbError("vgId:%d failed to init file handle lock since %s", pCfg->tsdbId, strerror(code)); terrno = TAOS_SYSTEM_ERROR(code); @@ -76,6 +82,7 @@ int *tsdbOpenFileH(STsdbRepo *pRepo) { DIR * dir = NULL; int fid = 0; + SFileGroup fileGroup = {0}; STsdbFileH pFileH = pRepo->tsdbFileH; tDataDir = tsdbGetDataDirName(pRepo->rootDir); @@ -95,21 +102,22 @@ int *tsdbOpenFileH(STsdbRepo *pRepo) { while ((dp = readdir(dir)) != NULL) { if (strncmp(dp->d_name, ".", 1) == 0 || strncmp(dp->d_name, "..", 2) == 0) continue; sscanf(dp->d_name, "f%d", &fid); - - SFileGroup fileGroup = {0}; - if (tsdbSearchFGroup(pFileH, fid, TD_EQ) != NULL) continue; + if (tsdbSearchFGroup(pRepo->tsdbFileH, fid, TD_EQ) != NULL) return 0; + + fileGroup = {0}; fileGroup.fileId = fid; - for (int type = TSDB_FILE_TYPE_HEAD; type <= TSDB_FILE_TYPE_LAST; type++) { - fileGroup.headF.fname = tsdbGetDataFileName(pRepo, fid, type); - if (fileGroup.headF.fname == NULL) goto _err; - if (tsdbInitFile(fileGroup.headF)) - - + for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { + if (tsdbInitFile(&fileGroup.files[type], pRepo, fid, type) < 0) { + tsdbError("vgId:%d failed to init file fid %d type %d", REPO_ID(pRepo), fid, type); + goto _err; + } } - for (int type = TSDB_FILE_TYPE_NHEAD; type <= TSDB_FILE_TYPE_NLAST; type++) { - } + tsdbTrace("vgId:%d file group %d init", REPO_ID(pRepo), fid); + + pFileH->[pFileH->nFGroups++] = fileGroup; + qsort((void *)(pFileH->pFGroup), pFileH->nFGroups, sizeof(SFileGroup), compFGroup); } tfree(tDataDir); @@ -117,8 +125,11 @@ int *tsdbOpenFileH(STsdbRepo *pRepo) { return 0; _err: + for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) tsdbDestroyFile(&fileGroup.files[type]); + tfree(tDataDir); if (dir != NULL) closedir(tDataDir); + tsdbCloseFileH(pRepo); return -1; } @@ -126,17 +137,14 @@ void tsdbCloseFileH(STsdbRepo *pRepo) { STsdbFileH *pFileH = pRepo->tsdbFileH; for (int i = 0; i < pFileH->nFGroups; i++) { - // TODO - + SFileGroup *pFGroup = pFileH->pFGroup + i; + for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { + tsdbDestroyFile(pFGroup->files[type]); + } } } -/** - * Create the file group if the file group not exists. - * - * @return A pointer to - */ -SFileGroup *tsdbCreateFGroup(STsdbFileH *pFileH, char *dataDir, int fid, int maxTables) { +SFileGroup *tsdbCreateFGroupIfNeed(STsdbFileH *pFileH, char *dataDir, int fid, int maxTables) { if (pFileH->numOfFGroups >= pFileH->maxFGroups) return NULL; SFileGroup fGroup; @@ -158,35 +166,15 @@ SFileGroup *tsdbCreateFGroup(STsdbFileH *pFileH, char *dataDir, int fid, int max return pGroup; _err: - // TODO: deal with the err here + for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) tsdbDestroyFile(&pGroup->files[type]); return NULL; } -int tsdbRemoveFileGroup(STsdbFileH *pFileH, int fid) { - SFileGroup *pGroup = - bsearch((void *)&fid, (void *)(pFileH->fGroup), pFileH->numOfFGroups, sizeof(SFileGroup), keyFGroupCompFunc); - if (pGroup == NULL) return -1; - - // Remove from disk - for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { - remove(pGroup->files[type].fname); - } - - // Adjust the memory - int filesBehind = pFileH->numOfFGroups - (((char *)pGroup - (char *)(pFileH->fGroup)) / sizeof(SFileGroup) + 1); - if (filesBehind > 0) { - memmove((void *)pGroup, (void *)((char *)pGroup + sizeof(SFileGroup)), sizeof(SFileGroup) * filesBehind); - } - pFileH->numOfFGroups--; - - return 0; -} - -void tsdbInitFileGroupIter(STsdbFileH *pFileH, SFileGroupIter *pIter, int direction) { +void tsdbInitFileGroupIter(STsdbFileH *pFileH, SFileGroupIter *pIter, int direction) { // TODO pIter->direction = direction; pIter->base = pFileH->fGroup; pIter->numOfFGroups = pFileH->numOfFGroups; - if (pFileH->numOfFGroups == 0){ + if (pFileH->numOfFGroups == 0) { pIter->pFileGroup = NULL; } else { if (direction == TSDB_FGROUP_ITER_FORWARD) { @@ -197,25 +185,13 @@ void tsdbInitFileGroupIter(STsdbFileH *pFileH, SFileGroupIter *pIter, int direct } } -void tsdbFitRetention(STsdbRepo *pRepo) { - STsdbFileH *pFileH = pRepo->tsdbFileH; - SFileGroup *pGroup = pFileH->fGroup; - - int mfid = - tsdbGetKeyFileId(taosGetTimestamp(pRepo->config.precision), pRepo->config.daysPerFile, pRepo->config.precision) - pFileH->maxFGroups + 3; - - while (pFileH->numOfFGroups > 0 && pGroup[0].fileId < mfid) { - tsdbRemoveFileGroup(pFileH, pGroup[0].fileId); - } -} - -void tsdbSeekFileGroupIter(SFileGroupIter *pIter, int fid) { +void tsdbSeekFileGroupIter(SFileGroupIter *pIter, int fid) { // TODO if (pIter->numOfFGroups == 0) { assert(pIter->pFileGroup == NULL); return; } - - int flags = (pIter->direction == TSDB_FGROUP_ITER_FORWARD) ? TD_GE : TD_LE; + + int flags = (pIter->direction == TSDB_FGROUP_ITER_FORWARD) ? TD_GE : TD_LE; void *ptr = taosbsearch(&fid, pIter->base, pIter->numOfFGroups, sizeof(SFileGroup), keyFGroupCompFunc, flags); if (ptr == NULL) { pIter->pFileGroup = NULL; @@ -224,7 +200,7 @@ void tsdbSeekFileGroupIter(SFileGroupIter *pIter, int fid) { } } -SFileGroup *tsdbGetFileGroupNext(SFileGroupIter *pIter) { +SFileGroup *tsdbGetFileGroupNext(SFileGroupIter *pIter) {//TODO SFileGroup *ret = pIter->pFileGroup; if (ret == NULL) return NULL; @@ -264,20 +240,21 @@ void tsdbCloseFile(SFile *pFile) { } } -int tsdbCreateFile(char *dataDir, int fileId, const char *suffix, SFile *pFile) { +int tsdbCreateFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type) { memset((void *)pFile, 0, sizeof(SFile)); pFile->fd = -1; - tsdbGetFileName(dataDir, fileId, suffix, pFile->fname); - + pFile->fname = tsdbGetDataFileName(pRepo, fid, type); + if (pFile->fname == NULL) return -1; + if (access(pFile->fname, F_OK) == 0) { - // File already exists - return -1; + tsdbError("vgId:%d file %s already exists", REPO_ID(pRepo), fid); + terrno = TSDB_CODE_TDB_FILE_ALREADY_EXISTS; + goto _err; } if (tsdbOpenFile(pFile, O_RDWR | O_CREAT) < 0) { - // TODO: deal with the ERROR here - return -1; + goto _err; } pFile->info.size = TSDB_FILE_HEAD_SIZE; @@ -290,6 +267,9 @@ int tsdbCreateFile(char *dataDir, int fileId, const char *suffix, SFile *pFile) tsdbCloseFile(pFile); return 0; + +_err: + return -1; } SFileGroup *tsdbSearchFGroup(STsdbFileH *pFileH, int fid, int flags) { @@ -299,18 +279,46 @@ SFileGroup *tsdbSearchFGroup(STsdbFileH *pFileH, int fid, int flags) { return (SFileGroup *)ptr; } +void tsdbFitRetention(STsdbRepo *pRepo) { + STsdbCfg *pCfg = &(pRepo->config); + STsdbFileH *pFileH = pRepo->tsdbFileH; + SFileGroup *pGroup = pFileH->pFGroup; + + int mfid = TSDB_KEY_FILEID(taosGetTimestamp(pCfg->precision), pCfg->daysPerFile, pCfg->precision) - + TSDB_MAX_FILE(pCfg->keep, pCfg->daysPerFile); + + pthread_rwlock_wrlock(&(pFileH->fhlock)); + + while (pFileH->numOfFGroups > 0 && pGroup[0].fileId < mfid) { + tsdbRemoveFileGroup(pFileH, pGroup); + } + + pthread_rwlock_unlock(&(pFileH->fhlock)) +} + + // ---------------- LOCAL FUNCTIONS ---------------- -static int tsdbInitFile(char *dataDir, int fid, const char *suffix, SFile *pFile) { +static int tsdbInitFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type) { uint32_t version; - char buf[512] = "\0"; + char buf[512] = "\0"; + + pFile->fname = tsdbGetDataFileName(pRepo, fid, type); + if (pFile->fname == NULL) return -1; - tsdbGetFileName(dataDir, fid, suffix, pFile->fname); - if (access(pFile->fname, F_OK|R_OK|W_OK) < 0) return -1; pFile->fd = -1; - if (tsdbOpenFile(pFile, O_RDONLY) < 0) return -1; + if (tsdbOpenFile(pFile, O_RDONLY) < 0) goto _err; - if (tread(pFile->fd, buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) return -1; - if (!taosCheckChecksumWhole((uint8_t *)buf, TSDB_FILE_HEAD_SIZE)) return -1; + if (tread(pFile->fd, buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { + tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pRepo), TSDB_FILE_HEAD_SIZE, + pFile->fname, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + if (!taosCheckChecksumWhole((uint8_t *)buf, TSDB_FILE_HEAD_SIZE)) { + tsdbError("vgId:%d file %s head part is corrupted", REPO_ID(pRepo), pFile->fname); + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + goto _err; + } void *pBuf = buf; pBuf = taosDecodeFixedU32(pBuf, &version); @@ -319,21 +327,15 @@ static int tsdbInitFile(char *dataDir, int fid, const char *suffix, SFile *pFile tsdbCloseFile(pFile); return 0; +_err: + tsdbDestroyFile(pFile); + return -1; } -// static int tsdbOpenFGroup(STsdbFileH *pFileH, char *dataDir, int fid) { -// if (tsdbSearchFGroup(pFileH, fid) != NULL) return 0; - -// SFileGroup fGroup = {0}; -// fGroup.fileId = fid; - -// for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { -// if (tsdbInitFile(dataDir, fid, tsdbFileSuffix[type], &fGroup.files[type]) < 0) return -1; -// } -// pFileH->fGroup[pFileH->numOfFGroups++] = fGroup; -// qsort((void *)(pFileH->fGroup), pFileH->numOfFGroups, sizeof(SFileGroup), compFGroup); -// return 0; -// } +static void tsdbDestroyFile(SFile *pFile) { + tsdbCloseFile(pFile); + tfree(pFile->fname); +} static int compFGroup(const void *arg1, const void *arg2) { int val1 = ((SFileGroup *)arg1)->fileId; @@ -356,4 +358,24 @@ static int keyFGroupCompFunc(const void *key, const void *fgroup) { } else { return fid > pFGroup->fileId ? 1 : -1; } +} + +static void tsdbRemoveFileGroup(STsdbRepo *pRepo, SFileGroup *pFGroup) { + ASSERT(pFGroup != NULL); + STsdbFileH *pFileH = pRepo->tsdbFileH; + + SFileGroup fileGroup = *pFGroup; + + int nFilesLeft = pFileH->nFGroups - (POINTER_DISTANCE(pFGroup, pFileH->pFGroup) / sizeof(SFileGroup) + 1); + if (nFilesLeft > 0) { + memmove((void *)pFGroup, POINTER_SHIFT(pFGroup, sizeof(SFileGroup)), sizeof(SFileGroup) * nFilesLeft); + } + + pFileH->nFGroups--; + ASSERT(pFileH->nFGroups >= 0); + + for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { + remove(fileGroup.files[type].fname); + tsdbDestroyFile(&fileGroup.files[type]); + } } \ No newline at end of file diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 75bfe0d7ce..0c8bca9441 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -464,7 +464,7 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe return -1; } - if ((pGroup = tsdbCreateFGroup(pFileH, dataDir, fid, pCfg->maxTables)) == NULL) { + if ((pGroup = tsdbCreateFGroupIfNeed(pFileH, dataDir, fid, pCfg->maxTables)) == NULL) { tsdbError("vgId:%d failed to create file group %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno)); goto _err; } From f4ca77f7da0d97374ba854cc21393ace9749fb81 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 17 Jun 2020 04:06:17 +0000 Subject: [PATCH 29/65] TD-353 --- src/inc/tsdb.h | 4 +- src/tsdb/inc/tsdbMain.h | 41 ++++++++++---- src/tsdb/src/tsdbFile.c | 104 ++++++++++++++++++++++++++---------- src/tsdb/src/tsdbMain.c | 88 ++++++++++++++++++------------ src/tsdb/src/tsdbMemTable.c | 15 +++--- src/tsdb/src/tsdbMeta.c | 17 +++--- src/tsdb/src/tsdbRWHelper.c | 37 ++++++++++++- src/tsdb/src/tsdbRead.c | 4 +- src/util/inc/tref.h | 4 +- 9 files changed, 220 insertions(+), 94 deletions(-) diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index 96a04ea5ed..ee18396b61 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -78,9 +78,9 @@ STsdbCfg *tsdbGetCfg(const TSDB_REPO_T *repo); // --------- TSDB REPOSITORY DEFINITION int tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg); -int32_t tsdbDropRepo(TSDB_REPO_T *repo); +int32_t tsdbDropRepo(char *rootDir); TSDB_REPO_T *tsdbOpenRepo(char *rootDir, STsdbAppH *pAppH); -int32_t tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit); +void tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit); int32_t tsdbConfigRepo(TSDB_REPO_T *repo, STsdbCfg *pCfg); // --------- TSDB TABLE DEFINITION diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index c26beba21c..fa1b98ec94 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -50,7 +50,7 @@ typedef struct STable { uint64_t suid; struct STable* pSuper; // super table pointer uint8_t numOfSchemas; - STSchema schema[TSDB_MAX_TABLE_SCHEMAS]; + STSchema* schema[TSDB_MAX_TABLE_SCHEMAS]; STSchema* tagSchema; SKVRow tagVal; void* pIndex; // For TSDB_SUPER_TABLE, it is the skiplist index @@ -316,9 +316,9 @@ int tsdbTakeMemSnapshot(STsdbRepo* pRepo, SMemTable** pMem, SMemTable** pIMem); STsdbFileH* tsdbNewFileH(STsdbCfg* pCfg); void tsdbFreeFileH(STsdbFileH* pFileH); -int* tsdbOpenFileH(STsdbRepo* pRepo); +int tsdbOpenFileH(STsdbRepo* pRepo); void tsdbCloseFileH(STsdbRepo* pRepo); -SFileGroup* tsdbCreateFGroupIfNeed(STsdbFileH* pFileH, char* dataDir, int fid, int maxTables); +SFileGroup* tsdbCreateFGroupIfNeed(STsdbRepo* pRepo, char* dataDir, int fid, int maxTables); void tsdbInitFileGroupIter(STsdbFileH* pFileH, SFileGroupIter* pIter, int direction); void tsdbSeekFileGroupIter(SFileGroupIter* pIter, int fid); SFileGroup* tsdbGetFileGroupNext(SFileGroupIter* pIter); @@ -327,6 +327,7 @@ void tsdbCloseFile(SFile* pFile); int tsdbCreateFile(SFile* pFile, STsdbRepo* pRepo, int fid, int type); SFileGroup* tsdbSearchFGroup(STsdbFileH* pFileH, int fid, int flags); void tsdbFitRetention(STsdbRepo* pRepo); +int tsdbUpdateFileHeader(SFile* pFile, uint32_t version); // ------------------ tsdbRWHelper.c #define TSDB_HELPER_CLEAR_STATE 0x0 // Clear state @@ -345,14 +346,38 @@ void tsdbFitRetention(STsdbRepo* pRepo); #define helperRepo(h) (h)->pRepo #define helperState(h) (h)->state +int tsdbInitReadHelper(SRWHelper* pHelper, STsdbRepo* pRepo); +int tsdbInitWriteHelper(SRWHelper* pHelper, STsdbRepo* pRepo); +void tsdbDestroyHelper(SRWHelper* pHelper); +void tsdbResetHelper(SRWHelper* pHelper); +int tsdbSetAndOpenHelperFile(SRWHelper* pHelper, SFileGroup* pGroup); +int tsdbCloseHelperFile(SRWHelper* pHelper, bool hasError); +void tsdbSetHelperTable(SRWHelper* pHelper, STable* pTable, STsdbRepo* pRepo); +int tsdbWriteDataBlock(SRWHelper* pHelper, SDataCols* pDataCols); +int tsdbMoveLastBlockIfNeccessary(SRWHelper* pHelper); +int tsdbWriteCompInfo(SRWHelper* pHelper); +int tsdbWriteCompIdx(SRWHelper* pHelper); +int tsdbLoadCompIdx(SRWHelper* pHelper, void* target); +int tsdbLoadCompInfo(SRWHelper* pHelper, void* target); +int tsdbloadcompdata(srwhelper* phelper, scompblock* pcompblock, void* target); +void tsdbGetDataStatis(SRWHelper* pHelper, SDataStatis* pStatis, int numOfCols); +int tsdbLoadBlockDataCols(SRWHelper* pHelper, SDataCols* pDataCols, int blkIdx, int16_t* colIds, int numOfColIds); +int tsdbLoadBlockData(SRWHelper* pHelper, SCompBlock* pCompBlock, SDataCols* target); +int tsdbUpdateFileHeader(SFile* pFile, uint32_t version); +void* tsdbEncodeSFileInfo(void* buf, const STsdbFileInfo* pInfo); +void* tsdbDecodeSFileInfo(void* buf, STsdbFileInfo* pInfo); + // ------------------ tsdbMain.c #define REPO_ID(r) (r)->config.tsdbId #define IS_REPO_LOCKED(r) (r)->repoLocked -char* tsdbGetMetaFileName(char* rootDir); -char* tsdbGetDataFileName(STsdbRepo* pRepo, int fid, int type); -int tsdbLockRepo(STsdbRepo* pRepo); -int tsdbUnlockRepo(STsdbRepo* pRepo); +char* tsdbGetMetaFileName(char* rootDir); +char* tsdbGetDataFileName(STsdbRepo* pRepo, int fid, int type); +int tsdbLockRepo(STsdbRepo* pRepo); +int tsdbUnlockRepo(STsdbRepo* pRepo); +char* tsdbGetDataDirName(char* rootDir); +STsdbMeta* tsdbGetMeta(TSDB_REPO_T* pRepo); +STsdbFileH* tsdbGetFile(TSDB_REPO_T* pRepo); #if 0 @@ -389,8 +414,6 @@ void tsdbAdjustCacheBlocks(STsdbCache *pCache); int32_t tsdbGetMetaFileName(char *rootDir, char *fname); int tsdbUpdateFileHeader(SFile *pFile, uint32_t version); - -int compFGroupKey(const void *key, const void *fgroup); #endif #ifdef __cplusplus diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 25f8e032ed..8af9441227 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -31,11 +31,13 @@ const char *tsdbFileSuffix[] = {".head", ".data", ".last"}; -static int tsdbInitFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type); -static void tsdbDestroyFile(SFile *pFile); -static int compFGroup(const void *arg1, const void *arg2); -static int keyFGroupCompFunc(const void *key, const void *fgroup); -static void tsdbRemoveFileGroup(STsdbRepo *pRepo, SFileGroup *pFGroup); +static int tsdbInitFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type); +static void tsdbDestroyFile(SFile *pFile); +static int compFGroup(const void *arg1, const void *arg2); +static int keyFGroupCompFunc(const void *key, const void *fgroup); +static void tsdbRemoveFileGroup(STsdbRepo *pRepo, SFileGroup *pFGroup); +static void *tsdbEncodeSFileInfo(void *buf, const STsdbFileInfo *pInfo); +static void *tsdbDecodeSFileInfo(void *buf, STsdbFileInfo *pInfo); // ---------------- INTERNAL FUNCTIONS ---------------- STsdbFileH *tsdbNewFileH(STsdbCfg *pCfg) { @@ -75,7 +77,7 @@ void tsdbFreeFileH(STsdbFileH *pFileH) { } } -int *tsdbOpenFileH(STsdbRepo *pRepo) { +int tsdbOpenFileH(STsdbRepo *pRepo) { ASSERT(pRepo != NULL && pRepo->tsdbFileH != NULL); char *tDataDir = NULL; @@ -83,7 +85,7 @@ int *tsdbOpenFileH(STsdbRepo *pRepo) { int fid = 0; SFileGroup fileGroup = {0}; - STsdbFileH pFileH = pRepo->tsdbFileH; + STsdbFileH *pFileH = pRepo->tsdbFileH; tDataDir = tsdbGetDataDirName(pRepo->rootDir); if (tDataDir == NULL) { @@ -91,7 +93,7 @@ int *tsdbOpenFileH(STsdbRepo *pRepo) { goto _err; } - DIR *dir = opendir(tDataDir); + dir = opendir(tDataDir); if (dir == NULL) { tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), tDataDir, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); @@ -105,7 +107,7 @@ int *tsdbOpenFileH(STsdbRepo *pRepo) { if (tsdbSearchFGroup(pRepo->tsdbFileH, fid, TD_EQ) != NULL) return 0; - fileGroup = {0}; + memset((void *)(&fileGroup), 0, sizeof(SFileGroup)); fileGroup.fileId = fid; for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { if (tsdbInitFile(&fileGroup.files[type], pRepo, fid, type) < 0) { @@ -116,7 +118,7 @@ int *tsdbOpenFileH(STsdbRepo *pRepo) { tsdbTrace("vgId:%d file group %d init", REPO_ID(pRepo), fid); - pFileH->[pFileH->nFGroups++] = fileGroup; + pFileH->pFGroup[pFileH->nFGroups++] = fileGroup; qsort((void *)(pFileH->pFGroup), pFileH->nFGroups, sizeof(SFileGroup), compFGroup); } @@ -128,7 +130,7 @@ _err: for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) tsdbDestroyFile(&fileGroup.files[type]); tfree(tDataDir); - if (dir != NULL) closedir(tDataDir); + if (dir != NULL) closedir(dir); tsdbCloseFileH(pRepo); return -1; } @@ -139,28 +141,30 @@ void tsdbCloseFileH(STsdbRepo *pRepo) { for (int i = 0; i < pFileH->nFGroups; i++) { SFileGroup *pFGroup = pFileH->pFGroup + i; for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { - tsdbDestroyFile(pFGroup->files[type]); + tsdbDestroyFile(&pFGroup->files[type]); } } } -SFileGroup *tsdbCreateFGroupIfNeed(STsdbFileH *pFileH, char *dataDir, int fid, int maxTables) { - if (pFileH->numOfFGroups >= pFileH->maxFGroups) return NULL; +SFileGroup *tsdbCreateFGroupIfNeed(STsdbRepo *pRepo, char *dataDir, int fid, int maxTables) { + STsdbFileH *pFileH = pRepo->tsdbFileH; + + if (pFileH->nFGroups >= pFileH->maxFGroups) return NULL; SFileGroup fGroup; SFileGroup *pFGroup = &fGroup; - SFileGroup *pGroup = tsdbSearchFGroup(pFileH, fid); + SFileGroup *pGroup = tsdbSearchFGroup(pFileH, fid, TD_EQ); if (pGroup == NULL) { // if not exists, create one pFGroup->fileId = fid; for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { - if (tsdbCreateFile(dataDir, fid, tsdbFileSuffix[type], &(pFGroup->files[type])) < 0) + if (tsdbCreateFile(&pFGroup->files[type], pRepo, fid, type) < 0) goto _err; } - pFileH->fGroup[pFileH->numOfFGroups++] = fGroup; - qsort((void *)(pFileH->fGroup), pFileH->numOfFGroups, sizeof(SFileGroup), compFGroup); - return tsdbSearchFGroup(pFileH, fid); + pFileH->pFGroup[pFileH->nFGroups++] = fGroup; + qsort((void *)(pFileH->pFGroup), pFileH->nFGroups, sizeof(SFileGroup), compFGroup); + return tsdbSearchFGroup(pFileH, fid, TD_EQ); } return pGroup; @@ -172,15 +176,15 @@ _err: void tsdbInitFileGroupIter(STsdbFileH *pFileH, SFileGroupIter *pIter, int direction) { // TODO pIter->direction = direction; - pIter->base = pFileH->fGroup; - pIter->numOfFGroups = pFileH->numOfFGroups; - if (pFileH->numOfFGroups == 0) { + pIter->base = pFileH->pFGroup; + pIter->numOfFGroups = pFileH->nFGroups; + if (pFileH->nFGroups == 0) { pIter->pFileGroup = NULL; } else { if (direction == TSDB_FGROUP_ITER_FORWARD) { - pIter->pFileGroup = pFileH->fGroup; + pIter->pFileGroup = pFileH->pFGroup; } else { - pIter->pFileGroup = pFileH->fGroup + pFileH->numOfFGroups - 1; + pIter->pFileGroup = pFileH->pFGroup + pFileH->nFGroups - 1; } } } @@ -274,7 +278,7 @@ _err: SFileGroup *tsdbSearchFGroup(STsdbFileH *pFileH, int fid, int flags) { void *ptr = - taosbsearch((void *)(&fid), (void *)(pFileH->pFGroup), pFileH->nFGroups, sizeof(SFileGroup), keyFGroupCompFunc); + taosbsearch((void *)(&fid), (void *)(pFileH->pFGroup), pFileH->nFGroups, sizeof(SFileGroup), keyFGroupCompFunc, flags); if (ptr == NULL) return NULL; return (SFileGroup *)ptr; } @@ -289,13 +293,35 @@ void tsdbFitRetention(STsdbRepo *pRepo) { pthread_rwlock_wrlock(&(pFileH->fhlock)); - while (pFileH->numOfFGroups > 0 && pGroup[0].fileId < mfid) { - tsdbRemoveFileGroup(pFileH, pGroup); + while (pFileH->nFGroups > 0 && pGroup[0].fileId < mfid) { + tsdbRemoveFileGroup(pRepo, pGroup); } - pthread_rwlock_unlock(&(pFileH->fhlock)) + pthread_rwlock_unlock(&(pFileH->fhlock)); } +int tsdbUpdateFileHeader(SFile *pFile, uint32_t version) { + char buf[TSDB_FILE_HEAD_SIZE] = "\0"; + + void *pBuf = (void *)buf; + pBuf = taosEncodeFixedU32(pBuf, version); + pBuf = tsdbEncodeSFileInfo(pBuf, &(pFile->info)); + + taosCalcChecksumAppend(0, (uint8_t *)buf, TSDB_FILE_HEAD_SIZE); + + if (lseek(pFile->fd, 0, SEEK_SET) < 0) { + tsdbError("failed to lseek file %s since %s", pFile->fname, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + if (twrite(pFile->fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { + tsdbError("failed to write %d bytes to file %s since %s", TSDB_FILE_HEAD_SIZE, pFile->fname, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + return 0; +} // ---------------- LOCAL FUNCTIONS ---------------- static int tsdbInitFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type) { @@ -378,4 +404,26 @@ static void tsdbRemoveFileGroup(STsdbRepo *pRepo, SFileGroup *pFGroup) { remove(fileGroup.files[type].fname); tsdbDestroyFile(&fileGroup.files[type]); } +} + +static void *tsdbEncodeSFileInfo(void *buf, const STsdbFileInfo *pInfo) { + buf = taosEncodeFixedU32(buf, pInfo->offset); + buf = taosEncodeFixedU32(buf, pInfo->len); + buf = taosEncodeFixedU64(buf, pInfo->size); + buf = taosEncodeFixedU64(buf, pInfo->tombSize); + buf = taosEncodeFixedU32(buf, pInfo->totalBlocks); + buf = taosEncodeFixedU32(buf, pInfo->totalSubBlocks); + + return buf; +} + +static void *tsdbDecodeSFileInfo(void *buf, STsdbFileInfo *pInfo) { + buf = taosDecodeFixedU32(buf, &(pInfo->offset)); + buf = taosDecodeFixedU32(buf, &(pInfo->len)); + buf = taosDecodeFixedU64(buf, &(pInfo->size)); + buf = taosDecodeFixedU64(buf, &(pInfo->tombSize)); + buf = taosDecodeFixedU32(buf, &(pInfo->totalBlocks)); + buf = taosDecodeFixedU32(buf, &(pInfo->totalSubBlocks)); + + return buf; } \ No newline at end of file diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 4ecd87b250..fe7214f01e 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -29,6 +29,10 @@ #define TSDB_DATA_DIR_NAME "data" #define TSDB_META_FILE_NAME "meta" #define TSDB_META_FILE_INDEX 10000000 +#define IS_VALID_PRECISION(precision) \ + (((precision) >= TSDB_TIME_PRECISION_MILLI) && ((precision) <= TSDB_TIME_PRECISION_NANO)) +#define TSDB_DEFAULT_COMPRESSION TWO_STAGE_COMP +#define IS_VALID_COMPRESSION(compression) (((compression) >= NO_COMPRESSION) && ((compression) <= TWO_STAGE_COMP)) typedef struct { int32_t totalLen; @@ -42,6 +46,24 @@ typedef struct { SSubmitBlk *pBlock; } SSubmitMsgIter; +static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg); +static int32_t tsdbSetRepoEnv(char *rootDir, STsdbCfg *pCfg); +static int32_t tsdbUnsetRepoEnv(char *rootDir); +static int32_t tsdbSaveConfig(char *rootDir, STsdbCfg *pCfg); +static int tsdbLoadConfig(char *rootDir, STsdbCfg *pCfg); +static char * tsdbGetCfgFname(char *rootDir); +static STsdbRepo * tsdbNewRepo(char *rootDir, STsdbAppH *pAppH, STsdbCfg *pCfg); +static void tsdbFreeRepo(STsdbRepo *pRepo); +static int tsdbInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter); +static int32_t tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, TSKEY now, int32_t *affectedrows); +static SSubmitBlk *tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter); +static SDataRow tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter); +static int tsdbRestoreInfo(STsdbRepo *pRepo); +static int tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter); +static void tsdbAlterCompression(STsdbRepo *pRepo, int8_t compression); +static void tsdbAlterKeep(STsdbRepo *pRepo, int32_t keep); +static void tsdbAlterMaxTables(STsdbRepo *pRepo, int32_t maxTables); + // Function declaration int32_t tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg) { if (mkdir(rootDir, 0755) < 0) { @@ -122,8 +144,6 @@ void tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit) { tsdbCloseBufPool(pRepo); tsdbCloseMeta(pRepo); tsdbTrace("vgId:%d repository is closed", REPO_ID(pRepo)); - - return 0; } int32_t tsdbInsertData(TSDB_REPO_T *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg *pRsp) { @@ -136,7 +156,6 @@ int32_t tsdbInsertData(TSDB_REPO_T *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg * } SSubmitBlk *pBlock = NULL; - int32_t code = TSDB_CODE_SUCCESS; int32_t affectedrows = 0; TSKEY now = taosGetTimestamp(pRepo->config.precision); @@ -156,7 +175,7 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_ // STsdbMeta *pMeta = pRepo->tsdbMeta; STsdbFileH *pFileH = pRepo->tsdbFileH; uint32_t magic = 0; - char fname[256] = "\0"; + char *fname = NULL; struct stat fState; @@ -169,9 +188,9 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_ if (name[0] == 0) { // get the file from index or after, but not larger than eindex int fid = (*index) / 3; - if (pFileH->numOfFGroups == 0 || fid > pFileH->fGroup[pFileH->numOfFGroups - 1].fileId) { + if (pFileH->nFGroups == 0 || fid > pFileH->pFGroup[pFileH->nFGroups - 1].fileId) { if (*index <= TSDB_META_FILE_INDEX && TSDB_META_FILE_INDEX <= eindex) { - tsdbGetMetaFileName(pRepo->rootDir, fname); + fname = tsdbGetMetaFileName(pRepo->rootDir); *index = TSDB_META_FILE_INDEX; } else { tfree(sdup); @@ -179,7 +198,7 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_ } } else { SFileGroup *pFGroup = - taosbsearch(&fid, pFileH->fGroup, pFileH->numOfFGroups, sizeof(SFileGroup), compFGroupKey, TD_GE); + taosbsearch(&fid, pFileH->pFGroup, pFileH->nFGroups, sizeof(SFileGroup), compFGroupKey, TD_GE); if (pFGroup->fileId == fid) { strcpy(fname, pFGroup->files[(*index) % 3].fname); } else { @@ -195,10 +214,10 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_ strcpy(name, fname + strlen(prefix)); } else { // get the named file at the specified index. If not there, return 0 if (*index == TSDB_META_FILE_INDEX) { // get meta file - tsdbGetMetaFileName(pRepo->rootDir, fname); + fname = tsdbGetMetaFileName(pRepo->rootDir); } else { int fid = (*index) / 3; - SFileGroup *pFGroup = tsdbSearchFGroup(pFileH, fid); + SFileGroup *pFGroup = tsdbSearchFGroup(pFileH, fid, TD_EQ); if (pFGroup == NULL) { // not found tfree(sdup); return 0; @@ -218,6 +237,7 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_ *size = fState.st_size; magic = *size; + tfree(fname); return magic; } @@ -229,7 +249,7 @@ void tsdbStartStream(TSDB_REPO_T *repo) { STable *pTable = pMeta->tables[i]; if (pTable && pTable->type == TSDB_STREAM_TABLE) { pTable->cqhandle = (*pRepo->appH.cqCreateFunc)(pRepo->appH.cqH, TALBE_UID(pTable), TABLE_TID(pTable), pTable->sql, - tsdbGetTableSchema(pMeta, pTable)); + tsdbGetTableSchema(pTable)); } } } @@ -270,7 +290,7 @@ int32_t tsdbConfigRepo(TSDB_REPO_T *repo, STsdbCfg *pCfg) { tsdbAlterMaxTables(pRepo, pCfg->maxTables); } - if (configChanged) tsdbSaveConfig(pRepo); + if (configChanged) tsdbSaveConfig(pRepo->rootDir, &pRepo->config); return TSDB_CODE_SUCCESS; } @@ -302,7 +322,7 @@ char *tsdbGetDataFileName(STsdbRepo *pRepo, int fid, int type) { char *fname = malloc(tlen); if (fname == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; + return NULL; } sprintf(fname, "%s/%s/v%df%d.%s", pRepo->rootDir, TSDB_DATA_DIR_NAME, REPO_ID(pRepo), fid, tsdbFileSuffix[type]); @@ -331,6 +351,18 @@ int tsdbUnlockRepo(STsdbRepo *pRepo) { return 0; } +char *tsdbGetDataDirName(char *rootDir) { + int tlen = strlen(rootDir) + strlen(TSDB_DATA_DIR_NAME) + 2; + char *fname = calloc(1, tlen); + if (fname == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return NULL; + } + + snprintf(fname, tlen, "%s/%s", rootDir, TSDB_DATA_DIR_NAME); + return fname; +} + STsdbMeta * tsdbGetMeta(TSDB_REPO_T *pRepo) { return ((STsdbRepo *)pRepo)->tsdbMeta; } STsdbFileH * tsdbGetFile(TSDB_REPO_T *pRepo) { return ((STsdbRepo *)pRepo)->tsdbFileH; } STsdbRepoInfo *tsdbGetStatus(TSDB_REPO_T *pRepo) { return NULL; } @@ -413,7 +445,7 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) { } if (pCfg->minRowsPerFileBlock > pCfg->maxRowsPerFileBlock) { - tsdbError("vgId:%d invalid configuration! minRowsPerFileBlock %d maxRowsPerFileBlock %d" pCfg->tsdbId, + tsdbError("vgId:%d invalid configuration! minRowsPerFileBlock %d maxRowsPerFileBlock %d", pCfg->tsdbId, pCfg->minRowsPerFileBlock, pCfg->maxRowsPerFileBlock); goto _err; } @@ -488,7 +520,7 @@ static int32_t tsdbSaveConfig(char *rootDir, STsdbCfg *pCfg) { if (fd < 0) { tsdbError("vgId:%d failed to open file %s since %s", pCfg->tsdbId, fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); - goto _err + goto _err; } if (twrite(fd, (void *)pCfg, sizeof(STsdbCfg)) < sizeof(STsdbCfg)) { @@ -560,17 +592,6 @@ static char *tsdbGetCfgFname(char *rootDir) { return fname; } -static char *tsdbGetDataDirName(char *rootDir) { - int tlen = strlen(rootDir) + strlen(TSDB_DATA_DIR_NAME) + 2; - char *fname = calloc(1, tlen); - if (fname == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return NULL; - } - - snprintf(fname, tlen, "%s/%s", rootDir, TSDB_DATA_DIR_NAME); - return fname; -} static STsdbRepo *tsdbNewRepo(char *rootDir, STsdbAppH *pAppH, STsdbCfg *pCfg) { STsdbRepo *pRepo = (STsdbRepo *)calloc(1, sizeof(STsdbRepo)); @@ -626,8 +647,8 @@ static void tsdbFreeRepo(STsdbRepo *pRepo) { tsdbFreeFileH(pRepo->tsdbFileH); tsdbFreeBufPool(pRepo->pPool); tsdbFreeMeta(pRepo->tsdbMeta); - tsdbFreeMemTable(pRepo->mem); - tsdbFreeMemTable(pRepo->imem); + // tsdbFreeMemTable(pRepo->mem); + // tsdbFreeMemTable(pRepo->imem); tfree(pRepo->rootDir); pthread_mutex_destroy(&pRepo->mutex); free(pRepo); @@ -660,7 +681,7 @@ static int32_t tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, TSKEY STsdbMeta *pMeta = pRepo->tsdbMeta; int64_t points = 0; - STable *pTable == tsdbGetTableByUid(pMeta, pBlock->uid); + STable *pTable = tsdbGetTableByUid(pMeta, pBlock->uid); if (pTable == NULL || TABLE_TID(pTable) != pBlock->tid) { tsdbError("vgId:%d failed to get table to insert data, uid " PRIu64 " tid %d", REPO_ID(pRepo), pBlock->uid, pBlock->tid); @@ -676,7 +697,7 @@ static int32_t tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, TSKEY // Check schema version int32_t tversion = pBlock->sversion; - STSchema *pSchema = tsdbGetTableSchema(pMeta, pTable); + STSchema *pSchema = tsdbGetTableSchema(pTable); ASSERT(pSchema != NULL); int16_t nversion = schemaVersion(pSchema); if (tversion > nversion) { @@ -701,9 +722,9 @@ static int32_t tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, TSKEY tsdbClearTableCfg(pTableCfg); rpcFreeCont(msg); - pSchema = tsdbGetTableSchemaByVersion(pMeta, pTable, tversion); + pSchema = tsdbGetTableSchemaByVersion(pTable, tversion); } else if (tversion < nversion) { - pSchema = tsdbGetTableSchemaByVersion(pMeta, pTable, tversion); + pSchema = tsdbGetTableSchemaByVersion(pTable, tversion); if (pSchema == NULL) { tsdbError("vgId:%d table %s tid %d invalid schema version %d from client", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), tversion); @@ -829,8 +850,8 @@ static void tsdbAlterKeep(STsdbRepo *pRepo, int32_t keep) { pRepo->tsdbFileH->maxFGroups = maxFiles; } else { pRepo->config.keep = keep; - pRepo->tsdbFileH->fGroup = realloc(pRepo->tsdbFileH->fGroup, sizeof(SFileGroup)); - if (pRepo->tsdbFileH->fGroup == NULL) { + pRepo->tsdbFileH->pFGroup = realloc(pRepo->tsdbFileH->pFGroup, sizeof(SFileGroup)); + if (pRepo->tsdbFileH->pFGroup == NULL) { // TODO: deal with the error } pRepo->tsdbFileH->maxFGroups = maxFiles; @@ -846,7 +867,6 @@ static void tsdbAlterMaxTables(STsdbRepo *pRepo, int32_t maxTables) { STsdbMeta *pMeta = pRepo->tsdbMeta; - pMeta->maxTables = maxTables; pMeta->tables = realloc(pMeta->tables, maxTables * sizeof(STable *)); memset(&pMeta->tables[oldMaxTables], 0, sizeof(STable *) * (maxTables - oldMaxTables)); pRepo->config.maxTables = maxTables; diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 0c8bca9441..f8bb3850e2 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -92,7 +92,7 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { pRepo->mem->tData[TABLE_TID(pTable)] = pTableData; } - ASSERT(pTableData != NULL) && pTableData->uid == TALBE_UID(pTable); + ASSERT((pTableData != NULL) && pTableData->uid == TALBE_UID(pTable)); if (tSkipListPut(pTableData->pData, pNode) == NULL) { tsdbFreeBytes(pRepo, (void *)pNode, bytes); @@ -107,7 +107,7 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { ASSERT(pTableData->numOfRows == tSkipListGetSize(pTableData->pData)); STSchema *pSchema = tsdbGetTableSchema(pTable); - if (schemaNCols(pSchema) > pMemTable->maxCols) pMemTable->maxCols = schemaNCols; + if (schemaNCols(pSchema) > pMemTable->maxCols) pMemTable->maxCols = schemaNCols(pSchema); if (schemaTLen(pSchema) > pMemTable->maxRowBytes) pMemTable->maxRowBytes = schemaTLen(pSchema); } @@ -167,6 +167,8 @@ int tsdbTakeMemSnapshot(STsdbRepo *pRepo, SMemTable **pMem, SMemTable **pIMem) { tsdbRefMemTable(pRepo, *pIMem); if (tsdbUnlockRepo(pRepo) < 0) return -1; + + return 0; } // ---------------- LOCAL FUNCTIONS ---------------- @@ -174,11 +176,11 @@ static FORCE_INLINE STsdbBufBlock *tsdbGetCurrBufBlock(STsdbRepo *pRepo) { ASSERT(pRepo != NULL); if (pRepo->mem == NULL) return NULL; - SListNode *pNode = listTail(pRepo->mem); + SListNode *pNode = listTail(pRepo->mem->bufBlockList); if (pNode == NULL) return NULL; STsdbBufBlock *pBufBlock = NULL; - tdListNodeGetData(pMemTable->bufBlockList, pNode, (void *)(&pBufBlock)); + tdListNodeGetData(pRepo->mem->bufBlockList, pNode, (void *)(&pBufBlock)); return pBufBlock; } @@ -189,7 +191,7 @@ static void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) { int code = 0; if (pBufBlock != NULL && pBufBlock->remain < bytes) { - if (listNEles(pRepo->mem) >= pCfg->totalBlocks / 2) { // need to commit mem + if (listNEles(pRepo->mem->bufBlockList) >= pCfg->totalBlocks / 2) { // need to commit mem if (pRepo->imem) { code = pthread_join(pRepo->commitThread, NULL); if (code != 0) { @@ -358,6 +360,7 @@ static void *tsdbCommitData(void *arg) { STsdbCfg * pCfg = &pRepo->config; SDataCols * pDataCols = NULL; SCommitIter *iters = NULL; + SRWHelper whelper = {0}; ASSERT(pRepo->commit == 1); ASSERT(pMem != NULL); @@ -379,7 +382,7 @@ static void *tsdbCommitData(void *arg) { if ((pDataCols = tdNewDataCols(pMem->maxRowBytes, pMem->maxCols, pCfg->maxRowsPerFileBlock)) == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; tsdbError("vgId:%d failed to init data cols with maxRowBytes %d maxCols %d maxRowsPerFileBlock %d since %s", - REPO_ID(pRepo), pMeta->maxRowBytes, pMeta->maxCols, pCfg->maxRowsPerFileBlock, tstrerror(terrno)); + REPO_ID(pRepo), pMem->maxRowBytes, pMeta->maxCols, pCfg->maxRowsPerFileBlock, tstrerror(terrno)); goto _exit; } diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index f96b0ae1da..2be68cd5de 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -42,7 +42,6 @@ static int tsdbTableSetSName(STableCfg *config, char *sname, bool dup); static int tsdbTableSetSuperUid(STableCfg *config, uint64_t uid); static int tsdbTableSetTagValue(STableCfg *config, SKVRow row, bool dup); static int tsdbTableSetStreamSql(STableCfg *config, char *sql, bool dup); -static void tsdbClearTableCfg(STableCfg *config); static void * tsdbEncodeTableName(void *buf, tstr *name); static void * tsdbDecodeTableName(void *buf, tstr **name); static void * tsdbEncodeTable(void *buf, STable *pTable); @@ -122,7 +121,7 @@ int tsdbDropTable(TSDB_REPO_T *repo, STableId tableId) { tsdbTrace("vgId:%d, table %s is dropped! tid:%d, uid:%" PRId64, pRepo->config.tsdbId, varDataVal(pTable->name), tableId.tid, tableId.uid); - if (tsdbRemoveTableFromMeta(pMeta, pTable, true) < 0) return -1; + tsdbRemoveTableFromMeta(pMeta, pTable, true); return 0; } @@ -132,7 +131,7 @@ void *tsdbGetTableTagVal(TSDB_REPO_T *repo, const STableId *id, int32_t colId, i STsdbMeta *pMeta = tsdbGetMeta(repo); STable * pTable = tsdbGetTableByUid(pMeta, id->uid); - STSchema *pSchema = tsdbGetTableTagSchema(pMeta, pTable); + STSchema *pSchema = tsdbGetTableTagSchema(pTable); STColumn *pCol = tdGetColOfID(pSchema, colId); if (pCol == NULL) { return NULL; // No matched tag volumn @@ -255,7 +254,7 @@ int tsdbUpdateTagValue(TSDB_REPO_T *repo, SUpdateTableTagValMsg *pMsg) { return -1; } if (TABLE_TID(pTable) != htonl(pMsg->tid)) { - terrno = TSDB_CODE_TDB_INVALID_TABLE_ID + terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; return -1; } @@ -457,7 +456,7 @@ int tsdbUpdateTable(STsdbMeta *pMeta, STable *pTable, STableCfg *pCfg) { isChanged = true; } - STSchema *pTSchema = tsdbGetTableSchema(pMeta, pTable); + STSchema *pTSchema = tsdbGetTableSchema(pTable); if (schemaVersion(pTSchema) < schemaVersion(pCfg->schema)) { if (pTable->numOfSchemas < TSDB_MAX_TABLE_SCHEMAS) { pTable->schema[pTable->numOfSchemas++] = tdDupSchema(pCfg->schema); @@ -475,8 +474,8 @@ int tsdbUpdateTable(STsdbMeta *pMeta, STable *pTable, STableCfg *pCfg) { if (isChanged) { char *buf = malloc(1024 * 1024); int bufLen = 0; - tsdbEncodeTable(pTable, buf, &bufLen); - tsdbInsertMetaRecord(pMeta->mfh, pTable->tableId.uid, buf, bufLen); + tsdbEncodeTable(buf, pTable); + // tsdbInsertMetaRecord(pMeta->mfh, pTable->tableId.uid, buf, bufLen); free(buf); } @@ -966,7 +965,7 @@ static int tsdbTableSetStreamSql(STableCfg *config, char *sql, bool dup) { return 0; } -static void tsdbClearTableCfg(STableCfg *config) { +void tsdbClearTableCfg(STableCfg *config) { if (config) { if (config->schema) tdFreeSchema(config->schema); if (config->tagSchema) tdFreeSchema(config->tagSchema); @@ -985,7 +984,7 @@ static void *tsdbEncodeTableName(void *buf, tstr *name) { memcpy(pBuf, name->data, name->len); pBuf = POINTER_SHIFT(pBuf, name->len); - return POINTER_DISTANCE(pBuf, buf); + return pBuf; } static void *tsdbDecodeTableName(void *buf, tstr **name) { diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index d1a41ce550..de81606a08 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -20,6 +20,39 @@ #include "tscompression.h" #include "tsdbMain.h" +static bool tsdbShouldCreateNewLast(SRWHelper *pHelper); +static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDataCols, int rowsToWrite, + SCompBlock *pCompBlock, bool isLast, bool isSuperBlock); +static int compareKeyBlock(const void *arg1, const void *arg2); +static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDataCols); +static int compTSKEY(const void *key1, const void *key2); +static int tsdbAdjustInfoSizeIfNeeded(SRWHelper *pHelper, size_t esize); +static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx); +static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, int rowsAdded); +static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx); +static int tsdbGetRowsInRange(SDataCols *pDataCols, TSKEY minKey, TSKEY maxKey); +static void tsdbResetHelperFileImpl(SRWHelper *pHelper); +static int tsdbInitHelperFile(SRWHelper *pHelper); +static void tsdbDestroyHelperFile(SRWHelper *pHelper); +static void tsdbResetHelperTableImpl(SRWHelper *pHelper); +static void tsdbResetHelperTable(SRWHelper *pHelper); +static void tsdbInitHelperTable(SRWHelper *pHelper); +static void tsdbDestroyHelperTable(SRWHelper *pHelper); +static void tsdbResetHelperBlockImpl(SRWHelper *pHelper); +static void tsdbResetHelperBlock(SRWHelper *pHelper); +static int tsdbInitHelperBlock(SRWHelper *pHelper); +static int tsdbInitHelper(SRWHelper *pHelper, STsdbRepo *pRepo, tsdb_rw_helper_t type); +static int comparColIdCompCol(const void *arg1, const void *arg2); +static int comparColIdDataCol(const void *arg1, const void *arg2); +static int tsdbLoadSingleColumnData(int fd, SCompBlock *pCompBlock, SCompCol *pCompCol, void *buf); +static int tsdbLoadSingleBlockDataCols(SRWHelper *pHelper, SCompBlock *pCompBlock, int16_t *colIds, int numOfColIds, + SDataCols *pDataCols); +static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, char *content, int32_t len, int8_t comp, int numOfRows, + int maxPoints, char *buffer, int bufferSize); +static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *pDataCols); +static void *tsdbEncodeSCompIdx(void *buf, SCompIdx *pIdx); +static void *tsdbDecodeSCompIdx(void *buf, SCompIdx *pIdx); + // ---------------------- INTERNAL FUNCTIONS ---------------------- int tsdbInitReadHelper(SRWHelper *pHelper, STsdbRepo *pRepo) { return tsdbInitHelper(pHelper, pRepo, TSDB_READ_HELPER); @@ -69,7 +102,7 @@ int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { pHelper->files.headF = pGroup->files[TSDB_FILE_TYPE_HEAD]; pHelper->files.dataF = pGroup->files[TSDB_FILE_TYPE_DATA]; pHelper->files.lastF = pGroup->files[TSDB_FILE_TYPE_LAST]; - if (TSDB_HELPER_TYPE(pHelper) == TSDB_WRITE_HELPER) { + if (helperType(pHelper) == TSDB_WRITE_HELPER) { char *fnameDup = strdup(pHelper->files.headF.fname); if (fnameDup == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; @@ -437,7 +470,7 @@ int tsdbLoadCompInfo(SRWHelper *pHelper, void *target) { return 0; } -int tsdbLoadCompData(SRWHelper *pHelper, SCompBlock *pCompBlock, void *target) { +int tsdbloadcompdata(srwhelper *phelper, scompblock *pcompblock, void *target) { ASSERT(pCompBlock->numOfSubBlocks <= 1); int fd = (pCompBlock->last) ? pHelper->files.lastF.fd : pHelper->files.dataF.fd; diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 736be5e788..d6bab10913 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -589,7 +589,7 @@ static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlo pCheckInfo->pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pRepo->config.maxRowsPerFileBlock); } - tdInitDataCols(pCheckInfo->pDataCols, tsdbGetTableSchema(tsdbGetMeta(pQueryHandle->pTsdb), pCheckInfo->pTableObj)); + tdInitDataCols(pCheckInfo->pDataCols, tsdbGetTableSchema(pCheckInfo->pTableObj)); if (tsdbLoadBlockData(&(pQueryHandle->rhelper), pBlock, NULL) == 0) { SDataBlockLoadInfo* pBlockLoadInfo = &pQueryHandle->dataBlockLoadInfo; @@ -836,7 +836,7 @@ static void copyOneRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, char* pData = NULL; // the schema version info is embeded in SDataRow - STSchema* pSchema = tsdbGetTableSchemaByVersion(pMeta, pTable, dataRowVersion(row)); + STSchema* pSchema = tsdbGetTableSchemaByVersion(pTable, dataRowVersion(row)); int32_t numOfRowCols = schemaNCols(pSchema); int32_t i = 0, j = 0; diff --git a/src/util/inc/tref.h b/src/util/inc/tref.h index 9483c1cc35..fa2517e19a 100644 --- a/src/util/inc/tref.h +++ b/src/util/inc/tref.h @@ -31,7 +31,7 @@ typedef void (*_ref_fn_t)(const void* pObj); _ref_fn_t end; \ } _ref_func = {.begin = (s), .end = (e)}; -#define T_REF_INC(x) (atomic_add_fetch_16(&((x)->_ref.val), 1)); +#define T_REF_INC(x) (atomic_add_fetch_16(&((x)->_ref.val), 1)) #define T_REF_INC_WITH_CB(x, p) \ do { \ @@ -41,7 +41,7 @@ typedef void (*_ref_fn_t)(const void* pObj); } \ } while (0) -#define T_REF_DEC(x) (atomic_sub_fetch_16(&((x)->_ref.val), 1)); +#define T_REF_DEC(x) (atomic_sub_fetch_16(&((x)->_ref.val), 1)) #define T_REF_DEC_WITH_CB(x, p) \ do { \ From 5ff68c3c2f51c18b98cc06c83717edf120b657a2 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 17 Jun 2020 06:28:10 +0000 Subject: [PATCH 30/65] TD-353 --- src/tsdb/inc/tsdbMain.h | 49 ++++------------------- src/tsdb/src/tsdbFile.c | 60 ++++++++++++++++------------ src/tsdb/src/tsdbMain.c | 51 +++++++++++++++++++++-- src/tsdb/src/tsdbMemTable.c | 26 ++++++------ src/tsdb/src/tsdbMeta.c | 36 ++++++++--------- src/tsdb/src/tsdbRWHelper.c | 80 +++++++++++++++++++------------------ src/tsdb/src/tsdbRead.c | 2 +- 7 files changed, 164 insertions(+), 140 deletions(-) diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index fa1b98ec94..a4a127069d 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -117,6 +117,8 @@ typedef enum { TSDB_FILE_TYPE_DATA, TSDB_FILE_TYPE_LAST, TSDB_FILE_TYPE_MAX, + TSDB_FILE_TYPE_NHEAD, + TSDB_FILE_TYPE_NLAST } TSDB_FILE_TYPE; typedef struct { @@ -272,7 +274,7 @@ typedef struct { #define TABLE_TYPE(t) (t)->type #define TABLE_NAME(t) (t)->name #define TABLE_CHAR_NAME(t) TABLE_NAME(t)->data -#define TALBE_UID(t) (t)->tableId.uid +#define TABLE_UID(t) (t)->tableId.uid #define TABLE_TID(t) (t)->tableId.tid #define TABLE_SUID(t) (t)->suid #define TABLE_LASTKEY(t) (t)->lastKey @@ -328,6 +330,9 @@ int tsdbCreateFile(SFile* pFile, STsdbRepo* pRepo, int fid, int type); SFileGroup* tsdbSearchFGroup(STsdbFileH* pFileH, int fid, int flags); void tsdbFitRetention(STsdbRepo* pRepo); int tsdbUpdateFileHeader(SFile* pFile, uint32_t version); +void* tsdbEncodeSFileInfo(void* buf, const STsdbFileInfo* pInfo); +void* tsdbDecodeSFileInfo(void* buf, STsdbFileInfo* pInfo); +int tsdbCpySFile(SFile* src, SFile* dst); // ------------------ tsdbRWHelper.c #define TSDB_HELPER_CLEAR_STATE 0x0 // Clear state @@ -359,17 +364,16 @@ int tsdbWriteCompInfo(SRWHelper* pHelper); int tsdbWriteCompIdx(SRWHelper* pHelper); int tsdbLoadCompIdx(SRWHelper* pHelper, void* target); int tsdbLoadCompInfo(SRWHelper* pHelper, void* target); -int tsdbloadcompdata(srwhelper* phelper, scompblock* pcompblock, void* target); +int tsdbLoadCompData(SRWHelper* phelper, SCompBlock* pcompblock, void* target); void tsdbGetDataStatis(SRWHelper* pHelper, SDataStatis* pStatis, int numOfCols); int tsdbLoadBlockDataCols(SRWHelper* pHelper, SDataCols* pDataCols, int blkIdx, int16_t* colIds, int numOfColIds); int tsdbLoadBlockData(SRWHelper* pHelper, SCompBlock* pCompBlock, SDataCols* target); int tsdbUpdateFileHeader(SFile* pFile, uint32_t version); -void* tsdbEncodeSFileInfo(void* buf, const STsdbFileInfo* pInfo); -void* tsdbDecodeSFileInfo(void* buf, STsdbFileInfo* pInfo); // ------------------ tsdbMain.c #define REPO_ID(r) (r)->config.tsdbId #define IS_REPO_LOCKED(r) (r)->repoLocked +#define TSDB_SUBMIT_MSG_HEAD_SIZE sizeof(SSubmitMsg) char* tsdbGetMetaFileName(char* rootDir); char* tsdbGetDataFileName(STsdbRepo* pRepo, int fid, int type); @@ -379,43 +383,6 @@ char* tsdbGetDataDirName(char* rootDir); STsdbMeta* tsdbGetMeta(TSDB_REPO_T* pRepo); STsdbFileH* tsdbGetFile(TSDB_REPO_T* pRepo); -#if 0 - -// --------- Helper state - -int tsdbInitReadHelper(SRWHelper *pHelper, STsdbRepo *pRepo); -int tsdbInitWriteHelper(SRWHelper *pHelper, STsdbRepo *pRepo); -void tsdbDestroyHelper(SRWHelper *pHelper); -void tsdbResetHelper(SRWHelper *pHelper); - -// --------- For set operations -int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup); -void tsdbSetHelperTable(SRWHelper *pHelper, STable *pTable, STsdbRepo *pRepo); -int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError); - -// --------- For read operations -int tsdbLoadCompIdx(SRWHelper *pHelper, void *target); -int tsdbLoadCompInfo(SRWHelper *pHelper, void *target); -int tsdbLoadCompData(SRWHelper *pHelper, SCompBlock *pCompBlock, void *target); -int tsdbLoadBlockDataCols(SRWHelper *pHelper, SDataCols *pDataCols, int blkIdx, int16_t *colIds, int numOfColIds); -int tsdbLoadBlockData(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *target); -void tsdbGetDataStatis(SRWHelper *pHelper, SDataStatis *pStatis, int numOfCols); - -// --------- For write operations -int tsdbWriteDataBlock(SRWHelper *pHelper, SDataCols *pDataCols); -int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper); -int tsdbWriteCompInfo(SRWHelper *pHelper); -int tsdbWriteCompIdx(SRWHelper *pHelper); - -// --------- Other functions need to further organize -void tsdbFitRetention(STsdbRepo *pRepo); -int tsdbAlterCacheTotalBlocks(STsdbRepo *pRepo, int totalBlocks); -void tsdbAdjustCacheBlocks(STsdbCache *pCache); -int32_t tsdbGetMetaFileName(char *rootDir, char *fname); -int tsdbUpdateFileHeader(SFile *pFile, uint32_t version); - -#endif - #ifdef __cplusplus } #endif diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 8af9441227..4556e79db4 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -29,15 +29,13 @@ #include "tutil.h" #include "ttime.h" -const char *tsdbFileSuffix[] = {".head", ".data", ".last"}; +const char *tsdbFileSuffix[] = {".head", ".data", ".last", "", ".h", ".h"}; static int tsdbInitFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type); static void tsdbDestroyFile(SFile *pFile); static int compFGroup(const void *arg1, const void *arg2); static int keyFGroupCompFunc(const void *key, const void *fgroup); static void tsdbRemoveFileGroup(STsdbRepo *pRepo, SFileGroup *pFGroup); -static void *tsdbEncodeSFileInfo(void *buf, const STsdbFileInfo *pInfo); -static void *tsdbDecodeSFileInfo(void *buf, STsdbFileInfo *pInfo); // ---------------- INTERNAL FUNCTIONS ---------------- STsdbFileH *tsdbNewFileH(STsdbCfg *pCfg) { @@ -323,6 +321,40 @@ int tsdbUpdateFileHeader(SFile *pFile, uint32_t version) { return 0; } +void *tsdbEncodeSFileInfo(void *buf, const STsdbFileInfo *pInfo) { + buf = taosEncodeFixedU32(buf, pInfo->offset); + buf = taosEncodeFixedU32(buf, pInfo->len); + buf = taosEncodeFixedU64(buf, pInfo->size); + buf = taosEncodeFixedU64(buf, pInfo->tombSize); + buf = taosEncodeFixedU32(buf, pInfo->totalBlocks); + buf = taosEncodeFixedU32(buf, pInfo->totalSubBlocks); + + return buf; +} + +void *tsdbDecodeSFileInfo(void *buf, STsdbFileInfo *pInfo) { + buf = taosDecodeFixedU32(buf, &(pInfo->offset)); + buf = taosDecodeFixedU32(buf, &(pInfo->len)); + buf = taosDecodeFixedU64(buf, &(pInfo->size)); + buf = taosDecodeFixedU64(buf, &(pInfo->tombSize)); + buf = taosDecodeFixedU32(buf, &(pInfo->totalBlocks)); + buf = taosDecodeFixedU32(buf, &(pInfo->totalSubBlocks)); + + return buf; +} + +int tsdbCpySFile(SFile *src, SFile *dst) { + *dst = *src; + dst->fname = strdup(dst->fname); + + if (dst->fname == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + + return 0; +} + // ---------------- LOCAL FUNCTIONS ---------------- static int tsdbInitFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type) { uint32_t version; @@ -404,26 +436,4 @@ static void tsdbRemoveFileGroup(STsdbRepo *pRepo, SFileGroup *pFGroup) { remove(fileGroup.files[type].fname); tsdbDestroyFile(&fileGroup.files[type]); } -} - -static void *tsdbEncodeSFileInfo(void *buf, const STsdbFileInfo *pInfo) { - buf = taosEncodeFixedU32(buf, pInfo->offset); - buf = taosEncodeFixedU32(buf, pInfo->len); - buf = taosEncodeFixedU64(buf, pInfo->size); - buf = taosEncodeFixedU64(buf, pInfo->tombSize); - buf = taosEncodeFixedU32(buf, pInfo->totalBlocks); - buf = taosEncodeFixedU32(buf, pInfo->totalSubBlocks); - - return buf; -} - -static void *tsdbDecodeSFileInfo(void *buf, STsdbFileInfo *pInfo) { - buf = taosDecodeFixedU32(buf, &(pInfo->offset)); - buf = taosDecodeFixedU32(buf, &(pInfo->len)); - buf = taosDecodeFixedU64(buf, &(pInfo->size)); - buf = taosDecodeFixedU64(buf, &(pInfo->tombSize)); - buf = taosDecodeFixedU32(buf, &(pInfo->totalBlocks)); - buf = taosDecodeFixedU32(buf, &(pInfo->totalSubBlocks)); - - return buf; } \ No newline at end of file diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index fe7214f01e..e6cf865399 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -63,6 +63,8 @@ static int tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIt static void tsdbAlterCompression(STsdbRepo *pRepo, int8_t compression); static void tsdbAlterKeep(STsdbRepo *pRepo, int32_t keep); static void tsdbAlterMaxTables(STsdbRepo *pRepo, int32_t maxTables); +static int tsdbAlterCacheTotalBlocks(STsdbRepo *pRepo, int totalBlocks); +static int keyFGroupCompFunc(const void *key, const void *fgroup); // Function declaration int32_t tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg) { @@ -198,7 +200,7 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_ } } else { SFileGroup *pFGroup = - taosbsearch(&fid, pFileH->pFGroup, pFileH->nFGroups, sizeof(SFileGroup), compFGroupKey, TD_GE); + taosbsearch(&fid, pFileH->pFGroup, pFileH->nFGroups, sizeof(SFileGroup), keyFGroupCompFunc, TD_GE); if (pFGroup->fileId == fid) { strcpy(fname, pFGroup->files[(*index) % 3].fname); } else { @@ -248,7 +250,7 @@ void tsdbStartStream(TSDB_REPO_T *repo) { for (int i = 0; i < pRepo->config.maxTables; i++) { STable *pTable = pMeta->tables[i]; if (pTable && pTable->type == TSDB_STREAM_TABLE) { - pTable->cqhandle = (*pRepo->appH.cqCreateFunc)(pRepo->appH.cqH, TALBE_UID(pTable), TABLE_TID(pTable), pTable->sql, + pTable->cqhandle = (*pRepo->appH.cqCreateFunc)(pRepo->appH.cqH, TABLE_UID(pTable), TABLE_TID(pTable), pTable->sql, tsdbGetTableSchema(pTable)); } } @@ -744,7 +746,7 @@ static int32_t tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, TSKEY if (dataRowKey(row) < minKey || dataRowKey(row) > maxKey) { tsdbError("vgId:%d table %s tid %d uid %ld timestamp is out of range! now " PRId64 " maxKey " PRId64 " minKey " PRId64, - REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TALBE_UID(pTable), now, minKey, maxKey); + REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), now, minKey, maxKey); terrno = TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE; return -1; } @@ -802,7 +804,7 @@ static int tsdbRestoreInfo(STsdbRepo *pRepo) { SFileGroup *pFGroup = NULL; SFileGroupIter iter; - SRWHelper rhelper = {{0}}; + SRWHelper rhelper = {0}; if (tsdbInitReadHelper(&rhelper, pRepo) < 0) goto _err; @@ -874,6 +876,47 @@ static void tsdbAlterMaxTables(STsdbRepo *pRepo, int32_t maxTables) { tsdbTrace("vgId:%d, tsdb maxTables is changed from %d to %d!", pRepo->config.tsdbId, oldMaxTables, maxTables); } +static int keyFGroupCompFunc(const void *key, const void *fgroup) { + int fid = *(int *)key; + SFileGroup *pFGroup = (SFileGroup *)fgroup; + if (fid == pFGroup->fileId) { + return 0; + } else { + return fid > pFGroup->fileId ? 1 : -1; + } +} + +static int tsdbAlterCacheTotalBlocks(STsdbRepo *pRepo, int totalBlocks) { + // TODO + // STsdbCache *pCache = pRepo->tsdbCache; + // int oldNumOfBlocks = pCache->totalCacheBlocks; + + // tsdbLockRepo((TsdbRepoT *)pRepo); + + // ASSERT(pCache->totalCacheBlocks != totalBlocks); + + // if (pCache->totalCacheBlocks < totalBlocks) { + // ASSERT(pCache->totalCacheBlocks == pCache->pool.numOfCacheBlocks); + // int blocksToAdd = pCache->totalCacheBlocks - totalBlocks; + // pCache->totalCacheBlocks = totalBlocks; + // for (int i = 0; i < blocksToAdd; i++) { + // if (tsdbAddCacheBlockToPool(pCache) < 0) { + // tsdbUnLockRepo((TsdbRepoT *)pRepo); + // tsdbError("tsdbId:%d, failed to add cache block to cache pool", pRepo->config.tsdbId); + // return -1; + // } + // } + // } else { + // pCache->totalCacheBlocks = totalBlocks; + // tsdbAdjustCacheBlocks(pCache); + // } + // pRepo->config.totalBlocks = totalBlocks; + + // tsdbUnLockRepo((TsdbRepoT *)pRepo); + // tsdbTrace("vgId:%d, tsdb total cache blocks changed from %d to %d", pRepo->config.tsdbId, oldNumOfBlocks, totalBlocks); + return 0; +} + #if 0 TSKEY tsdbGetTableLastKey(TSDB_REPO_T *repo, uint64_t uid) { diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index f8bb3850e2..f342431378 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -51,14 +51,16 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { TSKEY key = dataRowKey(row); SMemTable * pMemTable = pRepo->mem; STableData *pTableData = NULL; + SSkipList * pSList = NULL; int bytes = 0; if (pMemTable != NULL && pMemTable->tData[TABLE_TID(pTable)] != NULL && - pMemTable->tData[TABLE_TID(pTable)]->uid == TALBE_UID(pTable)) { + pMemTable->tData[TABLE_TID(pTable)]->uid == TABLE_UID(pTable)) { pTableData = pMemTable->tData[TABLE_TID(pTable)]; + pSList = pTableData->pData; } - tSkipListNewNodeInfo(pTableData, &level, &headSize); + tSkipListNewNodeInfo(pSList, &level, &headSize); bytes = headSize + dataRowLen(row); SSkipListNode *pNode = tsdbAllocBytes(pRepo, bytes); @@ -75,7 +77,7 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { pMemTable = pRepo->mem; pTableData = pMemTable->tData[TABLE_TID(pTable)]; - if (pTableData == NULL || pTableData->uid != TALBE_UID(pTable)) { + if (pTableData == NULL || pTableData->uid != TABLE_UID(pTable)) { if (pTableData != NULL) { // destroy the table skiplist (may have race condition problem) pMemTable->tData[TABLE_TID(pTable)] = NULL; tsdbFreeTableData(pTableData); @@ -92,7 +94,7 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { pRepo->mem->tData[TABLE_TID(pTable)] = pTableData; } - ASSERT((pTableData != NULL) && pTableData->uid == TALBE_UID(pTable)); + ASSERT((pTableData != NULL) && pTableData->uid == TABLE_UID(pTable)); if (tSkipListPut(pTableData->pData, pNode) == NULL) { tsdbFreeBytes(pRepo, (void *)pNode, bytes); @@ -112,7 +114,7 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { } tsdbTrace("vgId:%d a row is inserted to table %s tid %d uid %" PRIu64 " key %" PRIu64, REPO_ID(pRepo), - TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TALBE_UID(pTable), key); + TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), key); return 0; } @@ -322,7 +324,7 @@ static STableData *tsdbNewTableData(STsdbCfg *pCfg, STable *pTable) { goto _err; } - pTableData->uid = TALBE_UID(pTable); + pTableData->uid = TABLE_UID(pTable); pTableData->keyFirst = INT64_MAX; pTableData->keyLast = 0; pTableData->numOfRows = 0; @@ -355,7 +357,6 @@ static char *tsdbGetTsTupleKey(const void *data) { return dataRowTuple(data); } static void *tsdbCommitData(void *arg) { STsdbRepo * pRepo = (STsdbRepo *)arg; - STsdbMeta * pMeta = pRepo->tsdbMeta; SMemTable * pMem = pRepo->imem; STsdbCfg * pCfg = &pRepo->config; SDataCols * pDataCols = NULL; @@ -382,7 +383,7 @@ static void *tsdbCommitData(void *arg) { if ((pDataCols = tdNewDataCols(pMem->maxRowBytes, pMem->maxCols, pCfg->maxRowsPerFileBlock)) == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; tsdbError("vgId:%d failed to init data cols with maxRowBytes %d maxCols %d maxRowsPerFileBlock %d since %s", - REPO_ID(pRepo), pMem->maxRowBytes, pMeta->maxCols, pCfg->maxRowsPerFileBlock, tstrerror(terrno)); + REPO_ID(pRepo), pMem->maxRowBytes, pMem->maxCols, pCfg->maxRowsPerFileBlock, tstrerror(terrno)); goto _exit; } @@ -446,7 +447,6 @@ static void tsdbGetFidKeyRange(int daysPerFile, int8_t precision, int fileId, TS static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHelper *pHelper, SDataCols *pDataCols) { char * dataDir = NULL; STsdbMeta * pMeta = pRepo->tsdbMeta; - STsdbFileH *pFileH = pRepo->tsdbFileH; STsdbCfg * pCfg = &pRepo->config; SFileGroup *pGroup = NULL; @@ -467,7 +467,7 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe return -1; } - if ((pGroup = tsdbCreateFGroupIfNeed(pFileH, dataDir, fid, pCfg->maxTables)) == NULL) { + if ((pGroup = tsdbCreateFGroupIfNeed(pRepo, dataDir, fid, pCfg->maxTables)) == NULL) { tsdbError("vgId:%d failed to create file group %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno)); goto _err; } @@ -485,7 +485,7 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe SCommitIter *pIter = iters + tid; if (pIter->pTable == NULL) continue; - tsdbSetHelperTable(pHelper, pTable, pRepo); + tsdbSetHelperTable(pHelper, pIter->pTable, pRepo); if (pIter->pIter != NULL) { tdInitDataCols(pDataCols, tsdbGetTableSchema(pIter->pTable)); @@ -574,7 +574,7 @@ static SCommitIter *tsdbCreateTableIters(STsdbRepo *pRepo) { if (tsdbUnlockRepoMeta(pRepo) < 0) goto _err; for (int i = 0; i < pCfg->maxTables; i++) { - if ((iters[i].pTable != NULL) && (pMem->tData[i] != NULL) && (TALBE_UID(iters[i].pTable) == pMem->tData[i]->uid)) { + if ((iters[i].pTable != NULL) && (pMem->tData[i] != NULL) && (TABLE_UID(iters[i].pTable) == pMem->tData[i]->uid)) { if ((iters[i].pIter = tSkipListCreateIter(pMem->tData[i]->pData)) == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _err; @@ -624,7 +624,7 @@ static int tsdbReadRowsFromCache(STsdbMeta *pMeta, STable *pTable, SSkipListIter if (dataRowKey(row) > maxKey) break; if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) { - pSchema = tsdbGetTableSchemaByVersion(pMeta, pTable, dataRowVersion(row)); + pSchema = tsdbGetTableSchemaByVersion(pTable, dataRowVersion(row)); if (pSchema == NULL) { // TODO: deal with the error here ASSERT(false); diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 2be68cd5de..99f5c3173d 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -58,7 +58,7 @@ int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg) { STable *pTable = tsdbGetTableByUid(pMeta, pCfg->tableId.uid); if (pTable != NULL) { tsdbError("vgId:%d table %s already exists, tid %d uid %" PRId64, REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), - TABLE_TID(pTable), TALBE_UID(pTable)); + TABLE_TID(pTable), TABLE_UID(pTable)); return TSDB_CODE_TDB_TABLE_ALREADY_EXIST; } @@ -121,7 +121,7 @@ int tsdbDropTable(TSDB_REPO_T *repo, STableId tableId) { tsdbTrace("vgId:%d, table %s is dropped! tid:%d, uid:%" PRId64, pRepo->config.tsdbId, varDataVal(pTable->name), tableId.tid, tableId.uid); - tsdbRemoveTableFromMeta(pMeta, pTable, true); + tsdbRemoveTableFromMeta(pRepo, pTable, true); return 0; } @@ -473,7 +473,6 @@ int tsdbUpdateTable(STsdbMeta *pMeta, STable *pTable, STableCfg *pCfg) { if (isChanged) { char *buf = malloc(1024 * 1024); - int bufLen = 0; tsdbEncodeTable(buf, pTable); // tsdbInsertMetaRecord(pMeta->mfh, pTable->tableId.uid, buf, bufLen); free(buf); @@ -536,7 +535,6 @@ static int tsdbCompareSchemaVersion(const void *key1, const void *key2) { static int tsdbRestoreTable(void *pHandle, void *cont, int contLen) { STsdbRepo *pRepo = (STsdbRepo *)pHandle; - STsdbMeta *pMeta = pRepo->tsdbMeta; STable * pTable = NULL; if (!taosCheckChecksumWhole((uint8_t *)cont, contLen)) { @@ -546,21 +544,22 @@ static int tsdbRestoreTable(void *pHandle, void *cont, int contLen) { tsdbDecodeTable(cont, &pTable); - if (tsdbAddTableToMeta(pMeta, pTable, false) < 0) { + if (tsdbAddTableToMeta(pRepo, pTable, false) < 0) { tsdbFreeTable(pTable); return -1; } tsdbTrace("vgId:%d table %s tid %d uid %" PRIu64 " is restored from file", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), - TABLE_TID(pTable), TALBE_UID(pTable)); + TABLE_TID(pTable), TABLE_UID(pTable)); return 0; } static void tsdbOrgMeta(void *pHandle) { STsdbRepo *pRepo = (STsdbRepo *)pHandle; + STsdbMeta *pMeta = pRepo->tsdbMeta; STsdbCfg * pCfg = &pRepo->config; - for (int i = 1; i < pMeta->maxTables; i++) { + for (int i = 1; i < pCfg->maxTables; i++) { STable *pTable = pMeta->tables[i]; if (pTable != NULL && pTable->type == TSDB_CHILD_TABLE) { tsdbAddTableIntoIndex(pMeta, pTable); @@ -572,7 +571,7 @@ static char *getTagIndexKey(const void *pData) { STable *pTable = *(STable **)pData; STSchema *pSchema = tsdbGetTableTagSchema(pTable); - STColumn *pCol = schemaColAt(DEFAULT_TAG_INDEX_COLUMN); + STColumn *pCol = schemaColAt(pSchema, DEFAULT_TAG_INDEX_COLUMN); void * res = tdGetKVRowValOfCol(pTable->tagVal, pCol->colId); return res; } @@ -596,7 +595,7 @@ static STable *tsdbNewTable(STableCfg *pCfg, bool isSuper) { goto _err; } STR_WITH_SIZE_TO_VARSTR(pTable->name, pCfg->sname, tsize); - TALBE_UID(pTable) = pCfg->superUid; + TABLE_UID(pTable) = pCfg->superUid; TABLE_TID(pTable) = -1; TABLE_SUID(pTable) = -1; pTable->pSuper = NULL; @@ -627,7 +626,7 @@ static STable *tsdbNewTable(STableCfg *pCfg, bool isSuper) { goto _err; } STR_WITH_SIZE_TO_VARSTR(pTable->name, pCfg->name, tsize); - TALBE_UID(pTable) = pCfg->tableId.uid; + TABLE_UID(pTable) = pCfg->tableId.uid; TABLE_TID(pTable) = pCfg->tableId.tid; if (pCfg->type == TSDB_CHILD_TABLE) { @@ -741,7 +740,7 @@ static int tsdbAddTableToMeta(STsdbRepo *pRepo, STable *pTable, bool addIdx) { if (addIdx && tsdbUnlockRepoMeta(pRepo) < 0) return -1; tsdbTrace("vgId:%d table %s tid %d uid %" PRIu64 " is added to meta", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), - TABLE_TID(pTable), TALBE_UID(pTable)); + TABLE_TID(pTable), TABLE_UID(pTable)); return 0; _err: @@ -778,7 +777,7 @@ static void tsdbRemoveTableFromMeta(STsdbRepo *pRepo, STable *pTable, bool rmFro pMeta->nTables--; } - taosHashRemove(pMeta->uid, (char *)(&(TALBE_UID(pTable))), sizeof(TALBE_UID(pTable))); + taosHashRemove(pMeta->uidMap, (char *)(&(TABLE_UID(pTable))), sizeof(TABLE_UID(pTable))); if (rmFromIdx) tsdbUnlockRepoMeta(pRepo); } @@ -804,8 +803,7 @@ static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable) { } pNode->level = level; - SSkipList *list = pSTable->pIndex; - (STable *)(SL_GET_NODE_DATA(pNode)) = pTable; + memcpy(SL_GET_NODE_DATA(pNode), &pTable, sizeof(STable *)); tSkipListPut(pSTable->pIndex, pNode); return 0; @@ -814,7 +812,7 @@ static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable) { static int tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable) { ASSERT(pTable->type == TSDB_CHILD_TABLE && pTable != NULL); - STable *pSTable = tsdbGetTableByUid(pMeta, pTable->superUid); + STable *pSTable = pTable->pSuper; ASSERT(pSTable != NULL); STSchema *pSchema = tsdbGetTableTagSchema(pTable); @@ -1008,7 +1006,7 @@ static void *tsdbEncodeTable(void *buf, STable *pTable) { buf = taosEncodeFixedU8(buf, pTable->type); buf = tsdbEncodeTableName(buf, pTable->name); - buf = taosEncodeFixedU64(buf, TALBE_UID(pTable)); + buf = taosEncodeFixedU64(buf, TABLE_UID(pTable)); buf = taosEncodeFixedI32(buf, TABLE_TID(pTable)); if (TABLE_TYPE(pTable) == TSDB_CHILD_TABLE) { @@ -1038,10 +1036,12 @@ static void *tsdbDecodeTable(void *buf, STable **pRTable) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return NULL; } + uint8_t type = 0; - buf = taosDecodeFixedU8(buf, &(pTable->type)); + buf = taosDecodeFixedU8(buf, &type); + pTable->type = type; buf = tsdbDecodeTableName(buf, &(pTable->name)); - buf = taosDecodeFixedU64(buf, &TALBE_UID(pTable)); + buf = taosDecodeFixedU64(buf, &TABLE_UID(pTable)); buf = taosDecodeFixedI32(buf, &TABLE_TID(pTable)); if (TABLE_TYPE(pTable) == TSDB_CHILD_TABLE) { diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index de81606a08..c7ebbf460c 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -52,6 +52,7 @@ static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, char *content, int static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *pDataCols); static void *tsdbEncodeSCompIdx(void *buf, SCompIdx *pIdx); static void *tsdbDecodeSCompIdx(void *buf, SCompIdx *pIdx); +static void tsdbDestroyHelperBlock(SRWHelper *pHelper); // ---------------------- INTERNAL FUNCTIONS ---------------------- int tsdbInitReadHelper(SRWHelper *pHelper, STsdbRepo *pRepo) { @@ -99,20 +100,13 @@ int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { // Set the files pHelper->files.fid = pGroup->fileId; - pHelper->files.headF = pGroup->files[TSDB_FILE_TYPE_HEAD]; - pHelper->files.dataF = pGroup->files[TSDB_FILE_TYPE_DATA]; - pHelper->files.lastF = pGroup->files[TSDB_FILE_TYPE_LAST]; + tsdbCpySFile(&pHelper->files.headF, &pGroup->files[TSDB_FILE_TYPE_HEAD]); + tsdbCpySFile(&pHelper->files.dataF, &pGroup->files[TSDB_FILE_TYPE_DATA]); + tsdbCpySFile(&pHelper->files.lastF, &pGroup->files[TSDB_FILE_TYPE_LAST]); if (helperType(pHelper) == TSDB_WRITE_HELPER) { - char *fnameDup = strdup(pHelper->files.headF.fname); - if (fnameDup == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - char *dataDir = dirname(fnameDup); - tsdbGetFileName(dataDir, pHelper->files.fid, ".h", pHelper->files.nHeadF.fname); - tsdbGetFileName(dataDir, pHelper->files.fid, ".l", pHelper->files.nLastF.fname); - free((void *)fnameDup); + pHelper->files.nHeadF.fname = tsdbGetDataFileName(pHelper->pRepo, pGroup->fileId, TSDB_FILE_TYPE_NHEAD); + pHelper->files.nLastF.fname = tsdbGetDataFileName(pHelper->pRepo, pGroup->fileId, TSDB_FILE_TYPE_NLAST); } // Open the files @@ -123,10 +117,10 @@ int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { // Create and open .h if (tsdbOpenFile(&(pHelper->files.nHeadF), O_WRONLY | O_CREAT) < 0) return -1; - // size_t tsize = TSDB_FILE_HEAD_SIZE + sizeof(SCompIdx) * pHelper->config.maxTables + sizeof(TSCKSUM); + // size_t tsize = TSDB_FILE_HEAD_SIZE + sizeof(SCompIdx) * pCfg->maxTables + sizeof(TSCKSUM); if (tsendfile(pHelper->files.nHeadF.fd, pHelper->files.headF.fd, NULL, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { tsdbError("vgId:%d failed to sendfile %d bytes from file %s to %s since %s", REPO_ID(pHelper->pRepo), - TSDB_FILE_HEAD_SIZE, hpHelper->files.headF.fname, pHelper->files.nHeadF.fname, strerror(errno)); + TSDB_FILE_HEAD_SIZE, pHelper->files.headF.fname, pHelper->files.nHeadF.fname, strerror(errno)); errno = TAOS_SYSTEM_ERROR(errno); goto _err; } @@ -136,7 +130,7 @@ int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { if (tsdbOpenFile(&(pHelper->files.nLastF), O_WRONLY | O_CREAT) < 0) goto _err; if (tsendfile(pHelper->files.nLastF.fd, pHelper->files.lastF.fd, NULL, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) tsdbError("vgId:%d failed to sendfile %d bytes from file %s to %s since %s", REPO_ID(pHelper->pRepo), - TSDB_FILE_HEAD_SIZE, hpHelper->files.lastF.fname, pHelper->files.nLastF.fname, strerror(errno)); + TSDB_FILE_HEAD_SIZE, pHelper->files.lastF.fname, pHelper->files.nLastF.fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); goto _err; } @@ -229,13 +223,15 @@ void tsdbSetHelperTable(SRWHelper *pHelper, STable *pTable, STsdbRepo *pRepo) { * -1 for failure */ int tsdbWriteDataBlock(SRWHelper *pHelper, SDataCols *pDataCols) { - ASSERT(TSDB_HELPER_TYPE(pHelper) == TSDB_WRITE_HELPER); + ASSERT(helperType(pHelper) == TSDB_WRITE_HELPER); ASSERT(pDataCols->numOfRows > 0); SCompBlock compBlock; int rowsToWrite = 0; TSKEY keyFirst = dataColsKeyFirst(pDataCols); + STsdbCfg *pCfg = &pHelper->pRepo->config; + ASSERT(helperHasState(pHelper, TSDB_HELPER_IDX_LOAD)); SCompIdx *pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; // for change purpose @@ -249,7 +245,7 @@ int tsdbWriteDataBlock(SRWHelper *pHelper, SDataCols *pDataCols) { SFile *pWFile = NULL; bool isLast = false; - if (rowsToWrite >= pHelper->config.minRowsPerFileBlock) { + if (rowsToWrite >= pCfg->minRowsPerFileBlock) { pWFile = &(pHelper->files.dataF); } else { isLast = true; @@ -293,7 +289,9 @@ _err: } int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper) { - ASSERT(TSDB_HELPER_TYPE(pHelper) == TSDB_WRITE_HELPER); + STsdbCfg *pCfg = &pHelper->pRepo->config; + + ASSERT(helperType(pHelper) == TSDB_WRITE_HELPER); SCompIdx * pIdx = pHelper->pCompIdx + pHelper->tableInfo.tid; SCompBlock compBlock; if ((pHelper->files.nLastF.fd > 0) && (pHelper->hasOldLastBlock)) { @@ -305,7 +303,7 @@ int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper) { if (pCompBlock->numOfSubBlocks > 1) { if (tsdbLoadBlockData(pHelper, blockAtIdx(pHelper, pIdx->numOfBlocks - 1), NULL) < 0) return -1; ASSERT(pHelper->pDataCols[0]->numOfRows > 0 && - pHelper->pDataCols[0]->numOfRows < pHelper->config.minRowsPerFileBlock); + pHelper->pDataCols[0]->numOfRows < pCfg->minRowsPerFileBlock); if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.nLastF), pHelper->pDataCols[0], pHelper->pDataCols[0]->numOfRows, &compBlock, true, true) < 0) return -1; @@ -355,16 +353,17 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) { } int tsdbWriteCompIdx(SRWHelper *pHelper) { - ASSERT(TSDB_HELPER_TYPE(pHelper) == TSDB_WRITE_HELPER); + STsdbCfg *pCfg = &pHelper->pRepo->config; + + ASSERT(helperType(pHelper) == TSDB_WRITE_HELPER); off_t offset = lseek(pHelper->files.nHeadF.fd, 0, SEEK_END); if (offset < 0) return -1; SFile *pFile = &(pHelper->files.nHeadF); pFile->info.offset = offset; - // TODO: change the implementation of pHelper->pBuffer void *buf = pHelper->pBuffer; - for (uint32_t i = 0; i < pHelper->config.maxTables; i++) { + for (uint32_t i = 0; i < pCfg->maxTables; i++) { SCompIdx *pCompIdx = pHelper->pCompIdx + i; if (pCompIdx->offset > 0) { int drift = POINTER_DISTANCE(buf, pHelper->pBuffer); @@ -386,6 +385,8 @@ int tsdbWriteCompIdx(SRWHelper *pHelper) { } int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { + STsdbCfg *pCfg = &(pHelper->pRepo->config); + ASSERT(pHelper->state == TSDB_HELPER_FILE_SET_AND_OPEN); if (!helperHasState(pHelper, TSDB_HELPER_IDX_LOAD)) { @@ -425,7 +426,7 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) { while (POINTER_DISTANCE(ptr, pHelper->pBuffer) < (pFile->info.len - sizeof(TSCKSUM))) { uint32_t tid = 0; if ((ptr = taosDecodeVariantU32(ptr, &tid)) == NULL) return -1; - ASSERT(tid > 0 && tid < pHelper->config.maxTables); + ASSERT(tid > 0 && tid < pCfg->maxTables); if ((ptr = tsdbDecodeSCompIdx(ptr, pHelper->pCompIdx + tid)) == NULL) return -1; @@ -470,7 +471,7 @@ int tsdbLoadCompInfo(SRWHelper *pHelper, void *target) { return 0; } -int tsdbloadcompdata(srwhelper *phelper, scompblock *pcompblock, void *target) { +int tsdbLoadCompData(SRWHelper *pHelper, SCompBlock *pCompBlock, void *target) { ASSERT(pCompBlock->numOfSubBlocks <= 1); int fd = (pCompBlock->last) ? pHelper->files.lastF.fd : pHelper->files.dataF.fd; @@ -614,12 +615,14 @@ static bool tsdbShouldCreateNewLast(SRWHelper *pHelper) { static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDataCols, int rowsToWrite, SCompBlock *pCompBlock, bool isLast, bool isSuperBlock) { - ASSERT(rowsToWrite > 0 && rowsToWrite <= pDataCols->numOfRows && rowsToWrite <= pHelper->config.maxRowsPerFileBlock); - ASSERT(isLast ? rowsToWrite < pHelper->config.minRowsPerFileBlock : true); - + STsdbCfg *pCfg = &(pHelper->pRepo->config); SCompData *pCompData = (SCompData *)(pHelper->pBuffer); int64_t offset = 0; + ASSERT(rowsToWrite > 0 && rowsToWrite <= pDataCols->numOfRows && rowsToWrite <= pCfg->maxRowsPerFileBlock); + ASSERT(isLast ? rowsToWrite < pCfg->minRowsPerFileBlock : true); + + offset = lseek(pFile->fd, 0, SEEK_END); if (offset < 0) { tsdbError("vgId:%d failed to write block to file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, strerror(errno)); @@ -668,8 +671,8 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa int32_t tlen = dataColGetNEleLen(pDataCol, rowsToWrite); - if (pHelper->config.compress) { - if (pHelper->config.compress == TWO_STAGE_COMP) { + if (pCfg->compression) { + if (pCfg->compression == TWO_STAGE_COMP) { pHelper->compBuffer = trealloc(pHelper->compBuffer, tlen + COMP_OVERFLOW_BYTES); if (pHelper->compBuffer == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; @@ -678,7 +681,7 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa } pCompCol->len = (*(tDataTypeDesc[pDataCol->type].compFunc))( - (char *)pDataCol->pData, tlen, rowsToWrite, tptr, tsizeof(pHelper->pBuffer) - lsize, pHelper->config.compress, + (char *)pDataCol->pData, tlen, rowsToWrite, tptr, tsizeof(pHelper->pBuffer) - lsize, pCfg->compression, pHelper->compBuffer, tsizeof(pHelper->compBuffer)); } else { pCompCol->len = tlen; @@ -710,7 +713,7 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa // Update pCompBlock membership vairables pCompBlock->last = isLast; pCompBlock->offset = offset; - pCompBlock->algorithm = pHelper->config.compress; + pCompBlock->algorithm = pCfg->compression; pCompBlock->numOfRows = rowsToWrite; pCompBlock->sversion = pHelper->tableInfo.sversion; pCompBlock->len = (int32_t)lsize; @@ -747,6 +750,7 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa // TODO: set pHelper->hasOldBlock int rowsWritten = 0; SCompBlock compBlock = {0}; + STsdbCfg * pCfg = &pHelper->pRepo->config; ASSERT(pDataCols->numOfRows > 0); TSKEY keyFirst = dataColsKeyFirst(pDataCols); @@ -760,13 +764,13 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa // ASSERT(compareKeyBlock((void *)&keyFirst, (void *)pCompBlock) == 0); if (keyFirst > blockAtIdx(pHelper, blkIdx)->keyLast) { // Merge with the last block by append - ASSERT(blockAtIdx(pHelper, blkIdx)->numOfRows < pHelper->config.minRowsPerFileBlock && + ASSERT(blockAtIdx(pHelper, blkIdx)->numOfRows < pCfg->minRowsPerFileBlock && blkIdx == pIdx->numOfBlocks - 1); - int defaultRowsToWrite = pHelper->config.maxRowsPerFileBlock * 4 / 5; // TODO: make a interface + int defaultRowsToWrite = pCfg->maxRowsPerFileBlock * 4 / 5; // TODO: make a interface rowsWritten = MIN((defaultRowsToWrite - blockAtIdx(pHelper, blkIdx)->numOfRows), pDataCols->numOfRows); if ((blockAtIdx(pHelper, blkIdx)->numOfSubBlocks < TSDB_MAX_SUBBLOCKS) && - (blockAtIdx(pHelper, blkIdx)->numOfRows + rowsWritten < pHelper->config.minRowsPerFileBlock) && + (blockAtIdx(pHelper, blkIdx)->numOfRows + rowsWritten < pCfg->minRowsPerFileBlock) && (pHelper->files.nLastF.fd) < 0) { if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.lastF), pDataCols, rowsWritten, &compBlock, true, false) < 0) goto _err; @@ -780,7 +784,7 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa // Write SFile *pWFile = NULL; bool isLast = false; - if (pHelper->pDataCols[0]->numOfRows >= pHelper->config.minRowsPerFileBlock) { + if (pHelper->pDataCols[0]->numOfRows >= pCfg->minRowsPerFileBlock) { pWFile = &(pHelper->files.dataF); } else { isLast = true; @@ -804,7 +808,7 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa int rows1 = tsdbGetRowsInRange(pDataCols, blockAtIdx(pHelper, blkIdx)->keyFirst, blockAtIdx(pHelper, blkIdx)->keyLast); // rows2: max number of rows the block can have more - int rows2 = pHelper->config.maxRowsPerFileBlock - blockAtIdx(pHelper, blkIdx)->numOfRows; + int rows2 = pCfg->maxRowsPerFileBlock - blockAtIdx(pHelper, blkIdx)->numOfRows; // rows3: number of rows between this block and the next block int rows3 = tsdbGetRowsInRange(pDataCols, blockAtIdx(pHelper, blkIdx)->keyFirst, keyLimit); @@ -812,7 +816,7 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa if ((rows2 >= rows1) && (blockAtIdx(pHelper, blkIdx)->numOfSubBlocks < TSDB_MAX_SUBBLOCKS) && ((!blockAtIdx(pHelper, blkIdx)->last) || - ((rows1 + blockAtIdx(pHelper, blkIdx)->numOfRows < pHelper->config.minRowsPerFileBlock) && + ((rows1 + blockAtIdx(pHelper, blkIdx)->numOfRows < pCfg->minRowsPerFileBlock) && (pHelper->files.nLastF.fd < 0)))) { rowsWritten = rows1; bool isLast = false; @@ -841,7 +845,7 @@ static int tsdbMergeDataWithBlock(SRWHelper *pHelper, int blkIdx, SDataCols *pDa while (true) { if (iter1 >= pHelper->pDataCols[0]->numOfRows && iter2 >= rows3) break; tdMergeTwoDataCols(pHelper->pDataCols[1], pHelper->pDataCols[0], &iter1, pHelper->pDataCols[0]->numOfRows, - pDataCols, &iter2, rowsWritten, pHelper->config.maxRowsPerFileBlock * 4 / 5); + pDataCols, &iter2, rowsWritten, pCfg->maxRowsPerFileBlock * 4 / 5); ASSERT(pHelper->pDataCols[1]->numOfRows > 0); if (tsdbWriteBlockToFile(pHelper, &(pHelper->files.dataF), pHelper->pDataCols[1], pHelper->pDataCols[1]->numOfRows, &compBlock, false, true) < 0) diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index d6bab10913..93feff9180 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -2119,7 +2119,7 @@ int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, const char* pT } SArray* res = taosArrayInit(8, sizeof(STableId)); - STSchema* pTagSchema = tsdbGetTableTagSchema(tsdbGetMeta(tsdb), pTable); + STSchema* pTagSchema = tsdbGetTableTagSchema(pTable); // no tags and tbname condition, all child tables of this stable are involved if (tbnameCond == NULL && (pTagCond == NULL || len == 0)) { From a2260415fe01a247a3eb57f6b0cd0a4f17b8162c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 17 Jun 2020 07:06:05 +0000 Subject: [PATCH 31/65] TD-353 --- src/util/src/tkvstore.c | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/src/util/src/tkvstore.c b/src/util/src/tkvstore.c index 3f27151aca..fb4ea2146f 100644 --- a/src/util/src/tkvstore.c +++ b/src/util/src/tkvstore.c @@ -226,12 +226,32 @@ _err: } int tdUpdateKVStoreRecord(SKVStore *pStore, uint64_t uid, void *cont, int contLen) { - SKVRecord *pRecord = taosHashGet(pStore->map, (void *)&uid, sizeof(uid)); - if (pRecord != NULL) { - pStore->info.tombSize += (pRecord->size + sizeof(SKVRecord)); + SKVRecord rInfo = {0}; + + rInfo.offset = lseek(pStore->fd, 0, SEEK_CUR); + if (rInfo.offset < 0) { + uError("failed to lseek file %s since %s", pStore->fname, strerror(errno)); + return -1; } - // TODO + rInfo.uid = uid; + rInfo.size = contLen; + + if (twrite(pStore->fd, cont, contLen) < contLen) { + uError("failed to write %d bytes to file %s since %s", contLen, pStore->fname, strerror(errno)); + return -1; + } + + pStore->info.size += (sizeof(SKVRecord) + contLen); + SKVRecord *pRecord = taosHashGet(pStore->map, (void *)&uid, sizeof(uid)); + if (pRecord != NULL) { // just to insert + pStore->info.nRecords++; + } else { + pStore->info.tombSize += pRecord->size; + } + + taosHashPut(pStore->map, (void *)(&uid), sizeof(uid), (void *)(&rInfo), sizeof(rInfo)); + return 0; } @@ -239,7 +259,7 @@ int tdDropKVStoreRecord(SKVStore *pStore, uint64_t uid) { SKVRecord rInfo = {0}; char buf[128] = "\0"; - SKVRecord *pRecord = taosHashGet(pStore->map, &uid, sizeof(uid)); + SKVRecord *pRecord = taosHashGet(pStore->map, (void *)(&uid), sizeof(uid)); if (pRecord == NULL) { uError("failed to drop KV store record with key " PRIu64 " since not find", uid); return -1; @@ -262,6 +282,8 @@ int tdDropKVStoreRecord(SKVStore *pStore, uint64_t uid) { pStore->info.nRecords--; pStore->info.tombSize += (rInfo.size + sizeof(SKVRecord) * 2); + taosHashRemove(pStore->map, (void *)(&uid), sizeof(uid)); + return 0; } From 580fb8e9395edd0d0692642ea352221f96224565 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 17 Jun 2020 15:35:07 +0800 Subject: [PATCH 32/65] [td-353] --- src/tsdb/src/tsdbRead.c | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 93feff9180..145ee5afa5 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -72,9 +72,11 @@ typedef struct STableCheckInfo { int32_t compSize; int32_t numOfBlocks; // number of qualified data blocks not the original blocks SDataCols* pDataCols; - int32_t chosen; // indicate which iterator should move forward bool initBuf; // whether to initialize the in-memory skip list iterator or not + SMemTable* mem; // in-mem buffer, hold the ref count + SMemTable* imem; // imem buffer, hold the ref count to avoid release + SSkipListIterator* iter; // mem buffer skip list iterator SSkipListIterator* iiter; // imem buffer skip list iterator } STableCheckInfo; @@ -247,22 +249,24 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh pCheckInfo->initBuf = true; int32_t order = pHandle->order; - + + tsdbTakeMemSnapshot(pHandle->pTsdb, &pCheckInfo->mem, &pCheckInfo->imem); + // no data in buffer, abort - if (pTable->mem == NULL && pTable->imem == NULL) { + if (pCheckInfo->mem == NULL && pCheckInfo->imem == NULL) { return false; } assert(pCheckInfo->iter == NULL && pCheckInfo->iiter == NULL); - if (pTable->mem) { - pCheckInfo->iter = tSkipListCreateIterFromVal(pTable->mem->pData, (const char*) &pCheckInfo->lastKey, - TSDB_DATA_TYPE_TIMESTAMP, order); + if (pCheckInfo->mem) { + pCheckInfo->iter = tSkipListCreateIterFromVal(pCheckInfo->mem->tData[pCheckInfo->tableId.tid]->pData, + (const char*) &pCheckInfo->lastKey, TSDB_DATA_TYPE_TIMESTAMP, order); } - if (pTable->imem) { - pCheckInfo->iiter = tSkipListCreateIterFromVal(pTable->imem->pData, (const char*) &pCheckInfo->lastKey, - TSDB_DATA_TYPE_TIMESTAMP, order); + if (pCheckInfo->imem) { + pCheckInfo->iiter = tSkipListCreateIterFromVal(pCheckInfo->imem->tData[pCheckInfo->tableId.tid]->pData, + (const char*) &pCheckInfo->lastKey, TSDB_DATA_TYPE_TIMESTAMP, order); } // both iterators are NULL, no data in buffer right now @@ -1822,8 +1826,8 @@ static int32_t getAllTableIdList(STable* pSuperTable, SArray* list) { while (tSkipListIterNext(iter)) { SSkipListNode* pNode = tSkipListIterGet(iter); - STableIndexElem* elem = (STableIndexElem*)(SL_GET_NODE_DATA((SSkipListNode*) pNode)); - taosArrayPush(list, &elem->pTable->tableId); + STable** pTable = (STable**) SL_GET_NODE_DATA((SSkipListNode*) pNode); + taosArrayPush(list, &(*pTable)->tableId); } tSkipListDestroyIter(iter); @@ -1842,8 +1846,8 @@ static void convertQueryResult(SArray* pRes, SArray* pTableList) { size_t size = taosArrayGetSize(pTableList); for (int32_t i = 0; i < size; ++i) { // todo speedup by using reserve space. - STableIndexElem* elem = taosArrayGet(pTableList, i); - taosArrayPush(pRes, &elem->pTable->tableId); + STable* pTable = taosArrayGetP(pTableList, i); + taosArrayPush(pRes, &pTable->tableId); } } @@ -2027,16 +2031,16 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC bool indexedNodeFilterFp(const void* pNode, void* param) { tQueryInfo* pInfo = (tQueryInfo*) param; - STableIndexElem* elem = (STableIndexElem*)(SL_GET_NODE_DATA((SSkipListNode*)pNode)); + STable* pTable = *(STable**)(SL_GET_NODE_DATA((SSkipListNode*)pNode)); char* val = NULL; int8_t type = pInfo->sch.type; if (pInfo->colIndex == TSDB_TBNAME_COLUMN_INDEX) { - val = (char*) elem->pTable->name; + val = (char*) pTable->name; type = TSDB_DATA_TYPE_BINARY; } else { - val = tdGetKVRowValOfCol(elem->pTable->tagVal, pInfo->sch.colId); + val = tdGetKVRowValOfCol(pTable->tagVal, pInfo->sch.colId); } //todo :the val is possible to be null, so check it out carefully @@ -2092,7 +2096,7 @@ static int32_t doQueryTableList(STable* pSTable, SArray* pRes, tExprNode* pExpr) .pExtInfo = pSTable->tagSchema, }; - SArray* pTableList = taosArrayInit(8, sizeof(STableIndexElem)); + SArray* pTableList = taosArrayInit(8, POINTER_BYTES); tExprTreeTraverse(pExpr, pSTable->pIndex, pTableList, &supp); tExprTreeDestroy(&pExpr, destroyHelper); From 3cb26dbef445cddccb5833280ad33c6bfd364738 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 17 Jun 2020 15:37:38 +0800 Subject: [PATCH 33/65] [td-353] --- src/tsdb/src/tsdbRead.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 145ee5afa5..e2f1bbcdb9 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -2207,6 +2207,10 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) { size_t size = taosArrayGetSize(pQueryHandle->pTableCheckInfo); for (int32_t i = 0; i < size; ++i) { STableCheckInfo* pTableCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); + + tsdbUnRefMemTable(pQueryHandle->pTsdb, pTableCheckInfo->mem); + tsdbUnRefMemTable(pQueryHandle->pTsdb, pTableCheckInfo->imem); + tSkipListDestroyIter(pTableCheckInfo->iter); if (pTableCheckInfo->pDataCols != NULL) { From b71ec85236db91dbbc49036a29c5f60de84dbf76 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 17 Jun 2020 07:45:08 +0000 Subject: [PATCH 34/65] TD-353 --- src/query/src/qast.c | 2 +- src/tsdb/inc/tsdbMain.h | 1 - src/tsdb/src/tsdbRWHelper.c | 45 ------------------------------------- src/tsdb/src/tsdbRead.c | 6 +++-- 4 files changed, 5 insertions(+), 49 deletions(-) diff --git a/src/query/src/qast.c b/src/query/src/qast.c index 1db965e8a3..83c60c9e88 100644 --- a/src/query/src/qast.c +++ b/src/query/src/qast.c @@ -772,7 +772,7 @@ static void tQueryIndexlessColumn(SSkipList* pSkipList, tQueryInfo* pQueryInfo, char * pData = SL_GET_NODE_DATA(pNode); // todo refactor: - tstr *name = ((STableIndexElem *)pData)->pTable->name; + tstr *name = (*(STable **)pData)->name; // todo speed up by using hash if (pQueryInfo->colIndex == TSDB_TBNAME_COLUMN_INDEX) { if (pQueryInfo->optr == TSDB_RELATION_IN) { diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index a4a127069d..a8c81ccc5b 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -368,7 +368,6 @@ int tsdbLoadCompData(SRWHelper* phelper, SCompBlock* pcompblock, void* target) void tsdbGetDataStatis(SRWHelper* pHelper, SDataStatis* pStatis, int numOfCols); int tsdbLoadBlockDataCols(SRWHelper* pHelper, SDataCols* pDataCols, int blkIdx, int16_t* colIds, int numOfColIds); int tsdbLoadBlockData(SRWHelper* pHelper, SCompBlock* pCompBlock, SDataCols* target); -int tsdbUpdateFileHeader(SFile* pFile, uint32_t version); // ------------------ tsdbMain.c #define REPO_ID(r) (r)->config.tsdbId diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index c7ebbf460c..b89a187259 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -559,51 +559,6 @@ _err: return -1; } -int tsdbUpdateFileHeader(SFile *pFile, uint32_t version) { - char buf[TSDB_FILE_HEAD_SIZE] = "\0"; - - void *pBuf = (void *)buf; - pBuf = taosEncodeFixedU32(pBuf, version); - pBuf = tsdbEncodeSFileInfo(pBuf, &(pFile->info)); - - taosCalcChecksumAppend(0, (uint8_t *)buf, TSDB_FILE_HEAD_SIZE); - - if (lseek(pFile->fd, 0, SEEK_SET) < 0) { - tsdbError("failed to lseek file %s since %s", pFile->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - if (twrite(pFile->fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { - tsdbError("failed to write %d bytes to file %s since %s", TSDB_FILE_HEAD_SIZE, pFile->fname, strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - return 0; -} - -void *tsdbEncodeSFileInfo(void *buf, const STsdbFileInfo *pInfo) { - buf = taosEncodeFixedU32(buf, pInfo->offset); - buf = taosEncodeFixedU32(buf, pInfo->len); - buf = taosEncodeFixedU64(buf, pInfo->size); - buf = taosEncodeFixedU64(buf, pInfo->tombSize); - buf = taosEncodeFixedU32(buf, pInfo->totalBlocks); - buf = taosEncodeFixedU32(buf, pInfo->totalSubBlocks); - - return buf; -} - -void *tsdbDecodeSFileInfo(void *buf, STsdbFileInfo *pInfo) { - buf = taosDecodeFixedU32(buf, &(pInfo->offset)); - buf = taosDecodeFixedU32(buf, &(pInfo->len)); - buf = taosDecodeFixedU64(buf, &(pInfo->size)); - buf = taosDecodeFixedU64(buf, &(pInfo->tombSize)); - buf = taosDecodeFixedU32(buf, &(pInfo->totalBlocks)); - buf = taosDecodeFixedU32(buf, &(pInfo->totalSubBlocks)); - - return buf; -} - // ---------------------- INTERNAL FUNCTIONS ---------------------- static bool tsdbShouldCreateNewLast(SRWHelper *pHelper) { ASSERT(pHelper->files.lastF.fd > 0); diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index e2f1bbcdb9..09eb981e28 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -589,8 +589,10 @@ static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlo int64_t st = taosGetTimestampUs(); if (pCheckInfo->pDataCols == NULL) { - STsdbMeta* pMeta = tsdbGetMeta(pRepo); - pCheckInfo->pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pRepo->config.maxRowsPerFileBlock); + // STsdbMeta* pMeta = tsdbGetMeta(pRepo); + // TODO + pCheckInfo->pDataCols = + tdNewDataCols(pCheckInfo->imem->maxRowBytes, pCheckInfo->imem->maxCols, pRepo->config.maxRowsPerFileBlock); } tdInitDataCols(pCheckInfo->pDataCols, tsdbGetTableSchema(pCheckInfo->pTableObj)); From cd062fdbeaa621a682c0734ad0b6dd2c07e318a2 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 17 Jun 2020 07:55:52 +0000 Subject: [PATCH 35/65] TD-353 --- src/util/src/tkvstore.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/src/tkvstore.c b/src/util/src/tkvstore.c index fb4ea2146f..41632d5b2b 100644 --- a/src/util/src/tkvstore.c +++ b/src/util/src/tkvstore.c @@ -386,7 +386,7 @@ static SKVStore *tdNewKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void if (pStore == NULL) goto _err; pStore->fname = strdup(fname); - if (pStore->map == NULL) { + if (pStore->fname == NULL) { terrno = TSDB_CODE_COM_OUT_OF_MEMORY; goto _err; } From 6c311f0d119dd7cf114249d7c802d107217137b8 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 17 Jun 2020 07:56:41 +0000 Subject: [PATCH 36/65] TD-353 --- src/util/src/tkvstore.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/util/src/tkvstore.c b/src/util/src/tkvstore.c index 41632d5b2b..be9c4c19b0 100644 --- a/src/util/src/tkvstore.c +++ b/src/util/src/tkvstore.c @@ -382,7 +382,7 @@ static void *tdDecodeStoreInfo(void *buf, SStoreInfo *pInfo) { } static SKVStore *tdNewKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void *appH) { - SKVStore *pStore = (SKVStore *)malloc(sizeof(SKVStore)); + SKVStore *pStore = (SKVStore *)calloc(1, sizeof(SKVStore)); if (pStore == NULL) goto _err; pStore->fname = strdup(fname); @@ -392,7 +392,9 @@ static SKVStore *tdNewKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void } pStore->fsnap = tdGetKVStoreSnapshotFname(fname); - if (pStore->fsnap == NULL) goto _err; + if (pStore->fsnap == NULL) { + goto _err; + } pStore->fnew = tdGetKVStoreNewFname(fname); if (pStore->fnew == NULL) goto _err; From 63124ada407f65399e25720f354b8a09f35c079a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 17 Jun 2020 08:34:46 +0000 Subject: [PATCH 37/65] TD-353 --- src/tsdb/inc/tsdbMain.h | 4 ++-- src/tsdb/src/tsdbMemTable.c | 8 +++----- src/tsdb/src/tsdbMeta.c | 24 ++++++++++++++++++++++++ src/tsdb/src/tsdbRWHelper.c | 10 ++++++---- src/tsdb/src/tsdbRead.c | 4 ++-- 5 files changed, 37 insertions(+), 13 deletions(-) diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index a8c81ccc5b..40f79e544f 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -70,6 +70,8 @@ typedef struct { SList* superList; SHashObj* uidMap; SKVStore* pStore; + int maxRowBytes; + int maxCols; } STsdbMeta; // ------------------ tsdbBuffer.c @@ -106,8 +108,6 @@ typedef struct { STableData** tData; SList* actList; SList* bufBlockList; - int maxCols; - int maxRowBytes; } SMemTable; // ------------------ tsdbFile.c diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index f342431378..ef9cc1b07e 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -108,9 +108,6 @@ int tsdbInsertRowToMem(STsdbRepo *pRepo, SDataRow row, STable *pTable) { pTableData->numOfRows++; ASSERT(pTableData->numOfRows == tSkipListGetSize(pTableData->pData)); - STSchema *pSchema = tsdbGetTableSchema(pTable); - if (schemaNCols(pSchema) > pMemTable->maxCols) pMemTable->maxCols = schemaNCols(pSchema); - if (schemaTLen(pSchema) > pMemTable->maxRowBytes) pMemTable->maxRowBytes = schemaTLen(pSchema); } tsdbTrace("vgId:%d a row is inserted to table %s tid %d uid %" PRIu64 " key %" PRIu64, REPO_ID(pRepo), @@ -360,6 +357,7 @@ static void *tsdbCommitData(void *arg) { SMemTable * pMem = pRepo->imem; STsdbCfg * pCfg = &pRepo->config; SDataCols * pDataCols = NULL; + STsdbMeta * pMeta = pRepo->tsdbMeta; SCommitIter *iters = NULL; SRWHelper whelper = {0}; ASSERT(pRepo->commit == 1); @@ -380,10 +378,10 @@ static void *tsdbCommitData(void *arg) { goto _exit; } - if ((pDataCols = tdNewDataCols(pMem->maxRowBytes, pMem->maxCols, pCfg->maxRowsPerFileBlock)) == NULL) { + if ((pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pCfg->maxRowsPerFileBlock)) == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; tsdbError("vgId:%d failed to init data cols with maxRowBytes %d maxCols %d maxRowsPerFileBlock %d since %s", - REPO_ID(pRepo), pMem->maxRowBytes, pMem->maxCols, pCfg->maxRowsPerFileBlock, tstrerror(terrno)); + REPO_ID(pRepo), pMeta->maxCols, pMeta->maxRowBytes, pCfg->maxRowsPerFileBlock, tstrerror(terrno)); goto _exit; } diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 99f5c3173d..4acb0a3625 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -737,6 +737,12 @@ static int tsdbAddTableToMeta(STsdbRepo *pRepo, STable *pTable, bool addIdx) { goto _err; } + if (TABLE_TYPE(pTable) != TSDB_CHILD_TABLE) { + STSchema *pSchema = tsdbGetTableSchema(pTable); + if (schemaNCols(pSchema) > pMeta->maxCols) pMeta->maxCols = schemaNCols(pSchema); + if (schemaTLen(pSchema) > pMeta->maxRowBytes) pMeta->maxRowBytes = schemaTLen(pSchema); + } + if (addIdx && tsdbUnlockRepoMeta(pRepo) < 0) return -1; tsdbTrace("vgId:%d table %s tid %d uid %" PRIu64 " is added to meta", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), @@ -754,6 +760,11 @@ static void tsdbRemoveTableFromMeta(STsdbRepo *pRepo, STable *pTable, bool rmFro SListIter lIter = {0}; SListNode *pNode = NULL; STable * tTable = NULL; + STsdbCfg * pCfg = &(pRepo->config); + + STSchema *pSchema = tsdbGetTableSchema(pTable); + int maxCols = schemaNCols(pSchema); + int maxRowBytes = schemaTLen(pSchema); if (rmFromIdx) tsdbWLockRepoMeta(pRepo); @@ -779,6 +790,19 @@ static void tsdbRemoveTableFromMeta(STsdbRepo *pRepo, STable *pTable, bool rmFro taosHashRemove(pMeta->uidMap, (char *)(&(TABLE_UID(pTable))), sizeof(TABLE_UID(pTable))); + if (maxCols == pMeta->maxCols || maxRowBytes == pMeta->maxRowBytes) { + maxCols = 0; + maxRowBytes = 0; + for (int i = 0; i < pCfg->maxTables; i++) { + STable *pTable = pMeta->tables[i]; + if (pTable != NULL) { + pSchema = tsdbGetTableSchema(pTable); + maxCols = MAX(maxCols, schemaNCols(pSchema)); + maxRowBytes = MAX(maxRowBytes, schemaTLen(pSchema)); + } + } + } + if (rmFromIdx) tsdbUnlockRepoMeta(pRepo); } diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index b89a187259..28f562973f 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -1096,11 +1096,12 @@ static void tsdbResetHelperBlock(SRWHelper *pHelper) { static int tsdbInitHelperBlock(SRWHelper *pHelper) { STsdbRepo *pRepo = helperRepo(pHelper); + STsdbMeta *pMeta = pHelper->pRepo->tsdbMeta; pHelper->pDataCols[0] = - tdNewDataCols(pRepo->imem->maxRowBytes, pRepo->imem->maxCols, pRepo->config.maxRowsPerFileBlock); + tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pRepo->config.maxRowsPerFileBlock); pHelper->pDataCols[1] = - tdNewDataCols(pRepo->imem->maxRowBytes, pRepo->imem->maxCols, pRepo->config.maxRowsPerFileBlock); + tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pRepo->config.maxRowsPerFileBlock); if (pHelper->pDataCols[0] == NULL || pHelper->pDataCols[1] == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return -1; @@ -1120,6 +1121,7 @@ static void tsdbDestroyHelperBlock(SRWHelper *pHelper) { static int tsdbInitHelper(SRWHelper *pHelper, STsdbRepo *pRepo, tsdb_rw_helper_t type) { STsdbCfg *pCfg = &pRepo->config; memset((void *)pHelper, 0, sizeof(*pHelper)); + STsdbMeta *pMeta = pRepo->tsdbMeta; helperType(pHelper) = type; helperRepo(pHelper) = pRepo; @@ -1135,8 +1137,8 @@ static int tsdbInitHelper(SRWHelper *pHelper, STsdbRepo *pRepo, tsdb_rw_helper_t if (tsdbInitHelperBlock(pHelper) < 0) goto _err; pHelper->pBuffer = - tmalloc(sizeof(SCompData) + (sizeof(SCompCol) + sizeof(TSCKSUM) + COMP_OVERFLOW_BYTES) * pRepo->imem->maxCols + - pRepo->imem->maxRowBytes * pCfg->maxRowsPerFileBlock + sizeof(TSCKSUM)); + tmalloc(sizeof(SCompData) + (sizeof(SCompCol) + sizeof(TSCKSUM) + COMP_OVERFLOW_BYTES) * pMeta->maxCols + + pMeta->maxRowBytes * pCfg->maxRowsPerFileBlock + sizeof(TSCKSUM)); if (pHelper->pBuffer == NULL) { terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _err; diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 09eb981e28..e5d4b2fdc0 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -589,10 +589,10 @@ static bool doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlo int64_t st = taosGetTimestampUs(); if (pCheckInfo->pDataCols == NULL) { - // STsdbMeta* pMeta = tsdbGetMeta(pRepo); + STsdbMeta* pMeta = tsdbGetMeta(pRepo); // TODO pCheckInfo->pDataCols = - tdNewDataCols(pCheckInfo->imem->maxRowBytes, pCheckInfo->imem->maxCols, pRepo->config.maxRowsPerFileBlock); + tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pRepo->config.maxRowsPerFileBlock); } tdInitDataCols(pCheckInfo->pDataCols, tsdbGetTableSchema(pCheckInfo->pTableObj)); From 3a759852fffc46fa6d808decab7f00eea8473d3c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 17 Jun 2020 09:39:05 +0000 Subject: [PATCH 38/65] TD-353 --- src/common/inc/tdataformat.h | 4 +- src/common/src/tdataformat.c | 25 +++--- src/tsdb/inc/tsdbMain.h | 2 +- src/tsdb/src/tsdbFile.c | 21 ++--- src/tsdb/src/tsdbMeta.c | 56 +++++++------ src/tsdb/src/tsdbRWHelper.c | 24 +++--- src/util/inc/tcoding.h | 145 ++++++++++++++++++++------------- src/util/src/tkvstore.c | 34 ++++---- src/util/tests/codingTests.cpp | 96 ++++++++++++++-------- 9 files changed, 241 insertions(+), 166 deletions(-) diff --git a/src/common/inc/tdataformat.h b/src/common/inc/tdataformat.h index ca9fd3ee33..b696aee2eb 100644 --- a/src/common/inc/tdataformat.h +++ b/src/common/inc/tdataformat.h @@ -83,7 +83,7 @@ typedef struct { #define tdFreeSchema(s) tfree((s)) STSchema *tdDupSchema(STSchema *pSchema); -void * tdEncodeSchema(void *buf, STSchema *pSchema); +int tdEncodeSchema(void **buf, STSchema *pSchema); void * tdDecodeSchema(void *buf, STSchema **pRSchema); static FORCE_INLINE int comparColId(const void *key1, const void *key2) { @@ -288,7 +288,7 @@ typedef struct { SKVRow tdKVRowDup(SKVRow row); int tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value); -void * tdEncodeKVRow(void *buf, SKVRow row); +int tdEncodeKVRow(void **buf, SKVRow row); void * tdDecodeKVRow(void *buf, SKVRow *row); static FORCE_INLINE int comparTagId(const void *key1, const void *key2) { diff --git a/src/common/src/tdataformat.c b/src/common/src/tdataformat.c index 77af404eda..d17ce585cd 100644 --- a/src/common/src/tdataformat.c +++ b/src/common/src/tdataformat.c @@ -34,18 +34,19 @@ STSchema *tdDupSchema(STSchema *pSchema) { /** * Encode a schema to dst, and return the next pointer */ -void *tdEncodeSchema(void *buf, STSchema *pSchema) { - buf = taosEncodeFixedI32(buf, schemaVersion(pSchema)); - buf = taosEncodeFixedI32(buf, schemaNCols(pSchema)); +int tdEncodeSchema(void **buf, STSchema *pSchema) { + int tlen = 0; + tlen += taosEncodeFixedI32(buf, schemaVersion(pSchema)); + tlen += taosEncodeFixedI32(buf, schemaNCols(pSchema)); for (int i = 0; i < schemaNCols(pSchema); i++) { STColumn *pCol = schemaColAt(pSchema, i); - buf = taosEncodeFixedI8(buf, colType(pCol)); - buf = taosEncodeFixedI16(buf, colColId(pCol)); - buf = taosEncodeFixedI32(buf, colBytes(pCol)); + tlen += taosEncodeFixedI8(buf, colType(pCol)); + tlen += taosEncodeFixedI16(buf, colColId(pCol)); + tlen += taosEncodeFixedI32(buf, colBytes(pCol)); } - return buf; + return tlen; } /** @@ -605,10 +606,14 @@ int tdSetKVRowDataOfCol(SKVRow *orow, int16_t colId, int8_t type, void *value) { return 0; } -void *tdEncodeKVRow(void *buf, SKVRow row) { +int tdEncodeKVRow(void **buf, SKVRow row) { // May change the encode purpose - kvRowCpy(buf, row); - return POINTER_SHIFT(buf, kvRowLen(row)); + if (buf != NULL) { + kvRowCpy(*buf, row); + *buf = POINTER_SHIFT(*buf, kvRowLen(row)); + } + + return kvRowLen(row); } void *tdDecodeKVRow(void *buf, SKVRow *row) { diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 40f79e544f..44e8002ac1 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -330,7 +330,7 @@ int tsdbCreateFile(SFile* pFile, STsdbRepo* pRepo, int fid, int type); SFileGroup* tsdbSearchFGroup(STsdbFileH* pFileH, int fid, int flags); void tsdbFitRetention(STsdbRepo* pRepo); int tsdbUpdateFileHeader(SFile* pFile, uint32_t version); -void* tsdbEncodeSFileInfo(void* buf, const STsdbFileInfo* pInfo); +int tsdbEncodeSFileInfo(void** buf, const STsdbFileInfo* pInfo); void* tsdbDecodeSFileInfo(void* buf, STsdbFileInfo* pInfo); int tsdbCpySFile(SFile* src, SFile* dst); diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 4556e79db4..e268aa1d65 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -302,8 +302,8 @@ int tsdbUpdateFileHeader(SFile *pFile, uint32_t version) { char buf[TSDB_FILE_HEAD_SIZE] = "\0"; void *pBuf = (void *)buf; - pBuf = taosEncodeFixedU32(pBuf, version); - pBuf = tsdbEncodeSFileInfo(pBuf, &(pFile->info)); + taosEncodeFixedU32(pBuf, version); + tsdbEncodeSFileInfo(pBuf, &(pFile->info)); taosCalcChecksumAppend(0, (uint8_t *)buf, TSDB_FILE_HEAD_SIZE); @@ -321,15 +321,16 @@ int tsdbUpdateFileHeader(SFile *pFile, uint32_t version) { return 0; } -void *tsdbEncodeSFileInfo(void *buf, const STsdbFileInfo *pInfo) { - buf = taosEncodeFixedU32(buf, pInfo->offset); - buf = taosEncodeFixedU32(buf, pInfo->len); - buf = taosEncodeFixedU64(buf, pInfo->size); - buf = taosEncodeFixedU64(buf, pInfo->tombSize); - buf = taosEncodeFixedU32(buf, pInfo->totalBlocks); - buf = taosEncodeFixedU32(buf, pInfo->totalSubBlocks); +int tsdbEncodeSFileInfo(void **buf, const STsdbFileInfo *pInfo) { + int tlen = 0; + tlen += taosEncodeFixedU32(buf, pInfo->offset); + tlen += taosEncodeFixedU32(buf, pInfo->len); + tlen += taosEncodeFixedU64(buf, pInfo->size); + tlen += taosEncodeFixedU64(buf, pInfo->tombSize); + tlen += taosEncodeFixedU32(buf, pInfo->totalBlocks); + tlen += taosEncodeFixedU32(buf, pInfo->totalSubBlocks); - return buf; + return tlen; } void *tsdbDecodeSFileInfo(void *buf, STsdbFileInfo *pInfo) { diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 4acb0a3625..9eb5d10371 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -42,9 +42,9 @@ static int tsdbTableSetSName(STableCfg *config, char *sname, bool dup); static int tsdbTableSetSuperUid(STableCfg *config, uint64_t uid); static int tsdbTableSetTagValue(STableCfg *config, SKVRow row, bool dup); static int tsdbTableSetStreamSql(STableCfg *config, char *sql, bool dup); -static void * tsdbEncodeTableName(void *buf, tstr *name); +static int tsdbEncodeTableName(void **buf, tstr *name); static void * tsdbDecodeTableName(void *buf, tstr **name); -static void * tsdbEncodeTable(void *buf, STable *pTable); +static int tsdbEncodeTable(void **buf, STable *pTable); static void * tsdbDecodeTable(void *buf, STable **pRTable); // ------------------ OUTER FUNCTIONS ------------------ @@ -85,6 +85,9 @@ int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg) { } if (tsdbAddTableToMeta(pRepo, table, true) < 0) goto _err; + int tlen = tsdbEncodeTable(NULL, pTable); + ASSERT(tlen > 0); + // // Write to meta file // int bufLen = 0; // char *buf = malloc(1024 * 1024); @@ -472,10 +475,11 @@ int tsdbUpdateTable(STsdbMeta *pMeta, STable *pTable, STableCfg *pCfg) { } if (isChanged) { - char *buf = malloc(1024 * 1024); - tsdbEncodeTable(buf, pTable); - // tsdbInsertMetaRecord(pMeta->mfh, pTable->tableId.uid, buf, bufLen); - free(buf); + // TODO + // char *buf = malloc(1024 * 1024); + // tsdbEncodeTable(buf, pTable); + // // tsdbInsertMetaRecord(pMeta->mfh, pTable->tableId.uid, buf, bufLen); + // free(buf); } return TSDB_CODE_SUCCESS; @@ -999,14 +1003,17 @@ void tsdbClearTableCfg(STableCfg *config) { } } -static void *tsdbEncodeTableName(void *buf, tstr *name) { - void *pBuf = buf; +static int tsdbEncodeTableName(void **buf, tstr *name) { + int tlen = 0; - pBuf = taosEncodeFixedI16(pBuf, name->len); - memcpy(pBuf, name->data, name->len); - pBuf = POINTER_SHIFT(pBuf, name->len); + tlen += taosEncodeFixedI16(buf, name->len); + if (buf != NULL) { + memcpy(*buf, name->data, name->len); + *buf = POINTER_SHIFT(*buf, name->len); + } + tlen += name->len; - return pBuf; + return tlen; } static void *tsdbDecodeTableName(void *buf, tstr **name) { @@ -1025,33 +1032,34 @@ static void *tsdbDecodeTableName(void *buf, tstr **name) { return buf; } -static void *tsdbEncodeTable(void *buf, STable *pTable) { +static int tsdbEncodeTable(void **buf, STable *pTable) { ASSERT(pTable != NULL); + int tlen = 0; - buf = taosEncodeFixedU8(buf, pTable->type); - buf = tsdbEncodeTableName(buf, pTable->name); - buf = taosEncodeFixedU64(buf, TABLE_UID(pTable)); - buf = taosEncodeFixedI32(buf, TABLE_TID(pTable)); + tlen = taosEncodeFixedU8(buf, pTable->type); + tlen = tsdbEncodeTableName(buf, pTable->name); + tlen = taosEncodeFixedU64(buf, TABLE_UID(pTable)); + tlen = taosEncodeFixedI32(buf, TABLE_TID(pTable)); if (TABLE_TYPE(pTable) == TSDB_CHILD_TABLE) { - buf = taosEncodeFixedU64(buf, TABLE_SUID(pTable)); - buf = tdEncodeKVRow(buf, pTable->tagVal); + tlen += taosEncodeFixedU64(buf, TABLE_SUID(pTable)); + tlen += tdEncodeKVRow(buf, pTable->tagVal); } else { - buf = taosEncodeFixedU8(buf, pTable->numOfSchemas); + tlen += taosEncodeFixedU8(buf, pTable->numOfSchemas); for (int i = 0; i < pTable->numOfSchemas; i++) { - buf = tdEncodeSchema(buf, pTable->schema[i]); + tlen += tdEncodeSchema(buf, pTable->schema[i]); } if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) { - buf = tdEncodeSchema(buf, pTable->tagSchema); + tlen += tdEncodeSchema(buf, pTable->tagSchema); } if (TABLE_TYPE(pTable) == TSDB_STREAM_TABLE) { - buf = taosEncodeString(buf, pTable->sql); + tlen += taosEncodeString(buf, pTable->sql); } } - return buf; + return tlen; } static void *tsdbDecodeTable(void *buf, STable **pRTable) { diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 28f562973f..ac3358cf2a 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -50,7 +50,7 @@ static int tsdbLoadSingleBlockDataCols(SRWHelper *pHelper, SCompBlock *pCompBl static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, char *content, int32_t len, int8_t comp, int numOfRows, int maxPoints, char *buffer, int bufferSize); static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *pDataCols); -static void *tsdbEncodeSCompIdx(void *buf, SCompIdx *pIdx); +static int tsdbEncodeSCompIdx(void **buf, SCompIdx *pIdx); static void *tsdbDecodeSCompIdx(void *buf, SCompIdx *pIdx); static void tsdbDestroyHelperBlock(SRWHelper *pHelper); @@ -371,8 +371,8 @@ int tsdbWriteCompIdx(SRWHelper *pHelper) { pHelper->pBuffer = trealloc(pHelper->pBuffer, tsizeof(pHelper->pBuffer) * 2); } buf = POINTER_SHIFT(pHelper->pBuffer, drift); - buf = taosEncodeVariantU32(buf, i); - buf = tsdbEncodeSCompIdx(buf, pCompIdx); + taosEncodeVariantU32(&buf, i); + tsdbEncodeSCompIdx(&buf, pCompIdx); } } @@ -1298,15 +1298,17 @@ _err: return -1; } -static void *tsdbEncodeSCompIdx(void *buf, SCompIdx *pIdx) { - buf = taosEncodeVariantU32(buf, pIdx->len); - buf = taosEncodeVariantU32(buf, pIdx->offset); - buf = taosEncodeFixedU8(buf, pIdx->hasLast); - buf = taosEncodeVariantU32(buf, pIdx->numOfBlocks); - buf = taosEncodeFixedU64(buf, pIdx->uid); - buf = taosEncodeFixedU64(buf, pIdx->maxKey); +static int tsdbEncodeSCompIdx(void **buf, SCompIdx *pIdx) { + int tlen = 0; - return buf; + tlen += taosEncodeVariantU32(buf, pIdx->len); + tlen += taosEncodeVariantU32(buf, pIdx->offset); + tlen += taosEncodeFixedU8(buf, pIdx->hasLast); + tlen += taosEncodeVariantU32(buf, pIdx->numOfBlocks); + tlen += taosEncodeFixedU64(buf, pIdx->uid); + tlen += taosEncodeFixedU64(buf, pIdx->maxKey); + + return tlen; } static void *tsdbDecodeSCompIdx(void *buf, SCompIdx *pIdx) { diff --git a/src/util/inc/tcoding.h b/src/util/inc/tcoding.h index e22c959a56..907c69bf01 100644 --- a/src/util/inc/tcoding.h +++ b/src/util/inc/tcoding.h @@ -33,9 +33,13 @@ static const int32_t TNUMBER = 1; #define ZIGZAGD(T, v) ((v) >> 1) ^ -((T)((v)&1)) // zigzag decode // ---- Fixed U8 -static FORCE_INLINE void *taosEncodeFixedU8(void *buf, uint8_t value) { - ((uint8_t *)buf)[0] = value; - return POINTER_SHIFT(buf, sizeof(value)); +static FORCE_INLINE int taosEncodeFixedU8(void **buf, uint8_t value) { + if (buf != NULL) { + ((uint8_t *)(*buf))[0] = value; + *buf = POINTER_SHIFT(*buf, sizeof(value)); + } + + return (int)sizeof(value); } static FORCE_INLINE void *taosDecodeFixedU8(void *buf, uint8_t *value) { @@ -44,9 +48,12 @@ static FORCE_INLINE void *taosDecodeFixedU8(void *buf, uint8_t *value) { } // ---- Fixed I8 -static FORCE_INLINE void *taosEncodeFixedI8(void *buf, int8_t value) { - ((int8_t *)buf)[0] = value; - return POINTER_SHIFT(buf, sizeof(value)); +static FORCE_INLINE int taosEncodeFixedI8(void **buf, int8_t value) { + if (buf != NULL) { + ((int8_t *)(*buf))[0] = value; + *buf = POINTER_SHIFT(*buf, sizeof(value)); + } + return (int)sizeof(value); } static FORCE_INLINE void *taosDecodeFixedI8(void *buf, int8_t *value) { @@ -55,15 +62,18 @@ static FORCE_INLINE void *taosDecodeFixedI8(void *buf, int8_t *value) { } // ---- Fixed U16 -static FORCE_INLINE void *taosEncodeFixedU16(void *buf, uint16_t value) { - if (IS_LITTLE_ENDIAN()) { - memcpy(buf, &value, sizeof(value)); - } else { - ((uint8_t *)buf)[0] = value & 0xff; - ((uint8_t *)buf)[1] = (value >> 8) & 0xff; +static FORCE_INLINE int taosEncodeFixedU16(void **buf, uint16_t value) { + if (buf != NULL) { + if (IS_LITTLE_ENDIAN()) { + memcpy(*buf, &value, sizeof(value)); + } else { + ((uint8_t *)(*buf))[0] = value & 0xff; + ((uint8_t *)(*buf))[1] = (value >> 8) & 0xff; + } + *buf = POINTER_SHIFT(*buf, sizeof(value)); } - return POINTER_SHIFT(buf, sizeof(value)); + return (int)sizeof(value); } static FORCE_INLINE void *taosDecodeFixedU16(void *buf, uint16_t *value) { @@ -78,7 +88,7 @@ static FORCE_INLINE void *taosDecodeFixedU16(void *buf, uint16_t *value) { } // ---- Fixed I16 -static FORCE_INLINE void *taosEncodeFixedI16(void *buf, int16_t value) { +static FORCE_INLINE int taosEncodeFixedI16(void **buf, int16_t value) { return taosEncodeFixedU16(buf, ZIGZAGE(int16_t, value)); } @@ -90,17 +100,20 @@ static FORCE_INLINE void *taosDecodeFixedI16(void *buf, int16_t *value) { } // ---- Fixed U32 -static FORCE_INLINE void *taosEncodeFixedU32(void *buf, uint32_t value) { - if (IS_LITTLE_ENDIAN()) { - memcpy(buf, &value, sizeof(value)); - } else { - ((uint8_t *)buf)[0] = value & 0xff; - ((uint8_t *)buf)[1] = (value >> 8) & 0xff; - ((uint8_t *)buf)[2] = (value >> 16) & 0xff; - ((uint8_t *)buf)[3] = (value >> 24) & 0xff; +static FORCE_INLINE int taosEncodeFixedU32(void **buf, uint32_t value) { + if (buf != NULL) { + if (IS_LITTLE_ENDIAN()) { + memcpy(*buf, &value, sizeof(value)); + } else { + ((uint8_t *)(*buf))[0] = value & 0xff; + ((uint8_t *)(*buf))[1] = (value >> 8) & 0xff; + ((uint8_t *)(*buf))[2] = (value >> 16) & 0xff; + ((uint8_t *)(*buf))[3] = (value >> 24) & 0xff; + } + *buf = POINTER_SHIFT(*buf, sizeof(value)); } - return POINTER_SHIFT(buf, sizeof(value)); + return (int)sizeof(value); } static FORCE_INLINE void *taosDecodeFixedU32(void *buf, uint32_t *value) { @@ -117,7 +130,7 @@ static FORCE_INLINE void *taosDecodeFixedU32(void *buf, uint32_t *value) { } // ---- Fixed I32 -static FORCE_INLINE void *taosEncodeFixedI32(void *buf, int32_t value) { +static FORCE_INLINE int taosEncodeFixedI32(void **buf, int32_t value) { return taosEncodeFixedU32(buf, ZIGZAGE(int32_t, value)); } @@ -129,21 +142,25 @@ static FORCE_INLINE void *taosDecodeFixedI32(void *buf, int32_t *value) { } // ---- Fixed U64 -static FORCE_INLINE void *taosEncodeFixedU64(void *buf, uint64_t value) { - if (IS_LITTLE_ENDIAN()) { - memcpy(buf, &value, sizeof(value)); - } else { - ((uint8_t *)buf)[0] = value & 0xff; - ((uint8_t *)buf)[1] = (value >> 8) & 0xff; - ((uint8_t *)buf)[2] = (value >> 16) & 0xff; - ((uint8_t *)buf)[3] = (value >> 24) & 0xff; - ((uint8_t *)buf)[4] = (value >> 32) & 0xff; - ((uint8_t *)buf)[5] = (value >> 40) & 0xff; - ((uint8_t *)buf)[6] = (value >> 48) & 0xff; - ((uint8_t *)buf)[7] = (value >> 56) & 0xff; +static FORCE_INLINE int taosEncodeFixedU64(void **buf, uint64_t value) { + if (buf != NULL) { + if (IS_LITTLE_ENDIAN()) { + memcpy(*buf, &value, sizeof(value)); + } else { + ((uint8_t *)(*buf))[0] = value & 0xff; + ((uint8_t *)(*buf))[1] = (value >> 8) & 0xff; + ((uint8_t *)(*buf))[2] = (value >> 16) & 0xff; + ((uint8_t *)(*buf))[3] = (value >> 24) & 0xff; + ((uint8_t *)(*buf))[4] = (value >> 32) & 0xff; + ((uint8_t *)(*buf))[5] = (value >> 40) & 0xff; + ((uint8_t *)(*buf))[6] = (value >> 48) & 0xff; + ((uint8_t *)(*buf))[7] = (value >> 56) & 0xff; + } + + *buf = POINTER_SHIFT(*buf, sizeof(value)); } - return POINTER_SHIFT(buf, sizeof(value)); + return (int)sizeof(value); } static FORCE_INLINE void *taosDecodeFixedU64(void *buf, uint64_t *value) { @@ -164,7 +181,7 @@ static FORCE_INLINE void *taosDecodeFixedU64(void *buf, uint64_t *value) { } // ---- Fixed I64 -static FORCE_INLINE void *taosEncodeFixedI64(void *buf, int64_t value) { +static FORCE_INLINE int taosEncodeFixedI64(void **buf, int64_t value) { return taosEncodeFixedU64(buf, ZIGZAGE(int64_t, value)); } @@ -176,18 +193,21 @@ static FORCE_INLINE void *taosDecodeFixedI64(void *buf, int64_t *value) { } // ---- Variant U16 -static FORCE_INLINE void *taosEncodeVariantU16(void *buf, uint16_t value) { +static FORCE_INLINE int taosEncodeVariantU16(void **buf, uint16_t value) { int i = 0; while (value >= ENCODE_LIMIT) { - ((uint8_t *)buf)[i] = (value | ENCODE_LIMIT); + if (buf != NULL) ((uint8_t *)(*buf))[i] = (value | ENCODE_LIMIT); value >>= 7; i++; ASSERT(i < 3); } - ((uint8_t *)buf)[i] = value; + if (buf != NULL) { + ((uint8_t *)(*buf))[i] = value; + *buf = POINTER_SHIFT(*buf, i + 1); + } - return POINTER_SHIFT(buf, i + 1); + return i + 1; } static FORCE_INLINE void *taosDecodeVariantU16(void *buf, uint16_t *value) { @@ -209,7 +229,7 @@ static FORCE_INLINE void *taosDecodeVariantU16(void *buf, uint16_t *value) { } // ---- Variant I16 -static FORCE_INLINE void *taosEncodeVariantI16(void *buf, int16_t value) { +static FORCE_INLINE int taosEncodeVariantI16(void **buf, int16_t value) { return taosEncodeVariantU16(buf, ZIGZAGE(int16_t, value)); } @@ -221,18 +241,21 @@ static FORCE_INLINE void *taosDecodeVariantI16(void *buf, int16_t *value) { } // ---- Variant U32 -static FORCE_INLINE void *taosEncodeVariantU32(void *buf, uint32_t value) { +static FORCE_INLINE int taosEncodeVariantU32(void **buf, uint32_t value) { int i = 0; while (value >= ENCODE_LIMIT) { - ((uint8_t *)buf)[i] = (value | ENCODE_LIMIT); + if (buf != NULL) ((uint8_t *)(*buf))[i] = (value | ENCODE_LIMIT); value >>= 7; i++; ASSERT(i < 5); } - ((uint8_t *)buf)[i] = value; + if (buf != NULL) { + ((uint8_t *)(*buf))[i] = value; + *buf = POINTER_SHIFT(*buf, i + 1); + } - return POINTER_SHIFT(buf, i + 1); + return i + 1; } static FORCE_INLINE void *taosDecodeVariantU32(void *buf, uint32_t *value) { @@ -254,7 +277,7 @@ static FORCE_INLINE void *taosDecodeVariantU32(void *buf, uint32_t *value) { } // ---- Variant I32 -static FORCE_INLINE void *taosEncodeVariantI32(void *buf, int32_t value) { +static FORCE_INLINE int taosEncodeVariantI32(void **buf, int32_t value) { return taosEncodeVariantU32(buf, ZIGZAGE(int32_t, value)); } @@ -266,18 +289,21 @@ static FORCE_INLINE void *taosDecodeVariantI32(void *buf, int32_t *value) { } // ---- Variant U64 -static FORCE_INLINE void *taosEncodeVariantU64(void *buf, uint64_t value) { +static FORCE_INLINE int taosEncodeVariantU64(void **buf, uint64_t value) { int i = 0; while (value >= ENCODE_LIMIT) { - ((uint8_t *)buf)[i] = (value | ENCODE_LIMIT); + if (buf != NULL) ((uint8_t *)(*buf))[i] = (value | ENCODE_LIMIT); value >>= 7; i++; ASSERT(i < 10); } - ((uint8_t *)buf)[i] = value; + if (buf != NULL) { + ((uint8_t *)(*buf))[i] = value; + *buf = POINTER_SHIFT(*buf, i + 1); + } - return POINTER_SHIFT(buf, i + 1); + return i + 1; } static FORCE_INLINE void *taosDecodeVariantU64(void *buf, uint64_t *value) { @@ -299,7 +325,7 @@ static FORCE_INLINE void *taosDecodeVariantU64(void *buf, uint64_t *value) { } // ---- Variant I64 -static FORCE_INLINE void *taosEncodeVariantI64(void *buf, int64_t value) { +static FORCE_INLINE int taosEncodeVariantI64(void **buf, int64_t value) { return taosEncodeVariantU64(buf, ZIGZAGE(int64_t, value)); } @@ -311,13 +337,18 @@ static FORCE_INLINE void *taosDecodeVariantI64(void *buf, int64_t *value) { } // ---- string -static FORCE_INLINE void *taosEncodeString(void *buf, char *value) { +static FORCE_INLINE int taosEncodeString(void **buf, char *value) { + int tlen = 0; size_t size = strlen(value); - buf = taosEncodeVariantU64(buf, size); - memcpy(buf, value, size); + tlen += taosEncodeVariantU64(buf, size); + if (buf != NULL) { + memcpy(*buf, value, size); + *buf = POINTER_SHIFT(*buf, size); + } + tlen += size; - return POINTER_SHIFT(buf, size); + return tlen; } static FORCE_INLINE void *taosDecodeString(void *buf, char **value) { diff --git a/src/util/src/tkvstore.c b/src/util/src/tkvstore.c index be9c4c19b0..a84a5ac011 100644 --- a/src/util/src/tkvstore.c +++ b/src/util/src/tkvstore.c @@ -41,7 +41,7 @@ typedef struct { } SKVRecord; static int tdInitKVStoreHeader(int fd, char *fname); -static void * tdEncodeStoreInfo(void *buf, SStoreInfo *pInfo); +static int tdEncodeStoreInfo(void **buf, SStoreInfo *pInfo); static void * tdDecodeStoreInfo(void *buf, SStoreInfo *pInfo); static SKVStore *tdNewKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void *appH); static char * tdGetKVStoreSnapshotFname(char *fdata); @@ -49,7 +49,7 @@ static char * tdGetKVStoreNewFname(char *fdata); static void tdFreeKVStore(SKVStore *pStore); static int tdUpdateKVStoreHeader(int fd, char *fname, SStoreInfo *pInfo); static int tdLoadKVStoreHeader(int fd, char *fname, SStoreInfo *pInfo); -static void * tdEncodeKVRecord(void *buf, SKVRecord *pRecord); +static int tdEncodeKVRecord(void **buf, SKVRecord *pRecord); static void * tdDecodeKVRecord(void *buf, SKVRecord *pRecord); static int tdRestoreKVStore(SKVStore *pStore); @@ -269,7 +269,8 @@ int tdDropKVStoreRecord(SKVStore *pStore, uint64_t uid) { rInfo.uid = pRecord->uid; rInfo.size = pRecord->size; - void *pBuf = tdEncodeKVRecord(buf, &rInfo); + void *pBuf = buf; + tdEncodeKVRecord(&pBuf, &rInfo); if (twrite(pStore->fd, buf, POINTER_DISTANCE(pBuf, buf)) < POINTER_DISTANCE(pBuf, buf)) { uError("failed to write %d bytes to file %s since %s", POINTER_DISTANCE(pBuf, buf), pStore->fname, strerror(errno)); @@ -344,7 +345,8 @@ static int tdUpdateKVStoreHeader(int fd, char *fname, SStoreInfo *pInfo) { return -1; } - void *pBuf = tdEncodeStoreInfo(buf, pInfo); + void *pBuf = buf; + tdEncodeStoreInfo(&pBuf, pInfo); ASSERT(POINTER_DISTANCE(pBuf, buf) + sizeof(TSCKSUM) <= TD_KVSTORE_HEADER_SIZE); taosCalcChecksumAppend(0, (uint8_t *)buf, TD_KVSTORE_HEADER_SIZE); @@ -363,13 +365,14 @@ static int tdInitKVStoreHeader(int fd, char *fname) { return tdUpdateKVStoreHeader(fd, fname, &info); } -static void *tdEncodeStoreInfo(void *buf, SStoreInfo *pInfo) { - buf = taosEncodeVariantI64(buf, pInfo->size); - buf = taosEncodeVariantI64(buf, pInfo->tombSize); - buf = taosEncodeVariantI64(buf, pInfo->nRecords); - buf = taosEncodeVariantI64(buf, pInfo->nDels); +static int tdEncodeStoreInfo(void **buf, SStoreInfo *pInfo) { + int tlen = 0; + tlen += taosEncodeVariantI64(buf, pInfo->size); + tlen += taosEncodeVariantI64(buf, pInfo->tombSize); + tlen += taosEncodeVariantI64(buf, pInfo->nRecords); + tlen += taosEncodeVariantI64(buf, pInfo->nDels); - return buf; + return tlen; } static void *tdDecodeStoreInfo(void *buf, SStoreInfo *pInfo) { @@ -450,12 +453,13 @@ static char *tdGetKVStoreNewFname(char *fdata) { return fname; } -static void *tdEncodeKVRecord(void *buf, SKVRecord *pRecord) { - buf = taosEncodeFixedU64(buf, pRecord->uid); - buf = taosEncodeFixedI64(buf, pRecord->offset); - buf = taosEncodeFixedI64(buf, pRecord->size); +static int tdEncodeKVRecord(void **buf, SKVRecord *pRecord) { + int tlen = 0; + tlen += taosEncodeFixedU64(buf, pRecord->uid); + tlen += taosEncodeFixedI64(buf, pRecord->offset); + tlen += taosEncodeFixedI64(buf, pRecord->size); - return buf; + return tlen; } static void *tdDecodeKVRecord(void *buf, SKVRecord *pRecord) { diff --git a/src/util/tests/codingTests.cpp b/src/util/tests/codingTests.cpp index 57e21a828c..0cd9524646 100644 --- a/src/util/tests/codingTests.cpp +++ b/src/util/tests/codingTests.cpp @@ -9,120 +9,144 @@ static bool test_fixed_uint16(uint16_t value) { char buf[20] = "\0"; uint16_t value_check = 0; - void *ptr1 = taosEncodeFixedU16(static_cast(buf), value); - void *ptr2 = taosDecodeFixedU16(static_cast(buf), &value_check); + void *pBuf = (void *)buf; - return ((ptr2 != NULL) && (value == value_check) && (ptr1 == ptr2)); + int tlen = taosEncodeFixedU16(static_cast(&pBuf), value); + void *ptr = taosDecodeFixedU16(static_cast(buf), &value_check); + + return ((ptr != NULL) && (value == value_check) && (pBuf == ptr) && POINTER_DISTANCE(pBuf, buf) == tlen); } static bool test_fixed_int16(int16_t value) { char buf[20] = "\0"; int16_t value_check = 0; - void *ptr1 = taosEncodeFixedI16(static_cast(buf), value); - void *ptr2 = taosDecodeFixedI16(static_cast(buf), &value_check); + void *pBuf = (void *)buf; - return ((ptr2 != NULL) && (value == value_check) && (ptr1 == ptr2)); + int tlen = taosEncodeFixedI16(static_cast(&pBuf), value); + void *ptr = taosDecodeFixedI16(static_cast(buf), &value_check); + + return ((ptr != NULL) && (value == value_check) && (pBuf == ptr) && POINTER_DISTANCE(pBuf, buf) == tlen); } static bool test_fixed_uint32(uint32_t value) { char buf[20] = "\0"; uint32_t value_check = 0; - void *ptr1 = taosEncodeFixedU32(static_cast(buf), value); - void *ptr2 = taosDecodeFixedU32(static_cast(buf), &value_check); + void *pBuf = (void *)buf; - return ((ptr2 != NULL) && (value == value_check) && (ptr1 == ptr2)); + int tlen = taosEncodeFixedU32(static_cast(&pBuf), value); + void *ptr = taosDecodeFixedU32(static_cast(buf), &value_check); + + return ((ptr != NULL) && (value == value_check) && (pBuf == ptr) && POINTER_DISTANCE(pBuf, buf) == tlen); } static bool test_fixed_int32(int32_t value) { char buf[20] = "\0"; int32_t value_check = 0; - void *ptr1 = taosEncodeFixedI32(static_cast(buf), value); - void *ptr2 = taosDecodeFixedI32(static_cast(buf), &value_check); + void *pBuf = (void *)buf; - return ((ptr2 != NULL) && (value == value_check) && (ptr1 == ptr2)); + int tlen = taosEncodeFixedI32(static_cast(&pBuf), value); + void *ptr = taosDecodeFixedI32(static_cast(buf), &value_check); + + return ((ptr != NULL) && (value == value_check) && (pBuf == ptr) && POINTER_DISTANCE(pBuf, buf) == tlen); } static bool test_fixed_uint64(uint64_t value) { char buf[20] = "\0"; uint64_t value_check = 0; - void *ptr1 = taosEncodeFixedU64(static_cast(buf), value); - void *ptr2 = taosDecodeFixedU64(static_cast(buf), &value_check); + void *pBuf = (void *)buf; - return ((ptr2 != NULL) && (value == value_check) && (ptr1 == ptr2)); + int tlen = taosEncodeFixedU64(static_cast(&pBuf), value); + void *ptr = taosDecodeFixedU64(static_cast(buf), &value_check); + + return ((ptr != NULL) && (value == value_check) && (pBuf == ptr) && POINTER_DISTANCE(pBuf, buf) == tlen); } static bool test_fixed_int64(int64_t value) { char buf[20] = "\0"; int64_t value_check = 0; - void *ptr1 = taosEncodeFixedI64(static_cast(buf), value); - void *ptr2 = taosDecodeFixedI64(static_cast(buf), &value_check); + void *pBuf = (void *)buf; - return ((ptr2 != NULL) && (value == value_check) && (ptr1 == ptr2)); + int tlen = taosEncodeFixedI64(static_cast(&pBuf), value); + void *ptr = taosDecodeFixedI64(static_cast(buf), &value_check); + + return ((ptr != NULL) && (value == value_check) && (pBuf == ptr) && POINTER_DISTANCE(pBuf, buf) == tlen); } static bool test_variant_uint16(uint16_t value) { char buf[20] = "\0"; uint16_t value_check = 0; - void *ptr1 = taosEncodeVariantU16(static_cast(buf), value); - void *ptr2 = taosDecodeVariantU16(static_cast(buf), &value_check); + void *pBuf = (void *)buf; - return ((ptr2 != NULL) && (value == value_check) && (ptr1 == ptr2)); + int tlen = taosEncodeVariantU16(static_cast(&pBuf), value); + void *ptr = taosDecodeVariantU16(static_cast(buf), &value_check); + + return ((ptr != NULL) && (value == value_check) && (pBuf == ptr) && POINTER_DISTANCE(pBuf, buf) == tlen); } static bool test_variant_int16(int16_t value) { char buf[20] = "\0"; int16_t value_check = 0; - void *ptr1 = taosEncodeVariantI16(static_cast(buf), value); - void *ptr2 = taosDecodeVariantI16(static_cast(buf), &value_check); + void *pBuf = (void *)buf; - return ((ptr2 != NULL) && (value == value_check) && (ptr1 == ptr2)); + int tlen = taosEncodeVariantI16(static_cast(&pBuf), value); + void *ptr = taosDecodeVariantI16(static_cast(buf), &value_check); + + return ((ptr != NULL) && (value == value_check) && (pBuf == ptr) && POINTER_DISTANCE(pBuf, buf) == tlen); } static bool test_variant_uint32(uint32_t value) { char buf[20] = "\0"; uint32_t value_check = 0; - void *ptr1 = taosEncodeVariantU32(static_cast(buf), value); - void *ptr2 = taosDecodeVariantU32(static_cast(buf), &value_check); + void *pBuf = (void *)buf; - return ((ptr2 != NULL) && (value == value_check) && (ptr1 == ptr2)); + int tlen = taosEncodeVariantU32(static_cast(&pBuf), value); + void *ptr = taosDecodeVariantU32(static_cast(buf), &value_check); + + return ((ptr != NULL) && (value == value_check) && (pBuf == ptr) && POINTER_DISTANCE(pBuf, buf) == tlen); } static bool test_variant_int32(int32_t value) { char buf[20] = "\0"; int32_t value_check = 0; - void *ptr1 = taosEncodeVariantI32(static_cast(buf), value); - void *ptr2 = taosDecodeVariantI32(static_cast(buf), &value_check); + void *pBuf = (void *)buf; - return ((ptr2 != NULL) && (value == value_check) && (ptr1 == ptr2)); + int tlen = taosEncodeVariantI32(static_cast(&pBuf), value); + void *ptr = taosDecodeVariantI32(static_cast(buf), &value_check); + + return ((ptr != NULL) && (value == value_check) && (pBuf == ptr) && POINTER_DISTANCE(pBuf, buf) == tlen); } static bool test_variant_uint64(uint64_t value) { char buf[20] = "\0"; uint64_t value_check = 0; - void *ptr1 = taosEncodeVariantU64(static_cast(buf), value); - void *ptr2 = taosDecodeVariantU64(static_cast(buf), &value_check); + void *pBuf = (void *)buf; - return ((ptr2 != NULL) && (value == value_check) && (ptr1 == ptr2)); + int tlen = taosEncodeVariantU64(static_cast(&pBuf), value); + void *ptr = taosDecodeVariantU64(static_cast(buf), &value_check); + + return ((ptr != NULL) && (value == value_check) && (pBuf == ptr) && POINTER_DISTANCE(pBuf, buf) == tlen); } static bool test_variant_int64(int64_t value) { char buf[20] = "\0"; int64_t value_check = 0; - void *ptr1 = taosEncodeVariantI64(static_cast(buf), value); - void *ptr2 = taosDecodeVariantI64(static_cast(buf), &value_check); + void *pBuf = (void *)buf; - return ((ptr2 != NULL) && (value == value_check) && (ptr1 == ptr2)); + int tlen = taosEncodeVariantI64(static_cast(&pBuf), value); + void *ptr = taosDecodeVariantI64(static_cast(buf), &value_check); + + return ((ptr != NULL) && (value == value_check) && (pBuf == ptr) && POINTER_DISTANCE(pBuf, buf) == tlen); } TEST(codingTest, fixed_encode_decode) { From 6eba0ff9f0381f4a33fe4e44c05ead620ccfadef Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 18 Jun 2020 02:02:38 +0000 Subject: [PATCH 39/65] TD-353 --- src/tsdb/inc/tsdbMain.h | 20 +++++++-- src/tsdb/src/tsdbMemTable.c | 87 +++++++++++++++++++++++++++++-------- src/tsdb/src/tsdbMeta.c | 57 ++++++++++++++++++------ src/tsdb/src/tsdbRWHelper.c | 1 + 4 files changed, 128 insertions(+), 37 deletions(-) diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 44e8002ac1..2602544bb4 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -110,6 +110,17 @@ typedef struct { SList* bufBlockList; } SMemTable; +enum { TSDB_UPDATE_META, TSDB_DROP_META }; +typedef struct __attribute__((packed)){ + char act; + uint64_t uid; +} SActObj; + +typedef struct { + int len; + char cont[]; +} SActCont; + // ------------------ tsdbFile.c extern const char* tsdbFileSuffix[]; typedef enum { @@ -302,10 +313,11 @@ void tsdbCloseBufPool(STsdbRepo* pRepo); SListNode* tsdbAllocBufBlockFromPool(STsdbRepo* pRepo); // ------------------ tsdbMemTable.c -int tsdbInsertRowToMem(STsdbRepo* pRepo, SDataRow row, STable* pTable); -int tsdbRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable); -int tsdbUnRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable); -int tsdbTakeMemSnapshot(STsdbRepo* pRepo, SMemTable** pMem, SMemTable** pIMem); +int tsdbInsertRowToMem(STsdbRepo* pRepo, SDataRow row, STable* pTable); +int tsdbRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable); +int tsdbUnRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable); +int tsdbTakeMemSnapshot(STsdbRepo* pRepo, SMemTable** pMem, SMemTable** pIMem); +void* tsdbAllocBytes(STsdbRepo* pRepo, int bytes); // ------------------ tsdbFile.c #define TSDB_KEY_FILEID(key, daysPerFile, precision) ((key) / tsMsPerDay[(precision)] / (daysPerFile)) diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index ef9cc1b07e..4b8c76702e 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -25,7 +25,6 @@ typedef struct { static FORCE_INLINE STsdbBufBlock *tsdbGetCurrBufBlock(STsdbRepo *pRepo); -static void * tsdbAllocBytes(STsdbRepo *pRepo, int bytes); static void tsdbFreeBytes(STsdbRepo *pRepo, void *ptr, int bytes); static SMemTable * tsdbNewMemTable(STsdbCfg *pCfg); static void tsdbFreeMemTable(SMemTable *pMemTable); @@ -33,6 +32,7 @@ static STableData *tsdbNewTableData(STsdbCfg *pCfg, STable *pTable); static void tsdbFreeTableData(STableData *pTableData); static char * tsdbGetTsTupleKey(const void *data); static void * tsdbCommitData(void *arg); +static int tsdbCommitMeta(STsdbRepo *pRepo); static void tsdbEndCommit(STsdbRepo *pRepo); static TSKEY tsdbNextIterKey(SCommitIter *pIter); static int tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey); @@ -170,21 +170,7 @@ int tsdbTakeMemSnapshot(STsdbRepo *pRepo, SMemTable **pMem, SMemTable **pIMem) { return 0; } -// ---------------- LOCAL FUNCTIONS ---------------- -static FORCE_INLINE STsdbBufBlock *tsdbGetCurrBufBlock(STsdbRepo *pRepo) { - ASSERT(pRepo != NULL); - if (pRepo->mem == NULL) return NULL; - - SListNode *pNode = listTail(pRepo->mem->bufBlockList); - if (pNode == NULL) return NULL; - - STsdbBufBlock *pBufBlock = NULL; - tdListNodeGetData(pRepo->mem->bufBlockList, pNode, (void *)(&pBufBlock)); - - return pBufBlock; -} - -static void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) { +void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) { STsdbCfg * pCfg = &pRepo->config; STsdbBufBlock *pBufBlock = tsdbGetCurrBufBlock(pRepo); int code = 0; @@ -256,6 +242,20 @@ static void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) { return ptr; } +// ---------------- LOCAL FUNCTIONS ---------------- +static FORCE_INLINE STsdbBufBlock *tsdbGetCurrBufBlock(STsdbRepo *pRepo) { + ASSERT(pRepo != NULL); + if (pRepo->mem == NULL) return NULL; + + SListNode *pNode = listTail(pRepo->mem->bufBlockList); + if (pNode == NULL) return NULL; + + STsdbBufBlock *pBufBlock = NULL; + tdListNodeGetData(pRepo->mem->bufBlockList, pNode, (void *)(&pBufBlock)); + + return pBufBlock; +} + static void tsdbFreeBytes(STsdbRepo *pRepo, void *ptr, int bytes) { STsdbBufBlock *pBufBlock = tsdbGetCurrBufBlock(pRepo); ASSERT(pBufBlock != NULL); @@ -396,10 +396,13 @@ static void *tsdbCommitData(void *arg) { } } - // TODO: Do retention actions - tsdbFitRetention(pRepo); + // Commit to update meta file + if (tsdbCommitMeta(pRepo) < 0) { + tsdbError("vgId:%d failed to commit data while committing meta data since %s", REPO_ID(pRepo), tstrerror(terrno)); + goto _exit; + } - // TODO: Commit action meta data + tsdbFitRetention(pRepo); _exit: tdFreeDataCols(pDataCols); @@ -411,6 +414,52 @@ _exit: return NULL; } +static int tsdbCommitMeta(STsdbRepo *pRepo) { + SMemTable *pMem = pRepo->imem; + STsdbMeta *pMeta = pRepo->tsdbMeta; + SActObj * pAct = NULL; + SActCont * pCont = NULL; + + if (listNEles(pMem->actList) > 0) { + if (tdKVStoreStartCommit(pMeta->pStore) < 0) { + tsdbError("vgId:%d failed to commit data while start commit meta since %s", REPO_ID(pRepo), tstrerror(terrno)); + goto _err; + } + + SListNode *pNode = NULL; + + while ((pNode = tdListPopHead(pMem->actList)) != NULL) { + pAct = (SActObj *)pNode->data; + if (pAct->act == TSDB_UPDATE_META) { + pCont = (SActCont *)POINTER_SHIFT(pAct, sizeof(SActObj)); + if (tdUpdateKVStoreRecord(pMeta->pStore, pAct->uid, (void *)(pCont->cont), pCont->len) < 0) { + tsdbError("vgId:%d failed to update meta with uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid, + tstrerror(terrno)); + goto _err; + } + } else if (pAct->act == TSDB_DROP_META) { + if (tdDropKVStoreRecord(pMeta->pStore, pAct->uid) < 0) { + tsdbError("vgId:%d failed to drop meta with uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid, + tstrerror(terrno)); + goto _err; + } + } else { + ASSERT(false); + } + } + + if (tdKVStoreEndCommit(pMeta->pStore) < 0) { + tsdbError("vgId:%d failed to commit data while end commit meta since %s", REPO_ID(pRepo), tstrerror(terrno)); + goto _err; + } + } + + return 0; + +_err: + return -1; +} + static void tsdbEndCommit(STsdbRepo *pRepo) { ASSERT(pRepo->commit == 1); tsdbLockRepo(pRepo); diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 9eb5d10371..98c390f5a5 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -46,6 +46,8 @@ static int tsdbEncodeTableName(void **buf, tstr *name); static void * tsdbDecodeTableName(void *buf, tstr **name); static int tsdbEncodeTable(void **buf, STable *pTable); static void * tsdbDecodeTable(void *buf, STable **pRTable); +static int tsdbGetTableEncodeSize(int8_t act, STable *pTable); +static void * tsdbInsertTableAct(STsdbRepo *pRepo, int8_t act, void *buf, STable *pTable); // ------------------ OUTER FUNCTIONS ------------------ int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg) { @@ -85,20 +87,18 @@ int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg) { } if (tsdbAddTableToMeta(pRepo, table, true) < 0) goto _err; - int tlen = tsdbEncodeTable(NULL, pTable); - ASSERT(tlen > 0); - - // // Write to meta file - // int bufLen = 0; - // char *buf = malloc(1024 * 1024); - // if (newSuper) { - // tsdbEncodeTable(super, buf, &bufLen); - // tsdbInsertMetaRecord(pMeta->mfh, super->tableId.uid, buf, bufLen); - // } - - // tsdbEncodeTable(table, buf, &bufLen); - // tsdbInsertMetaRecord(pMeta->mfh, table->tableId.uid, buf, bufLen); - // tfree(buf); + // Write to memtable action + int tlen1 = (newSuper) ? tsdbGetTableEncodeSize(TSDB_UPDATE_META, super) : 0; + int tlen2 = tsdbGetTableEncodeSize(TSDB_UPDATE_META, table); + int tlen = tlen1 + tlen2; + void *buf = tsdbAllocBytes(pRepo, tlen); + ASSERT(buf != NULL); + if (newSuper) { + void *pBuf = tsdbInsertTableAct(pRepo, TSDB_UPDATE_META, buf, super); + ASSERT(POINTER_DISTANCE(pBuf, buf) == tlen1); + buf = pBuf; + } + tsdbInsertTableAct(pRepo, TSDB_UPDATE_META, buf, super); return 0; @@ -1107,4 +1107,33 @@ static void *tsdbDecodeTable(void *buf, STable **pRTable) { *pRTable = pTable; return buf; +} + +static int tsdbGetTableEncodeSize(int8_t act, STable *pTable) { + int tlen = sizeof(SListNode) + sizeof(SActObj); + if (act == TSDB_UPDATE_META) tlen += (sizeof(SActCont) + tsdbEncodeTable(NULL, pTable) + sizeof(TSCKSUM)); + + return tlen; +} + +static void *tsdbInsertTableAct(STsdbRepo *pRepo, int8_t act, void *buf, STable *pTable) { + SListNode *pNode = (SListNode *)buf; + SActObj * pAct = (SActObj *)(pNode->data); + SActCont * pCont = (SActCont *)POINTER_SHIFT(pAct, sizeof(*pAct)); + void * pBuf = (void *)pCont; + + pNode->prev = pNode->next = NULL; + pAct->act = act; + pAct->uid = TABLE_UID(pTable); + + if (act == TSDB_UPDATE_META) { + pBuf = (void *)(pCont->cont); + pCont->len = tsdbEncodeTable(&pBuf, pTable) + sizeof(TSCKSUM); + taosCalcChecksumAppend(0, (uint8_t *)pCont->cont, pCont->len); + pBuf = POINTER_SHIFT(pBuf, sizeof(TSCKSUM)); + } + + tdListAppendNode(pRepo->mem->actList, pNode); + + return pBuf; } \ No newline at end of file diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index ac3358cf2a..08fe69b0ee 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -1136,6 +1136,7 @@ static int tsdbInitHelper(SRWHelper *pHelper, STsdbRepo *pRepo, tsdb_rw_helper_t // Init block part if (tsdbInitHelperBlock(pHelper) < 0) goto _err; + // TODO: pMeta->maxRowBytes and pMeta->maxCols may change here causing invalid write pHelper->pBuffer = tmalloc(sizeof(SCompData) + (sizeof(SCompCol) + sizeof(TSCKSUM) + COMP_OVERFLOW_BYTES) * pMeta->maxCols + pMeta->maxRowBytes * pCfg->maxRowsPerFileBlock + sizeof(TSCKSUM)); From 2e3441d1089e367651a641cdf124d56f92c0939b Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 18 Jun 2020 02:29:39 +0000 Subject: [PATCH 40/65] TD-353 --- src/tsdb/inc/tsdbMain.h | 1 + src/tsdb/src/tsdbMain.c | 4 ++- src/tsdb/src/tsdbMemTable.c | 70 ++++++++++++++++++++----------------- 3 files changed, 41 insertions(+), 34 deletions(-) diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 2602544bb4..5b17e53904 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -318,6 +318,7 @@ int tsdbRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable); int tsdbUnRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable); int tsdbTakeMemSnapshot(STsdbRepo* pRepo, SMemTable** pMem, SMemTable** pIMem); void* tsdbAllocBytes(STsdbRepo* pRepo, int bytes); +int tsdbAsyncCommit(STsdbRepo* pRepo); // ------------------ tsdbFile.c #define TSDB_KEY_FILEID(key, daysPerFile, precision) ((key) / tsMsPerDay[(precision)] / (daysPerFile)) diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index e6cf865399..36d8901bdf 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -135,12 +135,14 @@ _err: return NULL; } +// Note: all working thread and query thread must stopped when calling this function void tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit) { if (repo == NULL) return; STsdbRepo *pRepo = (STsdbRepo *)repo; - // TODO: wait for commit over + tsdbAsyncCommit(pRepo); + if (pRepo->commit) pthread_join(pRepo->commitThread, NULL); tsdbCloseFileH(pRepo); tsdbCloseBufPool(pRepo); diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 4b8c76702e..ce57111125 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -173,42 +173,10 @@ int tsdbTakeMemSnapshot(STsdbRepo *pRepo, SMemTable **pMem, SMemTable **pIMem) { void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) { STsdbCfg * pCfg = &pRepo->config; STsdbBufBlock *pBufBlock = tsdbGetCurrBufBlock(pRepo); - int code = 0; if (pBufBlock != NULL && pBufBlock->remain < bytes) { if (listNEles(pRepo->mem->bufBlockList) >= pCfg->totalBlocks / 2) { // need to commit mem - if (pRepo->imem) { - code = pthread_join(pRepo->commitThread, NULL); - if (code != 0) { - tsdbError("vgId:%d failed to thread join since %s", REPO_ID(pRepo), strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(errno); - return NULL; - } - - if (tsdbUnRefMemTable(pRepo, pRepo->imem) < 0) { - tsdbError("vgId:%d failed to unref memtable since %s", REPO_ID(pRepo), tstrerror(terrno)) - return NULL; - } - } - - ASSERT(pRepo->commit == 0); - SMemTable *pImem = pRepo->imem; - if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_START); - - if (tsdbLockRepo(pRepo) < 0) return NULL; - pRepo->imem = pRepo->mem; - pRepo->mem = NULL; - pRepo->commit = 1; - code = pthread_create(&pRepo->commitThread, NULL, tsdbCommitData, (void *)pRepo); - if (code != 0) { - tsdbError("vgId:%d failed to create commit thread since %s", REPO_ID(pRepo), strerror(errno)); - terrno = TAOS_SYSTEM_ERROR(code); - tsdbUnlockRepo(pRepo); - return NULL; - } - if (tsdbUnlockRepo(pRepo) < 0) return NULL; - - if (pImem && tsdbUnRefMemTable(pRepo, pImem) < 0) return NULL; + if (tsdbAsyncCommit(pRepo) < 0) return NULL; } else { if (tsdbLockRepo(pRepo) < 0) return NULL; SListNode *pNode = tsdbAllocBufBlockFromPool(pRepo); @@ -242,6 +210,42 @@ void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) { return ptr; } +int tsdbAsyncCommit(STsdbRepo *pRepo) { + SMemTable *pIMem = pRepo->imem; + int code = 0; + + if (pIMem != NULL) { + ASSERT(pRepo->commit); + code = pthread_join(pRepo->commitThread, NULL); + if (code != 0) { + tsdbError("vgId:%d failed to thread join since %s", REPO_ID(pRepo), strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + } + + ASSERT(pRepo->commit == 0); + if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_START); + if (pRepo->mem != NULL) { + if (tsdbLockRepo(pRepo) < 0) return -1; + pRepo->imem = pRepo->mem; + pRepo->mem = NULL; + pRepo->commit = 1; + code = pthread_create(&pRepo->commitThread, NULL, tsdbCommitData, (void *)pRepo); + if (code != 0) { + tsdbError("vgId:%d failed to create commit thread since %s", REPO_ID(pRepo), strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(code); + tsdbUnlockRepo(pRepo); + return -1; + } + if (tsdbUnlockRepo(pRepo) < 0) return -1; + } + + if (pIMem && tsdbUnRefMemTable(pRepo, pIMem) < 0) return -1; + + return 0; +} + // ---------------- LOCAL FUNCTIONS ---------------- static FORCE_INLINE STsdbBufBlock *tsdbGetCurrBufBlock(STsdbRepo *pRepo) { ASSERT(pRepo != NULL); From e8ddb5421c03f949fdd997491b8eee971201161a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 18 Jun 2020 05:25:30 +0000 Subject: [PATCH 41/65] TD-353 --- src/tsdb/src/tsdbMeta.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 98c390f5a5..a0a3aebfd4 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -1036,10 +1036,10 @@ static int tsdbEncodeTable(void **buf, STable *pTable) { ASSERT(pTable != NULL); int tlen = 0; - tlen = taosEncodeFixedU8(buf, pTable->type); - tlen = tsdbEncodeTableName(buf, pTable->name); - tlen = taosEncodeFixedU64(buf, TABLE_UID(pTable)); - tlen = taosEncodeFixedI32(buf, TABLE_TID(pTable)); + tlen += taosEncodeFixedU8(buf, pTable->type); + tlen += tsdbEncodeTableName(buf, pTable->name); + tlen += taosEncodeFixedU64(buf, TABLE_UID(pTable)); + tlen += taosEncodeFixedI32(buf, TABLE_TID(pTable)); if (TABLE_TYPE(pTable) == TSDB_CHILD_TABLE) { tlen += taosEncodeFixedU64(buf, TABLE_SUID(pTable)); From 32f7a2728b2a5feb375ce758a23dd84a66168433 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 18 Jun 2020 05:56:59 +0000 Subject: [PATCH 42/65] TD-353 --- src/tsdb/src/tsdbBuffer.c | 4 ++-- src/tsdb/src/tsdbMeta.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/tsdb/src/tsdbBuffer.c b/src/tsdb/src/tsdbBuffer.c index 95136454a0..0da0df9aa0 100644 --- a/src/tsdb/src/tsdbBuffer.c +++ b/src/tsdb/src/tsdbBuffer.c @@ -67,13 +67,13 @@ int tsdbOpenBufPool(STsdbRepo *pRepo) { ASSERT(pPool != NULL); - pPool->bufBlockSize = pCfg->cacheBlockSize; + pPool->bufBlockSize = pCfg->cacheBlockSize * 1024 * 1024; pPool->tBufBlocks = pCfg->totalBlocks; pPool->nBufBlocks = 0; pPool->index = 0; for (int i = 0; i < pCfg->totalBlocks; i++) { - STsdbBufBlock *pBufBlock = tsdbNewBufBlock(pCfg->cacheBlockSize); + STsdbBufBlock *pBufBlock = tsdbNewBufBlock(pPool->bufBlockSize); if (pBufBlock == NULL) goto _err; if (tdListAppend(pPool->bufBlockList, (void *)(&pBufBlock)) < 0) { diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index a0a3aebfd4..1bcc444628 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -98,7 +98,7 @@ int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg) { ASSERT(POINTER_DISTANCE(pBuf, buf) == tlen1); buf = pBuf; } - tsdbInsertTableAct(pRepo, TSDB_UPDATE_META, buf, super); + tsdbInsertTableAct(pRepo, TSDB_UPDATE_META, buf, table); return 0; @@ -722,7 +722,7 @@ static int tsdbAddTableToMeta(STsdbRepo *pRepo, STable *pTable, bool addIdx) { goto _err; } } else { - if (TABLE_TID(pTable) == TSDB_CHILD_TABLE && addIdx) { // add STABLE to the index + if (TABLE_TYPE(pTable) == TSDB_CHILD_TABLE && addIdx) { // add STABLE to the index if (tsdbAddTableIntoIndex(pMeta, pTable) < 0) { tsdbTrace("vgId:%d failed to add table %s to meta while add table to index since %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), tstrerror(terrno)); From 05bc6db54049d44b806e61048b8079e3fed296ac Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 18 Jun 2020 05:58:24 +0000 Subject: [PATCH 43/65] TD-353 --- src/tsdb/src/tsdbMain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 36d8901bdf..a01de1f4fe 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -485,7 +485,7 @@ static int32_t tsdbSetRepoEnv(char *rootDir, STsdbCfg *pCfg) { if (mkdir(dirName, 0755) < 0) { tsdbError("vgId:%d failed to create directory %s since %s", pCfg->tsdbId, dirName, strerror(errno)); - errno = TAOS_SYSTEM_ERROR(errno); + terrno = TAOS_SYSTEM_ERROR(errno); free(dirName); return -1; } From 65552dc480278ddc9a9fcf5d1390ffc1aa7fe570 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 18 Jun 2020 06:27:08 +0000 Subject: [PATCH 44/65] TD-353 --- src/tsdb/src/tsdbMain.c | 74 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 65 insertions(+), 9 deletions(-) diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index a01de1f4fe..6a6a78cd4e 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -65,6 +65,8 @@ static void tsdbAlterKeep(STsdbRepo *pRepo, int32_t keep); static void tsdbAlterMaxTables(STsdbRepo *pRepo, int32_t maxTables); static int tsdbAlterCacheTotalBlocks(STsdbRepo *pRepo, int totalBlocks); static int keyFGroupCompFunc(const void *key, const void *fgroup); +static int tsdbEncodeCfg(void **buf, STsdbCfg *pCfg); +static void * tsdbDecodeCfg(void *buf, STsdbCfg *pCfg); // Function declaration int32_t tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg) { @@ -141,8 +143,10 @@ void tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit) { STsdbRepo *pRepo = (STsdbRepo *)repo; - tsdbAsyncCommit(pRepo); - if (pRepo->commit) pthread_join(pRepo->commitThread, NULL); + if (toCommit) { + tsdbAsyncCommit(pRepo); + if (pRepo->commit) pthread_join(pRepo->commitThread, NULL); + } tsdbCloseFileH(pRepo); tsdbCloseBufPool(pRepo); @@ -179,7 +183,7 @@ uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_ // STsdbMeta *pMeta = pRepo->tsdbMeta; STsdbFileH *pFileH = pRepo->tsdbFileH; uint32_t magic = 0; - char *fname = NULL; + char * fname = NULL; struct stat fState; @@ -513,6 +517,8 @@ static int32_t tsdbUnsetRepoEnv(char *rootDir) { static int32_t tsdbSaveConfig(char *rootDir, STsdbCfg *pCfg) { int fd = -1; char *fname = NULL; + char buf[TSDB_FILE_HEAD_SIZE] = "\0"; + char *pBuf = buf; fname = tsdbGetCfgFname(rootDir); if (fname == NULL) { @@ -527,8 +533,13 @@ static int32_t tsdbSaveConfig(char *rootDir, STsdbCfg *pCfg) { goto _err; } - if (twrite(fd, (void *)pCfg, sizeof(STsdbCfg)) < sizeof(STsdbCfg)) { - tsdbError("vgId:%d failed to write %d bytes to file %s since %s", pCfg->tsdbId, sizeof(STsdbCfg), fname, + int tlen = tsdbEncodeCfg((void *)(&pBuf), pCfg); + ASSERT(tlen + sizeof(TSCKSUM) <= TSDB_FILE_HEAD_SIZE); + + taosCalcChecksumAppend(0, (uint8_t *)buf, TSDB_FILE_HEAD_SIZE); + + if (twrite(fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { + tsdbError("vgId:%d failed to write %d bytes to file %s since %s", pCfg->tsdbId, TSDB_FILE_HEAD_SIZE, fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -553,6 +564,7 @@ _err: static int tsdbLoadConfig(char *rootDir, STsdbCfg *pCfg) { char *fname = NULL; int fd = -1; + char buf[TSDB_FILE_HEAD_SIZE] = "\0"; fname = tsdbGetCfgFname(rootDir); if (fname == NULL) { @@ -567,12 +579,20 @@ static int tsdbLoadConfig(char *rootDir, STsdbCfg *pCfg) { goto _err; } - if (tread(fd, (void *)pCfg, sizeof(*pCfg)) < sizeof(*pCfg)) { - tsdbError("failed to read %d bytes from file %s since %s", sizeof(*pCfg), fname, strerror(errno)); + if (tread(fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) { + tsdbError("failed to read %d bytes from file %s since %s", TSDB_FILE_HEAD_SIZE, fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); goto _err; } + if (!taosCheckChecksumWhole((uint8_t *)buf, TSDB_FILE_HEAD_SIZE)) { + tsdbError("file %s is corrupted", fname); + terrno = TSDB_CODE_TDB_FILE_CORRUPTED; + goto _err; + } + + tsdbDecodeCfg(buf, pCfg); + tfree(fname); close(fd); @@ -596,7 +616,6 @@ static char *tsdbGetCfgFname(char *rootDir) { return fname; } - static STsdbRepo *tsdbNewRepo(char *rootDir, STsdbAppH *pAppH, STsdbCfg *pCfg) { STsdbRepo *pRepo = (STsdbRepo *)calloc(1, sizeof(STsdbRepo)); if (pRepo == NULL) { @@ -888,6 +907,42 @@ static int keyFGroupCompFunc(const void *key, const void *fgroup) { } } +static int tsdbEncodeCfg(void **buf, STsdbCfg *pCfg) { + int tlen = 0; + + tlen += taosEncodeVariantI32(buf, pCfg->tsdbId); + tlen += taosEncodeFixedI32(buf, pCfg->cacheBlockSize); + tlen += taosEncodeVariantI32(buf, pCfg->totalBlocks); + tlen += taosEncodeVariantI32(buf, pCfg->maxTables); + tlen += taosEncodeVariantI32(buf, pCfg->daysPerFile); + tlen += taosEncodeVariantI32(buf, pCfg->keep); + tlen += taosEncodeVariantI32(buf, pCfg->keep1); + tlen += taosEncodeVariantI32(buf, pCfg->keep2); + tlen += taosEncodeVariantI32(buf, pCfg->minRowsPerFileBlock); + tlen += taosEncodeVariantI32(buf, pCfg->maxRowsPerFileBlock); + tlen += taosEncodeFixedI8(buf, pCfg->precision); + tlen += taosEncodeFixedI8(buf, pCfg->compression); + + return tlen; +} + +static void *tsdbDecodeCfg(void *buf, STsdbCfg *pCfg) { + buf = taosDecodeVariantI32(buf, &(pCfg->tsdbId)); + buf = taosDecodeFixedI32(buf, &(pCfg->cacheBlockSize)); + buf = taosDecodeVariantI32(buf, &(pCfg->totalBlocks)); + buf = taosDecodeVariantI32(buf, &(pCfg->maxTables)); + buf = taosDecodeVariantI32(buf, &(pCfg->daysPerFile)); + buf = taosDecodeVariantI32(buf, &(pCfg->keep)); + buf = taosDecodeVariantI32(buf, &(pCfg->keep1)); + buf = taosDecodeVariantI32(buf, &(pCfg->keep2)); + buf = taosDecodeVariantI32(buf, &(pCfg->minRowsPerFileBlock)); + buf = taosDecodeVariantI32(buf, &(pCfg->maxRowsPerFileBlock)); + buf = taosDecodeFixedI8(buf, &(pCfg->precision)); + buf = taosDecodeFixedI8(buf, &(pCfg->compression)); + + return buf; +} + static int tsdbAlterCacheTotalBlocks(STsdbRepo *pRepo, int totalBlocks) { // TODO // STsdbCache *pCache = pRepo->tsdbCache; @@ -915,7 +970,8 @@ static int tsdbAlterCacheTotalBlocks(STsdbRepo *pRepo, int totalBlocks) { // pRepo->config.totalBlocks = totalBlocks; // tsdbUnLockRepo((TsdbRepoT *)pRepo); - // tsdbTrace("vgId:%d, tsdb total cache blocks changed from %d to %d", pRepo->config.tsdbId, oldNumOfBlocks, totalBlocks); + // tsdbTrace("vgId:%d, tsdb total cache blocks changed from %d to %d", pRepo->config.tsdbId, oldNumOfBlocks, + // totalBlocks); return 0; } From a60fc0f7ce8cece41e266ae5d18b2c793c10df31 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 18 Jun 2020 08:00:03 +0000 Subject: [PATCH 45/65] TD-353 --- src/tsdb/inc/tsdbMain.h | 2 +- src/tsdb/src/tsdbBuffer.c | 3 +- src/tsdb/src/tsdbMain.c | 103 +++++++++++++++++++++++--------------- src/tsdb/src/tsdbMeta.c | 38 ++++++++------ src/util/src/hash.c | 4 +- src/util/src/tkvstore.c | 12 ++--- src/util/src/tlist.c | 6 ++- 7 files changed, 97 insertions(+), 71 deletions(-) diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 5b17e53904..0a55b9050c 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -298,7 +298,7 @@ STSchema* tsdbGetTableSchema(STable* pTable); STable* tsdbGetTableByUid(STsdbMeta* pMeta, uint64_t uid); STSchema* tsdbGetTableSchemaByVersion(STable* pTable, int16_t version); STSchema* tsdbGetTableTagSchema(STable* pTable); -int tsdbUpdateTable(STsdbMeta* pMeta, STable* pTable, STableCfg* pCfg); +int tsdbUpdateTable(STsdbRepo* pRepo, STable* pTable, STableCfg* pCfg); int tsdbWLockRepoMeta(STsdbRepo* pRepo); int tsdbRLockRepoMeta(STsdbRepo* pRepo); int tsdbUnlockRepoMeta(STsdbRepo* pRepo); diff --git a/src/tsdb/src/tsdbBuffer.c b/src/tsdb/src/tsdbBuffer.c index 0da0df9aa0..686fc4bbec 100644 --- a/src/tsdb/src/tsdbBuffer.c +++ b/src/tsdb/src/tsdbBuffer.c @@ -67,7 +67,7 @@ int tsdbOpenBufPool(STsdbRepo *pRepo) { ASSERT(pPool != NULL); - pPool->bufBlockSize = pCfg->cacheBlockSize * 1024 * 1024; + pPool->bufBlockSize = pCfg->cacheBlockSize * 1024 * 1024; // MB pPool->tBufBlocks = pCfg->totalBlocks; pPool->nBufBlocks = 0; pPool->index = 0; @@ -106,6 +106,7 @@ void tsdbCloseBufPool(STsdbRepo *pRepo) { while ((pNode = tdListPopHead(pBufPool->bufBlockList)) != NULL) { tdListNodeGetData(pBufPool->bufBlockList, pNode, (void *)(&pBufBlock)); tsdbFreeBufBlock(pBufBlock); + free(pNode); } } diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 6a6a78cd4e..faf69d87d3 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -67,6 +67,7 @@ static int tsdbAlterCacheTotalBlocks(STsdbRepo *pRepo, int totalBlocks); static int keyFGroupCompFunc(const void *key, const void *fgroup); static int tsdbEncodeCfg(void **buf, STsdbCfg *pCfg); static void * tsdbDecodeCfg(void *buf, STsdbCfg *pCfg); +static int tsdbCheckTableSchema(STsdbRepo *pRepo, SSubmitBlk *pBlock, STable *pTable); // Function declaration int32_t tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg) { @@ -81,7 +82,7 @@ int32_t tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg) { if (tsdbSetRepoEnv(rootDir, pCfg) < 0) return -1; tsdbTrace( - "vgId%d tsdb env create succeed! cacheBlockSize %d totalBlocks %d maxTables %d daysPerFile %d keep " + "vgId:%d tsdb env create succeed! cacheBlockSize %d totalBlocks %d maxTables %d daysPerFile %d keep " "%d minRowsPerFileBlock %d maxRowsPerFileBlock %d precision %d compression %d", pCfg->tsdbId, pCfg->cacheBlockSize, pCfg->totalBlocks, pCfg->maxTables, pCfg->daysPerFile, pCfg->keep, pCfg->minRowsPerFileBlock, pCfg->maxRowsPerFileBlock, pCfg->precision, pCfg->compression); @@ -142,6 +143,7 @@ void tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit) { if (repo == NULL) return; STsdbRepo *pRepo = (STsdbRepo *)repo; + int vgId = REPO_ID(pRepo); if (toCommit) { tsdbAsyncCommit(pRepo); @@ -151,7 +153,8 @@ void tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit) { tsdbCloseFileH(pRepo); tsdbCloseBufPool(pRepo); tsdbCloseMeta(pRepo); - tsdbTrace("vgId:%d repository is closed", REPO_ID(pRepo)); + tsdbFreeRepo(pRepo); + tsdbTrace("vgId:%d repository is closed", vgId); } int32_t tsdbInsertData(TSDB_REPO_T *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg *pRsp) { @@ -170,7 +173,6 @@ int32_t tsdbInsertData(TSDB_REPO_T *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg * while ((pBlock = tsdbGetSubmitMsgNext(&msgIter)) != NULL) { if (tsdbInsertDataToTable(pRepo, pBlock, now, &affectedrows) < 0) { - pRsp->affectedRows = htonl(affectedrows); return -1; } } @@ -534,7 +536,7 @@ static int32_t tsdbSaveConfig(char *rootDir, STsdbCfg *pCfg) { } int tlen = tsdbEncodeCfg((void *)(&pBuf), pCfg); - ASSERT(tlen + sizeof(TSCKSUM) <= TSDB_FILE_HEAD_SIZE); + ASSERT((tlen + sizeof(TSCKSUM) <= TSDB_FILE_HEAD_SIZE) && (POINTER_DISTANCE(pBuf, buf) == tlen)); taosCalcChecksumAppend(0, (uint8_t *)buf, TSDB_FILE_HEAD_SIZE); @@ -718,42 +720,11 @@ static int32_t tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, TSKEY return -1; } - // Check schema version - int32_t tversion = pBlock->sversion; - STSchema *pSchema = tsdbGetTableSchema(pTable); - ASSERT(pSchema != NULL); - int16_t nversion = schemaVersion(pSchema); - if (tversion > nversion) { - tsdbTrace("vgId:%d table %s tid %d server schema version %d is older than clien version %d, try to config.", - REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), nversion, tversion); - void *msg = (*pRepo->appH.configFunc)(REPO_ID(pRepo), TABLE_TID(pTable)); - if (msg == NULL) return -1; - - // TODO: Deal with error her - STableCfg *pTableCfg = tsdbCreateTableCfgFromMsg(msg); - STable * pTableUpdate = NULL; - if (pTable->type == TSDB_CHILD_TABLE) { - pTableUpdate = tsdbGetTableByUid(pMeta, pTableCfg->superUid); - } else { - pTableUpdate = pTable; - } - - int32_t code = tsdbUpdateTable(pMeta, pTableUpdate, pTableCfg); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - tsdbClearTableCfg(pTableCfg); - rpcFreeCont(msg); - - pSchema = tsdbGetTableSchemaByVersion(pTable, tversion); - } else if (tversion < nversion) { - pSchema = tsdbGetTableSchemaByVersion(pTable, tversion); - if (pSchema == NULL) { - tsdbError("vgId:%d table %s tid %d invalid schema version %d from client", REPO_ID(pRepo), - TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), tversion); - terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION; - return -1; - } + // Check schema version and update schema if needed + if (tsdbCheckTableSchema(pRepo, pBlock, pTable) < 0) { + tsdbError("vgId:%d failed to insert data to table %s since %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), + tstrerror(terrno)); + return -1; } SSubmitBlkIter blkIter = {0}; @@ -777,6 +748,8 @@ static int32_t tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, TSKEY (*affectedrows)++; points++; } + + STSchema *pSchema = tsdbGetTableSchemaByVersion(pTable, pBlock->sversion); pRepo->stat.pointsWritten += points * schemaNCols(pSchema); pRepo->stat.totalStorage += points * schemaVLen(pSchema); @@ -820,6 +793,7 @@ static SDataRow tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter) { } static int tsdbRestoreInfo(STsdbRepo *pRepo) { + // TODO STsdbMeta * pMeta = pRepo->tsdbMeta; STsdbFileH *pFileH = pRepo->tsdbFileH; SFileGroup *pFGroup = NULL; @@ -943,6 +917,55 @@ static void *tsdbDecodeCfg(void *buf, STsdbCfg *pCfg) { return buf; } +static int tsdbCheckTableSchema(STsdbRepo *pRepo, SSubmitBlk *pBlock, STable *pTable) { + ASSERT(pTable != NULL); + + STSchema *pSchema = tsdbGetTableSchema(pTable); + int sversion = schemaVersion(pSchema); + + if (pBlock->sversion == sversion) return 0; + if (pBlock->sversion > sversion) { // need to config + tsdbTrace("vgId:%d table %s tid %d has version %d smaller than client version %d, try to config", REPO_ID(pRepo), + TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), sversion, pBlock->sversion); + if (pRepo->appH.configFunc) { + void *msg = (*pRepo->appH.configFunc)(REPO_ID(pRepo), TABLE_TID(pTable)); + if (msg == NULL) { + tsdbError("vgId:%d failed to config table %s tid %d since %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), + TABLE_TID(pTable), tstrerror(terrno)); + return -1; + } + + STableCfg *pTableCfg = tsdbCreateTableCfgFromMsg(msg); + if (pTableCfg == NULL) { + rpcFreeCont(msg); + return -1; + } + + if (tsdbUpdateTable(pRepo, (TABLE_TYPE(pTable) == TSDB_CHILD_TABLE) ? pTable->pSuper : pTable, pTableCfg) < 0) { + tsdbError("vgId:%d failed to update table %s since %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), + tstrerror(terrno)); + tsdbClearTableCfg(pTableCfg); + rpcFreeCont(msg); + return -1; + } + tsdbClearTableCfg(pTableCfg); + rpcFreeCont(msg); + } else { + terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION; + return -1; + } + } else { + if (tsdbGetTableSchemaByVersion(pTable, pBlock->sversion) == NULL) { + tsdbError("vgId:%d invalid submit schema version %d to table %s tid %d from client", REPO_ID(pRepo), + pBlock->sversion, TABLE_CHAR_NAME(pTable), TABLE_TID(pTable)); + } + terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION; + return -1; + } + + return 0; + } + static int tsdbAlterCacheTotalBlocks(STsdbRepo *pRepo, int totalBlocks) { // TODO // STsdbCache *pCache = pRepo->tsdbCache; diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 1bcc444628..b1e17fb259 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -74,7 +74,7 @@ int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg) { // TODO if (super->type != TSDB_SUPER_TABLE) return -1; if (super->tableId.uid != pCfg->superUid) return -1; - tsdbUpdateTable(pMeta, super, pCfg); + tsdbUpdateTable(pRepo, super, pCfg); } } @@ -279,7 +279,7 @@ int tsdbUpdateTagValue(TSDB_REPO_T *repo, SUpdateTableTagValMsg *pMsg) { STable * super = tsdbGetTableByUid(pMeta, pTableCfg->superUid); ASSERT(super != NULL); - int32_t code = tsdbUpdateTable(pMeta, super, pTableCfg); + int32_t code = tsdbUpdateTable(pRepo, super, pTableCfg); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -447,16 +447,21 @@ STSchema *tsdbGetTableTagSchema(STable *pTable) { } } -int tsdbUpdateTable(STsdbMeta *pMeta, STable *pTable, STableCfg *pCfg) { - ASSERT(pTable->type != TSDB_CHILD_TABLE); - bool isChanged = false; +int tsdbUpdateTable(STsdbRepo *pRepo, STable *pTable, STableCfg *pCfg) { + // TODO: this function can only be called when there is no query and commit on this table + ASSERT(TABLE_TYPE(pTable) != TSDB_CHILD_TABLE); + bool changed = false; + STsdbMeta *pMeta = pRepo->tsdbMeta; if (pTable->type == TSDB_SUPER_TABLE) { if (schemaVersion(pTable->tagSchema) < schemaVersion(pCfg->tagSchema)) { - int32_t code = tsdbUpdateTableTagSchema(pTable, pCfg->tagSchema); - if (code != TSDB_CODE_SUCCESS) return code; + if (tsdbUpdateTableTagSchema(pTable, pCfg->tagSchema) < 0) { + tsdbError("vgId:%d failed to update table %s tag schema since %s", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), + tstrerror(terrno)); + return -1; + } } - isChanged = true; + changed = true; } STSchema *pTSchema = tsdbGetTableSchema(pTable); @@ -471,18 +476,19 @@ int tsdbUpdateTable(STsdbMeta *pMeta, STable *pTable, STableCfg *pCfg) { pTable->schema[pTable->numOfSchemas - 1] = tSchema; } - isChanged = true; + pMeta->maxRowBytes = MAX(pMeta->maxRowBytes, dataRowMaxBytesFromSchema(pCfg->schema)); + pMeta->maxCols = MAX(pMeta->maxCols, schemaNCols(pCfg->schema)); + + changed = true; } - if (isChanged) { - // TODO - // char *buf = malloc(1024 * 1024); - // tsdbEncodeTable(buf, pTable); - // // tsdbInsertMetaRecord(pMeta->mfh, pTable->tableId.uid, buf, bufLen); - // free(buf); + if (changed) { + int tlen = tsdbGetTableEncodeSize(TSDB_UPDATE_META, pTable); + void *buf = tsdbAllocBytes(pRepo, tlen); + tsdbInsertTableAct(pRepo, TSDB_UPDATE_META, buf, pTable); } - return TSDB_CODE_SUCCESS; + return 0; } int tsdbWLockRepoMeta(STsdbRepo *pRepo) { diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 95d0ce7081..1de013c416 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -309,9 +309,7 @@ void taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen) { } void taosHashCleanup(SHashObj *pHashObj) { - if (pHashObj == NULL || pHashObj->capacity <= 0) { - return; - } + if (pHashObj == NULL) return; SHashNode *pNode, *pNext; diff --git a/src/util/src/tkvstore.c b/src/util/src/tkvstore.c index a84a5ac011..6eca141375 100644 --- a/src/util/src/tkvstore.c +++ b/src/util/src/tkvstore.c @@ -58,25 +58,21 @@ int tdCreateKVStore(char *fname) { if (fd < 0) { uError("failed to open file %s since %s", fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - if (tdInitKVStoreHeader(fd, fname) < 0) { - close(fd); return -1; } + if (tdInitKVStoreHeader(fd, fname) < 0) goto _err; + if (fsync(fd) < 0) { uError("failed to fsync file %s since %s", fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); - close(fd); - return -1; + goto _err; } if (close(fd) < 0) { uError("failed to close file %s since %s", fname, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); - return -1; + goto _err; } return 0; diff --git a/src/util/src/tlist.c b/src/util/src/tlist.c index 0c1b8bd59a..9e90815f61 100644 --- a/src/util/src/tlist.c +++ b/src/util/src/tlist.c @@ -39,8 +39,10 @@ void tdListEmpty(SList *list) { } void tdListFree(SList *list) { - tdListEmpty(list); - free(list); + if (list) { + tdListEmpty(list); + free(list); + } } void tdListPrependNode(SList *list, SListNode *node) { From 76b7e3c815ae3b9984aa08ec1c4485c0ec713991 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 18 Jun 2020 08:32:13 +0000 Subject: [PATCH 46/65] TD-353 --- src/tsdb/inc/tsdbMain.h | 1 + src/tsdb/src/tsdbFile.c | 41 ++++++++++---------- src/tsdb/src/tsdbMain.c | 84 +++++++++++++++++++++++++++++------------ 3 files changed, 81 insertions(+), 45 deletions(-) diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 0a55b9050c..2238aa7fc8 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -346,6 +346,7 @@ int tsdbUpdateFileHeader(SFile* pFile, uint32_t version); int tsdbEncodeSFileInfo(void** buf, const STsdbFileInfo* pInfo); void* tsdbDecodeSFileInfo(void* buf, STsdbFileInfo* pInfo); int tsdbCpySFile(SFile* src, SFile* dst); +void tsdbRemoveFileGroup(STsdbRepo* pRepo, SFileGroup* pFGroup); // ------------------ tsdbRWHelper.c #define TSDB_HELPER_CLEAR_STATE 0x0 // Clear state diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index e268aa1d65..4c5681de93 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -35,7 +35,6 @@ static int tsdbInitFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type); static void tsdbDestroyFile(SFile *pFile); static int compFGroup(const void *arg1, const void *arg2); static int keyFGroupCompFunc(const void *key, const void *fgroup); -static void tsdbRemoveFileGroup(STsdbRepo *pRepo, SFileGroup *pFGroup); // ---------------- INTERNAL FUNCTIONS ---------------- STsdbFileH *tsdbNewFileH(STsdbCfg *pCfg) { @@ -356,6 +355,26 @@ int tsdbCpySFile(SFile *src, SFile *dst) { return 0; } +void tsdbRemoveFileGroup(STsdbRepo *pRepo, SFileGroup *pFGroup) { + ASSERT(pFGroup != NULL); + STsdbFileH *pFileH = pRepo->tsdbFileH; + + SFileGroup fileGroup = *pFGroup; + + int nFilesLeft = pFileH->nFGroups - (POINTER_DISTANCE(pFGroup, pFileH->pFGroup) / sizeof(SFileGroup) + 1); + if (nFilesLeft > 0) { + memmove((void *)pFGroup, POINTER_SHIFT(pFGroup, sizeof(SFileGroup)), sizeof(SFileGroup) * nFilesLeft); + } + + pFileH->nFGroups--; + ASSERT(pFileH->nFGroups >= 0); + + for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { + remove(fileGroup.files[type].fname); + tsdbDestroyFile(&fileGroup.files[type]); + } +} + // ---------------- LOCAL FUNCTIONS ---------------- static int tsdbInitFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type) { uint32_t version; @@ -418,23 +437,3 @@ static int keyFGroupCompFunc(const void *key, const void *fgroup) { return fid > pFGroup->fileId ? 1 : -1; } } - -static void tsdbRemoveFileGroup(STsdbRepo *pRepo, SFileGroup *pFGroup) { - ASSERT(pFGroup != NULL); - STsdbFileH *pFileH = pRepo->tsdbFileH; - - SFileGroup fileGroup = *pFGroup; - - int nFilesLeft = pFileH->nFGroups - (POINTER_DISTANCE(pFGroup, pFileH->pFGroup) / sizeof(SFileGroup) + 1); - if (nFilesLeft > 0) { - memmove((void *)pFGroup, POINTER_SHIFT(pFGroup, sizeof(SFileGroup)), sizeof(SFileGroup) * nFilesLeft); - } - - pFileH->nFGroups--; - ASSERT(pFileH->nFGroups >= 0); - - for (int type = TSDB_FILE_TYPE_HEAD; type < TSDB_FILE_TYPE_MAX; type++) { - remove(fileGroup.files[type].fname); - tsdbDestroyFile(&fileGroup.files[type]); - } -} \ No newline at end of file diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index faf69d87d3..75f9be6771 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -61,7 +61,7 @@ static SDataRow tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter); static int tsdbRestoreInfo(STsdbRepo *pRepo); static int tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter); static void tsdbAlterCompression(STsdbRepo *pRepo, int8_t compression); -static void tsdbAlterKeep(STsdbRepo *pRepo, int32_t keep); +static int tsdbAlterKeep(STsdbRepo *pRepo, int32_t keep); static void tsdbAlterMaxTables(STsdbRepo *pRepo, int32_t maxTables); static int tsdbAlterCacheTotalBlocks(STsdbRepo *pRepo, int totalBlocks); static int keyFGroupCompFunc(const void *key, const void *fgroup); @@ -270,10 +270,11 @@ STsdbCfg *tsdbGetCfg(const TSDB_REPO_T *repo) { } int32_t tsdbConfigRepo(TSDB_REPO_T *repo, STsdbCfg *pCfg) { + // TODO: think about multithread cases STsdbRepo *pRepo = (STsdbRepo *)repo; STsdbCfg * pRCfg = &pRepo->config; - if (tsdbCheckAndSetDefaultCfg(pCfg) < 0) return TSDB_CODE_TDB_INVALID_CONFIG; + if (tsdbCheckAndSetDefaultCfg(pCfg) < 0) return -1; ASSERT(pRCfg->tsdbId == pCfg->tsdbId); ASSERT(pRCfg->cacheBlockSize == pCfg->cacheBlockSize); @@ -284,25 +285,33 @@ int32_t tsdbConfigRepo(TSDB_REPO_T *repo, STsdbCfg *pCfg) { bool configChanged = false; if (pRCfg->compression != pCfg->compression) { - configChanged = true; tsdbAlterCompression(pRepo, pCfg->compression); + configChanged = true; } if (pRCfg->keep != pCfg->keep) { + if (tsdbAlterKeep(pRepo, pCfg->keep) < 0) { + tsdbError("vgId:%d failed to configure repo when alter keep since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } configChanged = true; - tsdbAlterKeep(pRepo, pCfg->keep); } if (pRCfg->totalBlocks != pCfg->totalBlocks) { - configChanged = true; tsdbAlterCacheTotalBlocks(pRepo, pCfg->totalBlocks); + configChanged = true; } if (pRCfg->maxTables != pCfg->maxTables) { - configChanged = true; tsdbAlterMaxTables(pRepo, pCfg->maxTables); + configChanged = true; } - if (configChanged) tsdbSaveConfig(pRepo->rootDir, &pRepo->config); + if (configChanged) { + if (tsdbSaveConfig(pRepo->rootDir, &pRepo->config) < 0) { + tsdbError("vgId:%d failed to configure repository while save config since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } + } - return TSDB_CODE_SUCCESS; + return 0; } void tsdbReportStat(void *repo, int64_t *totalPoints, int64_t *totalStorage, int64_t *compStorage) { @@ -351,6 +360,7 @@ int tsdbLockRepo(STsdbRepo *pRepo) { } int tsdbUnlockRepo(STsdbRepo *pRepo) { + ASSERT(IS_REPO_LOCKED(pRepo)); pRepo->repoLocked = false; int code = pthread_mutex_unlock(&pRepo->mutex); if (code != 0) { @@ -832,31 +842,57 @@ static int tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter) { } static void tsdbAlterCompression(STsdbRepo *pRepo, int8_t compression) { - int8_t oldCompRession = pRepo->config.compression; + int8_t ocompression = pRepo->config.compression; pRepo->config.compression = compression; - tsdbTrace("vgId:%d tsdb compression is changed from %d to %d", oldCompRession, compression); + tsdbTrace("vgId:%d tsdb compression is changed from %d to %d", ocompression, compression); } -static void tsdbAlterKeep(STsdbRepo *pRepo, int32_t keep) { - STsdbCfg *pCfg = &pRepo->config; - int oldKeep = pCfg->keep; +static int tsdbAlterKeep(STsdbRepo *pRepo, int32_t keep) { + STsdbCfg * pCfg = &pRepo->config; + STsdbFileH *pFileH = pRepo->tsdbFileH; + int okeep = pCfg->keep; + SFileGroup *pFGroup = NULL; - int maxFiles = keep / pCfg->maxTables + 3; - if (pRepo->config.keep > keep) { - pRepo->config.keep = keep; - pRepo->tsdbFileH->maxFGroups = maxFiles; - } else { - pRepo->config.keep = keep; - pRepo->tsdbFileH->pFGroup = realloc(pRepo->tsdbFileH->pFGroup, sizeof(SFileGroup)); - if (pRepo->tsdbFileH->pFGroup == NULL) { - // TODO: deal with the error + ASSERT(pCfg->keep != keep); + int maxFiles = TSDB_MAX_FILE(keep, pCfg->daysPerFile); + + if (maxFiles != pFileH->maxFGroups) { + pthread_rwlock_wrlock(&(pFileH->fhlock)); + + pCfg->keep = keep; + pFGroup = (SFileGroup *)calloc(maxFiles, sizeof(SFileGroup)); + if (pFGroup == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + pthread_rwlock_unlock(&(pFileH->fhlock)); + return -1; } - pRepo->tsdbFileH->maxFGroups = maxFiles; + + int mfid = TSDB_KEY_FILEID(taosGetTimestamp(pCfg->precision), pCfg->daysPerFile, pCfg->precision) - + TSDB_MAX_FILE(keep, pCfg->daysPerFile); + + int i = 0; + for (; i < pFileH->nFGroups; i++) { + if (pFileH->pFGroup[i].fileId >= mfid) break; + tsdbRemoveFileGroup(pRepo, &(pFileH->pFGroup[i])); + } + + for (int j = 0; i < pFileH->nFGroups; i++, j++) { + pFGroup[j] = pFileH->pFGroup[i]; + } + + free(pFileH->pFGroup); + pFileH->pFGroup = pFGroup; + + pthread_rwlock_unlock(&(pFileH->fhlock)); } - tsdbTrace("vgId:%d, keep is changed from %d to %d", pRepo->config.tsdbId, oldKeep, keep); + + tsdbTrace("vgId:%d keep is changed from %d to %d", REPO_ID(pRepo), okeep, keep); + + return 0; } static void tsdbAlterMaxTables(STsdbRepo *pRepo, int32_t maxTables) { + // TODO int oldMaxTables = pRepo->config.maxTables; if (oldMaxTables < pRepo->config.maxTables) { // TODO From 40b48c89806637bb50b8bee236aa1c5dee50b69d Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 18 Jun 2020 09:06:42 +0000 Subject: [PATCH 47/65] TD-353 --- src/tsdb/src/tsdbMain.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 75f9be6771..6c94a791fc 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -148,6 +148,8 @@ void tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit) { if (toCommit) { tsdbAsyncCommit(pRepo); if (pRepo->commit) pthread_join(pRepo->commitThread, NULL); + } else { + // TODO } tsdbCloseFileH(pRepo); From f890114a4187d7e024d4a054c9a726b65b271522 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 18 Jun 2020 09:24:09 +0000 Subject: [PATCH 48/65] TD-353 --- src/tsdb/src/tsdbMeta.c | 36 +++++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index b1e17fb259..b4dfe48190 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -110,21 +110,38 @@ _err: int tsdbDropTable(TSDB_REPO_T *repo, STableId tableId) { STsdbRepo *pRepo = (STsdbRepo *)repo; - if (pRepo == NULL) return -1; - STsdbMeta *pMeta = pRepo->tsdbMeta; - if (pMeta == NULL) return -1; + uint64_t uid = tableId.uid; - STable *pTable = tsdbGetTableByUid(pMeta, tableId.uid); + STable *pTable = tsdbGetTableByUid(pMeta, uid); if (pTable == NULL) { - tsdbError("vgId:%d, failed to drop table since table not exists! tid:%d, uid:" PRId64, pRepo->config.tsdbId, - tableId.tid, tableId.uid); + tsdbError("vgId:%d failed to drop table since table not exists! tid:%d uid %" PRId64, REPO_ID(pRepo), tableId.tid, + uid); + terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; return -1; } + if (TABLE_TYPE(pTable) == TSDB_STREAM_TABLE) { + if (pTable->cqhandle) pRepo->appH.cqDropFunc(pTable->cqhandle); + } + + if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) { + SSkipListIterator *pIter = tSkipListCreateIter(pTable->pIndex); + while (tSkipListIterNext(pIter)) { + STable *tTable = *(STable **)SL_GET_NODE_DATA(tSkipListIterGet(pIter)); + ASSERT(TABLE_TYPE(tTable) == TSDB_CHILD_TABLE); + int tlen = tsdbGetTableEncodeSize(TSDB_DROP_META, tTable); + void *buf = tsdbAllocBytes(pRepo, tlen); + ASSERT(buf != NULL); + tsdbInsertTableAct(pRepo, TSDB_DROP_META, buf, tTable); + tsdbRemoveTableFromMeta(pRepo, tTable, true); + } + } + + tsdbRemoveTableFromMeta(pRepo, pTable, true); + tsdbTrace("vgId:%d, table %s is dropped! tid:%d, uid:%" PRId64, pRepo->config.tsdbId, varDataVal(pTable->name), tableId.tid, tableId.uid); - tsdbRemoveTableFromMeta(pRepo, pTable, true); return 0; } @@ -528,6 +545,9 @@ void tsdbRefTable(STable *pTable) { T_REF_INC(pTable); } void tsdbUnRefTable(STable *pTable) { if (T_REF_DEC(pTable) == 0) { + if (TABLE_TYPE(pTable) == TSDB_CHILD_TABLE) { + tsdbUnRefTable(pTable->pSuper); + } tsdbFreeTable(pTable); } } @@ -814,6 +834,7 @@ static void tsdbRemoveTableFromMeta(STsdbRepo *pRepo, STable *pTable, bool rmFro } if (rmFromIdx) tsdbUnlockRepoMeta(pRepo); + tsdbUnRefTable(pTable); } static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable) { @@ -840,6 +861,7 @@ static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable) { memcpy(SL_GET_NODE_DATA(pNode), &pTable, sizeof(STable *)); tSkipListPut(pSTable->pIndex, pNode); + T_REF_INC(pSTable); return 0; } From 2f9e141512c104f266298277c957a9840d750dd3 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 18 Jun 2020 10:02:23 +0000 Subject: [PATCH 49/65] TD-353 --- src/tsdb/inc/tsdbMain.h | 5 ++--- src/tsdb/src/tsdbFile.c | 23 +++-------------------- src/tsdb/src/tsdbMain.c | 13 ++----------- src/tsdb/src/tsdbMemTable.c | 5 ++++- src/tsdb/src/tsdbRWHelper.c | 16 +++++----------- 5 files changed, 16 insertions(+), 46 deletions(-) diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 2238aa7fc8..0af8f787fa 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -142,7 +142,7 @@ typedef struct { } STsdbFileInfo; typedef struct { - char* fname; + char fname[TSDB_FILENAME_LEN]; int fd; STsdbFileInfo info; @@ -345,7 +345,6 @@ void tsdbFitRetention(STsdbRepo* pRepo); int tsdbUpdateFileHeader(SFile* pFile, uint32_t version); int tsdbEncodeSFileInfo(void** buf, const STsdbFileInfo* pInfo); void* tsdbDecodeSFileInfo(void* buf, STsdbFileInfo* pInfo); -int tsdbCpySFile(SFile* src, SFile* dst); void tsdbRemoveFileGroup(STsdbRepo* pRepo, SFileGroup* pFGroup); // ------------------ tsdbRWHelper.c @@ -389,7 +388,7 @@ int tsdbLoadBlockData(SRWHelper* pHelper, SCompBlock* pCompBlock, SDataCols* t #define TSDB_SUBMIT_MSG_HEAD_SIZE sizeof(SSubmitMsg) char* tsdbGetMetaFileName(char* rootDir); -char* tsdbGetDataFileName(STsdbRepo* pRepo, int fid, int type); +void tsdbGetDataFileName(STsdbRepo* pRepo, int fid, int type, char* fname); int tsdbLockRepo(STsdbRepo* pRepo); int tsdbUnlockRepo(STsdbRepo* pRepo); char* tsdbGetDataDirName(char* rootDir); diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 4c5681de93..63b219ebcb 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -245,8 +245,7 @@ int tsdbCreateFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type) { memset((void *)pFile, 0, sizeof(SFile)); pFile->fd = -1; - pFile->fname = tsdbGetDataFileName(pRepo, fid, type); - if (pFile->fname == NULL) return -1; + tsdbGetDataFileName(pRepo, fid, type, pFile->fname); if (access(pFile->fname, F_OK) == 0) { tsdbError("vgId:%d file %s already exists", REPO_ID(pRepo), fid); @@ -343,18 +342,6 @@ void *tsdbDecodeSFileInfo(void *buf, STsdbFileInfo *pInfo) { return buf; } -int tsdbCpySFile(SFile *src, SFile *dst) { - *dst = *src; - dst->fname = strdup(dst->fname); - - if (dst->fname == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return -1; - } - - return 0; -} - void tsdbRemoveFileGroup(STsdbRepo *pRepo, SFileGroup *pFGroup) { ASSERT(pFGroup != NULL); STsdbFileH *pFileH = pRepo->tsdbFileH; @@ -380,8 +367,7 @@ static int tsdbInitFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type) { uint32_t version; char buf[512] = "\0"; - pFile->fname = tsdbGetDataFileName(pRepo, fid, type); - if (pFile->fname == NULL) return -1; + tsdbGetDataFileName(pRepo, fid, type, pFile->fname); pFile->fd = -1; if (tsdbOpenFile(pFile, O_RDONLY) < 0) goto _err; @@ -410,10 +396,7 @@ _err: return -1; } -static void tsdbDestroyFile(SFile *pFile) { - tsdbCloseFile(pFile); - tfree(pFile->fname); -} +static void tsdbDestroyFile(SFile *pFile) { tsdbCloseFile(pFile); } static int compFGroup(const void *arg1, const void *arg2) { int val1 = ((SFileGroup *)arg1)->fileId; diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 6c94a791fc..91cb21b93a 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -337,17 +337,8 @@ char *tsdbGetMetaFileName(char *rootDir) { return fname; } -char *tsdbGetDataFileName(STsdbRepo *pRepo, int fid, int type) { - int tlen = strlen(pRepo->rootDir) + strlen(tsdbFileSuffix[type]) + 24; - - char *fname = malloc(tlen); - if (fname == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - return NULL; - } - - sprintf(fname, "%s/%s/v%df%d.%s", pRepo->rootDir, TSDB_DATA_DIR_NAME, REPO_ID(pRepo), fid, tsdbFileSuffix[type]); - return fname; +void tsdbGetDataFileName(STsdbRepo *pRepo, int fid, int type, char *fname) { + snprintf(fname, TSDB_FILENAME_LEN, "%s/%s/v%df%d.%s", pRepo->rootDir, TSDB_DATA_DIR_NAME, REPO_ID(pRepo), fid, tsdbFileSuffix[type]); } int tsdbLockRepo(STsdbRepo *pRepo) { diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index ce57111125..72a1b307de 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -499,6 +499,7 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe char * dataDir = NULL; STsdbMeta * pMeta = pRepo->tsdbMeta; STsdbCfg * pCfg = &pRepo->config; + STsdbFileH *pFileH = pRepo->tsdbFileH; SFileGroup *pGroup = NULL; TSKEY minKey = 0, maxKey = 0; @@ -588,10 +589,12 @@ static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHe } tsdbCloseHelperFile(pHelper, 0); - // TODO: make it atomic with some methods + + pthread_rwlock_wrlock(&(pFileH->fhlock)); pGroup->files[TSDB_FILE_TYPE_HEAD] = pHelper->files.headF; pGroup->files[TSDB_FILE_TYPE_DATA] = pHelper->files.dataF; pGroup->files[TSDB_FILE_TYPE_LAST] = pHelper->files.lastF; + pthread_rwlock_unlock(&(pFileH->fhlock)); return 0; diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 08fe69b0ee..aee82ddc50 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -100,13 +100,12 @@ int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) { // Set the files pHelper->files.fid = pGroup->fileId; - tsdbCpySFile(&pHelper->files.headF, &pGroup->files[TSDB_FILE_TYPE_HEAD]); - tsdbCpySFile(&pHelper->files.dataF, &pGroup->files[TSDB_FILE_TYPE_DATA]); - tsdbCpySFile(&pHelper->files.lastF, &pGroup->files[TSDB_FILE_TYPE_LAST]); + pHelper->files.headF = pGroup->files[TSDB_FILE_TYPE_HEAD]; + pHelper->files.dataF = pGroup->files[TSDB_FILE_TYPE_DATA]; + pHelper->files.lastF = pGroup->files[TSDB_FILE_TYPE_LAST]; if (helperType(pHelper) == TSDB_WRITE_HELPER) { - - pHelper->files.nHeadF.fname = tsdbGetDataFileName(pHelper->pRepo, pGroup->fileId, TSDB_FILE_TYPE_NHEAD); - pHelper->files.nLastF.fname = tsdbGetDataFileName(pHelper->pRepo, pGroup->fileId, TSDB_FILE_TYPE_NLAST); + tsdbGetDataFileName(pHelper->pRepo, pGroup->fileId, TSDB_FILE_TYPE_NHEAD, pHelper->files.nHeadF.fname); + tsdbGetDataFileName(pHelper->pRepo, pGroup->fileId, TSDB_FILE_TYPE_NLAST, pHelper->files.nLastF.fname); } // Open the files @@ -1036,15 +1035,10 @@ static int tsdbGetRowsInRange(SDataCols *pDataCols, TSKEY minKey, TSKEY maxKey) static void tsdbResetHelperFileImpl(SRWHelper *pHelper) { memset((void *)&pHelper->files, 0, sizeof(pHelper->files)); pHelper->files.fid = -1; - tfree(pHelper->files.headF.fname); pHelper->files.headF.fd = -1; - tfree(pHelper->files.dataF.fname); pHelper->files.dataF.fd = -1; - tfree(pHelper->files.lastF.fname); pHelper->files.lastF.fd = -1; - tfree(pHelper->files.nHeadF.fname); pHelper->files.nHeadF.fd = -1; - tfree(pHelper->files.nLastF.fname); pHelper->files.nLastF.fd = -1; } From fe34e6a274a10016c52de132b50ee05d2dacc1a6 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 18 Jun 2020 10:53:54 +0000 Subject: [PATCH 50/65] TD-353 --- src/tsdb/src/tsdbFile.c | 4 ++-- src/tsdb/src/tsdbMemTable.c | 2 +- src/util/src/tkvstore.c | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index 63b219ebcb..b22d5fa834 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -300,8 +300,8 @@ int tsdbUpdateFileHeader(SFile *pFile, uint32_t version) { char buf[TSDB_FILE_HEAD_SIZE] = "\0"; void *pBuf = (void *)buf; - taosEncodeFixedU32(pBuf, version); - tsdbEncodeSFileInfo(pBuf, &(pFile->info)); + taosEncodeFixedU32((void *)(&pBuf), version); + tsdbEncodeSFileInfo((void *)(&pBuf), &(pFile->info)); taosCalcChecksumAppend(0, (uint8_t *)buf, TSDB_FILE_HEAD_SIZE); diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index 72a1b307de..f2e3407b66 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -161,7 +161,7 @@ int tsdbTakeMemSnapshot(STsdbRepo *pRepo, SMemTable **pMem, SMemTable **pIMem) { if (tsdbLockRepo(pRepo) < 0) return -1; *pMem = pRepo->mem; - *pIMem = pRepo->mem; + *pIMem = pRepo->imem; tsdbRefMemTable(pRepo, *pMem); tsdbRefMemTable(pRepo, *pIMem); diff --git a/src/util/src/tkvstore.c b/src/util/src/tkvstore.c index 1c7d93b307..1ffc86bf5e 100644 --- a/src/util/src/tkvstore.c +++ b/src/util/src/tkvstore.c @@ -241,9 +241,9 @@ int tdUpdateKVStoreRecord(SKVStore *pStore, uint64_t uid, void *cont, int contLe pStore->info.size += (sizeof(SKVRecord) + contLen); SKVRecord *pRecord = taosHashGet(pStore->map, (void *)&uid, sizeof(uid)); if (pRecord != NULL) { // just to insert - pStore->info.nRecords++; - } else { pStore->info.tombSize += pRecord->size; + } else { + pStore->info.nRecords++; } taosHashPut(pStore->map, (void *)(&uid), sizeof(uid), (void *)(&rInfo), sizeof(rInfo)); From eb8f74e9528907d4603dc7ff54569d1ec26a863c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 18 Jun 2020 15:49:24 +0000 Subject: [PATCH 51/65] TD-353 --- src/tsdb/src/tsdbFile.c | 3 ++- src/tsdb/src/tsdbMain.c | 1 - src/util/src/tkvstore.c | 16 ++++++++++++++-- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index b22d5fa834..feec663229 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -80,6 +80,7 @@ int tsdbOpenFileH(STsdbRepo *pRepo) { char *tDataDir = NULL; DIR * dir = NULL; int fid = 0; + int vid = 0; SFileGroup fileGroup = {0}; STsdbFileH *pFileH = pRepo->tsdbFileH; @@ -100,7 +101,7 @@ int tsdbOpenFileH(STsdbRepo *pRepo) { struct dirent *dp = NULL; while ((dp = readdir(dir)) != NULL) { if (strncmp(dp->d_name, ".", 1) == 0 || strncmp(dp->d_name, "..", 2) == 0) continue; - sscanf(dp->d_name, "f%d", &fid); + sscanf(dp->d_name, "v%df%d", &vid, &fid); if (tsdbSearchFGroup(pRepo->tsdbFileH, fid, TD_EQ) != NULL) return 0; diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 91cb21b93a..8cd5b53a56 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -134,7 +134,6 @@ TSDB_REPO_T *tsdbOpenRepo(char *rootDir, STsdbAppH *pAppH) { _err: tsdbCloseRepo(pRepo, false); - tsdbFreeRepo(pRepo); return NULL; } diff --git a/src/util/src/tkvstore.c b/src/util/src/tkvstore.c index 1ffc86bf5e..8fa2c387d4 100644 --- a/src/util/src/tkvstore.c +++ b/src/util/src/tkvstore.c @@ -223,6 +223,8 @@ _err: int tdUpdateKVStoreRecord(SKVStore *pStore, uint64_t uid, void *cont, int contLen) { SKVRecord rInfo = {0}; + char buf[64] = "\0"; + char * pBuf = buf; rInfo.offset = lseek(pStore->fd, 0, SEEK_CUR); if (rInfo.offset < 0) { @@ -233,6 +235,16 @@ int tdUpdateKVStoreRecord(SKVStore *pStore, uint64_t uid, void *cont, int contLe rInfo.uid = uid; rInfo.size = contLen; + int tlen = tdEncodeKVRecord((void *)(&pBuf), &rInfo); + ASSERT(tlen == POINTER_DISTANCE(pBuf, buf)); + ASSERT(tlen == sizeof(SKVRecord)); + + if (twrite(pStore->fd, buf, tlen) < tlen) { + uError("failed to write %d bytes to file %s since %s", tlen, pStore->fname, strerror(errno)); + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + if (twrite(pStore->fd, cont, contLen) < contLen) { uError("failed to write %d bytes to file %s since %s", contLen, pStore->fname, strerror(errno)); return -1; @@ -534,7 +546,7 @@ static int tdRestoreKVStore(SKVStore *pStore) { while (taosHashIterNext(pIter)) { SKVRecord *pRecord = taosHashIterGet(pIter); - if (lseek(pStore->fd, pRecord->offset, SEEK_SET) < 0) { + if (lseek(pStore->fd, pRecord->offset + sizeof(SKVRecord), SEEK_SET) < 0) { uError("failed to lseek file %s since %s, offset %" PRId64, pStore->fname, strerror(errno), pRecord->offset); terrno = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -551,7 +563,7 @@ static int tdRestoreKVStore(SKVStore *pStore) { if ((*pStore->iFunc)(pStore->appH, buf, pRecord->size) < 0) { uError("failed to restore record uid %" PRIu64 " in kv store %s at offset %" PRId64 " size %" PRId64 " since %s", - pStore->fname, pRecord->uid, pRecord->offset, pRecord->size, tstrerror(terrno)); + pRecord->uid, pStore->fname, pRecord->offset, pRecord->size, tstrerror(terrno)); goto _err; } } From 496c5da86cbb51d89db6f2d1d718fdc9ba588256 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 18 Jun 2020 16:10:50 +0000 Subject: [PATCH 52/65] TD-353 --- src/tsdb/src/tsdbMeta.c | 1 + src/util/src/tlist.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index b4dfe48190..f22a2c093f 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -415,6 +415,7 @@ int tsdbCloseMeta(STsdbRepo *pRepo) { while ((pNode = tdListPopHead(pMeta->superList)) != NULL) { tdListNodeGetData(pMeta->superList, pNode, (void *)(&pTable)); tsdbFreeTable(pTable); + listNodeFree(pNode); } tsdbTrace("vgId:%d TSDB meta is closed", REPO_ID(pRepo)); diff --git a/src/util/src/tlist.c b/src/util/src/tlist.c index 9e90815f61..f402c2307e 100644 --- a/src/util/src/tlist.c +++ b/src/util/src/tlist.c @@ -83,7 +83,7 @@ int tdListPrepend(SList *list, void *data) { } int tdListAppend(SList *list, void *data) { - SListNode *node = (SListNode *)malloc(sizeof(SListNode) + list->eleSize); + SListNode *node = (SListNode *)calloc(1, sizeof(SListNode) + list->eleSize); if (node == NULL) return -1; memcpy((void *)(node->data), data, list->eleSize); From 7f9df4c361e9a5606c631a356c51a430aaf198be Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 19 Jun 2020 01:41:14 +0000 Subject: [PATCH 53/65] TD-353 --- src/tsdb/src/tsdbMeta.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index f22a2c093f..8aed53895b 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -112,6 +112,8 @@ int tsdbDropTable(TSDB_REPO_T *repo, STableId tableId) { STsdbRepo *pRepo = (STsdbRepo *)repo; STsdbMeta *pMeta = pRepo->tsdbMeta; uint64_t uid = tableId.uid; + int tid = 0; + char * tbname = NULL; STable *pTable = tsdbGetTableByUid(pMeta, uid); if (pTable == NULL) { @@ -121,6 +123,13 @@ int tsdbDropTable(TSDB_REPO_T *repo, STableId tableId) { return -1; } + tid = TABLE_TID(pTable); + tbname = strdup(TABLE_CHAR_NAME(pTable)); + if (tbname == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + return -1; + } + if (TABLE_TYPE(pTable) == TSDB_STREAM_TABLE) { if (pTable->cqhandle) pRepo->appH.cqDropFunc(pTable->cqhandle); } @@ -140,8 +149,8 @@ int tsdbDropTable(TSDB_REPO_T *repo, STableId tableId) { tsdbRemoveTableFromMeta(pRepo, pTable, true); - tsdbTrace("vgId:%d, table %s is dropped! tid:%d, uid:%" PRId64, pRepo->config.tsdbId, varDataVal(pTable->name), - tableId.tid, tableId.uid); + tsdbTrace("vgId:%d, table %s is dropped! tid:%d, uid:%" PRId64, pRepo->config.tsdbId, tbname, tid, uid); + free(tbname); return 0; } From b51aceefdec2734063a98e6a345d2f13942dc896 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 19 Jun 2020 02:03:32 +0000 Subject: [PATCH 54/65] TD-353 --- src/tsdb/src/tsdbFile.c | 2 +- src/tsdb/src/tsdbMain.c | 8 ++++---- src/tsdb/src/tsdbMeta.c | 6 +++--- src/tsdb/src/tsdbRWHelper.c | 10 ++++++---- src/util/src/tkvstore.c | 4 ++-- 5 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index feec663229..ef4f8234f1 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -249,7 +249,7 @@ int tsdbCreateFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type) { tsdbGetDataFileName(pRepo, fid, type, pFile->fname); if (access(pFile->fname, F_OK) == 0) { - tsdbError("vgId:%d file %s already exists", REPO_ID(pRepo), fid); + tsdbError("vgId:%d file %s already exists", REPO_ID(pRepo), pFile->fname); terrno = TSDB_CODE_TDB_FILE_ALREADY_EXISTS; goto _err; } diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 8cd5b53a56..b37b565b42 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -710,7 +710,7 @@ static int32_t tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, TSKEY STable *pTable = tsdbGetTableByUid(pMeta, pBlock->uid); if (pTable == NULL || TABLE_TID(pTable) != pBlock->tid) { - tsdbError("vgId:%d failed to get table to insert data, uid " PRIu64 " tid %d", REPO_ID(pRepo), pBlock->uid, + tsdbError("vgId:%d failed to get table to insert data, uid %" PRIu64 " tid %d", REPO_ID(pRepo), pBlock->uid, pBlock->tid); terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; return -1; @@ -738,8 +738,8 @@ static int32_t tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, TSKEY tsdbInitSubmitBlkIter(pBlock, &blkIter); while ((row = tsdbGetSubmitBlkNext(&blkIter)) != NULL) { if (dataRowKey(row) < minKey || dataRowKey(row) > maxKey) { - tsdbError("vgId:%d table %s tid %d uid %ld timestamp is out of range! now " PRId64 " maxKey " PRId64 - " minKey " PRId64, + tsdbError("vgId:%d table %s tid %d uid %" PRIu64 " timestamp is out of range! now %" PRId64 " minKey %" PRId64 + " maxKey %" PRId64, REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable), now, minKey, maxKey); terrno = TSDB_CODE_TDB_TIMESTAMP_OUT_OF_RANGE; return -1; @@ -836,7 +836,7 @@ static int tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter) { static void tsdbAlterCompression(STsdbRepo *pRepo, int8_t compression) { int8_t ocompression = pRepo->config.compression; pRepo->config.compression = compression; - tsdbTrace("vgId:%d tsdb compression is changed from %d to %d", ocompression, compression); + tsdbTrace("vgId:%d tsdb compression is changed from %d to %d", REPO_ID(pRepo), ocompression, compression); } static int tsdbAlterKeep(STsdbRepo *pRepo, int32_t keep) { diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 8aed53895b..86d299a945 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -317,9 +317,9 @@ int tsdbUpdateTagValue(TSDB_REPO_T *repo, SUpdateTableTagValMsg *pMsg) { if (schemaVersion(pTagSchema) > tversion) { tsdbError( - "vgId:%d failed to update tag value of table %s since version out of date, client tag version:%d server tag " - "version:%d", - pRepo->config.tsdbId, varDataVal(pTable->name), tversion, schemaVersion(pTable->tagSchema)); + "vgId:%d failed to update tag value of table %s since version out of date, client tag version %d server tag " + "version %d", + REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), tversion, schemaVersion(pTable->tagSchema)); return TSDB_CODE_TDB_TAG_VER_OUT_OF_DATE; } if (schemaColAt(pTagSchema, DEFAULT_TAG_INDEX_COLUMN)->colId == htons(pMsg->colId)) { diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index aee82ddc50..9b87b94289 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -659,7 +659,8 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa // Write the whole block to file if (twrite(pFile->fd, (void *)pCompData, lsize) < lsize) { - tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(helperRepo(pHelper)), lsize, strerror(errno)); + tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(helperRepo(pHelper)), lsize, pFile->fname, + strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); goto _err; } @@ -678,8 +679,9 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa tsdbTrace("vgId:%d tid:%d a block of data is written to file %s, offset %" PRId64 " numOfRows %d len %d numOfCols %" PRId16 " keyFirst %" PRId64 " keyLast %" PRId64, - REPO_ID(helperRepo(pHelper)), pHelper->tableInfo.tid, pFile->fname, pCompBlock->offset, - pCompBlock->numOfRows, pCompBlock->len, pCompBlock->numOfCols, pCompBlock->keyFirst, pCompBlock->keyLast); + REPO_ID(helperRepo(pHelper)), pHelper->tableInfo.tid, pFile->fname, (int64_t)(pCompBlock->offset), + (int)(pCompBlock->numOfRows), pCompBlock->len, pCompBlock->numOfCols, pCompBlock->keyFirst, + pCompBlock->keyLast); return 0; @@ -1239,7 +1241,7 @@ static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDa int32_t tsize = sizeof(SCompData) + sizeof(SCompCol) * pCompBlock->numOfCols + sizeof(TSCKSUM); if (!taosCheckChecksumWhole((uint8_t *)pCompData, tsize)) { tsdbError("vgId:%d file %s block data is corrupted offset %" PRId64 " len %d", REPO_ID(pHelper->pRepo), - pFile->fname, pCompBlock->offset, pCompBlock->len); + pFile->fname, (int64_t)(pCompBlock->offset), pCompBlock->len); terrno = TSDB_CODE_TDB_FILE_CORRUPTED; goto _err; } diff --git a/src/util/src/tkvstore.c b/src/util/src/tkvstore.c index 6df929ac38..1f71fba3cb 100644 --- a/src/util/src/tkvstore.c +++ b/src/util/src/tkvstore.c @@ -119,7 +119,7 @@ SKVStore *tdOpenKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void *appH if (terrno != TSDB_CODE_COM_FILE_CORRUPTED) goto _err; } else { if (ftruncate(pStore->fd, info.size) < 0) { - uError("failed to truncate %s to " PRId64 " size since %s", pStore->fname, info.size, strerror(errno)); + uError("failed to truncate %s to %" PRId64 " size since %s", pStore->fname, info.size, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); goto _err; } @@ -492,7 +492,7 @@ static int tdRestoreKVStore(SKVStore *pStore) { ssize_t tsize = tread(pStore->fd, tbuf, sizeof(SKVRecord)); if (tsize == 0) break; if (tsize < sizeof(SKVRecord)) { - uError("failed to read %d bytes from file %s at offset %" PRId64 "since %s", sizeof(SKVRecord), pStore->fname, + uError("failed to read %zu bytes from file %s at offset %" PRId64 "since %s", sizeof(SKVRecord), pStore->fname, pStore->info.size, strerror(errno)); terrno = TAOS_SYSTEM_ERROR(errno); goto _err; From f1ce02255615cc41e0cbd25b695342f078f341e3 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 19 Jun 2020 02:41:47 +0000 Subject: [PATCH 55/65] TD-353 --- src/tsdb/src/tsdbMain.c | 2 +- src/tsdb/src/tsdbRWHelper.c | 57 ++++++++++++++++++++----------------- src/util/src/tkvstore.c | 1 - 3 files changed, 32 insertions(+), 28 deletions(-) diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index b37b565b42..aa4df032e8 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -337,7 +337,7 @@ char *tsdbGetMetaFileName(char *rootDir) { } void tsdbGetDataFileName(STsdbRepo *pRepo, int fid, int type, char *fname) { - snprintf(fname, TSDB_FILENAME_LEN, "%s/%s/v%df%d.%s", pRepo->rootDir, TSDB_DATA_DIR_NAME, REPO_ID(pRepo), fid, tsdbFileSuffix[type]); + snprintf(fname, TSDB_FILENAME_LEN, "%s/%s/v%df%d%s", pRepo->rootDir, TSDB_DATA_DIR_NAME, REPO_ID(pRepo), fid, tsdbFileSuffix[type]); } int tsdbLockRepo(STsdbRepo *pRepo) { diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 9b87b94289..208eafe49d 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -148,44 +148,49 @@ _err: int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) { if (pHelper->files.headF.fd > 0) { - fsync(pHelper->files.headF.fd); close(pHelper->files.headF.fd); pHelper->files.headF.fd = -1; } if (pHelper->files.dataF.fd > 0) { - if (!hasError) tsdbUpdateFileHeader(&(pHelper->files.dataF), 0); - fsync(pHelper->files.dataF.fd); + if ((helperType(pHelper) == TSDB_WRITE_HELPER)) { + tsdbUpdateFileHeader(&(pHelper->files.dataF), 0); + fsync(pHelper->files.dataF.fd); + } close(pHelper->files.dataF.fd); pHelper->files.dataF.fd = -1; } if (pHelper->files.lastF.fd > 0) { - fsync(pHelper->files.lastF.fd); - close(pHelper->files.lastF.fd); + if ((helperType(pHelper) == TSDB_WRITE_HELPER)) { + fsync(pHelper->files.lastF.fd); + close(pHelper->files.lastF.fd); + } pHelper->files.lastF.fd = -1; } - if (pHelper->files.nHeadF.fd > 0) { - if (!hasError) tsdbUpdateFileHeader(&(pHelper->files.nHeadF), 0); - fsync(pHelper->files.nHeadF.fd); - close(pHelper->files.nHeadF.fd); - pHelper->files.nHeadF.fd = -1; - if (hasError) { - remove(pHelper->files.nHeadF.fname); - } else { - rename(pHelper->files.nHeadF.fname, pHelper->files.headF.fname); - pHelper->files.headF.info = pHelper->files.nHeadF.info; + if (helperType(pHelper) == TSDB_WRITE_HELPER) { + if (pHelper->files.nHeadF.fd > 0) { + if (!hasError) tsdbUpdateFileHeader(&(pHelper->files.nHeadF), 0); + fsync(pHelper->files.nHeadF.fd); + close(pHelper->files.nHeadF.fd); + pHelper->files.nHeadF.fd = -1; + if (hasError) { + remove(pHelper->files.nHeadF.fname); + } else { + rename(pHelper->files.nHeadF.fname, pHelper->files.headF.fname); + pHelper->files.headF.info = pHelper->files.nHeadF.info; + } } - } - if (pHelper->files.nLastF.fd > 0) { - if (!hasError) tsdbUpdateFileHeader(&(pHelper->files.nLastF), 0); - fsync(pHelper->files.nLastF.fd); - close(pHelper->files.nLastF.fd); - pHelper->files.nLastF.fd = -1; - if (hasError) { - remove(pHelper->files.nLastF.fname); - } else { - rename(pHelper->files.nLastF.fname, pHelper->files.lastF.fname); - pHelper->files.lastF.info = pHelper->files.nLastF.info; + if (pHelper->files.nLastF.fd > 0) { + if (!hasError) tsdbUpdateFileHeader(&(pHelper->files.nLastF), 0); + fsync(pHelper->files.nLastF.fd); + close(pHelper->files.nLastF.fd); + pHelper->files.nLastF.fd = -1; + if (hasError) { + remove(pHelper->files.nLastF.fname); + } else { + rename(pHelper->files.nLastF.fname, pHelper->files.lastF.fname); + pHelper->files.lastF.info = pHelper->files.nLastF.info; + } } } return 0; diff --git a/src/util/src/tkvstore.c b/src/util/src/tkvstore.c index 1f71fba3cb..ee34ca68fd 100644 --- a/src/util/src/tkvstore.c +++ b/src/util/src/tkvstore.c @@ -578,6 +578,5 @@ static int tdRestoreKVStore(SKVStore *pStore) { _err: taosHashDestroyIter(pIter); tfree(buf); - taosHashDestroyIter(pIter); return -1; } \ No newline at end of file From dc1a04e6dd0d79bfb89b958c3be9e8785e34a6c0 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 19 Jun 2020 03:05:14 +0000 Subject: [PATCH 56/65] TD-353 --- src/tsdb/src/tsdbMain.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index aa4df032e8..5b0a233e13 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -147,9 +147,11 @@ void tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit) { if (toCommit) { tsdbAsyncCommit(pRepo); if (pRepo->commit) pthread_join(pRepo->commitThread, NULL); - } else { - // TODO } + tsdbUnRefMemTable(pRepo, pRepo->mem); + tsdbUnRefMemTable(pRepo, pRepo->imem); + pRepo->mem = NULL; + pRepo->imem = NULL; tsdbCloseFileH(pRepo); tsdbCloseBufPool(pRepo); From 7ee009477393ad27f84906bab0827b516144599a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 19 Jun 2020 03:42:17 +0000 Subject: [PATCH 57/65] TD-353 --- src/tsdb/src/tsdbFile.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tsdb/src/tsdbFile.c b/src/tsdb/src/tsdbFile.c index ef4f8234f1..ffac8da944 100644 --- a/src/tsdb/src/tsdbFile.c +++ b/src/tsdb/src/tsdbFile.c @@ -103,7 +103,7 @@ int tsdbOpenFileH(STsdbRepo *pRepo) { if (strncmp(dp->d_name, ".", 1) == 0 || strncmp(dp->d_name, "..", 2) == 0) continue; sscanf(dp->d_name, "v%df%d", &vid, &fid); - if (tsdbSearchFGroup(pRepo->tsdbFileH, fid, TD_EQ) != NULL) return 0; + if (tsdbSearchFGroup(pRepo->tsdbFileH, fid, TD_EQ) != NULL) continue; memset((void *)(&fileGroup), 0, sizeof(SFileGroup)); fileGroup.fileId = fid; From ca9ba4276bf348e31e7bb44d9694ab408e9b55c3 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 19 Jun 2020 05:45:00 +0000 Subject: [PATCH 58/65] TD-353 --- src/util/src/tskiplist.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/src/tskiplist.c b/src/util/src/tskiplist.c index a65d9b0f31..3a474fe6c1 100644 --- a/src/util/src/tskiplist.c +++ b/src/util/src/tskiplist.c @@ -279,7 +279,7 @@ SSkipListNode *tSkipListPut(SSkipList *pSkipList, SSkipListNode *pNode) { pthread_rwlock_unlock(pSkipList->lock); } - return forward[0]; + return NULL; } tSkipListDoInsert(pSkipList, forward, pNode); From cbb70cc56370e9160fc790eafaf9c09117bedbb9 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 19 Jun 2020 06:32:29 +0000 Subject: [PATCH 59/65] TD-353 --- src/tsdb/src/tsdbMeta.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 86d299a945..398d891f78 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -894,7 +894,7 @@ static int tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable) { SSkipListNode *pNode = taosArrayGetP(res, i); // STableIndexElem* pElem = (STableIndexElem*) SL_GET_NODE_DATA(pNode); - if ((STable *)SL_GET_NODE_DATA(pNode) == pTable) { // this is the exact what we need + if (*(STable **)SL_GET_NODE_DATA(pNode) == pTable) { // this is the exact what we need tSkipListRemoveNode(pSTable->pIndex, pNode); } } From f948a51f6bdd5219c2dafd3dc19f166ad472e0b2 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 19 Jun 2020 08:55:19 +0000 Subject: [PATCH 60/65] TD-353 --- src/tsdb/src/tsdbMain.c | 25 +++++++++++++++++++++---- src/tsdb/src/tsdbRead.c | 4 ++-- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/src/tsdb/src/tsdbMain.c b/src/tsdb/src/tsdbMain.c index 5b0a233e13..80e5e8ea0d 100644 --- a/src/tsdb/src/tsdbMain.c +++ b/src/tsdb/src/tsdbMain.c @@ -62,7 +62,7 @@ static int tsdbRestoreInfo(STsdbRepo *pRepo); static int tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter); static void tsdbAlterCompression(STsdbRepo *pRepo, int8_t compression); static int tsdbAlterKeep(STsdbRepo *pRepo, int32_t keep); -static void tsdbAlterMaxTables(STsdbRepo *pRepo, int32_t maxTables); +static int tsdbAlterMaxTables(STsdbRepo *pRepo, int32_t maxTables); static int tsdbAlterCacheTotalBlocks(STsdbRepo *pRepo, int totalBlocks); static int keyFGroupCompFunc(const void *key, const void *fgroup); static int tsdbEncodeCfg(void **buf, STsdbCfg *pCfg); @@ -303,7 +303,10 @@ int32_t tsdbConfigRepo(TSDB_REPO_T *repo, STsdbCfg *pCfg) { configChanged = true; } if (pRCfg->maxTables != pCfg->maxTables) { - tsdbAlterMaxTables(pRepo, pCfg->maxTables); + if (tsdbAlterMaxTables(pRepo, pCfg->maxTables) < 0) { + tsdbError("vgId:%d failed to configure repo when alter maxTables since %s", REPO_ID(pRepo), tstrerror(terrno)); + return -1; + } configChanged = true; } @@ -885,11 +888,12 @@ static int tsdbAlterKeep(STsdbRepo *pRepo, int32_t keep) { return 0; } -static void tsdbAlterMaxTables(STsdbRepo *pRepo, int32_t maxTables) { +static int tsdbAlterMaxTables(STsdbRepo *pRepo, int32_t maxTables) { // TODO int oldMaxTables = pRepo->config.maxTables; if (oldMaxTables < pRepo->config.maxTables) { - // TODO + terrno = TSDB_CODE_TDB_INVALID_ACTION; + return -1; } STsdbMeta *pMeta = pRepo->tsdbMeta; @@ -898,7 +902,20 @@ static void tsdbAlterMaxTables(STsdbRepo *pRepo, int32_t maxTables) { memset(&pMeta->tables[oldMaxTables], 0, sizeof(STable *) * (maxTables - oldMaxTables)); pRepo->config.maxTables = maxTables; + if (pRepo->mem) { + pRepo->mem->tData = realloc(pRepo->mem->tData, maxTables * sizeof(STableData *)); + memset(POINTER_SHIFT(pRepo->mem->tData, sizeof(STableData *) * oldMaxTables), 0, + sizeof(STableData *) * (maxTables - oldMaxTables)); + } + + if (pRepo->imem) { + pRepo->imem->tData = realloc(pRepo->imem->tData, maxTables * sizeof(STableData *)); + memset(POINTER_SHIFT(pRepo->imem->tData, sizeof(STableData *) * oldMaxTables), 0, + sizeof(STableData *) * (maxTables - oldMaxTables)); + } + tsdbTrace("vgId:%d, tsdb maxTables is changed from %d to %d!", pRepo->config.tsdbId, oldMaxTables, maxTables); + return 0; } static int keyFGroupCompFunc(const void *key, const void *fgroup) { diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index a2dd23dcc1..d1981943e6 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -259,12 +259,12 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh assert(pCheckInfo->iter == NULL && pCheckInfo->iiter == NULL); - if (pCheckInfo->mem) { + if (pCheckInfo->mem && pCheckInfo->mem->tData[pCheckInfo->tableId.tid] != NULL) { pCheckInfo->iter = tSkipListCreateIterFromVal(pCheckInfo->mem->tData[pCheckInfo->tableId.tid]->pData, (const char*) &pCheckInfo->lastKey, TSDB_DATA_TYPE_TIMESTAMP, order); } - if (pCheckInfo->imem) { + if (pCheckInfo->imem && pCheckInfo->imem->tData[pCheckInfo->tableId.tid] != NULL) { pCheckInfo->iiter = tSkipListCreateIterFromVal(pCheckInfo->imem->tData[pCheckInfo->tableId.tid]->pData, (const char*) &pCheckInfo->lastKey, TSDB_DATA_TYPE_TIMESTAMP, order); } From fad50b6e4b5efd1a63a85113c788f784287c58ce Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 19 Jun 2020 10:32:44 +0000 Subject: [PATCH 61/65] fix compile error --- src/tsdb/src/tsdbRWHelper.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 208eafe49d..9c0e05d274 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -152,7 +152,7 @@ int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) { pHelper->files.headF.fd = -1; } if (pHelper->files.dataF.fd > 0) { - if ((helperType(pHelper) == TSDB_WRITE_HELPER)) { + if (helperType(pHelper) == TSDB_WRITE_HELPER) { tsdbUpdateFileHeader(&(pHelper->files.dataF), 0); fsync(pHelper->files.dataF.fd); } From 4dc80e772d5e2682d348bbae51f64f33041ddc90 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Fri, 19 Jun 2020 10:55:31 +0000 Subject: [PATCH 62/65] fix compile error --- src/tsdb/src/tsdbRWHelper.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tsdb/src/tsdbRWHelper.c b/src/tsdb/src/tsdbRWHelper.c index 9c0e05d274..c1bfa2939b 100644 --- a/src/tsdb/src/tsdbRWHelper.c +++ b/src/tsdb/src/tsdbRWHelper.c @@ -160,7 +160,7 @@ int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError) { pHelper->files.dataF.fd = -1; } if (pHelper->files.lastF.fd > 0) { - if ((helperType(pHelper) == TSDB_WRITE_HELPER)) { + if (helperType(pHelper) == TSDB_WRITE_HELPER) { fsync(pHelper->files.lastF.fd); close(pHelper->files.lastF.fd); } From 8bcefa8d50382170af1c592f7e400a1667aaec46 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 20 Jun 2020 02:05:46 +0000 Subject: [PATCH 63/65] TD-353 --- src/tsdb/src/tsdbMemTable.c | 52 +++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/src/tsdb/src/tsdbMemTable.c b/src/tsdb/src/tsdbMemTable.c index f2e3407b66..e928cdc385 100644 --- a/src/tsdb/src/tsdbMemTable.c +++ b/src/tsdb/src/tsdbMemTable.c @@ -371,33 +371,35 @@ static void *tsdbCommitData(void *arg) { pMem->keyFirst, pMem->keyLast, pMem->numOfRows); // Create the iterator to read from cache - iters = tsdbCreateTableIters(pRepo); - if (iters == NULL) { - tsdbError("vgId:%d failed to create commit iterator since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _exit; - } - - if (tsdbInitWriteHelper(&whelper, pRepo) < 0) { - tsdbError("vgId:%d failed to init write helper since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _exit; - } - - if ((pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pCfg->maxRowsPerFileBlock)) == NULL) { - terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; - tsdbError("vgId:%d failed to init data cols with maxRowBytes %d maxCols %d maxRowsPerFileBlock %d since %s", - REPO_ID(pRepo), pMeta->maxCols, pMeta->maxRowBytes, pCfg->maxRowsPerFileBlock, tstrerror(terrno)); - goto _exit; - } - - int sfid = TSDB_KEY_FILEID(pMem->keyFirst, pCfg->daysPerFile, pCfg->precision); - int efid = TSDB_KEY_FILEID(pMem->keyLast, pCfg->daysPerFile, pCfg->precision); - - // Loop to commit to each file - for (int fid = sfid; fid <= efid; fid++) { - if (tsdbCommitToFile(pRepo, fid, iters, &whelper, pDataCols) < 0) { - tsdbError("vgId:%d failed to commit to file %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno)); + if (pMem->numOfRows > 0) { + iters = tsdbCreateTableIters(pRepo); + if (iters == NULL) { + tsdbError("vgId:%d failed to create commit iterator since %s", REPO_ID(pRepo), tstrerror(terrno)); goto _exit; } + + if (tsdbInitWriteHelper(&whelper, pRepo) < 0) { + tsdbError("vgId:%d failed to init write helper since %s", REPO_ID(pRepo), tstrerror(terrno)); + goto _exit; + } + + if ((pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pCfg->maxRowsPerFileBlock)) == NULL) { + terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; + tsdbError("vgId:%d failed to init data cols with maxRowBytes %d maxCols %d maxRowsPerFileBlock %d since %s", + REPO_ID(pRepo), pMeta->maxCols, pMeta->maxRowBytes, pCfg->maxRowsPerFileBlock, tstrerror(terrno)); + goto _exit; + } + + int sfid = TSDB_KEY_FILEID(pMem->keyFirst, pCfg->daysPerFile, pCfg->precision); + int efid = TSDB_KEY_FILEID(pMem->keyLast, pCfg->daysPerFile, pCfg->precision); + + // Loop to commit to each file + for (int fid = sfid; fid <= efid; fid++) { + if (tsdbCommitToFile(pRepo, fid, iters, &whelper, pDataCols) < 0) { + tsdbError("vgId:%d failed to commit to file %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno)); + goto _exit; + } + } } // Commit to update meta file From 22482771c42b1dcd4232f1919a7a6f37abe0ef23 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 20 Jun 2020 04:36:35 +0000 Subject: [PATCH 64/65] TD-353 --- src/tsdb/inc/tsdbMain.h | 2 +- src/tsdb/src/tsdbMeta.c | 23 ++++++++++++++--------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/src/tsdb/inc/tsdbMain.h b/src/tsdb/inc/tsdbMain.h index 0af8f787fa..cd64e48347 100644 --- a/src/tsdb/inc/tsdbMain.h +++ b/src/tsdb/inc/tsdbMain.h @@ -53,7 +53,7 @@ typedef struct STable { STSchema* schema[TSDB_MAX_TABLE_SCHEMAS]; STSchema* tagSchema; SKVRow tagVal; - void* pIndex; // For TSDB_SUPER_TABLE, it is the skiplist index + SSkipList* pIndex; // For TSDB_SUPER_TABLE, it is the skiplist index void* eventHandler; // TODO void* streamHandler; // TODO TSKEY lastKey; // lastkey inserted in this table, initialized as 0, TODO: make a structure diff --git a/src/tsdb/src/tsdbMeta.c b/src/tsdb/src/tsdbMeta.c index 398d891f78..beb8f33052 100644 --- a/src/tsdb/src/tsdbMeta.c +++ b/src/tsdb/src/tsdbMeta.c @@ -31,7 +31,7 @@ static STable *tsdbNewTable(STableCfg *pCfg, bool isSuper); static void tsdbFreeTable(STable *pTable); static int tsdbUpdateTableTagSchema(STable *pTable, STSchema *newSchema); static int tsdbAddTableToMeta(STsdbRepo *pRepo, STable *pTable, bool addIdx); -static void tsdbRemoveTableFromMeta(STsdbRepo *pRepo, STable *pTable, bool rmFromIdx); +static void tsdbRemoveTableFromMeta(STsdbRepo *pRepo, STable *pTable, bool rmFromIdx, bool lock); static int tsdbAddTableIntoIndex(STsdbMeta *pMeta, STable *pTable); static int tsdbRemoveTableFromIndex(STsdbMeta *pMeta, STable *pTable); static int tsdbInitTableCfg(STableCfg *config, ETableType type, uint64_t uid, int32_t tid); @@ -123,6 +123,8 @@ int tsdbDropTable(TSDB_REPO_T *repo, STableId tableId) { return -1; } + tsdbTrace("vgId:%d try to drop table %s type %d", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable), TABLE_TYPE(pTable)); + tid = TABLE_TID(pTable); tbname = strdup(TABLE_CHAR_NAME(pTable)); if (tbname == NULL) { @@ -143,11 +145,11 @@ int tsdbDropTable(TSDB_REPO_T *repo, STableId tableId) { void *buf = tsdbAllocBytes(pRepo, tlen); ASSERT(buf != NULL); tsdbInsertTableAct(pRepo, TSDB_DROP_META, buf, tTable); - tsdbRemoveTableFromMeta(pRepo, tTable, true); + tsdbRemoveTableFromMeta(pRepo, tTable, false, true); } } - tsdbRemoveTableFromMeta(pRepo, pTable, true); + tsdbRemoveTableFromMeta(pRepo, pTable, true, true); tsdbTrace("vgId:%d, table %s is dropped! tid:%d, uid:%" PRId64, pRepo->config.tsdbId, tbname, tid, uid); free(tbname); @@ -708,6 +710,7 @@ _err: static void tsdbFreeTable(STable *pTable) { if (pTable) { + tsdbTrace("table %s is destroyed", TABLE_CHAR_NAME(pTable)); tfree(TABLE_NAME(pTable)); if (TABLE_TYPE(pTable) != TSDB_CHILD_TABLE) { for (int i = 0; i < TSDB_MAX_TABLE_SCHEMAS; i++) { @@ -723,6 +726,7 @@ static void tsdbFreeTable(STable *pTable) { tSkipListDestroy(pTable->pIndex); tfree(pTable->sql); + free(pTable); } } @@ -790,12 +794,12 @@ static int tsdbAddTableToMeta(STsdbRepo *pRepo, STable *pTable, bool addIdx) { return 0; _err: - tsdbRemoveTableFromMeta(pRepo, pTable, false); + tsdbRemoveTableFromMeta(pRepo, pTable, false, false); if (addIdx) tsdbUnlockRepoMeta(pRepo); return -1; } -static void tsdbRemoveTableFromMeta(STsdbRepo *pRepo, STable *pTable, bool rmFromIdx) { +static void tsdbRemoveTableFromMeta(STsdbRepo *pRepo, STable *pTable, bool rmFromIdx, bool lock) { STsdbMeta *pMeta = pRepo->tsdbMeta; SListIter lIter = {0}; SListNode *pNode = NULL; @@ -806,17 +810,17 @@ static void tsdbRemoveTableFromMeta(STsdbRepo *pRepo, STable *pTable, bool rmFro int maxCols = schemaNCols(pSchema); int maxRowBytes = schemaTLen(pSchema); - if (rmFromIdx) tsdbWLockRepoMeta(pRepo); + if (lock) tsdbWLockRepoMeta(pRepo); if (TABLE_TYPE(pTable) == TSDB_SUPER_TABLE) { tdListInitIter(pMeta->superList, &lIter, TD_LIST_BACKWARD); while ((pNode = tdListNext(&lIter)) != NULL) { - tdListNodeGetData(pMeta->superList, pNode, (void *)(tTable)); + tdListNodeGetData(pMeta->superList, pNode, (void *)(&tTable)); if (pTable == tTable) { - break; tdListPopNode(pMeta->superList, pNode); free(pNode); + break; } } } else { @@ -843,7 +847,8 @@ static void tsdbRemoveTableFromMeta(STsdbRepo *pRepo, STable *pTable, bool rmFro } } - if (rmFromIdx) tsdbUnlockRepoMeta(pRepo); + if (lock) tsdbUnlockRepoMeta(pRepo); + tsdbTrace("vgId:%d table %s is removed from meta", REPO_ID(pRepo), TABLE_CHAR_NAME(pTable)); tsdbUnRefTable(pTable); } From fa6b4f3126dd72355cd4006056e6f574b37460f5 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Sat, 20 Jun 2020 05:10:11 +0000 Subject: [PATCH 65/65] TD-353 --- src/vnode/src/vnodeWrite.c | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/src/vnode/src/vnodeWrite.c b/src/vnode/src/vnodeWrite.c index 68e98757cc..6d2b050163 100644 --- a/src/vnode/src/vnodeWrite.c +++ b/src/vnode/src/vnodeWrite.c @@ -110,10 +110,11 @@ static int32_t vnodeProcessSubmitMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pR } static int32_t vnodeProcessCreateTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pRet) { + int code = TSDB_CODE_SUCCESS; STableCfg *pCfg = tsdbCreateTableCfgFromMsg((SMDCreateTableMsg *)pCont); if (pCfg == NULL) return terrno; - int32_t code = tsdbCreateTable(pVnode->tsdb, pCfg); + if (tsdbCreateTable(pVnode->tsdb, pCfg) < 0) code = terrno; tsdbClearTableCfg(pCfg); return code; @@ -121,15 +122,12 @@ static int32_t vnodeProcessCreateTableMsg(SVnodeObj *pVnode, void *pCont, SRspRe static int32_t vnodeProcessDropTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pRet) { SMDDropTableMsg *pTable = pCont; - int32_t code = 0; + int32_t code = TSDB_CODE_SUCCESS; vTrace("vgId:%d, table:%s, start to drop", pVnode->vgId, pTable->tableId); - STableId tableId = { - .uid = htobe64(pTable->uid), - .tid = htonl(pTable->sid) - }; + STableId tableId = {.uid = htobe64(pTable->uid), .tid = htonl(pTable->sid)}; - code = tsdbDropTable(pVnode->tsdb, tableId); + if (tsdbDropTable(pVnode->tsdb, tableId) < 0) code = terrno; return code; } @@ -140,24 +138,24 @@ static int32_t vnodeProcessAlterTableMsg(SVnodeObj *pVnode, void *pCont, SRspRet static int32_t vnodeProcessDropStableMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pRet) { SMDDropSTableMsg *pTable = pCont; - int32_t code = 0; + int32_t code = TSDB_CODE_SUCCESS; vTrace("vgId:%d, stable:%s, start to drop", pVnode->vgId, pTable->tableId); - - STableId stableId = { - .uid = htobe64(pTable->uid), - .tid = -1 - }; - code = tsdbDropTable(pVnode->tsdb, stableId); - + STableId stableId = {.uid = htobe64(pTable->uid), .tid = -1}; + + if (tsdbDropTable(pVnode->tsdb, stableId) < 0) code = terrno; + vTrace("vgId:%d, stable:%s, drop stable result:%s", pVnode->vgId, pTable->tableId, tstrerror(code)); - + return code; } static int32_t vnodeProcessUpdateTagValMsg(SVnodeObj *pVnode, void *pCont, SRspRet *pRet) { - return tsdbUpdateTagValue(pVnode->tsdb, (SUpdateTableTagValMsg *)pCont); + if (tsdbUpdateTagValue(pVnode->tsdb, (SUpdateTableTagValMsg *)pCont) < 0) { + return terrno; + } + return TSDB_CODE_SUCCESS; } int vnodeWriteToQueue(void *param, void *data, int type) {