xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

enh(tsdb/cache/read): lazy init table map hash and entries

This commit is contained in:
Minglei Jin 2023-08-01 15:53:42 +08:00
parent 347e78ecb5
commit b3fae65d90
2 changed files with 34 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
source/dnode/vnode/src/tsdb/tsdbCache.c
View File

@ -1592,6 +1592,29 @@ _err:
return code; return code;
} }
static void freeTableInfoFunc(void *param) {
void **p = (void **)param;
taosMemoryFreeClear(*p);
}
static STableLoadInfo *getTableLoadInfo(SCacheRowsReader *pReader, uint64_t uid) {
STableLoadInfo *pInfo = NULL;
if (!pReader->pTableMap) {
pReader->pTableMap = tSimpleHashInit(pReader->numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT));
tSimpleHashSetFreeFp(pReader->pTableMap, freeTableInfoFunc);
}
pInfo = *(STableLoadInfo **)tSimpleHashGet(pReader->pTableMap, &uid, sizeof(uid));
if (!pInfo) {
pInfo = taosMemoryCalloc(1, sizeof(STableLoadInfo));
tSimpleHashPut(pReader->pTableMap, &uid, sizeof(uint64_t), &pInfo, POINTER_BYTES);
}
return pInfo;
}
static int32_t loadTombFromBlk(const TTombBlkArray *pTombBlkArray, SCacheRowsReader *pReader, void *pFileReader, static int32_t loadTombFromBlk(const TTombBlkArray *pTombBlkArray, SCacheRowsReader *pReader, void *pFileReader,
bool isFile) { bool isFile) {
int32_t code = 0; int32_t code = 0;
@ -1618,7 +1641,7 @@ static int32_t loadTombFromBlk(const TTombBlkArray *pTombBlkArray, SCacheRowsRea
} }
uint64_t uid = uidList[j]; uint64_t uid = uidList[j];
STableLoadInfo *pInfo = *(STableLoadInfo **)tSimpleHashGet(pReader->pTableMap, &uid, sizeof(uid)); STableLoadInfo *pInfo = getTableLoadInfo(pReader, uid);
if (pInfo->pTombData == NULL) { if (pInfo->pTombData == NULL) {
pInfo->pTombData = taosArrayInit(4, sizeof(SDelData)); pInfo->pTombData = taosArrayInit(4, sizeof(SDelData));
} }
@ -1660,13 +1683,16 @@ static int32_t loadTombFromBlk(const TTombBlkArray *pTombBlkArray, SCacheRowsRea
} }
if (newTable) { if (newTable) {
pInfo = *(STableLoadInfo **)tSimpleHashGet(pReader->pTableMap, &uid, sizeof(uid)); pInfo = getTableLoadInfo(pReader, uid);
if (pInfo->pTombData == NULL) { if (pInfo->pTombData == NULL) {
pInfo->pTombData = taosArrayInit(4, sizeof(SDelData)); pInfo->pTombData = taosArrayInit(4, sizeof(SDelData));
} }
} }
if (record.version <= pReader->info.verRange.maxVer) { if (record.version <= pReader->info.verRange.maxVer) {
tsdbError("tomb xx load/cache: vgId:%d fid:%d commit %" PRId64 "~%" PRId64 "~%" PRId64 " tomb records",
TD_VID(pReader->pTsdb->pVnode), pReader->pCurFileSet->fid, record.skey, record.ekey, uid);
SDelData delData = {.version = record.version, .sKey = record.skey, .eKey = record.ekey}; SDelData delData = {.version = record.version, .sKey = record.skey, .eKey = record.ekey};
taosArrayPush(pInfo->pTombData, &delData); taosArrayPush(pInfo->pTombData, &delData);
} }
@ -2458,13 +2484,14 @@ static int32_t nextRowIterGet(CacheNextRowIter *pIter, TSDBROW **ppRow, bool *pI
pIter->pSkyline = taosArrayInit(32, sizeof(TSDBKEY)); pIter->pSkyline = taosArrayInit(32, sizeof(TSDBKEY));
uint64_t uid = pIter->idx.uid; uint64_t uid = pIter->idx.uid;
STableLoadInfo *pInfo = *(STableLoadInfo **)tSimpleHashGet(pIter->pr->pTableMap, &uid, sizeof(uid)); STableLoadInfo *pInfo = getTableLoadInfo(pIter->pr, uid);
SArray *pTombData = pInfo->pTombData; SArray *pTombData = pInfo->pTombData;
if (pTombData) {
taosArrayAddAll(pTombData, pIter->pMemDelData);
code = tsdbBuildDeleteSkyline(pTombData, 0, (int32_t)(TARRAY_SIZE(pTombData) - 1), pIter->pSkyline); if (pTombData) {
taosArrayAddAll(pIter->pMemDelData, pTombData);
} }
code = tsdbBuildDeleteSkyline(pIter->pMemDelData, 0, (int32_t)(TARRAY_SIZE(pIter->pMemDelData) - 1),
pIter->pSkyline);
pIter->iSkyline = taosArrayGetSize(pIter->pSkyline) - 1; pIter->iSkyline = taosArrayGetSize(pIter->pSkyline) - 1;
} }

15
source/dnode/vnode/src/tsdb/tsdbCacheRead.c
View File

@ -143,11 +143,6 @@ static int32_t uidComparFunc(const void* p1, const void* p2) {
} }
} }
static void freeTableInfoFunc(void* param) {
void** p = (void**)param;
taosMemoryFreeClear(*p);
}
int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList, int32_t numOfTables, int32_t numOfCols, int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList, int32_t numOfTables, int32_t numOfCols,
SArray* pCidList, int32_t* pSlotIds, uint64_t suid, void** pReader, const char* idstr) { SArray* pCidList, int32_t* pSlotIds, uint64_t suid, void** pReader, const char* idstr) {
*pReader = NULL; *pReader = NULL;
@ -173,25 +168,17 @@ int32_t tsdbCacherowsReaderOpen(void* pVnode, int32_t type, void* pTableIdList,
p->pTableList = pTableIdList; p->pTableList = pTableIdList;
p->numOfTables = numOfTables; p->numOfTables = numOfTables;
p->pTableMap = tSimpleHashInit(numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT));
if (p->pTableMap == NULL) {
tsdbCacherowsReaderClose(p);
return TSDB_CODE_OUT_OF_MEMORY;
}
p->uidList = taosMemoryMalloc(numOfTables * sizeof(uint64_t)); p->uidList = taosMemoryMalloc(numOfTables * sizeof(uint64_t));
if (p->uidList == NULL) { if (p->uidList == NULL) {
tsdbCacherowsReaderClose(p); tsdbCacherowsReaderClose(p);
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
for (int32_t i = 0; i < numOfTables; ++i) { for (int32_t i = 0; i < numOfTables; ++i) {
uint64_t uid = p->pTableList[i].uid; uint64_t uid = p->pTableList[i].uid;
p->uidList[i] = uid; p->uidList[i] = uid;
STableLoadInfo* pInfo = taosMemoryCalloc(1, sizeof(STableLoadInfo));
tSimpleHashPut(p->pTableMap, &uid, sizeof(uint64_t), &pInfo, POINTER_BYTES);
} }
tSimpleHashSetFreeFp(p->pTableMap, freeTableInfoFunc);
taosSort(p->uidList, numOfTables, sizeof(uint64_t), uidComparFunc); taosSort(p->uidList, numOfTables, sizeof(uint64_t), uidComparFunc);
int32_t code = setTableSchema(p, suid, idstr); int32_t code = setTableSchema(p, suid, idstr);