xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

opti:use suid+uid->tags index to optimize time cost

This commit is contained in:
wangmm0220 2022-08-12 11:33:53 +08:00
parent 32f9bc0dd0
commit e102f81f92
5 changed files with 103 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
source/dnode/vnode/inc/vnode.h
View File

@ -91,9 +91,9 @@ typedef struct SMetaEntry SMetaEntry;
void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags);
void metaReaderClear(SMetaReader *pReader);
int32_t metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid);
//int32_t metaGetTableEntryByUidTest(void *pReader, SArray *uidList);
int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList, SArray *tags);
int32_t metaReadNext(SMetaReader *pReader);
const void *metaGetTableTagVal(SMetaEntry *pEntry, int16_t type, STagVal *tagVal);
const void *metaGetTableTagVal(void *tag, int16_t type, STagVal *tagVal);
int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName);
typedef struct SMetaFltParam {

70
source/dnode/vnode/src/meta/metaQuery.c
View File

@ -59,6 +59,7 @@ int metaGetTableEntryByUidTest(void* meta, SArray *uidList) {
SArray* uidVersion = taosArrayInit(taosArrayGetSize(uidList), sizeof(STbDbKey));
SMeta *pMeta = meta;
int64_t version;
SHashObj *uHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
int64_t stt1 = taosGetTimestampUs();
for(int i = 0; i < taosArrayGetSize(uidList); i++) {
@ -73,10 +74,31 @@ int metaGetTableEntryByUidTest(void* meta, SArray *uidList) {
STbDbKey tbDbKey = {.version = version, .uid = *uid};
taosArrayPush(uidVersion, &tbDbKey);
taosHashPut(uHash, uid, sizeof(int64_t), ppVal, sizeof(int64_t));
}
int64_t stt2 = taosGetTimestampUs();
qDebug("metaGetTableEntryByUidTest1 rows:%d, cost:%ld us", taosArrayGetSize(uidList), stt2-stt1);
TBC *pCur = NULL;
tdbTbcOpen(pMeta->pTbDb, &pCur, NULL);
tdbTbcMoveToFirst(pCur);
void *pKey = NULL;
int kLen = 0;
while(1){
SMetaReader pReader = {0};
int32_t ret = tdbTbcNext(pCur, &pKey, &kLen, &pReader.pBuf, &pReader.szBuf);
if (ret < 0) break;
STbDbKey *tmp = (STbDbKey*)pKey;
int64_t *ver = (int64_t*)taosHashGet(uHash, &tmp->uid, sizeof(int64_t));
if(ver == NULL || *ver != tmp->version) continue;
taosArrayPush(readerList, &pReader);
}
tdbTbcClose(pCur);
taosArrayClear(readerList);
int64_t stt3 = taosGetTimestampUs();
qDebug("metaGetTableEntryByUidTest2 rows:%d, cost:%ld us", taosArrayGetSize(uidList), stt3-stt2);
for(int i = 0; i < taosArrayGetSize(uidVersion); i++) {
SMetaReader pReader = {0};
@ -87,8 +109,8 @@ int metaGetTableEntryByUidTest(void* meta, SArray *uidList) {
}
taosArrayPush(readerList, &pReader);
}
int64_t stt3 = taosGetTimestampUs();
qDebug("metaGetTableEntryByUidTest2 rows:%d, cost:%ld us", taosArrayGetSize(uidList), stt3-stt2);
int64_t stt4 = taosGetTimestampUs();
qDebug("metaGetTableEntryByUidTest3 rows:%d, cost:%ld us", taosArrayGetSize(uidList), stt4-stt3);
for(int i = 0; i < taosArrayGetSize(readerList); i++){
SMetaReader* pReader = taosArrayGet(readerList, i);
@ -100,8 +122,8 @@ int metaGetTableEntryByUidTest(void* meta, SArray *uidList) {
}
metaReaderClear(pReader);
}
int64_t stt4 = taosGetTimestampUs();
qDebug("metaGetTableEntryByUidTest3 rows:%d, cost:%ld us", taosArrayGetSize(readerList), stt4-stt3);
int64_t stt5 = taosGetTimestampUs();
qDebug("metaGetTableEntryByUidTest4 rows:%d, cost:%ld us", taosArrayGetSize(readerList), stt5-stt4);
return 0;
}
@ -801,9 +823,8 @@ SArray *metaGetSmaTbUids(SMeta *pMeta) {
#endif
const void *metaGetTableTagVal(SMetaEntry *pEntry, int16_t type, STagVal *val) {
ASSERT(pEntry->type == TSDB_CHILD_TABLE);
STag *tag = (STag *)pEntry->ctbEntry.pTags;
const void *metaGetTableTagVal(void *pTag, int16_t type, STagVal *val) {
STag *tag = (STag*) pTag;
if (type == TSDB_DATA_TYPE_JSON) {
return tag;
}
@ -940,3 +961,38 @@ END:
return ret;
}
int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList, SArray *tags) {
SMCtbCursor *pCur = metaOpenCtbCursor(pMeta, suid);
SHashObj *uHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
size_t len = taosArrayGetSize(uidList);
if(len > 0){
for(int i = 0; i < len; i++){
int64_t *uid = taosArrayGet(uidList, i);
taosHashPut(uHash, uid, sizeof(int64_t), &i, sizeof(i));
}
}
while (1) {
tb_uid_t id = metaCtbCursorNext(pCur);
if (id == 0) {
break;
}
if(len > 0 && taosHashGet(uHash, &id, sizeof(int64_t)) == NULL){
continue;
}
void* tag = taosMemoryMalloc(pCur->vLen);
memcpy(tag, pCur->pVal, pCur->vLen);
taosArrayPush(tags, &tag);
if(len == 0){
taosArrayPush(uidList, &id);
}
}
taosHashCleanup(uHash);
metaCloseCtbCursor(pCur);
return TSDB_CODE_SUCCESS;
}

5
source/dnode/vnode/src/meta/metaTable.c
View File

@ -864,6 +864,9 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA
metaUpdateTagIdx(pMeta, &ctbEntry);
}
SCtbIdxKey ctbIdxKey = {.suid = ctbEntry.ctbEntry.suid, .uid = uid};
tdbTbUpsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), ctbEntry.ctbEntry.pTags, ((STag*)(ctbEntry.ctbEntry.pTags))->len, &pMeta->txn);
tDecoderClear(&dc1);
tDecoderClear(&dc2);
if (ctbEntry.ctbEntry.pTags) taosMemoryFree((void *)ctbEntry.ctbEntry.pTags);
@ -1061,7 +1064,7 @@ static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME) {
static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME) {
SCtbIdxKey ctbIdxKey = {.suid = pME->ctbEntry.suid, .uid = pME->uid};
return tdbTbInsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), NULL, 0, &pMeta->txn);
return tdbTbInsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), pME->ctbEntry.pTags, ((STag*)(pME->ctbEntry.pTags))->len, &pMeta->txn);
}
int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void *pTagData, int32_t nTagData, int8_t type, tb_uid_t uid,

79
source/libs/executor/src/executil.c
View File

@ -221,7 +221,7 @@ EDealRes doTranslateTagExpr(SNode** pNode, void* pContext) {
STagVal tagVal = {0};
tagVal.cid = pSColumnNode->colId;
const char* p = metaGetTableTagVal(&mr->me, pSColumnNode->node.resType.type, &tagVal);
const char* p = metaGetTableTagVal(mr->me.ctbEntry.pTags, pSColumnNode->node.resType.type, &tagVal);
if (p == NULL) {
res->node.resType.type = TSDB_DATA_TYPE_NULL;
} else if (pSColumnNode->node.resType.type == TSDB_DATA_TYPE_JSON) {
@ -343,10 +343,11 @@ static int32_t createResultData(SDataType* pType, int32_t numOfRows, SScalarPara
return TSDB_CODE_SUCCESS;
}
SColumnInfoData* getColInfoResult(void* metaHandle, SArray* pTableList, SNode* pTagCond){
SColumnInfoData* getColInfoResult(void* metaHandle, uint64_t suid, SArray* uidList, SNode* pTagCond){
int32_t code = TSDB_CODE_SUCCESS;
SArray* pBlockList = NULL;
SSDataBlock* pResBlock = NULL;
SArray* tags = NULL;
SScalarParam output = {0};
tagFilterAssist ctx = {0};
@ -375,35 +376,30 @@ SColumnInfoData* getColInfoResult(void* metaHandle, SArray* pTableList, SNode* p
blockDataAppendColInfo(pResBlock, &colInfo);
}
int32_t rows = taosArrayGetSize(pTableList);
int64_t stt = taosGetTimestampUs();
tags = taosArrayInit(8, POINTER_BYTES);
code = metaGetTableTags(metaHandle, suid, uidList, tags);
if (code != TSDB_CODE_SUCCESS) {
terrno = code;
goto end;
}
int32_t rows = taosArrayGetSize(uidList);
int64_t stt1 = taosGetTimestampUs();
qDebug("generate tag meta rows:%d, cost:%ld us", rows, stt1-stt);
code = blockDataEnsureCapacity(pResBlock, rows);
if (code != TSDB_CODE_SUCCESS) {
terrno = code;
goto end;
}
// int64_t stt = taosGetTimestampUs();
// SArray* arrAssist = taosArrayInit(rows, sizeof(SMetaReader));
// SArray* uidList = taosArrayInit(rows, sizeof(uint64_t));
// for (int32_t i = 0; i < rows; i++) {
// STableKeyInfo* info = taosArrayGet(pTableList, i);
// taosArrayPush(uidList, &info->uid);
// }
// code = metaGetTableEntryByUidTest(metaHandle, uidList);
//
// int64_t stt1 = taosGetTimestampUs();
// qDebug("generate tag meta rows:%d, cost:%ld us", rows, stt1-stt);
int64_t st = taosGetTimestampUs();
for (int32_t i = 0; i < rows; i++) {
STableKeyInfo* info = taosArrayGet(pTableList, i);
uint64_t* uid = taosArrayGet(uidList, i);
void* tag = taosArrayGet(tags, i);
// int64_t stt = taosGetTimestampUs();
SMetaReader mr = {0};
metaReaderInit(&mr, metaHandle, 0);
code = metaGetTableEntryByUid(&mr, info->uid);
// int64_t stt1 = taosGetTimestampUs();
// qDebug("generate tag get meta rows:%d, cost:%ld ms", rows, stt1-stt);
// SMetaReader *mr = taosArrayGet(arrAssist, i);
@ -412,7 +408,7 @@ SColumnInfoData* getColInfoResult(void* metaHandle, SArray* pTableList, SNode* p
STagVal tagVal = {0};
tagVal.cid = pColInfo->info.colId;
// int64_t t1 = taosGetTimestampUs();
const char* p = metaGetTableTagVal(&mr.me, pColInfo->info.type, &tagVal);
const char* p = metaGetTableTagVal(tag, pColInfo->info.type, &tagVal);
// int64_t t2 = taosGetTimestampUs();
// qDebug("generate tag inner1 rows:%d, cost:%ld ms", rows, t2-t1);
@ -430,7 +426,6 @@ SColumnInfoData* getColInfoResult(void* metaHandle, SArray* pTableList, SNode* p
}
// int64_t stt2 = taosGetTimestampUs();
// qDebug("generate tag get block rows:%d, cost:%ld us", rows, stt2-stt1);
metaReaderClear(&mr);
}
pResBlock->info.rows = rows;
@ -454,6 +449,7 @@ SColumnInfoData* getColInfoResult(void* metaHandle, SArray* pTableList, SNode* p
qDebug("calculate tag block rows:%d, cost:%ld us", rows, st2-st1);
end:
taosArrayDestroy(tags);
taosHashCleanup(ctx.colHash);
taosArrayDestroy(ctx.cInfoList);
blockDataDestroy(pResBlock);
@ -471,8 +467,8 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode,
}
uint64_t tableUid = pScanNode->uid;
pListInfo->suid = pScanNode->suid;
SArray* res = taosArrayInit(8, sizeof(uint64_t));
if (pScanNode->tableType == TSDB_SUPER_TABLE) {
if (pTagIndexCond) {
@ -480,42 +476,24 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode,
.metaEx = metaHandle, .idx = tsdbGetIdx(metaHandle), .ivtIdx = tsdbGetIvtIdx(metaHandle), .suid = tableUid};
int64_t stt = taosGetTimestampUs();
SArray* res = taosArrayInit(8, sizeof(uint64_t));
SIdxFltStatus status = SFLT_NOT_INDEX;
code = doFilterTag(pTagIndexCond, &metaArg, res, &status);
if (code != 0 || status == SFLT_NOT_INDEX) {
qError("failed to get tableIds from index, reason:%s, suid:%" PRIu64, tstrerror(code), tableUid);
// code = TSDB_CODE_INDEX_REBUILDING;
code = vnodeGetAllTableList(pVnode, tableUid, pListInfo->pTableList);
} else {
qDebug("success to get tableIds, size:%d, suid:%" PRIu64, (int)taosArrayGetSize(res), tableUid);
}
for (int i = 0; i < taosArrayGetSize(res); i++) {
STableKeyInfo info = {.uid = *(uint64_t*)taosArrayGet(res, i), .groupId = 0};
taosArrayPush(pListInfo->pTableList, &info);
}
taosArrayDestroy(res);
int64_t stt1 = taosGetTimestampUs();
qDebug("generate table list, cost:%ld us", stt1-stt);
} else {
code = vnodeGetAllTableList(pVnode, tableUid, pListInfo->pTableList);
}
if (code != TSDB_CODE_SUCCESS) {
qError("failed to get tableIds, reason:%s, suid:%" PRIu64, tstrerror(code), tableUid);
terrno = code;
return code;
}
} else { // Create one table group.
STableKeyInfo info = {.uid = tableUid, .groupId = 0};
taosArrayPush(pListInfo->pTableList, &info);
taosArrayPush(res, &tableUid);
}
if (pTagCond && taosArrayGetSize(pListInfo->pTableList) > 0) {
SColumnInfoData* pColInfoData = getColInfoResult(metaHandle, pListInfo->pTableList, pTagCond);
if (pTagCond) {
SColumnInfoData* pColInfoData = getColInfoResult(metaHandle, pListInfo->suid, res, pTagCond);
if(terrno != TDB_CODE_SUCCESS){
colDataDestroy(pColInfoData);
taosArrayDestroy(res);
return terrno;
}
@ -532,6 +510,13 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode,
colDataDestroy(pColInfoData);
}
for (int i = 0; i < taosArrayGetSize(res); i++) {
STableKeyInfo info = {.uid = *(uint64_t*)taosArrayGet(res, i), .groupId = 0};
taosArrayPush(pListInfo->pTableList, &info);
}
taosArrayDestroy(res);
pListInfo->pGroupList = taosArrayInit(4, POINTER_BYTES);
if (pListInfo->pGroupList == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;

4
source/libs/executor/src/scanoperator.c
View File

@ -439,7 +439,7 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, SExprInfo* pPseudoExpr, int
} else { // these are tags
STagVal tagVal = {0};
tagVal.cid = pExpr->base.pParam[0].pCol->colId;
const char* p = metaGetTableTagVal(&mr.me, pColInfoData->info.type, &tagVal);
const char* p = metaGetTableTagVal(mr.me.ctbEntry.pTags, pColInfoData->info.type, &tagVal);
char* data = NULL;
if (pColInfoData->info.type != TSDB_DATA_TYPE_JSON && p != NULL) {
@ -2569,7 +2569,7 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) {
} else { // it is a tag value
STagVal val = {0};
val.cid = pExprInfo[j].base.pParam[0].pCol->colId;
const char* p = metaGetTableTagVal(&mr.me, pDst->info.type, &val);
const char* p = metaGetTableTagVal(mr.me.ctbEntry.pTags, pDst->info.type, &val);
char* data = NULL;
if (pDst->info.type != TSDB_DATA_TYPE_JSON && p != NULL) {