diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index e969634a9f..6ad47799e1 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -91,9 +91,9 @@ typedef struct SMetaEntry SMetaEntry; void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags); void metaReaderClear(SMetaReader *pReader); int32_t metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid); -//int32_t metaGetTableEntryByUidTest(void *pReader, SArray *uidList); +int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList, SArray *tags); int32_t metaReadNext(SMetaReader *pReader); -const void *metaGetTableTagVal(SMetaEntry *pEntry, int16_t type, STagVal *tagVal); +const void *metaGetTableTagVal(void *tag, int16_t type, STagVal *tagVal); int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName); typedef struct SMetaFltParam { diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index f5771c17bd..df22913a0d 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -59,6 +59,7 @@ int metaGetTableEntryByUidTest(void* meta, SArray *uidList) { SArray* uidVersion = taosArrayInit(taosArrayGetSize(uidList), sizeof(STbDbKey)); SMeta *pMeta = meta; int64_t version; + SHashObj *uHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); int64_t stt1 = taosGetTimestampUs(); for(int i = 0; i < taosArrayGetSize(uidList); i++) { @@ -73,10 +74,31 @@ int metaGetTableEntryByUidTest(void* meta, SArray *uidList) { STbDbKey tbDbKey = {.version = version, .uid = *uid}; taosArrayPush(uidVersion, &tbDbKey); + taosHashPut(uHash, uid, sizeof(int64_t), ppVal, sizeof(int64_t)); } int64_t stt2 = taosGetTimestampUs(); qDebug("metaGetTableEntryByUidTest1 rows:%d, cost:%ld us", taosArrayGetSize(uidList), stt2-stt1); + TBC *pCur = NULL; + tdbTbcOpen(pMeta->pTbDb, &pCur, NULL); + tdbTbcMoveToFirst(pCur); + void *pKey = NULL; + int kLen = 0; + + while(1){ + SMetaReader pReader = {0}; + int32_t ret = tdbTbcNext(pCur, &pKey, &kLen, &pReader.pBuf, &pReader.szBuf); + if (ret < 0) break; + STbDbKey *tmp = (STbDbKey*)pKey; + int64_t *ver = (int64_t*)taosHashGet(uHash, &tmp->uid, sizeof(int64_t)); + if(ver == NULL || *ver != tmp->version) continue; + taosArrayPush(readerList, &pReader); + } + tdbTbcClose(pCur); + + taosArrayClear(readerList); + int64_t stt3 = taosGetTimestampUs(); + qDebug("metaGetTableEntryByUidTest2 rows:%d, cost:%ld us", taosArrayGetSize(uidList), stt3-stt2); for(int i = 0; i < taosArrayGetSize(uidVersion); i++) { SMetaReader pReader = {0}; @@ -87,8 +109,8 @@ int metaGetTableEntryByUidTest(void* meta, SArray *uidList) { } taosArrayPush(readerList, &pReader); } - int64_t stt3 = taosGetTimestampUs(); - qDebug("metaGetTableEntryByUidTest2 rows:%d, cost:%ld us", taosArrayGetSize(uidList), stt3-stt2); + int64_t stt4 = taosGetTimestampUs(); + qDebug("metaGetTableEntryByUidTest3 rows:%d, cost:%ld us", taosArrayGetSize(uidList), stt4-stt3); for(int i = 0; i < taosArrayGetSize(readerList); i++){ SMetaReader* pReader = taosArrayGet(readerList, i); @@ -100,8 +122,8 @@ int metaGetTableEntryByUidTest(void* meta, SArray *uidList) { } metaReaderClear(pReader); } - int64_t stt4 = taosGetTimestampUs(); - qDebug("metaGetTableEntryByUidTest3 rows:%d, cost:%ld us", taosArrayGetSize(readerList), stt4-stt3); + int64_t stt5 = taosGetTimestampUs(); + qDebug("metaGetTableEntryByUidTest4 rows:%d, cost:%ld us", taosArrayGetSize(readerList), stt5-stt4); return 0; } @@ -801,9 +823,8 @@ SArray *metaGetSmaTbUids(SMeta *pMeta) { #endif -const void *metaGetTableTagVal(SMetaEntry *pEntry, int16_t type, STagVal *val) { - ASSERT(pEntry->type == TSDB_CHILD_TABLE); - STag *tag = (STag *)pEntry->ctbEntry.pTags; +const void *metaGetTableTagVal(void *pTag, int16_t type, STagVal *val) { + STag *tag = (STag*) pTag; if (type == TSDB_DATA_TYPE_JSON) { return tag; } @@ -940,3 +961,38 @@ END: return ret; } + +int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList, SArray *tags) { + SMCtbCursor *pCur = metaOpenCtbCursor(pMeta, suid); + + SHashObj *uHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); + size_t len = taosArrayGetSize(uidList); + if(len > 0){ + for(int i = 0; i < len; i++){ + int64_t *uid = taosArrayGet(uidList, i); + taosHashPut(uHash, uid, sizeof(int64_t), &i, sizeof(i)); + } + } + while (1) { + tb_uid_t id = metaCtbCursorNext(pCur); + if (id == 0) { + break; + } + + if(len > 0 && taosHashGet(uHash, &id, sizeof(int64_t)) == NULL){ + continue; + } + + void* tag = taosMemoryMalloc(pCur->vLen); + memcpy(tag, pCur->pVal, pCur->vLen); + taosArrayPush(tags, &tag); + + if(len == 0){ + taosArrayPush(uidList, &id); + } + } + + taosHashCleanup(uHash); + metaCloseCtbCursor(pCur); + return TSDB_CODE_SUCCESS; +} diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 7427f79509..86e2f38697 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -864,6 +864,9 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA metaUpdateTagIdx(pMeta, &ctbEntry); } + SCtbIdxKey ctbIdxKey = {.suid = ctbEntry.ctbEntry.suid, .uid = uid}; + tdbTbUpsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), ctbEntry.ctbEntry.pTags, ((STag*)(ctbEntry.ctbEntry.pTags))->len, &pMeta->txn); + tDecoderClear(&dc1); tDecoderClear(&dc2); if (ctbEntry.ctbEntry.pTags) taosMemoryFree((void *)ctbEntry.ctbEntry.pTags); @@ -1061,7 +1064,7 @@ static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME) { static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME) { SCtbIdxKey ctbIdxKey = {.suid = pME->ctbEntry.suid, .uid = pME->uid}; - return tdbTbInsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), NULL, 0, &pMeta->txn); + return tdbTbInsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), pME->ctbEntry.pTags, ((STag*)(pME->ctbEntry.pTags))->len, &pMeta->txn); } int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void *pTagData, int32_t nTagData, int8_t type, tb_uid_t uid, diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index f798433f51..c8d607185e 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -221,7 +221,7 @@ EDealRes doTranslateTagExpr(SNode** pNode, void* pContext) { STagVal tagVal = {0}; tagVal.cid = pSColumnNode->colId; - const char* p = metaGetTableTagVal(&mr->me, pSColumnNode->node.resType.type, &tagVal); + const char* p = metaGetTableTagVal(mr->me.ctbEntry.pTags, pSColumnNode->node.resType.type, &tagVal); if (p == NULL) { res->node.resType.type = TSDB_DATA_TYPE_NULL; } else if (pSColumnNode->node.resType.type == TSDB_DATA_TYPE_JSON) { @@ -343,10 +343,11 @@ static int32_t createResultData(SDataType* pType, int32_t numOfRows, SScalarPara return TSDB_CODE_SUCCESS; } -SColumnInfoData* getColInfoResult(void* metaHandle, SArray* pTableList, SNode* pTagCond){ +SColumnInfoData* getColInfoResult(void* metaHandle, uint64_t suid, SArray* uidList, SNode* pTagCond){ int32_t code = TSDB_CODE_SUCCESS; SArray* pBlockList = NULL; SSDataBlock* pResBlock = NULL; + SArray* tags = NULL; SScalarParam output = {0}; tagFilterAssist ctx = {0}; @@ -375,35 +376,30 @@ SColumnInfoData* getColInfoResult(void* metaHandle, SArray* pTableList, SNode* p blockDataAppendColInfo(pResBlock, &colInfo); } - int32_t rows = taosArrayGetSize(pTableList); + int64_t stt = taosGetTimestampUs(); + tags = taosArrayInit(8, POINTER_BYTES); + code = metaGetTableTags(metaHandle, suid, uidList, tags); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + goto end; + } + + int32_t rows = taosArrayGetSize(uidList); + + int64_t stt1 = taosGetTimestampUs(); + qDebug("generate tag meta rows:%d, cost:%ld us", rows, stt1-stt); + code = blockDataEnsureCapacity(pResBlock, rows); if (code != TSDB_CODE_SUCCESS) { terrno = code; goto end; } -// int64_t stt = taosGetTimestampUs(); -// SArray* arrAssist = taosArrayInit(rows, sizeof(SMetaReader)); -// SArray* uidList = taosArrayInit(rows, sizeof(uint64_t)); -// for (int32_t i = 0; i < rows; i++) { -// STableKeyInfo* info = taosArrayGet(pTableList, i); -// taosArrayPush(uidList, &info->uid); -// } - -// code = metaGetTableEntryByUidTest(metaHandle, uidList); - -// -// int64_t stt1 = taosGetTimestampUs(); -// qDebug("generate tag meta rows:%d, cost:%ld us", rows, stt1-stt); - int64_t st = taosGetTimestampUs(); for (int32_t i = 0; i < rows; i++) { - STableKeyInfo* info = taosArrayGet(pTableList, i); - + uint64_t* uid = taosArrayGet(uidList, i); + void* tag = taosArrayGet(tags, i); // int64_t stt = taosGetTimestampUs(); - SMetaReader mr = {0}; - metaReaderInit(&mr, metaHandle, 0); - code = metaGetTableEntryByUid(&mr, info->uid); // int64_t stt1 = taosGetTimestampUs(); // qDebug("generate tag get meta rows:%d, cost:%ld ms", rows, stt1-stt); // SMetaReader *mr = taosArrayGet(arrAssist, i); @@ -412,7 +408,7 @@ SColumnInfoData* getColInfoResult(void* metaHandle, SArray* pTableList, SNode* p STagVal tagVal = {0}; tagVal.cid = pColInfo->info.colId; // int64_t t1 = taosGetTimestampUs(); - const char* p = metaGetTableTagVal(&mr.me, pColInfo->info.type, &tagVal); + const char* p = metaGetTableTagVal(tag, pColInfo->info.type, &tagVal); // int64_t t2 = taosGetTimestampUs(); // qDebug("generate tag inner1 rows:%d, cost:%ld ms", rows, t2-t1); @@ -430,7 +426,6 @@ SColumnInfoData* getColInfoResult(void* metaHandle, SArray* pTableList, SNode* p } // int64_t stt2 = taosGetTimestampUs(); // qDebug("generate tag get block rows:%d, cost:%ld us", rows, stt2-stt1); - metaReaderClear(&mr); } pResBlock->info.rows = rows; @@ -454,6 +449,7 @@ SColumnInfoData* getColInfoResult(void* metaHandle, SArray* pTableList, SNode* p qDebug("calculate tag block rows:%d, cost:%ld us", rows, st2-st1); end: + taosArrayDestroy(tags); taosHashCleanup(ctx.colHash); taosArrayDestroy(ctx.cInfoList); blockDataDestroy(pResBlock); @@ -471,8 +467,8 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, } uint64_t tableUid = pScanNode->uid; - pListInfo->suid = pScanNode->suid; + SArray* res = taosArrayInit(8, sizeof(uint64_t)); if (pScanNode->tableType == TSDB_SUPER_TABLE) { if (pTagIndexCond) { @@ -480,42 +476,24 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, .metaEx = metaHandle, .idx = tsdbGetIdx(metaHandle), .ivtIdx = tsdbGetIvtIdx(metaHandle), .suid = tableUid}; int64_t stt = taosGetTimestampUs(); - SArray* res = taosArrayInit(8, sizeof(uint64_t)); SIdxFltStatus status = SFLT_NOT_INDEX; code = doFilterTag(pTagIndexCond, &metaArg, res, &status); if (code != 0 || status == SFLT_NOT_INDEX) { qError("failed to get tableIds from index, reason:%s, suid:%" PRIu64, tstrerror(code), tableUid); - // code = TSDB_CODE_INDEX_REBUILDING; - code = vnodeGetAllTableList(pVnode, tableUid, pListInfo->pTableList); - } else { - qDebug("success to get tableIds, size:%d, suid:%" PRIu64, (int)taosArrayGetSize(res), tableUid); } - for (int i = 0; i < taosArrayGetSize(res); i++) { - STableKeyInfo info = {.uid = *(uint64_t*)taosArrayGet(res, i), .groupId = 0}; - taosArrayPush(pListInfo->pTableList, &info); - } - taosArrayDestroy(res); int64_t stt1 = taosGetTimestampUs(); qDebug("generate table list, cost:%ld us", stt1-stt); - } else { - code = vnodeGetAllTableList(pVnode, tableUid, pListInfo->pTableList); - } - - if (code != TSDB_CODE_SUCCESS) { - qError("failed to get tableIds, reason:%s, suid:%" PRIu64, tstrerror(code), tableUid); - terrno = code; - return code; } } else { // Create one table group. - STableKeyInfo info = {.uid = tableUid, .groupId = 0}; - taosArrayPush(pListInfo->pTableList, &info); + taosArrayPush(res, &tableUid); } - if (pTagCond && taosArrayGetSize(pListInfo->pTableList) > 0) { - SColumnInfoData* pColInfoData = getColInfoResult(metaHandle, pListInfo->pTableList, pTagCond); + if (pTagCond) { + SColumnInfoData* pColInfoData = getColInfoResult(metaHandle, pListInfo->suid, res, pTagCond); if(terrno != TDB_CODE_SUCCESS){ colDataDestroy(pColInfoData); + taosArrayDestroy(res); return terrno; } @@ -532,6 +510,13 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, colDataDestroy(pColInfoData); } + for (int i = 0; i < taosArrayGetSize(res); i++) { + STableKeyInfo info = {.uid = *(uint64_t*)taosArrayGet(res, i), .groupId = 0}; + taosArrayPush(pListInfo->pTableList, &info); + } + + taosArrayDestroy(res); + pListInfo->pGroupList = taosArrayInit(4, POINTER_BYTES); if (pListInfo->pGroupList == NULL) { return TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index a211542de1..61f6474190 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -439,7 +439,7 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, SExprInfo* pPseudoExpr, int } else { // these are tags STagVal tagVal = {0}; tagVal.cid = pExpr->base.pParam[0].pCol->colId; - const char* p = metaGetTableTagVal(&mr.me, pColInfoData->info.type, &tagVal); + const char* p = metaGetTableTagVal(mr.me.ctbEntry.pTags, pColInfoData->info.type, &tagVal); char* data = NULL; if (pColInfoData->info.type != TSDB_DATA_TYPE_JSON && p != NULL) { @@ -2569,7 +2569,7 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { } else { // it is a tag value STagVal val = {0}; val.cid = pExprInfo[j].base.pParam[0].pCol->colId; - const char* p = metaGetTableTagVal(&mr.me, pDst->info.type, &val); + const char* p = metaGetTableTagVal(mr.me.ctbEntry.pTags, pDst->info.type, &val); char* data = NULL; if (pDst->info.type != TSDB_DATA_TYPE_JSON && p != NULL) {