xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

refactor: do some internal refactor.

This commit is contained in:
Haojun Liao 2023-01-14 23:17:57 +08:00
parent 7aee35c457
commit 82f4db302e
2 changed files with 48 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
source/dnode/vnode/src/meta/metaQuery.c
View File

@ -1368,7 +1368,7 @@ int32_t metaGetTableTagsByUids(SMeta *pMeta, int64_t suid, SArray *uidList, SHas
taosHashPut(tags, id, sizeof(tb_uid_t), val, len); taosHashPut(tags, id, sizeof(tb_uid_t), val, len);
tdbFree(val); tdbFree(val);
} else { } else {
metaError("vgId:%d, failed to table IDs, suid: %" PRId64 ", uid: %" PRId64 "", TD_VID(pMeta->pVnode), suid, metaError("vgId:%d, failed to table tags, suid: %" PRId64 ", uid: %" PRId64 "", TD_VID(pMeta->pVnode), suid,
*id); *id);
} }
} }
@ -1381,31 +1381,35 @@ int32_t metaGetTableTagsByUids(SMeta *pMeta, int64_t suid, SArray *uidList, SHas
int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList, SHashObj *tags) { int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList, SHashObj *tags) {
SMCtbCursor *pCur = metaOpenCtbCursor(pMeta, suid, 1); SMCtbCursor *pCur = metaOpenCtbCursor(pMeta, suid, 1);
SHashObj *uHash = NULL; // If len > 0 means there already have uids, and we only want the
size_t len = taosArrayGetSize(uidList); // len > 0 means there already have uids // tags of the specified tables, of which uid in the uid list. Otherwise, all table tags are retrieved and kept
// in the hash map, that may require a lot of memory
SHashObj *pSepecifiedUidMap = NULL;
size_t len = taosArrayGetSize(uidList);
if (len > 0) { if (len > 0) {
uHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); pSepecifiedUidMap = taosHashInit(len / 0.7, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
for (int i = 0; i < len; i++) { for (int i = 0; i < len; i++) {
int64_t *uid = taosArrayGet(uidList, i); int64_t *uid = taosArrayGet(uidList, i);
taosHashPut(uHash, uid, sizeof(int64_t), &i, sizeof(i)); taosHashPut(pSepecifiedUidMap, uid, sizeof(int64_t), 0, 0);
} }
} }
while (1) { while (1) {
tb_uid_t id = metaCtbCursorNext(pCur); tb_uid_t uid = metaCtbCursorNext(pCur);
if (id == 0) { if (uid == 0) {
break; break;
} }
if (len > 0 && taosHashGet(uHash, &id, sizeof(int64_t)) == NULL) { if (len > 0 && taosHashGet(pSepecifiedUidMap, &uid, sizeof(int64_t)) == NULL) {
continue; continue;
} else if (len == 0) { } else if (len == 0) {
taosArrayPush(uidList, &id); taosArrayPush(uidList, &uid);
} }
taosHashPut(tags, &id, sizeof(int64_t), pCur->pVal, pCur->vLen); taosHashPut(tags, &uid, sizeof(uint64_t), pCur->pVal, pCur->vLen);
} }
taosHashCleanup(uHash); taosHashCleanup(pSepecifiedUidMap);
metaCloseCtbCursor(pCur, 1); metaCloseCtbCursor(pCur, 1);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }

59
source/libs/executor/src/executil.c
View File

@ -43,9 +43,9 @@ typedef struct tagFilterAssist {
SArray* cInfoList; SArray* cInfoList;
} tagFilterAssist; } tagFilterAssist;
static int32_t removeInvalidTable(SArray* uids, SHashObj* tags); static int32_t removeInvalidUid(SArray* uids, SHashObj* tags);
static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list, SNode* pTagCond, SHashObj* tags); static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list, SNode* pTagCond, SHashObj* tags);
static int32_t optimizeTbnameInCondImpl(void* metaHandle, int64_t suid, SArray* list, SNode* pTagCond); static int32_t optimizeTbnameInCondImpl(void* metaHandle, int64_t suid, SArray* pExistedUidList, SNode* pTagCond);
static int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, static int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond,
SNode* pTagIndexCond, STableListInfo* pListInfo); SNode* pTagIndexCond, STableListInfo* pListInfo);
@ -433,6 +433,7 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, int64_t suid, SArray*
int32_t filter = optimizeTbnameInCond(metaHandle, suid, uidList, pTagCond, tags); int32_t filter = optimizeTbnameInCond(metaHandle, suid, uidList, pTagCond, tags);
if (filter == -1) { if (filter == -1) {
// here we retrieve all tags from the vnode table-meta store
code = metaGetTableTags(metaHandle, suid, uidList, tags); code = metaGetTableTags(metaHandle, suid, uidList, tags);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
qError("failed to get table tags from meta, reason:%s, suid:%" PRIu64, tstrerror(code), suid); qError("failed to get table tags from meta, reason:%s, suid:%" PRIu64, tstrerror(code), suid);
@ -440,22 +441,23 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, int64_t suid, SArray*
goto end; goto end;
} }
} }
if (suid != 0) { if (suid != 0) {
removeInvalidTable(uidList, tags); removeInvalidUid(uidList, tags);
} }
int32_t rows = taosArrayGetSize(uidList); int32_t size = taosArrayGetSize(uidList);
if (rows == 0) { if (size == 0) {
goto end; goto end;
} }
code = blockDataEnsureCapacity(pResBlock, rows); code = blockDataEnsureCapacity(pResBlock, size);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
terrno = code; terrno = code;
goto end; goto end;
} }
for (int32_t i = 0; i < rows; i++) { for (int32_t i = 0; i < size; i++) {
int64_t* uid = taosArrayGet(uidList, i); int64_t* uid = taosArrayGet(uidList, i);
for (int32_t j = 0; j < taosArrayGetSize(pResBlock->pDataBlock); j++) { for (int32_t j = 0; j < taosArrayGetSize(pResBlock->pDataBlock); j++) {
SColumnInfoData* pColInfo = (SColumnInfoData*)taosArrayGet(pResBlock->pDataBlock, j); SColumnInfoData* pColInfo = (SColumnInfoData*)taosArrayGet(pResBlock->pDataBlock, j);
@ -468,13 +470,14 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, int64_t suid, SArray*
qDebug("tagfilter uid:%ld, tbname:%s", *uid, str + 2); qDebug("tagfilter uid:%ld, tbname:%s", *uid, str + 2);
#endif #endif
} else { } else {
void* tag = taosHashGet(tags, uid, sizeof(int64_t)); void* pTagsVal = taosHashGet(tags, uid, sizeof(uint64_t));
if (tag == NULL) { if (pTagsVal == NULL) {
continue; continue;
} }
STagVal tagVal = {0}; STagVal tagVal = {0};
tagVal.cid = pColInfo->info.colId; tagVal.cid = pColInfo->info.colId;
const char* p = metaGetTableTagVal(tag, pColInfo->info.type, &tagVal); const char* p = metaGetTableTagVal(pTagsVal, pColInfo->info.type, &tagVal);
if (p == NULL || (pColInfo->info.type == TSDB_DATA_TYPE_JSON && ((STag*)p)->nTag == 0)) { if (p == NULL || (pColInfo->info.type == TSDB_DATA_TYPE_JSON && ((STag*)p)->nTag == 0)) {
colDataAppend(pColInfo, i, p, true); colDataAppend(pColInfo, i, p, true);
@ -504,7 +507,7 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, int64_t suid, SArray*
} }
} }
pResBlock->info.rows = rows; pResBlock->info.rows = size;
// int64_t st1 = taosGetTimestampUs(); // int64_t st1 = taosGetTimestampUs();
// qDebug("generate tag block rows:%d, cost:%ld us", rows, st1-st); // qDebug("generate tag block rows:%d, cost:%ld us", rows, st1-st);
@ -513,7 +516,7 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, int64_t suid, SArray*
taosArrayPush(pBlockList, &pResBlock); taosArrayPush(pBlockList, &pResBlock);
SDataType type = {.type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)}; SDataType type = {.type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)};
code = createResultData(&type, rows, &output); code = createResultData(&type, size, &output);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
terrno = code; terrno = code;
qError("failed to create result, reason:%s", tstrerror(code)); qError("failed to create result, reason:%s", tstrerror(code));
@ -850,7 +853,7 @@ static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list
ret = optimizeTbnameInCondImpl(metaHandle, suid, list, cond); ret = optimizeTbnameInCondImpl(metaHandle, suid, list, cond);
if (ret != -1) { if (ret != -1) {
metaGetTableTagsByUids(metaHandle, suid, list, tags); metaGetTableTagsByUids(metaHandle, suid, list, tags);
removeInvalidTable(list, tags); removeInvalidUid(list, tags);
} }
} }
@ -882,7 +885,7 @@ static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list
if (hasTbnameCond) { if (hasTbnameCond) {
ret = metaGetTableTagsByUids(metaHandle, suid, list, tags); ret = metaGetTableTagsByUids(metaHandle, suid, list, tags);
removeInvalidTable(list, tags); removeInvalidUid(list, tags);
} }
return ret; return ret;
@ -891,12 +894,15 @@ static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list
/* /*
* handle invalid uid * handle invalid uid
*/ */
static int32_t removeInvalidTable(SArray* uids, SHashObj* tags) { static int32_t removeInvalidUid(SArray* uids, SHashObj* tags) {
if (taosArrayGetSize(uids) <= 0) return 0; int32_t size = taosArrayGetSize(uids);
if (size <= 0) {
return 0;
}
SArray* validUid = taosArrayInit(taosArrayGetSize(uids), sizeof(int64_t)); SArray* validUid = taosArrayInit(size, sizeof(int64_t));
for (int32_t i = 0; i < taosArrayGetSize(uids); i++) { for (int32_t i = 0; i < size; i++) {
int64_t* uid = taosArrayGet(uids, i); int64_t* uid = taosArrayGet(uids, i);
if (taosHashGet(tags, uid, sizeof(int64_t)) != NULL) { if (taosHashGet(tags, uid, sizeof(int64_t)) != NULL) {
taosArrayPush(validUid, uid); taosArrayPush(validUid, uid);
@ -908,7 +914,8 @@ static int32_t removeInvalidTable(SArray* uids, SHashObj* tags) {
return 0; return 0;
} }
static int32_t optimizeTbnameInCondImpl(void* metaHandle, int64_t suid, SArray* list, SNode* pTagCond) { // only return uid that does not contained in pExistedUidList
static int32_t optimizeTbnameInCondImpl(void* metaHandle, int64_t suid, SArray* pExistedUidList, SNode* pTagCond) {
if (nodeType(pTagCond) != QUERY_NODE_OPERATOR) { if (nodeType(pTagCond) != QUERY_NODE_OPERATOR) {
return -1; return -1;
} }
@ -931,11 +938,11 @@ static int32_t optimizeTbnameInCondImpl(void* metaHandle, int64_t suid, SArray*
SArray* pTbList = getTableNameList(pList); SArray* pTbList = getTableNameList(pList);
int32_t numOfTables = taosArrayGetSize(pTbList); int32_t numOfTables = taosArrayGetSize(pTbList);
SHashObj* uHash = NULL; SHashObj* uHash = NULL;
size_t listlen = taosArrayGetSize(list); // len > 0 means there already have uids size_t numOfExisted = taosArrayGetSize(pExistedUidList); // len > 0 means there already have uids
if (listlen > 0) { if (numOfExisted > 0) {
uHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); uHash = taosHashInit(numOfExisted / 0.7, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
for (int i = 0; i < listlen; i++) { for (int i = 0; i < numOfExisted; i++) {
int64_t* uid = taosArrayGet(list, i); int64_t* uid = taosArrayGet(pExistedUidList, i);
taosHashPut(uHash, uid, sizeof(int64_t), &i, sizeof(i)); taosHashPut(uHash, uid, sizeof(int64_t), &i, sizeof(i));
} }
} }
@ -948,7 +955,7 @@ static int32_t optimizeTbnameInCondImpl(void* metaHandle, int64_t suid, SArray*
ETableType tbType = TSDB_TABLE_MAX; ETableType tbType = TSDB_TABLE_MAX;
if (metaGetTableTypeByName(metaHandle, name, &tbType) == 0 && tbType == TSDB_CHILD_TABLE) { if (metaGetTableTypeByName(metaHandle, name, &tbType) == 0 && tbType == TSDB_CHILD_TABLE) {
if (NULL == uHash || taosHashGet(uHash, &uid, sizeof(uid)) == NULL) { if (NULL == uHash || taosHashGet(uHash, &uid, sizeof(uid)) == NULL) {
taosArrayPush(list, &uid); taosArrayPush(pExistedUidList, &uid);
} }
} else { } else {
taosArrayDestroy(pTbList); taosArrayDestroy(pTbList);
@ -1057,7 +1064,7 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode,
goto _end; goto _end;
} }
if (!pTagCond) { // no tag condition exists, let's fetch all tables of this super table if (!pTagCond) { // no tag filter condition exists, let's fetch all tables of this super table
ASSERT(pTagIndexCond == NULL); ASSERT(pTagIndexCond == NULL);
vnodeGetCtbIdList(pVnode, pScanNode->suid, res); vnodeGetCtbIdList(pVnode, pScanNode->suid, res);
} else { } else {