xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

refactor: opt tag filter perf.

This commit is contained in:
Haojun Liao 2023-02-02 10:11:36 +08:00
parent 49dedc3c3f
commit 6a5ce7d663
3 changed files with 113 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
include/common/tcommon.h
View File

@ -367,6 +367,12 @@ typedef struct SSortExecInfo {
int32_t readBytes; // read io bytes int32_t readBytes; // read io bytes
} SSortExecInfo; } SSortExecInfo;
typedef struct SFilterTableInfo {
char* name;
uint64_t uid;
void* pTagVal;
} SFilterTableInfo;
// stream special block column // stream special block column
#define START_TS_COLUMN_INDEX 0 #define START_TS_COLUMN_INDEX 0

15
source/dnode/vnode/src/meta/metaQuery.c
View File

@ -1346,13 +1346,14 @@ static int32_t metaGetTableTagByUid(SMeta *pMeta, int64_t suid, int64_t uid, voi
return ret; return ret;
} }
int32_t metaGetTableTagsByUids(SMeta *pMeta, int64_t suid, SArray *uidList, SHashObj *tags) { int32_t metaGetTableTagsByUids(SMeta *pMeta, int64_t suid, SArray *uidList, SHashObj *tags) {
const int32_t LIMIT = 128; const int32_t LIMIT = 128;
int32_t isLock = false; int32_t isLock = false;
int32_t sz = uidList ? taosArrayGetSize(uidList) : 0; int32_t sz = uidList ? taosArrayGetSize(uidList) : 0;
for (int i = 0; i < sz; i++) { for (int i = 0; i < sz; i++) {
tb_uid_t *id = taosArrayGet(uidList, i); SFilterTableInfo *p = taosArrayGet(uidList, i);
if (i % LIMIT == 0) { if (i % LIMIT == 0) {
if (isLock) metaULock(pMeta); if (isLock) metaULock(pMeta);
@ -1361,20 +1362,20 @@ int32_t metaGetTableTagsByUids(SMeta *pMeta, int64_t suid, SArray *uidList, SHas
isLock = true; isLock = true;
} }
if (taosHashGet(tags, id, sizeof(tb_uid_t)) == NULL) { // if (taosHashGet(tags, &p->uid, sizeof(tb_uid_t)) == NULL) {
void *val = NULL; void *val = NULL;
int32_t len = 0; int32_t len = 0;
if (metaGetTableTagByUid(pMeta, suid, *id, &val, &len, false) == 0) { if (metaGetTableTagByUid(pMeta, suid, p->uid, &val, &len, false) == 0) {
taosHashPut(tags, id, sizeof(tb_uid_t), val, len); p->pTagVal = taosMemoryMalloc(len);
memcpy(p->pTagVal, val, len);
tdbFree(val); tdbFree(val);
} else { } else {
metaError("vgId:%d, failed to table tags, suid: %" PRId64 ", uid: %" PRId64 "", TD_VID(pMeta->pVnode), suid, metaError("vgId:%d, failed to table tags, suid: %" PRId64 ", uid: %" PRId64 "", TD_VID(pMeta->pVnode), suid,
*id); p->uid);
} }
} }
} // }
if (isLock) metaULock(pMeta); if (isLock) metaULock(pMeta);
return 0; return 0;
} }

163
source/libs/executor/src/executil.c
View File

@ -44,8 +44,8 @@ typedef struct tagFilterAssist {
} tagFilterAssist; } tagFilterAssist;
static int32_t removeInvalidUid(SArray* uids, SHashObj* tags); static int32_t removeInvalidUid(SArray* uids, SHashObj* tags);
static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list, SNode* pTagCond, SHashObj* tags); static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* pRes, SNode* pTagCond, SHashObj* tags);
static int32_t optimizeTbnameInCondImpl(void* metaHandle, int64_t suid, SArray* pExistedUidList, SNode* pTagCond); static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* pExistedUidList, SNode* pTagCond);
static int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, static int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond,
SNode* pTagIndexCond, STableListInfo* pListInfo); SNode* pTagIndexCond, STableListInfo* pListInfo);
@ -416,6 +416,48 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, int64_t suid, SArray*
nodesRewriteExprPostOrder(&pTagCond, getColumn, (void*)&ctx); nodesRewriteExprPostOrder(&pTagCond, getColumn, (void*)&ctx);
SDataType type = {.type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)};
// int64_t stt = taosGetTimestampUs();
// tags = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
SArray* pRes = taosArrayInit(10, sizeof(SFilterTableInfo));
int32_t filter = optimizeTbnameInCond(metaHandle, suid, pRes, pTagCond, tags);
if (filter == 0) { // tbname in filter is activated, do nothing and return
int32_t numOfRows = taosArrayGetSize(pRes);
code = createResultData(&type, numOfRows, &output);
if (code != TSDB_CODE_SUCCESS) {
terrno = code;
qError("failed to create result, reason:%s", tstrerror(code));
goto end;
}
bool* b = (bool*)output.columnData->pData;
taosArrayEnsureCap(uidList, numOfRows);
for(int32_t i = 0; i < numOfRows; ++i) {
b[i] = true;
SFilterTableInfo* pInfo = taosArrayGet(pRes, i);
taosArrayPush(uidList, &pInfo->uid);
}
terrno = 0;
goto end;
} else {
// if (filter == -1) {
// here we retrieve all tags from the vnode table-meta store
code = metaGetTableTags(metaHandle, suid, uidList, tags);
if (code != TSDB_CODE_SUCCESS) {
qError("failed to get table tags from meta, reason:%s, suid:%" PRIu64, tstrerror(code), suid);
terrno = code;
goto end;
}
}
if (suid != 0) {
// removeInvalidUid(uidList, tags);
}
pResBlock = createDataBlock(); pResBlock = createDataBlock();
if (pResBlock == NULL) { if (pResBlock == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -428,25 +470,7 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, int64_t suid, SArray*
blockDataAppendColInfo(pResBlock, &colInfo); blockDataAppendColInfo(pResBlock, &colInfo);
} }
// int64_t stt = taosGetTimestampUs(); int32_t size = taosArrayGetSize(pRes);
tags = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
int32_t filter = optimizeTbnameInCond(metaHandle, suid, uidList, pTagCond, tags);
if (filter == -1) {
// here we retrieve all tags from the vnode table-meta store
code = metaGetTableTags(metaHandle, suid, uidList, tags);
if (code != TSDB_CODE_SUCCESS) {
qError("failed to get table tags from meta, reason:%s, suid:%" PRIu64, tstrerror(code), suid);
terrno = code;
goto end;
}
}
if (suid != 0) {
removeInvalidUid(uidList, tags);
}
int32_t size = taosArrayGetSize(uidList);
if (size == 0) { if (size == 0) {
goto end; goto end;
} }
@ -457,27 +481,32 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, int64_t suid, SArray*
goto end; goto end;
} }
int32_t numOfCols = taosArrayGetSize(pResBlock->pDataBlock);
for (int32_t i = 0; i < size; i++) { for (int32_t i = 0; i < size; i++) {
int64_t* uid = taosArrayGet(uidList, i); SFilterTableInfo* p1 = taosArrayGet(pRes, i);
for (int32_t j = 0; j < taosArrayGetSize(pResBlock->pDataBlock); j++) {
for (int32_t j = 0; j < numOfCols; j++) {
SColumnInfoData* pColInfo = (SColumnInfoData*)taosArrayGet(pResBlock->pDataBlock, j); SColumnInfoData* pColInfo = (SColumnInfoData*)taosArrayGet(pResBlock->pDataBlock, j);
if (pColInfo->info.colId == -1) { // tbname if (pColInfo->info.colId == -1) { // tbname
char str[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; char str[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
metaGetTableNameByUid(metaHandle, *uid, str); STR_TO_VARSTR(str, p1->name);
// metaGetTableNameByUid(metaHandle, *uid, str);
colDataAppend(pColInfo, i, str, false); colDataAppend(pColInfo, i, str, false);
#if TAG_FILTER_DEBUG #if TAG_FILTER_DEBUG
qDebug("tagfilter uid:%ld, tbname:%s", *uid, str + 2); qDebug("tagfilter uid:%ld, tbname:%s", *uid, str + 2);
#endif #endif
} else { } else {
void* pTagsVal = taosHashGet(tags, uid, sizeof(uint64_t)); // void* pTagsVal = taosHashGet(tags, uid, sizeof(uint64_t));
if (pTagsVal == NULL) { // if (pTagsVal == NULL) {
continue; // continue;
} // }
STagVal tagVal = {0}; STagVal tagVal = {0};
tagVal.cid = pColInfo->info.colId; tagVal.cid = pColInfo->info.colId;
const char* p = metaGetTableTagVal(pTagsVal, pColInfo->info.type, &tagVal); const char* p = metaGetTableTagVal(p1->pTagVal, pColInfo->info.type, &tagVal);
if (p == NULL || (pColInfo->info.type == TSDB_DATA_TYPE_JSON && ((STag*)p)->nTag == 0)) { if (p == NULL || (pColInfo->info.type == TSDB_DATA_TYPE_JSON && ((STag*)p)->nTag == 0)) {
colDataAppend(pColInfo, i, p, true); colDataAppend(pColInfo, i, p, true);
@ -485,14 +514,12 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, int64_t suid, SArray*
colDataAppend(pColInfo, i, p, false); colDataAppend(pColInfo, i, p, false);
} else if (IS_VAR_DATA_TYPE(pColInfo->info.type)) { } else if (IS_VAR_DATA_TYPE(pColInfo->info.type)) {
char* tmp = alloca(tagVal.nData + VARSTR_HEADER_SIZE + 1); char* tmp = alloca(tagVal.nData + VARSTR_HEADER_SIZE + 1);
// char* tmp = taosMemoryCalloc(tagVal.nData + VARSTR_HEADER_SIZE + 1, 1);
varDataSetLen(tmp, tagVal.nData); varDataSetLen(tmp, tagVal.nData);
memcpy(tmp + VARSTR_HEADER_SIZE, tagVal.pData, tagVal.nData); memcpy(tmp + VARSTR_HEADER_SIZE, tagVal.pData, tagVal.nData);
colDataAppend(pColInfo, i, tmp, false); colDataAppend(pColInfo, i, tmp, false);
#if TAG_FILTER_DEBUG #if TAG_FILTER_DEBUG
qDebug("tagfilter varch:%s", tmp + 2); qDebug("tagfilter varch:%s", tmp + 2);
#endif #endif
// taosMemoryFree(tmp);
} else { } else {
colDataAppend(pColInfo, i, (const char*)&tagVal.i64, false); colDataAppend(pColInfo, i, (const char*)&tagVal.i64, false);
#if TAG_FILTER_DEBUG #if TAG_FILTER_DEBUG
@ -515,14 +542,6 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, int64_t suid, SArray*
pBlockList = taosArrayInit(2, POINTER_BYTES); pBlockList = taosArrayInit(2, POINTER_BYTES);
taosArrayPush(pBlockList, &pResBlock); taosArrayPush(pBlockList, &pResBlock);
SDataType type = {.type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)};
code = createResultData(&type, size, &output);
if (code != TSDB_CODE_SUCCESS) {
terrno = code;
qError("failed to create result, reason:%s", tstrerror(code));
goto end;
}
code = scalarCalculate(pTagCond, pBlockList, &output); code = scalarCalculate(pTagCond, pBlockList, &output);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
qError("failed to calculate scalar, reason:%s", tstrerror(code)); qError("failed to calculate scalar, reason:%s", tstrerror(code));
@ -847,13 +866,26 @@ static int tableUidCompare(const void* a, const void* b) {
return u1 < u2 ? -1 : 1; return u1 < u2 ? -1 : 1;
} }
static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list, SNode* cond, SHashObj* tags) { static int32_t filterTableInfoCompare(const void* a, const void* b) {
SFilterTableInfo* p1 = (SFilterTableInfo*) a;
SFilterTableInfo* p2 = (SFilterTableInfo*) b;
if (p1->uid == p2->uid) {
return 0;
}
return p1->uid < p2->uid? -1:1;
}
static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* pRes, SNode* cond, SHashObj* tags) {
int32_t ret = -1; int32_t ret = -1;
if (nodeType(cond) == QUERY_NODE_OPERATOR) { if (nodeType(cond) == QUERY_NODE_OPERATOR) {
ret = optimizeTbnameInCondImpl(metaHandle, suid, list, cond); ret = optimizeTbnameInCondImpl(metaHandle, pRes, cond);
if (ret != -1) { if (ret == 0) {
metaGetTableTagsByUids(metaHandle, suid, list, tags); // metaGetTableTagsByUids(metaHandle, suid, pRes, tags);
removeInvalidUid(list, tags); // removeInvalidUid(pRes, tags);
} else { // ret == -1
// do nothing
} }
} }
@ -873,19 +905,19 @@ static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* list
SListCell* cell = pList->pHead; SListCell* cell = pList->pHead;
for (int i = 0; i < len; i++) { for (int i = 0; i < len; i++) {
if (cell == NULL) break; if (cell == NULL) break;
if (optimizeTbnameInCondImpl(metaHandle, suid, list, cell->pNode) == 0) { if (optimizeTbnameInCondImpl(metaHandle, pRes, cell->pNode) == 0) {
hasTbnameCond = true; hasTbnameCond = true;
break; break;
} }
cell = cell->pNext; cell = cell->pNext;
} }
taosArraySort(list, tableUidCompare); taosArraySort(pRes, filterTableInfoCompare);
taosArrayRemoveDuplicate(list, tableUidCompare, NULL); taosArrayRemoveDuplicate(pRes, filterTableInfoCompare, NULL);
if (hasTbnameCond) { if (hasTbnameCond) {
ret = metaGetTableTagsByUids(metaHandle, suid, list, tags); ret = metaGetTableTagsByUids(metaHandle, suid, pRes, tags);
removeInvalidUid(list, tags); removeInvalidUid(pRes, tags);
} }
return ret; return ret;
@ -900,12 +932,12 @@ static int32_t removeInvalidUid(SArray* uids, SHashObj* tags) {
return 0; return 0;
} }
SArray* validUid = taosArrayInit(size, sizeof(int64_t)); SArray* validUid = taosArrayInit(size, sizeof(SFilterTableInfo));
for (int32_t i = 0; i < size; i++) { for (int32_t i = 0; i < size; i++) {
int64_t* uid = taosArrayGet(uids, i); SFilterTableInfo* p = taosArrayGet(uids, i);
if (taosHashGet(tags, uid, sizeof(int64_t)) != NULL) { if (taosHashGet(tags, &p->uid, sizeof(int64_t)) != NULL) {
taosArrayPush(validUid, uid); taosArrayPush(validUid, p);
} }
} }
@ -915,7 +947,7 @@ static int32_t removeInvalidUid(SArray* uids, SHashObj* tags) {
} }
// only return uid that does not contained in pExistedUidList // only return uid that does not contained in pExistedUidList
static int32_t optimizeTbnameInCondImpl(void* metaHandle, int64_t suid, SArray* pExistedUidList, SNode* pTagCond) { static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* pExistedUidList, SNode* pTagCond) {
if (nodeType(pTagCond) != QUERY_NODE_OPERATOR) { if (nodeType(pTagCond) != QUERY_NODE_OPERATOR) {
return -1; return -1;
} }
@ -938,12 +970,13 @@ static int32_t optimizeTbnameInCondImpl(void* metaHandle, int64_t suid, SArray*
SArray* pTbList = getTableNameList(pList); SArray* pTbList = getTableNameList(pList);
int32_t numOfTables = taosArrayGetSize(pTbList); int32_t numOfTables = taosArrayGetSize(pTbList);
SHashObj* uHash = NULL; SHashObj* uHash = NULL;
size_t numOfExisted = taosArrayGetSize(pExistedUidList); // len > 0 means there already have uids size_t numOfExisted = taosArrayGetSize(pExistedUidList); // len > 0 means there already have uids
if (numOfExisted > 0) { if (numOfExisted > 0) {
uHash = taosHashInit(numOfExisted / 0.7, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); uHash = taosHashInit(numOfExisted / 0.7, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
for (int i = 0; i < numOfExisted; i++) { for (int i = 0; i < numOfExisted; i++) {
int64_t* uid = taosArrayGet(pExistedUidList, i); SFilterTableInfo* pTInfo = taosArrayGet(pExistedUidList, i);
taosHashPut(uHash, uid, sizeof(int64_t), &i, sizeof(i)); taosHashPut(uHash, &pTInfo->uid, sizeof(uint64_t), &i, sizeof(i));
} }
} }
@ -955,7 +988,8 @@ static int32_t optimizeTbnameInCondImpl(void* metaHandle, int64_t suid, SArray*
ETableType tbType = TSDB_TABLE_MAX; ETableType tbType = TSDB_TABLE_MAX;
if (metaGetTableTypeByName(metaHandle, name, &tbType) == 0 && tbType == TSDB_CHILD_TABLE) { if (metaGetTableTypeByName(metaHandle, name, &tbType) == 0 && tbType == TSDB_CHILD_TABLE) {
if (NULL == uHash || taosHashGet(uHash, &uid, sizeof(uid)) == NULL) { if (NULL == uHash || taosHashGet(uHash, &uid, sizeof(uid)) == NULL) {
taosArrayPush(pExistedUidList, &uid); SFilterTableInfo s = {.uid = uid, .name = name, .pTagVal = NULL};
taosArrayPush(pExistedUidList, &s);
} }
} else { } else {
taosArrayDestroy(pTbList); taosArrayDestroy(pTbList);
@ -992,39 +1026,40 @@ static void genTagFilterDigest(const SNode* pTagCond, T_MD5_CTX* pContext) {
taosMemoryFree(payload); taosMemoryFree(payload);
} }
static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* res, SNode* pTagCond, void* metaHandle) { static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pRes, SNode* pTagCond, void* metaHandle) {
if (pTagCond == NULL) { if (pTagCond == NULL) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
terrno = TDB_CODE_SUCCESS; terrno = TDB_CODE_SUCCESS;
SColumnInfoData* pColInfoData = getColInfoResult(metaHandle, pListInfo->suid, res, pTagCond); SColumnInfoData* pColInfoData = getColInfoResult(metaHandle, pListInfo->suid, pRes, pTagCond);
if (terrno != TDB_CODE_SUCCESS) { if (terrno != TDB_CODE_SUCCESS) {
colDataDestroy(pColInfoData); colDataDestroy(pColInfoData);
taosMemoryFreeClear(pColInfoData); taosMemoryFreeClear(pColInfoData);
taosArrayDestroy(res); taosArrayDestroy(pRes);
qError("failed to getColInfoResult, code: %s", tstrerror(terrno)); qError("failed to getColInfoResult, code: %s", tstrerror(terrno));
return terrno; return terrno;
} }
int32_t i = 0; int32_t i = 0;
int32_t len = taosArrayGetSize(res); int32_t len = taosArrayGetSize(pRes);
if (pColInfoData != NULL) { if (pColInfoData != NULL) {
bool* pResult = (bool*)pColInfoData->pData; bool* pResult = (bool*)pColInfoData->pData;
SArray* p = taosArrayInit(taosArrayGetSize(res), sizeof(uint64_t)); SArray* p = taosArrayInit(taosArrayGetSize(pRes), sizeof(uint64_t));
while (i < len && pColInfoData) { while (i < len && pColInfoData) {
int64_t* uid = taosArrayGet(res, i); int64_t* uid = taosArrayGet(pRes, i);
qDebug("tagfilter get uid:%" PRId64 ", res:%d", *uid, pResult[i]); qDebug("tagfilter get uid:%" PRId64 ", res:%d", *uid, pResult[i]);
if (pResult[i]) { if (pResult[i]) {
taosArrayPush(p, uid); taosArrayPush(p, uid);
} }
i += 1; i += 1;
} }
taosArraySwap(res, p); taosArraySwap(pRes, p);
taosArrayDestroy(p); taosArrayDestroy(p);
} }