xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix(query): set correct tag value during tag filter and do some internal refactor.

This commit is contained in:
Haojun Liao 2023-02-10 16:03:11 +08:00
parent e46657ada2
commit 72287a3b5f
5 changed files with 297 additions and 157 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
include/common/tcommon.h
View File

@ -367,11 +367,11 @@ typedef struct SSortExecInfo {
int32_t readBytes; // read io bytes int32_t readBytes; // read io bytes
} SSortExecInfo; } SSortExecInfo;
typedef struct SFilterTableInfo { typedef struct STUidTagInfo {
char* name; char* name;
uint64_t uid; uint64_t uid;
void* pTagVal; void* pTagVal;
} SFilterTableInfo; } STUidTagInfo;
// stream special block column // stream special block column

5
source/common/src/tdatablock.c
View File

@ -1546,7 +1546,10 @@ size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize) {
} }
void colDataDestroy(SColumnInfoData* pColData) { void colDataDestroy(SColumnInfoData* pColData) {
if (!pColData) return; if (!pColData) {
return;
}
if (IS_VAR_DATA_TYPE(pColData->info.type)) { if (IS_VAR_DATA_TYPE(pColData->info.type)) {
taosMemoryFreeClear(pColData->varmeta.offset); taosMemoryFreeClear(pColData->varmeta.offset);
} else { } else {

2
source/dnode/vnode/inc/vnode.h
View File

@ -104,7 +104,7 @@ void metaReaderClear(SMetaReader *pReader);
int32_t metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid); int32_t metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid);
int32_t metaGetTableEntryByUidCache(SMetaReader *pReader, tb_uid_t uid); int32_t metaGetTableEntryByUidCache(SMetaReader *pReader, tb_uid_t uid);
int metaGetTableEntryByName(SMetaReader *pReader, const char *name); int metaGetTableEntryByName(SMetaReader *pReader, const char *name);
int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList, SHashObj *tags); int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList);
int32_t metaGetTableTagsByUids(SMeta *pMeta, int64_t suid, SArray *uidList); int32_t metaGetTableTagsByUids(SMeta *pMeta, int64_t suid, SArray *uidList);
int32_t metaReadNext(SMetaReader *pReader); int32_t metaReadNext(SMetaReader *pReader);
const void *metaGetTableTagVal(void *tag, int16_t type, STagVal *tagVal); const void *metaGetTableTagVal(void *tag, int16_t type, STagVal *tagVal);

26
source/dnode/vnode/src/meta/metaQuery.c
View File

@ -1378,7 +1378,7 @@ int32_t metaGetTableTagsByUids(SMeta *pMeta, int64_t suid, SArray *uidList) {
int32_t isLock = false; int32_t isLock = false;
int32_t sz = uidList ? taosArrayGetSize(uidList) : 0; int32_t sz = uidList ? taosArrayGetSize(uidList) : 0;
for (int i = 0; i < sz; i++) { for (int i = 0; i < sz; i++) {
SFilterTableInfo *p = taosArrayGet(uidList, i); STUidTagInfo *p = taosArrayGet(uidList, i);
if (i % LIMIT == 0) { if (i % LIMIT == 0) {
if (isLock) metaULock(pMeta); if (isLock) metaULock(pMeta);
@ -1404,18 +1404,18 @@ int32_t metaGetTableTagsByUids(SMeta *pMeta, int64_t suid, SArray *uidList) {
return 0; return 0;
} }
int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList, SHashObj *tags) { int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *pUidTagInfo) {
SMCtbCursor *pCur = metaOpenCtbCursor(pMeta, suid, 1); SMCtbCursor *pCur = metaOpenCtbCursor(pMeta, suid, 1);
// If len > 0 means there already have uids, and we only want the // If len > 0 means there already have uids, and we only want the
// tags of the specified tables, of which uid in the uid list. Otherwise, all table tags are retrieved and kept // tags of the specified tables, of which uid in the uid list. Otherwise, all table tags are retrieved and kept
// in the hash map, that may require a lot of memory // in the hash map, that may require a lot of memory
SHashObj *pSepecifiedUidMap = NULL; SHashObj *pSepecifiedUidMap = NULL;
size_t len = taosArrayGetSize(uidList); size_t numOfElems = taosArrayGetSize(pUidTagInfo);
if (len > 0) { if (numOfElems > 0) {
pSepecifiedUidMap = taosHashInit(len / 0.7, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); pSepecifiedUidMap = taosHashInit(numOfElems / 0.7, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
for (int i = 0; i < len; i++) { for (int i = 0; i < numOfElems; i++) {
int64_t *uid = taosArrayGet(uidList, i); int64_t *uid = taosArrayGet(pUidTagInfo, i);
taosHashPut(pSepecifiedUidMap, uid, sizeof(int64_t), 0, 0); taosHashPut(pSepecifiedUidMap, uid, sizeof(int64_t), 0, 0);
} }
} }
@ -1426,13 +1426,15 @@ int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList, SHashObj
break; break;
} }
if (len > 0 && taosHashGet(pSepecifiedUidMap, &uid, sizeof(int64_t)) == NULL) { if (numOfElems > 0 && taosHashGet(pSepecifiedUidMap, &uid, sizeof(int64_t)) == NULL) {
continue; continue;
} else if (len == 0) { } else if (numOfElems == 0) {
taosArrayPush(uidList, &uid); STUidTagInfo info = {.uid = uid, .pTagVal = pCur->pVal};
} info.pTagVal = taosMemoryMalloc(pCur->vLen);
memcpy(info.pTagVal, pCur->pVal, pCur->vLen);
taosHashPut(tags, &uid, sizeof(uint64_t), pCur->pVal, pCur->vLen); taosArrayPush(pUidTagInfo, &info);
}
} }
taosHashCleanup(pSepecifiedUidMap); taosHashCleanup(pSepecifiedUidMap);

417
source/libs/executor/src/executil.c
View File

@ -44,10 +44,11 @@ typedef struct tagFilterAssist {
} tagFilterAssist; } tagFilterAssist;
static int32_t removeInvalidUid(SArray* uids, SHashObj* tags); static int32_t removeInvalidUid(SArray* uids, SHashObj* tags);
static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* pRes, SNode* pTagCond, SHashObj* tags); static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* pRes, SNode* pTagCond);
static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* pExistedUidList, SNode* pTagCond); static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* pExistedUidList, SNode* pTagCond);
static int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, static int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond,
SNode* pTagIndexCond, STableListInfo* pListInfo); SNode* pTagIndexCond, STableListInfo* pListInfo);
static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, SArray* pUidTagList);
static int64_t getLimit(const SNode* pLimit) { return NULL == pLimit ? -1 : ((SLimitNode*)pLimit)->limit; } static int64_t getLimit(const SNode* pLimit) { return NULL == pLimit ? -1 : ((SLimitNode*)pLimit)->limit; }
static int64_t getOffset(const SNode* pLimit) { return NULL == pLimit ? -1 : ((SLimitNode*)pLimit)->offset; } static int64_t getOffset(const SNode* pLimit) { return NULL == pLimit ? -1 : ((SLimitNode*)pLimit)->offset; }
@ -393,7 +394,7 @@ static int32_t createResultData(SDataType* pType, int32_t numOfRows, SScalarPara
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
static SColumnInfoData* getColInfoResult(void* metaHandle, int64_t suid, SArray* uidList, SNode* pTagCond) { static void getColInfoResult(void* metaHandle, int64_t suid, SArray* pUidList, SNode* pTagCond) {
int32_t code = TSDB_CODE_SUCCESS; int32_t code = TSDB_CODE_SUCCESS;
SArray* pBlockList = NULL; SArray* pBlockList = NULL;
SSDataBlock* pResBlock = NULL; SSDataBlock* pResBlock = NULL;
@ -401,7 +402,6 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, int64_t suid, SArray*
SScalarParam output = {0}; SScalarParam output = {0};
tagFilterAssist ctx = {0}; tagFilterAssist ctx = {0};
ctx.colHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_SMALLINT), false, HASH_NO_LOCK); ctx.colHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_SMALLINT), false, HASH_NO_LOCK);
if (ctx.colHash == NULL) { if (ctx.colHash == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -419,10 +419,10 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, int64_t suid, SArray*
SDataType type = {.type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)}; SDataType type = {.type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)};
// int64_t stt = taosGetTimestampUs(); // int64_t stt = taosGetTimestampUs();
SArray* pRes = taosArrayInit(10, sizeof(SFilterTableInfo)); SArray* pUidTagList = taosArrayInit(10, sizeof(STUidTagInfo));
int32_t filter = optimizeTbnameInCond(metaHandle, suid, pRes, pTagCond, tags); int32_t filter = optimizeTbnameInCond(metaHandle, suid, pUidTagList, pTagCond);
if (filter == 0) { // tbname in filter is activated, do nothing and return if (filter == 0) { // tbname in filter is activated, do nothing and return
int32_t numOfRows = taosArrayGetSize(pRes); int32_t numOfRows = taosArrayGetSize(pUidTagList);
code = createResultData(&type, numOfRows, &output); code = createResultData(&type, numOfRows, &output);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
terrno = code; terrno = code;
@ -430,20 +430,26 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, int64_t suid, SArray*
goto end; goto end;
} }
bool* b = (bool*)output.columnData->pData; taosArrayEnsureCap(pUidList, numOfRows);
taosArrayEnsureCap(uidList, numOfRows);
for(int32_t i = 0; i < numOfRows; ++i) { for(int32_t i = 0; i < numOfRows; ++i) {
b[i] = true; STUidTagInfo* pInfo = taosArrayGet(pUidTagList, i);
SFilterTableInfo* pInfo = taosArrayGet(pRes, i); taosArrayPush(pUidList, &pInfo->uid);
taosArrayPush(uidList, &pInfo->uid);
} }
terrno = 0; terrno = 0;
goto end; goto end;
} else { } else {
// here we retrieve all tags from the vnode table-meta store // here we retrieve all tags from the vnode table-meta store
code = metaGetTableTags(metaHandle, suid, uidList, tags); int32_t numOfExisted = taosArrayGetSize(pUidList);
if (numOfExisted) {
for(int32_t i = 0; i < numOfExisted; ++i) {
uint64_t* uid = taosArrayGet(pUidList, i);
STUidTagInfo info = {.uid = *uid};
taosArrayPush(pUidTagList, &info);
}
}
code = metaGetTableTags(metaHandle, suid, pUidTagList);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
qError("failed to get table tags from meta, reason:%s, suid:%" PRIu64, tstrerror(code), suid); qError("failed to get table tags from meta, reason:%s, suid:%" PRIu64, tstrerror(code), suid);
terrno = code; terrno = code;
@ -451,10 +457,6 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, int64_t suid, SArray*
} }
} }
if (suid != 0) {
// removeInvalidUid(uidList, tags);
}
pResBlock = createDataBlock(); pResBlock = createDataBlock();
if (pResBlock == NULL) { if (pResBlock == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY; terrno = TSDB_CODE_OUT_OF_MEMORY;
@ -467,12 +469,12 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, int64_t suid, SArray*
blockDataAppendColInfo(pResBlock, &colInfo); blockDataAppendColInfo(pResBlock, &colInfo);
} }
int32_t size = taosArrayGetSize(pRes); int32_t numOfTables = taosArrayGetSize(pUidTagList);
if (size == 0) { if (numOfTables == 0) {
goto end; goto end;
} }
code = blockDataEnsureCapacity(pResBlock, size); code = blockDataEnsureCapacity(pResBlock, numOfTables);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
terrno = code; terrno = code;
goto end; goto end;
@ -480,8 +482,8 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, int64_t suid, SArray*
int32_t numOfCols = taosArrayGetSize(pResBlock->pDataBlock); int32_t numOfCols = taosArrayGetSize(pResBlock->pDataBlock);
for (int32_t i = 0; i < size; i++) { for (int32_t i = 0; i < numOfTables; i++) {
SFilterTableInfo* p1 = taosArrayGet(pRes, i); STUidTagInfo* p1 = taosArrayGet(pUidTagList, i);
for (int32_t j = 0; j < numOfCols; j++) { for (int32_t j = 0; j < numOfCols; j++) {
SColumnInfoData* pColInfo = (SColumnInfoData*)taosArrayGet(pResBlock->pDataBlock, j); SColumnInfoData* pColInfo = (SColumnInfoData*)taosArrayGet(pResBlock->pDataBlock, j);
@ -496,10 +498,14 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, int64_t suid, SArray*
} else { } else {
STagVal tagVal = {0}; STagVal tagVal = {0};
tagVal.cid = pColInfo->info.colId; tagVal.cid = pColInfo->info.colId;
if (p1->pTagVal == NULL) {
colDataAppendNULL(pColInfo, i);
}
const char* p = metaGetTableTagVal(p1->pTagVal, pColInfo->info.type, &tagVal); const char* p = metaGetTableTagVal(p1->pTagVal, pColInfo->info.type, &tagVal);
if (p == NULL || (pColInfo->info.type == TSDB_DATA_TYPE_JSON && ((STag*)p)->nTag == 0)) { if (p == NULL || (pColInfo->info.type == TSDB_DATA_TYPE_JSON && ((STag*)p)->nTag == 0)) {
colDataAppend(pColInfo, i, p, true); colDataAppendNULL(pColInfo, i);
} else if (pColInfo->info.type == TSDB_DATA_TYPE_JSON) { } else if (pColInfo->info.type == TSDB_DATA_TYPE_JSON) {
colDataAppend(pColInfo, i, p, false); colDataAppend(pColInfo, i, p, false);
} else if (IS_VAR_DATA_TYPE(pColInfo->info.type)) { } else if (IS_VAR_DATA_TYPE(pColInfo->info.type)) {
@ -524,22 +530,39 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, int64_t suid, SArray*
} }
} }
pResBlock->info.rows = size; pResBlock->info.rows = numOfTables;
// int64_t st1 = taosGetTimestampUs(); // int64_t st1 = taosGetTimestampUs();
// qDebug("generate tag block rows:%d, cost:%ld us", rows, st1-st); // qDebug("generate tag block rows:%d, cost:%ld us", rows, st1-st);
pBlockList = taosArrayInit(2, POINTER_BYTES); pBlockList = taosArrayInit(2, POINTER_BYTES);
taosArrayPush(pBlockList, &pResBlock); taosArrayPush(pBlockList, &pResBlock);
code = createResultData(&type, numOfTables, &output);
if (code != TSDB_CODE_SUCCESS) {
terrno = code;
goto end;
}
code = scalarCalculate(pTagCond, pBlockList, &output); code = scalarCalculate(pTagCond, pBlockList, &output);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
qError("failed to calculate scalar, reason:%s", tstrerror(code)); qError("failed to calculate scalar, reason:%s", tstrerror(code));
terrno = code; terrno = code;
goto end; goto end;
} }
// int64_t st2 = taosGetTimestampUs();
// qDebug("calculate tag block rows:%d, cost:%ld us", rows, st2-st1); taosArrayClear(pUidList);
bool* pResult = (bool*)output.columnData->pData;
for(int32_t i = 0; i < numOfTables; ++i) {
uint64_t uid = ((STUidTagInfo*)taosArrayGet(pUidTagList, i))->uid;
qDebug("tagfilter get uid:%" PRId64 ", res:%d", uid, pResult[i]);
if (pResult[i]) {
taosArrayPush(pUidList, &uid);
}
i += 1;
}
end: end:
taosHashCleanup(tags); taosHashCleanup(tags);
@ -547,7 +570,10 @@ end:
taosArrayDestroy(ctx.cInfoList); taosArrayDestroy(ctx.cInfoList);
blockDataDestroy(pResBlock); blockDataDestroy(pResBlock);
taosArrayDestroy(pBlockList); taosArrayDestroy(pBlockList);
return output.columnData;
colDataDestroy(output.columnData);
taosMemoryFreeClear(output.columnData);
// return output.columnData;
} }
static void releaseColInfoData(void* pCol) { static void releaseColInfoData(void* pCol) {
@ -604,77 +630,28 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis
blockDataAppendColInfo(pResBlock, &colInfo); blockDataAppendColInfo(pResBlock, &colInfo);
} }
SArray* pUidTagList = taosArrayInit(8, sizeof(STUidTagInfo));
uidList = taosArrayInit(rows, sizeof(uint64_t)); uidList = taosArrayInit(rows, sizeof(uint64_t));
for (int32_t i = 0; i < rows; ++i) { for (int32_t i = 0; i < rows; ++i) {
STableKeyInfo* pkeyInfo = taosArrayGet(pTableListInfo->pTableList, i); STableKeyInfo* pkeyInfo = taosArrayGet(pTableListInfo->pTableList, i);
taosArrayPush(uidList, &pkeyInfo->uid); STUidTagInfo info = {.uid = pkeyInfo->uid};
taosArrayPush(pUidTagList, &info);
} }
// int64_t stt = taosGetTimestampUs(); // int64_t stt = taosGetTimestampUs();
tags = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); tags = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
code = metaGetTableTags(metaHandle, pTableListInfo->suid, uidList, tags); code = metaGetTableTags(metaHandle, pTableListInfo->suid, pUidTagList);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto end; goto end;
} }
// int64_t stt1 = taosGetTimestampUs(); int32_t numOfTables = taosArrayGetSize(pUidTagList);
// qDebug("generate tag meta rows:%d, cost:%ld us", rows, stt1-stt); pResBlock = createTagValBlockForFilter(ctx.cInfoList, numOfTables, pUidTagList);
code = blockDataEnsureCapacity(pResBlock, rows);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
goto end; goto end;
} }
// int64_t st = taosGetTimestampUs();
for (int32_t i = 0; i < rows; i++) {
int64_t* uid = taosArrayGet(uidList, i);
for (int32_t j = 0; j < taosArrayGetSize(pResBlock->pDataBlock); j++) {
SColumnInfoData* pColInfo = (SColumnInfoData*)taosArrayGet(pResBlock->pDataBlock, j);
if (pColInfo->info.colId == -1) { // tbname
char str[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
metaGetTableNameByUid(metaHandle, *uid, str);
colDataAppend(pColInfo, i, str, false);
#if TAG_FILTER_DEBUG
qDebug("tagfilter uid:%ld, tbname:%s", *uid, str + 2);
#endif
} else {
void* tag = taosHashGet(tags, uid, sizeof(int64_t));
ASSERT(tag);
STagVal tagVal = {0};
tagVal.cid = pColInfo->info.colId;
const char* p = metaGetTableTagVal(tag, pColInfo->info.type, &tagVal);
if (p == NULL || (pColInfo->info.type == TSDB_DATA_TYPE_JSON && ((STag*)p)->nTag == 0)) {
colDataAppend(pColInfo, i, p, true);
} else if (pColInfo->info.type == TSDB_DATA_TYPE_JSON) {
colDataAppend(pColInfo, i, p, false);
} else if (IS_VAR_DATA_TYPE(pColInfo->info.type)) {
char* tmp = taosMemoryCalloc(tagVal.nData + VARSTR_HEADER_SIZE + 1, 1);
varDataSetLen(tmp, tagVal.nData);
memcpy(tmp + VARSTR_HEADER_SIZE, tagVal.pData, tagVal.nData);
colDataAppend(pColInfo, i, tmp, false);
#if TAG_FILTER_DEBUG
qDebug("tagfilter varch:%s", tmp + 2);
#endif
taosMemoryFree(tmp);
} else {
colDataAppend(pColInfo, i, (const char*)&tagVal.i64, false);
#if TAG_FILTER_DEBUG
if (pColInfo->info.type == TSDB_DATA_TYPE_INT) {
qDebug("tagfilter int:%d", *(int*)(&tagVal.i64));
} else if (pColInfo->info.type == TSDB_DATA_TYPE_DOUBLE) {
qDebug("tagfilter double:%f", *(double*)(&tagVal.i64));
}
#endif
}
}
}
}
pResBlock->info.rows = rows;
// int64_t st1 = taosGetTimestampUs(); // int64_t st1 = taosGetTimestampUs();
// qDebug("generate tag block rows:%d, cost:%ld us", rows, st1-st); // qDebug("generate tag block rows:%d, cost:%ld us", rows, st1-st);
@ -857,8 +834,8 @@ static int tableUidCompare(const void* a, const void* b) {
} }
static int32_t filterTableInfoCompare(const void* a, const void* b) { static int32_t filterTableInfoCompare(const void* a, const void* b) {
SFilterTableInfo* p1 = (SFilterTableInfo*) a; STUidTagInfo* p1 = (STUidTagInfo*) a;
SFilterTableInfo* p2 = (SFilterTableInfo*) b; STUidTagInfo* p2 = (STUidTagInfo*) b;
if (p1->uid == p2->uid) { if (p1->uid == p2->uid) {
return 0; return 0;
@ -867,7 +844,7 @@ static int32_t filterTableInfoCompare(const void* a, const void* b) {
return p1->uid < p2->uid? -1:1; return p1->uid < p2->uid? -1:1;
} }
static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* pRes, SNode* cond, SHashObj* tags) { static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* pRes, SNode* cond) {
int32_t ret = -1; int32_t ret = -1;
int32_t ntype = nodeType(cond); int32_t ntype = nodeType(cond);
@ -903,12 +880,13 @@ static int32_t optimizeTbnameInCond(void* metaHandle, int64_t suid, SArray* pRes
if (hasTbnameCond) { if (hasTbnameCond) {
ret = metaGetTableTagsByUids(metaHandle, suid, pRes); ret = metaGetTableTagsByUids(metaHandle, suid, pRes);
removeInvalidUid(pRes, tags); // removeInvalidUid(pRes, tags);
} }
return ret; return ret;
} }
#if 0
/* /*
* handle invalid uid * handle invalid uid
*/ */
@ -918,10 +896,10 @@ static int32_t removeInvalidUid(SArray* uids, SHashObj* tags) {
return 0; return 0;
} }
SArray* validUid = taosArrayInit(size, sizeof(SFilterTableInfo)); SArray* validUid = taosArrayInit(size, sizeof(STUidTagInfo));
for (int32_t i = 0; i < size; i++) { for (int32_t i = 0; i < size; i++) {
SFilterTableInfo* p = taosArrayGet(uids, i); STUidTagInfo* p = taosArrayGet(uids, i);
if (taosHashGet(tags, &p->uid, sizeof(int64_t)) != NULL) { if (taosHashGet(tags, &p->uid, sizeof(int64_t)) != NULL) {
taosArrayPush(validUid, p); taosArrayPush(validUid, p);
} }
@ -932,6 +910,8 @@ static int32_t removeInvalidUid(SArray* uids, SHashObj* tags) {
return 0; return 0;
} }
#endif
// only return uid that does not contained in pExistedUidList // only return uid that does not contained in pExistedUidList
static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* pExistedUidList, SNode* pTagCond) { static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* pExistedUidList, SNode* pTagCond) {
if (nodeType(pTagCond) != QUERY_NODE_OPERATOR) { if (nodeType(pTagCond) != QUERY_NODE_OPERATOR) {
@ -961,7 +941,7 @@ static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* pExistedUidLis
if (numOfExisted > 0) { if (numOfExisted > 0) {
uHash = taosHashInit(numOfExisted / 0.7, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); uHash = taosHashInit(numOfExisted / 0.7, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
for (int i = 0; i < numOfExisted; i++) { for (int i = 0; i < numOfExisted; i++) {
SFilterTableInfo* pTInfo = taosArrayGet(pExistedUidList, i); STUidTagInfo* pTInfo = taosArrayGet(pExistedUidList, i);
taosHashPut(uHash, &pTInfo->uid, sizeof(uint64_t), &i, sizeof(i)); taosHashPut(uHash, &pTInfo->uid, sizeof(uint64_t), &i, sizeof(i));
} }
} }
@ -974,7 +954,7 @@ static int32_t optimizeTbnameInCondImpl(void* metaHandle, SArray* pExistedUidLis
ETableType tbType = TSDB_TABLE_MAX; ETableType tbType = TSDB_TABLE_MAX;
if (metaGetTableTypeByName(metaHandle, name, &tbType) == 0 && tbType == TSDB_CHILD_TABLE) { if (metaGetTableTypeByName(metaHandle, name, &tbType) == 0 && tbType == TSDB_CHILD_TABLE) {
if (NULL == uHash || taosHashGet(uHash, &uid, sizeof(uid)) == NULL) { if (NULL == uHash || taosHashGet(uHash, &uid, sizeof(uid)) == NULL) {
SFilterTableInfo s = {.uid = uid, .name = name, .pTagVal = NULL}; STUidTagInfo s = {.uid = uid, .name = name, .pTagVal = NULL};
taosArrayPush(pExistedUidList, &s); taosArrayPush(pExistedUidList, &s);
} }
} else { } else {
@ -1012,45 +992,198 @@ static void genTagFilterDigest(const SNode* pTagCond, T_MD5_CTX* pContext) {
taosMemoryFree(payload); taosMemoryFree(payload);
} }
static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pRes, SNode* pTagCond, void* metaHandle) { static SSDataBlock* createTagValBlockForFilter(SArray* pColList, int32_t numOfTables, SArray* pUidTagList) {
SSDataBlock* pResBlock = createDataBlock();
if (pResBlock == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
for (int32_t i = 0; i < taosArrayGetSize(pColList); ++i) {
SColumnInfoData colInfo = {0};
colInfo.info = *(SColumnInfo*)taosArrayGet(pColList, i);
blockDataAppendColInfo(pResBlock, &colInfo);
}
int32_t code = blockDataEnsureCapacity(pResBlock, numOfTables);
if (code != TSDB_CODE_SUCCESS) {
terrno = code;
return NULL;
}
pResBlock->info.rows = numOfTables;
int32_t numOfCols = taosArrayGetSize(pResBlock->pDataBlock);
for (int32_t i = 0; i < numOfTables; i++) {
STUidTagInfo* p1 = taosArrayGet(pUidTagList, i);
for (int32_t j = 0; j < numOfCols; j++) {
SColumnInfoData* pColInfo = (SColumnInfoData*)taosArrayGet(pResBlock->pDataBlock, j);
if (pColInfo->info.colId == -1) { // tbname
char str[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0};
STR_TO_VARSTR(str, p1->name);
colDataAppend(pColInfo, i, str, false);
#if TAG_FILTER_DEBUG
qDebug("tagfilter uid:%ld, tbname:%s", *uid, str + 2);
#endif
} else {
STagVal tagVal = {0};
tagVal.cid = pColInfo->info.colId;
if (p1->pTagVal == NULL) {
colDataAppendNULL(pColInfo, i);
}
const char* p = metaGetTableTagVal(p1->pTagVal, pColInfo->info.type, &tagVal);
if (p == NULL || (pColInfo->info.type == TSDB_DATA_TYPE_JSON && ((STag*)p)->nTag == 0)) {
colDataAppendNULL(pColInfo, i);
} else if (pColInfo->info.type == TSDB_DATA_TYPE_JSON) {
colDataAppend(pColInfo, i, p, false);
} else if (IS_VAR_DATA_TYPE(pColInfo->info.type)) {
char* tmp = alloca(tagVal.nData + VARSTR_HEADER_SIZE + 1);
varDataSetLen(tmp, tagVal.nData);
memcpy(tmp + VARSTR_HEADER_SIZE, tagVal.pData, tagVal.nData);
colDataAppend(pColInfo, i, tmp, false);
#if TAG_FILTER_DEBUG
qDebug("tagfilter varch:%s", tmp + 2);
#endif
} else {
colDataAppend(pColInfo, i, (const char*)&tagVal.i64, false);
#if TAG_FILTER_DEBUG
if (pColInfo->info.type == TSDB_DATA_TYPE_INT) {
qDebug("tagfilter int:%d", *(int*)(&tagVal.i64));
} else if (pColInfo->info.type == TSDB_DATA_TYPE_DOUBLE) {
qDebug("tagfilter double:%f", *(double*)(&tagVal.i64));
}
#endif
}
}
}
}
return pResBlock;
}
static void doSetQualifiedUid(SArray* pUidList, const SArray* pUidTagList, bool* pResultList) {
taosArrayClear(pUidList);
int32_t numOfTables = taosArrayGetSize(pUidTagList);
for(int32_t i = 0; i < numOfTables; ++i) {
uint64_t uid = ((STUidTagInfo*)taosArrayGet(pUidTagList, i))->uid;
qDebug("tagfilter get uid:%" PRId64 ", res:%d", uid, pResultList[i]);
if (pResultList[i]) {
taosArrayPush(pUidList, &uid);
}
i += 1;
}
}
static void copyExistedUids(SArray* pUidTagList, const SArray* pUidList) {
int32_t numOfExisted = taosArrayGetSize(pUidList);
if (numOfExisted) {
for(int32_t i = 0; i < numOfExisted; ++i) {
uint64_t* uid = taosArrayGet(pUidList, i);
STUidTagInfo info = {.uid = *uid};
taosArrayPush(pUidTagList, &info);
}
}
}
static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SNode* pTagCond, void* metaHandle) {
if (pTagCond == NULL) { if (pTagCond == NULL) {
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
terrno = TDB_CODE_SUCCESS; terrno = TDB_CODE_SUCCESS;
SColumnInfoData* pColInfoData = getColInfoResult(metaHandle, pListInfo->suid, pRes, pTagCond);
if (terrno != TDB_CODE_SUCCESS) { int32_t code = TSDB_CODE_SUCCESS;
colDataDestroy(pColInfoData); SArray* pBlockList = NULL;
taosMemoryFreeClear(pColInfoData); SSDataBlock* pResBlock = NULL;
taosArrayDestroy(pRes); SScalarParam output = {0};
qError("failed to getColInfoResult, code: %s", tstrerror(terrno));
return terrno; tagFilterAssist ctx = {0};
ctx.colHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_SMALLINT), false, HASH_NO_LOCK);
if (ctx.colHash == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto end;
} }
int32_t i = 0; ctx.cInfoList = taosArrayInit(4, sizeof(SColumnInfo));
int32_t len = taosArrayGetSize(pRes); if (ctx.cInfoList == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto end;
}
if (pColInfoData != NULL) { nodesRewriteExprPostOrder(&pTagCond, getColumn, (void*)&ctx);
bool* pResult = (bool*)pColInfoData->pData;
SArray* p = taosArrayInit(taosArrayGetSize(pRes), sizeof(uint64_t));
while (i < len && pColInfoData) { SDataType type = {.type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)};
int64_t* uid = taosArrayGet(pRes, i);
qDebug("tagfilter get uid:%" PRId64 ", res:%d", *uid, pResult[i]);
if (pResult[i]) { // int64_t stt = taosGetTimestampUs();
taosArrayPush(p, uid); SArray* pUidTagList = taosArrayInit(10, sizeof(STUidTagInfo));
} int32_t filter = optimizeTbnameInCond(metaHandle, pListInfo->suid, pUidTagList, pTagCond);
if (filter == 0) { // tbname in filter is activated, do nothing and return
i += 1; int32_t numOfRows = taosArrayGetSize(pUidTagList);
taosArrayEnsureCap(pUidList, numOfRows);
for(int32_t i = 0; i < numOfRows; ++i) {
STUidTagInfo* pInfo = taosArrayGet(pUidTagList, i);
taosArrayPush(pUidList, &pInfo->uid);
} }
taosArraySwap(pRes, p); terrno = 0;
taosArrayDestroy(p); goto end;
} else {
// here we retrieve all tags from the vnode table-meta store
copyExistedUids(pUidTagList, pUidList);
code = metaGetTableTags(metaHandle, pListInfo->suid, pUidTagList);
if (code != TSDB_CODE_SUCCESS) {
qError("failed to get table tags from meta, reason:%s, suid:%" PRIu64, tstrerror(code), pListInfo->suid);
terrno = code;
goto end;
}
} }
colDataDestroy(pColInfoData); int32_t numOfTables = taosArrayGetSize(pUidTagList);
taosMemoryFreeClear(pColInfoData); if (numOfTables == 0) {
goto end;
}
pResBlock = createTagValBlockForFilter(ctx.cInfoList, numOfTables, pUidTagList);
if (code != TSDB_CODE_SUCCESS) {
goto end;
}
// int64_t st1 = taosGetTimestampUs();
// qDebug("generate tag block rows:%d, cost:%ld us", rows, st1-st);
pBlockList = taosArrayInit(2, POINTER_BYTES);
taosArrayPush(pBlockList, &pResBlock);
code = createResultData(&type, numOfTables, &output);
if (code != TSDB_CODE_SUCCESS) {
terrno = code;
goto end;
}
code = scalarCalculate(pTagCond, pBlockList, &output);
if (code != TSDB_CODE_SUCCESS) {
qError("failed to calculate scalar, reason:%s", tstrerror(code));
terrno = code;
goto end;
}
doSetQualifiedUid(pUidList, pUidTagList, (bool*) output.columnData->pData);
end:
taosHashCleanup(ctx.colHash);
taosArrayDestroy(ctx.cInfoList);
blockDataDestroy(pResBlock);
taosArrayDestroy(pBlockList);
colDataDestroy(output.columnData);
taosMemoryFreeClear(output.columnData);
return TSDB_CODE_SUCCESS; return TSDB_CODE_SUCCESS;
} }
@ -1062,36 +1195,37 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode,
uint64_t tableUid = pScanNode->uid; uint64_t tableUid = pScanNode->uid;
pListInfo->suid = pScanNode->suid; pListInfo->suid = pScanNode->suid;
SArray* res = taosArrayInit(8, sizeof(uint64_t));
SArray* pRes = taosArrayInit(8, sizeof(uint64_t));
if (pScanNode->tableType != TSDB_SUPER_TABLE) { if (pScanNode->tableType != TSDB_SUPER_TABLE) {
if (metaIsTableExist(metaHandle, tableUid)) { if (metaIsTableExist(metaHandle, tableUid)) {
taosArrayPush(res, &tableUid); taosArrayPush(pRes, &tableUid);
} }
code = doFilterByTagCond(pListInfo, res, pTagCond, metaHandle); code = doFilterByTagCond(pListInfo, pRes, pTagCond, metaHandle);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return code; return code;
} }
} else { } else {
T_MD5_CTX context = {0}; T_MD5_CTX context = {0};
if (tsTagFilterCache) { if (tsTagFilterCache) {
// try to retrieve the result from meta cache // try to retrieve the result from meta cache
genTagFilterDigest(pTagCond, &context); genTagFilterDigest(pTagCond, &context);
SArray* pUidList = taosArrayInit(8, sizeof(uint64_t));
bool acquired = false; bool acquired = false;
metaGetCachedTableUidList(metaHandle, pScanNode->suid, context.digest, tListLen(context.digest), res, &acquired); metaGetCachedTableUidList(metaHandle, pScanNode->suid, context.digest, tListLen(context.digest), pUidList, &acquired);
if (acquired) { if (acquired) {
qDebug("retrieve table uid list from cache, numOfTables:%d", (int32_t)taosArrayGetSize(res)); qDebug("retrieve table uid list from cache, numOfTables:%d", (int32_t)taosArrayGetSize(pUidList));
goto _end; goto _end;
} }
} }
if (!pTagCond) { // no tag filter condition exists, let's fetch all tables of this super table if (!pTagCond) { // no tag filter condition exists, let's fetch all tables of this super table
ASSERT(pTagIndexCond == NULL); ASSERT(pTagIndexCond == NULL);
vnodeGetCtbIdList(pVnode, pScanNode->suid, res); vnodeGetCtbIdList(pVnode, pScanNode->suid, pRes);
} else { } else {
// failed to find the result in the cache, let try to calculate the results // failed to find the result in the cache, let try to calculate the results
if (pTagIndexCond) { if (pTagIndexCond) {
@ -1099,7 +1233,7 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode,
.metaEx = metaHandle, .idx = tsdbGetIdx(metaHandle), .ivtIdx = tsdbGetIvtIdx(metaHandle), .suid = tableUid}; .metaEx = metaHandle, .idx = tsdbGetIdx(metaHandle), .ivtIdx = tsdbGetIvtIdx(metaHandle), .suid = tableUid};
SIdxFltStatus status = SFLT_NOT_INDEX; SIdxFltStatus status = SFLT_NOT_INDEX;
code = doFilterTag(pTagIndexCond, &metaArg, res, &status); code = doFilterTag(pTagIndexCond, &metaArg, pRes, &status);
if (code != 0 || status == SFLT_NOT_INDEX) { // temporarily disable it for performance sake if (code != 0 || status == SFLT_NOT_INDEX) { // temporarily disable it for performance sake
// qError("failed to get tableIds from index, reason:%s, suid:%" PRIu64, tstrerror(code), tableUid); // qError("failed to get tableIds from index, reason:%s, suid:%" PRIu64, tstrerror(code), tableUid);
code = TDB_CODE_SUCCESS; code = TDB_CODE_SUCCESS;
@ -1107,43 +1241,44 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode,
} }
} }
code = doFilterByTagCond(pListInfo, res, pTagCond, metaHandle); code = doFilterByTagCond(pListInfo, pRes, pTagCond, metaHandle);
if (code != TSDB_CODE_SUCCESS) { if (code != TSDB_CODE_SUCCESS) {
return code; return code;
} }
// let's add the filter results into meta-cache // let's add the filter results into meta-cache
numOfTables = taosArrayGetSize(res); numOfTables = taosArrayGetSize(pRes);
size_t size = numOfTables * sizeof(uint64_t) + sizeof(int32_t);
char* pPayload = taosMemoryMalloc(size);
*(int32_t*)pPayload = numOfTables;
if (numOfTables > 0) {
memcpy(pPayload + sizeof(int32_t), taosArrayGet(res, 0), numOfTables * sizeof(uint64_t));
}
if (tsTagFilterCache) { if (tsTagFilterCache) {
metaUidFilterCachePut(metaHandle, pScanNode->suid, context.digest, tListLen(context.digest), pPayload, size, 1); size_t size = numOfTables * sizeof(uint64_t) + sizeof(int32_t);
} char* pPayload = taosMemoryMalloc(size);
taosMemoryFree(pPayload); // todo convert to uid list
if (numOfTables > 0) {
*(int32_t*)pPayload = numOfTables;
memcpy(pPayload + sizeof(int32_t), taosArrayGet(pRes, 0), numOfTables * sizeof(uint64_t));
}
metaUidFilterCachePut(metaHandle, pScanNode->suid, context.digest, tListLen(context.digest), pPayload, size, 1);
taosMemoryFree(pPayload);
}
} }
_end: _end:
numOfTables = taosArrayGetSize(res); numOfTables = taosArrayGetSize(pRes);
for (int i = 0; i < numOfTables; i++) { for (int i = 0; i < numOfTables; i++) {
STableKeyInfo info = {.uid = *(uint64_t*)taosArrayGet(res, i), .groupId = 0}; STableKeyInfo info = {.uid = *(uint64_t*)taosArrayGet(pRes, i), .groupId = 0};
void* p = taosArrayPush(pListInfo->pTableList, &info); void* p = taosArrayPush(pListInfo->pTableList, &info);
if (p == NULL) { if (p == NULL) {
taosArrayDestroy(res); taosArrayDestroy(pRes);
return TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY;
} }
qTrace("tagfilter get uid:%" PRIu64 "", info.uid); qTrace("tagfilter get uid:%" PRIu64 "", info.uid);
} }
taosArrayDestroy(res); taosArrayDestroy(pRes);
return code; return code;
} }