From e0393347920bf12328688a7824ef227321e30162 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 10 Aug 2022 20:18:54 +0800 Subject: [PATCH 01/33] opti: logic in get table list --- source/common/src/tdatablock.c | 2 + source/libs/executor/src/executil.c | 154 ++++++++++++++++++++++++++-- 2 files changed, 148 insertions(+), 8 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index dba30bb876..7692e02a37 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1343,12 +1343,14 @@ SSDataBlock* createDataBlock() { SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); if (pBlock == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; } pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); if (pBlock->pDataBlock == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFree(pBlock); + return NULL; } return pBlock; diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 34247d3b47..2b2c4cc201 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -298,6 +298,143 @@ int32_t isQualifiedTable(STableKeyInfo* info, SNode* pTagCond, void* metaHandle, return TSDB_CODE_SUCCESS; } +typedef struct tagFilterAssist{ + SHashObj *colHash; + int32_t index; + SArray *cInfoList; +}tagFilterAssist; + +static EDealRes getColumn(SNode* pNode, void* pContext) { + if (QUERY_NODE_COLUMN == nodeType(pNode)) { + tagFilterAssist *pData = (tagFilterAssist *)pContext; + SColumnNode* pSColumnNode = (SColumnNode*)pNode; + void *data = taosHashGet(pData->colHash, &pSColumnNode->colId, sizeof(pSColumnNode->colId)); + if(!data){ + taosHashPut(pData->colHash, &pSColumnNode->colId, sizeof(pSColumnNode->colId), &pNode, sizeof(pNode)); + pSColumnNode->slotId = pData->index++; + SColumnInfo cInfo = {.colId = pSColumnNode->colId, .type = pSColumnNode->node.resType.type, .bytes = pSColumnNode->node.resType.bytes}; + taosArrayPush(pData->cInfoList, &cInfo); + } + + return DEAL_RES_END; + } + return DEAL_RES_CONTINUE; +} + +static int32_t createResultData(SDataType* pType, int32_t numOfRows, SScalarParam* pParam) { + SColumnInfoData* pColumnData = taosMemoryCalloc(1, sizeof(SColumnInfoData)); + if (pColumnData == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return terrno; + } + + pColumnData->info.type = pType->type; + pColumnData->info.bytes = pType->bytes; + pColumnData->info.scale = pType->scale; + pColumnData->info.precision = pType->precision; + + int32_t code = colInfoDataEnsureCapacity(pColumnData, numOfRows); + if (code != TSDB_CODE_SUCCESS) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(pColumnData); + return terrno; + } + + pParam->columnData = pColumnData; + pParam->colAlloced = true; + return TSDB_CODE_SUCCESS; +} + +SColumnInfoData* getColInfoResult(void* metaHandle, SArray* pTableList, SNode* pTagCond){ + int32_t code = TSDB_CODE_SUCCESS; + SArray* pBlockList = NULL; + SSDataBlock* pResBlock = NULL; + SScalarParam output = {0}; + + tagFilterAssist ctx = {0}; + ctx.colHash = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_SMALLINT), false, HASH_NO_LOCK); + if(ctx.colHash == NULL){ + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + ctx.index = 0; + ctx.cInfoList = taosArrayInit(4, sizeof(SColumnInfo)); + if(ctx.cInfoList == NULL){ + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + nodesWalkExprPostOrder(pTagCond, getColumn, (void *)&ctx); + + pResBlock = createDataBlock(); + if (pResBlock == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto end; + } + + for (int32_t i = 0; i < taosArrayGetSize(ctx.cInfoList); ++i) { + SColumnInfoData colInfo = {{0}, 0}; + colInfo.info = *(SColumnInfo*)taosArrayGet(ctx.cInfoList, i); + blockDataAppendColInfo(pResBlock, &colInfo); + } + + int32_t rows = taosArrayGetSize(pTableList); + code = blockDataEnsureCapacity(pResBlock, rows); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + goto end; + } + + for (int32_t i = 0; i < rows; i++) { + STableKeyInfo* info = taosArrayGet(pTableList, i); + + SMetaReader mr = {0}; + metaReaderInit(&mr, metaHandle, 0); + code = metaGetTableEntryByUid(&mr, info->uid); + if (TSDB_CODE_SUCCESS == code) { + for(int32_t j = 0; j < taosArrayGetSize(pResBlock->pDataBlock); j++){ + SColumnInfoData* pColInfo = (SColumnInfoData*)taosArrayGet(pResBlock->pDataBlock, j); + STagVal tagVal = {0}; + tagVal.cid = pColInfo->info.colId; + const char* p = metaGetTableTagVal(&mr.me, pColInfo->info.type, &tagVal); + + if (IS_VAR_DATA_TYPE(pColInfo->info.type)) { + char* tmp = taosMemoryCalloc(tagVal.nData + VARSTR_HEADER_SIZE, 1); + memcpy(varDataVal(tmp), tagVal.pData, tagVal.nData); + varDataSetLen(tmp, tagVal.nData); + colDataAppend(pColInfo, i, tmp, p == NULL); + taosMemoryFree(tmp); + } else { + colDataAppend(pColInfo, i, p, p == NULL); + } + } + + } + metaReaderClear(&mr); + } + + pBlockList = taosArrayInit(2, POINTER_BYTES); + taosArrayPush(pBlockList, &pResBlock); + + SDataType type = {.type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)}; + code = createResultData(&type, rows, &output); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } + + code = scalarCalculate(pTagCond, pBlockList, &output); + if(code != TSDB_CODE_SUCCESS){ + terrno = code; + } + +end: + taosArrayDestroy(pBlockList); + taosHashCleanup(ctx.colHash); + taosArrayDestroy(ctx.cInfoList); + blockDataDestroy(pResBlock); + taosArrayDestroy(pBlockList); + return output.columnData; +} + int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SNode* pTagCond, SNode* pTagIndexCond, STableListInfo* pListInfo) { int32_t code = TSDB_CODE_SUCCESS; @@ -347,22 +484,23 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, } if (pTagCond) { + SColumnInfoData* pColInfoData = getColInfoResult(metaHandle, pListInfo->pTableList, pTagCond); + if(terrno != TDB_CODE_SUCCESS){ + colDataDestroy(pColInfoData); + return terrno; + } + int32_t i = 0; while (i < taosArrayGetSize(pListInfo->pTableList)) { - STableKeyInfo* info = taosArrayGet(pListInfo->pTableList, i); + void* var = POINTER_SHIFT(pColInfoData->pData, i * pColInfoData->info.bytes); - bool qualified = true; - code = isQualifiedTable(info, pTagCond, metaHandle, &qualified); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - if (!qualified) { + if (*(bool*)var == false) { taosArrayRemove(pListInfo->pTableList, i); continue; } i++; } + colDataDestroy(pColInfoData); } pListInfo->pGroupList = taosArrayInit(4, POINTER_BYTES); From e7f4755386f19c149dc8569372a75700b1237556 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 10 Aug 2022 20:26:59 +0800 Subject: [PATCH 02/33] opti: add log for time cost --- source/libs/executor/src/executil.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 2b2c4cc201..63bd30120e 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -384,6 +384,7 @@ SColumnInfoData* getColInfoResult(void* metaHandle, SArray* pTableList, SNode* p goto end; } + int64_t st = taosGetTimestampUs(); for (int32_t i = 0; i < rows; i++) { STableKeyInfo* info = taosArrayGet(pTableList, i); @@ -411,6 +412,8 @@ SColumnInfoData* getColInfoResult(void* metaHandle, SArray* pTableList, SNode* p } metaReaderClear(&mr); } + int64_t st1 = taosGetTimestampUs(); + qDebug("generate tag block rows:%d, cost:%lf ms", rows, st1-st); pBlockList = taosArrayInit(2, POINTER_BYTES); taosArrayPush(pBlockList, &pResBlock); @@ -425,6 +428,8 @@ SColumnInfoData* getColInfoResult(void* metaHandle, SArray* pTableList, SNode* p if(code != TSDB_CODE_SUCCESS){ terrno = code; } + int64_t st2 = taosGetTimestampUs(); + qDebug("calculate tag block rows:%d, cost:%lf ms", rows, st2-st1); end: taosArrayDestroy(pBlockList); From 8be139626889e79f7aab95ba99530c1fbfcb440e Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 10 Aug 2022 21:31:08 +0800 Subject: [PATCH 03/33] opti: add log for time cost --- source/libs/executor/src/executil.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 63bd30120e..2a2a387e31 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -412,6 +412,8 @@ SColumnInfoData* getColInfoResult(void* metaHandle, SArray* pTableList, SNode* p } metaReaderClear(&mr); } + pResBlock->info.rows = rows; + int64_t st1 = taosGetTimestampUs(); qDebug("generate tag block rows:%d, cost:%lf ms", rows, st1-st); @@ -432,7 +434,6 @@ SColumnInfoData* getColInfoResult(void* metaHandle, SArray* pTableList, SNode* p qDebug("calculate tag block rows:%d, cost:%lf ms", rows, st2-st1); end: - taosArrayDestroy(pBlockList); taosHashCleanup(ctx.colHash); taosArrayDestroy(ctx.cInfoList); blockDataDestroy(pResBlock); From 4f03e2e4f5f4c550805e43b2107f40786eb994d6 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 11 Aug 2022 14:35:38 +0800 Subject: [PATCH 04/33] opti:logic for get table list --- source/libs/executor/src/executil.c | 48 ++++++++++++++++------------- 1 file changed, 27 insertions(+), 21 deletions(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 2a2a387e31..4713eaefe5 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -315,8 +315,6 @@ static EDealRes getColumn(SNode* pNode, void* pContext) { SColumnInfo cInfo = {.colId = pSColumnNode->colId, .type = pSColumnNode->node.resType.type, .bytes = pSColumnNode->node.resType.bytes}; taosArrayPush(pData->cInfoList, &cInfo); } - - return DEAL_RES_END; } return DEAL_RES_CONTINUE; } @@ -388,34 +386,42 @@ SColumnInfoData* getColInfoResult(void* metaHandle, SArray* pTableList, SNode* p for (int32_t i = 0; i < rows; i++) { STableKeyInfo* info = taosArrayGet(pTableList, i); +// int64_t stt = taosGetTimestampUs(); SMetaReader mr = {0}; metaReaderInit(&mr, metaHandle, 0); code = metaGetTableEntryByUid(&mr, info->uid); - if (TSDB_CODE_SUCCESS == code) { - for(int32_t j = 0; j < taosArrayGetSize(pResBlock->pDataBlock); j++){ - SColumnInfoData* pColInfo = (SColumnInfoData*)taosArrayGet(pResBlock->pDataBlock, j); - STagVal tagVal = {0}; - tagVal.cid = pColInfo->info.colId; - const char* p = metaGetTableTagVal(&mr.me, pColInfo->info.type, &tagVal); +// int64_t stt1 = taosGetTimestampUs(); +// qDebug("generate tag get meta rows:%d, cost:%ld ms", rows, stt1-stt); - if (IS_VAR_DATA_TYPE(pColInfo->info.type)) { - char* tmp = taosMemoryCalloc(tagVal.nData + VARSTR_HEADER_SIZE, 1); - memcpy(varDataVal(tmp), tagVal.pData, tagVal.nData); - varDataSetLen(tmp, tagVal.nData); - colDataAppend(pColInfo, i, tmp, p == NULL); - taosMemoryFree(tmp); - } else { - colDataAppend(pColInfo, i, p, p == NULL); - } + for(int32_t j = 0; j < taosArrayGetSize(pResBlock->pDataBlock); j++){ + SColumnInfoData* pColInfo = (SColumnInfoData*)taosArrayGet(pResBlock->pDataBlock, j); + STagVal tagVal = {0}; + tagVal.cid = pColInfo->info.colId; +// int64_t t1 = taosGetTimestampUs(); + const char* p = metaGetTableTagVal(&mr.me, pColInfo->info.type, &tagVal); +// int64_t t2 = taosGetTimestampUs(); +// qDebug("generate tag inner1 rows:%d, cost:%ld ms", rows, t2-t1); + + if (p == NULL){ + colDataAppend(pColInfo, i, p, true); + } else if (IS_VAR_DATA_TYPE(pColInfo->info.type)) { + char *tmp = (char*)(tagVal.pData - VARSTR_HEADER_SIZE); + varDataSetLen(tmp, tagVal.nData); + colDataAppend(pColInfo, i, tmp, p == NULL); + } else { + colDataAppend(pColInfo, i, p, false); } - +// int64_t t3 = taosGetTimestampUs(); +// qDebug("generate tag inner2 rows:%d, cost:%ld ms", rows, t3-t2); } +// int64_t stt2 = taosGetTimestampUs(); +// qDebug("generate tag get block rows:%d, cost:%ld us", rows, stt2-stt1); metaReaderClear(&mr); } pResBlock->info.rows = rows; int64_t st1 = taosGetTimestampUs(); - qDebug("generate tag block rows:%d, cost:%lf ms", rows, st1-st); + qDebug("generate tag block rows:%d, cost:%ld us", rows, st1-st); pBlockList = taosArrayInit(2, POINTER_BYTES); taosArrayPush(pBlockList, &pResBlock); @@ -431,7 +437,7 @@ SColumnInfoData* getColInfoResult(void* metaHandle, SArray* pTableList, SNode* p terrno = code; } int64_t st2 = taosGetTimestampUs(); - qDebug("calculate tag block rows:%d, cost:%lf ms", rows, st2-st1); + qDebug("calculate tag block rows:%d, cost:%ld us", rows, st2-st1); end: taosHashCleanup(ctx.colHash); @@ -489,7 +495,7 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, taosArrayPush(pListInfo->pTableList, &info); } - if (pTagCond) { + if (pTagCond && taosArrayGetSize(pListInfo->pTableList) > 0) { SColumnInfoData* pColInfoData = getColInfoResult(metaHandle, pListInfo->pTableList, pTagCond); if(terrno != TDB_CODE_SUCCESS){ colDataDestroy(pColInfoData); From 32f9bc0dd0ac02d948b5999747f3090c51cb6bf1 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Thu, 11 Aug 2022 18:29:28 +0800 Subject: [PATCH 05/33] opti:logic for get table list --- source/dnode/vnode/inc/vnode.h | 1 + source/dnode/vnode/src/meta/metaQuery.c | 52 +++++++++++++++++++++++++ source/libs/executor/src/executil.c | 19 ++++++++- 3 files changed, 71 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 18a7583f4c..e969634a9f 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -91,6 +91,7 @@ typedef struct SMetaEntry SMetaEntry; void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags); void metaReaderClear(SMetaReader *pReader); int32_t metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid); +//int32_t metaGetTableEntryByUidTest(void *pReader, SArray *uidList); int32_t metaReadNext(SMetaReader *pReader); const void *metaGetTableTagVal(SMetaEntry *pEntry, int16_t type, STagVal *tagVal); int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName); diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index eed0ae5e14..f5771c17bd 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -53,6 +53,58 @@ _err: return -1; } +int metaGetTableEntryByUidTest(void* meta, SArray *uidList) { + + SArray* readerList = taosArrayInit(taosArrayGetSize(uidList), sizeof(SMetaReader)); + SArray* uidVersion = taosArrayInit(taosArrayGetSize(uidList), sizeof(STbDbKey)); + SMeta *pMeta = meta; + int64_t version; + + int64_t stt1 = taosGetTimestampUs(); + for(int i = 0; i < taosArrayGetSize(uidList); i++) { + void* ppVal = NULL; + int vlen = 0; + uint64_t * uid = taosArrayGet(uidList, i); + // query uid.idx + if (tdbTbGet(pMeta->pUidIdx, uid, sizeof(*uid), &ppVal, &vlen) < 0) { + continue; + } + version = *(int64_t *)ppVal; + + STbDbKey tbDbKey = {.version = version, .uid = *uid}; + taosArrayPush(uidVersion, &tbDbKey); + } + int64_t stt2 = taosGetTimestampUs(); + qDebug("metaGetTableEntryByUidTest1 rows:%d, cost:%ld us", taosArrayGetSize(uidList), stt2-stt1); + + for(int i = 0; i < taosArrayGetSize(uidVersion); i++) { + SMetaReader pReader = {0}; + + STbDbKey *tbDbKey = taosArrayGet(uidVersion, i); + // query table.db + if (tdbTbGet(pMeta->pTbDb, tbDbKey, sizeof(STbDbKey), &pReader.pBuf, &pReader.szBuf) < 0) { + continue; + } + taosArrayPush(readerList, &pReader); + } + int64_t stt3 = taosGetTimestampUs(); + qDebug("metaGetTableEntryByUidTest2 rows:%d, cost:%ld us", taosArrayGetSize(uidList), stt3-stt2); + + for(int i = 0; i < taosArrayGetSize(readerList); i++){ + SMetaReader* pReader = taosArrayGet(readerList, i); + metaReaderInit(pReader, meta, 0); + // decode the entry + tDecoderInit(&pReader->coder, pReader->pBuf, pReader->szBuf); + + if (metaDecodeEntry(&pReader->coder, &pReader->me) < 0) { + } + metaReaderClear(pReader); + } + int64_t stt4 = taosGetTimestampUs(); + qDebug("metaGetTableEntryByUidTest3 rows:%d, cost:%ld us", taosArrayGetSize(readerList), stt4-stt3); + return 0; +} + int metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid) { SMeta *pMeta = pReader->pMeta; int64_t version; diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 4713eaefe5..f798433f51 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -382,6 +382,20 @@ SColumnInfoData* getColInfoResult(void* metaHandle, SArray* pTableList, SNode* p goto end; } +// int64_t stt = taosGetTimestampUs(); +// SArray* arrAssist = taosArrayInit(rows, sizeof(SMetaReader)); +// SArray* uidList = taosArrayInit(rows, sizeof(uint64_t)); +// for (int32_t i = 0; i < rows; i++) { +// STableKeyInfo* info = taosArrayGet(pTableList, i); +// taosArrayPush(uidList, &info->uid); +// } + +// code = metaGetTableEntryByUidTest(metaHandle, uidList); + +// +// int64_t stt1 = taosGetTimestampUs(); +// qDebug("generate tag meta rows:%d, cost:%ld us", rows, stt1-stt); + int64_t st = taosGetTimestampUs(); for (int32_t i = 0; i < rows; i++) { STableKeyInfo* info = taosArrayGet(pTableList, i); @@ -392,7 +406,7 @@ SColumnInfoData* getColInfoResult(void* metaHandle, SArray* pTableList, SNode* p code = metaGetTableEntryByUid(&mr, info->uid); // int64_t stt1 = taosGetTimestampUs(); // qDebug("generate tag get meta rows:%d, cost:%ld ms", rows, stt1-stt); - +// SMetaReader *mr = taosArrayGet(arrAssist, i); for(int32_t j = 0; j < taosArrayGetSize(pResBlock->pDataBlock); j++){ SColumnInfoData* pColInfo = (SColumnInfoData*)taosArrayGet(pResBlock->pDataBlock, j); STagVal tagVal = {0}; @@ -465,6 +479,7 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SIndexMetaArg metaArg = { .metaEx = metaHandle, .idx = tsdbGetIdx(metaHandle), .ivtIdx = tsdbGetIvtIdx(metaHandle), .suid = tableUid}; + int64_t stt = taosGetTimestampUs(); SArray* res = taosArrayInit(8, sizeof(uint64_t)); SIdxFltStatus status = SFLT_NOT_INDEX; code = doFilterTag(pTagIndexCond, &metaArg, res, &status); @@ -481,6 +496,8 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, taosArrayPush(pListInfo->pTableList, &info); } taosArrayDestroy(res); + int64_t stt1 = taosGetTimestampUs(); + qDebug("generate table list, cost:%ld us", stt1-stt); } else { code = vnodeGetAllTableList(pVnode, tableUid, pListInfo->pTableList); } From e102f81f923963b7684a6003cf742da43aea6d5a Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 12 Aug 2022 11:33:53 +0800 Subject: [PATCH 06/33] opti:use suid+uid->tags index to optimize time cost --- source/dnode/vnode/inc/vnode.h | 4 +- source/dnode/vnode/src/meta/metaQuery.c | 70 +++++++++++++++++++--- source/dnode/vnode/src/meta/metaTable.c | 5 +- source/libs/executor/src/executil.c | 79 ++++++++++--------------- source/libs/executor/src/scanoperator.c | 4 +- 5 files changed, 103 insertions(+), 59 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index e969634a9f..6ad47799e1 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -91,9 +91,9 @@ typedef struct SMetaEntry SMetaEntry; void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags); void metaReaderClear(SMetaReader *pReader); int32_t metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid); -//int32_t metaGetTableEntryByUidTest(void *pReader, SArray *uidList); +int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList, SArray *tags); int32_t metaReadNext(SMetaReader *pReader); -const void *metaGetTableTagVal(SMetaEntry *pEntry, int16_t type, STagVal *tagVal); +const void *metaGetTableTagVal(void *tag, int16_t type, STagVal *tagVal); int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName); typedef struct SMetaFltParam { diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index f5771c17bd..df22913a0d 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -59,6 +59,7 @@ int metaGetTableEntryByUidTest(void* meta, SArray *uidList) { SArray* uidVersion = taosArrayInit(taosArrayGetSize(uidList), sizeof(STbDbKey)); SMeta *pMeta = meta; int64_t version; + SHashObj *uHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); int64_t stt1 = taosGetTimestampUs(); for(int i = 0; i < taosArrayGetSize(uidList); i++) { @@ -73,10 +74,31 @@ int metaGetTableEntryByUidTest(void* meta, SArray *uidList) { STbDbKey tbDbKey = {.version = version, .uid = *uid}; taosArrayPush(uidVersion, &tbDbKey); + taosHashPut(uHash, uid, sizeof(int64_t), ppVal, sizeof(int64_t)); } int64_t stt2 = taosGetTimestampUs(); qDebug("metaGetTableEntryByUidTest1 rows:%d, cost:%ld us", taosArrayGetSize(uidList), stt2-stt1); + TBC *pCur = NULL; + tdbTbcOpen(pMeta->pTbDb, &pCur, NULL); + tdbTbcMoveToFirst(pCur); + void *pKey = NULL; + int kLen = 0; + + while(1){ + SMetaReader pReader = {0}; + int32_t ret = tdbTbcNext(pCur, &pKey, &kLen, &pReader.pBuf, &pReader.szBuf); + if (ret < 0) break; + STbDbKey *tmp = (STbDbKey*)pKey; + int64_t *ver = (int64_t*)taosHashGet(uHash, &tmp->uid, sizeof(int64_t)); + if(ver == NULL || *ver != tmp->version) continue; + taosArrayPush(readerList, &pReader); + } + tdbTbcClose(pCur); + + taosArrayClear(readerList); + int64_t stt3 = taosGetTimestampUs(); + qDebug("metaGetTableEntryByUidTest2 rows:%d, cost:%ld us", taosArrayGetSize(uidList), stt3-stt2); for(int i = 0; i < taosArrayGetSize(uidVersion); i++) { SMetaReader pReader = {0}; @@ -87,8 +109,8 @@ int metaGetTableEntryByUidTest(void* meta, SArray *uidList) { } taosArrayPush(readerList, &pReader); } - int64_t stt3 = taosGetTimestampUs(); - qDebug("metaGetTableEntryByUidTest2 rows:%d, cost:%ld us", taosArrayGetSize(uidList), stt3-stt2); + int64_t stt4 = taosGetTimestampUs(); + qDebug("metaGetTableEntryByUidTest3 rows:%d, cost:%ld us", taosArrayGetSize(uidList), stt4-stt3); for(int i = 0; i < taosArrayGetSize(readerList); i++){ SMetaReader* pReader = taosArrayGet(readerList, i); @@ -100,8 +122,8 @@ int metaGetTableEntryByUidTest(void* meta, SArray *uidList) { } metaReaderClear(pReader); } - int64_t stt4 = taosGetTimestampUs(); - qDebug("metaGetTableEntryByUidTest3 rows:%d, cost:%ld us", taosArrayGetSize(readerList), stt4-stt3); + int64_t stt5 = taosGetTimestampUs(); + qDebug("metaGetTableEntryByUidTest4 rows:%d, cost:%ld us", taosArrayGetSize(readerList), stt5-stt4); return 0; } @@ -801,9 +823,8 @@ SArray *metaGetSmaTbUids(SMeta *pMeta) { #endif -const void *metaGetTableTagVal(SMetaEntry *pEntry, int16_t type, STagVal *val) { - ASSERT(pEntry->type == TSDB_CHILD_TABLE); - STag *tag = (STag *)pEntry->ctbEntry.pTags; +const void *metaGetTableTagVal(void *pTag, int16_t type, STagVal *val) { + STag *tag = (STag*) pTag; if (type == TSDB_DATA_TYPE_JSON) { return tag; } @@ -940,3 +961,38 @@ END: return ret; } + +int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList, SArray *tags) { + SMCtbCursor *pCur = metaOpenCtbCursor(pMeta, suid); + + SHashObj *uHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); + size_t len = taosArrayGetSize(uidList); + if(len > 0){ + for(int i = 0; i < len; i++){ + int64_t *uid = taosArrayGet(uidList, i); + taosHashPut(uHash, uid, sizeof(int64_t), &i, sizeof(i)); + } + } + while (1) { + tb_uid_t id = metaCtbCursorNext(pCur); + if (id == 0) { + break; + } + + if(len > 0 && taosHashGet(uHash, &id, sizeof(int64_t)) == NULL){ + continue; + } + + void* tag = taosMemoryMalloc(pCur->vLen); + memcpy(tag, pCur->pVal, pCur->vLen); + taosArrayPush(tags, &tag); + + if(len == 0){ + taosArrayPush(uidList, &id); + } + } + + taosHashCleanup(uHash); + metaCloseCtbCursor(pCur); + return TSDB_CODE_SUCCESS; +} diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 7427f79509..86e2f38697 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -864,6 +864,9 @@ static int metaUpdateTableTagVal(SMeta *pMeta, int64_t version, SVAlterTbReq *pA metaUpdateTagIdx(pMeta, &ctbEntry); } + SCtbIdxKey ctbIdxKey = {.suid = ctbEntry.ctbEntry.suid, .uid = uid}; + tdbTbUpsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), ctbEntry.ctbEntry.pTags, ((STag*)(ctbEntry.ctbEntry.pTags))->len, &pMeta->txn); + tDecoderClear(&dc1); tDecoderClear(&dc2); if (ctbEntry.ctbEntry.pTags) taosMemoryFree((void *)ctbEntry.ctbEntry.pTags); @@ -1061,7 +1064,7 @@ static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME) { static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME) { SCtbIdxKey ctbIdxKey = {.suid = pME->ctbEntry.suid, .uid = pME->uid}; - return tdbTbInsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), NULL, 0, &pMeta->txn); + return tdbTbInsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), pME->ctbEntry.pTags, ((STag*)(pME->ctbEntry.pTags))->len, &pMeta->txn); } int metaCreateTagIdxKey(tb_uid_t suid, int32_t cid, const void *pTagData, int32_t nTagData, int8_t type, tb_uid_t uid, diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index f798433f51..c8d607185e 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -221,7 +221,7 @@ EDealRes doTranslateTagExpr(SNode** pNode, void* pContext) { STagVal tagVal = {0}; tagVal.cid = pSColumnNode->colId; - const char* p = metaGetTableTagVal(&mr->me, pSColumnNode->node.resType.type, &tagVal); + const char* p = metaGetTableTagVal(mr->me.ctbEntry.pTags, pSColumnNode->node.resType.type, &tagVal); if (p == NULL) { res->node.resType.type = TSDB_DATA_TYPE_NULL; } else if (pSColumnNode->node.resType.type == TSDB_DATA_TYPE_JSON) { @@ -343,10 +343,11 @@ static int32_t createResultData(SDataType* pType, int32_t numOfRows, SScalarPara return TSDB_CODE_SUCCESS; } -SColumnInfoData* getColInfoResult(void* metaHandle, SArray* pTableList, SNode* pTagCond){ +SColumnInfoData* getColInfoResult(void* metaHandle, uint64_t suid, SArray* uidList, SNode* pTagCond){ int32_t code = TSDB_CODE_SUCCESS; SArray* pBlockList = NULL; SSDataBlock* pResBlock = NULL; + SArray* tags = NULL; SScalarParam output = {0}; tagFilterAssist ctx = {0}; @@ -375,35 +376,30 @@ SColumnInfoData* getColInfoResult(void* metaHandle, SArray* pTableList, SNode* p blockDataAppendColInfo(pResBlock, &colInfo); } - int32_t rows = taosArrayGetSize(pTableList); + int64_t stt = taosGetTimestampUs(); + tags = taosArrayInit(8, POINTER_BYTES); + code = metaGetTableTags(metaHandle, suid, uidList, tags); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + goto end; + } + + int32_t rows = taosArrayGetSize(uidList); + + int64_t stt1 = taosGetTimestampUs(); + qDebug("generate tag meta rows:%d, cost:%ld us", rows, stt1-stt); + code = blockDataEnsureCapacity(pResBlock, rows); if (code != TSDB_CODE_SUCCESS) { terrno = code; goto end; } -// int64_t stt = taosGetTimestampUs(); -// SArray* arrAssist = taosArrayInit(rows, sizeof(SMetaReader)); -// SArray* uidList = taosArrayInit(rows, sizeof(uint64_t)); -// for (int32_t i = 0; i < rows; i++) { -// STableKeyInfo* info = taosArrayGet(pTableList, i); -// taosArrayPush(uidList, &info->uid); -// } - -// code = metaGetTableEntryByUidTest(metaHandle, uidList); - -// -// int64_t stt1 = taosGetTimestampUs(); -// qDebug("generate tag meta rows:%d, cost:%ld us", rows, stt1-stt); - int64_t st = taosGetTimestampUs(); for (int32_t i = 0; i < rows; i++) { - STableKeyInfo* info = taosArrayGet(pTableList, i); - + uint64_t* uid = taosArrayGet(uidList, i); + void* tag = taosArrayGet(tags, i); // int64_t stt = taosGetTimestampUs(); - SMetaReader mr = {0}; - metaReaderInit(&mr, metaHandle, 0); - code = metaGetTableEntryByUid(&mr, info->uid); // int64_t stt1 = taosGetTimestampUs(); // qDebug("generate tag get meta rows:%d, cost:%ld ms", rows, stt1-stt); // SMetaReader *mr = taosArrayGet(arrAssist, i); @@ -412,7 +408,7 @@ SColumnInfoData* getColInfoResult(void* metaHandle, SArray* pTableList, SNode* p STagVal tagVal = {0}; tagVal.cid = pColInfo->info.colId; // int64_t t1 = taosGetTimestampUs(); - const char* p = metaGetTableTagVal(&mr.me, pColInfo->info.type, &tagVal); + const char* p = metaGetTableTagVal(tag, pColInfo->info.type, &tagVal); // int64_t t2 = taosGetTimestampUs(); // qDebug("generate tag inner1 rows:%d, cost:%ld ms", rows, t2-t1); @@ -430,7 +426,6 @@ SColumnInfoData* getColInfoResult(void* metaHandle, SArray* pTableList, SNode* p } // int64_t stt2 = taosGetTimestampUs(); // qDebug("generate tag get block rows:%d, cost:%ld us", rows, stt2-stt1); - metaReaderClear(&mr); } pResBlock->info.rows = rows; @@ -454,6 +449,7 @@ SColumnInfoData* getColInfoResult(void* metaHandle, SArray* pTableList, SNode* p qDebug("calculate tag block rows:%d, cost:%ld us", rows, st2-st1); end: + taosArrayDestroy(tags); taosHashCleanup(ctx.colHash); taosArrayDestroy(ctx.cInfoList); blockDataDestroy(pResBlock); @@ -471,8 +467,8 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, } uint64_t tableUid = pScanNode->uid; - pListInfo->suid = pScanNode->suid; + SArray* res = taosArrayInit(8, sizeof(uint64_t)); if (pScanNode->tableType == TSDB_SUPER_TABLE) { if (pTagIndexCond) { @@ -480,42 +476,24 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, .metaEx = metaHandle, .idx = tsdbGetIdx(metaHandle), .ivtIdx = tsdbGetIvtIdx(metaHandle), .suid = tableUid}; int64_t stt = taosGetTimestampUs(); - SArray* res = taosArrayInit(8, sizeof(uint64_t)); SIdxFltStatus status = SFLT_NOT_INDEX; code = doFilterTag(pTagIndexCond, &metaArg, res, &status); if (code != 0 || status == SFLT_NOT_INDEX) { qError("failed to get tableIds from index, reason:%s, suid:%" PRIu64, tstrerror(code), tableUid); - // code = TSDB_CODE_INDEX_REBUILDING; - code = vnodeGetAllTableList(pVnode, tableUid, pListInfo->pTableList); - } else { - qDebug("success to get tableIds, size:%d, suid:%" PRIu64, (int)taosArrayGetSize(res), tableUid); } - for (int i = 0; i < taosArrayGetSize(res); i++) { - STableKeyInfo info = {.uid = *(uint64_t*)taosArrayGet(res, i), .groupId = 0}; - taosArrayPush(pListInfo->pTableList, &info); - } - taosArrayDestroy(res); int64_t stt1 = taosGetTimestampUs(); qDebug("generate table list, cost:%ld us", stt1-stt); - } else { - code = vnodeGetAllTableList(pVnode, tableUid, pListInfo->pTableList); - } - - if (code != TSDB_CODE_SUCCESS) { - qError("failed to get tableIds, reason:%s, suid:%" PRIu64, tstrerror(code), tableUid); - terrno = code; - return code; } } else { // Create one table group. - STableKeyInfo info = {.uid = tableUid, .groupId = 0}; - taosArrayPush(pListInfo->pTableList, &info); + taosArrayPush(res, &tableUid); } - if (pTagCond && taosArrayGetSize(pListInfo->pTableList) > 0) { - SColumnInfoData* pColInfoData = getColInfoResult(metaHandle, pListInfo->pTableList, pTagCond); + if (pTagCond) { + SColumnInfoData* pColInfoData = getColInfoResult(metaHandle, pListInfo->suid, res, pTagCond); if(terrno != TDB_CODE_SUCCESS){ colDataDestroy(pColInfoData); + taosArrayDestroy(res); return terrno; } @@ -532,6 +510,13 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, colDataDestroy(pColInfoData); } + for (int i = 0; i < taosArrayGetSize(res); i++) { + STableKeyInfo info = {.uid = *(uint64_t*)taosArrayGet(res, i), .groupId = 0}; + taosArrayPush(pListInfo->pTableList, &info); + } + + taosArrayDestroy(res); + pListInfo->pGroupList = taosArrayInit(4, POINTER_BYTES); if (pListInfo->pGroupList == NULL) { return TSDB_CODE_OUT_OF_MEMORY; diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index a211542de1..61f6474190 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -439,7 +439,7 @@ int32_t addTagPseudoColumnData(SReadHandle* pHandle, SExprInfo* pPseudoExpr, int } else { // these are tags STagVal tagVal = {0}; tagVal.cid = pExpr->base.pParam[0].pCol->colId; - const char* p = metaGetTableTagVal(&mr.me, pColInfoData->info.type, &tagVal); + const char* p = metaGetTableTagVal(mr.me.ctbEntry.pTags, pColInfoData->info.type, &tagVal); char* data = NULL; if (pColInfoData->info.type != TSDB_DATA_TYPE_JSON && p != NULL) { @@ -2569,7 +2569,7 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { } else { // it is a tag value STagVal val = {0}; val.cid = pExprInfo[j].base.pParam[0].pCol->colId; - const char* p = metaGetTableTagVal(&mr.me, pDst->info.type, &val); + const char* p = metaGetTableTagVal(mr.me.ctbEntry.pTags, pDst->info.type, &val); char* data = NULL; if (pDst->info.type != TSDB_DATA_TYPE_JSON && p != NULL) { From 95ef68bbc1c8720c3badb1f587529a34adbd89bd Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 12 Aug 2022 11:39:19 +0800 Subject: [PATCH 07/33] opti:use suid+uid->tags index to optimize time cost --- source/libs/executor/src/executil.c | 2 +- source/util/src/tarray.c | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index c8d607185e..eb6222f15b 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -449,7 +449,7 @@ SColumnInfoData* getColInfoResult(void* metaHandle, uint64_t suid, SArray* uidLi qDebug("calculate tag block rows:%d, cost:%ld us", rows, st2-st1); end: - taosArrayDestroy(tags); + taosArrayDestroyP(tags, taosMemoryFree); taosHashCleanup(ctx.colHash); taosArrayDestroy(ctx.cInfoList); blockDataDestroy(pResBlock); diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c index 3c4a0a20bd..454739348e 100644 --- a/source/util/src/tarray.c +++ b/source/util/src/tarray.c @@ -386,6 +386,7 @@ void* taosArrayDestroy(SArray* pArray) { } void taosArrayDestroyP(SArray* pArray, FDelete fp) { + if(!pArray) return; for (int32_t i = 0; i < pArray->size; i++) { fp(*(void**)TARRAY_GET_ELEM(pArray, i)); } From eec58b8ef0ee974dd41533d4000edb8a8ac32b5b Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 12 Aug 2022 13:53:24 +0800 Subject: [PATCH 08/33] opti:use suid+uid->tags index to optimize time cost --- source/common/src/tdatablock.c | 1 + source/dnode/vnode/src/meta/metaOpen.c | 2 +- source/libs/executor/src/executil.c | 13 ++++++++----- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 7692e02a37..196376e675 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -1425,6 +1425,7 @@ size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize) { } void colDataDestroy(SColumnInfoData* pColData) { + if(!pColData) return; if (IS_VAR_DATA_TYPE(pColData->info.type)) { taosMemoryFreeClear(pColData->varmeta.offset); } else { diff --git a/source/dnode/vnode/src/meta/metaOpen.c b/source/dnode/vnode/src/meta/metaOpen.c index 941d2c6d72..89071d0881 100644 --- a/source/dnode/vnode/src/meta/metaOpen.c +++ b/source/dnode/vnode/src/meta/metaOpen.c @@ -87,7 +87,7 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) { } // open pCtbIdx - ret = tdbTbOpen("ctb.idx", sizeof(SCtbIdxKey), 0, ctbIdxKeyCmpr, pMeta->pEnv, &pMeta->pCtbIdx); + ret = tdbTbOpen("ctb.idx", sizeof(SCtbIdxKey), -1, ctbIdxKeyCmpr, pMeta->pEnv, &pMeta->pCtbIdx); if (ret < 0) { metaError("vgId:%d, failed to open meta child table index since %s", TD_VID(pVnode), tstrerror(terrno)); goto _err; diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index eb6222f15b..81192cf2b7 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -385,7 +385,9 @@ SColumnInfoData* getColInfoResult(void* metaHandle, uint64_t suid, SArray* uidLi } int32_t rows = taosArrayGetSize(uidList); - + if(rows == 0){ + goto end; + } int64_t stt1 = taosGetTimestampUs(); qDebug("generate tag meta rows:%d, cost:%ld us", rows, stt1-stt); @@ -397,8 +399,7 @@ SColumnInfoData* getColInfoResult(void* metaHandle, uint64_t suid, SArray* uidLi int64_t st = taosGetTimestampUs(); for (int32_t i = 0; i < rows; i++) { - uint64_t* uid = taosArrayGet(uidList, i); - void* tag = taosArrayGet(tags, i); + void* tag = taosArrayGetP(tags, i); // int64_t stt = taosGetTimestampUs(); // int64_t stt1 = taosGetTimestampUs(); // qDebug("generate tag get meta rows:%d, cost:%ld ms", rows, stt1-stt); @@ -498,16 +499,18 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, } int32_t i = 0; - while (i < taosArrayGetSize(pListInfo->pTableList)) { + while (i < taosArrayGetSize(res) && pColInfoData) { void* var = POINTER_SHIFT(pColInfoData->pData, i * pColInfoData->info.bytes); if (*(bool*)var == false) { - taosArrayRemove(pListInfo->pTableList, i); + taosArrayRemove(res, i); continue; } i++; } colDataDestroy(pColInfoData); + }else{ + vnodeGetCtbIdList(pVnode, pScanNode->suid, res); } for (int i = 0; i < taosArrayGetSize(res); i++) { From 78e87a0aaec281f4efaae19aa06616ae2809d037 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 12 Aug 2022 14:38:54 +0800 Subject: [PATCH 09/33] fix:memory leak --- source/libs/executor/src/executil.c | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 81192cf2b7..42383887ab 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -343,7 +343,7 @@ static int32_t createResultData(SDataType* pType, int32_t numOfRows, SScalarPara return TSDB_CODE_SUCCESS; } -SColumnInfoData* getColInfoResult(void* metaHandle, uint64_t suid, SArray* uidList, SNode* pTagCond){ +static SColumnInfoData* getColInfoResult(void* metaHandle, uint64_t suid, SArray* uidList, SNode* pTagCond){ int32_t code = TSDB_CODE_SUCCESS; SArray* pBlockList = NULL; SSDataBlock* pResBlock = NULL; @@ -400,18 +400,11 @@ SColumnInfoData* getColInfoResult(void* metaHandle, uint64_t suid, SArray* uidLi int64_t st = taosGetTimestampUs(); for (int32_t i = 0; i < rows; i++) { void* tag = taosArrayGetP(tags, i); -// int64_t stt = taosGetTimestampUs(); -// int64_t stt1 = taosGetTimestampUs(); -// qDebug("generate tag get meta rows:%d, cost:%ld ms", rows, stt1-stt); -// SMetaReader *mr = taosArrayGet(arrAssist, i); for(int32_t j = 0; j < taosArrayGetSize(pResBlock->pDataBlock); j++){ SColumnInfoData* pColInfo = (SColumnInfoData*)taosArrayGet(pResBlock->pDataBlock, j); STagVal tagVal = {0}; tagVal.cid = pColInfo->info.colId; -// int64_t t1 = taosGetTimestampUs(); const char* p = metaGetTableTagVal(tag, pColInfo->info.type, &tagVal); -// int64_t t2 = taosGetTimestampUs(); -// qDebug("generate tag inner1 rows:%d, cost:%ld ms", rows, t2-t1); if (p == NULL){ colDataAppend(pColInfo, i, p, true); @@ -422,11 +415,7 @@ SColumnInfoData* getColInfoResult(void* metaHandle, uint64_t suid, SArray* uidLi } else { colDataAppend(pColInfo, i, p, false); } -// int64_t t3 = taosGetTimestampUs(); -// qDebug("generate tag inner2 rows:%d, cost:%ld ms", rows, t3-t2); } -// int64_t stt2 = taosGetTimestampUs(); -// qDebug("generate tag get block rows:%d, cost:%ld us", rows, stt2-stt1); } pResBlock->info.rows = rows; @@ -494,6 +483,7 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SColumnInfoData* pColInfoData = getColInfoResult(metaHandle, pListInfo->suid, res, pTagCond); if(terrno != TDB_CODE_SUCCESS){ colDataDestroy(pColInfoData); + taosMemoryFreeClear(pColInfoData); taosArrayDestroy(res); return terrno; } @@ -509,6 +499,7 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, i++; } colDataDestroy(pColInfoData); + taosMemoryFreeClear(pColInfoData); }else{ vnodeGetCtbIdList(pVnode, pScanNode->suid, res); } From 4f97d72502bfba48732bbdbb3c532172ec4357cb Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 12 Aug 2022 15:02:34 +0800 Subject: [PATCH 10/33] fix:memory leak --- source/libs/executor/src/executil.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 42383887ab..c157eded69 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -474,6 +474,8 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, int64_t stt1 = taosGetTimestampUs(); qDebug("generate table list, cost:%ld us", stt1-stt); + }else if(!pTagCond){ + vnodeGetCtbIdList(pVnode, pScanNode->suid, res); } } else { // Create one table group. taosArrayPush(res, &tableUid); @@ -500,8 +502,6 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, } colDataDestroy(pColInfoData); taosMemoryFreeClear(pColInfoData); - }else{ - vnodeGetCtbIdList(pVnode, pScanNode->suid, res); } for (int i = 0; i < taosArrayGetSize(res); i++) { From 37f6d1195c26e715e260cbbe5c9b598c10ce37b8 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Fri, 12 Aug 2022 21:57:29 +0800 Subject: [PATCH 11/33] fix:error in get table list --- source/dnode/vnode/src/meta/metaTable.c | 1 + source/libs/executor/src/executil.c | 85 ++++++++++++++++++------- 2 files changed, 62 insertions(+), 24 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 4a92b5233a..a253381449 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -1065,6 +1065,7 @@ static int metaUpdateTtlIdx(SMeta *pMeta, const SMetaEntry *pME) { static int metaUpdateCtbIdx(SMeta *pMeta, const SMetaEntry *pME) { SCtbIdxKey ctbIdxKey = {.suid = pME->ctbEntry.suid, .uid = pME->uid}; + return tdbTbInsert(pMeta->pCtbIdx, &ctbIdxKey, sizeof(ctbIdxKey), pME->ctbEntry.pTags, ((STag*)(pME->ctbEntry.pTags))->len, &pMeta->txn); } diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index c157eded69..2a43751707 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -304,18 +304,37 @@ typedef struct tagFilterAssist{ SArray *cInfoList; }tagFilterAssist; -static EDealRes getColumn(SNode* pNode, void* pContext) { - if (QUERY_NODE_COLUMN == nodeType(pNode)) { - tagFilterAssist *pData = (tagFilterAssist *)pContext; - SColumnNode* pSColumnNode = (SColumnNode*)pNode; - void *data = taosHashGet(pData->colHash, &pSColumnNode->colId, sizeof(pSColumnNode->colId)); - if(!data){ - taosHashPut(pData->colHash, &pSColumnNode->colId, sizeof(pSColumnNode->colId), &pNode, sizeof(pNode)); - pSColumnNode->slotId = pData->index++; - SColumnInfo cInfo = {.colId = pSColumnNode->colId, .type = pSColumnNode->node.resType.type, .bytes = pSColumnNode->node.resType.bytes}; - taosArrayPush(pData->cInfoList, &cInfo); +static EDealRes getColumn(SNode** pNode, void* pContext) { + SColumnNode* pSColumnNode = NULL; + if (QUERY_NODE_COLUMN == nodeType((*pNode))) { + pSColumnNode = *(SColumnNode**)pNode; + }else if(QUERY_NODE_FUNCTION == nodeType((*pNode))){ + SFunctionNode* pFuncNode = *(SFunctionNode**)(pNode); + if (pFuncNode->funcType == FUNCTION_TYPE_TBNAME) { + pSColumnNode = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); + if (NULL == pSColumnNode) { + return DEAL_RES_ERROR; + } + pSColumnNode->colId = -1; + pSColumnNode->colType = COLUMN_TYPE_TBNAME; + pSColumnNode->node.resType.type = TSDB_DATA_TYPE_VARCHAR; + pSColumnNode->node.resType.bytes = TSDB_TABLE_FNAME_LEN - 1 + VARSTR_HEADER_SIZE; + nodesDestroyNode(*pNode); + *pNode = (SNode*)pSColumnNode; } + }else{ + return DEAL_RES_CONTINUE; } + + tagFilterAssist *pData = (tagFilterAssist *)pContext; + void *data = taosHashGet(pData->colHash, &pSColumnNode->colId, sizeof(pSColumnNode->colId)); + if(!data){ + taosHashPut(pData->colHash, &pSColumnNode->colId, sizeof(pSColumnNode->colId), pNode, sizeof((*pNode))); + pSColumnNode->slotId = pData->index++; + SColumnInfo cInfo = {.colId = pSColumnNode->colId, .type = pSColumnNode->node.resType.type, .bytes = pSColumnNode->node.resType.bytes}; + taosArrayPush(pData->cInfoList, &cInfo); + } + return DEAL_RES_CONTINUE; } @@ -362,7 +381,8 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, uint64_t suid, SArray terrno = TSDB_CODE_OUT_OF_MEMORY; goto end; } - nodesWalkExprPostOrder(pTagCond, getColumn, (void *)&ctx); + + nodesRewriteExprPostOrder(&pTagCond, getColumn, (void *)&ctx); pResBlock = createDataBlock(); if (pResBlock == NULL) { @@ -400,20 +420,30 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, uint64_t suid, SArray int64_t st = taosGetTimestampUs(); for (int32_t i = 0; i < rows; i++) { void* tag = taosArrayGetP(tags, i); + int64_t* uid = taosArrayGet(uidList, i); for(int32_t j = 0; j < taosArrayGetSize(pResBlock->pDataBlock); j++){ SColumnInfoData* pColInfo = (SColumnInfoData*)taosArrayGet(pResBlock->pDataBlock, j); - STagVal tagVal = {0}; - tagVal.cid = pColInfo->info.colId; - const char* p = metaGetTableTagVal(tag, pColInfo->info.type, &tagVal); + if(pColInfo->info.colId == -1){ // tbname + char str[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + metaGetTableNameByUid(metaHandle, *uid, str); + colDataAppend(pColInfo, i, str, false); + qDebug("tbnameget uid:%ld, tbname:%s", *uid, str+2); + }else{ + STagVal tagVal = {0}; + tagVal.cid = pColInfo->info.colId; + const char* p = metaGetTableTagVal(tag, pColInfo->info.type, &tagVal); - if (p == NULL){ - colDataAppend(pColInfo, i, p, true); - } else if (IS_VAR_DATA_TYPE(pColInfo->info.type)) { - char *tmp = (char*)(tagVal.pData - VARSTR_HEADER_SIZE); - varDataSetLen(tmp, tagVal.nData); - colDataAppend(pColInfo, i, tmp, p == NULL); - } else { - colDataAppend(pColInfo, i, p, false); + if (p == NULL){ + colDataAppend(pColInfo, i, p, true); + } else if (IS_VAR_DATA_TYPE(pColInfo->info.type)) { + char *tmp = taosMemoryMalloc(tagVal.nData + VARSTR_HEADER_SIZE); + varDataSetLen(tmp, tagVal.nData); + memcpy(tmp + VARSTR_HEADER_SIZE, tagVal.pData, tagVal.nData); + colDataAppend(pColInfo, i, tmp, false); + taosMemoryFree(tmp); + } else { + colDataAppend(pColInfo, i, (const char*)&tagVal.i64, false); + } } } } @@ -491,14 +521,20 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, } int32_t i = 0; - while (i < taosArrayGetSize(res) && pColInfoData) { - void* var = POINTER_SHIFT(pColInfoData->pData, i * pColInfoData->info.bytes); + int32_t j = 0; + int32_t len = taosArrayGetSize(res); + while (i < taosArrayGetSize(res) && j < len && pColInfoData) { + void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); + int64_t* uid = taosArrayGet(res, i); + qDebug("tbnameget get uid:%ld, res:%d", *uid, *(bool*)var); if (*(bool*)var == false) { taosArrayRemove(res, i); + j++; continue; } i++; + j++; } colDataDestroy(pColInfoData); taosMemoryFreeClear(pColInfoData); @@ -507,6 +543,7 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, for (int i = 0; i < taosArrayGetSize(res); i++) { STableKeyInfo info = {.uid = *(uint64_t*)taosArrayGet(res, i), .groupId = 0}; taosArrayPush(pListInfo->pTableList, &info); + qDebug("tbnameget get uid:%ld", info.uid); } taosArrayDestroy(res); From 4eccd81e9420af0ce6c1812e531f3a518494c43e Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 16 Aug 2022 14:39:03 +0800 Subject: [PATCH 12/33] fix:error in tag filter optimization --- source/libs/executor/src/executil.c | 4 +++- tests/system-test/2-query/json_tag.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 2a43751707..2e82ab3d89 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -433,8 +433,10 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, uint64_t suid, SArray tagVal.cid = pColInfo->info.colId; const char* p = metaGetTableTagVal(tag, pColInfo->info.type, &tagVal); - if (p == NULL){ + if (p == NULL || (pColInfo->info.type == TSDB_DATA_TYPE_JSON && ((STag*)p)->nTag == 0)){ colDataAppend(pColInfo, i, p, true); + } else if (pColInfo->info.type == TSDB_DATA_TYPE_JSON) { + colDataAppend(pColInfo, i, p, false); } else if (IS_VAR_DATA_TYPE(pColInfo->info.type)) { char *tmp = taosMemoryMalloc(tagVal.nData + VARSTR_HEADER_SIZE); varDataSetLen(tmp, tagVal.nData); diff --git a/tests/system-test/2-query/json_tag.py b/tests/system-test/2-query/json_tag.py index d9d7ef2300..30179062f5 100644 --- a/tests/system-test/2-query/json_tag.py +++ b/tests/system-test/2-query/json_tag.py @@ -216,7 +216,7 @@ class TDTestCase: # test where with json tag tdSql.query("select * from jsons1_1 where jtag is not null") - tdSql.query("select * from jsons1 where jtag='{\"tag1\":11,\"tag2\":\"\"}'") + tdSql.error("select * from jsons1 where jtag='{\"tag1\":11,\"tag2\":\"\"}'") tdSql.error("select * from jsons1 where jtag->'tag1'={}") # test json error From c4d25ccb90874c49e9602189611157eddf99201e Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Tue, 16 Aug 2022 14:48:31 +0800 Subject: [PATCH 13/33] fix:remove log for time cost --- source/dnode/vnode/src/meta/metaQuery.c | 146 ++++++++++++------------ source/libs/executor/src/executil.c | 22 ++-- 2 files changed, 84 insertions(+), 84 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index df22913a0d..adf9088cae 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -53,79 +53,79 @@ _err: return -1; } -int metaGetTableEntryByUidTest(void* meta, SArray *uidList) { - - SArray* readerList = taosArrayInit(taosArrayGetSize(uidList), sizeof(SMetaReader)); - SArray* uidVersion = taosArrayInit(taosArrayGetSize(uidList), sizeof(STbDbKey)); - SMeta *pMeta = meta; - int64_t version; - SHashObj *uHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); - - int64_t stt1 = taosGetTimestampUs(); - for(int i = 0; i < taosArrayGetSize(uidList); i++) { - void* ppVal = NULL; - int vlen = 0; - uint64_t * uid = taosArrayGet(uidList, i); - // query uid.idx - if (tdbTbGet(pMeta->pUidIdx, uid, sizeof(*uid), &ppVal, &vlen) < 0) { - continue; - } - version = *(int64_t *)ppVal; - - STbDbKey tbDbKey = {.version = version, .uid = *uid}; - taosArrayPush(uidVersion, &tbDbKey); - taosHashPut(uHash, uid, sizeof(int64_t), ppVal, sizeof(int64_t)); - } - int64_t stt2 = taosGetTimestampUs(); - qDebug("metaGetTableEntryByUidTest1 rows:%d, cost:%ld us", taosArrayGetSize(uidList), stt2-stt1); - - TBC *pCur = NULL; - tdbTbcOpen(pMeta->pTbDb, &pCur, NULL); - tdbTbcMoveToFirst(pCur); - void *pKey = NULL; - int kLen = 0; - - while(1){ - SMetaReader pReader = {0}; - int32_t ret = tdbTbcNext(pCur, &pKey, &kLen, &pReader.pBuf, &pReader.szBuf); - if (ret < 0) break; - STbDbKey *tmp = (STbDbKey*)pKey; - int64_t *ver = (int64_t*)taosHashGet(uHash, &tmp->uid, sizeof(int64_t)); - if(ver == NULL || *ver != tmp->version) continue; - taosArrayPush(readerList, &pReader); - } - tdbTbcClose(pCur); - - taosArrayClear(readerList); - int64_t stt3 = taosGetTimestampUs(); - qDebug("metaGetTableEntryByUidTest2 rows:%d, cost:%ld us", taosArrayGetSize(uidList), stt3-stt2); - for(int i = 0; i < taosArrayGetSize(uidVersion); i++) { - SMetaReader pReader = {0}; - - STbDbKey *tbDbKey = taosArrayGet(uidVersion, i); - // query table.db - if (tdbTbGet(pMeta->pTbDb, tbDbKey, sizeof(STbDbKey), &pReader.pBuf, &pReader.szBuf) < 0) { - continue; - } - taosArrayPush(readerList, &pReader); - } - int64_t stt4 = taosGetTimestampUs(); - qDebug("metaGetTableEntryByUidTest3 rows:%d, cost:%ld us", taosArrayGetSize(uidList), stt4-stt3); - - for(int i = 0; i < taosArrayGetSize(readerList); i++){ - SMetaReader* pReader = taosArrayGet(readerList, i); - metaReaderInit(pReader, meta, 0); - // decode the entry - tDecoderInit(&pReader->coder, pReader->pBuf, pReader->szBuf); - - if (metaDecodeEntry(&pReader->coder, &pReader->me) < 0) { - } - metaReaderClear(pReader); - } - int64_t stt5 = taosGetTimestampUs(); - qDebug("metaGetTableEntryByUidTest4 rows:%d, cost:%ld us", taosArrayGetSize(readerList), stt5-stt4); - return 0; -} +//int metaGetTableEntryByUidTest(void* meta, SArray *uidList) { +// +// SArray* readerList = taosArrayInit(taosArrayGetSize(uidList), sizeof(SMetaReader)); +// SArray* uidVersion = taosArrayInit(taosArrayGetSize(uidList), sizeof(STbDbKey)); +// SMeta *pMeta = meta; +// int64_t version; +// SHashObj *uHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); +// +// int64_t stt1 = taosGetTimestampUs(); +// for(int i = 0; i < taosArrayGetSize(uidList); i++) { +// void* ppVal = NULL; +// int vlen = 0; +// uint64_t * uid = taosArrayGet(uidList, i); +// // query uid.idx +// if (tdbTbGet(pMeta->pUidIdx, uid, sizeof(*uid), &ppVal, &vlen) < 0) { +// continue; +// } +// version = *(int64_t *)ppVal; +// +// STbDbKey tbDbKey = {.version = version, .uid = *uid}; +// taosArrayPush(uidVersion, &tbDbKey); +// taosHashPut(uHash, uid, sizeof(int64_t), ppVal, sizeof(int64_t)); +// } +// int64_t stt2 = taosGetTimestampUs(); +// qDebug("metaGetTableEntryByUidTest1 rows:%d, cost:%ld us", taosArrayGetSize(uidList), stt2-stt1); +// +// TBC *pCur = NULL; +// tdbTbcOpen(pMeta->pTbDb, &pCur, NULL); +// tdbTbcMoveToFirst(pCur); +// void *pKey = NULL; +// int kLen = 0; +// +// while(1){ +// SMetaReader pReader = {0}; +// int32_t ret = tdbTbcNext(pCur, &pKey, &kLen, &pReader.pBuf, &pReader.szBuf); +// if (ret < 0) break; +// STbDbKey *tmp = (STbDbKey*)pKey; +// int64_t *ver = (int64_t*)taosHashGet(uHash, &tmp->uid, sizeof(int64_t)); +// if(ver == NULL || *ver != tmp->version) continue; +// taosArrayPush(readerList, &pReader); +// } +// tdbTbcClose(pCur); +// +// taosArrayClear(readerList); +// int64_t stt3 = taosGetTimestampUs(); +// qDebug("metaGetTableEntryByUidTest2 rows:%d, cost:%ld us", taosArrayGetSize(uidList), stt3-stt2); +// for(int i = 0; i < taosArrayGetSize(uidVersion); i++) { +// SMetaReader pReader = {0}; +// +// STbDbKey *tbDbKey = taosArrayGet(uidVersion, i); +// // query table.db +// if (tdbTbGet(pMeta->pTbDb, tbDbKey, sizeof(STbDbKey), &pReader.pBuf, &pReader.szBuf) < 0) { +// continue; +// } +// taosArrayPush(readerList, &pReader); +// } +// int64_t stt4 = taosGetTimestampUs(); +// qDebug("metaGetTableEntryByUidTest3 rows:%d, cost:%ld us", taosArrayGetSize(uidList), stt4-stt3); +// +// for(int i = 0; i < taosArrayGetSize(readerList); i++){ +// SMetaReader* pReader = taosArrayGet(readerList, i); +// metaReaderInit(pReader, meta, 0); +// // decode the entry +// tDecoderInit(&pReader->coder, pReader->pBuf, pReader->szBuf); +// +// if (metaDecodeEntry(&pReader->coder, &pReader->me) < 0) { +// } +// metaReaderClear(pReader); +// } +// int64_t stt5 = taosGetTimestampUs(); +// qDebug("metaGetTableEntryByUidTest4 rows:%d, cost:%ld us", taosArrayGetSize(readerList), stt5-stt4); +// return 0; +//} int metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid) { SMeta *pMeta = pReader->pMeta; diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 2e82ab3d89..449f357685 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -396,7 +396,7 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, uint64_t suid, SArray blockDataAppendColInfo(pResBlock, &colInfo); } - int64_t stt = taosGetTimestampUs(); +// int64_t stt = taosGetTimestampUs(); tags = taosArrayInit(8, POINTER_BYTES); code = metaGetTableTags(metaHandle, suid, uidList, tags); if (code != TSDB_CODE_SUCCESS) { @@ -408,8 +408,8 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, uint64_t suid, SArray if(rows == 0){ goto end; } - int64_t stt1 = taosGetTimestampUs(); - qDebug("generate tag meta rows:%d, cost:%ld us", rows, stt1-stt); +// int64_t stt1 = taosGetTimestampUs(); +// qDebug("generate tag meta rows:%d, cost:%ld us", rows, stt1-stt); code = blockDataEnsureCapacity(pResBlock, rows); if (code != TSDB_CODE_SUCCESS) { @@ -417,7 +417,7 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, uint64_t suid, SArray goto end; } - int64_t st = taosGetTimestampUs(); +// int64_t st = taosGetTimestampUs(); for (int32_t i = 0; i < rows; i++) { void* tag = taosArrayGetP(tags, i); int64_t* uid = taosArrayGet(uidList, i); @@ -451,8 +451,8 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, uint64_t suid, SArray } pResBlock->info.rows = rows; - int64_t st1 = taosGetTimestampUs(); - qDebug("generate tag block rows:%d, cost:%ld us", rows, st1-st); +// int64_t st1 = taosGetTimestampUs(); +// qDebug("generate tag block rows:%d, cost:%ld us", rows, st1-st); pBlockList = taosArrayInit(2, POINTER_BYTES); taosArrayPush(pBlockList, &pResBlock); @@ -467,8 +467,8 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, uint64_t suid, SArray if(code != TSDB_CODE_SUCCESS){ terrno = code; } - int64_t st2 = taosGetTimestampUs(); - qDebug("calculate tag block rows:%d, cost:%ld us", rows, st2-st1); +// int64_t st2 = taosGetTimestampUs(); +// qDebug("calculate tag block rows:%d, cost:%ld us", rows, st2-st1); end: taosArrayDestroyP(tags, taosMemoryFree); @@ -497,15 +497,15 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, SIndexMetaArg metaArg = { .metaEx = metaHandle, .idx = tsdbGetIdx(metaHandle), .ivtIdx = tsdbGetIvtIdx(metaHandle), .suid = tableUid}; - int64_t stt = taosGetTimestampUs(); +// int64_t stt = taosGetTimestampUs(); SIdxFltStatus status = SFLT_NOT_INDEX; code = doFilterTag(pTagIndexCond, &metaArg, res, &status); if (code != 0 || status == SFLT_NOT_INDEX) { qError("failed to get tableIds from index, reason:%s, suid:%" PRIu64, tstrerror(code), tableUid); } - int64_t stt1 = taosGetTimestampUs(); - qDebug("generate table list, cost:%ld us", stt1-stt); +// int64_t stt1 = taosGetTimestampUs(); +// qDebug("generate table list, cost:%ld us", stt1-stt); }else if(!pTagCond){ vnodeGetCtbIdList(pVnode, pScanNode->suid, res); } From 7c82b1221f69fdfcc36bf36bac0a2be849400585 Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Tue, 16 Aug 2022 17:04:35 +0800 Subject: [PATCH 14/33] fix: return error if stable's name duplicate with child table --- source/dnode/vnode/src/meta/metaTable.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index e56b8ad939..079c89cc36 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -176,6 +176,15 @@ int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { // validate req metaReaderInit(&mr, pMeta, 0); if (metaGetTableEntryByName(&mr, pReq->name) == 0) { + if (mr.me.type == TSDB_SUPER_TABLE) { + metaReaderClear(&mr); + return 0; + } else { + terrno = TSDB_CODE_TDB_STB_ALREADY_EXIST; + metaReaderClear(&mr); + return -1; + } + /* // TODO: just for pass case #if 0 terrno = TSDB_CODE_TDB_STB_ALREADY_EXIST; @@ -185,6 +194,7 @@ int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { metaReaderClear(&mr); return 0; #endif + */ } metaReaderClear(&mr); From 4c4f5bdf5fbd0cd4eeb0ef783b259a2824199de1 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 16 Aug 2022 19:28:16 +0800 Subject: [PATCH 15/33] fix: the supertable and subtable can have same name --- source/dnode/vnode/src/meta/metaTable.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 079c89cc36..a1c1215a92 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -180,14 +180,14 @@ int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { metaReaderClear(&mr); return 0; } else { - terrno = TSDB_CODE_TDB_STB_ALREADY_EXIST; + terrno = TSDB_CODE_TDB_TABLE_ALREADY_EXIST; metaReaderClear(&mr); return -1; } /* // TODO: just for pass case #if 0 - terrno = TSDB_CODE_TDB_STB_ALREADY_EXIST; + terrno = TSDB_CODE_TDB_TABLE_ALREADY_EXIST; metaReaderClear(&mr); return -1; #else From e73a311a08be329ea15c35e021695afe32d80ea1 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 16 Aug 2022 19:30:32 +0800 Subject: [PATCH 16/33] test: add case for dupliate supertable name --- tests/script/tsim/db/basic2.sim | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/script/tsim/db/basic2.sim b/tests/script/tsim/db/basic2.sim index 114adf98e6..2ba7d806af 100644 --- a/tests/script/tsim/db/basic2.sim +++ b/tests/script/tsim/db/basic2.sim @@ -3,6 +3,21 @@ system sh/deploy.sh -n dnode1 -i 1 system sh/exec.sh -n dnode1 -s start sql connect +print =============== conflict stb +sql create database db vgroups 1; +sql use db; +sql create table stb (ts timestamp, i int) tags (j int); +sql_error create table stb using stb tags (1); +sql_error create table stb (ts timestamp, i int); + +sql create table ctb (ts timestamp, i int); +sql_error create table ctb (ts timestamp, i int) tags (j int); + +sql create table ntb (ts timestamp, i int); +sql_error create table ntb (ts timestamp, i int) tags (j int); + +sql drop database db + print =============== create database d1 sql create database d1 sql use d1 From eee4c0853d6df2bff2a997305f709a104eafdca8 Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 17 Aug 2022 10:34:53 +0800 Subject: [PATCH 17/33] refactor(sync): add syncNodeAppendEntriesOnePeer --- include/libs/sync/sync.h | 11 +-- source/libs/sync/inc/syncReplication.h | 2 + source/libs/sync/src/syncReplication.c | 115 +++++++++++++++++++++++++ 3 files changed, 123 insertions(+), 5 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 6d8895eb96..7cd2ebdede 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -26,11 +26,12 @@ extern "C" { extern bool gRaftDetailLog; -#define SYNC_RESP_TTL_MS 10000000 -#define SYNC_SPEED_UP_HB_TIMER 400 -#define SYNC_SPEED_UP_AFTER_MS (1000 * 20) -#define SYNC_SLOW_DOWN_RANGE 100 -#define SYNC_MAX_READ_RANGE 10 +#define SYNC_RESP_TTL_MS 10000000 +#define SYNC_SPEED_UP_HB_TIMER 400 +#define SYNC_SPEED_UP_AFTER_MS (1000 * 20) +#define SYNC_SLOW_DOWN_RANGE 100 +#define SYNC_MAX_READ_RANGE 10 +#define SYNC_MAX_PROGRESS_WAIT_MS 4000 #define SYNC_MAX_BATCH_SIZE 1 #define SYNC_INDEX_BEGIN 0 diff --git a/source/libs/sync/inc/syncReplication.h b/source/libs/sync/inc/syncReplication.h index 21821be6c7..edce124ee5 100644 --- a/source/libs/sync/inc/syncReplication.h +++ b/source/libs/sync/inc/syncReplication.h @@ -55,6 +55,8 @@ int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode); int32_t syncNodeAppendEntriesPeersSnapshot(SSyncNode* pSyncNode); int32_t syncNodeAppendEntriesPeersSnapshot2(SSyncNode* pSyncNode); +int32_t syncNodeAppendEntriesOnePeer(SSyncNode* pSyncNode, SRaftId* pDestId, SyncIndex nextIndex); + int32_t syncNodeReplicate(SSyncNode* pSyncNode, bool isTimer); int32_t syncNodeAppendEntries(SSyncNode* pSyncNode, const SRaftId* destRaftId, const SyncAppendEntries* pMsg); int32_t syncNodeAppendEntriesBatch(SSyncNode* pSyncNode, const SRaftId* destRaftId, const SyncAppendEntriesBatch* pMsg); diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index 24f75de5d3..886f7ad199 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -116,6 +116,120 @@ int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode) { return ret; } +int32_t syncNodeAppendEntriesOnePeer(SSyncNode* pSyncNode, SRaftId* pDestId, SyncIndex nextIndex) { + int32_t ret = 0; + + // pre index, pre term + SyncIndex preLogIndex = syncNodeGetPreIndex(pSyncNode, nextIndex); + SyncTerm preLogTerm = syncNodeGetPreTerm(pSyncNode, nextIndex); + if (preLogTerm == SYNC_TERM_INVALID) { + SyncIndex newNextIndex = syncNodeGetLastIndex(pSyncNode) + 1; + // SyncIndex newNextIndex = nextIndex + 1; + + syncIndexMgrSetIndex(pSyncNode->pNextIndex, pDestId, newNextIndex); + syncIndexMgrSetIndex(pSyncNode->pMatchIndex, pDestId, SYNC_INDEX_INVALID); + sError("vgId:%d, sync get pre term error, nextIndex:%" PRId64 ", update next-index:%" PRId64 + ", match-index:%d, raftid:%" PRId64, + pSyncNode->vgId, nextIndex, newNextIndex, SYNC_INDEX_INVALID, pDestId->addr); + return -1; + } + + // entry pointer array + SSyncRaftEntry* entryPArr[SYNC_MAX_BATCH_SIZE]; + memset(entryPArr, 0, sizeof(entryPArr)); + + // get entry batch + int32_t getCount = 0; + SyncIndex getEntryIndex = nextIndex; + for (int32_t i = 0; i < pSyncNode->pRaftCfg->batchSize; ++i) { + SSyncRaftEntry* pEntry = NULL; + int32_t code = pSyncNode->pLogStore->syncLogGetEntry(pSyncNode->pLogStore, getEntryIndex, &pEntry); + if (code == 0) { + ASSERT(pEntry != NULL); + entryPArr[i] = pEntry; + getCount++; + getEntryIndex++; + + } else { + break; + } + } + + // event log + do { + char logBuf[128]; + char host[64]; + uint16_t port; + syncUtilU642Addr(pDestId->addr, host, sizeof(host), &port); + snprintf(logBuf, sizeof(logBuf), "build batch:%d for %s:%d", getCount, host, port); + syncNodeEventLog(pSyncNode, logBuf); + } while (0); + + // build msg + SyncAppendEntriesBatch* pMsg = syncAppendEntriesBatchBuild(entryPArr, getCount, pSyncNode->vgId); + ASSERT(pMsg != NULL); + + // free entries + for (int32_t i = 0; i < pSyncNode->pRaftCfg->batchSize; ++i) { + SSyncRaftEntry* pEntry = entryPArr[i]; + if (pEntry != NULL) { + syncEntryDestory(pEntry); + entryPArr[i] = NULL; + } + } + + // prepare msg + pMsg->srcId = pSyncNode->myRaftId; + pMsg->destId = *pDestId; + pMsg->term = pSyncNode->pRaftStore->currentTerm; + pMsg->prevLogIndex = preLogIndex; + pMsg->prevLogTerm = preLogTerm; + pMsg->commitIndex = pSyncNode->commitIndex; + pMsg->privateTerm = 0; + pMsg->dataCount = getCount; + + // send msg + syncNodeAppendEntriesBatch(pSyncNode, pDestId, pMsg); + + // speed up + if (pMsg->dataCount > 0 && pSyncNode->commitIndex - pMsg->prevLogIndex > SYNC_SLOW_DOWN_RANGE) { + ret = 1; + +#if 0 + do { + char logBuf[128]; + char host[64]; + uint16_t port; + syncUtilU642Addr(pDestId->addr, host, sizeof(host), &port); + snprintf(logBuf, sizeof(logBuf), "maybe speed up for %s:%d, pre-index:%ld", host, port, pMsg->prevLogIndex); + syncNodeEventLog(pSyncNode, logBuf); + } while (0); +#endif + } + + syncAppendEntriesBatchDestroy(pMsg); + + return ret; +} + +int32_t syncNodeAppendEntriesPeersSnapshot2(SSyncNode* pSyncNode) { + if (pSyncNode->state != TAOS_SYNC_STATE_LEADER) { + return -1; + } + + int32_t ret = 0; + for (int i = 0; i < pSyncNode->peersNum; ++i) { + SRaftId* pDestId = &(pSyncNode->peersId[i]); + + // next index + SyncIndex nextIndex = syncIndexMgrGetIndex(pSyncNode->pNextIndex, pDestId); + ret = syncNodeAppendEntriesOnePeer(pSyncNode, pDestId, nextIndex); + } + + return ret; +} + +#if 0 int32_t syncNodeAppendEntriesPeersSnapshot2(SSyncNode* pSyncNode) { if (pSyncNode->state != TAOS_SYNC_STATE_LEADER) { return -1; @@ -221,6 +335,7 @@ int32_t syncNodeAppendEntriesPeersSnapshot2(SSyncNode* pSyncNode) { return ret; } +#endif int32_t syncNodeAppendEntriesPeersSnapshot(SSyncNode* pSyncNode) { ASSERT(pSyncNode->state == TAOS_SYNC_STATE_LEADER); From 2bf2d5e6e1226252a0aaef69b5d4af1b1e692539 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 17 Aug 2022 11:39:55 +0800 Subject: [PATCH 18/33] fix:error in get table list by tag filter --- source/dnode/vnode/src/meta/metaQuery.c | 20 +- source/libs/executor/src/executil.c | 29 ++- source/libs/scalar/src/sclvector.c | 5 +- tests/system-test/2-query/json_tag.py | 295 ++++++++++++++++++++++++ 4 files changed, 336 insertions(+), 13 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index b302897907..9f2cf409be 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -965,10 +965,11 @@ END: int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList, SArray *tags) { SMCtbCursor *pCur = metaOpenCtbCursor(pMeta, suid); - SHashObj *uHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); - size_t len = taosArrayGetSize(uidList); - if (len > 0) { - for (int i = 0; i < len; i++) { + SHashObj *uHash = NULL; + size_t len = taosArrayGetSize(uidList); // len > 0 means there already have uids + if(len > 0){ + uHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + for(int i = 0; i < len; i++){ int64_t *uid = taosArrayGet(uidList, i); taosHashPut(uHash, uid, sizeof(int64_t), &i, sizeof(i)); } @@ -985,13 +986,22 @@ int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList, SArray *t void *tag = taosMemoryMalloc(pCur->vLen); memcpy(tag, pCur->pVal, pCur->vLen); - taosArrayPush(tags, &tag); if (len == 0) { taosArrayPush(uidList, &id); + taosArrayPush(tags, &tag); + }else{ + taosHashPut(uHash, &id, sizeof(int64_t), &tag, POINTER_BYTES); } } + for(int i = 0; i < len; i++){ + int64_t *uid = taosArrayGet(uidList, i); + void **tag = taosHashGet(uHash, uid, POINTER_BYTES); + taosArrayPush(tags, tag); + } + + taosHashCleanup(uHash); metaCloseCtbCursor(pCur); return TSDB_CODE_SUCCESS; diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 449f357685..f8164e0947 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -321,6 +321,8 @@ static EDealRes getColumn(SNode** pNode, void* pContext) { pSColumnNode->node.resType.bytes = TSDB_TABLE_FNAME_LEN - 1 + VARSTR_HEADER_SIZE; nodesDestroyNode(*pNode); *pNode = (SNode*)pSColumnNode; + }else{ + return DEAL_RES_CONTINUE; } }else{ return DEAL_RES_CONTINUE; @@ -423,11 +425,16 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, uint64_t suid, SArray int64_t* uid = taosArrayGet(uidList, i); for(int32_t j = 0; j < taosArrayGetSize(pResBlock->pDataBlock); j++){ SColumnInfoData* pColInfo = (SColumnInfoData*)taosArrayGet(pResBlock->pDataBlock, j); + + char str[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + metaGetTableNameByUid(metaHandle, *uid, str); + colDataAppend(pColInfo, i, str, false); + if(pColInfo->info.colId == -1){ // tbname - char str[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; - metaGetTableNameByUid(metaHandle, *uid, str); - colDataAppend(pColInfo, i, str, false); - qDebug("tbnameget uid:%ld, tbname:%s", *uid, str+2); +// char str[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; +// metaGetTableNameByUid(metaHandle, *uid, str); +// colDataAppend(pColInfo, i, str, false); +// qDebug("tagfilter uid:%ld, tbname:%s", *uid, str+2); }else{ STagVal tagVal = {0}; tagVal.cid = pColInfo->info.colId; @@ -445,6 +452,16 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, uint64_t suid, SArray taosMemoryFree(tmp); } else { colDataAppend(pColInfo, i, (const char*)&tagVal.i64, false); + + if(pColInfo->info.type == TSDB_DATA_TYPE_TINYINT){ + int8_t tint = *(int8_t*)(&tagVal.i64); + qDebug("tagfilter uid:%ld, tbname:%s, tint:%d", *uid, str+2, tint); + + }else if(pColInfo->info.type == TSDB_DATA_TYPE_INT){ + int nint = *(int*)(&tagVal.i64); + qDebug("tagfilter uid:%ld, tbname:%s nint:+%d", *uid, str+2, nint); + + } } } } @@ -529,7 +546,7 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); int64_t* uid = taosArrayGet(res, i); - qDebug("tbnameget get uid:%ld, res:%d", *uid, *(bool*)var); + qDebug("tagfilter get uid:%ld, res:%d", *uid, *(bool*)var); if (*(bool*)var == false) { taosArrayRemove(res, i); j++; @@ -545,7 +562,7 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, for (int i = 0; i < taosArrayGetSize(res); i++) { STableKeyInfo info = {.uid = *(uint64_t*)taosArrayGet(res, i), .groupId = 0}; taosArrayPush(pListInfo->pTableList, &info); - qDebug("tbnameget get uid:%ld", info.uid); + qDebug("tagfilter get uid:%ld", info.uid); } taosArrayDestroy(res); diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index aaa70ef5ae..4fc0005e4d 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -1672,8 +1672,9 @@ void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, colDataAppendInt8(pOut->columnData, i, (int8_t*)&result);\ }else{\ bool res = filterDoCompare(fp, optr, pLeftData, pRightData);\ - colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);\ - }\ + colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); \ + if(GET_PARAM_TYPE(pLeft) == TSDB_DATA_TYPE_BIGINT){qDebug("tagfilter left:%d, right:%d, res:%d", *(int64_t*)(pLeftData), *(int64_t*)(pRightData), res);} \ + } \ if(freeLeft) taosMemoryFreeClear(pLeftData);\ if(freeRight) taosMemoryFreeClear(pRightData);\ } diff --git a/tests/system-test/2-query/json_tag.py b/tests/system-test/2-query/json_tag.py index 856d764747..9b4379c011 100644 --- a/tests/system-test/2-query/json_tag.py +++ b/tests/system-test/2-query/json_tag.py @@ -698,3 +698,298 @@ class TDTestCase: tdCases.addWindows(__file__, TDTestCase()) tdCases.addLinux(__file__, TDTestCase()) + +08/16 17:52:17.220901 00059267 QRY tagfilter uid:4871209028224417813, tbname:t3 nint:+2 +08/16 17:52:17.220973 00059267 QRY tagfilter uid:4871209028224417813, tbname:t3, tint:3 +08/16 17:52:17.220992 00059267 QRY tagfilter uid:4871209028228349987, tbname:t10 nint:+3 +08/16 17:52:17.221014 00059267 QRY tagfilter uid:4871209028228349987, tbname:t10, tint:4 +08/16 17:52:17.221042 00059267 QRY tagfilter uid:4871209028232282161, tbname:t17 nint:+4 +08/16 17:52:17.221057 00059267 QRY tagfilter uid:4871209028232282161, tbname:t17, tint:5 +08/16 17:52:17.221580 00059267 QRY tagfilter uid:4871209028236083263, tbname:t24 nint:+5 +08/16 17:52:17.221627 00059267 QRY tagfilter uid:4871209028236083263, tbname:t24, tint:6 +08/16 17:52:17.221641 00059267 QRY tagfilter uid:4871209028240015437, tbname:t31 nint:+2 +08/16 17:52:17.221654 00059267 QRY tagfilter uid:4871209028240015437, tbname:t31, tint:3 +08/16 17:52:17.221974 00059267 QRY tagfilter uid:4871209028244013147, tbname:t38 nint:+3 +08/16 17:52:17.222022 00059267 QRY tagfilter uid:4871209028244013147, tbname:t38, tint:4 +08/16 17:52:17.222040 00059267 QRY tagfilter uid:4871209028247879785, tbname:t45 nint:+4 +08/16 17:52:17.222054 00059267 QRY tagfilter uid:4871209028247879785, tbname:t45, tint:5 +08/16 17:52:17.222068 00059267 QRY tagfilter uid:4871209028251877495, tbname:t52 nint:+5 +08/16 17:52:17.222081 00059267 QRY tagfilter uid:4871209028251877495, tbname:t52, tint:6 +08/16 17:52:17.222167 00059267 QRY tagfilter uid:4871209028255744133, tbname:t59 nint:+2 +08/16 17:52:17.222192 00059267 QRY tagfilter uid:4871209028255744133, tbname:t59, tint:3 +08/16 17:52:17.222210 00059267 QRY tagfilter uid:4871209028259741843, tbname:t66 nint:+3 +08/16 17:52:17.222490 00059267 QRY tagfilter uid:4871209028259741843, tbname:t66, tint:4 +08/16 17:52:17.222522 00059267 QRY tagfilter uid:4871209028263805089, tbname:t73 nint:+4 +08/16 17:52:17.222551 00059267 QRY tagfilter uid:4871209028263805089, tbname:t73, tint:5 +08/16 17:52:17.222809 00059267 QRY tagfilter uid:4871209028267802799, tbname:t80 nint:+5 +08/16 17:52:17.222855 00059267 QRY tagfilter uid:4871209028267802799, tbname:t80, tint:6 +08/16 17:52:17.222874 00059267 QRY tagfilter uid:4871209028272062653, tbname:t87 nint:+2 +08/16 17:52:17.222934 00059267 QRY tagfilter uid:4871209028272062653, tbname:t87, tint:3 +08/16 17:52:17.222976 00059267 QRY tagfilter uid:4871209028276322508, tbname:t94 nint:+3 +08/16 17:52:17.222995 00059267 QRY tagfilter uid:4871209028276322508, tbname:t94, tint:4 +08/16 17:52:17.223013 00059267 QRY tagfilter uid:4871209028224942103, tbname:t4 nint:+4 +08/16 17:52:17.223029 00059267 QRY tagfilter uid:4871209028224942103, tbname:t4, tint:5 +08/16 17:52:17.223047 00059267 QRY tagfilter uid:4871209028228874277, tbname:t11 nint:+5 +08/16 17:52:17.223064 00059267 QRY tagfilter uid:4871209028228874277, tbname:t11, tint:6 +08/16 17:52:17.223079 00059267 QRY tagfilter uid:4871209028232871987, tbname:t18 nint:+2 +08/16 17:52:17.223093 00059267 QRY tagfilter uid:4871209028232871987, tbname:t18, tint:3 +08/16 17:52:17.223109 00059267 QRY tagfilter uid:4871209028236607553, tbname:t25 nint:+3 +08/16 17:52:17.223125 00059267 QRY tagfilter uid:4871209028236607553, tbname:t25, tint:4 +08/16 17:52:17.223141 00059267 QRY tagfilter uid:4871209028240605263, tbname:t32 nint:+4 +08/16 17:52:17.223156 00059267 QRY tagfilter uid:4871209028240605263, tbname:t32, tint:5 +08/16 17:52:17.223172 00059267 QRY tagfilter uid:4871209028244602973, tbname:t39 nint:+5 +08/16 17:52:17.223188 00059267 QRY tagfilter uid:4871209028244602973, tbname:t39, tint:6 +08/16 17:52:17.223204 00059267 QRY tagfilter uid:4871209028248404075, tbname:t46 nint:+2 +08/16 17:52:17.223219 00059267 QRY tagfilter uid:4871209028248404075, tbname:t46, tint:3 +08/16 17:52:17.223564 00059267 QRY tagfilter uid:4871209028252401785, tbname:t53 nint:+3 +08/16 17:52:17.223598 00059267 QRY tagfilter uid:4871209028252401785, tbname:t53, tint:4 +08/16 17:52:17.223614 00059267 QRY tagfilter uid:4871209028256333959, tbname:t60 nint:+4 +08/16 17:52:17.223638 00059267 QRY tagfilter uid:4871209028256333959, tbname:t60, tint:5 +08/16 17:52:17.223771 00059267 QRY tagfilter uid:4871209028260397205, tbname:t67 nint:+5 +08/16 17:52:17.223813 00059267 QRY tagfilter uid:4871209028260397205, tbname:t67, tint:6 +08/16 17:52:17.223828 00059267 QRY tagfilter uid:4871209028264525987, tbname:t74 nint:+2 +08/16 17:52:17.223858 00059267 QRY tagfilter uid:4871209028264525987, tbname:t74, tint:3 +08/16 17:52:17.223895 00059267 QRY tagfilter uid:4871209028268392625, tbname:t81 nint:+3 +08/16 17:52:17.223917 00059267 QRY tagfilter uid:4871209028268392625, tbname:t81, tint:4 +08/16 17:52:17.223953 00059267 QRY tagfilter uid:4871209028272718015, tbname:t88 nint:+4 +08/16 17:52:17.223977 00059267 QRY tagfilter uid:4871209028272718015, tbname:t88, tint:5 +08/16 17:52:17.223991 00059267 QRY tagfilter uid:4871209028276977870, tbname:t95 nint:+5 +08/16 17:52:17.224003 00059267 QRY tagfilter uid:4871209028276977870, tbname:t95, tint:6 +08/16 17:52:17.224018 00059267 QRY tagfilter uid:4871209028225466393, tbname:t5 nint:+2 +08/16 17:52:17.224030 00059267 QRY tagfilter uid:4871209028225466393, tbname:t5, tint:3 +08/16 17:52:17.224044 00059267 QRY tagfilter uid:4871209028229398567, tbname:t12 nint:+3 +08/16 17:52:17.224056 00059267 QRY tagfilter uid:4871209028229398567, tbname:t12, tint:4 +08/16 17:52:17.224071 00059267 QRY tagfilter uid:4871209028233396277, tbname:t19 nint:+4 +08/16 17:52:17.224084 00059267 QRY tagfilter uid:4871209028233396277, tbname:t19, tint:5 +08/16 17:52:17.224097 00059267 QRY tagfilter uid:4871209028237197379, tbname:t26 nint:+5 +08/16 17:52:17.224110 00059267 QRY tagfilter uid:4871209028237197379, tbname:t26, tint:6 +08/16 17:52:17.224125 00059267 QRY tagfilter uid:4871209028241195089, tbname:t33 nint:+2 +08/16 17:52:17.224139 00059267 QRY tagfilter uid:4871209028241195089, tbname:t33, tint:3 +08/16 17:52:17.224188 00059267 QRY tagfilter uid:4871209028245127263, tbname:t40 nint:+3 +08/16 17:52:17.224209 00059267 QRY tagfilter uid:4871209028245127263, tbname:t40, tint:4 +08/16 17:52:17.224459 00059267 QRY tagfilter uid:4871209028248993901, tbname:t47 nint:+4 +08/16 17:52:17.224479 00059267 QRY tagfilter uid:4871209028248993901, tbname:t47, tint:5 +08/16 17:52:17.224494 00059267 QRY tagfilter uid:4871209028252926075, tbname:t54 nint:+5 +08/16 17:52:17.224507 00059267 QRY tagfilter uid:4871209028252926075, tbname:t54, tint:6 +08/16 17:52:17.224522 00059267 QRY tagfilter uid:4871209028256858249, tbname:t61 nint:+2 +08/16 17:52:17.224535 00059267 QRY tagfilter uid:4871209028256858249, tbname:t61, tint:3 +08/16 17:52:17.224549 00059267 QRY tagfilter uid:4871209028260987031, tbname:t68 nint:+3 +08/16 17:52:17.224564 00059267 QRY tagfilter uid:4871209028260987031, tbname:t68, tint:4 +08/16 17:52:17.224664 00059267 QRY tagfilter uid:4871209028265050277, tbname:t75 nint:+4 +08/16 17:52:17.224717 00059267 QRY tagfilter uid:4871209028265050277, tbname:t75, tint:5 +08/16 17:52:17.224731 00059267 QRY tagfilter uid:4871209028269047987, tbname:t82 nint:+5 +08/16 17:52:17.224769 00059267 QRY tagfilter uid:4871209028269047987, tbname:t82, tint:6 +08/16 17:52:17.224783 00059267 QRY tagfilter uid:4871209028273307841, tbname:t89 nint:+2 +08/16 17:52:17.224923 00059267 QRY tagfilter uid:4871209028273307841, tbname:t89, tint:3 +08/16 17:52:17.224990 00059267 QRY tagfilter uid:4871209028277502160, tbname:t96 nint:+3 +08/16 17:52:17.225006 00059267 QRY tagfilter uid:4871209028277502160, tbname:t96, tint:4 +08/16 17:52:17.225018 00059267 QRY tagfilter uid:4871209028225990683, tbname:t6 nint:+4 +08/16 17:52:17.225029 00059267 QRY tagfilter uid:4871209028225990683, tbname:t6, tint:5 +08/16 17:52:17.225039 00059267 QRY tagfilter uid:4871209028229922857, tbname:t13 nint:+5 +08/16 17:52:17.225053 00059267 QRY tagfilter uid:4871209028229922857, tbname:t13, tint:6 +08/16 17:52:17.225072 00059267 QRY tagfilter uid:4871209028233986103, tbname:t20 nint:+2 +08/16 17:52:17.225082 00059267 QRY tagfilter uid:4871209028233986103, tbname:t20, tint:3 +08/16 17:52:17.225093 00059267 QRY tagfilter uid:4871209028237721669, tbname:t27 nint:+3 +08/16 17:52:17.225103 00059267 QRY tagfilter uid:4871209028237721669, tbname:t27, tint:4 +08/16 17:52:17.225113 00059267 QRY tagfilter uid:4871209028241784915, tbname:t34 nint:+4 +08/16 17:52:17.225368 00059267 QRY tagfilter uid:4871209028241784915, tbname:t34, tint:5 +08/16 17:52:17.225409 00059267 QRY tagfilter uid:4871209028245717089, tbname:t41 nint:+5 +08/16 17:52:17.225437 00059267 QRY tagfilter uid:4871209028245717089, tbname:t41, tint:6 +08/16 17:52:17.225462 00059267 QRY tagfilter uid:4871209028249518191, tbname:t48 nint:+2 +08/16 17:52:17.225475 00059267 QRY tagfilter uid:4871209028249518191, tbname:t48, tint:3 +08/16 17:52:17.225486 00059267 QRY tagfilter uid:4871209028253450365, tbname:t55 nint:+3 +08/16 17:52:17.225496 00059267 QRY tagfilter uid:4871209028253450365, tbname:t55, tint:4 +08/16 17:52:17.225507 00059267 QRY tagfilter uid:4871209028257382539, tbname:t62 nint:+4 +08/16 17:52:17.225532 00059267 QRY tagfilter uid:4871209028257382539, tbname:t62, tint:5 +08/16 17:52:17.225544 00059267 QRY tagfilter uid:4871209028261511321, tbname:t69 nint:+5 +08/16 17:52:17.225554 00059267 QRY tagfilter uid:4871209028261511321, tbname:t69, tint:6 +08/16 17:52:17.225564 00059267 QRY tagfilter uid:4871209028265574567, tbname:t76 nint:+2 +08/16 17:52:17.225575 00059267 QRY tagfilter uid:4871209028265574567, tbname:t76, tint:3 +08/16 17:52:17.225585 00059267 QRY tagfilter uid:4871209028269637813, tbname:t83 nint:+3 +08/16 17:52:17.225615 00059267 QRY tagfilter uid:4871209028269637813, tbname:t83, tint:4 +08/16 17:52:17.225640 00059267 QRY tagfilter uid:4871209028273963203, tbname:t90 nint:+4 +08/16 17:52:17.225670 00059267 QRY tagfilter uid:4871209028273963203, tbname:t90, tint:5 +08/16 17:52:17.225698 00059267 QRY tagfilter uid:4871209028278026450, tbname:t97 nint:+5 +08/16 17:52:17.225709 00059267 QRY tagfilter uid:4871209028278026450, tbname:t97, tint:6 +08/16 17:52:17.226570 00059267 QRY tagfilter left:2, right:3, res:1 +08/16 17:52:17.226623 00059267 QRY tagfilter left:3, right:3, res:0 +08/16 17:52:17.226642 00059267 QRY tagfilter left:4, right:3, res:0 +08/16 17:52:17.226684 00059267 QRY tagfilter left:5, right:3, res:0 +08/16 17:52:17.226733 00059267 QRY tagfilter left:2, right:3, res:1 +08/16 17:52:17.226765 00059267 QRY tagfilter left:3, right:3, res:0 +08/16 17:52:17.226773 00059267 QRY tagfilter left:4, right:3, res:0 +08/16 17:52:17.226780 00059267 QRY tagfilter left:5, right:3, res:0 +08/16 17:52:17.226787 00059267 QRY tagfilter left:2, right:3, res:1 +08/16 17:52:17.226793 00059267 QRY tagfilter left:3, right:3, res:0 +08/16 17:52:17.226801 00059267 QRY tagfilter left:4, right:3, res:0 +08/16 17:52:17.226808 00059267 QRY tagfilter left:5, right:3, res:0 +08/16 17:52:17.226815 00059267 QRY tagfilter left:2, right:3, res:1 +08/16 17:52:17.226821 00059267 QRY tagfilter left:3, right:3, res:0 +08/16 17:52:17.226828 00059267 QRY tagfilter left:4, right:3, res:0 +08/16 17:52:17.226834 00059267 QRY tagfilter left:5, right:3, res:0 +08/16 17:52:17.226840 00059267 QRY tagfilter left:2, right:3, res:1 +08/16 17:52:17.226847 00059267 QRY tagfilter left:3, right:3, res:0 +08/16 17:52:17.226854 00059267 QRY tagfilter left:4, right:3, res:0 +08/16 17:52:17.226860 00059267 QRY tagfilter left:5, right:3, res:0 +08/16 17:52:17.226867 00059267 QRY tagfilter left:2, right:3, res:1 +08/16 17:52:17.226873 00059267 QRY tagfilter left:3, right:3, res:0 +08/16 17:52:17.226880 00059267 QRY tagfilter left:4, right:3, res:0 +08/16 17:52:17.226886 00059267 QRY tagfilter left:5, right:3, res:0 +08/16 17:52:17.226893 00059267 QRY tagfilter left:2, right:3, res:1 +08/16 17:52:17.226900 00059267 QRY tagfilter left:3, right:3, res:0 +08/16 17:52:17.226907 00059267 QRY tagfilter left:4, right:3, res:0 +08/16 17:52:17.226914 00059267 QRY tagfilter left:5, right:3, res:0 +08/16 17:52:17.226920 00059267 QRY tagfilter left:2, right:3, res:1 +08/16 17:52:17.226927 00059267 QRY tagfilter left:3, right:3, res:0 +08/16 17:52:17.226934 00059267 QRY tagfilter left:4, right:3, res:0 +08/16 17:52:17.226940 00059267 QRY tagfilter left:5, right:3, res:0 +08/16 17:52:17.227014 00059267 QRY tagfilter left:2, right:3, res:1 +08/16 17:52:17.227036 00059267 QRY tagfilter left:3, right:3, res:0 +08/16 17:52:17.227043 00059267 QRY tagfilter left:4, right:3, res:0 +08/16 17:52:17.227050 00059267 QRY tagfilter left:5, right:3, res:0 +08/16 17:52:17.227056 00059267 QRY tagfilter left:2, right:3, res:1 +08/16 17:52:17.227062 00059267 QRY tagfilter left:3, right:3, res:0 +08/16 17:52:17.227069 00059267 QRY tagfilter left:4, right:3, res:0 +08/16 17:52:17.227076 00059267 QRY tagfilter left:5, right:3, res:0 +08/16 17:52:17.227082 00059267 QRY tagfilter left:2, right:3, res:1 +08/16 17:52:17.227089 00059267 QRY tagfilter left:3, right:3, res:0 +08/16 17:52:17.227107 00059267 QRY tagfilter left:4, right:3, res:0 +08/16 17:52:17.227129 00059267 QRY tagfilter left:5, right:3, res:0 +08/16 17:52:17.227138 00059267 QRY tagfilter left:2, right:3, res:1 +08/16 17:52:17.227144 00059267 QRY tagfilter left:3, right:3, res:0 +08/16 17:52:17.227151 00059267 QRY tagfilter left:4, right:3, res:0 +08/16 17:52:17.227157 00059267 QRY tagfilter left:5, right:3, res:0 +08/16 17:52:17.227163 00059267 QRY tagfilter left:2, right:3, res:1 +08/16 17:52:17.227170 00059267 QRY tagfilter left:3, right:3, res:0 +08/16 17:52:17.227176 00059267 QRY tagfilter left:4, right:3, res:0 +08/16 17:52:17.227194 00059267 QRY tagfilter left:5, right:3, res:0 +08/16 17:52:17.227202 00059267 QRY tagfilter left:2, right:3, res:1 +08/16 17:52:17.227208 00059267 QRY tagfilter left:3, right:3, res:0 +08/16 17:52:17.227215 00059267 QRY tagfilter left:4, right:3, res:0 +08/16 17:52:17.227221 00059267 QRY tagfilter left:5, right:3, res:0 +08/16 17:52:17.227284 00059267 QRY tagfilter left:3, right:2, res:1 +08/16 17:52:17.227298 00059267 QRY tagfilter left:4, right:2, res:1 +08/16 17:52:17.227304 00059267 QRY tagfilter left:5, right:2, res:1 +08/16 17:52:17.227311 00059267 QRY tagfilter left:6, right:2, res:1 +08/16 17:52:17.227317 00059267 QRY tagfilter left:3, right:2, res:1 +08/16 17:52:17.227324 00059267 QRY tagfilter left:4, right:2, res:1 +08/16 17:52:17.227330 00059267 QRY tagfilter left:5, right:2, res:1 +08/16 17:52:17.227359 00059267 QRY tagfilter left:6, right:2, res:1 +08/16 17:52:17.227421 00059267 QRY tagfilter left:3, right:2, res:1 +08/16 17:52:17.227437 00059267 QRY tagfilter left:4, right:2, res:1 +08/16 17:52:17.227444 00059267 QRY tagfilter left:5, right:2, res:1 +08/16 17:52:17.227450 00059267 QRY tagfilter left:6, right:2, res:1 +08/16 17:52:17.227456 00059267 QRY tagfilter left:3, right:2, res:1 +08/16 17:52:17.227463 00059267 QRY tagfilter left:4, right:2, res:1 +08/16 17:52:17.227469 00059267 QRY tagfilter left:5, right:2, res:1 +08/16 17:52:17.227475 00059267 QRY tagfilter left:6, right:2, res:1 +08/16 17:52:17.227493 00059267 QRY tagfilter left:3, right:2, res:1 +08/16 17:52:17.227500 00059267 QRY tagfilter left:4, right:2, res:1 +08/16 17:52:17.227507 00059267 QRY tagfilter left:5, right:2, res:1 +08/16 17:52:17.227513 00059267 QRY tagfilter left:6, right:2, res:1 +08/16 17:52:17.227520 00059267 QRY tagfilter left:3, right:2, res:1 +08/16 17:52:17.227526 00059267 QRY tagfilter left:4, right:2, res:1 +08/16 17:52:17.227541 00059267 QRY tagfilter left:5, right:2, res:1 +08/16 17:52:17.227548 00059267 QRY tagfilter left:6, right:2, res:1 +08/16 17:52:17.227570 00059267 QRY tagfilter left:3, right:2, res:1 +08/16 17:52:17.227578 00059267 QRY tagfilter left:4, right:2, res:1 +08/16 17:52:17.227585 00059267 QRY tagfilter left:5, right:2, res:1 +08/16 17:52:17.227591 00059267 QRY tagfilter left:6, right:2, res:1 +08/16 17:52:17.227598 00059267 QRY tagfilter left:3, right:2, res:1 +08/16 17:52:17.227604 00059267 QRY tagfilter left:4, right:2, res:1 +08/16 17:52:17.227611 00059267 QRY tagfilter left:5, right:2, res:1 +08/16 17:52:17.227617 00059267 QRY tagfilter left:6, right:2, res:1 +08/16 17:52:17.227624 00059267 QRY tagfilter left:3, right:2, res:1 +08/16 17:52:17.227630 00059267 QRY tagfilter left:4, right:2, res:1 +08/16 17:52:17.227636 00059267 QRY tagfilter left:5, right:2, res:1 +08/16 17:52:17.227643 00059267 QRY tagfilter left:6, right:2, res:1 +08/16 17:52:17.227649 00059267 QRY tagfilter left:3, right:2, res:1 +08/16 17:52:17.227656 00059267 QRY tagfilter left:4, right:2, res:1 +08/16 17:52:17.227662 00059267 QRY tagfilter left:5, right:2, res:1 +08/16 17:52:17.227668 00059267 QRY tagfilter left:6, right:2, res:1 +08/16 17:52:17.227675 00059267 QRY tagfilter left:3, right:2, res:1 +08/16 17:52:17.227681 00059267 QRY tagfilter left:4, right:2, res:1 +08/16 17:52:17.227688 00059267 QRY tagfilter left:5, right:2, res:1 +08/16 17:52:17.227696 00059267 QRY tagfilter left:6, right:2, res:1 +08/16 17:52:17.227703 00059267 QRY tagfilter left:3, right:2, res:1 +08/16 17:52:17.227709 00059267 QRY tagfilter left:4, right:2, res:1 +08/16 17:52:17.227716 00059267 QRY tagfilter left:5, right:2, res:1 +08/16 17:52:17.227723 00059267 QRY tagfilter left:6, right:2, res:1 +08/16 17:52:17.227729 00059267 QRY tagfilter left:3, right:2, res:1 +08/16 17:52:17.227738 00059267 QRY tagfilter left:4, right:2, res:1 +08/16 17:52:17.227744 00059267 QRY tagfilter left:5, right:2, res:1 +08/16 17:52:17.227750 00059267 QRY tagfilter left:6, right:2, res:1 +08/16 17:52:17.227757 00059267 QRY tagfilter left:3, right:2, res:1 +08/16 17:52:17.227764 00059267 QRY tagfilter left:4, right:2, res:1 +08/16 17:52:17.227771 00059267 QRY tagfilter left:5, right:2, res:1 +08/16 17:52:17.227779 00059267 QRY tagfilter left:6, right:2, res:1 +08/16 17:52:17.227803 00059267 QRY tagfilter get uid:4871209028224417813, res:1 +08/16 17:52:17.227811 00059267 QRY tagfilter get uid:4871209028228349987, res:0 +08/16 17:52:17.227822 00059267 QRY tagfilter get uid:4871209028232282161, res:0 +08/16 17:52:17.227829 00059267 QRY tagfilter get uid:4871209028236083263, res:0 +08/16 17:52:17.227836 00059267 QRY tagfilter get uid:4871209028240015437, res:1 +08/16 17:52:17.227843 00059267 QRY tagfilter get uid:4871209028244013147, res:0 +08/16 17:52:17.227849 00059267 QRY tagfilter get uid:4871209028247879785, res:0 +08/16 17:52:17.227856 00059267 QRY tagfilter get uid:4871209028251877495, res:0 +08/16 17:52:17.227862 00059267 QRY tagfilter get uid:4871209028255744133, res:1 +08/16 17:52:17.227871 00059267 QRY tagfilter get uid:4871209028259741843, res:0 +08/16 17:52:17.227928 00059267 QRY tagfilter get uid:4871209028263805089, res:0 +08/16 17:52:17.227951 00059267 QRY tagfilter get uid:4871209028267802799, res:0 +08/16 17:52:17.227959 00059267 QRY tagfilter get uid:4871209028272062653, res:1 +08/16 17:52:17.227965 00059267 QRY tagfilter get uid:4871209028276322508, res:0 +08/16 17:52:17.227974 00059267 QRY tagfilter get uid:4871209028224942103, res:0 +08/16 17:52:17.227981 00059267 QRY tagfilter get uid:4871209028228874277, res:0 +08/16 17:52:17.227988 00059267 QRY tagfilter get uid:4871209028232871987, res:1 +08/16 17:52:17.227995 00059267 QRY tagfilter get uid:4871209028236607553, res:0 +08/16 17:52:17.228001 00059267 QRY tagfilter get uid:4871209028240605263, res:0 +08/16 17:52:17.228008 00059267 QRY tagfilter get uid:4871209028244602973, res:0 +08/16 17:52:17.228016 00059267 QRY tagfilter get uid:4871209028248404075, res:1 +08/16 17:52:17.228023 00059267 QRY tagfilter get uid:4871209028252401785, res:0 +08/16 17:52:17.228030 00059267 QRY tagfilter get uid:4871209028256333959, res:0 +08/16 17:52:17.228036 00059267 QRY tagfilter get uid:4871209028260397205, res:0 +08/16 17:52:17.228043 00059267 QRY tagfilter get uid:4871209028264525987, res:1 +08/16 17:52:17.228049 00059267 QRY tagfilter get uid:4871209028268392625, res:0 +08/16 17:52:17.228056 00059267 QRY tagfilter get uid:4871209028272718015, res:0 +08/16 17:52:17.228063 00059267 QRY tagfilter get uid:4871209028276977870, res:0 +08/16 17:52:17.228070 00059267 QRY tagfilter get uid:4871209028225466393, res:1 +08/16 17:52:17.228076 00059267 QRY tagfilter get uid:4871209028229398567, res:0 +08/16 17:52:17.228083 00059267 QRY tagfilter get uid:4871209028233396277, res:0 +08/16 17:52:17.228089 00059267 QRY tagfilter get uid:4871209028237197379, res:0 +08/16 17:52:17.228095 00059267 QRY tagfilter get uid:4871209028241195089, res:1 +08/16 17:52:17.228104 00059267 QRY tagfilter get uid:4871209028245127263, res:0 +08/16 17:52:17.228110 00059267 QRY tagfilter get uid:4871209028248993901, res:0 +08/16 17:52:17.228117 00059267 QRY tagfilter get uid:4871209028252926075, res:0 +08/16 17:52:17.228123 00059267 QRY tagfilter get uid:4871209028256858249, res:1 +08/16 17:52:17.228129 00059267 QRY tagfilter get uid:4871209028260987031, res:0 +08/16 17:52:17.228136 00059267 QRY tagfilter get uid:4871209028265050277, res:0 +08/16 17:52:17.228158 00059267 QRY tagfilter get uid:4871209028269047987, res:0 +08/16 17:52:17.228166 00059267 QRY tagfilter get uid:4871209028273307841, res:1 +08/16 17:52:17.228172 00059267 QRY tagfilter get uid:4871209028277502160, res:0 +08/16 17:52:17.228179 00059267 QRY tagfilter get uid:4871209028225990683, res:0 +08/16 17:52:17.228185 00059267 QRY tagfilter get uid:4871209028229922857, res:0 +08/16 17:52:17.228192 00059267 QRY tagfilter get uid:4871209028233986103, res:1 +08/16 17:52:17.228198 00059267 QRY tagfilter get uid:4871209028237721669, res:0 +08/16 17:52:17.228204 00059267 QRY tagfilter get uid:4871209028241784915, res:0 +08/16 17:52:17.228211 00059267 QRY tagfilter get uid:4871209028245717089, res:0 +08/16 17:52:17.228217 00059267 QRY tagfilter get uid:4871209028249518191, res:1 +08/16 17:52:17.228223 00059267 QRY tagfilter get uid:4871209028253450365, res:0 +08/16 17:52:17.228300 00059267 QRY tagfilter get uid:4871209028257382539, res:0 +08/16 17:52:17.228323 00059267 QRY tagfilter get uid:4871209028261511321, res:0 +08/16 17:52:17.228330 00059267 QRY tagfilter get uid:4871209028265574567, res:1 +08/16 17:52:17.228336 00059267 QRY tagfilter get uid:4871209028269637813, res:0 +08/16 17:52:17.228343 00059267 QRY tagfilter get uid:4871209028273963203, res:0 +08/16 17:52:17.228349 00059267 QRY tagfilter get uid:4871209028278026450, res:0 +08/16 17:52:17.228357 00059267 QRY tagfilter get uid:4871209028224417813 +08/16 17:52:17.228363 00059267 QRY tagfilter get uid:4871209028240015437 +08/16 17:52:17.228383 00059267 QRY tagfilter get uid:4871209028255744133 +08/16 17:52:17.228406 00059267 QRY tagfilter get uid:4871209028272062653 +08/16 17:52:17.228413 00059267 QRY tagfilter get uid:4871209028232871987 +08/16 17:52:17.228442 00059267 QRY tagfilter get uid:4871209028248404075 +08/16 17:52:17.228478 00059267 QRY tagfilter get uid:4871209028264525987 +08/16 17:52:17.228499 00059267 QRY tagfilter get uid:4871209028225466393 +08/16 17:52:17.228507 00059267 QRY tagfilter get uid:4871209028241195089 +08/16 17:52:17.228513 00059267 QRY tagfilter get uid:4871209028256858249 +08/16 17:52:17.228520 00059267 QRY tagfilter get uid:4871209028273307841 +08/16 17:52:17.228526 00059267 QRY tagfilter get uid:4871209028233986103 +08/16 17:52:17.228533 00059267 QRY tagfilter get uid:4871209028249518191 +08/16 17:52:17.228539 00059267 QRY tagfilter get uid:4871209028265574567 \ No newline at end of file From 171ce325c79d816f314561d05975ba93498ca411 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 17 Aug 2022 11:47:59 +0800 Subject: [PATCH 19/33] fix:error in get table list by tag filter --- source/libs/executor/src/executil.c | 22 +- source/libs/scalar/src/sclvector.c | 1 - tests/system-test/2-query/json_tag.py | 295 -------------------------- 3 files changed, 4 insertions(+), 314 deletions(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index f8164e0947..9f8f97c4f5 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -426,15 +426,11 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, uint64_t suid, SArray for(int32_t j = 0; j < taosArrayGetSize(pResBlock->pDataBlock); j++){ SColumnInfoData* pColInfo = (SColumnInfoData*)taosArrayGet(pResBlock->pDataBlock, j); - char str[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; - metaGetTableNameByUid(metaHandle, *uid, str); - colDataAppend(pColInfo, i, str, false); - if(pColInfo->info.colId == -1){ // tbname -// char str[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; -// metaGetTableNameByUid(metaHandle, *uid, str); -// colDataAppend(pColInfo, i, str, false); -// qDebug("tagfilter uid:%ld, tbname:%s", *uid, str+2); + char str[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; + metaGetTableNameByUid(metaHandle, *uid, str); + colDataAppend(pColInfo, i, str, false); + qDebug("tagfilter uid:%ld, tbname:%s", *uid, str+2); }else{ STagVal tagVal = {0}; tagVal.cid = pColInfo->info.colId; @@ -452,16 +448,6 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, uint64_t suid, SArray taosMemoryFree(tmp); } else { colDataAppend(pColInfo, i, (const char*)&tagVal.i64, false); - - if(pColInfo->info.type == TSDB_DATA_TYPE_TINYINT){ - int8_t tint = *(int8_t*)(&tagVal.i64); - qDebug("tagfilter uid:%ld, tbname:%s, tint:%d", *uid, str+2, tint); - - }else if(pColInfo->info.type == TSDB_DATA_TYPE_INT){ - int nint = *(int*)(&tagVal.i64); - qDebug("tagfilter uid:%ld, tbname:%s nint:+%d", *uid, str+2, nint); - - } } } } diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 4fc0005e4d..55699d5abd 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -1673,7 +1673,6 @@ void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, }else{\ bool res = filterDoCompare(fp, optr, pLeftData, pRightData);\ colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); \ - if(GET_PARAM_TYPE(pLeft) == TSDB_DATA_TYPE_BIGINT){qDebug("tagfilter left:%d, right:%d, res:%d", *(int64_t*)(pLeftData), *(int64_t*)(pRightData), res);} \ } \ if(freeLeft) taosMemoryFreeClear(pLeftData);\ if(freeRight) taosMemoryFreeClear(pRightData);\ diff --git a/tests/system-test/2-query/json_tag.py b/tests/system-test/2-query/json_tag.py index 9b4379c011..856d764747 100644 --- a/tests/system-test/2-query/json_tag.py +++ b/tests/system-test/2-query/json_tag.py @@ -698,298 +698,3 @@ class TDTestCase: tdCases.addWindows(__file__, TDTestCase()) tdCases.addLinux(__file__, TDTestCase()) - -08/16 17:52:17.220901 00059267 QRY tagfilter uid:4871209028224417813, tbname:t3 nint:+2 -08/16 17:52:17.220973 00059267 QRY tagfilter uid:4871209028224417813, tbname:t3, tint:3 -08/16 17:52:17.220992 00059267 QRY tagfilter uid:4871209028228349987, tbname:t10 nint:+3 -08/16 17:52:17.221014 00059267 QRY tagfilter uid:4871209028228349987, tbname:t10, tint:4 -08/16 17:52:17.221042 00059267 QRY tagfilter uid:4871209028232282161, tbname:t17 nint:+4 -08/16 17:52:17.221057 00059267 QRY tagfilter uid:4871209028232282161, tbname:t17, tint:5 -08/16 17:52:17.221580 00059267 QRY tagfilter uid:4871209028236083263, tbname:t24 nint:+5 -08/16 17:52:17.221627 00059267 QRY tagfilter uid:4871209028236083263, tbname:t24, tint:6 -08/16 17:52:17.221641 00059267 QRY tagfilter uid:4871209028240015437, tbname:t31 nint:+2 -08/16 17:52:17.221654 00059267 QRY tagfilter uid:4871209028240015437, tbname:t31, tint:3 -08/16 17:52:17.221974 00059267 QRY tagfilter uid:4871209028244013147, tbname:t38 nint:+3 -08/16 17:52:17.222022 00059267 QRY tagfilter uid:4871209028244013147, tbname:t38, tint:4 -08/16 17:52:17.222040 00059267 QRY tagfilter uid:4871209028247879785, tbname:t45 nint:+4 -08/16 17:52:17.222054 00059267 QRY tagfilter uid:4871209028247879785, tbname:t45, tint:5 -08/16 17:52:17.222068 00059267 QRY tagfilter uid:4871209028251877495, tbname:t52 nint:+5 -08/16 17:52:17.222081 00059267 QRY tagfilter uid:4871209028251877495, tbname:t52, tint:6 -08/16 17:52:17.222167 00059267 QRY tagfilter uid:4871209028255744133, tbname:t59 nint:+2 -08/16 17:52:17.222192 00059267 QRY tagfilter uid:4871209028255744133, tbname:t59, tint:3 -08/16 17:52:17.222210 00059267 QRY tagfilter uid:4871209028259741843, tbname:t66 nint:+3 -08/16 17:52:17.222490 00059267 QRY tagfilter uid:4871209028259741843, tbname:t66, tint:4 -08/16 17:52:17.222522 00059267 QRY tagfilter uid:4871209028263805089, tbname:t73 nint:+4 -08/16 17:52:17.222551 00059267 QRY tagfilter uid:4871209028263805089, tbname:t73, tint:5 -08/16 17:52:17.222809 00059267 QRY tagfilter uid:4871209028267802799, tbname:t80 nint:+5 -08/16 17:52:17.222855 00059267 QRY tagfilter uid:4871209028267802799, tbname:t80, tint:6 -08/16 17:52:17.222874 00059267 QRY tagfilter uid:4871209028272062653, tbname:t87 nint:+2 -08/16 17:52:17.222934 00059267 QRY tagfilter uid:4871209028272062653, tbname:t87, tint:3 -08/16 17:52:17.222976 00059267 QRY tagfilter uid:4871209028276322508, tbname:t94 nint:+3 -08/16 17:52:17.222995 00059267 QRY tagfilter uid:4871209028276322508, tbname:t94, tint:4 -08/16 17:52:17.223013 00059267 QRY tagfilter uid:4871209028224942103, tbname:t4 nint:+4 -08/16 17:52:17.223029 00059267 QRY tagfilter uid:4871209028224942103, tbname:t4, tint:5 -08/16 17:52:17.223047 00059267 QRY tagfilter uid:4871209028228874277, tbname:t11 nint:+5 -08/16 17:52:17.223064 00059267 QRY tagfilter uid:4871209028228874277, tbname:t11, tint:6 -08/16 17:52:17.223079 00059267 QRY tagfilter uid:4871209028232871987, tbname:t18 nint:+2 -08/16 17:52:17.223093 00059267 QRY tagfilter uid:4871209028232871987, tbname:t18, tint:3 -08/16 17:52:17.223109 00059267 QRY tagfilter uid:4871209028236607553, tbname:t25 nint:+3 -08/16 17:52:17.223125 00059267 QRY tagfilter uid:4871209028236607553, tbname:t25, tint:4 -08/16 17:52:17.223141 00059267 QRY tagfilter uid:4871209028240605263, tbname:t32 nint:+4 -08/16 17:52:17.223156 00059267 QRY tagfilter uid:4871209028240605263, tbname:t32, tint:5 -08/16 17:52:17.223172 00059267 QRY tagfilter uid:4871209028244602973, tbname:t39 nint:+5 -08/16 17:52:17.223188 00059267 QRY tagfilter uid:4871209028244602973, tbname:t39, tint:6 -08/16 17:52:17.223204 00059267 QRY tagfilter uid:4871209028248404075, tbname:t46 nint:+2 -08/16 17:52:17.223219 00059267 QRY tagfilter uid:4871209028248404075, tbname:t46, tint:3 -08/16 17:52:17.223564 00059267 QRY tagfilter uid:4871209028252401785, tbname:t53 nint:+3 -08/16 17:52:17.223598 00059267 QRY tagfilter uid:4871209028252401785, tbname:t53, tint:4 -08/16 17:52:17.223614 00059267 QRY tagfilter uid:4871209028256333959, tbname:t60 nint:+4 -08/16 17:52:17.223638 00059267 QRY tagfilter uid:4871209028256333959, tbname:t60, tint:5 -08/16 17:52:17.223771 00059267 QRY tagfilter uid:4871209028260397205, tbname:t67 nint:+5 -08/16 17:52:17.223813 00059267 QRY tagfilter uid:4871209028260397205, tbname:t67, tint:6 -08/16 17:52:17.223828 00059267 QRY tagfilter uid:4871209028264525987, tbname:t74 nint:+2 -08/16 17:52:17.223858 00059267 QRY tagfilter uid:4871209028264525987, tbname:t74, tint:3 -08/16 17:52:17.223895 00059267 QRY tagfilter uid:4871209028268392625, tbname:t81 nint:+3 -08/16 17:52:17.223917 00059267 QRY tagfilter uid:4871209028268392625, tbname:t81, tint:4 -08/16 17:52:17.223953 00059267 QRY tagfilter uid:4871209028272718015, tbname:t88 nint:+4 -08/16 17:52:17.223977 00059267 QRY tagfilter uid:4871209028272718015, tbname:t88, tint:5 -08/16 17:52:17.223991 00059267 QRY tagfilter uid:4871209028276977870, tbname:t95 nint:+5 -08/16 17:52:17.224003 00059267 QRY tagfilter uid:4871209028276977870, tbname:t95, tint:6 -08/16 17:52:17.224018 00059267 QRY tagfilter uid:4871209028225466393, tbname:t5 nint:+2 -08/16 17:52:17.224030 00059267 QRY tagfilter uid:4871209028225466393, tbname:t5, tint:3 -08/16 17:52:17.224044 00059267 QRY tagfilter uid:4871209028229398567, tbname:t12 nint:+3 -08/16 17:52:17.224056 00059267 QRY tagfilter uid:4871209028229398567, tbname:t12, tint:4 -08/16 17:52:17.224071 00059267 QRY tagfilter uid:4871209028233396277, tbname:t19 nint:+4 -08/16 17:52:17.224084 00059267 QRY tagfilter uid:4871209028233396277, tbname:t19, tint:5 -08/16 17:52:17.224097 00059267 QRY tagfilter uid:4871209028237197379, tbname:t26 nint:+5 -08/16 17:52:17.224110 00059267 QRY tagfilter uid:4871209028237197379, tbname:t26, tint:6 -08/16 17:52:17.224125 00059267 QRY tagfilter uid:4871209028241195089, tbname:t33 nint:+2 -08/16 17:52:17.224139 00059267 QRY tagfilter uid:4871209028241195089, tbname:t33, tint:3 -08/16 17:52:17.224188 00059267 QRY tagfilter uid:4871209028245127263, tbname:t40 nint:+3 -08/16 17:52:17.224209 00059267 QRY tagfilter uid:4871209028245127263, tbname:t40, tint:4 -08/16 17:52:17.224459 00059267 QRY tagfilter uid:4871209028248993901, tbname:t47 nint:+4 -08/16 17:52:17.224479 00059267 QRY tagfilter uid:4871209028248993901, tbname:t47, tint:5 -08/16 17:52:17.224494 00059267 QRY tagfilter uid:4871209028252926075, tbname:t54 nint:+5 -08/16 17:52:17.224507 00059267 QRY tagfilter uid:4871209028252926075, tbname:t54, tint:6 -08/16 17:52:17.224522 00059267 QRY tagfilter uid:4871209028256858249, tbname:t61 nint:+2 -08/16 17:52:17.224535 00059267 QRY tagfilter uid:4871209028256858249, tbname:t61, tint:3 -08/16 17:52:17.224549 00059267 QRY tagfilter uid:4871209028260987031, tbname:t68 nint:+3 -08/16 17:52:17.224564 00059267 QRY tagfilter uid:4871209028260987031, tbname:t68, tint:4 -08/16 17:52:17.224664 00059267 QRY tagfilter uid:4871209028265050277, tbname:t75 nint:+4 -08/16 17:52:17.224717 00059267 QRY tagfilter uid:4871209028265050277, tbname:t75, tint:5 -08/16 17:52:17.224731 00059267 QRY tagfilter uid:4871209028269047987, tbname:t82 nint:+5 -08/16 17:52:17.224769 00059267 QRY tagfilter uid:4871209028269047987, tbname:t82, tint:6 -08/16 17:52:17.224783 00059267 QRY tagfilter uid:4871209028273307841, tbname:t89 nint:+2 -08/16 17:52:17.224923 00059267 QRY tagfilter uid:4871209028273307841, tbname:t89, tint:3 -08/16 17:52:17.224990 00059267 QRY tagfilter uid:4871209028277502160, tbname:t96 nint:+3 -08/16 17:52:17.225006 00059267 QRY tagfilter uid:4871209028277502160, tbname:t96, tint:4 -08/16 17:52:17.225018 00059267 QRY tagfilter uid:4871209028225990683, tbname:t6 nint:+4 -08/16 17:52:17.225029 00059267 QRY tagfilter uid:4871209028225990683, tbname:t6, tint:5 -08/16 17:52:17.225039 00059267 QRY tagfilter uid:4871209028229922857, tbname:t13 nint:+5 -08/16 17:52:17.225053 00059267 QRY tagfilter uid:4871209028229922857, tbname:t13, tint:6 -08/16 17:52:17.225072 00059267 QRY tagfilter uid:4871209028233986103, tbname:t20 nint:+2 -08/16 17:52:17.225082 00059267 QRY tagfilter uid:4871209028233986103, tbname:t20, tint:3 -08/16 17:52:17.225093 00059267 QRY tagfilter uid:4871209028237721669, tbname:t27 nint:+3 -08/16 17:52:17.225103 00059267 QRY tagfilter uid:4871209028237721669, tbname:t27, tint:4 -08/16 17:52:17.225113 00059267 QRY tagfilter uid:4871209028241784915, tbname:t34 nint:+4 -08/16 17:52:17.225368 00059267 QRY tagfilter uid:4871209028241784915, tbname:t34, tint:5 -08/16 17:52:17.225409 00059267 QRY tagfilter uid:4871209028245717089, tbname:t41 nint:+5 -08/16 17:52:17.225437 00059267 QRY tagfilter uid:4871209028245717089, tbname:t41, tint:6 -08/16 17:52:17.225462 00059267 QRY tagfilter uid:4871209028249518191, tbname:t48 nint:+2 -08/16 17:52:17.225475 00059267 QRY tagfilter uid:4871209028249518191, tbname:t48, tint:3 -08/16 17:52:17.225486 00059267 QRY tagfilter uid:4871209028253450365, tbname:t55 nint:+3 -08/16 17:52:17.225496 00059267 QRY tagfilter uid:4871209028253450365, tbname:t55, tint:4 -08/16 17:52:17.225507 00059267 QRY tagfilter uid:4871209028257382539, tbname:t62 nint:+4 -08/16 17:52:17.225532 00059267 QRY tagfilter uid:4871209028257382539, tbname:t62, tint:5 -08/16 17:52:17.225544 00059267 QRY tagfilter uid:4871209028261511321, tbname:t69 nint:+5 -08/16 17:52:17.225554 00059267 QRY tagfilter uid:4871209028261511321, tbname:t69, tint:6 -08/16 17:52:17.225564 00059267 QRY tagfilter uid:4871209028265574567, tbname:t76 nint:+2 -08/16 17:52:17.225575 00059267 QRY tagfilter uid:4871209028265574567, tbname:t76, tint:3 -08/16 17:52:17.225585 00059267 QRY tagfilter uid:4871209028269637813, tbname:t83 nint:+3 -08/16 17:52:17.225615 00059267 QRY tagfilter uid:4871209028269637813, tbname:t83, tint:4 -08/16 17:52:17.225640 00059267 QRY tagfilter uid:4871209028273963203, tbname:t90 nint:+4 -08/16 17:52:17.225670 00059267 QRY tagfilter uid:4871209028273963203, tbname:t90, tint:5 -08/16 17:52:17.225698 00059267 QRY tagfilter uid:4871209028278026450, tbname:t97 nint:+5 -08/16 17:52:17.225709 00059267 QRY tagfilter uid:4871209028278026450, tbname:t97, tint:6 -08/16 17:52:17.226570 00059267 QRY tagfilter left:2, right:3, res:1 -08/16 17:52:17.226623 00059267 QRY tagfilter left:3, right:3, res:0 -08/16 17:52:17.226642 00059267 QRY tagfilter left:4, right:3, res:0 -08/16 17:52:17.226684 00059267 QRY tagfilter left:5, right:3, res:0 -08/16 17:52:17.226733 00059267 QRY tagfilter left:2, right:3, res:1 -08/16 17:52:17.226765 00059267 QRY tagfilter left:3, right:3, res:0 -08/16 17:52:17.226773 00059267 QRY tagfilter left:4, right:3, res:0 -08/16 17:52:17.226780 00059267 QRY tagfilter left:5, right:3, res:0 -08/16 17:52:17.226787 00059267 QRY tagfilter left:2, right:3, res:1 -08/16 17:52:17.226793 00059267 QRY tagfilter left:3, right:3, res:0 -08/16 17:52:17.226801 00059267 QRY tagfilter left:4, right:3, res:0 -08/16 17:52:17.226808 00059267 QRY tagfilter left:5, right:3, res:0 -08/16 17:52:17.226815 00059267 QRY tagfilter left:2, right:3, res:1 -08/16 17:52:17.226821 00059267 QRY tagfilter left:3, right:3, res:0 -08/16 17:52:17.226828 00059267 QRY tagfilter left:4, right:3, res:0 -08/16 17:52:17.226834 00059267 QRY tagfilter left:5, right:3, res:0 -08/16 17:52:17.226840 00059267 QRY tagfilter left:2, right:3, res:1 -08/16 17:52:17.226847 00059267 QRY tagfilter left:3, right:3, res:0 -08/16 17:52:17.226854 00059267 QRY tagfilter left:4, right:3, res:0 -08/16 17:52:17.226860 00059267 QRY tagfilter left:5, right:3, res:0 -08/16 17:52:17.226867 00059267 QRY tagfilter left:2, right:3, res:1 -08/16 17:52:17.226873 00059267 QRY tagfilter left:3, right:3, res:0 -08/16 17:52:17.226880 00059267 QRY tagfilter left:4, right:3, res:0 -08/16 17:52:17.226886 00059267 QRY tagfilter left:5, right:3, res:0 -08/16 17:52:17.226893 00059267 QRY tagfilter left:2, right:3, res:1 -08/16 17:52:17.226900 00059267 QRY tagfilter left:3, right:3, res:0 -08/16 17:52:17.226907 00059267 QRY tagfilter left:4, right:3, res:0 -08/16 17:52:17.226914 00059267 QRY tagfilter left:5, right:3, res:0 -08/16 17:52:17.226920 00059267 QRY tagfilter left:2, right:3, res:1 -08/16 17:52:17.226927 00059267 QRY tagfilter left:3, right:3, res:0 -08/16 17:52:17.226934 00059267 QRY tagfilter left:4, right:3, res:0 -08/16 17:52:17.226940 00059267 QRY tagfilter left:5, right:3, res:0 -08/16 17:52:17.227014 00059267 QRY tagfilter left:2, right:3, res:1 -08/16 17:52:17.227036 00059267 QRY tagfilter left:3, right:3, res:0 -08/16 17:52:17.227043 00059267 QRY tagfilter left:4, right:3, res:0 -08/16 17:52:17.227050 00059267 QRY tagfilter left:5, right:3, res:0 -08/16 17:52:17.227056 00059267 QRY tagfilter left:2, right:3, res:1 -08/16 17:52:17.227062 00059267 QRY tagfilter left:3, right:3, res:0 -08/16 17:52:17.227069 00059267 QRY tagfilter left:4, right:3, res:0 -08/16 17:52:17.227076 00059267 QRY tagfilter left:5, right:3, res:0 -08/16 17:52:17.227082 00059267 QRY tagfilter left:2, right:3, res:1 -08/16 17:52:17.227089 00059267 QRY tagfilter left:3, right:3, res:0 -08/16 17:52:17.227107 00059267 QRY tagfilter left:4, right:3, res:0 -08/16 17:52:17.227129 00059267 QRY tagfilter left:5, right:3, res:0 -08/16 17:52:17.227138 00059267 QRY tagfilter left:2, right:3, res:1 -08/16 17:52:17.227144 00059267 QRY tagfilter left:3, right:3, res:0 -08/16 17:52:17.227151 00059267 QRY tagfilter left:4, right:3, res:0 -08/16 17:52:17.227157 00059267 QRY tagfilter left:5, right:3, res:0 -08/16 17:52:17.227163 00059267 QRY tagfilter left:2, right:3, res:1 -08/16 17:52:17.227170 00059267 QRY tagfilter left:3, right:3, res:0 -08/16 17:52:17.227176 00059267 QRY tagfilter left:4, right:3, res:0 -08/16 17:52:17.227194 00059267 QRY tagfilter left:5, right:3, res:0 -08/16 17:52:17.227202 00059267 QRY tagfilter left:2, right:3, res:1 -08/16 17:52:17.227208 00059267 QRY tagfilter left:3, right:3, res:0 -08/16 17:52:17.227215 00059267 QRY tagfilter left:4, right:3, res:0 -08/16 17:52:17.227221 00059267 QRY tagfilter left:5, right:3, res:0 -08/16 17:52:17.227284 00059267 QRY tagfilter left:3, right:2, res:1 -08/16 17:52:17.227298 00059267 QRY tagfilter left:4, right:2, res:1 -08/16 17:52:17.227304 00059267 QRY tagfilter left:5, right:2, res:1 -08/16 17:52:17.227311 00059267 QRY tagfilter left:6, right:2, res:1 -08/16 17:52:17.227317 00059267 QRY tagfilter left:3, right:2, res:1 -08/16 17:52:17.227324 00059267 QRY tagfilter left:4, right:2, res:1 -08/16 17:52:17.227330 00059267 QRY tagfilter left:5, right:2, res:1 -08/16 17:52:17.227359 00059267 QRY tagfilter left:6, right:2, res:1 -08/16 17:52:17.227421 00059267 QRY tagfilter left:3, right:2, res:1 -08/16 17:52:17.227437 00059267 QRY tagfilter left:4, right:2, res:1 -08/16 17:52:17.227444 00059267 QRY tagfilter left:5, right:2, res:1 -08/16 17:52:17.227450 00059267 QRY tagfilter left:6, right:2, res:1 -08/16 17:52:17.227456 00059267 QRY tagfilter left:3, right:2, res:1 -08/16 17:52:17.227463 00059267 QRY tagfilter left:4, right:2, res:1 -08/16 17:52:17.227469 00059267 QRY tagfilter left:5, right:2, res:1 -08/16 17:52:17.227475 00059267 QRY tagfilter left:6, right:2, res:1 -08/16 17:52:17.227493 00059267 QRY tagfilter left:3, right:2, res:1 -08/16 17:52:17.227500 00059267 QRY tagfilter left:4, right:2, res:1 -08/16 17:52:17.227507 00059267 QRY tagfilter left:5, right:2, res:1 -08/16 17:52:17.227513 00059267 QRY tagfilter left:6, right:2, res:1 -08/16 17:52:17.227520 00059267 QRY tagfilter left:3, right:2, res:1 -08/16 17:52:17.227526 00059267 QRY tagfilter left:4, right:2, res:1 -08/16 17:52:17.227541 00059267 QRY tagfilter left:5, right:2, res:1 -08/16 17:52:17.227548 00059267 QRY tagfilter left:6, right:2, res:1 -08/16 17:52:17.227570 00059267 QRY tagfilter left:3, right:2, res:1 -08/16 17:52:17.227578 00059267 QRY tagfilter left:4, right:2, res:1 -08/16 17:52:17.227585 00059267 QRY tagfilter left:5, right:2, res:1 -08/16 17:52:17.227591 00059267 QRY tagfilter left:6, right:2, res:1 -08/16 17:52:17.227598 00059267 QRY tagfilter left:3, right:2, res:1 -08/16 17:52:17.227604 00059267 QRY tagfilter left:4, right:2, res:1 -08/16 17:52:17.227611 00059267 QRY tagfilter left:5, right:2, res:1 -08/16 17:52:17.227617 00059267 QRY tagfilter left:6, right:2, res:1 -08/16 17:52:17.227624 00059267 QRY tagfilter left:3, right:2, res:1 -08/16 17:52:17.227630 00059267 QRY tagfilter left:4, right:2, res:1 -08/16 17:52:17.227636 00059267 QRY tagfilter left:5, right:2, res:1 -08/16 17:52:17.227643 00059267 QRY tagfilter left:6, right:2, res:1 -08/16 17:52:17.227649 00059267 QRY tagfilter left:3, right:2, res:1 -08/16 17:52:17.227656 00059267 QRY tagfilter left:4, right:2, res:1 -08/16 17:52:17.227662 00059267 QRY tagfilter left:5, right:2, res:1 -08/16 17:52:17.227668 00059267 QRY tagfilter left:6, right:2, res:1 -08/16 17:52:17.227675 00059267 QRY tagfilter left:3, right:2, res:1 -08/16 17:52:17.227681 00059267 QRY tagfilter left:4, right:2, res:1 -08/16 17:52:17.227688 00059267 QRY tagfilter left:5, right:2, res:1 -08/16 17:52:17.227696 00059267 QRY tagfilter left:6, right:2, res:1 -08/16 17:52:17.227703 00059267 QRY tagfilter left:3, right:2, res:1 -08/16 17:52:17.227709 00059267 QRY tagfilter left:4, right:2, res:1 -08/16 17:52:17.227716 00059267 QRY tagfilter left:5, right:2, res:1 -08/16 17:52:17.227723 00059267 QRY tagfilter left:6, right:2, res:1 -08/16 17:52:17.227729 00059267 QRY tagfilter left:3, right:2, res:1 -08/16 17:52:17.227738 00059267 QRY tagfilter left:4, right:2, res:1 -08/16 17:52:17.227744 00059267 QRY tagfilter left:5, right:2, res:1 -08/16 17:52:17.227750 00059267 QRY tagfilter left:6, right:2, res:1 -08/16 17:52:17.227757 00059267 QRY tagfilter left:3, right:2, res:1 -08/16 17:52:17.227764 00059267 QRY tagfilter left:4, right:2, res:1 -08/16 17:52:17.227771 00059267 QRY tagfilter left:5, right:2, res:1 -08/16 17:52:17.227779 00059267 QRY tagfilter left:6, right:2, res:1 -08/16 17:52:17.227803 00059267 QRY tagfilter get uid:4871209028224417813, res:1 -08/16 17:52:17.227811 00059267 QRY tagfilter get uid:4871209028228349987, res:0 -08/16 17:52:17.227822 00059267 QRY tagfilter get uid:4871209028232282161, res:0 -08/16 17:52:17.227829 00059267 QRY tagfilter get uid:4871209028236083263, res:0 -08/16 17:52:17.227836 00059267 QRY tagfilter get uid:4871209028240015437, res:1 -08/16 17:52:17.227843 00059267 QRY tagfilter get uid:4871209028244013147, res:0 -08/16 17:52:17.227849 00059267 QRY tagfilter get uid:4871209028247879785, res:0 -08/16 17:52:17.227856 00059267 QRY tagfilter get uid:4871209028251877495, res:0 -08/16 17:52:17.227862 00059267 QRY tagfilter get uid:4871209028255744133, res:1 -08/16 17:52:17.227871 00059267 QRY tagfilter get uid:4871209028259741843, res:0 -08/16 17:52:17.227928 00059267 QRY tagfilter get uid:4871209028263805089, res:0 -08/16 17:52:17.227951 00059267 QRY tagfilter get uid:4871209028267802799, res:0 -08/16 17:52:17.227959 00059267 QRY tagfilter get uid:4871209028272062653, res:1 -08/16 17:52:17.227965 00059267 QRY tagfilter get uid:4871209028276322508, res:0 -08/16 17:52:17.227974 00059267 QRY tagfilter get uid:4871209028224942103, res:0 -08/16 17:52:17.227981 00059267 QRY tagfilter get uid:4871209028228874277, res:0 -08/16 17:52:17.227988 00059267 QRY tagfilter get uid:4871209028232871987, res:1 -08/16 17:52:17.227995 00059267 QRY tagfilter get uid:4871209028236607553, res:0 -08/16 17:52:17.228001 00059267 QRY tagfilter get uid:4871209028240605263, res:0 -08/16 17:52:17.228008 00059267 QRY tagfilter get uid:4871209028244602973, res:0 -08/16 17:52:17.228016 00059267 QRY tagfilter get uid:4871209028248404075, res:1 -08/16 17:52:17.228023 00059267 QRY tagfilter get uid:4871209028252401785, res:0 -08/16 17:52:17.228030 00059267 QRY tagfilter get uid:4871209028256333959, res:0 -08/16 17:52:17.228036 00059267 QRY tagfilter get uid:4871209028260397205, res:0 -08/16 17:52:17.228043 00059267 QRY tagfilter get uid:4871209028264525987, res:1 -08/16 17:52:17.228049 00059267 QRY tagfilter get uid:4871209028268392625, res:0 -08/16 17:52:17.228056 00059267 QRY tagfilter get uid:4871209028272718015, res:0 -08/16 17:52:17.228063 00059267 QRY tagfilter get uid:4871209028276977870, res:0 -08/16 17:52:17.228070 00059267 QRY tagfilter get uid:4871209028225466393, res:1 -08/16 17:52:17.228076 00059267 QRY tagfilter get uid:4871209028229398567, res:0 -08/16 17:52:17.228083 00059267 QRY tagfilter get uid:4871209028233396277, res:0 -08/16 17:52:17.228089 00059267 QRY tagfilter get uid:4871209028237197379, res:0 -08/16 17:52:17.228095 00059267 QRY tagfilter get uid:4871209028241195089, res:1 -08/16 17:52:17.228104 00059267 QRY tagfilter get uid:4871209028245127263, res:0 -08/16 17:52:17.228110 00059267 QRY tagfilter get uid:4871209028248993901, res:0 -08/16 17:52:17.228117 00059267 QRY tagfilter get uid:4871209028252926075, res:0 -08/16 17:52:17.228123 00059267 QRY tagfilter get uid:4871209028256858249, res:1 -08/16 17:52:17.228129 00059267 QRY tagfilter get uid:4871209028260987031, res:0 -08/16 17:52:17.228136 00059267 QRY tagfilter get uid:4871209028265050277, res:0 -08/16 17:52:17.228158 00059267 QRY tagfilter get uid:4871209028269047987, res:0 -08/16 17:52:17.228166 00059267 QRY tagfilter get uid:4871209028273307841, res:1 -08/16 17:52:17.228172 00059267 QRY tagfilter get uid:4871209028277502160, res:0 -08/16 17:52:17.228179 00059267 QRY tagfilter get uid:4871209028225990683, res:0 -08/16 17:52:17.228185 00059267 QRY tagfilter get uid:4871209028229922857, res:0 -08/16 17:52:17.228192 00059267 QRY tagfilter get uid:4871209028233986103, res:1 -08/16 17:52:17.228198 00059267 QRY tagfilter get uid:4871209028237721669, res:0 -08/16 17:52:17.228204 00059267 QRY tagfilter get uid:4871209028241784915, res:0 -08/16 17:52:17.228211 00059267 QRY tagfilter get uid:4871209028245717089, res:0 -08/16 17:52:17.228217 00059267 QRY tagfilter get uid:4871209028249518191, res:1 -08/16 17:52:17.228223 00059267 QRY tagfilter get uid:4871209028253450365, res:0 -08/16 17:52:17.228300 00059267 QRY tagfilter get uid:4871209028257382539, res:0 -08/16 17:52:17.228323 00059267 QRY tagfilter get uid:4871209028261511321, res:0 -08/16 17:52:17.228330 00059267 QRY tagfilter get uid:4871209028265574567, res:1 -08/16 17:52:17.228336 00059267 QRY tagfilter get uid:4871209028269637813, res:0 -08/16 17:52:17.228343 00059267 QRY tagfilter get uid:4871209028273963203, res:0 -08/16 17:52:17.228349 00059267 QRY tagfilter get uid:4871209028278026450, res:0 -08/16 17:52:17.228357 00059267 QRY tagfilter get uid:4871209028224417813 -08/16 17:52:17.228363 00059267 QRY tagfilter get uid:4871209028240015437 -08/16 17:52:17.228383 00059267 QRY tagfilter get uid:4871209028255744133 -08/16 17:52:17.228406 00059267 QRY tagfilter get uid:4871209028272062653 -08/16 17:52:17.228413 00059267 QRY tagfilter get uid:4871209028232871987 -08/16 17:52:17.228442 00059267 QRY tagfilter get uid:4871209028248404075 -08/16 17:52:17.228478 00059267 QRY tagfilter get uid:4871209028264525987 -08/16 17:52:17.228499 00059267 QRY tagfilter get uid:4871209028225466393 -08/16 17:52:17.228507 00059267 QRY tagfilter get uid:4871209028241195089 -08/16 17:52:17.228513 00059267 QRY tagfilter get uid:4871209028256858249 -08/16 17:52:17.228520 00059267 QRY tagfilter get uid:4871209028273307841 -08/16 17:52:17.228526 00059267 QRY tagfilter get uid:4871209028233986103 -08/16 17:52:17.228533 00059267 QRY tagfilter get uid:4871209028249518191 -08/16 17:52:17.228539 00059267 QRY tagfilter get uid:4871209028265574567 \ No newline at end of file From 96ab366beca54c8873cd08ad4a7898333d4613e2 Mon Sep 17 00:00:00 2001 From: Huo Linhe Date: Wed, 17 Aug 2022 13:55:04 +0800 Subject: [PATCH 20/33] fix: remove rust-bindings, use official connector instead Closes [TD-18455](https://jira.taosdata.com:18080/browse/TD-18455) --- cmake/rust-bindings_CMakeLists.txt.in | 12 --- contrib/CMakeLists.txt | 5 -- examples/rust/.gitignore | 2 + examples/rust/Cargo.toml | 18 +++++ examples/rust/examples/bind-tags.rs | 80 +++++++++++++++++++ examples/rust/examples/bind.rs | 74 ++++++++++++++++++ examples/rust/examples/query.rs | 106 ++++++++++++++++++++++++++ examples/rust/examples/subscribe.rs | 103 +++++++++++++++++++++++++ examples/rust/src/main.rs | 3 + 9 files changed, 386 insertions(+), 17 deletions(-) delete mode 100644 cmake/rust-bindings_CMakeLists.txt.in create mode 100644 examples/rust/.gitignore create mode 100644 examples/rust/Cargo.toml create mode 100644 examples/rust/examples/bind-tags.rs create mode 100644 examples/rust/examples/bind.rs create mode 100644 examples/rust/examples/query.rs create mode 100644 examples/rust/examples/subscribe.rs create mode 100644 examples/rust/src/main.rs diff --git a/cmake/rust-bindings_CMakeLists.txt.in b/cmake/rust-bindings_CMakeLists.txt.in deleted file mode 100644 index d16e86139b..0000000000 --- a/cmake/rust-bindings_CMakeLists.txt.in +++ /dev/null @@ -1,12 +0,0 @@ - -# rust-bindings -ExternalProject_Add(rust-bindings - GIT_REPOSITORY https://github.com/songtianyi/tdengine-rust-bindings.git - GIT_TAG 7ed7a97 - SOURCE_DIR "${TD_SOURCE_DIR}/examples/rust" - BINARY_DIR "${TD_SOURCE_DIR}/examples/rust" - CONFIGURE_COMMAND "" - BUILD_COMMAND "" - INSTALL_COMMAND "" - TEST_COMMAND "" - ) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 294b59fe95..a1eec81ee0 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -105,11 +105,6 @@ if(${BUILD_WITH_SQLITE}) cat("${TD_SUPPORT_DIR}/sqlite_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) endif(${BUILD_WITH_SQLITE}) -# rust-bindings -if(${RUST_BINDINGS}) - cat("${TD_SUPPORT_DIR}/rust-bindings_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) -endif(${RUST_BINDINGS}) - # lucene if(${BUILD_WITH_LUCENE}) cat("${TD_SUPPORT_DIR}/lucene_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) diff --git a/examples/rust/.gitignore b/examples/rust/.gitignore new file mode 100644 index 0000000000..96ef6c0b94 --- /dev/null +++ b/examples/rust/.gitignore @@ -0,0 +1,2 @@ +/target +Cargo.lock diff --git a/examples/rust/Cargo.toml b/examples/rust/Cargo.toml new file mode 100644 index 0000000000..1ed73e2fde --- /dev/null +++ b/examples/rust/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "rust" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +taos = "*" + +[dev-dependencies] +chrono = "0.4" +itertools = "0.10.3" +pretty_env_logger = "0.4.0" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +tokio = { version = "1", features = ["full"] } +anyhow = "1" diff --git a/examples/rust/examples/bind-tags.rs b/examples/rust/examples/bind-tags.rs new file mode 100644 index 0000000000..a1f7286625 --- /dev/null +++ b/examples/rust/examples/bind-tags.rs @@ -0,0 +1,80 @@ +use anyhow::Result; +use serde::Deserialize; +use taos::*; + +#[tokio::main] +async fn main() -> Result<()> { + let taos = TaosBuilder::from_dsn("taos://")?.build()?; + taos.exec_many([ + "drop database if exists test", + "create database test keep 36500", + "use test", + "create table tb1 (ts timestamp, c1 bool, c2 tinyint, c3 smallint, c4 int, c5 bigint, + c6 tinyint unsigned, c7 smallint unsigned, c8 int unsigned, c9 bigint unsigned, + c10 float, c11 double, c12 varchar(100), c13 nchar(100)) tags(t1 varchar(100))", + ]) + .await?; + let mut stmt = Stmt::init(&taos)?; + stmt.prepare( + "insert into ? using tb1 tags(?) values(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + )?; + stmt.set_tbname("d0")?; + stmt.set_tags(&[Value::VarChar("涛思".to_string())])?; + + let params = vec![ + ColumnView::from_millis_timestamp(vec![164000000000]), + ColumnView::from_bools(vec![true]), + ColumnView::from_tiny_ints(vec![i8::MAX]), + ColumnView::from_small_ints(vec![i16::MAX]), + ColumnView::from_ints(vec![i32::MAX]), + ColumnView::from_big_ints(vec![i64::MAX]), + ColumnView::from_unsigned_tiny_ints(vec![u8::MAX]), + ColumnView::from_unsigned_small_ints(vec![u16::MAX]), + ColumnView::from_unsigned_ints(vec![u32::MAX]), + ColumnView::from_unsigned_big_ints(vec![u64::MAX]), + ColumnView::from_floats(vec![f32::MAX]), + ColumnView::from_doubles(vec![f64::MAX]), + ColumnView::from_varchar(vec!["ABC"]), + ColumnView::from_nchar(vec!["涛思数据"]), + ]; + let rows = stmt.bind(¶ms)?.add_batch()?.execute()?; + assert_eq!(rows, 1); + + #[derive(Debug, Deserialize)] + #[allow(dead_code)] + struct Row { + ts: String, + c1: bool, + c2: i8, + c3: i16, + c4: i32, + c5: i64, + c6: u8, + c7: u16, + c8: u32, + c9: u64, + c10: Option, + c11: f64, + c12: String, + c13: String, + t1: serde_json::Value, + } + + let rows: Vec = taos + .query("select * from tb1") + .await? + .deserialize() + .try_collect() + .await?; + let row = &rows[0]; + dbg!(&row); + assert_eq!(row.c5, i64::MAX); + assert_eq!(row.c8, u32::MAX); + assert_eq!(row.c9, u64::MAX); + assert_eq!(row.c10.unwrap(), f32::MAX); + // assert_eq!(row.c11, f64::MAX); + assert_eq!(row.c12, "ABC"); + assert_eq!(row.c13, "涛思数据"); + + Ok(()) +} diff --git a/examples/rust/examples/bind.rs b/examples/rust/examples/bind.rs new file mode 100644 index 0000000000..194938a319 --- /dev/null +++ b/examples/rust/examples/bind.rs @@ -0,0 +1,74 @@ +use anyhow::Result; +use serde::Deserialize; +use taos::*; + +#[tokio::main] +async fn main() -> Result<()> { + let taos = TaosBuilder::from_dsn("taos://")?.build()?; + taos.exec_many([ + "drop database if exists test_bindable", + "create database test_bindable keep 36500", + "use test_bindable", + "create table tb1 (ts timestamp, c1 bool, c2 tinyint, c3 smallint, c4 int, c5 bigint, + c6 tinyint unsigned, c7 smallint unsigned, c8 int unsigned, c9 bigint unsigned, + c10 float, c11 double, c12 varchar(100), c13 nchar(100))", + ]) + .await?; + let mut stmt = Stmt::init(&taos)?; + stmt.prepare("insert into tb1 values(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)")?; + let params = vec![ + ColumnView::from_millis_timestamp(vec![0]), + ColumnView::from_bools(vec![true]), + ColumnView::from_tiny_ints(vec![i8::MAX]), + ColumnView::from_small_ints(vec![i16::MAX]), + ColumnView::from_ints(vec![i32::MAX]), + ColumnView::from_big_ints(vec![i64::MAX]), + ColumnView::from_unsigned_tiny_ints(vec![u8::MAX]), + ColumnView::from_unsigned_small_ints(vec![u16::MAX]), + ColumnView::from_unsigned_ints(vec![u32::MAX]), + ColumnView::from_unsigned_big_ints(vec![u64::MAX]), + ColumnView::from_floats(vec![f32::MAX]), + ColumnView::from_doubles(vec![f64::MAX]), + ColumnView::from_varchar(vec!["ABC"]), + ColumnView::from_nchar(vec!["涛思数据"]), + ]; + let rows = stmt.bind(¶ms)?.add_batch()?.execute()?; + assert_eq!(rows, 1); + + #[derive(Debug, Deserialize)] + #[allow(dead_code)] + struct Row { + ts: String, + c1: bool, + c2: i8, + c3: i16, + c4: i32, + c5: i64, + c6: u8, + c7: u16, + c8: u32, + c9: u64, + c10: Option, + c11: f64, + c12: String, + c13: String, + } + + let rows: Vec = taos + .query("select * from tb1") + .await? + .deserialize() + .try_collect() + .await?; + let row = &rows[0]; + dbg!(&row); + assert_eq!(row.c5, i64::MAX); + assert_eq!(row.c8, u32::MAX); + assert_eq!(row.c9, u64::MAX); + assert_eq!(row.c10.unwrap(), f32::MAX); + // assert_eq!(row.c11, f64::MAX); + assert_eq!(row.c12, "ABC"); + assert_eq!(row.c13, "涛思数据"); + + Ok(()) +} diff --git a/examples/rust/examples/query.rs b/examples/rust/examples/query.rs new file mode 100644 index 0000000000..016b291abc --- /dev/null +++ b/examples/rust/examples/query.rs @@ -0,0 +1,106 @@ +use std::time::Duration; + +use chrono::{DateTime, Local}; +use taos::*; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + let dsn = "taos://"; + + let opts = PoolBuilder::new() + .max_size(5000) // max connections + .max_lifetime(Some(Duration::from_secs(60 * 60))) // lifetime of each connection + .min_idle(Some(1000)) // minimal idle connections + .connection_timeout(Duration::from_secs(2)); + + let pool = TaosBuilder::from_dsn(dsn)?.with_pool_builder(opts)?; + + let taos = pool.get()?; + + let db = "query"; + + // prepare database + taos.exec_many([ + format!("DROP DATABASE IF EXISTS `{db}`"), + format!("CREATE DATABASE `{db}`"), + format!("USE `{db}`"), + ]) + .await?; + + let inserted = taos.exec_many([ + // create super table + "CREATE TABLE `meters` (`ts` TIMESTAMP, `current` FLOAT, `voltage` INT, `phase` FLOAT) TAGS (`groupid` INT, `location` BINARY(16))", + // create child table + "CREATE TABLE `d0` USING `meters` TAGS(0, 'Los Angles')", + // insert into child table + "INSERT INTO `d0` values(now - 10s, 10, 116, 0.32)", + // insert with NULL values + "INSERT INTO `d0` values(now - 8s, NULL, NULL, NULL)", + // insert and automatically create table with tags if not exists + "INSERT INTO `d1` USING `meters` TAGS(1, 'San Francisco') values(now - 9s, 10.1, 119, 0.33)", + // insert many records in a single sql + "INSERT INTO `d1` values (now-8s, 10, 120, 0.33) (now - 6s, 10, 119, 0.34) (now - 4s, 11.2, 118, 0.322)", + ]).await?; + + assert_eq!(inserted, 6); + loop { + let count: usize = taos + .query_one("select count(*) from `meters`") + .await? + .unwrap_or_default(); + + if count >= 6 { + break; + } else { + println!("waiting for data"); + } + } + + let mut result = taos.query("select tbname, * from `meters`").await?; + + for field in result.fields() { + println!("got field: {}", field.name()); + } + + // Query option 1, use rows stream. + let mut rows = result.rows(); + let mut nrows = 0; + while let Some(row) = rows.try_next().await? { + for (col, (name, value)) in row.enumerate() { + println!( + "[{}] got value in col {} (named `{:>8}`): {}", + nrows, col, name, value + ); + } + nrows += 1; + } + + // Query options 2, use deserialization with serde. + #[derive(Debug, serde::Deserialize)] + #[allow(dead_code)] + struct Record { + tbname: String, + // deserialize timestamp to chrono::DateTime + ts: DateTime, + // float to f32 + current: Option, + // int to i32 + voltage: Option, + phase: Option, + groupid: i32, + // binary/varchar to String + location: String, + } + + let records: Vec = taos + .query("select tbname, * from `meters`") + .await? + .deserialize() + .try_collect() + .await?; + + dbg!(result.summary()); + assert_eq!(records.len(), 6); + dbg!(records); + Ok(()) +} diff --git a/examples/rust/examples/subscribe.rs b/examples/rust/examples/subscribe.rs new file mode 100644 index 0000000000..9e2e890405 --- /dev/null +++ b/examples/rust/examples/subscribe.rs @@ -0,0 +1,103 @@ +use std::time::Duration; + +use chrono::{DateTime, Local}; +use taos::*; + +// Query options 2, use deserialization with serde. +#[derive(Debug, serde::Deserialize)] +#[allow(dead_code)] +struct Record { + // deserialize timestamp to chrono::DateTime + ts: DateTime, + // float to f32 + current: Option, + // int to i32 + voltage: Option, + phase: Option, +} + +async fn prepare(taos: Taos) -> anyhow::Result<()> { + let inserted = taos.exec_many([ + // create child table + "CREATE TABLE `d0` USING `meters` TAGS(0, 'Los Angles')", + // insert into child table + "INSERT INTO `d0` values(now - 10s, 10, 116, 0.32)", + // insert with NULL values + "INSERT INTO `d0` values(now - 8s, NULL, NULL, NULL)", + // insert and automatically create table with tags if not exists + "INSERT INTO `d1` USING `meters` TAGS(1, 'San Francisco') values(now - 9s, 10.1, 119, 0.33)", + // insert many records in a single sql + "INSERT INTO `d1` values (now-8s, 10, 120, 0.33) (now - 6s, 10, 119, 0.34) (now - 4s, 11.2, 118, 0.322)", + ]).await?; + assert_eq!(inserted, 6); + Ok(()) +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + // std::env::set_var("RUST_LOG", "debug"); + pretty_env_logger::init(); + let dsn = "taos://localhost:6030"; + let builder = TaosBuilder::from_dsn(dsn)?; + + let taos = builder.build()?; + let db = "tmq"; + + // prepare database + taos.exec_many([ + "DROP TOPIC IF EXISTS tmq_meters".to_string(), + format!("DROP DATABASE IF EXISTS `{db}`"), + format!("CREATE DATABASE `{db}`"), + format!("USE `{db}`"), + // create super table + "CREATE TABLE `meters` (`ts` TIMESTAMP, `current` FLOAT, `voltage` INT, `phase` FLOAT) TAGS (`groupid` INT, `location` BINARY(16))".to_string(), + // create topic for subscription + format!("CREATE TOPIC tmq_meters with META AS DATABASE {db}") + ]) + .await?; + + let task = tokio::spawn(prepare(taos)); + + tokio::time::sleep(Duration::from_secs(1)).await; + + // subscribe + let tmq = TmqBuilder::from_dsn("taos://localhost:6030/?group.id=test")?; + + let mut consumer = tmq.build()?; + consumer.subscribe(["tmq_meters"]).await?; + + { + let mut stream = consumer.stream(); + + while let Some((offset, message)) = stream.try_next().await? { + // get information from offset + + // the topic + let topic = offset.topic(); + // the vgroup id, like partition id in kafka. + let vgroup_id = offset.vgroup_id(); + println!("* in vgroup id {vgroup_id} of topic {topic}\n"); + + if let Some(data) = message.into_data() { + while let Some(block) = data.fetch_raw_block().await? { + // one block for one table, get table name if needed + let name = block.table_name(); + let records: Vec = block.deserialize().try_collect()?; + println!( + "** table: {}, got {} records: {:#?}\n", + name.unwrap(), + records.len(), + records + ); + } + } + consumer.commit(offset).await?; + } + } + + consumer.unsubscribe().await; + + task.await??; + + Ok(()) +} diff --git a/examples/rust/src/main.rs b/examples/rust/src/main.rs new file mode 100644 index 0000000000..e7a11a969c --- /dev/null +++ b/examples/rust/src/main.rs @@ -0,0 +1,3 @@ +fn main() { + println!("Hello, world!"); +} From 4f16771de4067d9cecbeef1baafd7b530409e411 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 17 Aug 2022 14:45:31 +0800 Subject: [PATCH 21/33] fix:error in get table list by tag filter --- source/dnode/vnode/inc/vnode.h | 2 +- source/dnode/vnode/src/meta/metaQuery.c | 21 ++++----------------- source/libs/executor/src/executil.c | 9 +++++---- 3 files changed, 10 insertions(+), 22 deletions(-) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 9d92a9c1e4..c224f6ce7f 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -91,7 +91,7 @@ typedef struct SMetaEntry SMetaEntry; void metaReaderInit(SMetaReader *pReader, SMeta *pMeta, int32_t flags); void metaReaderClear(SMetaReader *pReader); int32_t metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid); -int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList, SArray *tags); +int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList, SHashObj *tags); int32_t metaReadNext(SMetaReader *pReader); const void *metaGetTableTagVal(void *tag, int16_t type, STagVal *tagVal); int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName); diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 9f2cf409be..29a4dd3e34 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -962,13 +962,13 @@ END: return ret; } -int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList, SArray *tags) { +int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList, SHashObj *tags) { SMCtbCursor *pCur = metaOpenCtbCursor(pMeta, suid); SHashObj *uHash = NULL; size_t len = taosArrayGetSize(uidList); // len > 0 means there already have uids if(len > 0){ - uHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + uHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); for(int i = 0; i < len; i++){ int64_t *uid = taosArrayGet(uidList, i); taosHashPut(uHash, uid, sizeof(int64_t), &i, sizeof(i)); @@ -982,26 +982,13 @@ int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList, SArray *t if (len > 0 && taosHashGet(uHash, &id, sizeof(int64_t)) == NULL) { continue; - } - - void *tag = taosMemoryMalloc(pCur->vLen); - memcpy(tag, pCur->pVal, pCur->vLen); - - if (len == 0) { + }else if (len == 0) { taosArrayPush(uidList, &id); - taosArrayPush(tags, &tag); - }else{ - taosHashPut(uHash, &id, sizeof(int64_t), &tag, POINTER_BYTES); } - } - for(int i = 0; i < len; i++){ - int64_t *uid = taosArrayGet(uidList, i); - void **tag = taosHashGet(uHash, uid, POINTER_BYTES); - taosArrayPush(tags, tag); + taosHashPut(tags, &id, sizeof(int64_t), pCur->pVal, pCur->vLen); } - taosHashCleanup(uHash); metaCloseCtbCursor(pCur); return TSDB_CODE_SUCCESS; diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 9f8f97c4f5..2547b5e228 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -368,7 +368,7 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, uint64_t suid, SArray int32_t code = TSDB_CODE_SUCCESS; SArray* pBlockList = NULL; SSDataBlock* pResBlock = NULL; - SArray* tags = NULL; + SHashObj * tags = NULL; SScalarParam output = {0}; tagFilterAssist ctx = {0}; @@ -399,7 +399,7 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, uint64_t suid, SArray } // int64_t stt = taosGetTimestampUs(); - tags = taosArrayInit(8, POINTER_BYTES); + tags = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); code = metaGetTableTags(metaHandle, suid, uidList, tags); if (code != TSDB_CODE_SUCCESS) { terrno = code; @@ -421,8 +421,9 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, uint64_t suid, SArray // int64_t st = taosGetTimestampUs(); for (int32_t i = 0; i < rows; i++) { - void* tag = taosArrayGetP(tags, i); int64_t* uid = taosArrayGet(uidList, i); + void* tag = taosHashGet(tags, uid, sizeof(int64_t)); + ASSERT(tag); for(int32_t j = 0; j < taosArrayGetSize(pResBlock->pDataBlock); j++){ SColumnInfoData* pColInfo = (SColumnInfoData*)taosArrayGet(pResBlock->pDataBlock, j); @@ -474,7 +475,7 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, uint64_t suid, SArray // qDebug("calculate tag block rows:%d, cost:%ld us", rows, st2-st1); end: - taosArrayDestroyP(tags, taosMemoryFree); + taosHashCleanup(tags); taosHashCleanup(ctx.colHash); taosArrayDestroy(ctx.cInfoList); blockDataDestroy(pResBlock); From e7377e410caa4a55a37ccc7efd985cfc43c8be16 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 17 Aug 2022 15:10:15 +0800 Subject: [PATCH 22/33] refactor(mnode): check drop and alter stb for stream and topic --- source/dnode/mnode/impl/src/mndStb.c | 167 +++++++++++++++++++++++++-- 1 file changed, 160 insertions(+), 7 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 3e747b66c8..dd2b595c29 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -1145,7 +1145,7 @@ static int32_t mndAddSuperTableTag(const SStbObj *pOld, SStbObj *pNew, SArray *p return 0; } -int32_t mndCheckColAndTagModifiable(SMnode *pMnode, const char *stbname, int64_t suid, col_id_t colId) { +static int32_t mndCheckAlterColForTopic(SMnode *pMnode, const char *stbFullName, int64_t suid, col_id_t colId) { SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; while (1) { @@ -1154,7 +1154,7 @@ int32_t mndCheckColAndTagModifiable(SMnode *pMnode, const char *stbname, int64_t if (pIter == NULL) break; mDebug("topic:%s, check tag and column modifiable, stb:%s suid:%" PRId64 " colId:%d, subType:%d sql:%s", - pTopic->name, stbname, suid, colId, pTopic->subType, pTopic->sql); + pTopic->name, stbFullName, suid, colId, pTopic->subType, pTopic->sql); if (pTopic->subType != TOPIC_SUB_TYPE__COLUMN) { sdbRelease(pSdb, pTopic); continue; @@ -1192,20 +1192,66 @@ int32_t mndCheckColAndTagModifiable(SMnode *pMnode, const char *stbname, int64_t sdbRelease(pSdb, pTopic); nodesDestroyNode(pAst); } + return 0; +} +static int32_t mndCheckAlterColForStream(SMnode *pMnode, const char *stbFullName, int64_t suid, col_id_t colId) { + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + while (1) { + SStreamObj *pStream = NULL; + pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream); + if (pIter == NULL) break; + + SNode *pAst = NULL; + if (nodesStringToNode(pStream->ast, &pAst) != 0) { + ASSERT(0); + return -1; + } + + SNodeList *pNodeList = NULL; + nodesCollectColumns((SSelectStmt *)pAst, SQL_CLAUSE_FROM, NULL, COLLECT_COL_TYPE_ALL, &pNodeList); + SNode *pNode = NULL; + FOREACH(pNode, pNodeList) { + SColumnNode *pCol = (SColumnNode *)pNode; + + if (pCol->tableId != suid) { + mDebug("stream:%s, check colId:%d passed", pStream->name, pCol->colId); + goto NEXT; + } + if (pCol->colId > 0 && pCol->colId == colId) { + sdbRelease(pSdb, pStream); + nodesDestroyNode(pAst); + terrno = TSDB_CODE_MND_STREAM_MUST_BE_DELETED; + mError("stream:%s, check colId:%d conflicted", pStream->name, pCol->colId); + return -1; + } + mDebug("stream:%s, check colId:%d passed", pStream->name, pCol->colId); + } + + NEXT: + sdbRelease(pSdb, pStream); + nodesDestroyNode(pAst); + } + return 0; +} + +static int32_t mndCheckAlterColForTSma(SMnode *pMnode, const char *stbFullName, int64_t suid, col_id_t colId) { + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; while (1) { SSmaObj *pSma = NULL; pIter = sdbFetch(pSdb, SDB_SMA, pIter, (void **)&pSma); if (pIter == NULL) break; - mDebug("tsma:%s, check tag and column modifiable, stb:%s suid:%" PRId64 " colId:%d, sql:%s", pSma->name, stbname, - suid, colId, pSma->sql); + mDebug("tsma:%s, check tag and column modifiable, stb:%s suid:%" PRId64 " colId:%d, sql:%s", pSma->name, + stbFullName, suid, colId, pSma->sql); SNode *pAst = NULL; if (nodesStringToNode(pSma->ast, &pAst) != 0) { terrno = TSDB_CODE_SDB_INVALID_DATA_CONTENT; mError("tsma:%s, check tag and column modifiable, stb:%s suid:%" PRId64 " colId:%d failed since parse AST err", - pSma->name, stbname, suid, colId); + pSma->name, stbFullName, suid, colId); return -1; } @@ -1218,7 +1264,7 @@ int32_t mndCheckColAndTagModifiable(SMnode *pMnode, const char *stbname, int64_t if ((pCol->tableId != suid) && (pSma->stbUid != suid)) { mDebug("tsma:%s, check colId:%d passed", pSma->name, pCol->colId); - goto NEXT2; + goto NEXT; } if ((pCol->colId) > 0 && (pCol->colId == colId)) { sdbRelease(pSdb, pSma); @@ -1230,11 +1276,24 @@ int32_t mndCheckColAndTagModifiable(SMnode *pMnode, const char *stbname, int64_t mDebug("tsma:%s, check colId:%d passed", pSma->name, pCol->colId); } - NEXT2: + NEXT: sdbRelease(pSdb, pSma); nodesDestroyNode(pAst); } + return 0; +} +int32_t mndCheckColAndTagModifiable(SMnode *pMnode, const char *stbFullName, int64_t suid, col_id_t colId) { + if (mndCheckAlterColForTopic(pMnode, stbFullName, suid, colId) < 0) { + return -1; + } + if (mndCheckAlterColForStream(pMnode, stbFullName, suid, colId) < 0) { + return -1; + } + + if (mndCheckAlterColForTSma(pMnode, stbFullName, suid, colId) < 0) { + return -1; + } return 0; } @@ -1930,6 +1989,90 @@ _OVER: return code; } +static int32_t mndCheckDropStbForTopic(SMnode *pMnode, const char *stbFullName, int64_t suid) { + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + while (1) { + SMqTopicObj *pTopic = NULL; + pIter = sdbFetch(pSdb, SDB_TOPIC, pIter, (void **)&pTopic); + if (pIter == NULL) break; + + if (pTopic->subType == TOPIC_SUB_TYPE__TABLE) { + if (pTopic->stbUid == suid) { + sdbRelease(pSdb, pTopic); + return -1; + } + } + + if (pTopic->subType != TOPIC_SUB_TYPE__COLUMN) { + sdbRelease(pSdb, pTopic); + continue; + } + + SNode *pAst = NULL; + if (nodesStringToNode(pTopic->ast, &pAst) != 0) { + ASSERT(0); + return -1; + } + + SNodeList *pNodeList = NULL; + nodesCollectColumns((SSelectStmt *)pAst, SQL_CLAUSE_FROM, NULL, COLLECT_COL_TYPE_ALL, &pNodeList); + SNode *pNode = NULL; + FOREACH(pNode, pNodeList) { + SColumnNode *pCol = (SColumnNode *)pNode; + + if (pCol->tableId != suid) { + mDebug("topic:%s, check colId:%d passed", pTopic->name, pCol->colId); + sdbRelease(pSdb, pTopic); + nodesDestroyNode(pAst); + return -1; + } else { + goto NEXT; + } + } + NEXT: + sdbRelease(pSdb, pTopic); + nodesDestroyNode(pAst); + } + return 0; +} + +static int32_t mndCheckDropStbForStream(SMnode *pMnode, const char *stbFullName, int64_t suid) { + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + while (1) { + SStreamObj *pStream = NULL; + pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream); + if (pIter == NULL) break; + + SNode *pAst = NULL; + if (nodesStringToNode(pStream->ast, &pAst) != 0) { + ASSERT(0); + return -1; + } + + SNodeList *pNodeList = NULL; + nodesCollectColumns((SSelectStmt *)pAst, SQL_CLAUSE_FROM, NULL, COLLECT_COL_TYPE_ALL, &pNodeList); + SNode *pNode = NULL; + FOREACH(pNode, pNodeList) { + SColumnNode *pCol = (SColumnNode *)pNode; + + if (pCol->tableId != suid) { + mDebug("stream:%s, check colId:%d passed", pStream->name, pCol->colId); + sdbRelease(pSdb, pStream); + nodesDestroyNode(pAst); + return -1; + } else { + goto NEXT; + } + } + NEXT: + sdbRelease(pSdb, pStream); + nodesDestroyNode(pAst); + } + return 0; +} + static int32_t mndProcessDropStbReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; int32_t code = -1; @@ -1971,6 +2114,16 @@ static int32_t mndProcessDropStbReq(SRpcMsg *pReq) { goto _OVER; } + if (mndCheckDropStbForTopic(pMnode, dropReq.name, pStb->uid) < 0) { + terrno = TSDB_CODE_MND_TOPIC_MUST_BE_DELETED; + goto _OVER; + } + + if (mndCheckDropStbForStream(pMnode, dropReq.name, pStb->uid) < 0) { + terrno = TSDB_CODE_MND_STREAM_MUST_BE_DELETED; + goto _OVER; + } + code = mndDropStb(pMnode, pReq, pDb, pStb); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; From a036d64ae0dc7d50cf5c4bf21df6ef9949396baa Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 17 Aug 2022 15:28:49 +0800 Subject: [PATCH 23/33] avoid filter invalid table --- source/dnode/vnode/src/meta/metaQuery.c | 141 ++++++++++++------------ 1 file changed, 72 insertions(+), 69 deletions(-) diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 29a4dd3e34..4e85cde7ca 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -53,79 +53,79 @@ _err: return -1; } -//int metaGetTableEntryByUidTest(void* meta, SArray *uidList) { +// int metaGetTableEntryByUidTest(void* meta, SArray *uidList) { // -// SArray* readerList = taosArrayInit(taosArrayGetSize(uidList), sizeof(SMetaReader)); -// SArray* uidVersion = taosArrayInit(taosArrayGetSize(uidList), sizeof(STbDbKey)); -// SMeta *pMeta = meta; -// int64_t version; -// SHashObj *uHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); +// SArray* readerList = taosArrayInit(taosArrayGetSize(uidList), sizeof(SMetaReader)); +// SArray* uidVersion = taosArrayInit(taosArrayGetSize(uidList), sizeof(STbDbKey)); +// SMeta *pMeta = meta; +// int64_t version; +// SHashObj *uHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); // -// int64_t stt1 = taosGetTimestampUs(); -// for(int i = 0; i < taosArrayGetSize(uidList); i++) { -// void* ppVal = NULL; -// int vlen = 0; -// uint64_t * uid = taosArrayGet(uidList, i); -// // query uid.idx -// if (tdbTbGet(pMeta->pUidIdx, uid, sizeof(*uid), &ppVal, &vlen) < 0) { -// continue; -// } -// version = *(int64_t *)ppVal; +// int64_t stt1 = taosGetTimestampUs(); +// for(int i = 0; i < taosArrayGetSize(uidList); i++) { +// void* ppVal = NULL; +// int vlen = 0; +// uint64_t * uid = taosArrayGet(uidList, i); +// // query uid.idx +// if (tdbTbGet(pMeta->pUidIdx, uid, sizeof(*uid), &ppVal, &vlen) < 0) { +// continue; +// } +// version = *(int64_t *)ppVal; // -// STbDbKey tbDbKey = {.version = version, .uid = *uid}; -// taosArrayPush(uidVersion, &tbDbKey); -// taosHashPut(uHash, uid, sizeof(int64_t), ppVal, sizeof(int64_t)); -// } -// int64_t stt2 = taosGetTimestampUs(); -// qDebug("metaGetTableEntryByUidTest1 rows:%d, cost:%ld us", taosArrayGetSize(uidList), stt2-stt1); +// STbDbKey tbDbKey = {.version = version, .uid = *uid}; +// taosArrayPush(uidVersion, &tbDbKey); +// taosHashPut(uHash, uid, sizeof(int64_t), ppVal, sizeof(int64_t)); +// } +// int64_t stt2 = taosGetTimestampUs(); +// qDebug("metaGetTableEntryByUidTest1 rows:%d, cost:%ld us", taosArrayGetSize(uidList), stt2-stt1); // -// TBC *pCur = NULL; -// tdbTbcOpen(pMeta->pTbDb, &pCur, NULL); -// tdbTbcMoveToFirst(pCur); -// void *pKey = NULL; -// int kLen = 0; +// TBC *pCur = NULL; +// tdbTbcOpen(pMeta->pTbDb, &pCur, NULL); +// tdbTbcMoveToFirst(pCur); +// void *pKey = NULL; +// int kLen = 0; // -// while(1){ -// SMetaReader pReader = {0}; -// int32_t ret = tdbTbcNext(pCur, &pKey, &kLen, &pReader.pBuf, &pReader.szBuf); -// if (ret < 0) break; -// STbDbKey *tmp = (STbDbKey*)pKey; -// int64_t *ver = (int64_t*)taosHashGet(uHash, &tmp->uid, sizeof(int64_t)); -// if(ver == NULL || *ver != tmp->version) continue; -// taosArrayPush(readerList, &pReader); -// } -// tdbTbcClose(pCur); +// while(1){ +// SMetaReader pReader = {0}; +// int32_t ret = tdbTbcNext(pCur, &pKey, &kLen, &pReader.pBuf, &pReader.szBuf); +// if (ret < 0) break; +// STbDbKey *tmp = (STbDbKey*)pKey; +// int64_t *ver = (int64_t*)taosHashGet(uHash, &tmp->uid, sizeof(int64_t)); +// if(ver == NULL || *ver != tmp->version) continue; +// taosArrayPush(readerList, &pReader); +// } +// tdbTbcClose(pCur); // -// taosArrayClear(readerList); -// int64_t stt3 = taosGetTimestampUs(); -// qDebug("metaGetTableEntryByUidTest2 rows:%d, cost:%ld us", taosArrayGetSize(uidList), stt3-stt2); -// for(int i = 0; i < taosArrayGetSize(uidVersion); i++) { -// SMetaReader pReader = {0}; +// taosArrayClear(readerList); +// int64_t stt3 = taosGetTimestampUs(); +// qDebug("metaGetTableEntryByUidTest2 rows:%d, cost:%ld us", taosArrayGetSize(uidList), stt3-stt2); +// for(int i = 0; i < taosArrayGetSize(uidVersion); i++) { +// SMetaReader pReader = {0}; // -// STbDbKey *tbDbKey = taosArrayGet(uidVersion, i); -// // query table.db -// if (tdbTbGet(pMeta->pTbDb, tbDbKey, sizeof(STbDbKey), &pReader.pBuf, &pReader.szBuf) < 0) { -// continue; -// } -// taosArrayPush(readerList, &pReader); -// } -// int64_t stt4 = taosGetTimestampUs(); -// qDebug("metaGetTableEntryByUidTest3 rows:%d, cost:%ld us", taosArrayGetSize(uidList), stt4-stt3); +// STbDbKey *tbDbKey = taosArrayGet(uidVersion, i); +// // query table.db +// if (tdbTbGet(pMeta->pTbDb, tbDbKey, sizeof(STbDbKey), &pReader.pBuf, &pReader.szBuf) < 0) { +// continue; +// } +// taosArrayPush(readerList, &pReader); +// } +// int64_t stt4 = taosGetTimestampUs(); +// qDebug("metaGetTableEntryByUidTest3 rows:%d, cost:%ld us", taosArrayGetSize(uidList), stt4-stt3); // -// for(int i = 0; i < taosArrayGetSize(readerList); i++){ -// SMetaReader* pReader = taosArrayGet(readerList, i); -// metaReaderInit(pReader, meta, 0); -// // decode the entry -// tDecoderInit(&pReader->coder, pReader->pBuf, pReader->szBuf); +// for(int i = 0; i < taosArrayGetSize(readerList); i++){ +// SMetaReader* pReader = taosArrayGet(readerList, i); +// metaReaderInit(pReader, meta, 0); +// // decode the entry +// tDecoderInit(&pReader->coder, pReader->pBuf, pReader->szBuf); // -// if (metaDecodeEntry(&pReader->coder, &pReader->me) < 0) { -// } -// metaReaderClear(pReader); -// } -// int64_t stt5 = taosGetTimestampUs(); -// qDebug("metaGetTableEntryByUidTest4 rows:%d, cost:%ld us", taosArrayGetSize(readerList), stt5-stt4); -// return 0; -//} +// if (metaDecodeEntry(&pReader->coder, &pReader->me) < 0) { +// } +// metaReaderClear(pReader); +// } +// int64_t stt5 = taosGetTimestampUs(); +// qDebug("metaGetTableEntryByUidTest4 rows:%d, cost:%ld us", taosArrayGetSize(readerList), stt5-stt4); +// return 0; +// } int metaGetTableEntryByUid(SMetaReader *pReader, tb_uid_t uid) { SMeta *pMeta = pReader->pMeta; @@ -824,7 +824,7 @@ SArray *metaGetSmaTbUids(SMeta *pMeta) { #endif const void *metaGetTableTagVal(void *pTag, int16_t type, STagVal *val) { - STag *tag = (STag*) pTag; + STag *tag = (STag *)pTag; if (type == TSDB_DATA_TYPE_JSON) { return tag; } @@ -926,6 +926,9 @@ int32_t metaFilterTableIds(SMeta *pMeta, SMetaFltParam *param, SArray *pUids) { break; } } + if (p->suid != pKey->suid) { + break; + } first = false; if (p != NULL) { int32_t cmp = (*param->filterFunc)(p->data, pKey->data, pKey->type); @@ -966,10 +969,10 @@ int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList, SHashObj SMCtbCursor *pCur = metaOpenCtbCursor(pMeta, suid); SHashObj *uHash = NULL; - size_t len = taosArrayGetSize(uidList); // len > 0 means there already have uids - if(len > 0){ + size_t len = taosArrayGetSize(uidList); // len > 0 means there already have uids + if (len > 0) { uHash = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); - for(int i = 0; i < len; i++){ + for (int i = 0; i < len; i++) { int64_t *uid = taosArrayGet(uidList, i); taosHashPut(uHash, uid, sizeof(int64_t), &i, sizeof(i)); } @@ -982,7 +985,7 @@ int32_t metaGetTableTags(SMeta *pMeta, uint64_t suid, SArray *uidList, SHashObj if (len > 0 && taosHashGet(uHash, &id, sizeof(int64_t)) == NULL) { continue; - }else if (len == 0) { + } else if (len == 0) { taosArrayPush(uidList, &id); } From 2ca5bdc708ccbeb497912b01dd4b05b09f69d4dc Mon Sep 17 00:00:00 2001 From: Minghao Li Date: Wed, 17 Aug 2022 15:39:38 +0800 Subject: [PATCH 24/33] refactor(sync): add syncNodeDynamicQuorum --- include/libs/sync/sync.h | 14 ++-- include/libs/sync/syncTools.h | 1 + source/libs/sync/inc/syncIndexMgr.h | 17 +++-- source/libs/sync/inc/syncInt.h | 2 + source/libs/sync/src/syncAppendEntries.c | 9 +++ source/libs/sync/src/syncAppendEntriesReply.c | 12 ++++ source/libs/sync/src/syncCommit.c | 58 ++++++++++++++++ source/libs/sync/src/syncIndexMgr.c | 66 ++++++++++++++++++- source/libs/sync/src/syncMain.c | 11 ++-- source/libs/sync/src/syncMessage.c | 2 + .../sync/test/syncAppendEntriesReplyTest.cpp | 3 + 11 files changed, 178 insertions(+), 17 deletions(-) diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index 7cd2ebdede..952066df46 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -26,12 +26,14 @@ extern "C" { extern bool gRaftDetailLog; -#define SYNC_RESP_TTL_MS 10000000 -#define SYNC_SPEED_UP_HB_TIMER 400 -#define SYNC_SPEED_UP_AFTER_MS (1000 * 20) -#define SYNC_SLOW_DOWN_RANGE 100 -#define SYNC_MAX_READ_RANGE 10 -#define SYNC_MAX_PROGRESS_WAIT_MS 4000 +#define SYNC_RESP_TTL_MS 10000000 +#define SYNC_SPEED_UP_HB_TIMER 400 +#define SYNC_SPEED_UP_AFTER_MS (1000 * 20) +#define SYNC_SLOW_DOWN_RANGE 100 +#define SYNC_MAX_READ_RANGE 2 +#define SYNC_MAX_PROGRESS_WAIT_MS 4000 +#define SYNC_MAX_START_TIME_RANGE_MS (1000 * 20) +#define SYNC_MAX_RECV_TIME_RANGE_MS 1000 #define SYNC_MAX_BATCH_SIZE 1 #define SYNC_INDEX_BEGIN 0 diff --git a/include/libs/sync/syncTools.h b/include/libs/sync/syncTools.h index cd2c2d4a4f..6c95c3c6d7 100644 --- a/include/libs/sync/syncTools.h +++ b/include/libs/sync/syncTools.h @@ -423,6 +423,7 @@ typedef struct SyncAppendEntriesReply { SyncTerm privateTerm; bool success; SyncIndex matchIndex; + int64_t startTime; } SyncAppendEntriesReply; SyncAppendEntriesReply* syncAppendEntriesReplyBuild(int32_t vgId); diff --git a/source/libs/sync/inc/syncIndexMgr.h b/source/libs/sync/inc/syncIndexMgr.h index 1f60a9d57e..fb85b89419 100644 --- a/source/libs/sync/inc/syncIndexMgr.h +++ b/source/libs/sync/inc/syncIndexMgr.h @@ -29,8 +29,12 @@ extern "C" { // SIndexMgr ----------------------------- typedef struct SSyncIndexMgr { SRaftId (*replicas)[TSDB_MAX_REPLICA]; - SyncIndex index[TSDB_MAX_REPLICA]; - SyncTerm privateTerm[TSDB_MAX_REPLICA]; // for advanced function + SyncIndex index[TSDB_MAX_REPLICA]; + SyncTerm privateTerm[TSDB_MAX_REPLICA]; // for advanced function + + int64_t startTimeArr[TSDB_MAX_REPLICA]; + int64_t recvTimeArr[TSDB_MAX_REPLICA]; + int32_t replicaNum; SSyncNode *pSyncNode; } SSyncIndexMgr; @@ -41,8 +45,13 @@ void syncIndexMgrDestroy(SSyncIndexMgr *pSyncIndexMgr); void syncIndexMgrClear(SSyncIndexMgr *pSyncIndexMgr); void syncIndexMgrSetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, SyncIndex index); SyncIndex syncIndexMgrGetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId); -cJSON * syncIndexMgr2Json(SSyncIndexMgr *pSyncIndexMgr); -char * syncIndexMgr2Str(SSyncIndexMgr *pSyncIndexMgr); +cJSON *syncIndexMgr2Json(SSyncIndexMgr *pSyncIndexMgr); +char *syncIndexMgr2Str(SSyncIndexMgr *pSyncIndexMgr); + +void syncIndexMgrSetStartTime(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, int64_t startTime); +int64_t syncIndexMgrGetStartTime(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId); +void syncIndexMgrSetRecvTime(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, int64_t recvTime); +int64_t syncIndexMgrGetRecvTime(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId); // void syncIndexMgrSetTerm(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, SyncTerm term); // SyncTerm syncIndexMgrGetTerm(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId); diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 3e247e5d79..de43c81654 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -269,6 +269,8 @@ int32_t syncNodeLeaderTransfer(SSyncNode* pSyncNode); int32_t syncNodeLeaderTransferTo(SSyncNode* pSyncNode, SNodeInfo newLeader); int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* pEntry); +int32_t syncNodeDynamicQuorum(const SSyncNode* pSyncNode); + // trace log void syncLogSendRequestVote(SSyncNode* pSyncNode, const SyncRequestVote* pMsg, const char* s); void syncLogRecvRequestVote(SSyncNode* pSyncNode, const SyncRequestVote* pMsg, const char* s); diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 4f93d8197d..e000ba8bf8 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -148,6 +148,7 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { pReply->term = ths->pRaftStore->currentTerm; pReply->success = false; pReply->matchIndex = SYNC_INDEX_INVALID; + pReply->startTime = ths->startTime; // msg event log syncLogSendAppendEntriesReply(ths, pReply, ""); @@ -290,6 +291,8 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { pReply->matchIndex = pMsg->prevLogIndex; } + pReply->startTime = ths->startTime; + // msg event log syncLogSendAppendEntriesReply(ths, pReply, ""); @@ -603,6 +606,7 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc pReply->privateTerm = ths->pNewNodeReceiver->privateTerm; pReply->success = true; pReply->matchIndex = matchIndex; + pReply->startTime = ths->startTime; // msg event log syncLogSendAppendEntriesReply(ths, pReply, ""); @@ -651,6 +655,7 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc pReply->privateTerm = ths->pNewNodeReceiver->privateTerm; pReply->success = false; pReply->matchIndex = ths->commitIndex; + pReply->startTime = ths->startTime; // msg event log syncLogSendAppendEntriesReply(ths, pReply, ""); @@ -729,6 +734,7 @@ int32_t syncNodeOnAppendEntriesSnapshot2Cb(SSyncNode* ths, SyncAppendEntriesBatc pReply->privateTerm = ths->pNewNodeReceiver->privateTerm; pReply->success = true; pReply->matchIndex = hasAppendEntries ? pMsg->prevLogIndex + pMsg->dataCount : pMsg->prevLogIndex; + pReply->startTime = ths->startTime; // msg event log syncLogSendAppendEntriesReply(ths, pReply, ""); @@ -874,6 +880,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs pReply->privateTerm = ths->pNewNodeReceiver->privateTerm; pReply->success = true; pReply->matchIndex = matchIndex; + pReply->startTime = ths->startTime; // msg event log syncLogSendAppendEntriesReply(ths, pReply, ""); @@ -919,6 +926,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs pReply->privateTerm = ths->pNewNodeReceiver->privateTerm; pReply->success = false; pReply->matchIndex = SYNC_INDEX_INVALID; + pReply->startTime = ths->startTime; // msg event log syncLogSendAppendEntriesReply(ths, pReply, ""); @@ -984,6 +992,7 @@ int32_t syncNodeOnAppendEntriesSnapshotCb(SSyncNode* ths, SyncAppendEntries* pMs pReply->privateTerm = ths->pNewNodeReceiver->privateTerm; pReply->success = true; pReply->matchIndex = hasAppendEntries ? pMsg->prevLogIndex + 1 : pMsg->prevLogIndex; + pReply->startTime = ths->startTime; // msg event log syncLogSendAppendEntriesReply(ths, pReply, ""); diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index 4928c54bd7..9253ed0129 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -64,6 +64,10 @@ int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* p ASSERT(pMsg->term == ths->pRaftStore->currentTerm); + // update time + syncIndexMgrSetStartTime(ths->pNextIndex, &(pMsg->srcId), pMsg->startTime); + syncIndexMgrSetRecvTime(ths->pNextIndex, &(pMsg->srcId), taosGetTimestampMs()); + SyncIndex beforeNextIndex = syncIndexMgrGetIndex(ths->pNextIndex, &(pMsg->srcId)); SyncIndex beforeMatchIndex = syncIndexMgrGetIndex(ths->pMatchIndex, &(pMsg->srcId)); @@ -170,6 +174,10 @@ int32_t syncNodeOnAppendEntriesReplySnapshot2Cb(SSyncNode* ths, SyncAppendEntrie ASSERT(pMsg->term == ths->pRaftStore->currentTerm); + // update time + syncIndexMgrSetStartTime(ths->pNextIndex, &(pMsg->srcId), pMsg->startTime); + syncIndexMgrSetRecvTime(ths->pNextIndex, &(pMsg->srcId), taosGetTimestampMs()); + SyncIndex beforeNextIndex = syncIndexMgrGetIndex(ths->pNextIndex, &(pMsg->srcId)); SyncIndex beforeMatchIndex = syncIndexMgrGetIndex(ths->pMatchIndex, &(pMsg->srcId)); @@ -330,6 +338,10 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries ASSERT(pMsg->term == ths->pRaftStore->currentTerm); + // update time + syncIndexMgrSetStartTime(ths->pNextIndex, &(pMsg->srcId), pMsg->startTime); + syncIndexMgrSetRecvTime(ths->pNextIndex, &(pMsg->srcId), taosGetTimestampMs()); + SyncIndex beforeNextIndex = syncIndexMgrGetIndex(ths->pNextIndex, &(pMsg->srcId)); SyncIndex beforeMatchIndex = syncIndexMgrGetIndex(ths->pMatchIndex, &(pMsg->srcId)); diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c index 3a94ed9713..3829ea0730 100644 --- a/source/libs/sync/src/syncCommit.c +++ b/source/libs/sync/src/syncCommit.c @@ -133,6 +133,63 @@ bool syncAgreeIndex(SSyncNode* pSyncNode, SRaftId* pRaftId, SyncIndex index) { return false; } +static inline int64_t syncNodeAbs64(int64_t a, int64_t b) { + ASSERT(a >= 0); + ASSERT(b >= 0); + + int64_t c = a > b ? a - b : b - a; + return c; +} + +int32_t syncNodeDynamicQuorum(const SSyncNode* pSyncNode) { + int32_t quorum = 1; // self + + int64_t timeNow = taosGetTimestampMs(); + for (int i = 0; i < pSyncNode->peersNum; ++i) { + int64_t peerStartTime = syncIndexMgrGetStartTime(pSyncNode->pNextIndex, &(pSyncNode->peersId)[i]); + int64_t peerRecvTime = syncIndexMgrGetRecvTime(pSyncNode->pNextIndex, &(pSyncNode->peersId)[i]); + + int64_t recvTimeDiff = syncNodeAbs64(peerRecvTime, timeNow); + int64_t startTimeDiff = syncNodeAbs64(peerStartTime, pSyncNode->startTime); + + int32_t addQuorum = 0; + + if (recvTimeDiff < SYNC_MAX_RECV_TIME_RANGE_MS) { + addQuorum = 1; + } else { + addQuorum = 0; + } + + if (startTimeDiff > SYNC_MAX_START_TIME_RANGE_MS) { + addQuorum = 0; + } + + quorum += addQuorum; + } + + ASSERT(quorum <= pSyncNode->replicaNum); + + if (quorum < pSyncNode->quorum) { + quorum = pSyncNode->quorum; + } + + return quorum; +} + +bool syncAgree(SSyncNode* pSyncNode, SyncIndex index) { + int agreeCount = 0; + for (int i = 0; i < pSyncNode->replicaNum; ++i) { + if (syncAgreeIndex(pSyncNode, &(pSyncNode->replicasId[i]), index)) { + ++agreeCount; + } + if (agreeCount >= syncNodeDynamicQuorum(pSyncNode)) { + return true; + } + } + return false; +} + +/* bool syncAgree(SSyncNode* pSyncNode, SyncIndex index) { int agreeCount = 0; for (int i = 0; i < pSyncNode->replicaNum; ++i) { @@ -145,3 +202,4 @@ bool syncAgree(SSyncNode* pSyncNode, SyncIndex index) { } return false; } +*/ diff --git a/source/libs/sync/src/syncIndexMgr.c b/source/libs/sync/src/syncIndexMgr.c index 8c820fcd9c..07c4fa8429 100644 --- a/source/libs/sync/src/syncIndexMgr.c +++ b/source/libs/sync/src/syncIndexMgr.c @@ -47,6 +47,13 @@ void syncIndexMgrDestroy(SSyncIndexMgr *pSyncIndexMgr) { void syncIndexMgrClear(SSyncIndexMgr *pSyncIndexMgr) { memset(pSyncIndexMgr->index, 0, sizeof(pSyncIndexMgr->index)); memset(pSyncIndexMgr->privateTerm, 0, sizeof(pSyncIndexMgr->privateTerm)); + + // int64_t timeNow = taosGetMonotonicMs(); + for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { + pSyncIndexMgr->startTimeArr[i] = 0; + pSyncIndexMgr->recvTimeArr[i] = 0; + } + /* for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { pSyncIndexMgr->index[i] = 0; @@ -68,7 +75,8 @@ void syncIndexMgrSetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, char host[128]; uint16_t port; syncUtilU642Addr(pRaftId->addr, host, sizeof(host), &port); - sError("vgId:%d, index mgr set for %s:%d, index:%" PRId64 " error", pSyncIndexMgr->pSyncNode->vgId, host, port, index); + sError("vgId:%d, index mgr set for %s:%d, index:%" PRId64 " error", pSyncIndexMgr->pSyncNode->vgId, host, port, + index); } SyncIndex syncIndexMgrGetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId) { @@ -125,11 +133,65 @@ cJSON *syncIndexMgr2Json(SSyncIndexMgr *pSyncIndexMgr) { char *syncIndexMgr2Str(SSyncIndexMgr *pSyncIndexMgr) { cJSON *pJson = syncIndexMgr2Json(pSyncIndexMgr); - char * serialized = cJSON_Print(pJson); + char *serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } +void syncIndexMgrSetStartTime(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, int64_t startTime) { + for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { + if (syncUtilSameId(&((*(pSyncIndexMgr->replicas))[i]), pRaftId)) { + (pSyncIndexMgr->startTimeArr)[i] = startTime; + return; + } + } + + // maybe config change + // ASSERT(0); + char host[128]; + uint16_t port; + syncUtilU642Addr(pRaftId->addr, host, sizeof(host), &port); + sError("vgId:%d, index mgr set for %s:%d, start-time:%" PRId64 " error", pSyncIndexMgr->pSyncNode->vgId, host, port, + startTime); +} + +int64_t syncIndexMgrGetStartTime(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId) { + for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { + if (syncUtilSameId(&((*(pSyncIndexMgr->replicas))[i]), pRaftId)) { + int64_t startTime = (pSyncIndexMgr->startTimeArr)[i]; + return startTime; + } + } + ASSERT(0); +} + +void syncIndexMgrSetRecvTime(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, int64_t recvTime) { + for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { + if (syncUtilSameId(&((*(pSyncIndexMgr->replicas))[i]), pRaftId)) { + (pSyncIndexMgr->recvTimeArr)[i] = recvTime; + return; + } + } + + // maybe config change + // ASSERT(0); + char host[128]; + uint16_t port; + syncUtilU642Addr(pRaftId->addr, host, sizeof(host), &port); + sError("vgId:%d, index mgr set for %s:%d, recv-time:%" PRId64 " error", pSyncIndexMgr->pSyncNode->vgId, host, port, + recvTime); +} + +int64_t syncIndexMgrGetRecvTime(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId) { + for (int i = 0; i < pSyncIndexMgr->replicaNum; ++i) { + if (syncUtilSameId(&((*(pSyncIndexMgr->replicas))[i]), pRaftId)) { + int64_t recvTime = (pSyncIndexMgr->recvTimeArr)[i]; + return recvTime; + } + } + ASSERT(0); +} + // for debug ------------------- void syncIndexMgrPrint(SSyncIndexMgr *pObj) { char *serialized = syncIndexMgr2Str(pObj); diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 1991560d42..a00b59d292 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -1682,13 +1682,13 @@ inline void syncNodeEventLog(const SSyncNode* pSyncNode, char* str) { ", sby:%d, " "stgy:%d, bch:%d, " "r-num:%d, " - "lcfg:%" PRId64 ", chging:%d, rsto:%d, elt:%" PRId64 ", hb:%" PRId64 ", %s", + "lcfg:%" PRId64 ", chging:%d, rsto:%d, dquorum:%d, elt:%" PRId64 ", hb:%" PRId64 ", %s", pSyncNode->vgId, syncUtilState2String(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, logBeginIndex, logLastIndex, snapshot.lastApplyIndex, snapshot.lastApplyTerm, pSyncNode->pRaftCfg->isStandBy, pSyncNode->pRaftCfg->snapshotStrategy, pSyncNode->pRaftCfg->batchSize, pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex, pSyncNode->changing, - pSyncNode->restoreFinish, pSyncNode->electTimerLogicClockUser, pSyncNode->heartbeatTimerLogicClockUser, - printStr); + pSyncNode->restoreFinish, syncNodeDynamicQuorum(pSyncNode), pSyncNode->electTimerLogicClockUser, + pSyncNode->heartbeatTimerLogicClockUser, printStr); } else { snprintf(logBuf, sizeof(logBuf), "%s", str); } @@ -1706,12 +1706,13 @@ inline void syncNodeEventLog(const SSyncNode* pSyncNode, char* str) { ", sby:%d, " "stgy:%d, bch:%d, " "r-num:%d, " - "lcfg:%" PRId64 ", chging:%d, rsto:%d, %s", + "lcfg:%" PRId64 ", chging:%d, rsto:%d, dquorum:%d, elt:%" PRId64 ", hb:%" PRId64 ", %s", pSyncNode->vgId, syncUtilState2String(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm, pSyncNode->commitIndex, logBeginIndex, logLastIndex, snapshot.lastApplyIndex, snapshot.lastApplyTerm, pSyncNode->pRaftCfg->isStandBy, pSyncNode->pRaftCfg->snapshotStrategy, pSyncNode->pRaftCfg->batchSize, pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex, pSyncNode->changing, - pSyncNode->restoreFinish, printStr); + pSyncNode->restoreFinish, syncNodeDynamicQuorum(pSyncNode), pSyncNode->electTimerLogicClockUser, + pSyncNode->heartbeatTimerLogicClockUser, printStr); } else { snprintf(s, len, "%s", str); } diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index 13adaf055c..b42aba560f 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -1947,6 +1947,8 @@ cJSON* syncAppendEntriesReply2Json(const SyncAppendEntriesReply* pMsg) { cJSON_AddNumberToObject(pRoot, "success", pMsg->success); snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->matchIndex); cJSON_AddStringToObject(pRoot, "matchIndex", u64buf); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->startTime); + cJSON_AddStringToObject(pRoot, "startTime", u64buf); } cJSON* pJson = cJSON_CreateObject(); diff --git a/source/libs/sync/test/syncAppendEntriesReplyTest.cpp b/source/libs/sync/test/syncAppendEntriesReplyTest.cpp index d41e99a3cd..72d3fd5ef3 100644 --- a/source/libs/sync/test/syncAppendEntriesReplyTest.cpp +++ b/source/libs/sync/test/syncAppendEntriesReplyTest.cpp @@ -24,6 +24,7 @@ SyncAppendEntriesReply *createMsg() { pMsg->matchIndex = 77; pMsg->term = 33; pMsg->privateTerm = 44; + pMsg->startTime = taosGetTimestampMs(); return pMsg; } @@ -89,6 +90,8 @@ void test5() { } int main() { + gRaftDetailLog = true; + tsAsyncLog = 0; sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; logTest(); From 359fa4bc56ea43fe90bf39aec8ec048deaee4040 Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 17 Aug 2022 15:50:33 +0800 Subject: [PATCH 25/33] fix:error in get table list by tag filter --- source/libs/executor/src/executil.c | 1 + 1 file changed, 1 insertion(+) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 2547b5e228..fc10551618 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -506,6 +506,7 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, code = doFilterTag(pTagIndexCond, &metaArg, res, &status); if (code != 0 || status == SFLT_NOT_INDEX) { qError("failed to get tableIds from index, reason:%s, suid:%" PRIu64, tstrerror(code), tableUid); + code = TDB_CODE_SUCCESS; } // int64_t stt1 = taosGetTimestampUs(); From 6453e24c846dd5da55660fb54b19e6455d22a48c Mon Sep 17 00:00:00 2001 From: Huo Linhe Date: Wed, 17 Aug 2022 17:20:54 +0800 Subject: [PATCH 26/33] chore: add lost source code for tmq example of C --- docs/examples/c/tmq_example.c | 275 ++++++++++++++++++++++++++++++++++ 1 file changed, 275 insertions(+) create mode 100644 docs/examples/c/tmq_example.c diff --git a/docs/examples/c/tmq_example.c b/docs/examples/c/tmq_example.c new file mode 100644 index 0000000000..19adaad116 --- /dev/null +++ b/docs/examples/c/tmq_example.c @@ -0,0 +1,275 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#include +#include "taos.h" + +static int running = 1; +static char dbName[64] = "tmqdb"; +static char stbName[64] = "stb"; +static char topicName[64] = "topicname"; + +static int32_t msg_process(TAOS_RES* msg) { + char buf[1024]; + int32_t rows = 0; + + const char* topicName = tmq_get_topic_name(msg); + const char* dbName = tmq_get_db_name(msg); + int32_t vgroupId = tmq_get_vgroup_id(msg); + + printf("topic: %s\n", topicName); + printf("db: %s\n", dbName); + printf("vgroup id: %d\n", vgroupId); + + while (1) { + TAOS_ROW row = taos_fetch_row(msg); + if (row == NULL) break; + + TAOS_FIELD* fields = taos_fetch_fields(msg); + int32_t numOfFields = taos_field_count(msg); + int32_t* length = taos_fetch_lengths(msg); + int32_t precision = taos_result_precision(msg); + rows++; + taos_print_row(buf, row, fields, numOfFields); + printf("row content: %s\n", buf); + } + + return rows; +} + +static int32_t init_env() { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + if (pConn == NULL) { + return -1; + } + + TAOS_RES* pRes; + // drop database if exists + printf("create database\n"); + pRes = taos_query(pConn, "drop database if exists tmqdb"); + if (taos_errno(pRes) != 0) { + printf("error in drop tmqdb, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + // create database + pRes = taos_query(pConn, "create database tmqdb"); + if (taos_errno(pRes) != 0) { + printf("error in create tmqdb, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + // create super table + printf("create super table\n"); + pRes = taos_query( + pConn, "create table tmqdb.stb (ts timestamp, c1 int, c2 float, c3 varchar(16)) tags(t1 int, t3 varchar(16))"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table stb, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + // create sub tables + printf("create sub tables\n"); + pRes = taos_query(pConn, "create table tmqdb.ctb0 using tmqdb.stb tags(0, 'subtable0')"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table ctb0, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table tmqdb.ctb1 using tmqdb.stb tags(1, 'subtable1')"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table ctb1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table tmqdb.ctb2 using tmqdb.stb tags(2, 'subtable2')"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table ctb2, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table tmqdb.ctb3 using tmqdb.stb tags(3, 'subtable3')"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table ctb3, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + // insert data + printf("insert data into sub tables\n"); + pRes = taos_query(pConn, "insert into tmqdb.ctb0 values(now, 0, 0, 'a0')(now+1s, 0, 0, 'a00')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ctb0, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into tmqdb.ctb1 values(now, 1, 1, 'a1')(now+1s, 11, 11, 'a11')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ctb0, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into tmqdb.ctb2 values(now, 2, 2, 'a1')(now+1s, 22, 22, 'a22')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ctb0, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into tmqdb.ctb3 values(now, 3, 3, 'a1')(now+1s, 33, 33, 'a33')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ctb0, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + taos_close(pConn); + return 0; +} + +int32_t create_topic() { + printf("create topic\n"); + TAOS_RES* pRes; + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + if (pConn == NULL) { + return -1; + } + + pRes = taos_query(pConn, "use tmqdb"); + if (taos_errno(pRes) != 0) { + printf("error in use tmqdb, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create topic topicname as select ts, c1, c2, c3, tbname from tmqdb.stb where c1 > 1"); + if (taos_errno(pRes) != 0) { + printf("failed to create topic topicname, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + + taos_close(pConn); + return 0; +} + +void tmq_commit_cb_print(tmq_t* tmq, int32_t code, void* param) { + printf("tmq_commit_cb_print() code: %d, tmq: %p, param: %p\n", code, tmq, param); +} + +tmq_t* build_consumer() { + tmq_conf_res_t code; + tmq_conf_t* conf = tmq_conf_new(); + code = tmq_conf_set(conf, "enable.auto.commit", "true"); + if (TMQ_CONF_OK != code) return NULL; + code = tmq_conf_set(conf, "auto.commit.interval.ms", "1000"); + if (TMQ_CONF_OK != code) return NULL; + code = tmq_conf_set(conf, "group.id", "cgrpName"); + if (TMQ_CONF_OK != code) return NULL; + code = tmq_conf_set(conf, "client.id", "user defined name"); + if (TMQ_CONF_OK != code) return NULL; + code = tmq_conf_set(conf, "td.connect.user", "root"); + if (TMQ_CONF_OK != code) return NULL; + code = tmq_conf_set(conf, "td.connect.pass", "taosdata"); + if (TMQ_CONF_OK != code) return NULL; + code = tmq_conf_set(conf, "auto.offset.reset", "earliest"); + if (TMQ_CONF_OK != code) return NULL; + code = tmq_conf_set(conf, "experimental.snapshot.enable", "false"); + if (TMQ_CONF_OK != code) return NULL; + + tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL); + + tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); + tmq_conf_destroy(conf); + return tmq; +} + +tmq_list_t* build_topic_list() { + tmq_list_t* topicList = tmq_list_new(); + int32_t code = tmq_list_append(topicList, "topicname"); + if (code) { + return NULL; + } + return topicList; +} + +void basic_consume_loop(tmq_t* tmq) { + int32_t totalRows = 0; + int32_t msgCnt = 0; + int32_t timeout = 5000; + while (running) { + TAOS_RES* tmqmsg = tmq_consumer_poll(tmq, timeout); + if (tmqmsg) { + msgCnt++; + totalRows += msg_process(tmqmsg); + taos_free_result(tmqmsg); + } else { + break; + } + } + + fprintf(stderr, "%d msg consumed, include %d rows\n", msgCnt, totalRows); +} + +int main(int argc, char* argv[]) { + int32_t code; + + if (init_env() < 0) { + return -1; + } + + if (create_topic() < 0) { + return -1; + } + + tmq_t* tmq = build_consumer(); + if (NULL == tmq) { + fprintf(stderr, "%% build_consumer() fail!\n"); + return -1; + } + + tmq_list_t* topic_list = build_topic_list(); + if (NULL == topic_list) { + return -1; + } + + if ((code = tmq_subscribe(tmq, topic_list))) { + fprintf(stderr, "%% Failed to tmq_subscribe(): %s\n", tmq_err2str(code)); + } + tmq_list_destroy(topic_list); + + basic_consume_loop(tmq); + + code = tmq_consumer_close(tmq); + if (code) { + fprintf(stderr, "%% Failed to close consumer: %s\n", tmq_err2str(code)); + } else { + fprintf(stderr, "%% Consumer closed\n"); + } + + return 0; +} From 0bc12a8320157fe71872b61e468b31719d9a4f50 Mon Sep 17 00:00:00 2001 From: Huo Linhe Date: Wed, 17 Aug 2022 17:27:45 +0800 Subject: [PATCH 27/33] chore: fix source code file name typo --- docs/zh/07-develop/_sub_c.mdx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/07-develop/_sub_c.mdx b/docs/zh/07-develop/_sub_c.mdx index b8f73a8ff1..b0667268e9 100644 --- a/docs/zh/07-develop/_sub_c.mdx +++ b/docs/zh/07-develop/_sub_c.mdx @@ -1,3 +1,3 @@ ```c -{{#include docs/examples/c/tmq-example.c}} +{{#include docs/examples/c/tmq_example.c}} ``` From ebc79f906e182dd3a4b798dbc48153d648017c8b Mon Sep 17 00:00:00 2001 From: ShuaiQChang Date: Wed, 17 Aug 2022 17:39:40 +0800 Subject: [PATCH 28/33] fix: docs error --- docs/zh/07-develop/07-tmq.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/zh/07-develop/07-tmq.mdx b/docs/zh/07-develop/07-tmq.mdx index 25f37121e8..da8bf5e20e 100644 --- a/docs/zh/07-develop/07-tmq.mdx +++ b/docs/zh/07-develop/07-tmq.mdx @@ -724,7 +724,6 @@ consumer.close(); - ```go @@ -769,6 +768,7 @@ consumer.Unsubscribe(); // 关闭消费 consumer.Close(); ``` + @@ -809,7 +809,7 @@ SHOW SUBSCRIPTIONS; - + From 5e38aab3d7bb0dc389351e93897fdcde763d300e Mon Sep 17 00:00:00 2001 From: Minglei Jin Date: Wed, 17 Aug 2022 17:51:49 +0800 Subject: [PATCH 29/33] fix: plus 1 with tdb strlen --- source/dnode/vnode/src/meta/metaTable.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index e8755e0035..0e0b354cef 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -192,7 +192,7 @@ int metaCreateSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { // validate req void *pData = NULL; int nData = 0; - if (tdbTbGet(pMeta->pNameIdx, pReq->name, strlen(pReq->name), &pData, &nData) == 0) { + if (tdbTbGet(pMeta->pNameIdx, pReq->name, strlen(pReq->name) + 1, &pData, &nData) == 0) { tb_uid_t uid = *(tb_uid_t *)pData; tdbFree(pData); SMetaInfo info; From e3c11172ec7ccacefc2f643e249f7500726e691d Mon Sep 17 00:00:00 2001 From: wangmm0220 Date: Wed, 17 Aug 2022 19:01:22 +0800 Subject: [PATCH 30/33] fix:error in get table list by tag filter --- source/libs/executor/src/executil.c | 20 +++++++++++++++++++- source/libs/scalar/src/scalar.c | 3 +++ source/libs/scalar/src/sclvector.c | 4 ++-- 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index fc10551618..1dabea0d6b 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -334,7 +334,13 @@ static EDealRes getColumn(SNode** pNode, void* pContext) { taosHashPut(pData->colHash, &pSColumnNode->colId, sizeof(pSColumnNode->colId), pNode, sizeof((*pNode))); pSColumnNode->slotId = pData->index++; SColumnInfo cInfo = {.colId = pSColumnNode->colId, .type = pSColumnNode->node.resType.type, .bytes = pSColumnNode->node.resType.bytes}; +#if TAG_FILTER_DEBUG + qDebug("tagfilter build column info, slotId:%d, colId:%d, type:%d", pSColumnNode->slotId, cInfo.colId, cInfo.type); +#endif taosArrayPush(pData->cInfoList, &cInfo); + }else{ + SColumnNode* col = *(SColumnNode**)data; + pSColumnNode->slotId = col->slotId; } return DEAL_RES_CONTINUE; @@ -431,7 +437,9 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, uint64_t suid, SArray char str[TSDB_TABLE_FNAME_LEN + VARSTR_HEADER_SIZE] = {0}; metaGetTableNameByUid(metaHandle, *uid, str); colDataAppend(pColInfo, i, str, false); +#if TAG_FILTER_DEBUG qDebug("tagfilter uid:%ld, tbname:%s", *uid, str+2); +#endif }else{ STagVal tagVal = {0}; tagVal.cid = pColInfo->info.colId; @@ -442,13 +450,23 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, uint64_t suid, SArray } else if (pColInfo->info.type == TSDB_DATA_TYPE_JSON) { colDataAppend(pColInfo, i, p, false); } else if (IS_VAR_DATA_TYPE(pColInfo->info.type)) { - char *tmp = taosMemoryMalloc(tagVal.nData + VARSTR_HEADER_SIZE); + char *tmp = taosMemoryCalloc(tagVal.nData + VARSTR_HEADER_SIZE + 1, 1); varDataSetLen(tmp, tagVal.nData); memcpy(tmp + VARSTR_HEADER_SIZE, tagVal.pData, tagVal.nData); colDataAppend(pColInfo, i, tmp, false); +#if TAG_FILTER_DEBUG + qDebug("tagfilter varch:%s", tmp+2); +#endif taosMemoryFree(tmp); } else { colDataAppend(pColInfo, i, (const char*)&tagVal.i64, false); +#if TAG_FILTER_DEBUG + if(pColInfo->info.type == TSDB_DATA_TYPE_INT){ + qDebug("tagfilter int:%d", *(int*)(&tagVal.i64)); + }else if(pColInfo->info.type == TSDB_DATA_TYPE_DOUBLE){ + qDebug("tagfilter double:%f", *(double *)(&tagVal.i64)); + } +#endif } } } diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index d0c5a76f4b..54376561de 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -292,6 +292,9 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t } SColumnInfoData *columnData = (SColumnInfoData *)taosArrayGet(block->pDataBlock, ref->slotId); +#if TAG_FILTER_DEBUG + qDebug("tagfilter column info, slotId:%d, colId:%d, type:%d", ref->slotId, columnData->info.colId, columnData->info.type); +#endif param->numOfRows = block->info.rows; param->columnData = columnData; break; diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index 55699d5abd..aaa70ef5ae 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -1672,8 +1672,8 @@ void vectorBitOr(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, colDataAppendInt8(pOut->columnData, i, (int8_t*)&result);\ }else{\ bool res = filterDoCompare(fp, optr, pLeftData, pRightData);\ - colDataAppendInt8(pOut->columnData, i, (int8_t*)&res); \ - } \ + colDataAppendInt8(pOut->columnData, i, (int8_t*)&res);\ + }\ if(freeLeft) taosMemoryFreeClear(pLeftData);\ if(freeRight) taosMemoryFreeClear(pRightData);\ } From 853e6e29888afcf14ce4a743c9b82f1c19850bb9 Mon Sep 17 00:00:00 2001 From: Liu Jicong Date: Wed, 17 Aug 2022 19:19:58 +0800 Subject: [PATCH 31/33] refactor(mnode): drop stream task --- examples/c/stream_demo.c | 7 +++--- include/common/tcommon.h | 1 + include/libs/stream/tstream.h | 5 +++++ include/util/taoserror.h | 1 + source/dnode/mnode/impl/src/mndScheduler.c | 2 ++ source/dnode/mnode/impl/src/mndStb.c | 11 ++++++---- source/dnode/vnode/src/tq/tq.c | 14 +++--------- source/libs/stream/inc/streamInc.h | 1 - source/libs/stream/src/stream.c | 8 +++++-- source/libs/stream/src/streamMeta.c | 25 ++++++++++++++++------ source/libs/stream/src/streamQueue.c | 4 +++- source/libs/stream/src/streamTask.c | 4 ++-- source/util/src/terror.c | 1 + 13 files changed, 53 insertions(+), 31 deletions(-) diff --git a/examples/c/stream_demo.c b/examples/c/stream_demo.c index dd4fbc8d2d..2fcf4dd62c 100644 --- a/examples/c/stream_demo.c +++ b/examples/c/stream_demo.c @@ -98,10 +98,9 @@ int32_t create_stream() { /*const char* sql = "select min(k), max(k), sum(k) as sum_of_k from st1";*/ /*const char* sql = "select sum(k) from tu1 interval(10m)";*/ /*pRes = tmq_create_stream(pConn, "stream1", "out1", sql);*/ - pRes = - taos_query(pConn, - "create stream stream1 trigger max_delay 10s into outstb as select _wstart, sum(k) from st1 partition " - "by tbname session(ts, 10s) "); + pRes = taos_query(pConn, + "create stream stream1 trigger max_delay 10s watermark 10s into outstb as select _wstart start, " + "count(k) from st1 partition by tbname interval(20s) "); if (taos_errno(pRes) != 0) { printf("failed to create stream stream1, reason:%s\n", taos_errstr(pRes)); return -1; diff --git a/include/common/tcommon.h b/include/common/tcommon.h index e04d9d5e86..dbe020f7ec 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -60,6 +60,7 @@ enum { STREAM_INPUT__DATA_RETRIEVE, STREAM_INPUT__GET_RES, STREAM_INPUT__CHECKPOINT, + STREAM_INPUT__DESTROY, }; typedef enum EStreamType { diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index e6fcb021d5..484d0991f2 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -53,6 +53,7 @@ enum { TASK_SCHED_STATUS__WAITING, TASK_SCHED_STATUS__ACTIVE, TASK_SCHED_STATUS__FAILED, + TASK_SCHED_STATUS__DROPPING, }; enum { @@ -127,6 +128,10 @@ typedef struct { int8_t type; } SStreamCheckpoint; +typedef struct { + int8_t type; +} SStreamTaskDestroy; + typedef struct { int8_t type; SSDataBlock* pBlock; diff --git a/include/util/taoserror.h b/include/util/taoserror.h index d7ec3697af..c3796fbadd 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -291,6 +291,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_STREAM_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x03F1) #define TSDB_CODE_MND_INVALID_STREAM_OPTION TAOS_DEF_ERROR_CODE(0, 0x03F2) #define TSDB_CODE_MND_STREAM_MUST_BE_DELETED TAOS_DEF_ERROR_CODE(0, 0x03F3) +#define TSDB_CODE_MND_STREAM_TASK_DROPPED TAOS_DEF_ERROR_CODE(0, 0x03F4) // mnode-sma #define TSDB_CODE_MND_SMA_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0480) diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index a24b7ef459..3bfd7eb596 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -424,6 +424,8 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { } mndAddTaskToTaskSet(taskSourceLevel, pTask); + pTask->triggerParam = 0; + // source pTask->taskLevel = TASK_LEVEL__SOURCE; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index dd2b595c29..59c6d65953 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -2021,8 +2021,7 @@ static int32_t mndCheckDropStbForTopic(SMnode *pMnode, const char *stbFullName, FOREACH(pNode, pNodeList) { SColumnNode *pCol = (SColumnNode *)pNode; - if (pCol->tableId != suid) { - mDebug("topic:%s, check colId:%d passed", pTopic->name, pCol->colId); + if (pCol->tableId == suid) { sdbRelease(pSdb, pTopic); nodesDestroyNode(pAst); return -1; @@ -2045,6 +2044,11 @@ static int32_t mndCheckDropStbForStream(SMnode *pMnode, const char *stbFullName, pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream); if (pIter == NULL) break; + if (pStream->smaId == 0 && pStream->targetStbUid == suid) { + sdbRelease(pSdb, pStream); + return -1; + } + SNode *pAst = NULL; if (nodesStringToNode(pStream->ast, &pAst) != 0) { ASSERT(0); @@ -2057,8 +2061,7 @@ static int32_t mndCheckDropStbForStream(SMnode *pMnode, const char *stbFullName, FOREACH(pNode, pNodeList) { SColumnNode *pCol = (SColumnNode *)pNode; - if (pCol->tableId != suid) { - mDebug("stream:%s, check colId:%d passed", pStream->name, pCol->colId); + if (pCol->tableId == suid) { sdbRelease(pSdb, pStream); nodesDestroyNode(pAst); return -1; diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index a98fea1988..c6bc8e6e59 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -628,8 +628,6 @@ int32_t tqProcessVgChangeReq(STQ* pTq, int64_t version, char* msg, int32_t msgLe } int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask) { - int32_t code = 0; - if (pTask->taskLevel == TASK_LEVEL__AGG) { ASSERT(taosArrayGetSize(pTask->childEpInfo) != 0); } @@ -640,8 +638,7 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask) { pTask->outputQueue = streamQueueOpen(); if (pTask->inputQueue == NULL || pTask->outputQueue == NULL) { - code = -1; - goto FAIL; + return -1; } pTask->inputStatus = TASK_INPUT_STATUS__NORMAL; @@ -686,14 +683,9 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask) { streamSetupTrigger(pTask); - tqInfo("deploy stream task on vg %d, task id %d, child id %d", TD_VID(pTq->pVnode), pTask->taskId, + tqInfo("expand stream task on vg %d, task id %d, child id %d", TD_VID(pTq->pVnode), pTask->taskId, pTask->selfChildId); - -FAIL: - if (pTask->inputQueue) streamQueueClose(pTask->inputQueue); - if (pTask->outputQueue) streamQueueClose(pTask->outputQueue); - // TODO free executor - return code; + return 0; } int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen) { diff --git a/source/libs/stream/inc/streamInc.h b/source/libs/stream/inc/streamInc.h index 3776cb261f..6e30eeaa86 100644 --- a/source/libs/stream/inc/streamInc.h +++ b/source/libs/stream/inc/streamInc.h @@ -32,7 +32,6 @@ typedef struct { static SStreamGlobalEnv streamEnv; -int32_t streamExec(SStreamTask* pTask); int32_t streamPipelineExec(SStreamTask* pTask, int32_t batchNum, bool dispatch); int32_t streamDispatch(SStreamTask* pTask); diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 6da7d4fd59..d6e87c2736 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -185,7 +185,9 @@ int32_t streamProcessDispatchReq(SStreamTask* pTask, SStreamDispatchReq* pReq, S tFreeStreamDispatchReq(pReq); if (exec) { - streamTryExec(pTask); + if (streamTryExec(pTask) < 0) { + return -1; + } if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { streamDispatch(pTask); @@ -221,7 +223,9 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp) { } int32_t streamProcessRunReq(SStreamTask* pTask) { - streamTryExec(pTask); + if (streamTryExec(pTask) < 0) { + return -1; + } if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { streamDispatch(pTask); diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 64a9537e6c..f34f68ffc6 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -99,16 +99,19 @@ int32_t streamMetaAddSerializedTask(SStreamMeta* pMeta, int64_t startVer, char* goto FAIL; } - taosHashPut(pMeta->pTasks, &pTask->taskId, sizeof(int32_t), &pTask, sizeof(void*)); + if (taosHashPut(pMeta->pTasks, &pTask->taskId, sizeof(int32_t), &pTask, sizeof(void*)) < 0) { + goto FAIL; + } if (tdbTbUpsert(pMeta->pTaskDb, &pTask->taskId, sizeof(int32_t), msg, msgLen, &pMeta->txn) < 0) { + taosHashRemove(pMeta->pTasks, &pTask->taskId, sizeof(int32_t)); ASSERT(0); - return -1; + goto FAIL; } return 0; FAIL: - if (pTask) taosMemoryFree(pTask); + if (pTask) tFreeSStreamTask(pTask); return -1; } @@ -158,11 +161,21 @@ int32_t streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId) { SStreamTask* pTask = *ppTask; taosHashRemove(pMeta->pTasks, &taskId, sizeof(int32_t)); atomic_store_8(&pTask->taskStatus, TASK_STATUS__DROPPING); + + if (tdbTbDelete(pMeta->pTaskDb, &taskId, sizeof(int32_t), &pMeta->txn) < 0) { + /*return -1;*/ + } + + while (1) { + int8_t schedStatus = + atomic_val_compare_exchange_8(&pTask->schedStatus, TASK_SCHED_STATUS__INACTIVE, TASK_SCHED_STATUS__DROPPING); + if (schedStatus == TASK_SCHED_STATUS__INACTIVE) { + tFreeSStreamTask(pTask); + break; + } + } } - if (tdbTbDelete(pMeta->pTaskDb, &taskId, sizeof(int32_t), &pMeta->txn) < 0) { - /*return -1;*/ - } return 0; } diff --git a/source/libs/stream/src/streamQueue.c b/source/libs/stream/src/streamQueue.c index 6819e5329f..45b78a8c6e 100644 --- a/source/libs/stream/src/streamQueue.c +++ b/source/libs/stream/src/streamQueue.c @@ -38,7 +38,9 @@ void streamQueueClose(SStreamQueue* queue) { if (qItem) { taosFreeQitem(qItem); } else { - return; + break; } } + taosFreeQall(queue->qall); + taosCloseQueue(queue->queue); } diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index 638d39e5cc..0c35c1408e 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -152,8 +152,8 @@ int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask) { } void tFreeSStreamTask(SStreamTask* pTask) { - streamQueueClose(pTask->inputQueue); - streamQueueClose(pTask->outputQueue); + if (pTask->inputQueue) streamQueueClose(pTask->inputQueue); + if (pTask->outputQueue) streamQueueClose(pTask->outputQueue); if (pTask->exec.qmsg) taosMemoryFree(pTask->exec.qmsg); if (pTask->exec.executor) qDestroyTask(pTask->exec.executor); taosMemoryFree(pTask); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 7b06967940..f6b62b5ea8 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -293,6 +293,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_CGROUP_USED, "Consumer group being TAOS_DEFINE_ERROR(TSDB_CODE_MND_STREAM_ALREADY_EXIST, "Stream already exists") TAOS_DEFINE_ERROR(TSDB_CODE_MND_STREAM_NOT_EXIST, "Stream not exist") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_STREAM_OPTION, "Invalid stream option") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_STREAM_MUST_BE_DELETED, "Stream must be dropped first") // mnode-sma TAOS_DEFINE_ERROR(TSDB_CODE_MND_SMA_ALREADY_EXIST, "SMA already exists") From b604131eee73aa02f7db086c537d312cba63ae44 Mon Sep 17 00:00:00 2001 From: huolibo Date: Wed, 17 Aug 2022 22:30:02 +0800 Subject: [PATCH 32/33] docs: grafana plugin document (#16181) * docs: grafana for TDengine 3.0 * docs: add colon * docs: remove taosKeeper --- .../{15-taosKeeper.md => 14-taosKeeper.md} | 4 ++-- docs/zh/14-reference/14-taosx.md | 4 ---- docs/zh/20-third-party/01-grafana.mdx | 8 ++++++-- docs/zh/20-third-party/import_dashboard.webp | Bin 0 -> 6484 bytes 4 files changed, 8 insertions(+), 8 deletions(-) rename docs/zh/14-reference/{15-taosKeeper.md => 14-taosKeeper.md} (95%) delete mode 100644 docs/zh/14-reference/14-taosx.md create mode 100644 docs/zh/20-third-party/import_dashboard.webp diff --git a/docs/zh/14-reference/15-taosKeeper.md b/docs/zh/14-reference/14-taosKeeper.md similarity index 95% rename from docs/zh/14-reference/15-taosKeeper.md rename to docs/zh/14-reference/14-taosKeeper.md index d3f96bc5a9..f1165c9d0f 100644 --- a/docs/zh/14-reference/15-taosKeeper.md +++ b/docs/zh/14-reference/14-taosKeeper.md @@ -25,7 +25,7 @@ taosKeeper 安装方式: taosKeeper 需要在操作系统终端执行,该工具支持 [配置文件启动](#配置文件启动)。 -**在运行 taosKeeper 之前要确保 TDengine 集群与 taosAdapter 已经在正确运行。** +**在运行 taosKeeper 之前要确保 TDengine 集群与 taosAdapter 已经在正确运行。** 并且 TDengine 已经开启监控服务,具体请参考:[TDengine 监控配置](../config/#监控相关)。