From 4c13db3eec21fd563a3c2f2f6ac06d2bd0501a17 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 21 Oct 2020 18:01:00 +0800 Subject: [PATCH 01/56] [td-1770] --- src/client/src/tscSubquery.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 90490f07d8..de63465e0c 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -524,13 +524,7 @@ static void issueTSCompQuery(SSqlObj* pSql, SJoinSupporter* pSupporter, SSqlObj* tscProcessSql(pSql); } -static bool checkForDuplicateTagVal(SQueryInfo* pQueryInfo, SJoinSupporter* p1, SSqlObj* pPSqlObj) { - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - SSchema* pSchema = tscGetTableTagSchema(pTableMetaInfo->pTableMeta);// todo: tags mismatch, tags not completed - SColumn *pCol = taosArrayGetP(pTableMetaInfo->tagColList, 0); - SSchema *pColSchema = &pSchema[pCol->colIndex.columnIndex]; - +static bool checkForDuplicateTagVal(SSchema* pColSchema, SJoinSupporter* p1, SSqlObj* pPSqlObj) { for(int32_t i = 1; i < p1->num; ++i) { STidTags* prev = (STidTags*) varDataVal(p1->pIdTagList + (i - 1) * p1->tagSize); STidTags* p = (STidTags*) varDataVal(p1->pIdTagList + i * p1->tagSize); @@ -564,7 +558,7 @@ static int32_t getIntersectionOfTableTuple(SQueryInfo* pQueryInfo, SSqlObj* pPar *s1 = taosArrayInit(p1->num, p1->tagSize - sizeof(int16_t)); *s2 = taosArrayInit(p2->num, p2->tagSize - sizeof(int16_t)); - if (!(checkForDuplicateTagVal(pQueryInfo, p1, pParentSql) && checkForDuplicateTagVal(pQueryInfo, p2, pParentSql))) { + if (!(checkForDuplicateTagVal(pColSchema, p1, pParentSql) && checkForDuplicateTagVal(pColSchema, p2, pParentSql))) { return TSDB_CODE_QRY_DUP_JOIN_KEY; } From 8668c15f9892683f1df8d6a3d34197d8db616b78 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 22 Oct 2020 12:27:01 +0800 Subject: [PATCH 02/56] [td-1373] add log --- src/client/src/tscServer.c | 2 ++ src/client/src/tscSubquery.c | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 966eee6b01..257a5b2b54 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -486,9 +486,11 @@ int tscBuildFetchMsg(SSqlObj *pSql, SSqlInfo *pInfo) { assert(pVgroupInfo->vgroups[vgIndex].vgId > 0 && vgIndex < pTableMetaInfo->vgroupList->numOfVgroups); pRetrieveMsg->header.vgId = htonl(pVgroupInfo->vgroups[vgIndex].vgId); + tscDebug("%p build fetch msg from vgId:%d, vgIndex:%d", pSql, pVgroupInfo->vgroups[vgIndex].vgId, vgIndex); } else { STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; pRetrieveMsg->header.vgId = htonl(pTableMeta->vgroupInfo.vgId); + tscDebug("%p build fetch msg from only one vgroup, vgId:%d", pSql, pTableMeta->vgroupInfo.vgId); } pSql->cmd.payloadLen = sizeof(SRetrieveTableMsg); diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index de63465e0c..4ba35a98f7 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -474,10 +474,11 @@ static void issueTSCompQuery(SSqlObj* pSql, SJoinSupporter* pSupporter, SSqlObj* SSqlCmd* pCmd = &pSql->cmd; tscClearSubqueryInfo(pCmd); tscFreeSqlResult(pSql); - + SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(pCmd, 0); + assert(pQueryInfo->numOfTables == 1); + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - tscInitQueryInfo(pQueryInfo); TSDB_QUERY_CLEAR_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_TAG_FILTER_QUERY); From d1a84d027c6f93243b37fc2628f160346364b771 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 22 Oct 2020 12:42:23 +0800 Subject: [PATCH 03/56] [td-1373] --- src/client/src/tscServer.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 257a5b2b54..545440b6ea 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -486,7 +486,8 @@ int tscBuildFetchMsg(SSqlObj *pSql, SSqlInfo *pInfo) { assert(pVgroupInfo->vgroups[vgIndex].vgId > 0 && vgIndex < pTableMetaInfo->vgroupList->numOfVgroups); pRetrieveMsg->header.vgId = htonl(pVgroupInfo->vgroups[vgIndex].vgId); - tscDebug("%p build fetch msg from vgId:%d, vgIndex:%d", pSql, pVgroupInfo->vgroups[vgIndex].vgId, vgIndex); + tscDebug("%p build fetch msg from vgId:%d, vgIndex:%d, %d, %d", pSql, pVgroupInfo->vgroups[vgIndex].vgId, vgIndex, + pVgroupInfo->vgroups[0].vgId, pVgroupInfo->vgroups[1].vgId); } else { STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; pRetrieveMsg->header.vgId = htonl(pTableMeta->vgroupInfo.vgId); @@ -563,7 +564,9 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char assert(index < pTableMetaInfo->vgroupList->numOfVgroups); pVgroupInfo = &pTableMetaInfo->vgroupList->vgroups[index]; } - tscDebug("%p query on stable, vgIndex:%d, numOfVgroups:%d", pSql, index, pTableMetaInfo->vgroupList->numOfVgroups); + tscDebug("%p query on stable, vgIndex:%d, numOfVgroups:%d, %d, %d", pSql, index, + pTableMetaInfo->vgroupList->numOfVgroups, pTableMetaInfo->vgroupList->vgroups[0].vgId, + pTableMetaInfo->vgroupList->vgroups[1].vgId); } else { pVgroupInfo = &pTableMeta->vgroupInfo; } @@ -608,8 +611,8 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char } } - tscDebug("%p vgId:%d, query on table:%s, tid:%d, uid:%" PRIu64, pSql, htonl(pQueryMsg->head.vgId), pTableMetaInfo->name, - pTableMeta->id.tid, pTableMeta->id.uid); + tscDebug("%p vgId:%d, query on table:%s, tid:%d, uid:%" PRIu64", pTableMeta:%p", pSql, htonl(pQueryMsg->head.vgId), pTableMetaInfo->name, + pTableMeta->id.tid, pTableMeta->id.uid, pTableMeta); return pMsg; } From 8fb14fc5cbc485da20c6a7818af60dd23cf9e189 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 22 Oct 2020 14:08:11 +0800 Subject: [PATCH 04/56] [td-1373] --- src/client/src/tscServer.c | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 545440b6ea..cb044b453f 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -481,13 +481,21 @@ int tscBuildFetchMsg(SSqlObj *pSql, SSqlInfo *pInfo) { if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { int32_t vgIndex = pTableMetaInfo->vgroupIndex; - - SVgroupsInfo* pVgroupInfo = pTableMetaInfo->vgroupList; - assert(pVgroupInfo->vgroups[vgIndex].vgId > 0 && vgIndex < pTableMetaInfo->vgroupList->numOfVgroups); + if (pTableMetaInfo->pVgroupTables == NULL) { + SVgroupsInfo *pVgroupInfo = pTableMetaInfo->vgroupList; + assert(pVgroupInfo->vgroups[vgIndex].vgId > 0 && vgIndex < pTableMetaInfo->vgroupList->numOfVgroups); - pRetrieveMsg->header.vgId = htonl(pVgroupInfo->vgroups[vgIndex].vgId); - tscDebug("%p build fetch msg from vgId:%d, vgIndex:%d, %d, %d", pSql, pVgroupInfo->vgroups[vgIndex].vgId, vgIndex, - pVgroupInfo->vgroups[0].vgId, pVgroupInfo->vgroups[1].vgId); + pRetrieveMsg->header.vgId = htonl(pVgroupInfo->vgroups[vgIndex].vgId); + tscDebug("%p build fetch msg from vgId:%d, vgIndex:%d", pSql, pVgroupInfo->vgroups[vgIndex].vgId, vgIndex); + } else { + int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables); + assert(vgIndex >= 0 && vgIndex < numOfVgroups); + + SVgroupTableInfo* pTableIdList = taosArrayGet(pTableMetaInfo->pVgroupTables, vgIndex); + + pRetrieveMsg->header.vgId = htonl(pTableIdList->vgInfo.vgId); + tscDebug("%p build fetch msg from vgId:%d, vgIndex:%d", pSql, pTableIdList->vgInfo.vgId, vgIndex); + } } else { STableMeta* pTableMeta = pTableMetaInfo->pTableMeta; pRetrieveMsg->header.vgId = htonl(pTableMeta->vgroupInfo.vgId); @@ -564,9 +572,7 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char assert(index < pTableMetaInfo->vgroupList->numOfVgroups); pVgroupInfo = &pTableMetaInfo->vgroupList->vgroups[index]; } - tscDebug("%p query on stable, vgIndex:%d, numOfVgroups:%d, %d, %d", pSql, index, - pTableMetaInfo->vgroupList->numOfVgroups, pTableMetaInfo->vgroupList->vgroups[0].vgId, - pTableMetaInfo->vgroupList->vgroups[1].vgId); + tscDebug("%p query on stable, vgIndex:%d, numOfVgroups:%d", pSql, index, pTableMetaInfo->vgroupList->numOfVgroups); } else { pVgroupInfo = &pTableMeta->vgroupInfo; } @@ -611,8 +617,8 @@ static char *doSerializeTableInfo(SQueryTableMsg* pQueryMsg, SSqlObj *pSql, char } } - tscDebug("%p vgId:%d, query on table:%s, tid:%d, uid:%" PRIu64", pTableMeta:%p", pSql, htonl(pQueryMsg->head.vgId), pTableMetaInfo->name, - pTableMeta->id.tid, pTableMeta->id.uid, pTableMeta); + tscDebug("%p vgId:%d, query on table:%s, tid:%d, uid:%" PRIu64, pSql, htonl(pQueryMsg->head.vgId), pTableMetaInfo->name, + pTableMeta->id.tid, pTableMeta->id.uid); return pMsg; } @@ -667,12 +673,12 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->limit = htobe64(pQueryInfo->limit.limit); pQueryMsg->offset = htobe64(pQueryInfo->limit.offset); pQueryMsg->numOfCols = htons((int16_t)taosArrayGetSize(pQueryInfo->colList)); - pQueryMsg->interval.interval = htobe64(pQueryInfo->interval.interval); - pQueryMsg->interval.sliding = htobe64(pQueryInfo->interval.sliding); + pQueryMsg->interval.interval = htobe64(pQueryInfo->interval.interval); + pQueryMsg->interval.sliding = htobe64(pQueryInfo->interval.sliding); pQueryMsg->interval.offset = htobe64(pQueryInfo->interval.offset); pQueryMsg->interval.intervalUnit = pQueryInfo->interval.intervalUnit; - pQueryMsg->interval.slidingUnit = pQueryInfo->interval.slidingUnit; - pQueryMsg->interval.offsetUnit = pQueryInfo->interval.offsetUnit; + pQueryMsg->interval.slidingUnit = pQueryInfo->interval.slidingUnit; + pQueryMsg->interval.offsetUnit = pQueryInfo->interval.offsetUnit; pQueryMsg->numOfGroupCols = htons(pQueryInfo->groupbyExpr.numOfGroupCols); pQueryMsg->numOfTags = htonl(numOfTags); pQueryMsg->tagNameRelType = htons(pQueryInfo->tagCond.relType); From de006209f92050382e65d9d7ba8d539089240f9a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 22 Oct 2020 14:32:31 +0800 Subject: [PATCH 05/56] [td-1373] --- src/client/src/tscSubquery.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 4ba35a98f7..2a684f8608 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -863,7 +863,14 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR assert(pQueryInfo->numOfTables == 1); // for projection query, need to try next vnode if current vnode is exhausted - if ((++pTableMetaInfo->vgroupIndex) < pTableMetaInfo->vgroupList->numOfVgroups) { + int32_t numOfVgroups = 0; // TODO refactor + if (pTableMetaInfo->pVgroupTables != NULL) { + numOfVgroups = taosArrayGetSize(pTableMetaInfo->pVgroupTables); + } else { + numOfVgroups = pTableMetaInfo->vgroupList->numOfVgroups; + } + + if ((++pTableMetaInfo->vgroupIndex) < numOfVgroups) { pState->numOfRemain = 1; pState->numOfSub = 1; From 056967250312324dd3d25796c225a93aeb04e0c7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 22 Oct 2020 14:40:17 +0800 Subject: [PATCH 06/56] [td-1373] --- src/client/src/tscServer.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index cb044b453f..ebd6b7920c 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -861,7 +861,8 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { int32_t numOfBlocks = 0; if (pQueryInfo->tsBuf != NULL) { - STSVnodeBlockInfo *pBlockInfo = tsBufGetVnodeBlockInfo(pQueryInfo->tsBuf, pTableMetaInfo->vgroupIndex); + int32_t vnodeId = htonl(pQueryMsg->head.vgId); + STSVnodeBlockInfo *pBlockInfo = tsBufGetVnodeBlockInfo(pQueryInfo->tsBuf, vnodeId); assert(QUERY_IS_JOIN_QUERY(pQueryInfo->type) && pBlockInfo != NULL); // this query should not be sent // todo refactor From 0f2c6f31810792e25be3b2c3ff90634f27f06e8d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 22 Oct 2020 15:49:15 +0800 Subject: [PATCH 07/56] [td-1373] --- src/client/src/tscFunctionImpl.c | 4 +-- src/client/src/tscSubquery.c | 6 ++-- src/query/inc/qTsbuf.h | 4 +++ src/query/src/qExecutor.c | 8 +++-- src/query/src/qTsbuf.c | 52 ++++++++++++++++++++++++-------- 5 files changed, 54 insertions(+), 20 deletions(-) diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 1b4f92d3fc..81893841f7 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -4025,11 +4025,11 @@ static void ts_comp_function(SQLFunctionCtx *pCtx) { // primary ts must be existed, so no need to check its existance if (pCtx->order == TSDB_ORDER_ASC) { - tsBufAppend(pTSbuf, 0, &pCtx->tag, input, pCtx->size * TSDB_KEYSIZE); + tsBufAppend(pTSbuf, pCtx->param[0].i64Key, &pCtx->tag, input, pCtx->size * TSDB_KEYSIZE); } else { for (int32_t i = pCtx->size - 1; i >= 0; --i) { char *d = GET_INPUT_CHAR_INDEX(pCtx, i); - tsBufAppend(pTSbuf, 0, &pCtx->tag, d, TSDB_KEYSIZE); + tsBufAppend(pTSbuf, pCtx->param[0].i64Key, &pCtx->tag, d, TSDB_KEYSIZE); } } diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 2a684f8608..ca778d4f92 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -158,9 +158,9 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ tsBufDestroy(pSupporter1->pTSBuf); tsBufDestroy(pSupporter2->pTSBuf); - tscDebug("%p input1:%" PRId64 ", input2:%" PRId64 ", final:%" PRId64 " for secondary query after ts blocks " - "intersecting, skey:%" PRId64 ", ekey:%" PRId64, pSql, numOfInput1, numOfInput2, output1->numOfTotal, - win->skey, win->ekey); + tscDebug("%p input1:%" PRId64 ", input2:%" PRId64 ", final:%" PRId64 " in %d vnodes for secondary query after ts blocks " + "intersecting, skey:%" PRId64 ", ekey:%" PRId64 ", numOfVnode:%d", pSql, numOfInput1, numOfInput2, output1->numOfTotal, + output1->numOfVnodes, win->skey, win->ekey, tsBufGetNumOfVnodes(output1)); return output1->numOfTotal; } diff --git a/src/query/inc/qTsbuf.h b/src/query/inc/qTsbuf.h index 46e6f79014..be3d97dad7 100644 --- a/src/query/inc/qTsbuf.h +++ b/src/query/inc/qTsbuf.h @@ -136,6 +136,10 @@ void tsBufSetCursor(STSBuf* pTSBuf, STSCursor* pCur); */ void tsBufDisplay(STSBuf* pTSBuf); +int32_t tsBufGetNumOfVnodes(STSBuf* pTSBuf); + +void tsBufGetVnodeIdList(STSBuf* pTSBuf, int32_t* num, int32_t** vnodeId); + #ifdef __cplusplus } #endif diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 3cb8db9faf..224ecfe08c 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -1005,9 +1005,10 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } + SQInfo* pQInfo = GET_QINFO_ADDR(pQuery); for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { char *dataBlock = getDataBlock(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->rows, pDataBlock); - setExecParams(pQuery, &pCtx[k], dataBlock, tsCols, pDataBlockInfo, pStatis, &sasArray[k], k); + setExecParams(pQuery, &pCtx[k], dataBlock, tsCols, pDataBlockInfo, pStatis, &sasArray[k], k, pQInfo->vgId); } int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order); @@ -1469,7 +1470,7 @@ static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBl } void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY *tsCol, SDataBlockInfo* pBlockInfo, - SDataStatis *pStatis, void *param, int32_t colIndex) { + SDataStatis *pStatis, void *param, int32_t colIndex, int32_t vgId) { int32_t functionId = pQuery->pSelectExpr[colIndex].base.functionId; int32_t colId = pQuery->pSelectExpr[colIndex].base.colInfo.colId; @@ -1542,6 +1543,9 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY } } } + } else if (functionId == TSDB_FUNC_TS_COMP) { + pCtx->param[0].i64Key = vgId; + pCtx->param[0].nType = TSDB_DATA_TYPE_BIGINT; } #if defined(_DEBUG_VIEW) diff --git a/src/query/src/qTsbuf.c b/src/query/src/qTsbuf.c index b264f6cdc9..30cf070c0a 100644 --- a/src/query/src/qTsbuf.c +++ b/src/query/src/qTsbuf.c @@ -561,6 +561,19 @@ static void tsBufGetBlock(STSBuf* pTSBuf, int32_t vnodeIndex, int32_t blockIndex pCur->tsIndex = (pCur->order == TSDB_ORDER_ASC) ? 0 : pBlock->numOfElem - 1; } +static int32_t doUpdateVnodeInfo(STSBuf* pTSBuf, int64_t offset, STSVnodeBlockInfo* pVInfo) { + if (offset < 0 || offset >= getDataStartOffset()) { + return -1; + } + + if (fseek(pTSBuf->f, (int32_t)offset, SEEK_SET) != 0) { + return -1; + } + + fwrite(pVInfo, sizeof(STSVnodeBlockInfo), 1, pTSBuf->f); + return 0; +} + STSVnodeBlockInfo* tsBufGetVnodeBlockInfo(STSBuf* pTSBuf, int32_t vnodeId) { int32_t j = tsBufFindVnodeIndexFromId(pTSBuf->pData, pTSBuf->numOfVnodes, vnodeId); if (j == -1) { @@ -915,19 +928,6 @@ static int32_t getDataStartOffset() { return sizeof(STSBufFileHeader) + TS_COMP_FILE_VNODE_MAX * sizeof(STSVnodeBlockInfo); } -static int32_t doUpdateVnodeInfo(STSBuf* pTSBuf, int64_t offset, STSVnodeBlockInfo* pVInfo) { - if (offset < 0 || offset >= getDataStartOffset()) { - return -1; - } - - if (fseek(pTSBuf->f, (int32_t)offset, SEEK_SET) != 0) { - return -1; - } - - fwrite(pVInfo, sizeof(STSVnodeBlockInfo), 1, pTSBuf->f); - return 0; -} - // update prev vnode length info in file static void TSBufUpdateVnodeInfo(STSBuf* pTSBuf, int32_t index, STSVnodeBlockInfo* pBlockInfo) { int32_t offset = sizeof(STSBufFileHeader) + index * sizeof(STSVnodeBlockInfo); @@ -969,3 +969,29 @@ static STSBuf* allocResForTSBuf(STSBuf* pTSBuf) { pTSBuf->fileSize += getDataStartOffset(); return pTSBuf; } + +int32_t tsBufGetNumOfVnodes(STSBuf* pTSBuf) { + if (pTSBuf == NULL) { + return 0; + } + + return pTSBuf->pData->len; +} + +void tsBufGetVnodeIdList(STSBuf* pTSBuf, int32_t* num, int32_t** vnodeId) { + int32_t size = tsBugGetNumOfVnodes(pTSBuf); + if (num != NULL) { + *num = size; + } + + *vnodeId = NULL; + if (size == 0) { + return; + } + + (*vnodeId) = malloc(tsBugGetNumOfVnodes(pTSBuf) * sizeof(int32_t)); + + for(int32_t i = 0; i < size; ++i) { + (*vnodeId)[i] = pTSBuf->pData[i].info.vnode; + } +} \ No newline at end of file From d3522fecf4be6a1147ce5317fd28891df81584ae Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 22 Oct 2020 15:52:08 +0800 Subject: [PATCH 08/56] [td-1373] --- src/query/src/qExecutor.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 224ecfe08c..331c8faa53 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -184,7 +184,7 @@ static void resetMergeResultBuf(SQuery *pQuery, SQLFunctionCtx *pCtx, SResultInf static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t functionId); static void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY *tsCol, SDataBlockInfo* pBlockInfo, - SDataStatis *pStatis, void *param, int32_t colIndex); + SDataStatis *pStatis, void *param, int32_t colIndex, int32_t vgId); static void initCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv); static void destroyTableQueryInfoImpl(STableQueryInfo *pTableQueryInfo); @@ -1005,7 +1005,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } - SQInfo* pQInfo = GET_QINFO_ADDR(pQuery); + SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv); for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { char *dataBlock = getDataBlock(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->rows, pDataBlock); setExecParams(pQuery, &pCtx[k], dataBlock, tsCols, pDataBlockInfo, pStatis, &sasArray[k], k, pQInfo->vgId); @@ -1287,9 +1287,10 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS groupbyColumnData = getGroupbyColumnData(pQuery, &type, &bytes, pDataBlock); } + SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv); for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { char *dataBlock = getDataBlock(pRuntimeEnv, &sasArray[k], k, pDataBlockInfo->rows, pDataBlock); - setExecParams(pQuery, &pCtx[k], dataBlock, tsCols, pDataBlockInfo, pStatis, &sasArray[k], k); + setExecParams(pQuery, &pCtx[k], dataBlock, tsCols, pDataBlockInfo, pStatis, &sasArray[k], k, pQInfo->vgId); } // set the input column data From ad499c5541c497539c26c16f473e1e4fe9a32b62 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 22 Oct 2020 15:53:25 +0800 Subject: [PATCH 09/56] [td-1373] --- src/query/src/qTsbuf.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/query/src/qTsbuf.c b/src/query/src/qTsbuf.c index 30cf070c0a..95fd88f1d8 100644 --- a/src/query/src/qTsbuf.c +++ b/src/query/src/qTsbuf.c @@ -979,7 +979,7 @@ int32_t tsBufGetNumOfVnodes(STSBuf* pTSBuf) { } void tsBufGetVnodeIdList(STSBuf* pTSBuf, int32_t* num, int32_t** vnodeId) { - int32_t size = tsBugGetNumOfVnodes(pTSBuf); + int32_t size = tsBufGetNumOfVnodes(pTSBuf); if (num != NULL) { *num = size; } @@ -989,7 +989,7 @@ void tsBufGetVnodeIdList(STSBuf* pTSBuf, int32_t* num, int32_t** vnodeId) { return; } - (*vnodeId) = malloc(tsBugGetNumOfVnodes(pTSBuf) * sizeof(int32_t)); + (*vnodeId) = malloc(tsBufGetNumOfVnodes(pTSBuf) * sizeof(int32_t)); for(int32_t i = 0; i < size; ++i) { (*vnodeId)[i] = pTSBuf->pData[i].info.vnode; From e06d7083c1564ead1411f03ea6b4e83bc691a3e3 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 22 Oct 2020 16:45:17 +0800 Subject: [PATCH 10/56] [td-1373] --- src/client/src/tscSubquery.c | 4 +--- src/query/inc/qTsbuf.h | 2 +- src/query/src/qTsbuf.c | 9 ++++----- src/query/tests/tsBufTest.cpp | 10 ++++------ 4 files changed, 10 insertions(+), 15 deletions(-) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index ca778d4f92..e5d7f03007 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -761,9 +761,7 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow pSupporter->pTSBuf = pBuf; } else { assert(pQueryInfo->numOfTables == 1); // for subquery, only one - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); - - tsBufMerge(pSupporter->pTSBuf, pBuf, pTableMetaInfo->vgroupIndex); + tsBufMerge(pSupporter->pTSBuf, pBuf); tsBufDestroy(pBuf); } diff --git a/src/query/inc/qTsbuf.h b/src/query/inc/qTsbuf.h index be3d97dad7..8bd775dac5 100644 --- a/src/query/inc/qTsbuf.h +++ b/src/query/inc/qTsbuf.h @@ -111,7 +111,7 @@ STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_ void* tsBufDestroy(STSBuf* pTSBuf); void tsBufAppend(STSBuf* pTSBuf, int32_t vnodeId, tVariant* tag, const char* pData, int32_t len); -int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf, int32_t vnodeIdx); +int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf); STSBuf* tsBufClone(STSBuf* pTSBuf); diff --git a/src/query/src/qTsbuf.c b/src/query/src/qTsbuf.c index 95fd88f1d8..7c2f8ebb5e 100644 --- a/src/query/src/qTsbuf.c +++ b/src/query/src/qTsbuf.c @@ -715,7 +715,7 @@ STSElem tsBufGetElem(STSBuf* pTSBuf) { * @param vnodeId * @return */ -int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf, int32_t vnodeId) { +int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) { if (pDestBuf == NULL || pSrcBuf == NULL || pSrcBuf->numOfVnodes <= 0) { return 0; } @@ -725,14 +725,13 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf, int32_t vnodeId) { } // src can only have one vnode index - if (pSrcBuf->numOfVnodes > 1) { - return -1; - } - + assert(pSrcBuf->numOfVnodes == 1); + // there are data in buffer, flush to disk first tsBufFlush(pDestBuf); // compared with the last vnode id + int32_t vnodeId = tsBufGetLastVnodeInfo((STSBuf*) pSrcBuf)->info.vnode; if (vnodeId != tsBufGetLastVnodeInfo(pDestBuf)->info.vnode) { int32_t oldSize = pDestBuf->numOfVnodes; int32_t newSize = oldSize + pSrcBuf->numOfVnodes; diff --git a/src/query/tests/tsBufTest.cpp b/src/query/tests/tsBufTest.cpp index b78c5314f2..f22f61e890 100644 --- a/src/query/tests/tsBufTest.cpp +++ b/src/query/tests/tsBufTest.cpp @@ -416,8 +416,8 @@ void mergeDiffVnodeBufferTest() { int64_t* list = createTsList(num, start, step); t.i64Key = i; - tsBufAppend(pTSBuf1, 0, &t, (const char*)list, num * sizeof(int64_t)); - tsBufAppend(pTSBuf2, 0, &t, (const char*)list, num * sizeof(int64_t)); + tsBufAppend(pTSBuf1, 1, &t, (const char*)list, num * sizeof(int64_t)); + tsBufAppend(pTSBuf2, 9, &t, (const char*)list, num * sizeof(int64_t)); free(list); @@ -426,7 +426,7 @@ void mergeDiffVnodeBufferTest() { tsBufFlush(pTSBuf2); - tsBufMerge(pTSBuf1, pTSBuf2, 9); + tsBufMerge(pTSBuf1, pTSBuf2); EXPECT_EQ(pTSBuf1->numOfVnodes, 2); EXPECT_EQ(pTSBuf1->numOfTotal, numOfTags * 2 * num); @@ -459,8 +459,6 @@ void mergeIdenticalVnodeBufferTest() { start += step * num; } - - for (int32_t i = numOfTags; i < numOfTags * 2; ++i) { int64_t* list = createTsList(num, start, step); @@ -473,7 +471,7 @@ void mergeIdenticalVnodeBufferTest() { tsBufFlush(pTSBuf2); - tsBufMerge(pTSBuf1, pTSBuf2, 12); + tsBufMerge(pTSBuf1, pTSBuf2); EXPECT_EQ(pTSBuf1->numOfVnodes, 1); EXPECT_EQ(pTSBuf1->numOfTotal, numOfTags * 2 * num); From b90807cec9954da09a90a492bc41b1961eae2755 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 22 Oct 2020 17:33:43 +0800 Subject: [PATCH 11/56] [td-1373] --- src/client/inc/tscUtil.h | 4 +- src/client/src/tscServer.c | 2 +- src/client/src/tscSubquery.c | 27 ++++++++++++ src/client/src/tscUtil.c | 83 ++++++++++++++++++++++++++++-------- 4 files changed, 97 insertions(+), 19 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 3df493349e..42096f058a 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -215,7 +215,7 @@ SQueryInfo *tscGetQueryInfoDetailSafely(SSqlCmd *pCmd, int32_t subClauseIndex); void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache); STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, STableMeta* pTableMeta, - SVgroupsInfo* vgroupList, SArray* pTagCols); + SVgroupsInfo* vgroupList, SArray* pTagCols, SArray* pVgroupTables); STableMetaInfo* tscAddEmptyMetaInfo(SQueryInfo *pQueryInfo); int32_t tscAddSubqueryInfo(SSqlCmd *pCmd); @@ -224,6 +224,8 @@ void tscInitQueryInfo(SQueryInfo* pQueryInfo); void tscClearSubqueryInfo(SSqlCmd* pCmd); void tscFreeVgroupTableInfo(SArray* pVgroupTables); +SArray* tscCloneVgroupTableInfo(SArray* pVgroupTables); +void tscRemoveVgroupTableGroup(SArray* pVgroupTable, int32_t index); int tscGetSTableVgroupInfo(SSqlObj* pSql, int32_t clauseIndex); int tscGetTableMeta(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo); diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index ebd6b7920c..55af431a0f 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -2283,7 +2283,7 @@ int tscGetSTableVgroupInfo(SSqlObj *pSql, int32_t clauseIndex) { for (int32_t i = 0; i < pQueryInfo->numOfTables; ++i) { STableMetaInfo *pMInfo = tscGetMetaInfo(pQueryInfo, i); STableMeta *pTableMeta = taosCacheAcquireByData(tscMetaCache, pMInfo->pTableMeta); - tscAddTableMetaInfo(pNewQueryInfo, pMInfo->name, pTableMeta, NULL, pMInfo->tagColList); + tscAddTableMetaInfo(pNewQueryInfo, pMInfo->name, pTableMeta, NULL, pMInfo->tagColList, pMInfo->pVgroupTables); } if ((code = tscAllocPayload(&pNew->cmd, TSDB_DEFAULT_PAYLOAD_SIZE)) != TSDB_CODE_SUCCESS) { diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index e5d7f03007..3ac480ae03 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -360,6 +360,31 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { pExpr->numOfParams = 1; } + int32_t num = 0; + int32_t *list = NULL; + tsBufGetVnodeIdList(pSupporter->pTSBuf, &num, &list); + + for(int32_t k = 0; k < taosArrayGetSize(pTableMetaInfo->pVgroupTables);) { + SVgroupTableInfo* p = taosArrayGet(pTableMetaInfo->pVgroupTables, k); + + bool found = false; + for(int32_t f = 0; f < num; ++f) { + if (p->vgInfo.vgId == list[f]) { + found = true; + break; + } + } + + if (!found) { + tscRemoveVgroupTableGroup(pTableMetaInfo->pVgroupTables, k); + } else { + k++; + } + } + + taosTFree(list); + assert(taosArrayGetSize(pTableMetaInfo->pVgroupTables) > 0); + size_t numOfCols = taosArrayGetSize(pNewQueryInfo->colList); tscDebug("%p subquery:%p tableIndex:%d, vgroupIndex:%d, type:%d, exprInfo:%" PRIzu ", colList:%" PRIzu ", fieldsInfo:%d, name:%s", pSql, pNew, 0, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, taosArrayGetSize(pNewQueryInfo->exprList), @@ -828,6 +853,8 @@ static void tsCompRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow // launch the query the retrieve actual results from vnode along with the filtered timestamp SQueryInfo* pPQueryInfo = tscGetQueryInfoDetail(&pParentSql->cmd, pParentSql->cmd.clauseIndex); updateQueryTimeRange(pPQueryInfo, &win); + + //update the vgroup that involved in real data query tscLaunchRealSubqueries(pParentSql); } diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 7e1c378584..eb3ad31235 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1680,19 +1680,64 @@ void tscClearSubqueryInfo(SSqlCmd* pCmd) { } void tscFreeVgroupTableInfo(SArray* pVgroupTables) { - if (pVgroupTables != NULL) { - size_t num = taosArrayGetSize(pVgroupTables); - for (size_t i = 0; i < num; i++) { - SVgroupTableInfo* pInfo = taosArrayGet(pVgroupTables, i); - - for(int32_t j = 0; j < pInfo->vgInfo.numOfEps; ++j) { - taosTFree(pInfo->vgInfo.epAddr[j].fqdn); - } - - taosArrayDestroy(pInfo->itemList); - } - taosArrayDestroy(pVgroupTables); + if (pVgroupTables == NULL) { + return; } + + size_t num = taosArrayGetSize(pVgroupTables); + for (size_t i = 0; i < num; i++) { + SVgroupTableInfo* pInfo = taosArrayGet(pVgroupTables, i); + + for(int32_t j = 0; j < pInfo->vgInfo.numOfEps; ++j) { + taosTFree(pInfo->vgInfo.epAddr[j].fqdn); + } + + taosArrayDestroy(pInfo->itemList); + } + + taosArrayDestroy(pVgroupTables); +} + +void tscRemoveVgroupTableGroup(SArray* pVgroupTable, int32_t index) { + assert(pVgroupTable != NULL && index >= 0); + + size_t size = taosArrayGetSize(pVgroupTable); + assert(size > index); + + SVgroupTableInfo* pInfo = taosArrayGet(pVgroupTable, index); + for(int32_t j = 0; j < pInfo->vgInfo.numOfEps; ++j) { + taosTFree(pInfo->vgInfo.epAddr[j].fqdn); + } + + taosArrayDestroy(pInfo->itemList); + + + taosArrayRemove(pVgroupTable, index); +} + +SArray* tscCloneVgroupTableInfo(SArray* pVgroupTables) { + if (pVgroupTables == NULL) { + return NULL; + } + + size_t num = taosArrayGetSize(pVgroupTables); + SArray* pa = taosArrayInit(num, sizeof(SVgroupTableInfo)); + + for (size_t i = 0; i < num; i++) { + SVgroupTableInfo* pInfo = taosArrayGet(pVgroupTables, i); + + for(int32_t j = 0; j < pInfo->vgInfo.numOfEps; ++j) { + taosTFree(pInfo->vgInfo.epAddr[j].fqdn); + } + + SVgroupTableInfo info = {0}; + info.vgInfo = pInfo->vgInfo; + info.itemList = taosArrayClone(pInfo->itemList); + + taosArrayPush(pa, &info); + } + + return pa; } void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool removeFromCache) { @@ -1710,7 +1755,7 @@ void clearAllTableMetaInfo(SQueryInfo* pQueryInfo, const char* address, bool rem } STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, STableMeta* pTableMeta, - SVgroupsInfo* vgroupList, SArray* pTagCols) { + SVgroupsInfo* vgroupList, SArray* pTagCols, SArray* pVgroupTables) { void* pAlloc = realloc(pQueryInfo->pTableMetaInfo, (pQueryInfo->numOfTables + 1) * POINTER_BYTES); if (pAlloc == NULL) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -1744,13 +1789,15 @@ STableMetaInfo* tscAddTableMetaInfo(SQueryInfo* pQueryInfo, const char* name, ST if (pTagCols != NULL) { tscColumnListCopy(pTableMetaInfo->tagColList, pTagCols, -1); } + + pTableMetaInfo->pVgroupTables = tscCloneVgroupTableInfo(pVgroupTables); pQueryInfo->numOfTables += 1; return pTableMetaInfo; } STableMetaInfo* tscAddEmptyMetaInfo(SQueryInfo* pQueryInfo) { - return tscAddTableMetaInfo(pQueryInfo, NULL, NULL, NULL, NULL); + return tscAddTableMetaInfo(pQueryInfo, NULL, NULL, NULL, NULL, NULL); } void tscClearTableMetaInfo(STableMetaInfo* pTableMetaInfo, bool removeFromCache) { @@ -1824,7 +1871,7 @@ SSqlObj* createSimpleSubObj(SSqlObj* pSql, void (*fp)(), void* param, int32_t cm assert(pSql->cmd.clauseIndex == 0); STableMetaInfo* pMasterTableMetaInfo = tscGetTableMetaInfoFromCmd(&pSql->cmd, pSql->cmd.clauseIndex, 0); - tscAddTableMetaInfo(pQueryInfo, pMasterTableMetaInfo->name, NULL, NULL, NULL); + tscAddTableMetaInfo(pQueryInfo, pMasterTableMetaInfo->name, NULL, NULL, NULL, NULL); registerSqlObj(pNew); return pNew; @@ -1989,14 +2036,16 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int16_t tableIndex, void (*fp)(), void STableMeta* pTableMeta = taosCacheAcquireByData(tscMetaCache, pTableMetaInfo->pTableMeta); // get by name may failed due to the cache cleanup assert(pTableMeta != NULL); - pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pTableMeta, pTableMetaInfo->vgroupList, pTableMetaInfo->tagColList); + pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pTableMeta, pTableMetaInfo->vgroupList, + pTableMetaInfo->tagColList, pTableMetaInfo->pVgroupTables); } else { // transfer the ownership of pTableMeta to the newly create sql object. STableMetaInfo* pPrevInfo = tscGetTableMetaInfoFromCmd(&pPrevSql->cmd, pPrevSql->cmd.clauseIndex, 0); STableMeta* pPrevTableMeta = taosCacheTransfer(tscMetaCache, (void**)&pPrevInfo->pTableMeta); SVgroupsInfo* pVgroupsInfo = pPrevInfo->vgroupList; - pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pPrevTableMeta, pVgroupsInfo, pTableMetaInfo->tagColList); + pFinalInfo = tscAddTableMetaInfo(pNewQueryInfo, name, pPrevTableMeta, pVgroupsInfo, pTableMetaInfo->tagColList, + pTableMetaInfo->pVgroupTables); } if (pFinalInfo->pTableMeta == NULL) { From daac5036fcbe40d6a60b912a1b48f400f88df6ff Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 22 Oct 2020 17:46:08 +0800 Subject: [PATCH 12/56] [td-1373] --- src/client/src/tscSubquery.c | 50 ++++++++++++++++++++++++++---------- 1 file changed, 36 insertions(+), 14 deletions(-) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 3ac480ae03..7262fdd9f2 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -362,28 +362,50 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { int32_t num = 0; int32_t *list = NULL; - tsBufGetVnodeIdList(pSupporter->pTSBuf, &num, &list); + tsBufGetVnodeIdList(pNewQueryInfo->tsBuf, &num, &list); - for(int32_t k = 0; k < taosArrayGetSize(pTableMetaInfo->pVgroupTables);) { - SVgroupTableInfo* p = taosArrayGet(pTableMetaInfo->pVgroupTables, k); + if (pTableMetaInfo->pVgroupTables != NULL) { + for(int32_t k = 0; k < taosArrayGetSize(pTableMetaInfo->pVgroupTables);) { + SVgroupTableInfo* p = taosArrayGet(pTableMetaInfo->pVgroupTables, k); - bool found = false; - for(int32_t f = 0; f < num; ++f) { - if (p->vgInfo.vgId == list[f]) { - found = true; - break; + bool found = false; + for(int32_t f = 0; f < num; ++f) { + if (p->vgInfo.vgId == list[f]) { + found = true; + break; + } + } + + if (!found) { + tscRemoveVgroupTableGroup(pTableMetaInfo->pVgroupTables, k); + } else { + k++; } } - if (!found) { - tscRemoveVgroupTableGroup(pTableMetaInfo->pVgroupTables, k); - } else { - k++; - } + assert(taosArrayGetSize(pTableMetaInfo->pVgroupTables) > 0); + } else { // TODO remove unnecessarily accessed vnode +// pTableMetaInfo->vgroupList-> +// for(int32_t k = 0; k < taosArrayGetSize(pTableMetaInfo->pVgroupTables);) { +// SVgroupTableInfo* p = taosArrayGet(pTableMetaInfo->pVgroupTables, k); +// +// bool found = false; +// for(int32_t f = 0; f < num; ++f) { +// if (p->vgInfo.vgId == list[f]) { +// found = true; +// break; +// } +// } +// +// if (!found) { +// tscRemoveVgroupTableGroup(pTableMetaInfo->pVgroupTables, k); +// } else { +// k++; +// } +// } } taosTFree(list); - assert(taosArrayGetSize(pTableMetaInfo->pVgroupTables) > 0); size_t numOfCols = taosArrayGetSize(pNewQueryInfo->colList); tscDebug("%p subquery:%p tableIndex:%d, vgroupIndex:%d, type:%d, exprInfo:%" PRIzu ", colList:%" PRIzu ", fieldsInfo:%d, name:%s", From 18f646b7989ca80dab03bfa840738d55a125d773 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 22 Oct 2020 17:49:49 +0800 Subject: [PATCH 13/56] [td-1373] --- src/query/src/qTsbuf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/src/qTsbuf.c b/src/query/src/qTsbuf.c index 7c2f8ebb5e..fa8f83e4dc 100644 --- a/src/query/src/qTsbuf.c +++ b/src/query/src/qTsbuf.c @@ -974,7 +974,7 @@ int32_t tsBufGetNumOfVnodes(STSBuf* pTSBuf) { return 0; } - return pTSBuf->pData->len; + return pTSBuf->numOfVnodes; } void tsBufGetVnodeIdList(STSBuf* pTSBuf, int32_t* num, int32_t** vnodeId) { From e3d9054f462968a07e093faf6a3339960c217c1e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 22 Oct 2020 23:14:01 +0800 Subject: [PATCH 14/56] [td-1373] --- src/client/inc/tscUtil.h | 1 + src/client/src/tscFunctionImpl.c | 2 +- src/client/src/tscSubquery.c | 10 +++++++++- src/client/src/tscUtil.c | 13 ++++++------- 4 files changed, 17 insertions(+), 9 deletions(-) diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 42096f058a..d86e1aa0fb 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -82,6 +82,7 @@ typedef struct SJoinSupporter { char* pIdTagList; // result of first stage tags int32_t totalLen; int32_t num; + SArray* pVgroupTables; } SJoinSupporter; typedef struct SVgroupTableInfo { diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 81893841f7..7a668e85bf 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -4048,7 +4048,7 @@ static void ts_comp_function_f(SQLFunctionCtx *pCtx, int32_t index) { STSBuf *pTSbuf = pInfo->pTSBuf; - tsBufAppend(pTSbuf, 0, &pCtx->tag, pData, TSDB_KEYSIZE); + tsBufAppend(pTSbuf, pCtx->param[0].i64Key, &pCtx->tag, pData, TSDB_KEYSIZE); SET_VAL(pCtx, pCtx->size, 1); pResInfo->hasResult = DATA_SET_FLAG; diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 7262fdd9f2..b58bca26ef 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -324,7 +324,8 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { tscFieldInfoUpdateOffset(pNewQueryInfo); STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pNewQueryInfo, 0); - + pTableMetaInfo->pVgroupTables = pSupporter->pVgroupTables; + /* * When handling the projection query, the offset value will be modified for table-table join, which is changed * during the timestamp intersection. @@ -384,6 +385,7 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { } assert(taosArrayGetSize(pTableMetaInfo->pVgroupTables) > 0); + TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_MULTITABLE_QUERY); } else { // TODO remove unnecessarily accessed vnode // pTableMetaInfo->vgroupList-> // for(int32_t k = 0; k < taosArrayGetSize(pTableMetaInfo->pVgroupTables);) { @@ -750,6 +752,12 @@ static void tidTagRetrieveCallback(void* param, TAOS_RES* tres, int32_t numOfRow STableMetaInfo* pTableMetaInfo2 = tscGetMetaInfo(pQueryInfo2, 0); tscBuildVgroupTableInfo(pParentSql, pTableMetaInfo2, s2); + SSqlObj* psub1 = pParentSql->pSubs[0]; + ((SJoinSupporter*)psub1->param)->pVgroupTables = tscCloneVgroupTableInfo(pTableMetaInfo1->pVgroupTables); + + SSqlObj* psub2 = pParentSql->pSubs[1]; + ((SJoinSupporter*)psub2->param)->pVgroupTables = tscCloneVgroupTableInfo(pTableMetaInfo2->pVgroupTables); + pParentSql->subState.numOfSub = 2; pParentSql->subState.numOfRemain = pParentSql->subState.numOfSub; diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index eb3ad31235..accb20d74f 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1710,8 +1710,6 @@ void tscRemoveVgroupTableGroup(SArray* pVgroupTable, int32_t index) { } taosArrayDestroy(pInfo->itemList); - - taosArrayRemove(pVgroupTable, index); } @@ -1726,14 +1724,15 @@ SArray* tscCloneVgroupTableInfo(SArray* pVgroupTables) { for (size_t i = 0; i < num; i++) { SVgroupTableInfo* pInfo = taosArrayGet(pVgroupTables, i); - for(int32_t j = 0; j < pInfo->vgInfo.numOfEps; ++j) { - taosTFree(pInfo->vgInfo.epAddr[j].fqdn); - } - SVgroupTableInfo info = {0}; info.vgInfo = pInfo->vgInfo; - info.itemList = taosArrayClone(pInfo->itemList); + for(int32_t j = 0; j < pInfo->vgInfo.numOfEps; ++j) { + info.vgInfo.epAddr[j].fqdn = strdup(pInfo->vgInfo.epAddr[j].fqdn); + } + + + info.itemList = taosArrayClone(pInfo->itemList); taosArrayPush(pa, &info); } From 204ba970f1e94c6621352a830a7214584f1dc231 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 23 Oct 2020 00:28:25 +0800 Subject: [PATCH 15/56] [td-1373] --- src/client/src/tscSchemaUtil.c | 2 +- src/client/src/tscSubquery.c | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscSchemaUtil.c b/src/client/src/tscSchemaUtil.c index dfd707344c..ac740555af 100644 --- a/src/client/src/tscSchemaUtil.c +++ b/src/client/src/tscSchemaUtil.c @@ -177,7 +177,7 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size pVgroupInfo->epAddr[i].port = pEpMsg->port; } - tscInitCorVgroupInfo(&pTableMeta->corVgroupInfo, &pTableMeta->vgroupInfo); + tscInitCorVgroupInfo(&pTableMeta->corVgroupInfo, pVgroupInfo); pTableMeta->sversion = pTableMetaMsg->sversion; pTableMeta->tversion = pTableMetaMsg->tversion; diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index b58bca26ef..a911b1e6d7 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -2142,6 +2142,8 @@ void tscBuildResFromSubqueries(SSqlObj *pSql) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex); size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); + pRes->numOfCols = numOfExprs; + pRes->tsrow = calloc(numOfExprs, POINTER_BYTES); pRes->buffer = calloc(numOfExprs, POINTER_BYTES); pRes->length = calloc(numOfExprs, sizeof(int32_t)); From 7582b7e4df6574ff66043b8d542a30edc5fdc76a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 23 Oct 2020 12:45:17 +0800 Subject: [PATCH 16/56] [td-1373] --- src/common/src/tvariant.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/common/src/tvariant.c b/src/common/src/tvariant.c index 005def6dc5..bfd6ca0184 100644 --- a/src/common/src/tvariant.c +++ b/src/common/src/tvariant.c @@ -151,8 +151,11 @@ void tVariantAssign(tVariant *pDst, const tVariant *pSrc) { if (pSrc->nType == TSDB_DATA_TYPE_NCHAR) { len = len * TSDB_NCHAR_SIZE; } - - pDst->pz = calloc(1, len); + + char* p = realloc(pDst->pz, len); + assert(p); + + memset(pDst->pz, 0, len); memcpy(pDst->pz, pSrc->pz, len); return; } From 22a6c4f13c95505a1d853a2479f87b054260505e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 23 Oct 2020 12:55:43 +0800 Subject: [PATCH 17/56] [td-1373] --- src/common/src/tvariant.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/common/src/tvariant.c b/src/common/src/tvariant.c index bfd6ca0184..78d1e38f1e 100644 --- a/src/common/src/tvariant.c +++ b/src/common/src/tvariant.c @@ -147,15 +147,20 @@ void tVariantAssign(tVariant *pDst, const tVariant *pSrc) { *pDst = *pSrc; if (pSrc->nType == TSDB_DATA_TYPE_BINARY || pSrc->nType == TSDB_DATA_TYPE_NCHAR) { - int32_t len = pSrc->nLen + 1; + int32_t allocLen = pSrc->nLen + 1; + int32_t len = pSrc->nLen; + if (pSrc->nType == TSDB_DATA_TYPE_NCHAR) { + allocLen = allocLen * TSDB_NCHAR_SIZE; len = len * TSDB_NCHAR_SIZE; } - char* p = realloc(pDst->pz, len); + char* p = realloc(pDst->pz, allocLen); assert(p); - memset(pDst->pz, 0, len); + memset(p, 0, allocLen); + pDst->pz = p; + memcpy(pDst->pz, pSrc->pz, len); return; } From b137e6e94654710de31bd67bf0c6ad99b870e273 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 23 Oct 2020 13:49:54 +0800 Subject: [PATCH 18/56] [td-1373] --- src/client/src/tscSubquery.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index a911b1e6d7..0dff717fe4 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -23,7 +23,6 @@ #include "tscSubquery.h" #include "tschemautil.h" #include "tsclient.h" -#include "tscSubquery.h" typedef struct SInsertSupporter { SSqlObj* pSql; @@ -305,7 +304,6 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { // set the second stage sub query for join process TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_JOIN_SEC_STAGE); - memcpy(&pQueryInfo->interval, &pSupporter->interval, sizeof(pQueryInfo->interval)); tscTagCondCopy(&pQueryInfo->tagCond, &pSupporter->tagCond); @@ -931,14 +929,16 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR pSql->cmd.command = TSDB_SQL_SELECT; pSql->fp = tscJoinQueryCallback; + + atomic_add_fetch_32(&pState->numOfRemain, 1); tscProcessSql(pSql); return; } } - if (atomic_sub_fetch_32(&pParentSql->subState.numOfRemain, 1) > 0) { - tscDebug("%p sub:%p completed, remain:%d, total:%d", pParentSql, tres, pParentSql->subState.numOfRemain, pState->numOfSub); + if (atomic_sub_fetch_32(&pState->numOfRemain, 1) > 0) { + tscDebug("%p sub:%p completed, remain:%d, total:%d", pParentSql, tres, pState->numOfRemain, pState->numOfSub); return; } From 186543bc15754431cf0daa52b7b67959610ee773 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 23 Oct 2020 13:52:48 +0800 Subject: [PATCH 19/56] [td-1373] --- src/client/src/tscSubquery.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 0dff717fe4..3e90a1567c 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -930,9 +930,7 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR pSql->cmd.command = TSDB_SQL_SELECT; pSql->fp = tscJoinQueryCallback; - atomic_add_fetch_32(&pState->numOfRemain, 1); tscProcessSql(pSql); - return; } } From b6d292da99fcd490a45f572ece20ca59b7893ffb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 23 Oct 2020 13:53:23 +0800 Subject: [PATCH 20/56] [td-1373] --- src/client/src/tscSubquery.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 3e90a1567c..75c5e2c98c 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -924,9 +924,6 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR } if ((++pTableMetaInfo->vgroupIndex) < numOfVgroups) { - pState->numOfRemain = 1; - pState->numOfSub = 1; - pSql->cmd.command = TSDB_SQL_SELECT; pSql->fp = tscJoinQueryCallback; From 503c7321fdbf25fedac73f8ab9a5a21f1413af65 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 23 Oct 2020 13:59:30 +0800 Subject: [PATCH 21/56] [td-1373] --- src/client/src/tscSubquery.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 75c5e2c98c..db76734467 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -923,6 +923,7 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR numOfVgroups = pTableMetaInfo->vgroupList->numOfVgroups; } + tscDebug("%p no result in current vnode anymore, try next vnode", pSql); if ((++pTableMetaInfo->vgroupIndex) < numOfVgroups) { pSql->cmd.command = TSDB_SQL_SELECT; pSql->fp = tscJoinQueryCallback; From 3833df542afda60b587126b3cf549faaa168dd2b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 23 Oct 2020 14:11:01 +0800 Subject: [PATCH 22/56] [td-1373] --- src/client/src/tscSubquery.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index db76734467..d141a3e807 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -923,13 +923,15 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR numOfVgroups = pTableMetaInfo->vgroupList->numOfVgroups; } - tscDebug("%p no result in current vnode anymore, try next vnode", pSql); if ((++pTableMetaInfo->vgroupIndex) < numOfVgroups) { + tscDebug("%p no result in current vnode anymore, try next vnode, vgIndex:%d", pSql, pTableMetaInfo->vgroupIndex); pSql->cmd.command = TSDB_SQL_SELECT; pSql->fp = tscJoinQueryCallback; tscProcessSql(pSql); return; + } else { + tscDebug("%p no result in current subquery anymore", pSql); } } @@ -1175,7 +1177,7 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) { * data instead of returning to its invoker */ if (pTableMetaInfo->vgroupIndex > 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { - pParentSql->subState.numOfRemain = pParentSql->subState.numOfSub; // reset the record value +// pParentSql->subState.numOfRemain = pParentSql->subState.numOfSub; // reset the record value pSql->fp = joinRetrieveFinalResCallback; // continue retrieve data pSql->cmd.command = TSDB_SQL_FETCH; From ce9dae676855d12294c89514cc3eb80ecca69816 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 23 Oct 2020 14:37:00 +0800 Subject: [PATCH 23/56] [td-1373] --- src/query/src/qExecutor.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 331c8faa53..7df85369ec 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -4769,9 +4769,9 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { if (pRuntimeEnv->pTSBuf != NULL) { if (pRuntimeEnv->cur.vgroupIndex == -1) { - STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, 0, &pRuntimeEnv->pCtx[0].tag); + STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, pQInfo->vgId, &pRuntimeEnv->pCtx[0].tag); - // failed to find data with the specified tag value + // failed to find data with the specified tag value and vnodeId if (elem.vnode < 0) { return false; } From 0e75d3983f757fbec0c03dffdcd58d9fc07cfe0c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 23 Oct 2020 15:01:04 +0800 Subject: [PATCH 24/56] [td-1373] --- src/query/inc/qTsbuf.h | 2 +- src/query/src/qExecutor.c | 2 +- src/query/src/qTsbuf.c | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/query/inc/qTsbuf.h b/src/query/inc/qTsbuf.h index 8bd775dac5..fe5de8b25a 100644 --- a/src/query/inc/qTsbuf.h +++ b/src/query/inc/qTsbuf.h @@ -106,7 +106,7 @@ typedef struct STSBufFileHeader { STSBuf* tsBufCreate(bool autoDelete, int32_t order); STSBuf* tsBufCreateFromFile(const char* path, bool autoDelete); -STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_t len, int32_t tsOrder); +STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_t len, int32_t tsOrder, int32_t vnodeId); void* tsBufDestroy(STSBuf* pTSBuf); diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 7df85369ec..251e225cfb 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -6325,7 +6325,7 @@ static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQ STSBuf *pTSBuf = NULL; if (pQueryMsg->tsLen > 0) { // open new file to save the result char *tsBlock = (char *) pQueryMsg + pQueryMsg->tsOffset; - pTSBuf = tsBufCreateFromCompBlocks(tsBlock, pQueryMsg->tsNumOfBlocks, pQueryMsg->tsLen, pQueryMsg->tsOrder); + pTSBuf = tsBufCreateFromCompBlocks(tsBlock, pQueryMsg->tsNumOfBlocks, pQueryMsg->tsLen, pQueryMsg->tsOrder, vgId); tsBufResetPos(pTSBuf); bool ret = tsBufNextPos(pTSBuf); diff --git a/src/query/src/qTsbuf.c b/src/query/src/qTsbuf.c index fa8f83e4dc..740bbfcf2d 100644 --- a/src/query/src/qTsbuf.c +++ b/src/query/src/qTsbuf.c @@ -803,14 +803,14 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf) { return 0; } -STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_t len, int32_t order) { +STSBuf* tsBufCreateFromCompBlocks(const char* pData, int32_t numOfBlocks, int32_t len, int32_t order, int32_t vnodeId) { STSBuf* pTSBuf = tsBufCreate(true, order); STSVnodeBlockInfo* pBlockInfo = &(addOneVnodeInfo(pTSBuf, 0)->info); pBlockInfo->numOfBlocks = numOfBlocks; pBlockInfo->compLen = len; pBlockInfo->offset = getDataStartOffset(); - pBlockInfo->vnode = 0; + pBlockInfo->vnode = vnodeId; // update prev vnode length info in file TSBufUpdateVnodeInfo(pTSBuf, pTSBuf->numOfVnodes - 1, pBlockInfo); From d5df0e03f4d61dc1b0a6c7303460c374e0f491e7 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 23 Oct 2020 15:13:34 +0800 Subject: [PATCH 25/56] [td-1373] --- src/query/src/qExecutor.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 251e225cfb..a71cb39826 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -1305,7 +1305,6 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS // from top to bottom in desc // from bottom to top in asc order if (pRuntimeEnv->pTSBuf != NULL) { - SQInfo *pQInfo = (SQInfo *)GET_QINFO_ADDR(pRuntimeEnv); qDebug("QInfo:%p process data rows, numOfRows:%d, query order:%d, ts comp order:%d", pQInfo, pDataBlockInfo->rows, pQuery->order.order, pRuntimeEnv->pTSBuf->cur.order); } From 30642fa1c6611a25ca7da759522392bf013dedce Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 23 Oct 2020 16:10:02 +0800 Subject: [PATCH 26/56] [td-1373] --- src/common/src/tvariant.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/common/src/tvariant.c b/src/common/src/tvariant.c index 78d1e38f1e..5e378c4621 100644 --- a/src/common/src/tvariant.c +++ b/src/common/src/tvariant.c @@ -144,8 +144,8 @@ void tVariantDestroy(tVariant *pVar) { void tVariantAssign(tVariant *pDst, const tVariant *pSrc) { if (pSrc == NULL || pDst == NULL) return; - *pDst = *pSrc; - + pDst->nType = pSrc->nType; + if (pSrc->nType == TSDB_DATA_TYPE_BINARY || pSrc->nType == TSDB_DATA_TYPE_NCHAR) { int32_t allocLen = pSrc->nLen + 1; int32_t len = pSrc->nLen; From a89368e6b6ed972870741ed7d8c7bd5f6df2f167 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 23 Oct 2020 16:24:00 +0800 Subject: [PATCH 27/56] [td-1373] --- src/common/src/tvariant.c | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/src/common/src/tvariant.c b/src/common/src/tvariant.c index 5e378c4621..e244744401 100644 --- a/src/common/src/tvariant.c +++ b/src/common/src/tvariant.c @@ -147,21 +147,15 @@ void tVariantAssign(tVariant *pDst, const tVariant *pSrc) { pDst->nType = pSrc->nType; if (pSrc->nType == TSDB_DATA_TYPE_BINARY || pSrc->nType == TSDB_DATA_TYPE_NCHAR) { - int32_t allocLen = pSrc->nLen + 1; - int32_t len = pSrc->nLen; - - if (pSrc->nType == TSDB_DATA_TYPE_NCHAR) { - allocLen = allocLen * TSDB_NCHAR_SIZE; - len = len * TSDB_NCHAR_SIZE; - } - - char* p = realloc(pDst->pz, allocLen); + int32_t len = pSrc->nLen + TSDB_NCHAR_SIZE; + char* p = realloc(pDst->pz, len); assert(p); - memset(p, 0, allocLen); + memset(p, 0, len); pDst->pz = p; - memcpy(pDst->pz, pSrc->pz, len); + memcpy(pDst->pz, pSrc->pz, pSrc->nLen); + pDst->nLen = pSrc->nLen; return; } From 3fff8aba71e783acdaead2303e1b97f035d7ee2a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 23 Oct 2020 22:54:47 +0800 Subject: [PATCH 28/56] [td-1373] --- src/client/src/tscSubquery.c | 78 +++++++++++++++++++++++++++++++----- src/query/src/qExecutor.c | 8 ++++ 2 files changed, 76 insertions(+), 10 deletions(-) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index d141a3e807..be8f417df7 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -465,6 +465,8 @@ static void quitAllSubquery(SSqlObj* pSqlObj, SJoinSupporter* pSupporter) { static void updateQueryTimeRange(SQueryInfo* pQueryInfo, STimeWindow* win) { assert(pQueryInfo->window.skey <= win->skey && pQueryInfo->window.ekey >= win->ekey); pQueryInfo->window = *win; + + } int32_t tscCompareTidTags(const void* p1, const void* p2) { @@ -963,19 +965,18 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR static SJoinSupporter* tscUpdateSubqueryStatus(SSqlObj* pSql, int32_t numOfFetch) { int32_t notInvolved = 0; - SJoinSupporter* pSupporter = NULL; SSubqueryState* pState = &pSql->subState; for(int32_t i = 0; i < pSql->subState.numOfSub; ++i) { if (pSql->pSubs[i] == NULL) { notInvolved++; - } else { - pSupporter = (SJoinSupporter*)pSql->pSubs[i]->param; +// } else { +// (SJoinSupporter*)pSql->pSubs[i]->param; } } pState->numOfRemain = numOfFetch; - return pSupporter; + return NULL; } void tscFetchDatablockFromSubquery(SSqlObj* pSql) { @@ -983,14 +984,16 @@ void tscFetchDatablockFromSubquery(SSqlObj* pSql) { int32_t numOfFetch = 0; bool hasData = true; + + // if the subquery is NULL, it does not involved in the final result generation for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { - // if the subquery is NULL, it does not involved in the final result generation SSqlObj* pSub = pSql->pSubs[i]; if (pSub == NULL) { continue; } - + SSqlRes *pRes = &pSub->res; + SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSub->cmd, 0); if (!tscHasReachLimitation(pQueryInfo, pRes)) { @@ -1014,8 +1017,61 @@ void tscFetchDatablockFromSubquery(SSqlObj* pSql) { tscBuildResFromSubqueries(pSql); return; } else if (numOfFetch <= 0) { - pSql->res.completed = true; - freeJoinSubqueryObj(pSql); + + bool tryNextVnode = false; + + SSqlObj* pp = pSql->pSubs[0]; + SQueryInfo* pi = tscGetQueryInfoDetail(&pp->cmd, 0); + if (tscNonOrderedProjectionQueryOnSTable(pi, 0)) { + + for(int32_t i = 0; i < pSql->subState.numOfSub; ++i) { + if (pSql->pSubs[i] != NULL) { + pSql->subState.numOfRemain++; + } + } + + } + + for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { + SSqlObj* pSub = pSql->pSubs[i]; + if (pSub == NULL) { + continue; + } + + SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSub->cmd, 0); + + if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + assert(pQueryInfo->numOfTables == 1); + + // for projection query, need to try next vnode if current vnode is exhausted + int32_t numOfVgroups = 0; // TODO refactor + if (pTableMetaInfo->pVgroupTables != NULL) { + numOfVgroups = taosArrayGetSize(pTableMetaInfo->pVgroupTables); + } else { + numOfVgroups = pTableMetaInfo->vgroupList->numOfVgroups; + } + + if ((++pTableMetaInfo->vgroupIndex) < numOfVgroups) { + tscDebug("%p no result in current vnode anymore, try next vnode, vgIndex:%d", pSub, + pTableMetaInfo->vgroupIndex); + pSub->cmd.command = TSDB_SQL_SELECT; + pSub->fp = tscJoinQueryCallback; + + tscProcessSql(pSub); + tryNextVnode = true; + } else { + tscDebug("%p no result in current subquery anymore", pSub); + } + } + } + + if (tryNextVnode) { + return; + } else { + pSql->res.completed = true; + freeJoinSubqueryObj(pSql); + } if (pSql->res.code == TSDB_CODE_SUCCESS) { (*pSql->fp)(pSql->param, pSql, 0); @@ -1027,15 +1083,17 @@ void tscFetchDatablockFromSubquery(SSqlObj* pSql) { } // TODO multi-vnode retrieve for projection query with limitation has bugs, since the global limiation is not handled + // retrieve data from current vnode. tscDebug("%p retrieve data from %d subqueries", pSql, numOfFetch); - SJoinSupporter* pSupporter = tscUpdateSubqueryStatus(pSql, numOfFetch); + SJoinSupporter* pSupporter = NULL; + tscUpdateSubqueryStatus(pSql, numOfFetch); for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { SSqlObj* pSql1 = pSql->pSubs[i]; if (pSql1 == NULL) { continue; } - + SSqlRes* pRes1 = &pSql1->res; SSqlCmd* pCmd1 = &pSql1->cmd; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index a71cb39826..dd14523c2a 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -4775,6 +4775,14 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { return false; } } else { + STSElem elem = tsBufGetElem(pRuntimeEnv->pTSBuf); + if (tVariantCompare(&elem.tag, &pRuntimeEnv->pCtx[0].tag) != 0) { + STSElem elem1 = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, pQInfo->vgId, &pRuntimeEnv->pCtx[0].tag); + if (elem1.vnode < 0) { + return false; + } + } + tsBufSetCursor(pRuntimeEnv->pTSBuf, &pRuntimeEnv->cur); } } From 9b2a53cec2dac86f07a78b97a1d37fd72e5d0dca Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 23 Oct 2020 23:19:21 +0800 Subject: [PATCH 29/56] [td-1373] --- src/query/src/qExecutor.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index dd14523c2a..cfef95ef00 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -4769,10 +4769,14 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { if (pRuntimeEnv->pTSBuf != NULL) { if (pRuntimeEnv->cur.vgroupIndex == -1) { STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, pQInfo->vgId, &pRuntimeEnv->pCtx[0].tag); - // failed to find data with the specified tag value and vnodeId if (elem.vnode < 0) { + qDebug("QInfo:%p failed to find tag:%s in ts_comp", pQInfo, pRuntimeEnv->pCtx[0].tag.pz); return false; + } else { + STSCursor cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); + qDebug("QInfo:%p find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pRuntimeEnv->pCtx[0].tag.pz, + cur.blockIndex, cur.tsIndex); } } else { STSElem elem = tsBufGetElem(pRuntimeEnv->pTSBuf); @@ -4784,6 +4788,9 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { } tsBufSetCursor(pRuntimeEnv->pTSBuf, &pRuntimeEnv->cur); + STSCursor cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); + qDebug("QInfo:%p continue scan ts_comp file, tag:%s blockIndex:%d, tsIndex:%d", pQInfo, pRuntimeEnv->pCtx[0].tag.pz, + cur.blockIndex, cur.tsIndex); } } From efc9c32edd54dbc91f2a576598573febc2deed75 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 23 Oct 2020 23:41:24 +0800 Subject: [PATCH 30/56] [td-1373] --- src/query/src/qExecutor.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index cfef95ef00..6291ef53f5 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -4785,12 +4785,13 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { if (elem1.vnode < 0) { return false; } - } - - tsBufSetCursor(pRuntimeEnv->pTSBuf, &pRuntimeEnv->cur); + } else { + tsBufSetCursor(pRuntimeEnv->pTSBuf, &pRuntimeEnv->cur); STSCursor cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); qDebug("QInfo:%p continue scan ts_comp file, tag:%s blockIndex:%d, tsIndex:%d", pQInfo, pRuntimeEnv->pCtx[0].tag.pz, cur.blockIndex, cur.tsIndex); + } + } } From c6b30184198ab7bd4a8cad18756de92158045da2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 24 Oct 2020 10:14:46 +0800 Subject: [PATCH 31/56] [td-1373] --- src/query/src/qExecutor.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 6291ef53f5..2aca661a6a 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -4771,7 +4771,7 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, pQInfo->vgId, &pRuntimeEnv->pCtx[0].tag); // failed to find data with the specified tag value and vnodeId if (elem.vnode < 0) { - qDebug("QInfo:%p failed to find tag:%s in ts_comp", pQInfo, pRuntimeEnv->pCtx[0].tag.pz); + qError("QInfo:%p failed to find tag:%s in ts_comp", pQInfo, pRuntimeEnv->pCtx[0].tag.pz); return false; } else { STSCursor cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); @@ -4781,9 +4781,16 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { } else { STSElem elem = tsBufGetElem(pRuntimeEnv->pTSBuf); if (tVariantCompare(&elem.tag, &pRuntimeEnv->pCtx[0].tag) != 0) { + STSElem elem1 = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, pQInfo->vgId, &pRuntimeEnv->pCtx[0].tag); + // failed to find data with the specified tag value and vnodeId if (elem1.vnode < 0) { + qError("QInfo:%p failed to find tag:%s in ts_comp", pQInfo, pRuntimeEnv->pCtx[0].tag.pz); return false; + } else { + STSCursor cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); + qDebug("QInfo:%p find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pRuntimeEnv->pCtx[0].tag.pz, + cur.blockIndex, cur.tsIndex); } } else { tsBufSetCursor(pRuntimeEnv->pTSBuf, &pRuntimeEnv->cur); @@ -5047,6 +5054,10 @@ static void sequentialTableProcess(SQInfo *pQInfo) { break; } + if (pRuntimeEnv->pTSBuf != NULL) { + pRuntimeEnv->cur = pRuntimeEnv->pTSBuf->cur; + } + } else { // all data in the result buffer are skipped due to the offset, continue to retrieve data from current meter if (pQuery->rec.rows == 0) { From 09c39cd2b878306cbb299083cfe28372b444eb55 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 24 Oct 2020 14:37:24 +0800 Subject: [PATCH 32/56] [td-1373] fix memory leak --- src/client/src/tscSubquery.c | 70 ++++++++++++++++++++++++++---------- src/query/src/qTsbuf.c | 7 ++-- 2 files changed, 55 insertions(+), 22 deletions(-) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index be8f417df7..ca7341d8e9 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -58,6 +58,8 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ pSubQueryInfo1->tsBuf = output1; pSubQueryInfo2->tsBuf = output2; + TSKEY st = taosGetTimestampUs(); + // no result generated, return directly if (pSupporter1->pTSBuf == NULL || pSupporter2->pTSBuf == NULL) { tscDebug("%p at least one ts-comp is empty, 0 for secondary query after ts blocks intersecting", pSql); @@ -123,6 +125,9 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ tsBufAppend(output1, elem1.vnode, &elem1.tag, (const char*)&elem1.ts, sizeof(elem1.ts)); tsBufAppend(output2, elem2.vnode, &elem2.tag, (const char*)&elem2.ts, sizeof(elem2.ts)); + + tVariantDestroy(&elem1.tag); + tVariantDestroy(&elem2.tag); } else { pLimit->offset -= 1; } @@ -157,9 +162,10 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ tsBufDestroy(pSupporter1->pTSBuf); tsBufDestroy(pSupporter2->pTSBuf); + TSKEY et = taosGetTimestampUs(); tscDebug("%p input1:%" PRId64 ", input2:%" PRId64 ", final:%" PRId64 " in %d vnodes for secondary query after ts blocks " - "intersecting, skey:%" PRId64 ", ekey:%" PRId64 ", numOfVnode:%d", pSql, numOfInput1, numOfInput2, output1->numOfTotal, - output1->numOfVnodes, win->skey, win->ekey, tsBufGetNumOfVnodes(output1)); + "intersecting, skey:%" PRId64 ", ekey:%" PRId64 ", numOfVnode:%d, elasped time:%"PRId64" us", pSql, numOfInput1, numOfInput2, output1->numOfTotal, + output1->numOfVnodes, win->skey, win->ekey, tsBufGetNumOfVnodes(output1), et - st); return output1->numOfTotal; } @@ -952,11 +958,21 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR // update the records for each subquery in parent sql object. for (int32_t i = 0; i < pState->numOfSub; ++i) { if (pParentSql->pSubs[i] == NULL) { + tscDebug("%p %p sub:%d not retrieve data", pParentSql, NULL, i); continue; } SSqlRes* pRes1 = &pParentSql->pSubs[i]->res; - pRes1->numOfClauseTotal += pRes1->numOfRows; + + if (pRes1->row > 0 && pRes1->numOfRows > 0) { + tscDebug("%p sub:%p index:%d numOfRows:%"PRId64" total:%"PRId64 " (not retrieve)", pParentSql, pParentSql->pSubs[i], i, + pRes1->numOfRows, pRes1->numOfTotal); + assert(pRes1->row < pRes1->numOfRows); + } else { + pRes1->numOfClauseTotal += pRes1->numOfRows; + tscDebug("%p sub:%p index:%d numOfRows:%"PRId64" total:%"PRId64, pParentSql, pParentSql->pSubs[i], i, + pRes1->numOfRows, pRes1->numOfTotal); + } } // data has retrieved to client, build the join results @@ -998,8 +1014,11 @@ void tscFetchDatablockFromSubquery(SSqlObj* pSql) { if (!tscHasReachLimitation(pQueryInfo, pRes)) { if (pRes->row >= pRes->numOfRows) { + // no data left in current result buffer hasData = false; + // The current query is completed for the active vnode, try next vnode if exists + // If it is completed, no need to fetch anymore. if (!pRes->completed) { numOfFetch++; } @@ -1016,20 +1035,24 @@ void tscFetchDatablockFromSubquery(SSqlObj* pSql) { if (hasData) { tscBuildResFromSubqueries(pSql); return; - } else if (numOfFetch <= 0) { + } + // If at least one subquery is completed in current vnode, try the next vnode in case of multi-vnode + // super table projection query. + if (numOfFetch <= 0) { bool tryNextVnode = false; SSqlObj* pp = pSql->pSubs[0]; SQueryInfo* pi = tscGetQueryInfoDetail(&pp->cmd, 0); - if (tscNonOrderedProjectionQueryOnSTable(pi, 0)) { + // get the number of subquery that need to retrieve the next vnode. + if (tscNonOrderedProjectionQueryOnSTable(pi, 0)) { for(int32_t i = 0; i < pSql->subState.numOfSub; ++i) { - if (pSql->pSubs[i] != NULL) { + SSqlObj* pSub = pSql->pSubs[i]; + if (pSub != NULL && pSub->res.row >= pSub->res.numOfRows && pSub->res.completed) { pSql->subState.numOfRemain++; } } - } for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { @@ -1040,7 +1063,7 @@ void tscFetchDatablockFromSubquery(SSqlObj* pSql) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSub->cmd, 0); - if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { + if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0) && pSub->res.row >= pSub->res.numOfRows && pSub->res.completed) { STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); assert(pQueryInfo->numOfTables == 1); @@ -1068,11 +1091,11 @@ void tscFetchDatablockFromSubquery(SSqlObj* pSql) { if (tryNextVnode) { return; - } else { - pSql->res.completed = true; - freeJoinSubqueryObj(pSql); } - + + pSql->res.completed = true; + freeJoinSubqueryObj(pSql); + if (pSql->res.code == TSDB_CODE_SUCCESS) { (*pSql->fp)(pSql->param, pSql, 0); } else { @@ -2130,7 +2153,7 @@ static char* getResultBlockPosition(SSqlCmd* pCmd, SSqlRes* pRes, int32_t column assert(pInfo->pSqlExpr != NULL); *bytes = pInfo->pSqlExpr->resBytes; - char* pData = pRes->data + pInfo->pSqlExpr->offset * pRes->numOfRows; + char* pData = pRes->data + pInfo->pSqlExpr->offset * pRes->numOfRows + pRes->row * (*bytes); return pData; } @@ -2142,11 +2165,13 @@ static void doBuildResFromSubqueries(SSqlObj* pSql) { int32_t numOfRes = INT32_MAX; for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { - if (pSql->pSubs[i] == NULL) { + SSqlObj* pSub = pSql->pSubs[i]; + if (pSub == NULL) { continue; } - numOfRes = (int32_t)(MIN(numOfRes, pSql->pSubs[i]->res.numOfRows)); + int32_t remain = pSub->res.numOfRows - pSub->res.row; + numOfRes = (int32_t)(MIN(numOfRes, remain)); } if (numOfRes == 0) { @@ -2172,14 +2197,23 @@ static void doBuildResFromSubqueries(SSqlObj* pSql) { size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); for(int32_t i = 0; i < numOfExprs; ++i) { SColumnIndex* pIndex = &pRes->pColumnIndex[i]; - SSqlRes *pRes1 = &pSql->pSubs[pIndex->tableIndex]->res; - SSqlCmd *pCmd1 = &pSql->pSubs[pIndex->tableIndex]->cmd; + SSqlRes* pRes1 = &pSql->pSubs[pIndex->tableIndex]->res; + SSqlCmd* pCmd1 = &pSql->pSubs[pIndex->tableIndex]->cmd; char* pData = getResultBlockPosition(pCmd1, pRes1, pIndex->columnIndex, &bytes); memcpy(data, pData, bytes * numOfRes); data += bytes * numOfRes; - pRes1->row = numOfRes; + } + + for(int32_t i = 0; i < pSql->subState.numOfSub; ++i) { + SSqlObj* pSub = pSql->pSubs[i]; + if (pSub == NULL) { + continue; + } + + pSub->res.row += numOfRes; + assert(pSub->res.row <= pSub->res.numOfRows); } pRes->numOfRows = numOfRes; diff --git a/src/query/src/qTsbuf.c b/src/query/src/qTsbuf.c index 740bbfcf2d..eefb0b56a4 100644 --- a/src/query/src/qTsbuf.c +++ b/src/query/src/qTsbuf.c @@ -400,11 +400,11 @@ void tsBufAppend(STSBuf* pTSBuf, int32_t vnodeId, tVariant* tag, const char* pDa if ((tVariantCompare(&pTSBuf->block.tag, tag) != 0) && ptsData->len > 0) { // new arrived data with different tags value, save current value into disk first writeDataToDisk(pTSBuf); + tVariantAssign(&pTSBuf->block.tag, tag); } else { expandBuffer(ptsData, len); } - tVariantAssign(&pTSBuf->block.tag, tag); memcpy(ptsData->rawBuf + ptsData->len, pData, (size_t)len); // todo check return value @@ -662,7 +662,7 @@ bool tsBufNextPos(STSBuf* pTSBuf) { return false; } - int32_t blockIndex = pCur->order == TSDB_ORDER_ASC ? 0 : pBlockInfo->numOfBlocks - 1; + int32_t blockIndex = (pCur->order == TSDB_ORDER_ASC) ? 0 : (pBlockInfo->numOfBlocks - 1); tsBufGetBlock(pTSBuf, pCur->vgroupIndex + step, blockIndex); break; @@ -688,8 +688,7 @@ void tsBufResetPos(STSBuf* pTSBuf) { } STSElem tsBufGetElem(STSBuf* pTSBuf) { - STSElem elem1 = {.vnode = -1}; - + STSElem elem1 = {.vnode = -1}; if (pTSBuf == NULL) { return elem1; } From c6cc433caff1eb937f879c5a2aefbed3d3dced87 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 24 Oct 2020 14:44:42 +0800 Subject: [PATCH 33/56] [td-1373] fix memory leak --- src/query/src/qTsbuf.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/query/src/qTsbuf.c b/src/query/src/qTsbuf.c index eefb0b56a4..42b6e3f6e0 100644 --- a/src/query/src/qTsbuf.c +++ b/src/query/src/qTsbuf.c @@ -400,11 +400,11 @@ void tsBufAppend(STSBuf* pTSBuf, int32_t vnodeId, tVariant* tag, const char* pDa if ((tVariantCompare(&pTSBuf->block.tag, tag) != 0) && ptsData->len > 0) { // new arrived data with different tags value, save current value into disk first writeDataToDisk(pTSBuf); - tVariantAssign(&pTSBuf->block.tag, tag); } else { expandBuffer(ptsData, len); } - + + tVariantAssign(&pTSBuf->block.tag, tag); memcpy(ptsData->rawBuf + ptsData->len, pData, (size_t)len); // todo check return value From c1bea6666d99295a60a84c29cc0943b7bf11820a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 24 Oct 2020 15:52:30 +0800 Subject: [PATCH 34/56] [td-1373] --- src/client/src/tscSubquery.c | 36 +++++++++++------------------ src/common/src/tvariant.c | 13 ++++------- src/query/src/qExecutor.c | 45 ++++++++++++++++++++++++++++-------- 3 files changed, 53 insertions(+), 41 deletions(-) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index ca7341d8e9..f4b6ea2c94 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -979,27 +979,12 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR tscBuildResFromSubqueries(pParentSql); } -static SJoinSupporter* tscUpdateSubqueryStatus(SSqlObj* pSql, int32_t numOfFetch) { - int32_t notInvolved = 0; - SSubqueryState* pState = &pSql->subState; - - for(int32_t i = 0; i < pSql->subState.numOfSub; ++i) { - if (pSql->pSubs[i] == NULL) { - notInvolved++; -// } else { -// (SJoinSupporter*)pSql->pSubs[i]->param; - } - } - - pState->numOfRemain = numOfFetch; - return NULL; -} - void tscFetchDatablockFromSubquery(SSqlObj* pSql) { assert(pSql->subState.numOfSub >= 1); int32_t numOfFetch = 0; - bool hasData = true; + bool hasData = true; + bool reachLimit = false; // if the subquery is NULL, it does not involved in the final result generation for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { @@ -1025,7 +1010,8 @@ void tscFetchDatablockFromSubquery(SSqlObj* pSql) { } } else { // has reach the limitation, no data anymore if (pRes->row >= pRes->numOfRows) { - hasData = false; + reachLimit = true; + hasData = false; break; } } @@ -1039,7 +1025,7 @@ void tscFetchDatablockFromSubquery(SSqlObj* pSql) { // If at least one subquery is completed in current vnode, try the next vnode in case of multi-vnode // super table projection query. - if (numOfFetch <= 0) { + if (numOfFetch <= 0 && !reachLimit) { bool tryNextVnode = false; SSqlObj* pp = pSql->pSubs[0]; @@ -1109,8 +1095,8 @@ void tscFetchDatablockFromSubquery(SSqlObj* pSql) { // retrieve data from current vnode. tscDebug("%p retrieve data from %d subqueries", pSql, numOfFetch); SJoinSupporter* pSupporter = NULL; - tscUpdateSubqueryStatus(pSql, numOfFetch); - + pSql->subState.numOfRemain = numOfFetch; + for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { SSqlObj* pSql1 = pSql->pSubs[i]; if (pSql1 == NULL) { @@ -1522,7 +1508,13 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); SSubqueryState *pState = &pSql->subState; - pState->numOfSub = pTableMetaInfo->vgroupList->numOfVgroups; + pState->numOfSub = 0; + if (pTableMetaInfo->pVgroupTables == NULL) { + pState->numOfSub = pTableMetaInfo->vgroupList->numOfVgroups; + } else { + pState->numOfSub = taosArrayGetSize(pTableMetaInfo->pVgroupTables); + } + assert(pState->numOfSub > 0); int32_t ret = tscLocalReducerEnvCreate(pSql, &pMemoryBuf, &pDesc, &pModel, nBufferSize); diff --git a/src/common/src/tvariant.c b/src/common/src/tvariant.c index e244744401..5c5cda3b5d 100644 --- a/src/common/src/tvariant.c +++ b/src/common/src/tvariant.c @@ -145,8 +145,9 @@ void tVariantAssign(tVariant *pDst, const tVariant *pSrc) { if (pSrc == NULL || pDst == NULL) return; pDst->nType = pSrc->nType; - - if (pSrc->nType == TSDB_DATA_TYPE_BINARY || pSrc->nType == TSDB_DATA_TYPE_NCHAR) { + if (pSrc->nType >= TSDB_DATA_TYPE_BOOL && pSrc->nType <= TSDB_DATA_TYPE_DOUBLE) { + pDst->i64Key = pSrc->i64Key; + } else if (pSrc->nType == TSDB_DATA_TYPE_BINARY || pSrc->nType == TSDB_DATA_TYPE_NCHAR) { int32_t len = pSrc->nLen + TSDB_NCHAR_SIZE; char* p = realloc(pDst->pz, len); assert(p); @@ -156,11 +157,7 @@ void tVariantAssign(tVariant *pDst, const tVariant *pSrc) { memcpy(pDst->pz, pSrc->pz, pSrc->nLen); pDst->nLen = pSrc->nLen; - return; - } - - // this is only for string array - if (pSrc->nType == TSDB_DATA_TYPE_ARRAY) { + } else if (pSrc->nType == TSDB_DATA_TYPE_ARRAY) { // this is only for string array size_t num = taosArrayGetSize(pSrc->arr); pDst->arr = taosArrayInit(num, sizeof(char*)); for(size_t i = 0; i < num; i++) { @@ -168,8 +165,6 @@ void tVariantAssign(tVariant *pDst, const tVariant *pSrc) { char* n = strdup(p); taosArrayPush(pDst->arr, &n); } - - return; } pDst->nLen = tDataTypeDesc[pDst->nType].nSize; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 2aca661a6a..3812cd31f6 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -4767,36 +4767,61 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { } if (pRuntimeEnv->pTSBuf != NULL) { + tVariant* pTag = &pRuntimeEnv->pCtx[0].tag; + if (pRuntimeEnv->cur.vgroupIndex == -1) { - STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, pQInfo->vgId, &pRuntimeEnv->pCtx[0].tag); + STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, pQInfo->vgId, pTag); // failed to find data with the specified tag value and vnodeId if (elem.vnode < 0) { - qError("QInfo:%p failed to find tag:%s in ts_comp", pQInfo, pRuntimeEnv->pCtx[0].tag.pz); + if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { + qError("QInfo:%p failed to find tag:%s in ts_comp", pQInfo, pTag->pz); + } else { + qError("QInfo:%p failed to find tag:%"PRId64" in ts_comp", pQInfo, pTag->i64Key); + } + return false; } else { STSCursor cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); - qDebug("QInfo:%p find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pRuntimeEnv->pCtx[0].tag.pz, - cur.blockIndex, cur.tsIndex); + + if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { + qDebug("QInfo:%p find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->pz, + cur.blockIndex, cur.tsIndex); + } else { + qDebug("QInfo:%p find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->i64Key, + cur.blockIndex, cur.tsIndex); + } + + } } else { STSElem elem = tsBufGetElem(pRuntimeEnv->pTSBuf); if (tVariantCompare(&elem.tag, &pRuntimeEnv->pCtx[0].tag) != 0) { - STSElem elem1 = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, pQInfo->vgId, &pRuntimeEnv->pCtx[0].tag); + STSElem elem1 = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, pQInfo->vgId, pTag); // failed to find data with the specified tag value and vnodeId if (elem1.vnode < 0) { - qError("QInfo:%p failed to find tag:%s in ts_comp", pQInfo, pRuntimeEnv->pCtx[0].tag.pz); + if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { + qError("QInfo:%p failed to find tag:%s in ts_comp", pQInfo, pTag->pz); + } else { + qError("QInfo:%p failed to find tag:%"PRId64" in ts_comp", pQInfo, pTag->i64Key); + } return false; } else { STSCursor cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); - qDebug("QInfo:%p find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pRuntimeEnv->pCtx[0].tag.pz, - cur.blockIndex, cur.tsIndex); + if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { + qDebug("QInfo:%p find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->pz, cur.blockIndex, cur.tsIndex); + } else { + qDebug("QInfo:%p find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->i64Key, cur.blockIndex, cur.tsIndex); + } } } else { tsBufSetCursor(pRuntimeEnv->pTSBuf, &pRuntimeEnv->cur); STSCursor cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); - qDebug("QInfo:%p continue scan ts_comp file, tag:%s blockIndex:%d, tsIndex:%d", pQInfo, pRuntimeEnv->pCtx[0].tag.pz, - cur.blockIndex, cur.tsIndex); + if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { + qDebug("QInfo:%p continue scan ts_comp file, tag:%s blockIndex:%d, tsIndex:%d", pQInfo, pTag->pz, cur.blockIndex, cur.tsIndex); + } else { + qDebug("QInfo:%p continue scan ts_comp file, tag:%"PRId64" blockIndex:%d, tsIndex:%d", pQInfo, pTag->i64Key, cur.blockIndex, cur.tsIndex); + } } } From 88de14360a3e106cdad0270565c959504e2c480e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 24 Oct 2020 15:55:55 +0800 Subject: [PATCH 35/56] [td-1373] fix memory leak --- src/common/src/tvariant.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/common/src/tvariant.c b/src/common/src/tvariant.c index 5c5cda3b5d..9eb9924932 100644 --- a/src/common/src/tvariant.c +++ b/src/common/src/tvariant.c @@ -145,9 +145,7 @@ void tVariantAssign(tVariant *pDst, const tVariant *pSrc) { if (pSrc == NULL || pDst == NULL) return; pDst->nType = pSrc->nType; - if (pSrc->nType >= TSDB_DATA_TYPE_BOOL && pSrc->nType <= TSDB_DATA_TYPE_DOUBLE) { - pDst->i64Key = pSrc->i64Key; - } else if (pSrc->nType == TSDB_DATA_TYPE_BINARY || pSrc->nType == TSDB_DATA_TYPE_NCHAR) { + if (pSrc->nType == TSDB_DATA_TYPE_BINARY || pSrc->nType == TSDB_DATA_TYPE_NCHAR) { int32_t len = pSrc->nLen + TSDB_NCHAR_SIZE; char* p = realloc(pDst->pz, len); assert(p); @@ -157,6 +155,12 @@ void tVariantAssign(tVariant *pDst, const tVariant *pSrc) { memcpy(pDst->pz, pSrc->pz, pSrc->nLen); pDst->nLen = pSrc->nLen; + return; + + } + + if (pSrc->nType >= TSDB_DATA_TYPE_BOOL && pSrc->nType <= TSDB_DATA_TYPE_DOUBLE) { + pDst->i64Key = pSrc->i64Key; } else if (pSrc->nType == TSDB_DATA_TYPE_ARRAY) { // this is only for string array size_t num = taosArrayGetSize(pSrc->arr); pDst->arr = taosArrayInit(num, sizeof(char*)); From c43e13ab0b68b45c616ad22d15c33bd2b465e05f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 24 Oct 2020 16:11:45 +0800 Subject: [PATCH 36/56] [td-1373] --- src/query/src/qExecutor.c | 68 +++++++++++++++++++++++++++++++++------ 1 file changed, 58 insertions(+), 10 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 3812cd31f6..9d921f5150 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -2629,8 +2629,13 @@ void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, void *pTable, void *tsdb) { SColumnInfo* pColInfo = doGetTagColumnInfoById(pQuery->tagColList, pQuery->numOfTags, tagColId); doSetTagValueInParam(tsdb, pTable, tagColId, &pRuntimeEnv->pCtx[0].tag, pColInfo->type, pColInfo->bytes); + + if (pRuntimeEnv->pCtx[0].tag.nType == TSDB_DATA_TYPE_BINARY || pRuntimeEnv->pCtx[0].tag.nType == TSDB_DATA_TYPE_NCHAR) {} qDebug("QInfo:%p set tag value for join comparison, colId:%" PRId64 ", val:%"PRId64, pQInfo, pExprInfo->base.arg->argValue.i64, - pRuntimeEnv->pCtx[0].tag.i64Key) + pRuntimeEnv->pCtx[0].tag.i64Key); + } else { + qDebug("QInfo:%p set tag value for join comparison, colId:%" PRId64 ", val:%s", pQInfo, pExprInfo->base.arg->argValue.i64, + pRuntimeEnv->pCtx[0].tag.pz); } } } @@ -3864,14 +3869,60 @@ int32_t setAdditionalInfo(SQInfo *pQInfo, void* pTable, STableQueryInfo *pTableQ // both the master and supplement scan needs to set the correct ts comp start position if (pRuntimeEnv->pTSBuf != NULL) { - if (pTableQueryInfo->cur.vgroupIndex == -1) { - tVariantAssign(&pTableQueryInfo->tag, &pRuntimeEnv->pCtx[0].tag); - tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, 0, &pTableQueryInfo->tag); + tVariant* pTag = &pRuntimeEnv->pCtx[0].tag; - // keep the cursor info of current meter - pTableQueryInfo->cur = pRuntimeEnv->pTSBuf->cur; + if (pRuntimeEnv->cur.vgroupIndex == -1) { + STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, pQInfo->vgId, pTag); + // failed to find data with the specified tag value and vnodeId + if (elem.vnode < 0) { + if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { + qError("QInfo:%p failed to find tag:%s in ts_comp", pQInfo, pTag->pz); + } else { + qError("QInfo:%p failed to find tag:%"PRId64" in ts_comp", pQInfo, pTag->i64Key); + } + + return false; + } else { + STSCursor cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); + + if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { + qDebug("QInfo:%p find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->pz, + cur.blockIndex, cur.tsIndex); + } else { + qDebug("QInfo:%p find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->i64Key, + cur.blockIndex, cur.tsIndex); + } + } } else { - tsBufSetCursor(pRuntimeEnv->pTSBuf, &pTableQueryInfo->cur); + STSElem elem = tsBufGetElem(pRuntimeEnv->pTSBuf); + if (tVariantCompare(&elem.tag, &pRuntimeEnv->pCtx[0].tag) != 0) { + + STSElem elem1 = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, pQInfo->vgId, pTag); + // failed to find data with the specified tag value and vnodeId + if (elem1.vnode < 0) { + if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { + qError("QInfo:%p failed to find tag:%s in ts_comp", pQInfo, pTag->pz); + } else { + qError("QInfo:%p failed to find tag:%"PRId64" in ts_comp", pQInfo, pTag->i64Key); + } + return false; + } else { + STSCursor cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); + if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { + qDebug("QInfo:%p find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->pz, cur.blockIndex, cur.tsIndex); + } else { + qDebug("QInfo:%p find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->i64Key, cur.blockIndex, cur.tsIndex); + } + } + } else { + tsBufSetCursor(pRuntimeEnv->pTSBuf, &pRuntimeEnv->cur); + STSCursor cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); + if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { + qDebug("QInfo:%p continue scan ts_comp file, tag:%s blockIndex:%d, tsIndex:%d", pQInfo, pTag->pz, cur.blockIndex, cur.tsIndex); + } else { + qDebug("QInfo:%p continue scan ts_comp file, tag:%"PRId64" blockIndex:%d, tsIndex:%d", pQInfo, pTag->i64Key, cur.blockIndex, cur.tsIndex); + } + } } } @@ -4790,8 +4841,6 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { qDebug("QInfo:%p find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->i64Key, cur.blockIndex, cur.tsIndex); } - - } } else { STSElem elem = tsBufGetElem(pRuntimeEnv->pTSBuf); @@ -4823,7 +4872,6 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { qDebug("QInfo:%p continue scan ts_comp file, tag:%"PRId64" blockIndex:%d, tsIndex:%d", pQInfo, pTag->i64Key, cur.blockIndex, cur.tsIndex); } } - } } From 289c125bf8c804704164793658e11f1110f74312 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 24 Oct 2020 16:30:36 +0800 Subject: [PATCH 37/56] [td-1373] --- src/query/src/qExecutor.c | 66 +++++++++++++-------------------------- 1 file changed, 21 insertions(+), 45 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 9d921f5150..99fb384161 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -2631,11 +2631,11 @@ void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, void *pTable, void *tsdb) { doSetTagValueInParam(tsdb, pTable, tagColId, &pRuntimeEnv->pCtx[0].tag, pColInfo->type, pColInfo->bytes); if (pRuntimeEnv->pCtx[0].tag.nType == TSDB_DATA_TYPE_BINARY || pRuntimeEnv->pCtx[0].tag.nType == TSDB_DATA_TYPE_NCHAR) {} - qDebug("QInfo:%p set tag value for join comparison, colId:%" PRId64 ", val:%"PRId64, pQInfo, pExprInfo->base.arg->argValue.i64, - pRuntimeEnv->pCtx[0].tag.i64Key); - } else { qDebug("QInfo:%p set tag value for join comparison, colId:%" PRId64 ", val:%s", pQInfo, pExprInfo->base.arg->argValue.i64, pRuntimeEnv->pCtx[0].tag.pz); + } else { + qDebug("QInfo:%p set tag value for join comparison, colId:%" PRId64 ", val:%"PRId64, pQInfo, pExprInfo->base.arg->argValue.i64, + pRuntimeEnv->pCtx[0].tag.i64Key); } } } @@ -3871,58 +3871,25 @@ int32_t setAdditionalInfo(SQInfo *pQInfo, void* pTable, STableQueryInfo *pTableQ if (pRuntimeEnv->pTSBuf != NULL) { tVariant* pTag = &pRuntimeEnv->pCtx[0].tag; - if (pRuntimeEnv->cur.vgroupIndex == -1) { - STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, pQInfo->vgId, pTag); + if (pTableQueryInfo->cur.vgroupIndex == -1) { + tVariantAssign(&pTableQueryInfo->tag, pTag); + STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, pQInfo->vgId, &pTableQueryInfo->tag); + // failed to find data with the specified tag value and vnodeId if (elem.vnode < 0) { if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { qError("QInfo:%p failed to find tag:%s in ts_comp", pQInfo, pTag->pz); } else { - qError("QInfo:%p failed to find tag:%"PRId64" in ts_comp", pQInfo, pTag->i64Key); + qError("QInfo:%p failed to find tag:%" PRId64 " in ts_comp", pQInfo, pTag->i64Key); } return false; - } else { - STSCursor cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); - - if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { - qDebug("QInfo:%p find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->pz, - cur.blockIndex, cur.tsIndex); - } else { - qDebug("QInfo:%p find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->i64Key, - cur.blockIndex, cur.tsIndex); - } } + + // keep the cursor info of current meter + pTableQueryInfo->cur = pRuntimeEnv->pTSBuf->cur; } else { - STSElem elem = tsBufGetElem(pRuntimeEnv->pTSBuf); - if (tVariantCompare(&elem.tag, &pRuntimeEnv->pCtx[0].tag) != 0) { - - STSElem elem1 = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, pQInfo->vgId, pTag); - // failed to find data with the specified tag value and vnodeId - if (elem1.vnode < 0) { - if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { - qError("QInfo:%p failed to find tag:%s in ts_comp", pQInfo, pTag->pz); - } else { - qError("QInfo:%p failed to find tag:%"PRId64" in ts_comp", pQInfo, pTag->i64Key); - } - return false; - } else { - STSCursor cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); - if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { - qDebug("QInfo:%p find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->pz, cur.blockIndex, cur.tsIndex); - } else { - qDebug("QInfo:%p find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->i64Key, cur.blockIndex, cur.tsIndex); - } - } - } else { - tsBufSetCursor(pRuntimeEnv->pTSBuf, &pRuntimeEnv->cur); - STSCursor cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); - if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { - qDebug("QInfo:%p continue scan ts_comp file, tag:%s blockIndex:%d, tsIndex:%d", pQInfo, pTag->pz, cur.blockIndex, cur.tsIndex); - } else { - qDebug("QInfo:%p continue scan ts_comp file, tag:%"PRId64" blockIndex:%d, tsIndex:%d", pQInfo, pTag->i64Key, cur.blockIndex, cur.tsIndex); - } - } + tsBufSetCursor(pRuntimeEnv->pTSBuf, &pTableQueryInfo->cur); } } @@ -4830,6 +4797,7 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { qError("QInfo:%p failed to find tag:%"PRId64" in ts_comp", pQInfo, pTag->i64Key); } + tVariantDestroy(&elem.tag); return false; } else { STSCursor cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); @@ -4854,6 +4822,10 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { } else { qError("QInfo:%p failed to find tag:%"PRId64" in ts_comp", pQInfo, pTag->i64Key); } + + tVariantDestroy(&elem.tag); + tVariantDestroy(&elem1.tag); + return false; } else { STSCursor cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); @@ -4863,6 +4835,8 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { qDebug("QInfo:%p find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->i64Key, cur.blockIndex, cur.tsIndex); } } + + tVariantDestroy(&elem1.tag); } else { tsBufSetCursor(pRuntimeEnv->pTSBuf, &pRuntimeEnv->cur); STSCursor cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); @@ -4872,6 +4846,8 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { qDebug("QInfo:%p continue scan ts_comp file, tag:%"PRId64" blockIndex:%d, tsIndex:%d", pQInfo, pTag->i64Key, cur.blockIndex, cur.tsIndex); } } + + tVariantDestroy(&elem.tag); } } From f4be8aae0e563d74c56329dae99d01f29b2a30d1 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 24 Oct 2020 16:37:24 +0800 Subject: [PATCH 38/56] [td-1373] --- src/query/src/qExecutor.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 99fb384161..bd3a2ae1f4 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -3887,9 +3887,21 @@ int32_t setAdditionalInfo(SQInfo *pQInfo, void* pTable, STableQueryInfo *pTableQ } // keep the cursor info of current meter - pTableQueryInfo->cur = pRuntimeEnv->pTSBuf->cur; + pTableQueryInfo->cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); + if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { + qDebug("QInfo:%p find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->pz, pTableQueryInfo->cur->blockIndex, pTableQueryInfo->cur->tsIndex); + } else { + qDebug("QInfo:%p find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->i64Key, pTableQueryInfo->cur->blockIndex, pTableQueryInfo->cur->tsIndex); + } + } else { tsBufSetCursor(pRuntimeEnv->pTSBuf, &pTableQueryInfo->cur); + + if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { + qDebug("QInfo:%p find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->pz, pTableQueryInfo->cur->blockIndex, pTableQueryInfo->cur->tsIndex); + } else { + qDebug("QInfo:%p find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->i64Key, pTableQueryInfo->cur->blockIndex, pTableQueryInfo->cur->tsIndex); + } } } From 79906e6a99113e887250aa8c3d4d33fa48ca6f7e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 24 Oct 2020 16:39:17 +0800 Subject: [PATCH 39/56] [td-1373] --- src/query/src/qExecutor.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index bd3a2ae1f4..e6ae61e70a 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -3889,18 +3889,18 @@ int32_t setAdditionalInfo(SQInfo *pQInfo, void* pTable, STableQueryInfo *pTableQ // keep the cursor info of current meter pTableQueryInfo->cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { - qDebug("QInfo:%p find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->pz, pTableQueryInfo->cur->blockIndex, pTableQueryInfo->cur->tsIndex); + qDebug("QInfo:%p find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->pz, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); } else { - qDebug("QInfo:%p find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->i64Key, pTableQueryInfo->cur->blockIndex, pTableQueryInfo->cur->tsIndex); + qDebug("QInfo:%p find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->i64Key, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); } } else { tsBufSetCursor(pRuntimeEnv->pTSBuf, &pTableQueryInfo->cur); if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { - qDebug("QInfo:%p find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->pz, pTableQueryInfo->cur->blockIndex, pTableQueryInfo->cur->tsIndex); + qDebug("QInfo:%p find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->pz, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); } else { - qDebug("QInfo:%p find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->i64Key, pTableQueryInfo->cur->blockIndex, pTableQueryInfo->cur->tsIndex); + qDebug("QInfo:%p find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", pQInfo, pTag->i64Key, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); } } } From e8223aef4061b53247d8ea013d1a0f8200a88bea Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 24 Oct 2020 16:44:18 +0800 Subject: [PATCH 40/56] [td-1373] --- src/query/src/qExecutor.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index e6ae61e70a..47a5e1522f 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -1410,6 +1410,8 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS item->lastKey = (QUERY_IS_ASC_QUERY(pQuery)? pDataBlockInfo->window.ekey:pDataBlockInfo->window.skey) + step; } + item->cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); + // todo refactor: extract method for(int32_t i = 0; i < pQuery->numOfOutput; ++i) { if (pQuery->pSelectExpr[i].base.functionId != TSDB_FUNC_ARITHM) { From 6b49d26693110381549d9b3e18c47d82e42aab73 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 24 Oct 2020 16:45:09 +0800 Subject: [PATCH 41/56] [td-1373] --- src/query/src/qExecutor.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 47a5e1522f..a994f815f9 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -1410,7 +1410,9 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS item->lastKey = (QUERY_IS_ASC_QUERY(pQuery)? pDataBlockInfo->window.ekey:pDataBlockInfo->window.skey) + step; } - item->cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); + if (pRuntimeEnv->pTSBuf != NULL) { + item->cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); + } // todo refactor: extract method for(int32_t i = 0; i < pQuery->numOfOutput; ++i) { From 6532bced2b2d40b63140f28c53624414339b60d2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 24 Oct 2020 17:00:48 +0800 Subject: [PATCH 42/56] [td-1373] fix invalid write and memory leaks --- src/client/src/tscLocalMerge.c | 3 ++- src/query/src/qExecutor.c | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c index 44ccb2471a..18d72e2d1e 100644 --- a/src/client/src/tscLocalMerge.c +++ b/src/client/src/tscLocalMerge.c @@ -698,7 +698,8 @@ int32_t tscLocalReducerEnvCreate(SSqlObj *pSql, tExtMemBuffer ***pMemBuffer, tOr pg *= 2; } - size_t numOfSubs = pTableMetaInfo->vgroupList->numOfVgroups; + size_t numOfSubs = pSql->subState.numOfSub; + assert(numOfSubs <= pTableMetaInfo->vgroupList->numOfVgroups); for (int32_t i = 0; i < numOfSubs; ++i) { (*pMemBuffer)[i] = createExtMemBuffer(nBufferSizes, rlen, pg, pModel); (*pMemBuffer)[i]->flushModel = MULTIPLE_APPEND_MODEL; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index a994f815f9..4e7f065dec 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -3877,6 +3877,7 @@ int32_t setAdditionalInfo(SQInfo *pQInfo, void* pTable, STableQueryInfo *pTableQ if (pTableQueryInfo->cur.vgroupIndex == -1) { tVariantAssign(&pTableQueryInfo->tag, pTag); + STSElem elem = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, pQInfo->vgId, &pTableQueryInfo->tag); // failed to find data with the specified tag value and vnodeId @@ -3887,6 +3888,7 @@ int32_t setAdditionalInfo(SQInfo *pQInfo, void* pTable, STableQueryInfo *pTableQ qError("QInfo:%p failed to find tag:%" PRId64 " in ts_comp", pQInfo, pTag->i64Key); } + tVariantDestroy(&elem.tag); return false; } From 36ae97a0e53458b39a60d679b6dd2d8981186d48 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 24 Oct 2020 17:13:00 +0800 Subject: [PATCH 43/56] [td-1373] refactor --- src/client/src/tscSubquery.c | 8 +++----- src/query/inc/qTsbuf.h | 11 +++-------- src/query/src/qExecutor.c | 12 ++---------- src/query/src/qTsbuf.c | 6 +++--- 4 files changed, 11 insertions(+), 26 deletions(-) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index f4b6ea2c94..938ec09beb 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -96,7 +96,7 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ tscInfo("%" PRId64 ", tags:%"PRId64" \t %" PRId64 ", tags:%"PRId64, elem1.ts, elem1.tag.i64Key, elem2.ts, elem2.tag.i64Key); #endif - int32_t res = tVariantCompare(&elem1.tag, &elem2.tag); + int32_t res = tVariantCompare(elem1.tag, elem2.tag); if (res == -1 || (res == 0 && tsCompare(order, elem1.ts, elem2.ts))) { if (!tsBufNextPos(pSupporter1->pTSBuf)) { break; @@ -123,11 +123,9 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ win->ekey = elem1.ts; } - tsBufAppend(output1, elem1.vnode, &elem1.tag, (const char*)&elem1.ts, sizeof(elem1.ts)); - tsBufAppend(output2, elem2.vnode, &elem2.tag, (const char*)&elem2.ts, sizeof(elem2.ts)); + tsBufAppend(output1, elem1.vnode, elem1.tag, (const char*)&elem1.ts, sizeof(elem1.ts)); + tsBufAppend(output2, elem2.vnode, elem2.tag, (const char*)&elem2.ts, sizeof(elem2.ts)); - tVariantDestroy(&elem1.tag); - tVariantDestroy(&elem2.tag); } else { pLimit->offset -= 1; } diff --git a/src/query/inc/qTsbuf.h b/src/query/inc/qTsbuf.h index fe5de8b25a..6c2a955f47 100644 --- a/src/query/inc/qTsbuf.h +++ b/src/query/inc/qTsbuf.h @@ -35,16 +35,9 @@ typedef struct STSList { int32_t len; } STSList; -typedef struct STSRawBlock { - int32_t vnode; - int64_t tag; - TSKEY* ts; - int32_t len; -} STSRawBlock; - typedef struct STSElem { TSKEY ts; - tVariant tag; + tVariant* tag; int32_t vnode; } STSElem; @@ -84,6 +77,7 @@ typedef struct STSBuf { char path[PATH_MAX]; uint32_t fileSize; + // todo use array STSVnodeBlockInfoEx* pData; uint32_t numOfAlloc; uint32_t numOfVnodes; @@ -121,6 +115,7 @@ void tsBufFlush(STSBuf* pTSBuf); void tsBufResetPos(STSBuf* pTSBuf); STSElem tsBufGetElem(STSBuf* pTSBuf); + bool tsBufNextPos(STSBuf* pTSBuf); STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t vnodeId, tVariant* tag); diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 4e7f065dec..37394b9769 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -1201,7 +1201,7 @@ static int32_t doTSJoinFilter(SQueryRuntimeEnv *pRuntimeEnv, int32_t offset) { SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; // compare tag first - if (tVariantCompare(&pCtx[0].tag, &elem.tag) != 0) { + if (tVariantCompare(&pCtx[0].tag, elem.tag) != 0) { return TS_JOIN_TAG_NOT_EQUALS; } @@ -3888,7 +3888,6 @@ int32_t setAdditionalInfo(SQInfo *pQInfo, void* pTable, STableQueryInfo *pTableQ qError("QInfo:%p failed to find tag:%" PRId64 " in ts_comp", pQInfo, pTag->i64Key); } - tVariantDestroy(&elem.tag); return false; } @@ -4815,7 +4814,6 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { qError("QInfo:%p failed to find tag:%"PRId64" in ts_comp", pQInfo, pTag->i64Key); } - tVariantDestroy(&elem.tag); return false; } else { STSCursor cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); @@ -4830,7 +4828,7 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { } } else { STSElem elem = tsBufGetElem(pRuntimeEnv->pTSBuf); - if (tVariantCompare(&elem.tag, &pRuntimeEnv->pCtx[0].tag) != 0) { + if (tVariantCompare(elem.tag, &pRuntimeEnv->pCtx[0].tag) != 0) { STSElem elem1 = tsBufGetElemStartPos(pRuntimeEnv->pTSBuf, pQInfo->vgId, pTag); // failed to find data with the specified tag value and vnodeId @@ -4841,9 +4839,6 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { qError("QInfo:%p failed to find tag:%"PRId64" in ts_comp", pQInfo, pTag->i64Key); } - tVariantDestroy(&elem.tag); - tVariantDestroy(&elem1.tag); - return false; } else { STSCursor cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); @@ -4854,7 +4849,6 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { } } - tVariantDestroy(&elem1.tag); } else { tsBufSetCursor(pRuntimeEnv->pTSBuf, &pRuntimeEnv->cur); STSCursor cur = tsBufGetCursor(pRuntimeEnv->pTSBuf); @@ -4864,8 +4858,6 @@ static bool multiTableMultioutputHelper(SQInfo *pQInfo, int32_t index) { qDebug("QInfo:%p continue scan ts_comp file, tag:%"PRId64" blockIndex:%d, tsIndex:%d", pQInfo, pTag->i64Key, cur.blockIndex, cur.tsIndex); } } - - tVariantDestroy(&elem.tag); } } diff --git a/src/query/src/qTsbuf.c b/src/query/src/qTsbuf.c index 42b6e3f6e0..ad29cef5c2 100644 --- a/src/query/src/qTsbuf.c +++ b/src/query/src/qTsbuf.c @@ -702,7 +702,7 @@ STSElem tsBufGetElem(STSBuf* pTSBuf) { elem1.vnode = pTSBuf->pData[pCur->vgroupIndex].info.vnode; elem1.ts = *(TSKEY*)(pTSBuf->tsData.rawBuf + pCur->tsIndex * TSDB_KEYSIZE); - tVariantAssign(&elem1.tag, &pBlock->tag); + elem1.tag = &pBlock->tag; return elem1; } @@ -913,8 +913,8 @@ void tsBufDisplay(STSBuf* pTSBuf) { while (tsBufNextPos(pTSBuf)) { STSElem elem = tsBufGetElem(pTSBuf); - if (elem.tag.nType == TSDB_DATA_TYPE_BIGINT) { - printf("%d-%" PRId64 "-%" PRId64 "\n", elem.vnode, elem.tag.i64Key, elem.ts); + if (elem.tag->nType == TSDB_DATA_TYPE_BIGINT) { + printf("%d-%" PRId64 "-%" PRId64 "\n", elem.vnode, elem.tag->i64Key, elem.ts); } } From 5b23931687fa988588afc8dfe7e938c71d54c0e3 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 24 Oct 2020 23:07:12 +0800 Subject: [PATCH 44/56] [td-1739] --- src/common/inc/tglobal.h | 7 ++++-- src/common/src/tglobal.c | 32 ++++++++++++++++++------ src/inc/query.h | 1 - src/inc/taoserror.h | 1 + src/query/src/qExecutor.c | 51 +++++++++++++++++++++++++++++++++++++++ src/util/inc/tconfig.h | 2 +- 6 files changed, 82 insertions(+), 12 deletions(-) diff --git a/src/common/inc/tglobal.h b/src/common/inc/tglobal.h index 515115c323..4636eaac08 100644 --- a/src/common/inc/tglobal.h +++ b/src/common/inc/tglobal.h @@ -44,14 +44,17 @@ extern int32_t tsMaxShellConns; extern int32_t tsShellActivityTimer; extern uint32_t tsMaxTmrCtrl; extern float tsNumOfThreadsPerCore; -extern float tsRatioOfQueryThreads; +extern float tsRatioOfQueryThreads; // todo remove it extern int8_t tsDaylight; extern char tsTimezone[]; extern char tsLocale[]; -extern char tsCharset[]; // default encode string +extern char tsCharset[]; // default encode string extern int32_t tsEnableCoreFile; extern int32_t tsCompressMsgSize; +//query buffer management +extern int32_t tsQueryBufferSize; // maximum allowed usage buffer for each data node during query processing + // client extern int32_t tsTableMetaKeepTimer; extern int32_t tsMaxSQLStringLen; diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index c24ba490ba..09ccdacc94 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -45,14 +45,14 @@ int32_t tsEnableTelemetryReporting = 1; char tsEmail[TSDB_FQDN_LEN] = {0}; // common -int32_t tsRpcTimer = 1000; -int32_t tsRpcMaxTime = 600; // seconds; -int32_t tsMaxShellConns = 5000; +int32_t tsRpcTimer = 1000; +int32_t tsRpcMaxTime = 600; // seconds; +int32_t tsMaxShellConns = 5000; int32_t tsMaxConnections = 5000; -int32_t tsShellActivityTimer = 3; // second -float tsNumOfThreadsPerCore = 1.0; -float tsRatioOfQueryThreads = 0.5; -int8_t tsDaylight = 0; +int32_t tsShellActivityTimer = 3; // second +float tsNumOfThreadsPerCore = 1.0f; +float tsRatioOfQueryThreads = 0.5f; +int8_t tsDaylight = 0; char tsTimezone[TSDB_TIMEZONE_LEN] = {0}; char tsLocale[TSDB_LOCALE_LEN] = {0}; char tsCharset[TSDB_LOCALE_LEN] = {0}; // default encode string @@ -99,6 +99,12 @@ float tsStreamComputDelayRatio = 0.1f; int32_t tsProjectExecInterval = 10000; // every 10sec, the projection will be executed once int64_t tsMaxRetentWindow = 24 * 3600L; // maximum time window tolerance +// the maximum allowed query buffer size during query processing for each data node. +// -1 no limit (default) +// 0 no query allowed, queries are disabled +// positive value (in MB) +int32_t tsQueryBufferSize = -1; + // db parameters int32_t tsCacheBlockSize = TSDB_DEFAULT_CACHE_BLOCK_SIZE; int32_t tsBlocksPerVnode = TSDB_DEFAULT_TOTAL_BLOCKS; @@ -676,7 +682,7 @@ static void doInitGlobalConfig(void) { cfg.minValue = TSDB_MIN_CACHE_BLOCK_SIZE; cfg.maxValue = TSDB_MAX_CACHE_BLOCK_SIZE; cfg.ptrLength = 0; - cfg.unitType = TAOS_CFG_UTYPE_Mb; + cfg.unitType = TAOS_CFG_UTYPE_MB; taosInitConfigOption(cfg); cfg.option = "blocks"; @@ -839,6 +845,16 @@ static void doInitGlobalConfig(void) { cfg.unitType = TAOS_CFG_UTYPE_NONE; taosInitConfigOption(cfg); + cfg.option = "queryBufferSize"; + cfg.ptr = &tsQueryBufferSize; + cfg.valType = TAOS_CFG_VTYPE_INT32; + cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; + cfg.minValue = -1; + cfg.maxValue = 10000000000000; //10TB + cfg.ptrLength = 0; + cfg.unitType = TAOS_CFG_UTYPE_BYTE; + taosInitConfigOption(cfg); + // locale & charset cfg.option = "timezone"; cfg.ptr = tsTimezone; diff --git a/src/inc/query.h b/src/inc/query.h index 0c18f85dc3..5e1de77889 100644 --- a/src/inc/query.h +++ b/src/inc/query.h @@ -78,7 +78,6 @@ int32_t qKillQuery(qinfo_t qinfo); int32_t qQueryCompleted(qinfo_t qinfo); - /** * destroy query info structure * @param qHandle diff --git a/src/inc/taoserror.h b/src/inc/taoserror.h index bb111d2da0..e64b797450 100644 --- a/src/inc/taoserror.h +++ b/src/inc/taoserror.h @@ -230,6 +230,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_READY, 0, 0x0707, "Query not TAOS_DEFINE_ERROR(TSDB_CODE_QRY_HAS_RSP, 0, 0x0708, "Query should response") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_IN_EXEC, 0, 0x0709, "Multiple retrieval of this query") TAOS_DEFINE_ERROR(TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW, 0, 0x070A, "Too many time window in query") +TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_ENOUGH_BUFFER, 0, 0x070B, "Query buffer limit has reached") // grant TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_EXPIRED, 0, 0x0800, "License expired") diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 37394b9769..0ed47ff66a 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -194,6 +194,8 @@ static void buildTagQueryResult(SQInfo *pQInfo); static int32_t setAdditionalInfo(SQInfo *pQInfo, void *pTable, STableQueryInfo *pTableQueryInfo); static int32_t flushFromResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo); +static int32_t checkForQueryBuf(int32_t numOfTables); +static void releaseQueryBuf(int32_t numOfTables); bool doFilterData(SQuery *pQuery, int32_t elemPos) { for (int32_t k = 0; k < pQuery->numOfFilterCols; ++k) { @@ -6492,6 +6494,8 @@ static void freeQInfo(SQInfo *pQInfo) { qDebug("QInfo:%p start to free QInfo", pQInfo); + releaseQueryBuf(pQInfo->tableqinfoGroupInfo.numOfTables); + teardownQueryRuntimeEnv(&pQInfo->runtimeEnv); SQuery *pQuery = pQInfo->runtimeEnv.pQuery; @@ -6726,6 +6730,11 @@ int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryMsg, qi assert(0); } + code = checkForQueryBuf(tableGroupInfo.numOfTables); + if (code != TSDB_CODE_SUCCESS) { // not enough query buffer, abort + goto _over; + } + (*pQInfo) = createQInfoImpl(pQueryMsg, pGroupbyExpr, pExprs, &tableGroupInfo, pTagColumnInfo, isSTableQuery); pExprs = NULL; pGroupbyExpr = NULL; @@ -7127,6 +7136,48 @@ static void buildTagQueryResult(SQInfo* pQInfo) { setQueryStatus(pQuery, QUERY_COMPLETED); } +static int64_t getQuerySupportBufSize(int32_t numOfTables) { + size_t s1 = sizeof(STableQueryInfo); + size_t s2 = sizeof(SHashNode); + +// size_t s3 = sizeof(STableCheckInfo); buffer consumption in tsdb + return (s1 + s2) * 1.5 * numOfTables; +} + +int32_t checkForQueryBuf(int32_t numOfTables) { + int64_t t = getQuerySupportBufSize(numOfTables); + if (tsQueryBufferSize < 0) { + return true; + } else if (tsQueryBufferSize > 0) { + + while(1) { + int64_t s = tsQueryBufferSize; + int64_t remain = s - t; + if (remain >= 0) { + if (atomic_val_compare_exchange_64(&tsQueryBufferSize, s, remain) == s) { + return TSDB_CODE_SUCCESS; + } + } else { + return TSDB_CODE_QRY_NOT_ENOUGH_BUFFER; + } + } + } + + // disable query processing if the value of tsQueryBufferSize is zero. + return TSDB_CODE_QRY_NOT_ENOUGH_BUFFER; +} + +void releaseQueryBuf(int32_t numOfTables) { + if (tsQueryBufferSize <= 0) { + return; + } + + int64_t t = getQuerySupportBufSize(numOfTables); + + // restore value is not enough buffer available + atomic_add_fetch_64(&tsQueryBufferSize, t); +} + void* qGetResultRetrieveMsg(qinfo_t qinfo) { SQInfo* pQInfo = (SQInfo*) qinfo; assert(pQInfo != NULL); diff --git a/src/util/inc/tconfig.h b/src/util/inc/tconfig.h index 0520cf29a8..33819f6a20 100644 --- a/src/util/inc/tconfig.h +++ b/src/util/inc/tconfig.h @@ -53,7 +53,7 @@ enum { TAOS_CFG_UTYPE_NONE, TAOS_CFG_UTYPE_PERCENT, TAOS_CFG_UTYPE_GB, - TAOS_CFG_UTYPE_Mb, + TAOS_CFG_UTYPE_MB, TAOS_CFG_UTYPE_BYTE, TAOS_CFG_UTYPE_SECOND, TAOS_CFG_UTYPE_MS From 00cc76aed969e0d24fc1a779aecc40fc21c76157 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 26 Oct 2020 14:42:56 +0800 Subject: [PATCH 45/56] [td-1569] --- src/util/src/tcache.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index 6e20c1708d..d7430f08ac 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -335,7 +335,7 @@ void *taosCacheTransfer(SCacheObj *pCacheObj, void **data) { } void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { - if (pCacheObj == NULL || taosHashGetSize(pCacheObj->pHashTable) + pCacheObj->numOfElemsInTrash == 0) { + if (pCacheObj == NULL) { return; } @@ -343,7 +343,12 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { uError("cache:%s, NULL data to release", pCacheObj->name); return; } - + + + // The operation of removal from hash table and addition to trashcan is not an atomic operation, + // therefore the check for the empty of both the hash table and the trashcan has a race condition. + // It happens when there is only one object in the cache, and two threads which has referenced this object + // start to free the it simultaneously. [TD-1569] size_t offset = offsetof(SCacheDataNode, data); SCacheDataNode *pNode = (SCacheDataNode *)((char *)(*data) - offset); From b1c2ff74b726fcafd24f0b1df50e91a4f15b0c3b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 26 Oct 2020 14:44:56 +0800 Subject: [PATCH 46/56] [td-1569] --- src/util/src/tcache.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/src/tcache.c b/src/util/src/tcache.c index d7430f08ac..5be7253f6d 100644 --- a/src/util/src/tcache.c +++ b/src/util/src/tcache.c @@ -348,7 +348,7 @@ void taosCacheRelease(SCacheObj *pCacheObj, void **data, bool _remove) { // The operation of removal from hash table and addition to trashcan is not an atomic operation, // therefore the check for the empty of both the hash table and the trashcan has a race condition. // It happens when there is only one object in the cache, and two threads which has referenced this object - // start to free the it simultaneously. [TD-1569] + // start to free the it simultaneously [TD-1569]. size_t offset = offsetof(SCacheDataNode, data); SCacheDataNode *pNode = (SCacheDataNode *)((char *)(*data) - offset); From 0f68ec7387417e0cb2fd13c376f425141929ae58 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 26 Oct 2020 14:54:34 +0800 Subject: [PATCH 47/56] [td-225] fix compiler error. --- src/query/tests/tsBufTest.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/query/tests/tsBufTest.cpp b/src/query/tests/tsBufTest.cpp index f22f61e890..8cd3a9cbef 100644 --- a/src/query/tests/tsBufTest.cpp +++ b/src/query/tests/tsBufTest.cpp @@ -304,7 +304,7 @@ void TSTraverse() { int32_t totalOutput = 10; while (1) { STSElem elem = tsBufGetElem(pTSBuf); - printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag.i64Key, elem.ts); + printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag->i64Key, elem.ts); if (!tsBufNextPos(pTSBuf)) { break; @@ -352,7 +352,7 @@ void TSTraverse() { totalOutput = 10; while (1) { STSElem elem = tsBufGetElem(pTSBuf); - printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag.i64Key, elem.ts); + printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag->i64Key, elem.ts); if (!tsBufNextPos(pTSBuf)) { break; @@ -480,7 +480,7 @@ void mergeIdenticalVnodeBufferTest() { STSElem elem = tsBufGetElem(pTSBuf1); EXPECT_EQ(elem.vnode, 12); - printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag.i64Key, elem.ts); + printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag->i64Key, elem.ts); } tsBufDestroy(pTSBuf1); From 5ce1ff42e38f68c3ff37badf61e3057dc473cfa8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 26 Oct 2020 15:26:15 +0800 Subject: [PATCH 48/56] [td-225] fix compiler error. --- src/common/src/tglobal.c | 2 +- src/query/src/qExecutor.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index 09ccdacc94..c067159b3b 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -850,7 +850,7 @@ static void doInitGlobalConfig(void) { cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.minValue = -1; - cfg.maxValue = 10000000000000; //10TB + cfg.maxValue = 10000000000000.0; //10TB cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_BYTE; taosInitConfigOption(cfg); diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index c53e0b4e45..3200dfd893 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -7147,7 +7147,7 @@ static int64_t getQuerySupportBufSize(int32_t numOfTables) { int32_t checkForQueryBuf(int32_t numOfTables) { int64_t t = getQuerySupportBufSize(numOfTables); if (tsQueryBufferSize < 0) { - return true; + return TSDB_CODE_SUCCESS; } else if (tsQueryBufferSize > 0) { while(1) { From afa66c6b77160efd5641a1915da237fddbf107d6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 26 Oct 2020 15:32:23 +0800 Subject: [PATCH 49/56] [td-225] fix compiler error. --- src/common/src/tglobal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index c067159b3b..e4d7d5af70 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -850,7 +850,7 @@ static void doInitGlobalConfig(void) { cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.minValue = -1; - cfg.maxValue = 10000000000000.0; //10TB + cfg.maxValue = 500000000000.0; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_BYTE; taosInitConfigOption(cfg); From 2e9fbee43c77ee19244fbd261253c4dd9b29fc20 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 26 Oct 2020 15:54:50 +0800 Subject: [PATCH 50/56] [td-225] fix compiler error. --- src/client/src/tscUtil.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index b42b4c13b4..46bb8570bb 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1722,7 +1722,7 @@ SArray* tscCloneVgroupTableInfo(SArray* pVgroupTables) { for (size_t i = 0; i < num; i++) { SVgroupTableInfo* pInfo = taosArrayGet(pVgroupTables, i); - SVgroupTableInfo info = {0}; + SVgroupTableInfo info = {{.vgId = 0, .epAddr = {0}, .numOfEps = 0}, .itemList = NULL}; info.vgInfo = pInfo->vgInfo; for(int32_t j = 0; j < pInfo->vgInfo.numOfEps; ++j) { From c5087aa44455db873ad2c48ad5d1d26918078fa8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 26 Oct 2020 16:06:23 +0800 Subject: [PATCH 51/56] [td-225] fix compiler error. --- src/client/src/tscUtil.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 46bb8570bb..ccaa7017e4 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1722,14 +1722,14 @@ SArray* tscCloneVgroupTableInfo(SArray* pVgroupTables) { for (size_t i = 0; i < num; i++) { SVgroupTableInfo* pInfo = taosArrayGet(pVgroupTables, i); - SVgroupTableInfo info = {{.vgId = 0, .epAddr = {0}, .numOfEps = 0}, .itemList = NULL}; - info.vgInfo = pInfo->vgInfo; + SVgroupTableInfo info; + memset(&info, 0, sizeof(SVgroupTableInfo)); + info.vgInfo = pInfo->vgInfo; for(int32_t j = 0; j < pInfo->vgInfo.numOfEps; ++j) { info.vgInfo.epAddr[j].fqdn = strdup(pInfo->vgInfo.epAddr[j].fqdn); } - info.itemList = taosArrayClone(pInfo->itemList); taosArrayPush(pa, &info); } From c615ca4ce6627cb1071f1afff6a2e35248ab0adc Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 26 Oct 2020 16:06:53 +0800 Subject: [PATCH 52/56] [td-225] fix compiler error. --- src/client/src/tscUtil.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index ccaa7017e4..f99adfffea 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1719,10 +1719,9 @@ SArray* tscCloneVgroupTableInfo(SArray* pVgroupTables) { size_t num = taosArrayGetSize(pVgroupTables); SArray* pa = taosArrayInit(num, sizeof(SVgroupTableInfo)); + SVgroupTableInfo info; for (size_t i = 0; i < num; i++) { SVgroupTableInfo* pInfo = taosArrayGet(pVgroupTables, i); - - SVgroupTableInfo info; memset(&info, 0, sizeof(SVgroupTableInfo)); info.vgInfo = pInfo->vgInfo; From e53c43356912061ec12ee6858f1c7236cb7ba1df Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 26 Oct 2020 16:41:32 +0800 Subject: [PATCH 53/56] [td-225] fix compiler error. --- src/common/src/tglobal.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/common/src/tglobal.c b/src/common/src/tglobal.c index e4d7d5af70..32569e3982 100644 --- a/src/common/src/tglobal.c +++ b/src/common/src/tglobal.c @@ -850,7 +850,7 @@ static void doInitGlobalConfig(void) { cfg.valType = TAOS_CFG_VTYPE_INT32; cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW; cfg.minValue = -1; - cfg.maxValue = 500000000000.0; + cfg.maxValue = 500000000000.0f; cfg.ptrLength = 0; cfg.unitType = TAOS_CFG_UTYPE_BYTE; taosInitConfigOption(cfg); From fc40984a691fd07feaec3329f3a2e104ae382d37 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 26 Oct 2020 16:44:44 +0800 Subject: [PATCH 54/56] [td-225] fix compiler error. --- src/query/src/qExecutor.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 3200dfd893..80f396544a 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -6494,7 +6494,7 @@ static void freeQInfo(SQInfo *pQInfo) { qDebug("QInfo:%p start to free QInfo", pQInfo); - releaseQueryBuf(pQInfo->tableqinfoGroupInfo.numOfTables); + releaseQueryBuf((int32_t) pQInfo->tableqinfoGroupInfo.numOfTables); teardownQueryRuntimeEnv(&pQInfo->runtimeEnv); From fcf59dd33d1df26096fbaefc412ff006227bde10 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 26 Oct 2020 16:49:02 +0800 Subject: [PATCH 55/56] [td-225] fix compiler error. --- src/query/src/qExecutor.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 80f396544a..3df319a5c2 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -194,8 +194,8 @@ static void buildTagQueryResult(SQInfo *pQInfo); static int32_t setAdditionalInfo(SQInfo *pQInfo, void *pTable, STableQueryInfo *pTableQueryInfo); static int32_t flushFromResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo); -static int32_t checkForQueryBuf(int32_t numOfTables); -static void releaseQueryBuf(int32_t numOfTables); +static int32_t checkForQueryBuf(size_t numOfTables); +static void releaseQueryBuf(size_t numOfTables); bool doFilterData(SQuery *pQuery, int32_t elemPos) { for (int32_t k = 0; k < pQuery->numOfFilterCols; ++k) { @@ -6494,7 +6494,7 @@ static void freeQInfo(SQInfo *pQInfo) { qDebug("QInfo:%p start to free QInfo", pQInfo); - releaseQueryBuf((int32_t) pQInfo->tableqinfoGroupInfo.numOfTables); + releaseQueryBuf(pQInfo->tableqinfoGroupInfo.numOfTables); teardownQueryRuntimeEnv(&pQInfo->runtimeEnv); @@ -7136,15 +7136,15 @@ static void buildTagQueryResult(SQInfo* pQInfo) { setQueryStatus(pQuery, QUERY_COMPLETED); } -static int64_t getQuerySupportBufSize(int32_t numOfTables) { +static int64_t getQuerySupportBufSize(size_t numOfTables) { size_t s1 = sizeof(STableQueryInfo); size_t s2 = sizeof(SHashNode); // size_t s3 = sizeof(STableCheckInfo); buffer consumption in tsdb - return (s1 + s2) * 1.5 * numOfTables; + return (int64_t)((s1 + s2) * 1.5 * numOfTables); } -int32_t checkForQueryBuf(int32_t numOfTables) { +int32_t checkForQueryBuf(size_t numOfTables) { int64_t t = getQuerySupportBufSize(numOfTables); if (tsQueryBufferSize < 0) { return TSDB_CODE_SUCCESS; @@ -7167,7 +7167,7 @@ int32_t checkForQueryBuf(int32_t numOfTables) { return TSDB_CODE_QRY_NOT_ENOUGH_BUFFER; } -void releaseQueryBuf(int32_t numOfTables) { +void releaseQueryBuf(size_t numOfTables) { if (tsQueryBufferSize <= 0) { return; } From 57f15cfcdfa1e3f83808a6ec7d19b535355d9564 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 26 Oct 2020 17:15:35 +0800 Subject: [PATCH 56/56] compile error in windows --- src/client/src/tscFunctionImpl.c | 6 +++--- src/client/src/tscSubquery.c | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 7a668e85bf..74ddcbd3fd 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -4025,11 +4025,11 @@ static void ts_comp_function(SQLFunctionCtx *pCtx) { // primary ts must be existed, so no need to check its existance if (pCtx->order == TSDB_ORDER_ASC) { - tsBufAppend(pTSbuf, pCtx->param[0].i64Key, &pCtx->tag, input, pCtx->size * TSDB_KEYSIZE); + tsBufAppend(pTSbuf, (int32_t)pCtx->param[0].i64Key, &pCtx->tag, input, pCtx->size * TSDB_KEYSIZE); } else { for (int32_t i = pCtx->size - 1; i >= 0; --i) { char *d = GET_INPUT_CHAR_INDEX(pCtx, i); - tsBufAppend(pTSbuf, pCtx->param[0].i64Key, &pCtx->tag, d, TSDB_KEYSIZE); + tsBufAppend(pTSbuf, (int32_t)pCtx->param[0].i64Key, &pCtx->tag, d, (int32_t)TSDB_KEYSIZE); } } @@ -4048,7 +4048,7 @@ static void ts_comp_function_f(SQLFunctionCtx *pCtx, int32_t index) { STSBuf *pTSbuf = pInfo->pTSBuf; - tsBufAppend(pTSbuf, pCtx->param[0].i64Key, &pCtx->tag, pData, TSDB_KEYSIZE); + tsBufAppend(pTSbuf, (int32_t)pCtx->param[0].i64Key, &pCtx->tag, pData, TSDB_KEYSIZE); SET_VAL(pCtx, pCtx->size, 1); pResInfo->hasResult = DATA_SET_FLAG; diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index cece5e4f5c..b2390dfe30 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -924,7 +924,7 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR // for projection query, need to try next vnode if current vnode is exhausted int32_t numOfVgroups = 0; // TODO refactor if (pTableMetaInfo->pVgroupTables != NULL) { - numOfVgroups = taosArrayGetSize(pTableMetaInfo->pVgroupTables); + numOfVgroups = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables); } else { numOfVgroups = pTableMetaInfo->vgroupList->numOfVgroups; } @@ -1054,7 +1054,7 @@ void tscFetchDatablockFromSubquery(SSqlObj* pSql) { // for projection query, need to try next vnode if current vnode is exhausted int32_t numOfVgroups = 0; // TODO refactor if (pTableMetaInfo->pVgroupTables != NULL) { - numOfVgroups = taosArrayGetSize(pTableMetaInfo->pVgroupTables); + numOfVgroups = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables); } else { numOfVgroups = pTableMetaInfo->vgroupList->numOfVgroups; } @@ -1510,7 +1510,7 @@ int32_t tscHandleMasterSTableQuery(SSqlObj *pSql) { if (pTableMetaInfo->pVgroupTables == NULL) { pState->numOfSub = pTableMetaInfo->vgroupList->numOfVgroups; } else { - pState->numOfSub = taosArrayGetSize(pTableMetaInfo->pVgroupTables); + pState->numOfSub = (int32_t)taosArrayGetSize(pTableMetaInfo->pVgroupTables); } assert(pState->numOfSub > 0); @@ -2160,7 +2160,7 @@ static void doBuildResFromSubqueries(SSqlObj* pSql) { continue; } - int32_t remain = pSub->res.numOfRows - pSub->res.row; + int32_t remain = (int32_t)(pSub->res.numOfRows - pSub->res.row); numOfRes = (int32_t)(MIN(numOfRes, remain)); } @@ -2222,7 +2222,7 @@ void tscBuildResFromSubqueries(SSqlObj *pSql) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex); size_t numOfExprs = tscSqlExprNumOfExprs(pQueryInfo); - pRes->numOfCols = numOfExprs; + pRes->numOfCols = (int32_t)numOfExprs; pRes->tsrow = calloc(numOfExprs, POINTER_BYTES); pRes->buffer = calloc(numOfExprs, POINTER_BYTES);