From 93c666bbb936e1a96f57b1bfa79210076151b69b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 30 Oct 2020 16:12:17 +0800 Subject: [PATCH 01/35] [TD-225] refactor codes. --- src/client/inc/tschemautil.h | 3 -- src/client/inc/tsclient.h | 8 --- src/client/src/tscLocal.c | 89 ++-------------------------------- src/client/src/tscSchemaUtil.c | 22 --------- src/client/src/tscServer.c | 2 +- src/client/src/tscUtil.c | 48 ++++++------------ src/query/inc/qExecutor.h | 2 +- src/query/inc/qSqlparser.h | 7 --- 8 files changed, 22 insertions(+), 159 deletions(-) diff --git a/src/client/inc/tschemautil.h b/src/client/inc/tschemautil.h index e5bdcecfa8..f6dc45398f 100644 --- a/src/client/inc/tschemautil.h +++ b/src/client/inc/tschemautil.h @@ -107,9 +107,6 @@ SSchema tscGetTbnameColumnSchema(); */ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size); -//todo tags value as well as the table id structure needs refactor -char *tsGetTagsValue(STableMeta *pMeta); - #ifdef __cplusplus } #endif diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index bbd029d675..d21179df3d 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -431,14 +431,6 @@ void tscResetSqlCmdObj(SSqlCmd *pCmd, bool removeFromCache); */ void tscFreeSqlResult(SSqlObj *pSql); -/** - * only free part of resources allocated during query. - * TODO remove it later - * Note: this function is multi-thread safe. - * @param pObj - */ -void tscPartiallyFreeSqlObj(SSqlObj *pSql); - /** * free sql object, release allocated resource * @param pObj diff --git a/src/client/src/tscLocal.c b/src/client/src/tscLocal.c index b4c3f3549b..44dffab56f 100644 --- a/src/client/src/tscLocal.c +++ b/src/client/src/tscLocal.c @@ -49,82 +49,6 @@ typedef struct SCreateBuilder { } SCreateBuilder; static void tscSetLocalQueryResult(SSqlObj *pSql, const char *val, const char *columnName, int16_t type, size_t valueLength); -static int32_t getToStringLength(const char *pData, int32_t length, int32_t type) { - char buf[512] = {0}; - - int32_t len = 0; - int32_t MAX_BOOL_TYPE_LENGTH = 5; // max(strlen("true"), strlen("false")); - switch (type) { - case TSDB_DATA_TYPE_BINARY: - return length; - case TSDB_DATA_TYPE_NCHAR: - return length; - case TSDB_DATA_TYPE_DOUBLE: { - double dv = 0; - dv = GET_DOUBLE_VAL(pData); - len = sprintf(buf, "%lf", dv); - if (strncasecmp("nan", buf, 3) == 0) { - len = 4; - } - } break; - case TSDB_DATA_TYPE_FLOAT: { - float fv = 0; - fv = GET_FLOAT_VAL(pData); - len = sprintf(buf, "%f", fv); - if (strncasecmp("nan", buf, 3) == 0) { - len = 4; - } - } break; - case TSDB_DATA_TYPE_TIMESTAMP: - case TSDB_DATA_TYPE_BIGINT: - len = sprintf(buf, "%" PRId64, *(int64_t *)pData); - break; - case TSDB_DATA_TYPE_BOOL: - len = MAX_BOOL_TYPE_LENGTH; - break; - default: - len = sprintf(buf, "%d", *(int32_t *)pData); - break; - }; - return len; -} - -/* - * we need to convert all data into string, so we need to sprintf all kinds of - * non-string data into string, and record its length to get the right - * maximum length. The length may be less or greater than its original binary length: - * For example: - * length((short) 1) == 1, less than sizeof(short) - * length((uint64_t) 123456789011) > 12, greater than sizsof(uint64_t) - */ -static int32_t tscMaxLengthOfTagsFields(SSqlObj *pSql) { - STableMeta *pMeta = tscGetTableMetaInfoFromCmd(&pSql->cmd, 0, 0)->pTableMeta; - - if (pMeta->tableType == TSDB_SUPER_TABLE || pMeta->tableType == TSDB_NORMAL_TABLE || - pMeta->tableType == TSDB_STREAM_TABLE) { - return 0; - } - - char * pTagValue = tsGetTagsValue(pMeta); - SSchema *pTagsSchema = tscGetTableTagSchema(pMeta); - - int32_t len = getToStringLength(pTagValue, pTagsSchema[0].bytes, pTagsSchema[0].type); - - pTagValue += pTagsSchema[0].bytes; - int32_t numOfTags = tscGetNumOfTags(pMeta); - - for (int32_t i = 1; i < numOfTags; ++i) { - int32_t tLen = getToStringLength(pTagValue, pTagsSchema[i].bytes, pTagsSchema[i].type); - if (len < tLen) { - len = tLen; - } - - pTagValue += pTagsSchema[i].bytes; - } - - return len; -} - static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) { SSqlRes *pRes = &pSql->res; @@ -186,8 +110,7 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) { return 0; } - // the following is handle display tags value for meters created according to metric - char *pTagValue = tsGetTagsValue(pMeta); + // the following is handle display tags for table created according to super table for (int32_t i = numOfRows; i < totalNumOfRows; ++i) { // field name TAOS_FIELD *pField = tscFieldInfoGetField(&pQueryInfo->fieldsInfo, 0); @@ -219,8 +142,6 @@ static int32_t tscSetValueToResObj(SSqlObj *pSql, int32_t rowLen) { char *target = pRes->data + tscFieldInfoGetOffset(pQueryInfo, 3) * totalNumOfRows + pField->bytes * i; const char *src = "TAG"; STR_WITH_MAXSIZE_TO_VARSTR(target, src, pField->bytes); - - pTagValue += pSchema[i].bytes; } return 0; @@ -286,10 +207,10 @@ static int32_t tscProcessDescribeTable(SSqlObj *pSql) { const int32_t TYPE_COLUMN_LENGTH = 16; const int32_t NOTE_COLUMN_MIN_LENGTH = 8; - int32_t noteFieldLen = tscMaxLengthOfTagsFields(pSql); - if (noteFieldLen == 0) { - noteFieldLen = NOTE_COLUMN_MIN_LENGTH; - } + int32_t noteFieldLen = NOTE_COLUMN_MIN_LENGTH;//tscMaxLengthOfTagsFields(pSql); +// if (noteFieldLen == 0) { +// noteFieldLen = NOTE_COLUMN_MIN_LENGTH; +// } int32_t rowLen = tscBuildTableSchemaResultFields(pSql, NUM_OF_DESC_TABLE_COLUMNS, TYPE_COLUMN_LENGTH, noteFieldLen); tscFieldInfoUpdateOffset(pQueryInfo); diff --git a/src/client/src/tscSchemaUtil.c b/src/client/src/tscSchemaUtil.c index 2a9a9de84b..ea493d2e25 100644 --- a/src/client/src/tscSchemaUtil.c +++ b/src/client/src/tscSchemaUtil.c @@ -197,28 +197,6 @@ STableMeta* tscCreateTableMetaFromMsg(STableMetaMsg* pTableMetaMsg, size_t* size return pTableMeta; } -/** - * the TableMeta data format in memory is as follows: - * - * +--------------------+ - * |STableMeta Body data| sizeof(STableMeta) - * +--------------------+ - * |Schema data | numOfTotalColumns * sizeof(SSchema) - * +--------------------+ - * |Tags data | tag_col_1.bytes + tag_col_2.bytes + .... - * +--------------------+ - * - * @param pTableMeta - * @return - */ -char* tsGetTagsValue(STableMeta* pTableMeta) { - int32_t offset = 0; -// int32_t numOfTotalCols = pTableMeta->numOfColumns + pTableMeta->numOfTags; -// uint32_t offset = sizeof(STableMeta) + numOfTotalCols * sizeof(SSchema); - - return ((char*)pTableMeta + offset); -} - // todo refactor UNUSED_FUNC static FORCE_INLINE char* skipSegments(char* input, char delim, int32_t num) { for (int32_t i = 0; i < num; ++i) { diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 2e7a99f2fb..e678464932 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -886,7 +886,7 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { STSVnodeBlockInfo *pBlockInfo = tsBufGetVnodeBlockInfo(pQueryInfo->tsBuf, vnodeId); assert(QUERY_IS_JOIN_QUERY(pQueryInfo->type) && pBlockInfo != NULL); // this query should not be sent - // todo refactor + // todo refactor, extract method and put into tsBuf.c if (fseek(pQueryInfo->tsBuf->f, pBlockInfo->offset, SEEK_SET) != 0) { int code = TAOS_SYSTEM_ERROR(ferror(pQueryInfo->tsBuf->f)); tscError("%p: fseek failed: %s", pSql, tstrerror(code)); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index 117874475b..dca23e0dcb 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -338,34 +338,6 @@ void tscFreeSqlResult(SSqlObj* pSql) { memset(&pSql->res, 0, sizeof(SSqlRes)); } -void tscPartiallyFreeSqlObj(SSqlObj* pSql) { - if (pSql == NULL || pSql->signature != pSql) { - return; - } - - SSqlCmd* pCmd = &pSql->cmd; - int32_t cmd = pCmd->command; - if (cmd < TSDB_SQL_INSERT || cmd == TSDB_SQL_RETRIEVE_LOCALMERGE || cmd == TSDB_SQL_RETRIEVE_EMPTY_RESULT || - cmd == TSDB_SQL_TABLE_JOIN_RETRIEVE) { - tscRemoveFromSqlList(pSql); - } - - // pSql->sqlstr will be used by tscBuildQueryStreamDesc -// if (pObj->signature == pObj) { - //pthread_mutex_lock(&pObj->mutex); - taosTFree(pSql->sqlstr); - //pthread_mutex_unlock(&pObj->mutex); -// } - - tscFreeSqlResult(pSql); - - taosTFree(pSql->pSubs); - pSql->subState.numOfSub = 0; - pSql->self = 0; - - tscResetSqlCmdObj(pCmd, false); -} - static void tscFreeSubobj(SSqlObj* pSql) { if (pSql->subState.numOfSub == 0) { return; @@ -434,22 +406,32 @@ void tscFreeSqlObj(SSqlObj* pSql) { tscDebug("%p start to free sqlObj", pSql); pSql->res.code = TSDB_CODE_TSC_QUERY_CANCELLED; + tscFreeSubobj(pSql); - tscPartiallyFreeSqlObj(pSql); + SSqlCmd* pCmd = &pSql->cmd; + int32_t cmd = pCmd->command; + if (cmd < TSDB_SQL_INSERT || cmd == TSDB_SQL_RETRIEVE_LOCALMERGE || cmd == TSDB_SQL_RETRIEVE_EMPTY_RESULT || + cmd == TSDB_SQL_TABLE_JOIN_RETRIEVE) { + tscRemoveFromSqlList(pSql); + } pSql->signature = NULL; pSql->fp = NULL; - - SSqlCmd* pCmd = &pSql->cmd; + taosTFree(pSql->sqlstr); + + taosTFree(pSql->pSubs); + pSql->subState.numOfSub = 0; + pSql->self = 0; + + tscFreeSqlResult(pSql); + tscResetSqlCmdObj(pCmd, false); memset(pCmd->payload, 0, (size_t)pCmd->allocSize); taosTFree(pCmd->payload); pCmd->allocSize = 0; - taosTFree(pSql->sqlstr); tsem_destroy(&pSql->rspSem); - free(pSql); } diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index b474bea987..e5f0d2cb2e 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -94,7 +94,7 @@ typedef struct SSingleColumnFilterInfo { SColumnFilterElem* pFilters; } SSingleColumnFilterInfo; -typedef struct STableQueryInfo { // todo merge with the STableQueryInfo struct +typedef struct STableQueryInfo { TSKEY lastKey; int32_t groupIndex; // group id in table list int16_t queryRangeSet; // denote if the query range is set, only available for interval query diff --git a/src/query/inc/qSqlparser.h b/src/query/inc/qSqlparser.h index bc8f9a5e23..25da04710d 100644 --- a/src/query/inc/qSqlparser.h +++ b/src/query/inc/qSqlparser.h @@ -223,13 +223,6 @@ typedef struct tSQLExprList { tSQLExprItem *a; /* One entry for each expression */ } tSQLExprList; -typedef struct tSQLExprListList { - int32_t nList; /* Number of expressions on the list */ - int32_t nAlloc; /* Number of entries allocated below */ - tSQLExprList **a; /* one entry for each row */ -} tSQLExprListList; - - /** * * @param yyp The parser From db6f936c934776f733a777c2526dd5929f6a8b7a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 30 Oct 2020 16:32:18 +0800 Subject: [PATCH 02/35] [TD-225] free content while in-compatible client connect to taosd and send hb message. --- src/mnode/src/mnodeShow.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/mnode/src/mnodeShow.c b/src/mnode/src/mnodeShow.c index 655fe71259..d759352f71 100644 --- a/src/mnode/src/mnodeShow.c +++ b/src/mnode/src/mnodeShow.c @@ -239,6 +239,7 @@ static int32_t mnodeProcessHeartBeatMsg(SMnodeMsg *pMsg) { SCMHeartBeatMsg *pHBMsg = pMsg->rpcMsg.pCont; if (taosCheckVersion(pHBMsg->clientVer, version, 3) != TSDB_CODE_SUCCESS) { + rpcFreeCont(pRsp); return TSDB_CODE_TSC_INVALID_VERSION; // todo change the error code } From 80b2acff4b9248fbb60e716ce8859e58d08d716d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 30 Oct 2020 22:43:34 +0800 Subject: [PATCH 03/35] [TD-225] fix memory leaks. --- src/client/src/tscSubquery.c | 8 +++++--- src/common/src/tvariant.c | 2 +- src/query/src/qExecutor.c | 3 ++- src/query/src/qTsbuf.c | 13 +++++++------ 4 files changed, 15 insertions(+), 11 deletions(-) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index c747eb2fe8..902ff989c7 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -520,9 +520,10 @@ void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArr taosArrayPush(result, &info); } - tscDebug("%p tid:%d, uid:%"PRIu64",vgId:%d added for vnode query", pSql, tt->tid, tt->uid, tt->vgId) STableIdInfo item = {.uid = tt->uid, .tid = tt->tid, .key = INT64_MIN}; taosArrayPush(vgTables, &item); + + tscDebug("%p tid:%d, uid:%"PRIu64",vgId:%d added, total:%d", pSql, tt->tid, tt->uid, tt->vgId, (int32_t) taosArrayGetSize(vgTables)); prev = tt; } @@ -602,11 +603,11 @@ static bool checkForDuplicateTagVal(SSchema* pColSchema, SJoinSupporter* p1, SSq } static int32_t getIntersectionOfTableTuple(SQueryInfo* pQueryInfo, SSqlObj* pParentSql, SArray** s1, SArray** s2) { - tscDebug("%p all subqueries retrieve complete, do tags match", pParentSql); - SJoinSupporter* p1 = pParentSql->pSubs[0]->param; SJoinSupporter* p2 = pParentSql->pSubs[1]->param; + tscDebug("%p all subquery retrieve complete, do tags match, %d, %d", pParentSql, p1->num, p2->num); + // sort according to the tag value qsort(p1->pIdTagList, p1->num, p1->tagSize, tagValCompar); qsort(p2->pIdTagList, p2->num, p2->tagSize, tagValCompar); @@ -655,6 +656,7 @@ static int32_t getIntersectionOfTableTuple(SQueryInfo* pQueryInfo, SSqlObj* pPar qsort((*s1)->pData, t1, size, tidTagsCompar); qsort((*s2)->pData, t2, size, tidTagsCompar); + tscDebug("%p tags match complete, result: %"PRId64", %"PRId64, pParentSql, t1, t2); return TSDB_CODE_SUCCESS; } diff --git a/src/common/src/tvariant.c b/src/common/src/tvariant.c index 9eb9924932..ca1644c0a2 100644 --- a/src/common/src/tvariant.c +++ b/src/common/src/tvariant.c @@ -108,7 +108,7 @@ void tVariantCreateFromBinary(tVariant *pVar, const char *pz, size_t len, uint32 break; } case TSDB_DATA_TYPE_BINARY: { // todo refactor, extract a method - pVar->pz = calloc(len, sizeof(char)); + pVar->pz = calloc(len + 1, sizeof(char)); memcpy(pVar->pz, pz, len); pVar->nLen = (int32_t)len; break; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 7a7c1ee2d0..3e79d2c9b9 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -3762,6 +3762,7 @@ void destroyTableQueryInfoImpl(STableQueryInfo *pTableQueryInfo) { return; } + tVariantDestroy(&pTableQueryInfo->tag); cleanupTimeWindowInfo(&pTableQueryInfo->windowResInfo); } @@ -6345,7 +6346,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou for(int32_t j = 0; j < s; ++j) { STableKeyInfo* info = taosArrayGet(pa, j); - void* buf = (char*)pQInfo->pBuf + index * sizeof(STableQueryInfo); + void* buf = (char*) pQInfo->pBuf + index * sizeof(STableQueryInfo); window.skey = info->lastKey; STableQueryInfo* item = createTableQueryInfo(&pQInfo->runtimeEnv, info->pTable, window, buf); diff --git a/src/query/src/qTsbuf.c b/src/query/src/qTsbuf.c index ad29cef5c2..ed7fa16cc4 100644 --- a/src/query/src/qTsbuf.c +++ b/src/query/src/qTsbuf.c @@ -156,7 +156,8 @@ void* tsBufDestroy(STSBuf* pTSBuf) { } else { // tscDebug("tsBuf %p destroyed, tmp file:%s, remains", pTSBuf, pTSBuf->path); } - + + tVariantDestroy(&pTSBuf->block.tag); free(pTSBuf); return NULL; } @@ -284,12 +285,12 @@ static void expandBuffer(STSList* ptsData, int32_t inputSize) { STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { STSBlock* pBlock = &pTSBuf->block; - + // clear the memory buffer - void* tmp = pBlock->payload; - memset(pBlock, 0, sizeof(STSBlock)); - pBlock->payload = tmp; - + pBlock->compLen = 0; + pBlock->padding = 0; + pBlock->numOfElem = 0; + if (order == TSDB_ORDER_DESC) { /* * set the right position for the reversed traverse, the reversed traverse is started from From b9f35001ca3cdcafde6e238283e051e0cb868814 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 31 Oct 2020 11:13:31 +0800 Subject: [PATCH 04/35] [TD-225] fix memory leaks. --- src/query/inc/qExecutor.h | 3 ++- src/query/src/qExecutor.c | 10 ++++++---- src/query/src/qUtil.c | 4 ++-- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index e5f0d2cb2e..af28d7b3bf 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -122,7 +122,8 @@ typedef struct SQueryCostInfo { uint32_t discardBlocks; uint64_t elapsedTime; uint64_t firstStageMergeTime; - uint64_t internalSupSize; + uint64_t winInfoSize; + uint64_t tableInfoSize; uint64_t numOfTimeWindows; } SQueryCostInfo; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 3e79d2c9b9..f1db2d01c1 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -469,7 +469,7 @@ static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWin } char *t = realloc(pWindowResInfo->pResult, (size_t)(newCap * sizeof(SWindowResult))); - pRuntimeEnv->summary.internalSupSize += (newCap - pWindowResInfo->capacity) * sizeof(SWindowResult); + pRuntimeEnv->summary.winInfoSize += (newCap - pWindowResInfo->capacity) * sizeof(SWindowResult); pRuntimeEnv->summary.numOfTimeWindows += (newCap - pWindowResInfo->capacity); if (t == NULL) { @@ -481,7 +481,7 @@ static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWin int32_t inc = (int32_t)newCap - pWindowResInfo->capacity; memset(&pWindowResInfo->pResult[pWindowResInfo->capacity], 0, sizeof(SWindowResult) * inc); - pRuntimeEnv->summary.internalSupSize += (pQuery->numOfOutput * sizeof(SResultInfo) + pRuntimeEnv->interBufSize) * inc; + pRuntimeEnv->summary.winInfoSize += (pQuery->numOfOutput * sizeof(SResultInfo) + pRuntimeEnv->interBufSize) * inc; for (int32_t i = pWindowResInfo->capacity; i < newCap; ++i) { int32_t ret = createQueryResultInfo(pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, pRuntimeEnv->interBufSize); @@ -4254,8 +4254,8 @@ static void queryCostStatis(SQInfo *pQInfo) { pQInfo, pSummary->elapsedTime, pSummary->firstStageMergeTime, pSummary->totalBlocks, pSummary->loadBlockStatis, pSummary->loadBlocks, pSummary->totalRows, pSummary->totalCheckedRows); - qDebug("QInfo:%p :cost summary: internal size:%"PRId64"B, numOfWin:%"PRId64, pQInfo, pSummary->internalSupSize, - pSummary->numOfTimeWindows); + qDebug("QInfo:%p :cost summary: windowInfo size:%"PRId64" k, numOfWin:%"PRId64", tableInfoSize:%"PRId64" k", pQInfo, pSummary->winInfoSize, + pSummary->numOfTimeWindows/1000, pSummary->tableInfoSize/1000); } static void updateOffsetVal(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBlockInfo) { @@ -6312,6 +6312,8 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou int tableIndex = 0; pQInfo->runtimeEnv.interBufSize = getOutputInterResultBufSize(pQuery); + pQInfo->runtimeEnv.summary.tableInfoSize += (pTableGroupInfo->numOfTables * sizeof(STableQueryInfo)); + pQInfo->pBuf = calloc(pTableGroupInfo->numOfTables, sizeof(STableQueryInfo)); if (pQInfo->pBuf == NULL) { goto _cleanup; diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index ac95afffb1..c454669474 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -56,8 +56,8 @@ int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, SQueryRuntimeEnv *pRun pWindowResInfo->interval = pRuntimeEnv->pQuery->interval.interval; - pSummary->internalSupSize += sizeof(SWindowResult) * threshold; - pSummary->internalSupSize += (pRuntimeEnv->pQuery->numOfOutput * sizeof(SResultInfo) + pRuntimeEnv->interBufSize) * pWindowResInfo->capacity; + pSummary->winInfoSize += sizeof(SWindowResult) * threshold; + pSummary->winInfoSize += (pRuntimeEnv->pQuery->numOfOutput * sizeof(SResultInfo) + pRuntimeEnv->interBufSize) * pWindowResInfo->capacity; pSummary->numOfTimeWindows = threshold; for (int32_t i = 0; i < pWindowResInfo->capacity; ++i) { From 17e35e7aeeda25a5e0975852d887b1c5f30fffb8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 31 Oct 2020 11:14:25 +0800 Subject: [PATCH 05/35] [TD-225] add logs; --- src/query/src/qResultbuf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/src/qResultbuf.c b/src/query/src/qResultbuf.c index b3e97459d3..3892c84c67 100644 --- a/src/query/src/qResultbuf.c +++ b/src/query/src/qResultbuf.c @@ -407,7 +407,7 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) { } if (pResultBuf->file != NULL) { - qDebug("QInfo:%p res output buffer closed, total:%" PRId64 " bytes, inmem size:%dbytes, file size:%"PRId64" bytes", + qDebug("QInfo:%p res output buffer closed, total:%" PRId64 " B, inmem size:%dbytes, file size:%"PRId64" bytes", pResultBuf->handle, pResultBuf->totalBufSize, listNEles(pResultBuf->lruList) * pResultBuf->pageSize, pResultBuf->fileSize); From a0ccfd3145503e36c33b1b4bbfbef84ebc586962 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 31 Oct 2020 11:43:52 +0800 Subject: [PATCH 06/35] [TD-225] add logs. --- src/query/src/qExecutor.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index f1db2d01c1..88abb2e90c 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -4254,8 +4254,8 @@ static void queryCostStatis(SQInfo *pQInfo) { pQInfo, pSummary->elapsedTime, pSummary->firstStageMergeTime, pSummary->totalBlocks, pSummary->loadBlockStatis, pSummary->loadBlocks, pSummary->totalRows, pSummary->totalCheckedRows); - qDebug("QInfo:%p :cost summary: windowInfo size:%"PRId64" k, numOfWin:%"PRId64", tableInfoSize:%"PRId64" k", pQInfo, pSummary->winInfoSize, - pSummary->numOfTimeWindows/1000, pSummary->tableInfoSize/1000); + qDebug("QInfo:%p :cost summary: windowInfo size:%f k, numOfWin:%"PRId64", tableInfoSize:%f k", pQInfo, pSummary->winInfoSize/1024.0, + pSummary->numOfTimeWindows, pSummary->tableInfoSize/1024.0); } static void updateOffsetVal(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBlockInfo) { From d371c840f0842be169446080e936c779381ab4f6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 31 Oct 2020 13:18:45 +0800 Subject: [PATCH 07/35] [TD-225] add logs. --- src/query/inc/qExecutor.h | 1 + src/query/src/qExecutor.c | 18 +++++++++--------- src/query/src/qUtil.c | 6 +++--- src/util/inc/hash.h | 3 ++- src/util/src/hash.c | 8 ++++++++ 5 files changed, 23 insertions(+), 13 deletions(-) diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index af28d7b3bf..bdaec7e2fa 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -124,6 +124,7 @@ typedef struct SQueryCostInfo { uint64_t firstStageMergeTime; uint64_t winInfoSize; uint64_t tableInfoSize; + uint64_t hashSize; uint64_t numOfTimeWindows; } SQueryCostInfo; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 88abb2e90c..3da3d6e28d 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -461,16 +461,16 @@ static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWin // more than the capacity, reallocate the resources if (pWindowResInfo->size >= pWindowResInfo->capacity) { - int64_t newCap = 0; + int64_t newCapacity = 0; if (pWindowResInfo->capacity > 10000) { - newCap = (int64_t)(pWindowResInfo->capacity * 1.25); + newCapacity = (int64_t)(pWindowResInfo->capacity * 1.25); } else { - newCap = (int64_t)(pWindowResInfo->capacity * 1.5); + newCapacity = (int64_t)(pWindowResInfo->capacity * 1.5); } - char *t = realloc(pWindowResInfo->pResult, (size_t)(newCap * sizeof(SWindowResult))); - pRuntimeEnv->summary.winInfoSize += (newCap - pWindowResInfo->capacity) * sizeof(SWindowResult); - pRuntimeEnv->summary.numOfTimeWindows += (newCap - pWindowResInfo->capacity); + char *t = realloc(pWindowResInfo->pResult, (size_t)(newCapacity * sizeof(SWindowResult))); + pRuntimeEnv->summary.winInfoSize += (newCapacity - pWindowResInfo->capacity) * sizeof(SWindowResult); + pRuntimeEnv->summary.numOfTimeWindows += (newCapacity - pWindowResInfo->capacity); if (t == NULL) { longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -478,19 +478,19 @@ static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWin pWindowResInfo->pResult = (SWindowResult *)t; - int32_t inc = (int32_t)newCap - pWindowResInfo->capacity; + int32_t inc = (int32_t)newCapacity - pWindowResInfo->capacity; memset(&pWindowResInfo->pResult[pWindowResInfo->capacity], 0, sizeof(SWindowResult) * inc); pRuntimeEnv->summary.winInfoSize += (pQuery->numOfOutput * sizeof(SResultInfo) + pRuntimeEnv->interBufSize) * inc; - for (int32_t i = pWindowResInfo->capacity; i < newCap; ++i) { + for (int32_t i = pWindowResInfo->capacity; i < newCapacity; ++i) { int32_t ret = createQueryResultInfo(pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, pRuntimeEnv->interBufSize); if (ret != TSDB_CODE_SUCCESS) { longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } } - pWindowResInfo->capacity = (int32_t)newCap; + pWindowResInfo->capacity = (int32_t)newCapacity; } // add a new result set for a new group diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index c454669474..3482c52052 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -49,16 +49,16 @@ int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, SQueryRuntimeEnv *pRun SQueryCostInfo* pSummary = &pRuntimeEnv->summary; // use the pointer arraylist - pWindowResInfo->pResult = calloc(threshold, sizeof(SWindowResult)); + pWindowResInfo->pResult = calloc(pWindowResInfo->capacity, sizeof(SWindowResult)); if (pWindowResInfo->pResult == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } pWindowResInfo->interval = pRuntimeEnv->pQuery->interval.interval; - pSummary->winInfoSize += sizeof(SWindowResult) * threshold; + pSummary->winInfoSize += sizeof(SWindowResult) * pWindowResInfo->capacity; pSummary->winInfoSize += (pRuntimeEnv->pQuery->numOfOutput * sizeof(SResultInfo) + pRuntimeEnv->interBufSize) * pWindowResInfo->capacity; - pSummary->numOfTimeWindows = threshold; + pSummary->numOfTimeWindows = pWindowResInfo->capacity; for (int32_t i = 0; i < pWindowResInfo->capacity; ++i) { int32_t code = createQueryResultInfo(pRuntimeEnv->pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, pRuntimeEnv->interBufSize); diff --git a/src/util/inc/hash.h b/src/util/inc/hash.h index 2a58bec883..d4e9b1677c 100644 --- a/src/util/inc/hash.h +++ b/src/util/inc/hash.h @@ -32,7 +32,6 @@ typedef void (*_hash_free_fn_t)(void *param); typedef struct SHashNode { char *key; -// struct SHashNode *prev; struct SHashNode *next; uint32_t hashVal; // the hash value of key, if hashVal == HASH_VALUE_IN_TRASH, this node is moved to trash uint32_t keyLen; // length of the key @@ -175,6 +174,8 @@ void* taosHashDestroyIter(SHashMutableIterator* iter); */ int32_t taosHashGetMaxOverflowLinkLength(const SHashObj *pHashObj); +size_t taosHashGetMemSize(const SHashObj *pHashObj); + #ifdef __cplusplus } #endif diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 625d4af1ac..9aadee0924 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -798,3 +798,11 @@ SHashNode *getNextHashNode(SHashMutableIterator *pIter) { return NULL; } + +size_t taosHashGetMemSize(const SHashObj *pHashObj) { + if (pHashObj == NULL) { + return 0; + } + + return (pHashObj->capacity * sizeof(SHashEntry) + POINTER_BYTES) + sizeof(SHashNode) * taosHashGetSize(pHashObj); +} From f5710472d5a806a6ad8dca6e81b2ed414a193867 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 31 Oct 2020 16:18:21 +0800 Subject: [PATCH 08/35] [TD-1844] --- src/client/src/tscSubquery.c | 173 +++++++++++++++++++++++++---------- 1 file changed, 124 insertions(+), 49 deletions(-) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 902ff989c7..d58b44b92f 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -32,11 +32,15 @@ typedef struct SInsertSupporter { static void freeJoinSubqueryObj(SSqlObj* pSql); static bool tscHasRemainDataInSubqueryResultSet(SSqlObj *pSql); -static bool tsCompare(int32_t order, int64_t left, int64_t right) { +static int32_t tsCompare(int32_t order, int64_t left, int64_t right) { + if (left == right) { + return 0; + } + if (order == TSDB_ORDER_ASC) { - return left < right; + return left < right? -1:1; } else { - return left > right; + return left > right? -1:1; } } @@ -51,10 +55,10 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ SLimitVal* pLimit = &pQueryInfo->limit; int32_t order = pQueryInfo->order.order; - + SQueryInfo* pSubQueryInfo1 = tscGetQueryInfoDetail(&pSql->pSubs[0]->cmd, 0); SQueryInfo* pSubQueryInfo2 = tscGetQueryInfoDetail(&pSql->pSubs[1]->cmd, 0); - + pSubQueryInfo1->tsBuf = output1; pSubQueryInfo2->tsBuf = output2; @@ -88,59 +92,106 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ int64_t numOfInput1 = 1; int64_t numOfInput2 = 1; - while (1) { - STSElem elem1 = tsBufGetElem(pSupporter1->pTSBuf); - STSElem elem2 = tsBufGetElem(pSupporter2->pTSBuf); + int32_t numOfVnodes = 0; + int32_t* idList = NULL; + tsBufGetVnodeIdList(pSupporter2->pTSBuf, &numOfVnodes, &idList); -#ifdef _DEBUG_VIEW - tscInfo("%" PRId64 ", tags:%"PRId64" \t %" PRId64 ", tags:%"PRId64, elem1.ts, elem1.tag.i64Key, elem2.ts, elem2.tag.i64Key); -#endif + bool completed = false; + while(1) { + STSElem elem = tsBufGetElem(pSupporter1->pTSBuf); - int32_t res = tVariantCompare(elem1.tag, elem2.tag); - if (res == -1 || (res == 0 && tsCompare(order, elem1.ts, elem2.ts))) { - if (!tsBufNextPos(pSupporter1->pTSBuf)) { + // no data in pSupporter1 anymore, jump out of loop + if (elem.vnode < 0 || completed) { + break; + } + + bool f = false; + for (int32_t i = 0; i < numOfVnodes; ++i) { + STSElem el = tsBufGetElemStartPos(pSupporter2->pTSBuf, idList[i], elem.tag); + if (el.vnode == idList[i]) { + f = true; break; } + } - numOfInput1++; - } else if ((res > 0) || (res == 0 && tsCompare(order, elem2.ts, elem1.ts))) { - if (!tsBufNextPos(pSupporter2->pTSBuf)) { - break; - } + /** + * there are elements in pSupporter2 with the same tag, continue + */ + if (f) { + while (1) { + STSElem elem1 = tsBufGetElem(pSupporter1->pTSBuf); + STSElem elem2 = tsBufGetElem(pSupporter2->pTSBuf); - numOfInput2++; - } else { - /* - * in case of stable query, limit/offset is not applied here. the limit/offset is applied to the - * final results which is acquired after the secondry merge of in the client. - */ - if (pLimit->offset == 0 || pQueryInfo->interval.interval > 0 || QUERY_IS_STABLE_QUERY(pQueryInfo->type)) { - if (win->skey > elem1.ts) { - win->skey = elem1.ts; + /* + * in case of stable query, limit/offset is not applied here. the limit/offset is applied to the + * final results which is acquired after the secondry merge of in the client. + */ + int32_t re = tsCompare(order, elem1.ts, elem2.ts); + if (re < 0) { + if (!tsBufNextPos(pSupporter1->pTSBuf)) { + completed = true; + break; + } + + numOfInput1++; + } else if (re > 0) { + if (!tsBufNextPos(pSupporter2->pTSBuf)) { + completed = true; + break; + } + + numOfInput2++; + } else { + if (pLimit->offset == 0 || pQueryInfo->interval.interval > 0 || QUERY_IS_STABLE_QUERY(pQueryInfo->type)) { + if (win->skey > elem1.ts) { + win->skey = elem1.ts; + } + + if (win->ekey < elem1.ts) { + win->ekey = elem1.ts; + } + + tsBufAppend(output1, elem1.vnode, elem1.tag, (const char*)&elem1.ts, sizeof(elem1.ts)); + tsBufAppend(output2, elem2.vnode, elem2.tag, (const char*)&elem2.ts, sizeof(elem2.ts)); + } else { + pLimit->offset -= 1; + } + + if (!tsBufNextPos(pSupporter1->pTSBuf)) { + completed = true; + break; + } + + numOfInput1++; + + if (!tsBufNextPos(pSupporter2->pTSBuf)) { + completed = true; + break; + } + + numOfInput2++; } - - if (win->ekey < elem1.ts) { - win->ekey = elem1.ts; + } + } else { // no data in pSupporter2, ignore current data in pSupporter2 + tVariant tag = {0}; + tVariantAssign(&tag, elem.tag); + + // ignore all records with the same tag + while (tsBufNextPos(pSupporter1->pTSBuf)) { + STSElem el1 = tsBufGetElem(pSupporter1->pTSBuf); + int32_t res = tVariantCompare(el1.tag, &tag); + + // it is a record with new tag + if (res != 0) { + break; } - - tsBufAppend(output1, elem1.vnode, elem1.tag, (const char*)&elem1.ts, sizeof(elem1.ts)); - tsBufAppend(output2, elem2.vnode, elem2.tag, (const char*)&elem2.ts, sizeof(elem2.ts)); - - } else { - pLimit->offset -= 1; } - if (!tsBufNextPos(pSupporter1->pTSBuf)) { + STSElem el1 = tsBufGetElem(pSupporter1->pTSBuf); + if (el1.vnode < 0) { // no data exists, abort + completed = true; break; } - - numOfInput1++; - - if (!tsBufNextPos(pSupporter2->pTSBuf)) { - break; - } - - numOfInput2++; } } @@ -162,8 +213,9 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ TSKEY et = taosGetTimestampUs(); tscDebug("%p input1:%" PRId64 ", input2:%" PRId64 ", final:%" PRId64 " in %d vnodes for secondary query after ts blocks " - "intersecting, skey:%" PRId64 ", ekey:%" PRId64 ", numOfVnode:%d, elasped time:%"PRId64" us", pSql, numOfInput1, numOfInput2, output1->numOfTotal, - output1->numOfVnodes, win->skey, win->ekey, tsBufGetNumOfVnodes(output1), et - st); + "intersecting, skey:%" PRId64 ", ekey:%" PRId64 ", numOfVnode:%d, elapsed time:%" PRId64 " us", + pSql, numOfInput1, numOfInput2, output1->numOfTotal, output1->numOfVnodes, win->skey, win->ekey, + tsBufGetNumOfVnodes(output1), et - st); return output1->numOfTotal; } @@ -517,18 +569,29 @@ void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArr vgTables = taosArrayInit(4, sizeof(STableIdInfo)); info.itemList = vgTables; + + if (taosArrayGetSize(result) > 0) { + SVgroupTableInfo* prevGroup = taosArrayGet(result, taosArrayGetSize(result) - 1); + tscDebug("%p vgId:%d, tables:%"PRId64, pSql, prevGroup->vgInfo.vgId, taosArrayGetSize(prevGroup->itemList)); + } + taosArrayPush(result, &info); } STableIdInfo item = {.uid = tt->uid, .tid = tt->tid, .key = INT64_MIN}; taosArrayPush(vgTables, &item); - tscDebug("%p tid:%d, uid:%"PRIu64",vgId:%d added, total:%d", pSql, tt->tid, tt->uid, tt->vgId, (int32_t) taosArrayGetSize(vgTables)); + tscTrace("%p tid:%d, uid:%"PRIu64",vgId:%d added", pSql, tt->tid, tt->uid, tt->vgId); prev = tt; } pTableMetaInfo->pVgroupTables = result; pTableMetaInfo->vgroupIndex = 0; + + if (taosArrayGetSize(result) > 0) { + SVgroupTableInfo* g = taosArrayGet(result, taosArrayGetSize(result) - 1); + tscDebug("%p vgId:%d, tables:%"PRId64, pSql, g->vgInfo.vgId, taosArrayGetSize(g->itemList)); + } } static void issueTSCompQuery(SSqlObj* pSql, SJoinSupporter* pSupporter, SSqlObj* pParent) { @@ -656,6 +719,18 @@ static int32_t getIntersectionOfTableTuple(SQueryInfo* pQueryInfo, SSqlObj* pPar qsort((*s1)->pData, t1, size, tidTagsCompar); qsort((*s2)->pData, t2, size, tidTagsCompar); +#if 0 + for(int32_t k = 0; k < t1; ++k) { + STidTags* p = (*s1)->pData + size * k; + printf("%d, tag:%s\n", p->vgId, ((tstr*)(p->tag))->data); + } + + for(int32_t k = 0; k < t1; ++k) { + STidTags* p = (*s2)->pData + size * k; + printf("%d, tag:%s\n", p->vgId, ((tstr*)(p->tag))->data); + } +#endif + tscDebug("%p tags match complete, result: %"PRId64", %"PRId64, pParentSql, t1, t2); return TSDB_CODE_SUCCESS; } From b8bead9a92d4754b8ee51e5b9d1851bc2e0f516a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 31 Oct 2020 16:33:05 +0800 Subject: [PATCH 09/35] [TD-225] add logs. --- src/query/src/qExecutor.c | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 3da3d6e28d..2eb0abd070 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -4246,6 +4246,23 @@ static void queryCostStatis(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQueryCostInfo *pSummary = &pRuntimeEnv->summary; + uint64_t hashSize = taosHashGetMemSize(pQInfo->runtimeEnv.windowResInfo.hashList); + + hashSize += taosHashGetMemSize(pQInfo->tableqinfoGroupInfo.map); + int32_t numOfGroup = GET_NUM_OF_TABLEGROUP(pQInfo); + for(int32_t i = 0; i < numOfGroup; ++i) { + SArray* pa = GET_TABLEGROUP(pQInfo, i); + + int32_t numOfTables = taosArrayGetSize(pa); + for(int32_t j = 0; j < numOfTables; ++j) { + STableQueryInfo* pTableQueryInfo = taosArrayGetP(pa, j); + + hashSize += taosHashGetMemSize(pTableQueryInfo->windowResInfo.hashList); + } + } + + pSummary->hashSize = hashSize; + // add the merge time pSummary->elapsedTime += pSummary->firstStageMergeTime; @@ -4254,8 +4271,8 @@ static void queryCostStatis(SQInfo *pQInfo) { pQInfo, pSummary->elapsedTime, pSummary->firstStageMergeTime, pSummary->totalBlocks, pSummary->loadBlockStatis, pSummary->loadBlocks, pSummary->totalRows, pSummary->totalCheckedRows); - qDebug("QInfo:%p :cost summary: windowInfo size:%f k, numOfWin:%"PRId64", tableInfoSize:%f k", pQInfo, pSummary->winInfoSize/1024.0, - pSummary->numOfTimeWindows, pSummary->tableInfoSize/1024.0); + qDebug("QInfo:%p :cost summary: windowInfo size:%.2f Kb, numOfWin:%"PRId64", tableInfoSize:%.2f Kb, hashTable:%.2f Kb", pQInfo, pSummary->winInfoSize/1024.0, + pSummary->numOfTimeWindows, pSummary->tableInfoSize/1024.0, pSummary->hashSize/1024.0); } static void updateOffsetVal(SQueryRuntimeEnv *pRuntimeEnv, SDataBlockInfo *pBlockInfo) { From a8e86c9a675845cb48a953448d8b3eccbd5a0283 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 31 Oct 2020 23:14:43 +0800 Subject: [PATCH 10/35] [TD-1870] --- src/query/inc/qExecutor.h | 5 +++- src/query/inc/qUtil.h | 9 ++++++ src/query/src/qExecutor.c | 57 +++++++++++++++++++++----------------- src/query/src/qResultbuf.c | 8 +++--- src/query/src/qUtil.c | 34 +++++++++++------------ src/util/src/hash.c | 2 +- 6 files changed, 66 insertions(+), 49 deletions(-) diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index bdaec7e2fa..ecedf4c13f 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -70,7 +70,8 @@ typedef struct SResultRec { typedef struct SWindowResInfo { SWindowResult* pResult; // result list - SHashObj* hashList; // hash list for quick access +// uint64_t uid; // table uid, in order to identify the result from global hash table +// SHashObj* hashList; // hash list for quick access int16_t type; // data type for hash key int32_t capacity; // max capacity int32_t curIndex; // current start active index @@ -177,6 +178,8 @@ typedef struct SQueryRuntimeEnv { int32_t interBufSize; // intermediate buffer sizse int32_t prevGroupId; // previous executed group id SDiskbasedResultBuf* pResultBuf; // query result buffer based on blocked-wised disk file + SHashObj* pWindowHashTable; // quick locate the window object for each result + char* keyBuf; // window key buffer } SQueryRuntimeEnv; enum { diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h index 32f26f66f5..fc997a44c5 100644 --- a/src/query/inc/qUtil.h +++ b/src/query/inc/qUtil.h @@ -15,6 +15,15 @@ #ifndef TDENGINE_QUERYUTIL_H #define TDENGINE_QUERYUTIL_H +#define SET_RES_WINDOW_KEY(_k, _ori, _len, _uid) \ + do { \ + assert(sizeof(_uid) == sizeof(uint64_t)); \ + *(uint64_t *)(_k) = (_uid); \ + memcpy((_k) + sizeof(uint64_t), (_ori), (_len)); \ + } while (0) + +#define GET_RES_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t)) + int32_t getOutputInterResultBufSize(SQuery* pQuery); void clearTimeWindowResBuf(SQueryRuntimeEnv* pRuntimeEnv, SWindowResult* pOneOutputRes); diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 2eb0abd070..32ed7190c6 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -448,10 +448,11 @@ static bool hasNullValue(SColIndex* pColIndex, SDataStatis *pStatis, SDataStatis } static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo, char *pData, - int16_t bytes, bool masterscan) { + int16_t bytes, bool masterscan, uint64_t uid) { SQuery *pQuery = pRuntimeEnv->pQuery; - int32_t *p1 = (int32_t *) taosHashGet(pWindowResInfo->hashList, pData, bytes); + SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, uid); + int32_t *p1 = (int32_t *) taosHashGet(pRuntimeEnv->pWindowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); if (p1 != NULL) { pWindowResInfo->curIndex = *p1; } else { @@ -495,7 +496,7 @@ static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWin // add a new result set for a new group pWindowResInfo->curIndex = pWindowResInfo->size++; - taosHashPut(pWindowResInfo->hashList, pData, bytes, (char *)&pWindowResInfo->curIndex, sizeof(int32_t)); + taosHashPut(pRuntimeEnv->pWindowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), (char *)&pWindowResInfo->curIndex, sizeof(int32_t)); } // too many time window in query @@ -555,7 +556,7 @@ static STimeWindow getActiveTimeWindow(SWindowResInfo *pWindowResInfo, int64_t t return w; } -static int32_t addNewWindowResultBuf(SWindowResult *pWindowRes, SDiskbasedResultBuf *pResultBuf, int32_t sid, +static int32_t addNewWindowResultBuf(SWindowResult *pWindowRes, SDiskbasedResultBuf *pResultBuf, int32_t tid, int32_t numOfRowsPerPage) { if (pWindowRes->pageId != -1) { return 0; @@ -565,10 +566,10 @@ static int32_t addNewWindowResultBuf(SWindowResult *pWindowRes, SDiskbasedResult // in the first scan, new space needed for results int32_t pageId = -1; - SIDList list = getDataBufPagesIdList(pResultBuf, sid); + SIDList list = getDataBufPagesIdList(pResultBuf, tid); if (taosArrayGetSize(list) == 0) { - pData = getNewDataBuf(pResultBuf, sid, &pageId); + pData = getNewDataBuf(pResultBuf, tid, &pageId); } else { SPageInfo* pi = getLastPageInfo(list); pData = getResBufPage(pResultBuf, pi->pageId); @@ -578,7 +579,7 @@ static int32_t addNewWindowResultBuf(SWindowResult *pWindowRes, SDiskbasedResult // release current page first, and prepare the next one releaseResBufPageInfo(pResultBuf, pi); - pData = getNewDataBuf(pResultBuf, sid, &pageId); + pData = getNewDataBuf(pResultBuf, tid, &pageId); if (pData != NULL) { assert(pData->num == 0); // number of elements must be 0 for new allocated buffer } @@ -600,13 +601,12 @@ static int32_t addNewWindowResultBuf(SWindowResult *pWindowRes, SDiskbasedResult return 0; } -static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo, int32_t sid, +static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo, SDataBlockInfo* pBockInfo, STimeWindow *win, bool masterscan, bool* newWind) { assert(win->skey <= win->ekey); SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; - SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&win->skey, - TSDB_KEYSIZE, masterscan); + SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&win->skey, TSDB_KEYSIZE, masterscan, pBockInfo->uid); if (pWindowRes == NULL) { *newWind = false; @@ -617,7 +617,7 @@ static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowRes // not assign result buffer yet, add new result buffer if (pWindowRes->pageId == -1) { - int32_t ret = addNewWindowResultBuf(pWindowRes, pResultBuf, sid, pRuntimeEnv->numOfRowsPerPage); + int32_t ret = addNewWindowResultBuf(pWindowRes, pResultBuf, pBockInfo->tid, pRuntimeEnv->numOfRowsPerPage); if (ret != TSDB_CODE_SUCCESS) { return -1; } @@ -1024,8 +1024,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * bool hasTimeWindow = false; STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery); - if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->tid, &win, masterScan, &hasTimeWindow) != - TSDB_CODE_SUCCESS) { + if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &win, masterScan, &hasTimeWindow) != TSDB_CODE_SUCCESS) { taosTFree(sasArray); return; } @@ -1053,8 +1052,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * // null data, failed to allocate more memory buffer hasTimeWindow = false; - if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->tid, &nextWin, masterScan, - &hasTimeWindow) != TSDB_CODE_SUCCESS) { + if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &nextWin, masterScan, &hasTimeWindow) != TSDB_CODE_SUCCESS) { break; } @@ -1112,12 +1110,13 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat len = varDataLen(pData); } else if (type == TSDB_DATA_TYPE_FLOAT || type == TSDB_DATA_TYPE_DOUBLE) { SQInfo* pQInfo = GET_QINFO_ADDR(pRuntimeEnv); - qError("QInfo:%p group by not supported on double/float/binary/nchar columns, abort", pQInfo); + qError("QInfo:%p group by not supported on double/float columns, abort", pQInfo); longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); } - SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, d, len, true); + uint64_t uid = 0; // uid is always set to be 0. + SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, d, len, true, uid); if (pWindowRes == NULL) { return -1; } @@ -1335,7 +1334,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery); bool hasTimeWindow = false; - int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->tid, &win, masterScan, &hasTimeWindow); + int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &win, masterScan, &hasTimeWindow); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code continue; } @@ -1363,7 +1362,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS // null data, failed to allocate more memory buffer hasTimeWindow = false; - if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo->tid, &nextWin, masterScan, &hasTimeWindow) != TSDB_CODE_SUCCESS) { + if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &nextWin, masterScan, &hasTimeWindow) != TSDB_CODE_SUCCESS) { break; } @@ -1764,6 +1763,10 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { tsdbCleanupQueryHandle(pRuntimeEnv->pSecQueryHandle); pRuntimeEnv->pTSBuf = tsBufDestroy(pRuntimeEnv->pTSBuf); + taosTFree(pRuntimeEnv->keyBuf); + + taosHashCleanup(pRuntimeEnv->pWindowHashTable); + pRuntimeEnv->pWindowHashTable = NULL; } #define IS_QUERY_KILLED(_q) ((_q)->code == TSDB_CODE_TSC_QUERY_CANCELLED) @@ -2262,7 +2265,7 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo * pW TSKEY k = QUERY_IS_ASC_QUERY(pQuery)? pBlockInfo->window.skey:pBlockInfo->window.ekey; STimeWindow win = getActiveTimeWindow(pWindowResInfo, k, pQuery); - if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pBlockInfo->tid, &win, masterScan, &hasTimeWindow) != + if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pBlockInfo, &win, masterScan, &hasTimeWindow) != TSDB_CODE_SUCCESS) { // todo handle error in set result for timewindow } @@ -3787,8 +3790,9 @@ void setExecutionContext(SQInfo *pQInfo, int32_t groupIndex, TSKEY nextKey) { return; } + uint64_t uid = 0; // uid is always set to be 0 SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&groupIndex, - sizeof(groupIndex), true); + sizeof(groupIndex), true, uid); if (pWindowRes == NULL) { return; } @@ -4246,7 +4250,7 @@ static void queryCostStatis(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQueryCostInfo *pSummary = &pRuntimeEnv->summary; - uint64_t hashSize = taosHashGetMemSize(pQInfo->runtimeEnv.windowResInfo.hashList); + uint64_t hashSize = taosHashGetMemSize(pQInfo->runtimeEnv.pWindowHashTable); hashSize += taosHashGetMemSize(pQInfo->tableqinfoGroupInfo.map); int32_t numOfGroup = GET_NUM_OF_TABLEGROUP(pQInfo); @@ -4255,9 +4259,9 @@ static void queryCostStatis(SQInfo *pQInfo) { int32_t numOfTables = taosArrayGetSize(pa); for(int32_t j = 0; j < numOfTables; ++j) { - STableQueryInfo* pTableQueryInfo = taosArrayGetP(pa, j); +// STableQueryInfo* pTableQueryInfo = taosArrayGetP(pa, j); - hashSize += taosHashGetMemSize(pTableQueryInfo->windowResInfo.hashList); +// hashSize += taosHashGetMemSize(pTableQueryInfo->windowResInfo.hashList); } } @@ -6323,7 +6327,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou pQInfo->tableqinfoGroupInfo.pGroupList = taosArrayInit(numOfGroups, POINTER_BYTES); pQInfo->tableqinfoGroupInfo.numOfTables = pTableGroupInfo->numOfTables; pQInfo->tableqinfoGroupInfo.map = taosHashInit(pTableGroupInfo->numOfTables, - taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false); + taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); } int tableIndex = 0; @@ -6331,6 +6335,9 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou pQInfo->runtimeEnv.interBufSize = getOutputInterResultBufSize(pQuery); pQInfo->runtimeEnv.summary.tableInfoSize += (pTableGroupInfo->numOfTables * sizeof(STableQueryInfo)); + pQInfo->runtimeEnv.pWindowHashTable = taosHashInit(pTableGroupInfo->numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + pQInfo->runtimeEnv.keyBuf = malloc(TSDB_MAX_BYTES_PER_ROW); + pQInfo->pBuf = calloc(pTableGroupInfo->numOfTables, sizeof(STableQueryInfo)); if (pQInfo->pBuf == NULL) { goto _cleanup; diff --git a/src/query/src/qResultbuf.c b/src/query/src/qResultbuf.c index 3892c84c67..bd7c4694d0 100644 --- a/src/query/src/qResultbuf.c +++ b/src/query/src/qResultbuf.c @@ -407,14 +407,14 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) { } if (pResultBuf->file != NULL) { - qDebug("QInfo:%p res output buffer closed, total:%" PRId64 " B, inmem size:%dbytes, file size:%"PRId64" bytes", - pResultBuf->handle, pResultBuf->totalBufSize, listNEles(pResultBuf->lruList) * pResultBuf->pageSize, + qDebug("QInfo:%p res output buffer closed, total:%.2f Kb, inmem size:%dbytes, file size:%"PRId64" bytes", + pResultBuf->handle, pResultBuf->totalBufSize/1024.0, listNEles(pResultBuf->lruList) * pResultBuf->pageSize, pResultBuf->fileSize); fclose(pResultBuf->file); } else { - qDebug("QInfo:%p res output buffer closed, total:%" PRId64 " bytes, no file created", pResultBuf->handle, - pResultBuf->totalBufSize); + qDebug("QInfo:%p res output buffer closed, total:%.2f Kb, no file created", pResultBuf->handle, + pResultBuf->totalBufSize/1024.0); } unlink(pResultBuf->path); diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index 3482c52052..8273a21395 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -36,12 +36,6 @@ int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, SQueryRuntimeEnv *pRun pWindowResInfo->threshold = threshold; pWindowResInfo->type = type; - _hash_fn_t fn = taosGetDefaultHashFunction(type); - pWindowResInfo->hashList = taosHashInit(threshold, fn, true, false); - if (pWindowResInfo->hashList == NULL) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } - pWindowResInfo->curIndex = -1; pWindowResInfo->size = 0; pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL; @@ -83,7 +77,7 @@ void cleanupTimeWindowInfo(SWindowResInfo *pWindowResInfo) { return; } if (pWindowResInfo->capacity == 0) { - assert(pWindowResInfo->hashList == NULL && pWindowResInfo->pResult == NULL); + assert(/*pWindowResInfo->hashList == NULL && */pWindowResInfo->pResult == NULL); return; } @@ -93,7 +87,7 @@ void cleanupTimeWindowInfo(SWindowResInfo *pWindowResInfo) { } } - taosHashCleanup(pWindowResInfo->hashList); +// taosHashCleanup(pWindowResInfo->hashList); taosTFree(pWindowResInfo->pResult); } @@ -108,11 +102,11 @@ void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowR } pWindowResInfo->curIndex = -1; - taosHashCleanup(pWindowResInfo->hashList); +// taosHashCleanup(pWindowResInfo->hashList); pWindowResInfo->size = 0; - _hash_fn_t fn = taosGetDefaultHashFunction(pWindowResInfo->type); - pWindowResInfo->hashList = taosHashInit(pWindowResInfo->capacity, fn, true, false); +// _hash_fn_t fn = taosGetDefaultHashFunction(pWindowResInfo->type); +// pWindowResInfo->hashList = taosHashInit(pWindowResInfo->capacity, fn, true, false); pWindowResInfo->startTime = TSKEY_INITIAL_VAL; pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL; @@ -127,10 +121,10 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { int32_t numOfClosed = numOfClosedTimeWindow(pWindowResInfo); assert(num >= 0 && num <= numOfClosed); - int16_t type = pWindowResInfo->type; - - char *key = NULL; - int16_t bytes = -1; + int16_t type = pWindowResInfo->type; + uint64_t uid = 0; // uid is always set to be 0. + char *key = NULL; + int16_t bytes = -1; for (int32_t i = 0; i < num; ++i) { SWindowResult *pResult = &pWindowResInfo->pResult[i]; @@ -145,7 +139,8 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { bytes = tDataTypeDesc[pWindowResInfo->type].nSize; } - taosHashRemove(pWindowResInfo->hashList, (const char *)key, bytes); + SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, uid); + taosHashRemove(pRuntimeEnv->pWindowHashTable, (const char *)pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); } else { break; } @@ -177,12 +172,15 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { bytes = tDataTypeDesc[pWindowResInfo->type].nSize; } - int32_t *p = (int32_t *)taosHashGet(pWindowResInfo->hashList, (const char *)key, bytes); + SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, uid); + int32_t *p = (int32_t *)taosHashGet(pRuntimeEnv->pWindowHashTable, (const char *)pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); assert(p != NULL); int32_t v = (*p - num); assert(v >= 0 && v <= pWindowResInfo->size); - taosHashPut(pWindowResInfo->hashList, (char *)key, bytes, (char *)&v, sizeof(int32_t)); + + SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, uid); + taosHashPut(pRuntimeEnv->pWindowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), (char *)&v, sizeof(int32_t)); } pWindowResInfo->curIndex = -1; diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 9aadee0924..8ffbb1f0f6 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -804,5 +804,5 @@ size_t taosHashGetMemSize(const SHashObj *pHashObj) { return 0; } - return (pHashObj->capacity * sizeof(SHashEntry) + POINTER_BYTES) + sizeof(SHashNode) * taosHashGetSize(pHashObj); + return (pHashObj->capacity * (sizeof(SHashEntry) + POINTER_BYTES)) + sizeof(SHashNode) * taosHashGetSize(pHashObj) + sizeof(SHashObj); } From 2314b1952b02d5778d12c4f3b0b0ab60f0ef3544 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 1 Nov 2020 23:08:28 +0800 Subject: [PATCH 11/35] [TD-1870] reduce memory consumption during query processing. --- src/client/src/tscLocalMerge.c | 2 +- src/query/inc/qExecutor.h | 18 ++++- src/query/inc/qFill.h | 2 +- src/query/inc/qUtil.h | 12 ++- src/query/src/qExecutor.c | 90 ++++++++++----------- src/query/src/qFill.c | 2 +- src/query/src/qUtil.c | 141 +++++++++++++++++++++++---------- 7 files changed, 171 insertions(+), 96 deletions(-) diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c index ea073e8192..8c835806a7 100644 --- a/src/client/src/tscLocalMerge.c +++ b/src/client/src/tscLocalMerge.c @@ -489,7 +489,7 @@ void tscDestroyLocalReducer(SSqlObj *pSql) { tscDebug("%p waiting for delete procedure, status: %d", pSql, status); } - pLocalReducer->pFillInfo = taosDestoryFillInfo(pLocalReducer->pFillInfo); + pLocalReducer->pFillInfo = taosDestroyFillInfo(pLocalReducer->pFillInfo); if (pLocalReducer->pCtx != NULL) { for (int32_t i = 0; i < pQueryInfo->fieldsInfo.numOfOutput; ++i) { diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index ecedf4c13f..ab20840cbe 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -40,6 +40,19 @@ typedef struct SGroupResInfo { int32_t rowId; } SGroupResInfo; +typedef struct SWindowResultPool { + int32_t elemSize; + int32_t blockSize; + int32_t numOfElemPerBlock; + + struct { + int32_t blockIndex; + int32_t pos; + } position; + + SArray* pData; // SArray +} SWindowResultPool; + typedef struct SSqlGroupbyExpr { int16_t tableIndex; SArray* columnInfo; // SArray, group by columns information @@ -69,9 +82,7 @@ typedef struct SResultRec { } SResultRec; typedef struct SWindowResInfo { - SWindowResult* pResult; // result list -// uint64_t uid; // table uid, in order to identify the result from global hash table -// SHashObj* hashList; // hash list for quick access + SWindowResult** pResult; // result list int16_t type; // data type for hash key int32_t capacity; // max capacity int32_t curIndex; // current start active index @@ -180,6 +191,7 @@ typedef struct SQueryRuntimeEnv { SDiskbasedResultBuf* pResultBuf; // query result buffer based on blocked-wised disk file SHashObj* pWindowHashTable; // quick locate the window object for each result char* keyBuf; // window key buffer + SWindowResultPool* pool; // window result object pool } SQueryRuntimeEnv; enum { diff --git a/src/query/inc/qFill.h b/src/query/inc/qFill.h index 6d44fee095..329ea9a789 100644 --- a/src/query/inc/qFill.h +++ b/src/query/inc/qFill.h @@ -71,7 +71,7 @@ SFillInfo* taosInitFillInfo(int32_t order, TSKEY skey, int32_t numOfTags, int32_ void taosResetFillInfo(SFillInfo* pFillInfo, TSKEY startTimestamp); -void* taosDestoryFillInfo(SFillInfo *pFillInfo); +void* taosDestroyFillInfo(SFillInfo *pFillInfo); void taosFillSetStartInfo(SFillInfo* pFillInfo, int32_t numOfRows, TSKEY endKey); diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h index fc997a44c5..3775780423 100644 --- a/src/query/inc/qUtil.h +++ b/src/query/inc/qUtil.h @@ -44,7 +44,7 @@ void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int static FORCE_INLINE SWindowResult *getWindowResult(SWindowResInfo *pWindowResInfo, int32_t slot) { assert(pWindowResInfo != NULL && slot >= 0 && slot < pWindowResInfo->size); - return &pWindowResInfo->pResult[slot]; + return pWindowResInfo->pResult[slot]; } #define curTimeWindowIndex(_winres) ((_winres)->curIndex) @@ -71,4 +71,14 @@ bool notNull_filter(SColumnFilterElem *pFilter, char* minval, char* maxval); __filter_func_t *getRangeFilterFuncArray(int32_t type); __filter_func_t *getValueFilterFuncArray(int32_t type); +size_t getWindowResultSize(SQueryRuntimeEnv* pRuntimeEnv); + +SWindowResultPool* initWindowResultPool(size_t size); +SWindowResult* getNewWindowResult(SWindowResultPool* p); +int64_t getWindowResultPoolMemSize(SWindowResultPool* p); +void* destroyWindowResultPool(SWindowResultPool* p); +int32_t getNumOfAllocatedWindowResult(SWindowResultPool* p); +int32_t getNumOfUsedWindowResult(SWindowResultPool* p); + + #endif // TDENGINE_QUERYUTIL_H diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 32ed7190c6..d5e1962eea 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -469,30 +469,32 @@ static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWin newCapacity = (int64_t)(pWindowResInfo->capacity * 1.5); } - char *t = realloc(pWindowResInfo->pResult, (size_t)(newCapacity * sizeof(SWindowResult))); - pRuntimeEnv->summary.winInfoSize += (newCapacity - pWindowResInfo->capacity) * sizeof(SWindowResult); - pRuntimeEnv->summary.numOfTimeWindows += (newCapacity - pWindowResInfo->capacity); + char *t = realloc(pWindowResInfo->pResult, (size_t)(newCapacity * POINTER_BYTES)); + // pRuntimeEnv->summary.winInfoSize += (newCapacity - pWindowResInfo->capacity) * sizeof(SWindowResult); + // pRuntimeEnv->summary.numOfTimeWindows += (newCapacity - pWindowResInfo->capacity); if (t == NULL) { longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } - pWindowResInfo->pResult = (SWindowResult *)t; + pWindowResInfo->pResult = (SWindowResult **)t; int32_t inc = (int32_t)newCapacity - pWindowResInfo->capacity; - memset(&pWindowResInfo->pResult[pWindowResInfo->capacity], 0, sizeof(SWindowResult) * inc); - - pRuntimeEnv->summary.winInfoSize += (pQuery->numOfOutput * sizeof(SResultInfo) + pRuntimeEnv->interBufSize) * inc; - - for (int32_t i = pWindowResInfo->capacity; i < newCapacity; ++i) { - int32_t ret = createQueryResultInfo(pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, pRuntimeEnv->interBufSize); - if (ret != TSDB_CODE_SUCCESS) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - } + memset(&pWindowResInfo->pResult[pWindowResInfo->capacity], 0, POINTER_BYTES * inc); pWindowResInfo->capacity = (int32_t)newCapacity; } +// pRuntimeEnv->summary.winInfoSize += (pQuery->numOfOutput * sizeof(SResultInfo) + pRuntimeEnv->interBufSize) * inc; + SWindowResult* pResult = getNewWindowResult(pRuntimeEnv->pool); + pWindowResInfo->pResult[pWindowResInfo->size] = pResult; +// for (int32_t i = pWindowResInfo->capacity; i < newCapacity; ++i) { + int32_t ret = createQueryResultInfo(pQuery, pResult, pRuntimeEnv->stableQuery, pRuntimeEnv->interBufSize); + if (ret != TSDB_CODE_SUCCESS) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } +// } + +// } // add a new result set for a new group pWindowResInfo->curIndex = pWindowResInfo->size++; @@ -632,7 +634,7 @@ static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowRes static bool getTimeWindowResStatus(SWindowResInfo *pWindowResInfo, int32_t slot) { assert(slot >= 0 && slot < pWindowResInfo->size); - return pWindowResInfo->pResult[slot].closed; + return pWindowResInfo->pResult[slot]->closed; } static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_search_fn_t searchFn, TSKEY ekey, int16_t pos, @@ -691,7 +693,7 @@ static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKe int64_t skey = TSKEY_INITIAL_VAL; for (i = 0; i < pWindowResInfo->size; ++i) { - SWindowResult *pResult = &pWindowResInfo->pResult[i]; + SWindowResult *pResult = pWindowResInfo->pResult[i]; if (pResult->closed) { numOfClosed += 1; continue; @@ -715,7 +717,7 @@ static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKe pWindowResInfo->curIndex = i; } - pWindowResInfo->prevSKey = pWindowResInfo->pResult[pWindowResInfo->curIndex].win.skey; + pWindowResInfo->prevSKey = pWindowResInfo->pResult[pWindowResInfo->curIndex]->win.skey; // the number of completed slots are larger than the threshold, return current generated results to client. if (numOfClosed > pWindowResInfo->threshold) { @@ -1756,7 +1758,7 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { taosTFree(pRuntimeEnv->pCtx); } - pRuntimeEnv->pFillInfo = taosDestoryFillInfo(pRuntimeEnv->pFillInfo); + pRuntimeEnv->pFillInfo = taosDestroyFillInfo(pRuntimeEnv->pFillInfo); destroyResultBuf(pRuntimeEnv->pResultBuf); tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle); @@ -1767,6 +1769,8 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { taosHashCleanup(pRuntimeEnv->pWindowHashTable); pRuntimeEnv->pWindowHashTable = NULL; + + pRuntimeEnv->pool = destroyWindowResultPool(pRuntimeEnv->pool); } #define IS_QUERY_KILLED(_q) ((_q)->code == TSDB_CODE_TSC_QUERY_CANCELLED) @@ -3326,11 +3330,13 @@ void switchCtxOrder(SQueryRuntimeEnv *pRuntimeEnv) { int32_t createQueryResultInfo(SQuery *pQuery, SWindowResult *pResultRow, bool isSTableQuery, size_t interBufSize) { int32_t numOfCols = pQuery->numOfOutput; - size_t size = numOfCols * sizeof(SResultInfo) + interBufSize; - pResultRow->resultInfo = calloc(1, size); - if (pResultRow->resultInfo == NULL) { - return TSDB_CODE_QRY_OUT_OF_MEMORY; - } +// size_t size = numOfCols * sizeof(SResultInfo) + interBufSize; + pResultRow->resultInfo = (SResultInfo*)((char*)pResultRow + sizeof(SWindowResult)); + +// pResultRow->resultInfo = calloc(1, size); +// if (pResultRow->resultInfo == NULL) { +// return TSDB_CODE_QRY_OUT_OF_MEMORY; +// } pResultRow->pageId = -1; pResultRow->rowId = -1; @@ -3698,7 +3704,7 @@ void finalizeQueryResult(SQueryRuntimeEnv *pRuntimeEnv) { } for (int32_t i = 0; i < pWindowResInfo->size; ++i) { - SWindowResult *buf = &pWindowResInfo->pResult[i]; + SWindowResult *buf = pWindowResInfo->pResult[i]; if (!isWindowResClosed(pWindowResInfo, i)) { continue; } @@ -4009,7 +4015,7 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResInfo *pResultInfo, int32_ qDebug("QInfo:%p start to copy data from windowResInfo to query buf", pQInfo); int32_t totalSet = numOfClosedTimeWindow(pResultInfo); - SWindowResult* result = pResultInfo->pResult; + SWindowResult** result = pResultInfo->pResult; if (orderType == TSDB_ORDER_ASC) { startIdx = pQInfo->groupIndex; @@ -4022,13 +4028,13 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResInfo *pResultInfo, int32_ SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo; for (int32_t i = startIdx; (i < totalSet) && (i >= 0); i += step) { - if (result[i].numOfRows == 0) { + if (result[i]->numOfRows == 0) { pQInfo->groupIndex += 1; pGroupResInfo->rowId = 0; continue; } - int32_t numOfRowsToCopy = result[i].numOfRows - pGroupResInfo->rowId; + int32_t numOfRowsToCopy = result[i]->numOfRows - pGroupResInfo->rowId; int32_t oldOffset = pGroupResInfo->rowId; /* @@ -4043,13 +4049,13 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResInfo *pResultInfo, int32_ pQInfo->groupIndex += 1; } - tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, result[i].pageId); + tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, result[i]->pageId); for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { int32_t size = pRuntimeEnv->pCtx[j].outputBytes; char *out = pQuery->sdata[j]->data + numOfResult * size; - char *in = getPosInResultPage(pRuntimeEnv, j, &result[i], page); + char *in = getPosInResultPage(pRuntimeEnv, j, result[i], page); memcpy(out, in + oldOffset * size, size * numOfRowsToCopy); } @@ -4096,7 +4102,7 @@ static void updateWindowResNumOfRes(SQueryRuntimeEnv *pRuntimeEnv) { } for (int32_t i = 0; i < pRuntimeEnv->windowResInfo.size; ++i) { - SWindowResult *pResult = &pRuntimeEnv->windowResInfo.pResult[i]; + SWindowResult *pResult = pRuntimeEnv->windowResInfo.pResult[i]; for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { int32_t functionId = pRuntimeEnv->pCtx[j].functionId; @@ -4251,31 +4257,22 @@ static void queryCostStatis(SQInfo *pQInfo) { SQueryCostInfo *pSummary = &pRuntimeEnv->summary; uint64_t hashSize = taosHashGetMemSize(pQInfo->runtimeEnv.pWindowHashTable); - hashSize += taosHashGetMemSize(pQInfo->tableqinfoGroupInfo.map); - int32_t numOfGroup = GET_NUM_OF_TABLEGROUP(pQInfo); - for(int32_t i = 0; i < numOfGroup; ++i) { - SArray* pa = GET_TABLEGROUP(pQInfo, i); - - int32_t numOfTables = taosArrayGetSize(pa); - for(int32_t j = 0; j < numOfTables; ++j) { -// STableQueryInfo* pTableQueryInfo = taosArrayGetP(pa, j); - -// hashSize += taosHashGetMemSize(pTableQueryInfo->windowResInfo.hashList); - } - } - pSummary->hashSize = hashSize; // add the merge time pSummary->elapsedTime += pSummary->firstStageMergeTime; + SWindowResultPool* p = pQInfo->runtimeEnv.pool; + pSummary->winInfoSize = getWindowResultPoolMemSize(p); + pSummary->numOfTimeWindows = getNumOfAllocatedWindowResult(p); + qDebug("QInfo:%p :cost summary: elapsed time:%"PRId64" us, first merge:%"PRId64" us, total blocks:%d, " "load block statis:%d, load data block:%d, total rows:%"PRId64 ", check rows:%"PRId64, pQInfo, pSummary->elapsedTime, pSummary->firstStageMergeTime, pSummary->totalBlocks, pSummary->loadBlockStatis, pSummary->loadBlocks, pSummary->totalRows, pSummary->totalCheckedRows); - qDebug("QInfo:%p :cost summary: windowInfo size:%.2f Kb, numOfWin:%"PRId64", tableInfoSize:%.2f Kb, hashTable:%.2f Kb", pQInfo, pSummary->winInfoSize/1024.0, + qDebug("QInfo:%p :cost summary: winResPool size:%.2f Kb, numOfWin:%"PRId64", tableInfoSize:%.2f Kb, hashTable:%.2f Kb", pQInfo, pSummary->winInfoSize/1024.0, pSummary->numOfTimeWindows, pSummary->tableInfoSize/1024.0, pSummary->hashSize/1024.0); } @@ -5034,9 +5031,9 @@ static void sequentialTableProcess(SQInfo *pQInfo) { } for (int32_t i = 0; i < pWindowResInfo->size; ++i) { - pWindowResInfo->pResult[i].closed = true; // enable return all results for group by normal columns + pWindowResInfo->pResult[i]->closed = true; // enable return all results for group by normal columns - SWindowResult *pResult = &pWindowResInfo->pResult[i]; + SWindowResult *pResult = pWindowResInfo->pResult[i]; for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { pResult->numOfRows = (uint16_t)(MAX(pResult->numOfRows, pResult->resultInfo[j].numOfRes)); } @@ -6337,6 +6334,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou pQInfo->runtimeEnv.pWindowHashTable = taosHashInit(pTableGroupInfo->numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); pQInfo->runtimeEnv.keyBuf = malloc(TSDB_MAX_BYTES_PER_ROW); + pQInfo->runtimeEnv.pool = initWindowResultPool(getWindowResultSize(&pQInfo->runtimeEnv)); pQInfo->pBuf = calloc(pTableGroupInfo->numOfTables, sizeof(STableQueryInfo)); if (pQInfo->pBuf == NULL) { diff --git a/src/query/src/qFill.c b/src/query/src/qFill.c index f186726c01..99fa3a8e0f 100644 --- a/src/query/src/qFill.c +++ b/src/query/src/qFill.c @@ -91,7 +91,7 @@ void taosResetFillInfo(SFillInfo* pFillInfo, TSKEY startTimestamp) { pFillInfo->numOfTotal = 0; } -void* taosDestoryFillInfo(SFillInfo* pFillInfo) { +void* taosDestroyFillInfo(SFillInfo* pFillInfo) { if (pFillInfo == NULL) { return NULL; } diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index 8273a21395..7ec39cc0c8 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -14,8 +14,9 @@ */ #include "os.h" -#include "hash.h" #include "taosmsg.h" +#include "hash.h" + #include "qExecutor.h" #include "qUtil.h" @@ -40,38 +41,29 @@ int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, SQueryRuntimeEnv *pRun pWindowResInfo->size = 0; pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL; - SQueryCostInfo* pSummary = &pRuntimeEnv->summary; +// SQueryCostInfo* pSummary = &pRuntimeEnv->summary; - // use the pointer arraylist - pWindowResInfo->pResult = calloc(pWindowResInfo->capacity, sizeof(SWindowResult)); + pWindowResInfo->pResult = calloc(pWindowResInfo->capacity, POINTER_BYTES); if (pWindowResInfo->pResult == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } pWindowResInfo->interval = pRuntimeEnv->pQuery->interval.interval; - pSummary->winInfoSize += sizeof(SWindowResult) * pWindowResInfo->capacity; - pSummary->winInfoSize += (pRuntimeEnv->pQuery->numOfOutput * sizeof(SResultInfo) + pRuntimeEnv->interBufSize) * pWindowResInfo->capacity; - pSummary->numOfTimeWindows = pWindowResInfo->capacity; +// pSummary->winInfoSize += POINTER_BYTES * pWindowResInfo->capacity; +// pSummary->winInfoSize += (pRuntimeEnv->pQuery->numOfOutput * sizeof(SResultInfo) + pRuntimeEnv->interBufSize) * pWindowResInfo->capacity; +// pSummary->numOfTimeWindows = pWindowResInfo->capacity; - for (int32_t i = 0; i < pWindowResInfo->capacity; ++i) { - int32_t code = createQueryResultInfo(pRuntimeEnv->pQuery, &pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, pRuntimeEnv->interBufSize); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - } +// for (int32_t i = 0; i < pWindowResInfo->capacity; ++i) { +// int32_t code = createQueryResultInfo(pRuntimeEnv->pQuery, pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, pRuntimeEnv->interBufSize); +// if (code != TSDB_CODE_SUCCESS) { +// return code; +// } +// } return TSDB_CODE_SUCCESS; } -void destroyTimeWindowRes(SWindowResult *pWindowRes) { - if (pWindowRes == NULL) { - return; - } - - free(pWindowRes->resultInfo); -} - void cleanupTimeWindowInfo(SWindowResInfo *pWindowResInfo) { if (pWindowResInfo == NULL) { return; @@ -81,13 +73,6 @@ void cleanupTimeWindowInfo(SWindowResInfo *pWindowResInfo) { return; } - if (pWindowResInfo->pResult != NULL) { - for (int32_t i = 0; i < pWindowResInfo->capacity; ++i) { - destroyTimeWindowRes(&pWindowResInfo->pResult[i]); - } - } - -// taosHashCleanup(pWindowResInfo->hashList); taosTFree(pWindowResInfo->pResult); } @@ -97,17 +82,13 @@ void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowR } for (int32_t i = 0; i < pWindowResInfo->size; ++i) { - SWindowResult *pWindowRes = &pWindowResInfo->pResult[i]; + SWindowResult *pWindowRes = pWindowResInfo->pResult[i]; clearTimeWindowResBuf(pRuntimeEnv, pWindowRes); } pWindowResInfo->curIndex = -1; -// taosHashCleanup(pWindowResInfo->hashList); pWindowResInfo->size = 0; -// _hash_fn_t fn = taosGetDefaultHashFunction(pWindowResInfo->type); -// pWindowResInfo->hashList = taosHashInit(pWindowResInfo->capacity, fn, true, false); - pWindowResInfo->startTime = TSKEY_INITIAL_VAL; pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL; } @@ -127,7 +108,7 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { int16_t bytes = -1; for (int32_t i = 0; i < num; ++i) { - SWindowResult *pResult = &pWindowResInfo->pResult[i]; + SWindowResult *pResult = pWindowResInfo->pResult[i]; if (pResult->closed) { // remove the window slot from hash table // todo refactor @@ -150,19 +131,19 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { // clear all the closed windows from the window list for (int32_t k = 0; k < remain; ++k) { - copyTimeWindowResBuf(pRuntimeEnv, &pWindowResInfo->pResult[k], &pWindowResInfo->pResult[num + k]); + copyTimeWindowResBuf(pRuntimeEnv, pWindowResInfo->pResult[k], pWindowResInfo->pResult[num + k]); } // move the unclosed window in the front of the window list for (int32_t k = remain; k < pWindowResInfo->size; ++k) { - SWindowResult *pWindowRes = &pWindowResInfo->pResult[k]; + SWindowResult *pWindowRes = pWindowResInfo->pResult[k]; clearTimeWindowResBuf(pRuntimeEnv, pWindowRes); } pWindowResInfo->size = remain; for (int32_t k = 0; k < pWindowResInfo->size; ++k) { - SWindowResult *pResult = &pWindowResInfo->pResult[k]; + SWindowResult *pResult = pWindowResInfo->pResult[k]; if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { key = varDataVal(pResult->key); @@ -198,7 +179,7 @@ void clearClosedTimeWindow(SQueryRuntimeEnv *pRuntimeEnv) { int32_t numOfClosedTimeWindow(SWindowResInfo *pWindowResInfo) { int32_t i = 0; - while (i < pWindowResInfo->size && pWindowResInfo->pResult[i].closed) { + while (i < pWindowResInfo->size && pWindowResInfo->pResult[i]->closed) { ++i; } @@ -209,11 +190,11 @@ void closeAllTimeWindow(SWindowResInfo *pWindowResInfo) { assert(pWindowResInfo->size >= 0 && pWindowResInfo->capacity >= pWindowResInfo->size); for (int32_t i = 0; i < pWindowResInfo->size; ++i) { - if (pWindowResInfo->pResult[i].closed) { + if (pWindowResInfo->pResult[i]->closed) { continue; } - pWindowResInfo->pResult[i].closed = true; + pWindowResInfo->pResult[i]->closed = true; } } @@ -229,19 +210,19 @@ void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_ } // get the result order - int32_t resultOrder = (pWindowResInfo->pResult[0].win.skey < pWindowResInfo->pResult[1].win.skey)? 1:-1; + int32_t resultOrder = (pWindowResInfo->pResult[0]->win.skey < pWindowResInfo->pResult[1]->win.skey)? 1:-1; if (order != resultOrder) { return; } int32_t i = 0; if (order == QUERY_ASC_FORWARD_STEP) { - TSKEY ekey = pWindowResInfo->pResult[i].win.ekey; + TSKEY ekey = pWindowResInfo->pResult[i]->win.ekey; while (i < pWindowResInfo->size && (ekey < lastKey)) { ++i; } } else if (order == QUERY_DESC_FORWARD_STEP) { - while (i < pWindowResInfo->size && (pWindowResInfo->pResult[i].win.skey > lastKey)) { + while (i < pWindowResInfo->size && (pWindowResInfo->pResult[i]->win.skey > lastKey)) { ++i; } } @@ -318,3 +299,77 @@ void copyTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *dst, con } } +size_t getWindowResultSize(SQueryRuntimeEnv* pRuntimeEnv) { + return (pRuntimeEnv->pQuery->numOfOutput * sizeof(SResultInfo)) + pRuntimeEnv->interBufSize + sizeof(SWindowResult); +} + +SWindowResultPool* initWindowResultPool(size_t size) { + SWindowResultPool* p = calloc(1, sizeof(SWindowResultPool)); + if (p == NULL) { + return NULL; + } + + p->numOfElemPerBlock = 128; + + p->elemSize = size; + p->blockSize = p->numOfElemPerBlock * p->elemSize; + + p->position.pos = 0; + + p->pData = taosArrayInit(8, POINTER_BYTES); + return p; +} + +SWindowResult* getNewWindowResult(SWindowResultPool* p) { + if (p == NULL) { + return NULL; + } + + void* ptr = NULL; + if (p->position.pos == 0) { + ptr = calloc(1, p->blockSize); + taosArrayPush(p->pData, &ptr); + + } else { + size_t last = taosArrayGetSize(p->pData); + + void** pBlock = taosArrayGet(p->pData, last - 1); + ptr = (*pBlock) + p->elemSize * p->position.pos; + } + + p->position.pos = (p->position.pos + 1)%p->numOfElemPerBlock; + return ptr; +} + +int64_t getWindowResultPoolMemSize(SWindowResultPool* p) { + if (p == NULL) { + return 0; + } + + return taosArrayGetSize(p->pData) * p->blockSize; +} + +int32_t getNumOfAllocatedWindowResult(SWindowResultPool* p) { + return taosArrayGetSize(p->pData) * p->numOfElemPerBlock; +} + +int32_t getNumOfUsedWindowResult(SWindowResultPool* p) { + return getNumOfAllocatedWindowResult(p) - p->numOfElemPerBlock + p->position.pos; +} + +void* destroyWindowResultPool(SWindowResultPool* p) { + if (p == NULL) { + return NULL; + } + + size_t size = taosArrayGetSize(p->pData); + for(int32_t i = 0; i < size; ++i) { + void** ptr = taosArrayGet(p->pData, i); + taosTFree(*ptr); + } + + taosArrayDestroy(p->pData); + + taosTFree(p); + return NULL; +} From 5bbd7dd9d640ca8d2127770a31789d2f72cecaf6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 2 Nov 2020 14:10:15 +0800 Subject: [PATCH 12/35] [TD-1870] --- src/client/inc/tscLocalMerge.h | 2 +- src/client/src/tscFunctionImpl.c | 384 +++++++++++++++---------------- src/client/src/tscLocalMerge.c | 14 +- src/query/inc/qExecutor.h | 14 +- src/query/inc/qUtil.h | 13 +- src/query/inc/tsqlfunction.h | 30 +-- src/query/src/qExecutor.c | 198 ++++++++-------- src/query/src/qResultbuf.c | 6 +- src/query/src/qUtil.c | 59 ++--- src/util/inc/hash.h | 9 +- src/util/src/hash.c | 49 ++-- 11 files changed, 379 insertions(+), 399 deletions(-) diff --git a/src/client/inc/tscLocalMerge.h b/src/client/inc/tscLocalMerge.h index 5baa66a9e0..edd83ad071 100644 --- a/src/client/inc/tscLocalMerge.h +++ b/src/client/inc/tscLocalMerge.h @@ -66,7 +66,7 @@ typedef struct SLocalReducer { SFillInfo* pFillInfo; // interpolation support structure char * pFinalRes; // result data after interpo tFilePage * discardData; - SResultInfo * pResInfo; + SResultRowCellInfo * pResInfo; bool discard; int32_t offset; // limit offset value bool orderPrjOnSTable; // projection query on stable diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index 12d3b7dfd3..b22c178466 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -167,7 +167,7 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_INTERP) { *type = (int16_t)dataType; *bytes = (int16_t)dataBytes; - *interBytes = *bytes + sizeof(SResultInfo); + *interBytes = 0;//*bytes; return TSDB_CODE_SUCCESS; } @@ -175,21 +175,21 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI if (functionId == TSDB_FUNC_TID_TAG) { // todo use struct *type = TSDB_DATA_TYPE_BINARY; *bytes = (int16_t)(dataBytes + sizeof(int16_t) + sizeof(int64_t) + sizeof(int32_t) + sizeof(int32_t) + VARSTR_HEADER_SIZE); - *interBytes = *bytes; + *interBytes = 0;//*bytes; return TSDB_CODE_SUCCESS; } if (functionId == TSDB_FUNC_COUNT) { *type = TSDB_DATA_TYPE_BIGINT; *bytes = sizeof(int64_t); - *interBytes = *bytes; + *interBytes = 0;//*bytes; return TSDB_CODE_SUCCESS; } if (functionId == TSDB_FUNC_ARITHM) { *type = TSDB_DATA_TYPE_DOUBLE; *bytes = sizeof(double); - *interBytes = *bytes; + *interBytes = 0;//*bytes; return TSDB_CODE_SUCCESS; } @@ -298,7 +298,7 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI } else if (functionId == TSDB_FUNC_FIRST || functionId == TSDB_FUNC_LAST) { *type = (int16_t)dataType; *bytes = (int16_t)dataBytes; - *interBytes = dataBytes + sizeof(SResultInfo); + *interBytes = dataBytes; } else if (functionId == TSDB_FUNC_SPREAD) { *type = (int16_t)TSDB_DATA_TYPE_DOUBLE; *bytes = sizeof(double); @@ -310,7 +310,7 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI } else if (functionId == TSDB_FUNC_LEASTSQR) { *type = TSDB_DATA_TYPE_BINARY; *bytes = TSDB_AVG_FUNCTION_INTER_BUFFER_SIZE; // string - *interBytes = *bytes + sizeof(SResultInfo); + *interBytes = *bytes; } else if (functionId == TSDB_FUNC_FIRST_DST || functionId == TSDB_FUNC_LAST_DST) { *type = TSDB_DATA_TYPE_BINARY; *bytes = (int16_t)(dataBytes + sizeof(SFirstLastInfo)); @@ -334,28 +334,25 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI return TSDB_CODE_SUCCESS; } -void setResultInfoBuf(SResultInfo *pResInfo, int32_t size, bool superTable, char* buf) { - assert(pResInfo->interResultBuf == NULL); - - pResInfo->bufLen = size; - pResInfo->superTableQ = superTable; - pResInfo->interResultBuf = buf; -} +//void setResultInfoBuf(SResultRowCellInfo *pResInfo, char* buf) { +// assert(GET_ROWCELL_INTERBUF(pResInfo) == NULL); +// GET_ROWCELL_INTERBUF(pResInfo) = buf; +//} // set the query flag to denote that query is completed static void no_next_step(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->complete = true; } static bool function_setup(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo->initialized) { return false; } memset(pCtx->aOutputBuf, 0, (size_t)pCtx->outputBytes); - initResultInfo(pResInfo); + initResultInfo(pResInfo, pCtx->interBufBytes); return true; } @@ -367,7 +364,7 @@ static bool function_setup(SQLFunctionCtx *pCtx) { * @param pCtx */ static void function_finalizer(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo->hasResult != DATA_SET_FLAG) { if (pCtx->outputType == TSDB_DATA_TYPE_BINARY || pCtx->outputType == TSDB_DATA_TYPE_NCHAR) { setVardataNull(pCtx->aOutputBuf, pCtx->outputType); @@ -431,7 +428,7 @@ static void count_function_f(SQLFunctionCtx *pCtx, int32_t index) { *((int64_t *)pCtx->aOutputBuf) += 1; // do not need it actually - SResultInfo *pInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx); pInfo->hasResult = DATA_SET_FLAG; } @@ -592,8 +589,8 @@ static void sum_function(SQLFunctionCtx *pCtx) { do_sum(pCtx); // keep the result data in output buffer, not in the intermediate buffer - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - if (pResInfo->hasResult == DATA_SET_FLAG && pResInfo->superTableQ) { + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + if (pResInfo->hasResult == DATA_SET_FLAG && pCtx->stableQuery) { // set the flag for super table query SSumInfo *pSum = (SSumInfo *)pCtx->aOutputBuf; pSum->hasResult = DATA_SET_FLAG; @@ -604,8 +601,8 @@ static void sum_function_f(SQLFunctionCtx *pCtx, int32_t index) { do_sum_f(pCtx, index); // keep the result data in output buffer, not in the intermediate buffer - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - if (pResInfo->hasResult == DATA_SET_FLAG && pResInfo->superTableQ) { + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + if (pResInfo->hasResult == DATA_SET_FLAG && pCtx->stableQuery) { SSumInfo *pSum = (SSumInfo *)pCtx->aOutputBuf; pSum->hasResult = DATA_SET_FLAG; } @@ -615,8 +612,7 @@ static int32_t sum_merge_impl(const SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; GET_TRUE_DATA_TYPE(); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - assert(pResInfo->superTableQ); + assert(pCtx->stableQuery); for (int32_t i = 0; i < pCtx->size; ++i) { char * input = GET_INPUT_CHAR_INDEX(pCtx, i); @@ -661,7 +657,7 @@ static void sum_func_second_merge(SQLFunctionCtx *pCtx) { int32_t notNullElems = sum_merge_impl(pCtx); SET_VAL(pCtx, notNullElems, 1); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); if (notNullElems > 0) { pResInfo->hasResult = DATA_SET_FLAG; @@ -755,9 +751,9 @@ static void avg_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; // NOTE: keep the intermediate result into the interResultBuf - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - SAvgInfo *pAvgInfo = (SAvgInfo *)pResInfo->interResultBuf; + SAvgInfo *pAvgInfo = (SAvgInfo *)GET_ROWCELL_INTERBUF(pResInfo); double * pVal = &pAvgInfo->sum; if (pCtx->preAggVals.isSet) { @@ -800,8 +796,8 @@ static void avg_function(SQLFunctionCtx *pCtx) { } // keep the data into the final output buffer for super table query since this execution may be the last one - if (pResInfo->superTableQ) { - memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SAvgInfo)); + if (pCtx->stableQuery) { + memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SAvgInfo)); } } @@ -814,9 +810,9 @@ static void avg_function_f(SQLFunctionCtx *pCtx, int32_t index) { SET_VAL(pCtx, 1, 1); // NOTE: keep the intermediate result into the interResultBuf - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - SAvgInfo *pAvgInfo = (SAvgInfo *)pResInfo->interResultBuf; + SAvgInfo *pAvgInfo = (SAvgInfo *)GET_ROWCELL_INTERBUF(pResInfo); if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { pAvgInfo->sum += GET_INT8_VAL(pData); @@ -839,16 +835,16 @@ static void avg_function_f(SQLFunctionCtx *pCtx, int32_t index) { pResInfo->hasResult = DATA_SET_FLAG; // keep the data into the final output buffer for super table query since this execution may be the last one - if (pResInfo->superTableQ) { - memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SAvgInfo)); + if (pCtx->stableQuery) { + memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SAvgInfo)); } } static void avg_func_merge(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - assert(pResInfo->superTableQ); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + assert(pCtx->stableQuery); - SAvgInfo *pAvgInfo = (SAvgInfo *)pResInfo->interResultBuf; + SAvgInfo *pAvgInfo = (SAvgInfo *)GET_ROWCELL_INTERBUF(pResInfo); char * input = GET_INPUT_CHAR(pCtx); for (int32_t i = 0; i < pCtx->size; ++i, input += pCtx->inputBytes) { @@ -864,12 +860,12 @@ static void avg_func_merge(SQLFunctionCtx *pCtx) { // if the data set hasResult is not set, the result is null if (pAvgInfo->num > 0) { pResInfo->hasResult = DATA_SET_FLAG; - memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SAvgInfo)); + memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SAvgInfo)); } } static void avg_func_second_merge(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); double *sum = (double*) pCtx->aOutputBuf; char * input = GET_INPUT_CHAR(pCtx); @@ -883,7 +879,7 @@ static void avg_func_second_merge(SQLFunctionCtx *pCtx) { *sum += pInput->sum; // keep the number of data into the temp buffer - *(int64_t *)pResInfo->interResultBuf += pInput->num; + *(int64_t *)GET_ROWCELL_INTERBUF(pResInfo) += pInput->num; } } @@ -891,21 +887,21 @@ static void avg_func_second_merge(SQLFunctionCtx *pCtx) { * the average value is calculated in finalize routine, since current routine does not know the exact number of points */ static void avg_finalizer(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); if (pCtx->currentStage == SECONDARY_STAGE_MERGE) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); - if (GET_INT64_VAL(pResInfo->interResultBuf) <= 0) { + if (GET_INT64_VAL(GET_ROWCELL_INTERBUF(pResInfo)) <= 0) { setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); return; // empty table } - *(double *)pCtx->aOutputBuf = (*(double *)pCtx->aOutputBuf) / *(int64_t *)pResInfo->interResultBuf; + *(double *)pCtx->aOutputBuf = (*(double *)pCtx->aOutputBuf) / *(int64_t *)GET_ROWCELL_INTERBUF(pResInfo); } else { // this is the secondary merge, only in the secondary merge, the input type is TSDB_DATA_TYPE_BINARY assert(pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_DOUBLE); - SAvgInfo *pAvgInfo = (SAvgInfo *)pResInfo->interResultBuf; + SAvgInfo *pAvgInfo = (SAvgInfo *)GET_ROWCELL_INTERBUF(pResInfo); if (pAvgInfo->num == 0) { // all data are NULL or empty table setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); @@ -1116,11 +1112,11 @@ static void min_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, notNullElems, 1); if (notNullElems > 0) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; // set the flag for super table query - if (pResInfo->superTableQ) { + if (pCtx->stableQuery) { *(pCtx->aOutputBuf + pCtx->inputBytes) = DATA_SET_FLAG; } } @@ -1133,11 +1129,11 @@ static void max_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, notNullElems, 1); if (notNullElems > 0) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; // set the flag for super table query - if (pResInfo->superTableQ) { + if (pCtx->stableQuery) { *(pCtx->aOutputBuf + pCtx->inputBytes) = DATA_SET_FLAG; } } @@ -1148,8 +1144,7 @@ static int32_t minmax_merge_impl(SQLFunctionCtx *pCtx, int32_t bytes, char *outp GET_TRUE_DATA_TYPE(); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - assert(pResInfo->superTableQ); + assert(pCtx->stableQuery); for (int32_t i = 0; i < pCtx->size; ++i) { char *input = GET_INPUT_CHAR_INDEX(pCtx, i); @@ -1210,7 +1205,7 @@ static void min_func_merge(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, notNullElems, 1); - if (notNullElems > 0) { // for super table query, SResultInfo is not used + if (notNullElems > 0) { // for super table query, SResultRowCellInfo is not used char *flag = pCtx->aOutputBuf + pCtx->inputBytes; *flag = DATA_SET_FLAG; } @@ -1221,7 +1216,7 @@ static void min_func_second_merge(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, notNullElems, 1); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); if (notNullElems > 0) { pResInfo->hasResult = DATA_SET_FLAG; } @@ -1242,7 +1237,7 @@ static void max_func_second_merge(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, numOfElem, 1); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); if (numOfElem > 0) { pResInfo->hasResult = DATA_SET_FLAG; } @@ -1297,8 +1292,8 @@ static void max_function_f(SQLFunctionCtx *pCtx, int32_t index) { SET_VAL(pCtx, 1, 1); minMax_function_f(pCtx, index, 0); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - if (pResInfo->hasResult == DATA_SET_FLAG && pResInfo->superTableQ) { + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + if (pResInfo->hasResult == DATA_SET_FLAG && pCtx->stableQuery) { char *flag = pCtx->aOutputBuf + pCtx->inputBytes; *flag = DATA_SET_FLAG; } @@ -1313,8 +1308,8 @@ static void min_function_f(SQLFunctionCtx *pCtx, int32_t index) { SET_VAL(pCtx, 1, 1); minMax_function_f(pCtx, index, 1); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - if (pResInfo->hasResult == DATA_SET_FLAG && pResInfo->superTableQ) { + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + if (pResInfo->hasResult == DATA_SET_FLAG && pCtx->stableQuery) { char *flag = pCtx->aOutputBuf + pCtx->inputBytes; *flag = DATA_SET_FLAG; } @@ -1330,7 +1325,7 @@ static void min_function_f(SQLFunctionCtx *pCtx, int32_t index) { static void stddev_function(SQLFunctionCtx *pCtx) { // the second stage to calculate standard deviation - SStddevInfo *pStd = GET_RES_INFO(pCtx)->interResultBuf; + SStddevInfo *pStd = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); if (pStd->stage == 0) { // the first stage is to calculate average value avg_function(pCtx); @@ -1381,8 +1376,8 @@ static void stddev_function(SQLFunctionCtx *pCtx) { static void stddev_function_f(SQLFunctionCtx *pCtx, int32_t index) { // the second stage to calculate standard deviation - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - SStddevInfo *pStd = pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SStddevInfo *pStd = GET_ROWCELL_INTERBUF(pResInfo); /* the first stage is to calculate average value */ if (pStd->stage == 0) { @@ -1433,8 +1428,8 @@ static void stddev_next_step(SQLFunctionCtx *pCtx) { * the stddevInfo and the average info struct share the same buffer area * And the position of each element in their struct is exactly the same matched */ - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - SStddevInfo *pStd = pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SStddevInfo *pStd = GET_ROWCELL_INTERBUF(pResInfo); if (pStd->stage == 0) { /* @@ -1449,7 +1444,7 @@ static void stddev_next_step(SQLFunctionCtx *pCtx) { pResInfo->initialized = true; // set it initialized to avoid re-initialization // save average value into tmpBuf, for second stage scan - SAvgInfo *pAvg = pResInfo->interResultBuf; + SAvgInfo *pAvg = GET_ROWCELL_INTERBUF(pResInfo); pStd->avg = GET_DOUBLE_VAL(pCtx->aOutputBuf); assert((isnan(pAvg->sum) && pAvg->num == 0) || (pStd->num == pAvg->num && pStd->avg == pAvg->sum)); @@ -1459,7 +1454,7 @@ static void stddev_next_step(SQLFunctionCtx *pCtx) { } static void stddev_finalizer(SQLFunctionCtx *pCtx) { - SStddevInfo *pStd = (SStddevInfo *)GET_RES_INFO(pCtx)->interResultBuf; + SStddevInfo *pStd = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); if (pStd->num <= 0) { setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); @@ -1505,7 +1500,7 @@ static void first_function(SQLFunctionCtx *pCtx) { TSKEY k = pCtx->ptsList[i]; DO_UPDATE_TAG_COLUMNS(pCtx, k); - SResultInfo *pInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx); pInfo->hasResult = DATA_SET_FLAG; pInfo->complete = true; @@ -1532,7 +1527,7 @@ static void first_function_f(SQLFunctionCtx *pCtx, int32_t index) { TSKEY ts = pCtx->ptsList[index]; DO_UPDATE_TAG_COLUMNS(pCtx, ts); - SResultInfo *pInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx); pInfo->hasResult = DATA_SET_FLAG; pInfo->complete = true; // get the first not-null data, completed } @@ -1576,7 +1571,7 @@ static void first_dist_function(SQLFunctionCtx *pCtx) { first_data_assign_impl(pCtx, data, i); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; notNullElems++; @@ -1604,8 +1599,7 @@ static void first_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) { static void first_dist_func_merge(SQLFunctionCtx *pCtx) { char *pData = GET_INPUT_CHAR(pCtx); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - assert(pCtx->size == 1 && pResInfo->superTableQ); + assert(pCtx->size == 1 && pCtx->stableQuery); SFirstLastInfo *pInput = (SFirstLastInfo *)(pData + pCtx->inputBytes); if (pInput->hasResult != DATA_SET_FLAG) { @@ -1620,8 +1614,8 @@ static void first_dist_func_merge(SQLFunctionCtx *pCtx) { } static void first_dist_func_second_merge(SQLFunctionCtx *pCtx) { - assert(pCtx->resultInfo->superTableQ); - + assert(pCtx->stableQuery); + char * pData = GET_INPUT_CHAR(pCtx); SFirstLastInfo *pInput = (SFirstLastInfo*) (pData + pCtx->outputBytes); if (pInput->hasResult != DATA_SET_FLAG) { @@ -1668,7 +1662,7 @@ static void last_function(SQLFunctionCtx *pCtx) { TSKEY ts = pCtx->ptsList[i]; DO_UPDATE_TAG_COLUMNS(pCtx, ts); - SResultInfo *pInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pInfo = GET_RES_INFO(pCtx); pInfo->hasResult = DATA_SET_FLAG; pInfo->complete = true; // set query completed on this column @@ -1691,7 +1685,7 @@ static void last_function_f(SQLFunctionCtx *pCtx, int32_t index) { TSKEY ts = pCtx->ptsList[index]; DO_UPDATE_TAG_COLUMNS(pCtx, ts); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; pResInfo->complete = true; // set query completed } @@ -1740,7 +1734,7 @@ static void last_dist_function(SQLFunctionCtx *pCtx) { last_data_assign_impl(pCtx, data, i); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; notNullElems++; @@ -1776,8 +1770,7 @@ static void last_dist_function_f(SQLFunctionCtx *pCtx, int32_t index) { static void last_dist_func_merge(SQLFunctionCtx *pCtx) { char *pData = GET_INPUT_CHAR(pCtx); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - assert(pCtx->size == 1 && pResInfo->superTableQ); + assert(pCtx->size == 1 && pCtx->stableQuery); // the input data is null SFirstLastInfo *pInput = (SFirstLastInfo *)(pData + pCtx->inputBytes); @@ -1833,11 +1826,11 @@ static void last_row_function(SQLFunctionCtx *pCtx) { // assign the last element in current data block assignVal(pCtx->aOutputBuf, pData + (pCtx->size - 1) * pCtx->inputBytes, pCtx->inputBytes, pCtx->inputType); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; // set the result to final result buffer in case of super table query - if (pResInfo->superTableQ) { + if (pCtx->stableQuery) { SLastrowInfo *pInfo1 = (SLastrowInfo *)(pCtx->aOutputBuf + pCtx->inputBytes); pInfo1->ts = pCtx->ptsList[pCtx->size - 1]; pInfo1->hasResult = DATA_SET_FLAG; @@ -1852,7 +1845,7 @@ static void last_row_function(SQLFunctionCtx *pCtx) { static void last_row_finalizer(SQLFunctionCtx *pCtx) { // do nothing at the first stage - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo->hasResult != DATA_SET_FLAG) { if (pCtx->outputType == TSDB_DATA_TYPE_BINARY || pCtx->outputType == TSDB_DATA_TYPE_NCHAR) { setVardataNull(pCtx->aOutputBuf, pCtx->outputType); @@ -2044,8 +2037,8 @@ static int32_t resDataAscComparFn(const void *pLeft, const void *pRight) { static int32_t resDataDescComparFn(const void *pLeft, const void *pRight) { return -resDataAscComparFn(pLeft, pRight); } static void copyTopBotRes(SQLFunctionCtx *pCtx, int32_t type) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - STopBotInfo *pRes = pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + STopBotInfo *pRes = GET_ROWCELL_INTERBUF(pResInfo); tValuePair **tvp = pRes->res; @@ -2135,18 +2128,18 @@ static void copyTopBotRes(SQLFunctionCtx *pCtx, int32_t type) { * top/bottom use the intermediate result buffer to keep the intermediate result */ static STopBotInfo *getTopBotOutputInfo(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); // only the first_stage_merge is directly written data into final output buffer - if (pResInfo->superTableQ && pCtx->currentStage != SECONDARY_STAGE_MERGE) { + if (pCtx->stableQuery && pCtx->currentStage != SECONDARY_STAGE_MERGE) { return (STopBotInfo*) pCtx->aOutputBuf; } else { // during normal table query and super table at the secondary_stage, result is written to intermediate buffer - return pResInfo->interResultBuf; + return GET_ROWCELL_INTERBUF(pResInfo); } } bool topbot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, const char *minval, const char *maxval) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); if (pResInfo == NULL) { return true; } @@ -2252,7 +2245,7 @@ static void top_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, notNullElems, 1); if (notNullElems > 0) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; } } @@ -2270,7 +2263,7 @@ static void top_function_f(SQLFunctionCtx *pCtx, int32_t index) { do_top_function_add(pRes, (int32_t)pCtx->param[0].i64Key, pData, pCtx->ptsList[index], pCtx->inputType, &pCtx->tagInfo, NULL, 0); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; } @@ -2285,8 +2278,7 @@ static void top_func_merge(SQLFunctionCtx *pCtx) { // remmap the input buffer may cause the struct pointer invalid, so rebuild the STopBotInfo is necessary buildTopBotStruct(pInput, pCtx); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - assert(pResInfo->superTableQ && pCtx->outputType == TSDB_DATA_TYPE_BINARY && pCtx->size == 1); + assert(pCtx->stableQuery && pCtx->outputType == TSDB_DATA_TYPE_BINARY && pCtx->size == 1); STopBotInfo *pOutput = getTopBotOutputInfo(pCtx); @@ -2314,7 +2306,7 @@ static void top_func_second_merge(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, pInput->num, pOutput->num); if (pOutput->num > 0) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; } } @@ -2343,7 +2335,7 @@ static void bottom_function(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, notNullElems, 1); if (notNullElems > 0) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; } } @@ -2359,7 +2351,7 @@ static void bottom_function_f(SQLFunctionCtx *pCtx, int32_t index) { do_bottom_function_add(pRes, (int32_t)pCtx->param[0].i64Key, pData, pCtx->ptsList[index], pCtx->inputType, &pCtx->tagInfo, NULL, 0); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; } @@ -2374,8 +2366,7 @@ static void bottom_func_merge(SQLFunctionCtx *pCtx) { // remmap the input buffer may cause the struct pointer invalid, so rebuild the STopBotInfo is necessary buildTopBotStruct(pInput, pCtx); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - assert(pResInfo->superTableQ && pCtx->outputType == TSDB_DATA_TYPE_BINARY && pCtx->size == 1); + assert(pCtx->stableQuery && pCtx->outputType == TSDB_DATA_TYPE_BINARY && pCtx->size == 1); STopBotInfo *pOutput = getTopBotOutputInfo(pCtx); @@ -2403,16 +2394,16 @@ static void bottom_func_second_merge(SQLFunctionCtx *pCtx) { SET_VAL(pCtx, pInput->num, pOutput->num); if (pOutput->num > 0) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; } } static void top_bottom_func_finalizer(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); // data in temporary list is less than the required number of results, not enough qualified number of results - STopBotInfo *pRes = pResInfo->interResultBuf; + STopBotInfo *pRes = GET_ROWCELL_INTERBUF(pResInfo); if (pRes->num == 0) { // no result assert(pResInfo->hasResult != DATA_SET_FLAG); // TODO: @@ -2443,8 +2434,8 @@ static bool percentile_function_setup(SQLFunctionCtx *pCtx) { } // in the first round, get the min-max value of all involved data - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - SPercentileInfo *pInfo = pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); SET_DOUBLE_VAL(&pInfo->minval, DBL_MAX); SET_DOUBLE_VAL(&pInfo->maxval, -DBL_MAX); pInfo->numOfElems = 0; @@ -2455,8 +2446,8 @@ static bool percentile_function_setup(SQLFunctionCtx *pCtx) { static void percentile_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; - SResultInfo * pResInfo = GET_RES_INFO(pCtx); - SPercentileInfo *pInfo = pResInfo->interResultBuf; + SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); + SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); // the first stage, only acquire the min/max value if (pInfo->stage == 0) { @@ -2546,9 +2537,9 @@ static void percentile_function_f(SQLFunctionCtx *pCtx, int32_t index) { return; } - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - SPercentileInfo *pInfo = (SPercentileInfo *)pResInfo->interResultBuf; + SPercentileInfo *pInfo = (SPercentileInfo *)GET_ROWCELL_INTERBUF(pResInfo); if (pInfo->stage == 0) { // TODO extract functions @@ -2595,8 +2586,8 @@ static void percentile_function_f(SQLFunctionCtx *pCtx, int32_t index) { static void percentile_finalizer(SQLFunctionCtx *pCtx) { double v = pCtx->param[0].nType == TSDB_DATA_TYPE_INT ? pCtx->param[0].i64Key : pCtx->param[0].dKey; - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - tMemBucket * pMemBucket = ((SPercentileInfo *)pResInfo->interResultBuf)->pMemBucket; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + tMemBucket * pMemBucket = ((SPercentileInfo *)GET_ROWCELL_INTERBUF(pResInfo))->pMemBucket; if (pMemBucket->total > 0) { // check for null *(double *)pCtx->aOutputBuf = getPercentile(pMemBucket, v); @@ -2609,8 +2600,8 @@ static void percentile_finalizer(SQLFunctionCtx *pCtx) { } static void percentile_next_step(SQLFunctionCtx *pCtx) { - SResultInfo * pResInfo = GET_RES_INFO(pCtx); - SPercentileInfo *pInfo = pResInfo->interResultBuf; + SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); + SPercentileInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); if (pInfo->stage == 0) { // all data are null, set it completed @@ -2627,12 +2618,12 @@ static void percentile_next_step(SQLFunctionCtx *pCtx) { ////////////////////////////////////////////////////////////////////////////////// static SAPercentileInfo *getAPerctInfo(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - if (pResInfo->superTableQ && pCtx->currentStage != SECONDARY_STAGE_MERGE) { + if (pCtx->stableQuery && pCtx->currentStage != SECONDARY_STAGE_MERGE) { return (SAPercentileInfo*) pCtx->aOutputBuf; } else { - return pResInfo->interResultBuf; + return GET_ROWCELL_INTERBUF(pResInfo); } } @@ -2651,7 +2642,7 @@ static bool apercentile_function_setup(SQLFunctionCtx *pCtx) { static void apercentile_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; - SResultInfo * pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); SAPercentileInfo *pInfo = getAPerctInfo(pCtx); for (int32_t i = 0; i < pCtx->size; ++i) { @@ -2704,8 +2695,8 @@ static void apercentile_function_f(SQLFunctionCtx *pCtx, int32_t index) { return; } - SResultInfo * pResInfo = GET_RES_INFO(pCtx); - SAPercentileInfo *pInfo = getAPerctInfo(pCtx); // pResInfo->interResultBuf; + SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); + SAPercentileInfo *pInfo = getAPerctInfo(pCtx); double v = 0; switch (pCtx->inputType) { @@ -2736,8 +2727,8 @@ static void apercentile_function_f(SQLFunctionCtx *pCtx, int32_t index) { } static void apercentile_func_merge(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - assert(pResInfo->superTableQ); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + assert(pCtx->stableQuery); SAPercentileInfo *pInput = (SAPercentileInfo *)GET_INPUT_CHAR(pCtx); @@ -2794,7 +2785,7 @@ static void apercentile_func_second_merge(SQLFunctionCtx *pCtx) { pOutput->pHisto = pRes; } - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); pResInfo->hasResult = DATA_SET_FLAG; SET_VAL(pCtx, 1, 1); } @@ -2802,8 +2793,8 @@ static void apercentile_func_second_merge(SQLFunctionCtx *pCtx) { static void apercentile_finalizer(SQLFunctionCtx *pCtx) { double v = (pCtx->param[0].nType == TSDB_DATA_TYPE_INT) ? pCtx->param[0].i64Key : pCtx->param[0].dKey; - SResultInfo * pResInfo = GET_RES_INFO(pCtx); - SAPercentileInfo *pOutput = pResInfo->interResultBuf; + SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); + SAPercentileInfo *pOutput = GET_ROWCELL_INTERBUF(pResInfo); if (pCtx->currentStage == SECONDARY_STAGE_MERGE) { if (pResInfo->hasResult == DATA_SET_FLAG) { // check for null @@ -2840,8 +2831,8 @@ static bool leastsquares_function_setup(SQLFunctionCtx *pCtx) { return false; } - SResultInfo * pResInfo = GET_RES_INFO(pCtx); - SLeastsquareInfo *pInfo = pResInfo->interResultBuf; + SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); + SLeastsquareInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); // 2*3 matrix pInfo->startVal = pCtx->param[0].dKey; @@ -2867,8 +2858,8 @@ static bool leastsquares_function_setup(SQLFunctionCtx *pCtx) { } static void leastsquares_function(SQLFunctionCtx *pCtx) { - SResultInfo * pResInfo = GET_RES_INFO(pCtx); - SLeastsquareInfo *pInfo = pResInfo->interResultBuf; + SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); + SLeastsquareInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); double(*param)[3] = pInfo->mat; double x = pInfo->startVal; @@ -2938,8 +2929,8 @@ static void leastsquares_function_f(SQLFunctionCtx *pCtx, int32_t index) { return; } - SResultInfo * pResInfo = GET_RES_INFO(pCtx); - SLeastsquareInfo *pInfo = pResInfo->interResultBuf; + SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); + SLeastsquareInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); double(*param)[3] = pInfo->mat; @@ -2988,8 +2979,8 @@ static void leastsquares_function_f(SQLFunctionCtx *pCtx, int32_t index) { static void leastsquares_finalizer(SQLFunctionCtx *pCtx) { // no data in query - SResultInfo * pResInfo = GET_RES_INFO(pCtx); - SLeastsquareInfo *pInfo = pResInfo->interResultBuf; + SResultRowCellInfo * pResInfo = GET_RES_INFO(pCtx); + SLeastsquareInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); if (pInfo->num == 0) { if (pCtx->outputType == TSDB_DATA_TYPE_BINARY || pCtx->outputType == TSDB_DATA_TYPE_NCHAR) { @@ -3054,7 +3045,7 @@ static void col_project_function(SQLFunctionCtx *pCtx) { } static void col_project_function_f(SQLFunctionCtx *pCtx, int32_t index) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); if (pCtx->numOfParams == 2) { // the number of output rows should not affect the final number of rows, so set it to be 0 return; } @@ -3486,7 +3477,7 @@ static bool spread_function_setup(SQLFunctionCtx *pCtx) { return false; } - SSpreadInfo *pInfo = GET_RES_INFO(pCtx)->interResultBuf; + SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); // this is the server-side setup function in client-side, the secondary merge do not need this procedure if (pCtx->currentStage == SECONDARY_STAGE_MERGE) { @@ -3501,8 +3492,8 @@ static bool spread_function_setup(SQLFunctionCtx *pCtx) { } static void spread_function(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - SSpreadInfo *pInfo = pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); int32_t numOfElems = 0; @@ -3568,8 +3559,8 @@ static void spread_function(SQLFunctionCtx *pCtx) { } // keep the data into the final output buffer for super table query since this execution may be the last one - if (pResInfo->superTableQ) { - memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SSpreadInfo)); + if (pCtx->stableQuery) { + memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SSpreadInfo)); } } @@ -3581,8 +3572,8 @@ static void spread_function_f(SQLFunctionCtx *pCtx, int32_t index) { SET_VAL(pCtx, 1, 1); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - SSpreadInfo *pInfo = pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); double val = 0.0; if (pCtx->inputType == TSDB_DATA_TYPE_TINYINT) { @@ -3611,16 +3602,16 @@ static void spread_function_f(SQLFunctionCtx *pCtx, int32_t index) { pResInfo->hasResult = DATA_SET_FLAG; pInfo->hasResult = DATA_SET_FLAG; - if (pResInfo->superTableQ) { - memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SSpreadInfo)); + if (pCtx->stableQuery) { + memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SSpreadInfo)); } } void spread_func_merge(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - assert(pResInfo->superTableQ); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + assert(pCtx->stableQuery); - SSpreadInfo *pResData = pResInfo->interResultBuf; + SSpreadInfo *pResData = GET_ROWCELL_INTERBUF(pResInfo); int32_t notNullElems = 0; for (int32_t i = 0; i < pCtx->size; ++i) { @@ -3644,7 +3635,7 @@ void spread_func_merge(SQLFunctionCtx *pCtx) { } if (notNullElems > 0) { - memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SSpreadInfo)); + memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SSpreadInfo)); pResInfo->hasResult = DATA_SET_FLAG; } } @@ -3675,7 +3666,7 @@ void spread_function_finalizer(SQLFunctionCtx *pCtx) { * here we do not check the input data types, because in case of metric query, * the type of intermediate data is binary */ - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); if (pCtx->currentStage == SECONDARY_STAGE_MERGE) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); @@ -3690,7 +3681,7 @@ void spread_function_finalizer(SQLFunctionCtx *pCtx) { assert((pCtx->inputType >= TSDB_DATA_TYPE_TINYINT && pCtx->inputType <= TSDB_DATA_TYPE_DOUBLE) || (pCtx->inputType == TSDB_DATA_TYPE_TIMESTAMP)); - SSpreadInfo *pInfo = GET_RES_INFO(pCtx)->interResultBuf; + SSpreadInfo *pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx)); if (pInfo->hasResult != DATA_SET_FLAG) { setNull(pCtx->aOutputBuf, pCtx->outputType, pCtx->outputBytes); return; @@ -3714,8 +3705,8 @@ static bool twa_function_setup(SQLFunctionCtx *pCtx) { return false; } - SResultInfo *pResInfo = GET_RES_INFO(pCtx); //->aOutputBuf + pCtx->outputBytes; - STwaInfo * pInfo = pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); //->aOutputBuf + pCtx->outputBytes; + STwaInfo * pInfo = GET_ROWCELL_INTERBUF(pResInfo); pInfo->lastKey = INT64_MIN; pInfo->type = pCtx->inputType; @@ -3754,8 +3745,8 @@ static void twa_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - STwaInfo * pInfo = pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + STwaInfo * pInfo = GET_ROWCELL_INTERBUF(pResInfo); int32_t i = 0; @@ -3808,7 +3799,7 @@ static void twa_function(SQLFunctionCtx *pCtx) { pResInfo->hasResult = DATA_SET_FLAG; } - if (pResInfo->superTableQ) { + if (pCtx->stableQuery) { memcpy(pCtx->aOutputBuf, pInfo, sizeof(STwaInfo)); } @@ -3825,8 +3816,8 @@ static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) { TSKEY *primaryKey = pCtx->ptsList; - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - STwaInfo *pInfo = pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + STwaInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); if (pInfo->lastKey == INT64_MIN) { pInfo->lastKey = pCtx->nStartQueryTimestamp; @@ -3848,14 +3839,13 @@ static void twa_function_f(SQLFunctionCtx *pCtx, int32_t index) { // pCtx->numOfIteratedElems += 1; pResInfo->hasResult = DATA_SET_FLAG; - if (pResInfo->superTableQ) { - memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(STwaInfo)); + if (pCtx->stableQuery) { + memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(STwaInfo)); } } static void twa_func_merge(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - assert(pResInfo->superTableQ); + assert(pCtx->stableQuery); STwaInfo *pBuf = (STwaInfo *)pCtx->aOutputBuf; char * indicator = pCtx->aInputElemBuf; @@ -3895,16 +3885,16 @@ static void twa_func_merge(SQLFunctionCtx *pCtx) { */ void twa_function_copy(SQLFunctionCtx *pCtx) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - memcpy(pResInfo->interResultBuf, pCtx->aInputElemBuf, (size_t)pCtx->inputBytes); + memcpy(GET_ROWCELL_INTERBUF(pResInfo), pCtx->aInputElemBuf, (size_t)pCtx->inputBytes); pResInfo->hasResult = ((STwaInfo *)pCtx->aInputElemBuf)->hasResult; } void twa_function_finalizer(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - STwaInfo *pInfo = (STwaInfo *)pResInfo->interResultBuf; + STwaInfo *pInfo = (STwaInfo *)GET_ROWCELL_INTERBUF(pResInfo); assert(pInfo->EKey >= pInfo->lastKey && pInfo->hasResult == pResInfo->hasResult); if (pInfo->hasResult != DATA_SET_FLAG) { @@ -3932,8 +3922,8 @@ void twa_function_finalizer(SQLFunctionCtx *pCtx) { */ static void interp_function(SQLFunctionCtx *pCtx) { // at this point, the value is existed, return directly - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - SInterpInfoDetail* pInfo = pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SInterpInfoDetail* pInfo = GET_ROWCELL_INTERBUF(pResInfo); if (pCtx->size == 1) { char *pData = GET_INPUT_CHAR(pCtx); @@ -4019,8 +4009,8 @@ static bool ts_comp_function_setup(SQLFunctionCtx *pCtx) { return false; // not initialized since it has been initialized } - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - STSCompInfo *pInfo = pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + STSCompInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); pInfo->pTSBuf = tsBufCreate(false, pCtx->order); pInfo->pTSBuf->tsOrder = pCtx->order; @@ -4028,8 +4018,8 @@ static bool ts_comp_function_setup(SQLFunctionCtx *pCtx) { } static void ts_comp_function(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - STSBuf * pTSbuf = ((STSCompInfo *)(pResInfo->interResultBuf))->pTSBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + STSBuf * pTSbuf = ((STSCompInfo *)(GET_ROWCELL_INTERBUF(pResInfo)))->pTSBuf; const char *input = GET_INPUT_CHAR(pCtx); @@ -4053,8 +4043,8 @@ static void ts_comp_function_f(SQLFunctionCtx *pCtx, int32_t index) { return; } - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - STSCompInfo *pInfo = pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + STSCompInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); STSBuf *pTSbuf = pInfo->pTSBuf; @@ -4065,9 +4055,9 @@ static void ts_comp_function_f(SQLFunctionCtx *pCtx, int32_t index) { } static void ts_comp_finalize(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); - STSCompInfo *pInfo = pResInfo->interResultBuf; + STSCompInfo *pInfo = GET_ROWCELL_INTERBUF(pResInfo); STSBuf * pTSbuf = pInfo->pTSBuf; tsBufFlush(pTSbuf); @@ -4116,8 +4106,8 @@ static bool rate_function_setup(SQLFunctionCtx *pCtx) { return false; } - SResultInfo *pResInfo = GET_RES_INFO(pCtx); //->aOutputBuf + pCtx->outputBytes; - SRateInfo * pInfo = pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); //->aOutputBuf + pCtx->outputBytes; + SRateInfo * pInfo = GET_ROWCELL_INTERBUF(pResInfo); pInfo->CorrectionValue = 0; pInfo->firstKey = INT64_MIN; @@ -4136,8 +4126,8 @@ static bool rate_function_setup(SQLFunctionCtx *pCtx) { static void rate_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); TSKEY *primaryKey = pCtx->ptsList; tscDebug("%p rate_function() size:%d, hasNull:%d", pCtx, pCtx->size, pCtx->hasNull); @@ -4200,8 +4190,8 @@ static void rate_function(SQLFunctionCtx *pCtx) { } // keep the data into the final output buffer for super table query since this execution may be the last one - if (pResInfo->superTableQ) { - memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SRateInfo)); + if (pCtx->stableQuery) { + memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo)); } } @@ -4212,8 +4202,8 @@ static void rate_function_f(SQLFunctionCtx *pCtx, int32_t index) { } // NOTE: keep the intermediate result into the interResultBuf - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); TSKEY *primaryKey = pCtx->ptsList; int64_t v = 0; @@ -4257,20 +4247,18 @@ static void rate_function_f(SQLFunctionCtx *pCtx, int32_t index) { pResInfo->hasResult = DATA_SET_FLAG; // keep the data into the final output buffer for super table query since this execution may be the last one - if (pResInfo->superTableQ) { - memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SRateInfo)); + if (pCtx->stableQuery) { + memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo)); } } static void rate_func_merge(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - assert(pResInfo->superTableQ); + assert(pCtx->stableQuery); tscDebug("rate_func_merge() size:%d", pCtx->size); - //SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf; SRateInfo *pBuf = (SRateInfo *)pCtx->aOutputBuf; char *indicator = pCtx->aInputElemBuf; @@ -4303,8 +4291,8 @@ static void rate_func_merge(SQLFunctionCtx *pCtx) { static void rate_func_copy(SQLFunctionCtx *pCtx) { assert(pCtx->inputType == TSDB_DATA_TYPE_BINARY); - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - memcpy(pResInfo->interResultBuf, pCtx->aInputElemBuf, (size_t)pCtx->inputBytes); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + memcpy(GET_ROWCELL_INTERBUF(pResInfo), pCtx->aInputElemBuf, (size_t)pCtx->inputBytes); pResInfo->hasResult = ((SRateInfo*)pCtx->aInputElemBuf)->hasResult; SRateInfo* pRateInfo = (SRateInfo*)pCtx->aInputElemBuf; @@ -4315,8 +4303,8 @@ static void rate_func_copy(SQLFunctionCtx *pCtx) { static void rate_finalizer(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); tscDebug("%p isIRate:%d firstKey:%" PRId64 " lastKey:%" PRId64 " firstValue:%" PRId64 " lastValue:%" PRId64 " CorrectionValue:%" PRId64 " hasResult:%d", pCtx, pRateInfo->isIRate, pRateInfo->firstKey, pRateInfo->lastKey, pRateInfo->firstValue, pRateInfo->lastValue, pRateInfo->CorrectionValue, pRateInfo->hasResult); @@ -4341,8 +4329,8 @@ static void rate_finalizer(SQLFunctionCtx *pCtx) { static void irate_function(SQLFunctionCtx *pCtx) { int32_t notNullElems = 0; - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); TSKEY *primaryKey = pCtx->ptsList; tscDebug("%p irate_function() size:%d, hasNull:%d", pCtx, pCtx->size, pCtx->hasNull); @@ -4404,8 +4392,8 @@ static void irate_function(SQLFunctionCtx *pCtx) { } // keep the data into the final output buffer for super table query since this execution may be the last one - if (pResInfo->superTableQ) { - memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SRateInfo)); + if (pCtx->stableQuery) { + memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo)); } } @@ -4416,8 +4404,8 @@ static void irate_function_f(SQLFunctionCtx *pCtx, int32_t index) { } // NOTE: keep the intermediate result into the interResultBuf - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); TSKEY *primaryKey = pCtx->ptsList; int64_t v = 0; @@ -4453,16 +4441,16 @@ static void irate_function_f(SQLFunctionCtx *pCtx, int32_t index) { pResInfo->hasResult = DATA_SET_FLAG; // keep the data into the final output buffer for super table query since this execution may be the last one - if (pResInfo->superTableQ) { - memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SRateInfo)); + if (pCtx->stableQuery) { + memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo)); } } static void do_sumrate_merge(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - assert(pResInfo->superTableQ); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + assert(pCtx->stableQuery); - SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf; + SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); char * input = GET_INPUT_CHAR(pCtx); for (int32_t i = 0; i < pCtx->size; ++i, input += pCtx->inputBytes) { @@ -4486,7 +4474,7 @@ static void do_sumrate_merge(SQLFunctionCtx *pCtx) { if (DATA_SET_FLAG == pRateInfo->hasResult) { pResInfo->hasResult = DATA_SET_FLAG; SET_VAL(pCtx, pRateInfo->num, 1); - memcpy(pCtx->aOutputBuf, pResInfo->interResultBuf, sizeof(SRateInfo)); + memcpy(pCtx->aOutputBuf, GET_ROWCELL_INTERBUF(pResInfo), sizeof(SRateInfo)); } } @@ -4501,10 +4489,10 @@ static void sumrate_func_second_merge(SQLFunctionCtx *pCtx) { } static void sumrate_finalizer(SQLFunctionCtx *pCtx) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - SRateInfo *pRateInfo = (SRateInfo *)pResInfo->interResultBuf; + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); + SRateInfo *pRateInfo = (SRateInfo *)GET_ROWCELL_INTERBUF(pResInfo); - tscDebug("%p sumrate_finalizer() superTableQ:%d num:%" PRId64 " sum:%f hasResult:%d", pCtx, pResInfo->superTableQ, pRateInfo->num, pRateInfo->sum, pRateInfo->hasResult); + tscDebug("%p sumrate_finalizer() superTableQ:%d num:%" PRId64 " sum:%f hasResult:%d", pCtx, pCtx->stableQuery, pRateInfo->num, pRateInfo->sum, pRateInfo->hasResult); if (pRateInfo->hasResult != DATA_SET_FLAG) { setNull(pCtx->aOutputBuf, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c index 8c835806a7..58c2b76b72 100644 --- a/src/client/src/tscLocalMerge.c +++ b/src/client/src/tscLocalMerge.c @@ -99,12 +99,12 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SLocalReducer *pReducer, tOrderDesc pCtx->param[1].i64Key = pQueryInfo->order.orderColId; } - SResultInfo *pResInfo = &pReducer->pResInfo[i]; - pResInfo->bufLen = pExpr->interBytes; - pResInfo->interResultBuf = calloc(1, (size_t) pResInfo->bufLen); +// SResultRowCellInfo *pResInfo = &pReducer->pResInfo[i]; + pCtx->interBufBytes = pExpr->interBytes; +// pResInfo->interResultBuf = calloc(1, (size_t) pCtx->interBufBytes); pCtx->resultInfo = &pReducer->pResInfo[i]; - pCtx->resultInfo->superTableQ = true; + pCtx->stableQuery = true; } int16_t n = 0; @@ -345,7 +345,7 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd size_t numOfCols = tscSqlExprNumOfExprs(pQueryInfo); pReducer->pTempBuffer->num = 0; - pReducer->pResInfo = calloc(numOfCols, sizeof(SResultInfo)); + pReducer->pResInfo = calloc(numOfCols, sizeof(SResultRowCellInfo)); tscCreateResPointerInfo(pRes, pQueryInfo); tscInitSqlContext(pCmd, pReducer, pDesc); @@ -512,7 +512,7 @@ void tscDestroyLocalReducer(SSqlObj *pSql) { if (pLocalReducer->pResInfo != NULL) { size_t num = tscSqlExprNumOfExprs(pQueryInfo); for (int32_t i = 0; i < num; ++i) { - taosTFree(pLocalReducer->pResInfo[i].interResultBuf); +// taosTFree(pLocalReducer->pResInfo[i].interResultBuf); } taosTFree(pLocalReducer->pResInfo); @@ -1072,7 +1072,7 @@ static int64_t getNumOfResultLocal(SQueryInfo *pQueryInfo, SQLFunctionCtx *pCtx) continue; } - SResultInfo* pResInfo = GET_RES_INFO(&pCtx[j]); + SResultRowCellInfo* pResInfo = GET_RES_INFO(&pCtx[j]); if (maxOutput < pResInfo->numOfRes) { maxOutput = pResInfo->numOfRes; } diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index ab20840cbe..64a0cb1a62 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -61,14 +61,14 @@ typedef struct SSqlGroupbyExpr { int16_t orderType; // order by type: asc/desc } SSqlGroupbyExpr; -typedef struct SWindowResult { +typedef struct SResultRow { int32_t pageId; // pageId & rowId is the position of current result in disk-based output buffer int32_t rowId:15; bool closed:1; // this result status: closed or opened uint16_t numOfRows; // number of rows of current time window - SResultInfo* resultInfo; // For each result column, there is a resultInfo + SResultRowCellInfo* pCellInfo; // For each result column, there is a resultInfo union {STimeWindow win; char* key;}; // start key of current time window -} SWindowResult; +} SResultRow; /** * If the number of generated results is greater than this value, @@ -82,7 +82,7 @@ typedef struct SResultRec { } SResultRec; typedef struct SWindowResInfo { - SWindowResult** pResult; // result list + SResultRow** pResult; // result list int16_t type; // data type for hash key int32_t capacity; // max capacity int32_t curIndex; // current start active index @@ -169,11 +169,11 @@ typedef struct SQuery { typedef struct SQueryRuntimeEnv { jmp_buf env; - SResultInfo* resultInfo; // todo refactor to merge with SWindowResInfo + SResultRowCellInfo* resultInfo; // todo refactor to merge with SWindowResInfo SQuery* pQuery; SQLFunctionCtx* pCtx; int32_t numOfRowsPerPage; - int16_t offset[TSDB_MAX_COLUMNS]; + uint16_t offset[TSDB_MAX_COLUMNS]; uint16_t scanFlag; // denotes reversed scan of data or not SFillInfo* pFillInfo; SWindowResInfo windowResInfo; @@ -192,6 +192,8 @@ typedef struct SQueryRuntimeEnv { SHashObj* pWindowHashTable; // quick locate the window object for each result char* keyBuf; // window key buffer SWindowResultPool* pool; // window result object pool + + int32_t* rowCellInfoOffset;// offset value for each row result cell info } SQueryRuntimeEnv; enum { diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h index 3775780423..698608a689 100644 --- a/src/query/inc/qUtil.h +++ b/src/query/inc/qUtil.h @@ -26,8 +26,9 @@ int32_t getOutputInterResultBufSize(SQuery* pQuery); -void clearTimeWindowResBuf(SQueryRuntimeEnv* pRuntimeEnv, SWindowResult* pOneOutputRes); -void copyTimeWindowResBuf(SQueryRuntimeEnv* pRuntimeEnv, SWindowResult* dst, const SWindowResult* src); +void clearResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* pRow); +void copyResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* dst, const SResultRow* src); +SResultRowCellInfo* getResultCell(SQueryRuntimeEnv* pRuntimeEnv, const SResultRow* pRow, int32_t index); int32_t initWindowResInfo(SWindowResInfo* pWindowResInfo, SQueryRuntimeEnv* pRuntimeEnv, int32_t size, int32_t threshold, int16_t type); @@ -42,7 +43,7 @@ void closeTimeWindow(SWindowResInfo* pWindowResInfo, int32_t slot); void closeAllTimeWindow(SWindowResInfo* pWindowResInfo); void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_t order); -static FORCE_INLINE SWindowResult *getWindowResult(SWindowResInfo *pWindowResInfo, int32_t slot) { +static FORCE_INLINE SResultRow *getWindowResult(SWindowResInfo *pWindowResInfo, int32_t slot) { assert(pWindowResInfo != NULL && slot >= 0 && slot < pWindowResInfo->size); return pWindowResInfo->pResult[slot]; } @@ -52,9 +53,9 @@ static FORCE_INLINE SWindowResult *getWindowResult(SWindowResInfo *pWindowResInf bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot); -int32_t createQueryResultInfo(SQuery *pQuery, SWindowResult *pResultRow, bool isSTableQuery, size_t interBufSize); +int32_t createQueryResultInfo(SQuery *pQuery, SResultRow *pResultRow); -static FORCE_INLINE char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int32_t columnIndex, SWindowResult *pResult, +static FORCE_INLINE char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int32_t columnIndex, SResultRow *pResult, tFilePage* page) { assert(pResult != NULL && pRuntimeEnv != NULL); @@ -74,7 +75,7 @@ __filter_func_t *getValueFilterFuncArray(int32_t type); size_t getWindowResultSize(SQueryRuntimeEnv* pRuntimeEnv); SWindowResultPool* initWindowResultPool(size_t size); -SWindowResult* getNewWindowResult(SWindowResultPool* p); +SResultRow* getNewWindowResult(SWindowResultPool* p); int64_t getWindowResultPoolMemSize(SWindowResultPool* p); void* destroyWindowResultPool(SWindowResultPool* p); int32_t getNumOfAllocatedWindowResult(SWindowResultPool* p); diff --git a/src/query/inc/tsqlfunction.h b/src/query/inc/tsqlfunction.h index 28b9a60102..3bd4aad276 100644 --- a/src/query/inc/tsqlfunction.h +++ b/src/query/inc/tsqlfunction.h @@ -145,15 +145,14 @@ typedef struct SInterpInfoDetail { int8_t primaryCol; } SInterpInfoDetail; -typedef struct SResultInfo { +typedef struct SResultRowCellInfo { int8_t hasResult; // result generated, not NULL value - bool initialized; // output buffer has been initialized - bool complete; // query has completed - bool superTableQ; // is super table query - uint32_t bufLen; // buffer size - uint64_t numOfRes; // num of output result in current buffer - void* interResultBuf; // output result buffer -} SResultInfo; + bool initialized; // output buffer has been initialized + bool complete; // query has completed + uint16_t numOfRes; // num of output result in current buffer +} SResultRowCellInfo; + +#define GET_ROWCELL_INTERBUF(_c) ((void*) ((char*)(_c) + sizeof(SResultRowCellInfo))) struct SQLFunctionCtx; @@ -175,9 +174,11 @@ typedef struct SQLFunctionCtx { int16_t inputBytes; int16_t outputType; - int16_t outputBytes; // size of results, determined by function and input column data type - bool hasNull; // null value exist in current block + int16_t outputBytes; // size of results, determined by function and input column data type + int32_t interBufBytes; // internal buffer size + bool hasNull; // null value exist in current block bool requireNull; // require null in some function + bool stableQuery; int16_t functionId; // function id void * aInputElemBuf; char * aOutputBuf; // final result output buffer, point to sdata->data @@ -189,7 +190,8 @@ typedef struct SQLFunctionCtx { void * ptsOutputBuf; // corresponding output buffer for timestamp of each result, e.g., top/bottom*/ SQLPreAggVal preAggVals; tVariant tag; - SResultInfo *resultInfo; + + SResultRowCellInfo *resultInfo; SExtTagsInfo tagInfo; } SQLFunctionCtx; @@ -274,16 +276,16 @@ bool topbot_datablock_filter(SQLFunctionCtx *pCtx, int32_t functionId, const cha (_r)->initialized = false; \ } while (0) -void setResultInfoBuf(SResultInfo *pResInfo, int32_t size, bool superTable, char* buf); +//void setResultInfoBuf(SResultRowCellInfo *pResInfo, char* buf); -static FORCE_INLINE void initResultInfo(SResultInfo *pResInfo) { +static FORCE_INLINE void initResultInfo(SResultRowCellInfo *pResInfo, uint32_t bufLen) { pResInfo->initialized = true; // the this struct has been initialized flag pResInfo->complete = false; pResInfo->hasResult = false; pResInfo->numOfRes = 0; - memset(pResInfo->interResultBuf, 0, (size_t)pResInfo->bufLen); + memset(GET_ROWCELL_INTERBUF(pResInfo), 0, (size_t)bufLen); } #ifdef __cplusplus diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index d5e1962eea..a573431bfc 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -178,9 +178,9 @@ static void getNextTimeWindow(SQuery* pQuery, STimeWindow* tw) { // todo move to utility static int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *group); -static void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult); -static void setWindowResOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult); -static void resetMergeResultBuf(SQuery *pQuery, SQLFunctionCtx *pCtx, SResultInfo *pResultInfo); +static void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult); +static void setWindowResOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult); +static void resetMergeResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx *pCtx, SResultRow *pRow); static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t functionId); static void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY *tsCol, SDataBlockInfo* pBlockInfo, @@ -255,7 +255,7 @@ int64_t getNumOfResult(SQueryRuntimeEnv *pRuntimeEnv) { continue; } - SResultInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); + SResultRowCellInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); if (pResInfo != NULL && maxOutput < pResInfo->numOfRes) { maxOutput = pResInfo->numOfRes; } @@ -272,7 +272,7 @@ void updateNumOfResult(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOfRes) { SQuery *pQuery = pRuntimeEnv->pQuery; for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - SResultInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); + SResultRowCellInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); int16_t functionId = pRuntimeEnv->pCtx[j].functionId; if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAGPRJ || @@ -447,7 +447,7 @@ static bool hasNullValue(SColIndex* pColIndex, SDataStatis *pStatis, SDataStatis return true; } -static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo, char *pData, +static SResultRow *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo, char *pData, int16_t bytes, bool masterscan, uint64_t uid) { SQuery *pQuery = pRuntimeEnv->pQuery; @@ -470,31 +470,27 @@ static SWindowResult *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWin } char *t = realloc(pWindowResInfo->pResult, (size_t)(newCapacity * POINTER_BYTES)); - // pRuntimeEnv->summary.winInfoSize += (newCapacity - pWindowResInfo->capacity) * sizeof(SWindowResult); + // pRuntimeEnv->summary.winInfoSize += (newCapacity - pWindowResInfo->capacity) * sizeof(SResultRow); // pRuntimeEnv->summary.numOfTimeWindows += (newCapacity - pWindowResInfo->capacity); if (t == NULL) { longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } - pWindowResInfo->pResult = (SWindowResult **)t; + pWindowResInfo->pResult = (SResultRow **)t; int32_t inc = (int32_t)newCapacity - pWindowResInfo->capacity; memset(&pWindowResInfo->pResult[pWindowResInfo->capacity], 0, POINTER_BYTES * inc); pWindowResInfo->capacity = (int32_t)newCapacity; } -// pRuntimeEnv->summary.winInfoSize += (pQuery->numOfOutput * sizeof(SResultInfo) + pRuntimeEnv->interBufSize) * inc; - SWindowResult* pResult = getNewWindowResult(pRuntimeEnv->pool); +// pRuntimeEnv->summary.winInfoSize += (pQuery->numOfOutput * sizeof(SResultRowCellInfo) + pRuntimeEnv->interBufSize) * inc; + SResultRow* pResult = getNewWindowResult(pRuntimeEnv->pool); pWindowResInfo->pResult[pWindowResInfo->size] = pResult; -// for (int32_t i = pWindowResInfo->capacity; i < newCapacity; ++i) { - int32_t ret = createQueryResultInfo(pQuery, pResult, pRuntimeEnv->stableQuery, pRuntimeEnv->interBufSize); - if (ret != TSDB_CODE_SUCCESS) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } -// } - -// } + int32_t ret = createQueryResultInfo(pQuery, pResult); + if (ret != TSDB_CODE_SUCCESS) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } // add a new result set for a new group pWindowResInfo->curIndex = pWindowResInfo->size++; @@ -522,7 +518,7 @@ static STimeWindow getActiveTimeWindow(SWindowResInfo *pWindowResInfo, int64_t t } } else { int32_t slot = curTimeWindowIndex(pWindowResInfo); - SWindowResult* pWindowRes = getWindowResult(pWindowResInfo, slot); + SResultRow* pWindowRes = getWindowResult(pWindowResInfo, slot); w = pWindowRes->win; } @@ -558,7 +554,7 @@ static STimeWindow getActiveTimeWindow(SWindowResInfo *pWindowResInfo, int64_t t return w; } -static int32_t addNewWindowResultBuf(SWindowResult *pWindowRes, SDiskbasedResultBuf *pResultBuf, int32_t tid, +static int32_t addNewWindowResultBuf(SResultRow *pWindowRes, SDiskbasedResultBuf *pResultBuf, int32_t tid, int32_t numOfRowsPerPage) { if (pWindowRes->pageId != -1) { return 0; @@ -608,7 +604,7 @@ static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowRes assert(win->skey <= win->ekey); SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; - SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&win->skey, TSDB_KEYSIZE, masterscan, pBockInfo->uid); + SResultRow *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&win->skey, TSDB_KEYSIZE, masterscan, pBockInfo->uid); if (pWindowRes == NULL) { *newWind = false; @@ -693,7 +689,7 @@ static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKe int64_t skey = TSKEY_INITIAL_VAL; for (i = 0; i < pWindowResInfo->size; ++i) { - SWindowResult *pResult = pWindowResInfo->pResult[i]; + SResultRow *pResult = pWindowResInfo->pResult[i]; if (pResult->closed) { numOfClosed += 1; continue; @@ -1118,7 +1114,7 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat } uint64_t uid = 0; // uid is always set to be 0. - SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, d, len, true, uid); + SResultRow *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, d, len, true, uid); if (pWindowRes == NULL) { return -1; } @@ -1232,7 +1228,7 @@ static int32_t doTSJoinFilter(SQueryRuntimeEnv *pRuntimeEnv, int32_t offset) { } static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t functionId) { - SResultInfo *pResInfo = GET_RES_INFO(pCtx); + SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); SQuery* pQuery = pRuntimeEnv->pQuery; // in case of timestamp column, always generated results. @@ -1519,7 +1515,8 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY * top/bottom values emerge, so does diff function */ if (functionId == TSDB_FUNC_TWA) { - STwaInfo *pTWAInfo = GET_RES_INFO(pCtx)->interResultBuf; + SResultRowCellInfo* pInfo = GET_RES_INFO(pCtx); + STwaInfo *pTWAInfo = (STwaInfo*) GET_ROWCELL_INTERBUF(pInfo); pTWAInfo->SKey = pQuery->window.skey; pTWAInfo->EKey = pQuery->window.ekey; } @@ -1533,7 +1530,9 @@ void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY pCtx->preAggVals.statis.max = pBlockInfo->window.ekey; } } else if (functionId == TSDB_FUNC_INTERP) { - SInterpInfoDetail *pInterpInfo = GET_RES_INFO(pCtx)->interResultBuf; + SResultRowCellInfo* pInfo = GET_RES_INFO(pCtx); + + SInterpInfoDetail *pInterpInfo = (SInterpInfoDetail *)GET_ROWCELL_INTERBUF(pInfo); pInterpInfo->type = (int8_t)pQuery->fillType; pInterpInfo->ts = pQuery->window.skey; pInterpInfo->primaryCol = (colId == PRIMARYKEY_TIMESTAMP_COL_INDEX); @@ -1610,26 +1609,29 @@ static int32_t setCtxTagColumnInfo(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx return TSDB_CODE_SUCCESS; } -static FORCE_INLINE void setWindowResultInfo(SResultInfo *pResultInfo, SQuery *pQuery, bool isStableQuery, char* buf) { - char* p = buf; - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - int32_t size = pQuery->pSelectExpr[i].interBytes; - setResultInfoBuf(&pResultInfo[i], size, isStableQuery, p); - - p += size; - } +static FORCE_INLINE void setResultRowCellInfo(SQueryRuntimeEnv* pRuntimeEnv, SResultRow *pRow, char* buf) { +// SQuery* pQuery = pRuntimeEnv->pQuery; +// +// char* p = buf; +// for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { +// int32_t size = pQuery->pSelectExpr[i].interBytes; +// SResultRowCellInfo* pInfo = getResultCell(pRuntimeEnv, pRow, i); +// setResultInfoBuf(pInfo, p); +// p += size; +// } } static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order) { qDebug("QInfo:%p setup runtime env", GET_QINFO_ADDR(pRuntimeEnv)); SQuery *pQuery = pRuntimeEnv->pQuery; - size_t size = pRuntimeEnv->interBufSize + pQuery->numOfOutput * sizeof(SResultInfo); + size_t size = pRuntimeEnv->interBufSize + pQuery->numOfOutput * sizeof(SResultRowCellInfo); pRuntimeEnv->resultInfo = calloc(1, size); pRuntimeEnv->pCtx = (SQLFunctionCtx *)calloc(pQuery->numOfOutput, sizeof(SQLFunctionCtx)); + pRuntimeEnv->rowCellInfoOffset = calloc(pQuery->numOfOutput, sizeof(int32_t)); - if (pRuntimeEnv->resultInfo == NULL || pRuntimeEnv->pCtx == NULL) { + if (pRuntimeEnv->resultInfo == NULL || pRuntimeEnv->pCtx == NULL || pRuntimeEnv->rowCellInfoOffset == NULL) { goto _clean; } @@ -1675,6 +1677,8 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order pCtx->order = pQuery->order.order; pCtx->functionId = pSqlFuncMsg->functionId; + pCtx->stableQuery = pRuntimeEnv->stableQuery; + pCtx->interBufBytes = pQuery->pSelectExpr[i].interBytes; pCtx->numOfParams = pSqlFuncMsg->numOfParams; for (int32_t j = 0; j < pCtx->numOfParams; ++j) { @@ -1704,13 +1708,15 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order if (i > 0) { pRuntimeEnv->offset[i] = pRuntimeEnv->offset[i - 1] + pRuntimeEnv->pCtx[i - 1].outputBytes; + pRuntimeEnv->rowCellInfoOffset[i] = pRuntimeEnv->rowCellInfoOffset[i - 1] + sizeof(SResultRowCellInfo) + pQuery->pSelectExpr[i - 1].interBytes; } + } - char* buf = (char*) pRuntimeEnv->resultInfo + sizeof(SResultInfo) * pQuery->numOfOutput; +// char* buf = (char*) pRuntimeEnv->resultInfo + sizeof(SResultRowCellInfo) * pQuery->numOfOutput; // set the intermediate result output buffer - setWindowResultInfo(pRuntimeEnv->resultInfo, pQuery, pRuntimeEnv->stableQuery, buf); +// setResultRowCellInfo(pRuntimeEnv, pRuntimeEnv->resultInfo, NULL); // if it is group by normal column, do not set output buffer, the output buffer is pResult if (!pRuntimeEnv->groupbyNormalCol && !pRuntimeEnv->stableQuery) { @@ -2655,7 +2661,7 @@ void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, void *pTable, void *tsdb) { } } -static void doMerge(SQueryRuntimeEnv *pRuntimeEnv, int64_t timestamp, SWindowResult *pWindowRes, bool mergeFlag) { +static void doMerge(SQueryRuntimeEnv *pRuntimeEnv, int64_t timestamp, SResultRow *pWindowRes, bool mergeFlag) { SQuery * pQuery = pRuntimeEnv->pQuery; SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx; @@ -2829,14 +2835,14 @@ int32_t tableResultComparFn(const void *pLeft, const void *pRight, void *param) } SWindowResInfo *pWindowResInfo1 = &supporter->pTableQueryInfo[left]->windowResInfo; - SWindowResult * pWindowRes1 = getWindowResult(pWindowResInfo1, leftPos); + SResultRow * pWindowRes1 = getWindowResult(pWindowResInfo1, leftPos); tFilePage *page1 = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes1->pageId); char *b1 = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes1, page1); TSKEY leftTimestamp = GET_INT64_VAL(b1); SWindowResInfo *pWindowResInfo2 = &supporter->pTableQueryInfo[right]->windowResInfo; - SWindowResult * pWindowRes2 = getWindowResult(pWindowResInfo2, rightPos); + SResultRow * pWindowRes2 = getWindowResult(pWindowResInfo2, rightPos); tFilePage *page2 = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes2->pageId); char *b2 = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes2, page2); @@ -2957,7 +2963,7 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { pQuery->rec.rows += offset; } -int64_t getNumOfResultWindowRes(SQuery *pQuery, SWindowResult *pWindowRes) { +int64_t getNumOfResultWindowRes(SQuery *pQuery, SResultRow *pResultRow) { for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { int32_t functionId = pQuery->pSelectExpr[j].base.functionId; @@ -2969,7 +2975,7 @@ int64_t getNumOfResultWindowRes(SQuery *pQuery, SWindowResult *pWindowRes) { continue; } - SResultInfo *pResultInfo = &pWindowRes->resultInfo[j]; + SResultRowCellInfo *pResultInfo = &pResultRow->pCellInfo[j]; assert(pResultInfo != NULL); if (pResultInfo->numOfRes > 0) { @@ -3038,18 +3044,19 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { SLoserTreeInfo *pTree = NULL; tLoserTreeCreate(&pTree, numOfTables, &cs, tableResultComparFn); - SResultInfo *pResultInfo = calloc(pQuery->numOfOutput, sizeof(SResultInfo)); - if (pResultInfo == NULL) { + SResultRow* pRow = calloc(1, getWindowResultSize(pRuntimeEnv)); + if (pRow == NULL) { longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } - char* buf = calloc(1, pRuntimeEnv->interBufSize); - if (buf == NULL) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } + pRow->pCellInfo = (SResultRowCellInfo*) ((char*) pRow + sizeof(SResultRow)); +// char* buf = (char*) pRow + sizeof(SResultRowCellInfo)*; +// if (buf == NULL) { +// longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); +// } - setWindowResultInfo(pResultInfo, pQuery, pRuntimeEnv->stableQuery, buf); - resetMergeResultBuf(pQuery, pRuntimeEnv->pCtx, pResultInfo); + setResultRowCellInfo(pRuntimeEnv, pRow, NULL); + resetMergeResultBuf(pRuntimeEnv, pRuntimeEnv->pCtx, pRow); pQInfo->groupResInfo.groupId = getGroupResultId(pQInfo->groupIndex); @@ -3064,8 +3071,8 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { taosTFree(pTableList); taosTFree(posList); taosTFree(pTree); - taosTFree(pResultInfo); - taosTFree(buf); +// taosTFree(pResultInfo); +// taosTFree(buf); longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); } @@ -3073,7 +3080,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { int32_t pos = pTree->pNode[0].index; SWindowResInfo *pWindowResInfo = &pTableList[pos]->windowResInfo; - SWindowResult *pWindowRes = getWindowResult(pWindowResInfo, cs.position[pos]); + SResultRow *pWindowRes = getWindowResult(pWindowResInfo, cs.position[pos]); tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pageId); char *b = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes, page); @@ -3101,7 +3108,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { return -1; } - resetMergeResultBuf(pQuery, pRuntimeEnv->pCtx, pResultInfo); + resetMergeResultBuf(pRuntimeEnv, pRuntimeEnv->pCtx, pRow); } doMerge(pRuntimeEnv, ts, pWindowRes, false); @@ -3123,7 +3130,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { } } else { // current page is not needed anymore - SWindowResult *pNextWindowRes = getWindowResult(pWindowResInfo, cs.position[pos]); + SResultRow *pNextWindowRes = getWindowResult(pWindowResInfo, cs.position[pos]); if (pNextWindowRes->pageId != currentPageId) { releaseResBufPage(pRuntimeEnv->pResultBuf, page); } @@ -3140,7 +3147,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { taosTFree(pTree); taosTFree(pTableList); taosTFree(posList); - taosTFree(pResultInfo); +// taosTFree(pResultInfo); return -1; } @@ -3158,8 +3165,8 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { taosTFree(posList); taosTFree(pTree); - taosTFree(pResultInfo); - taosTFree(buf); +// taosTFree(pResultInfo); +// taosTFree(buf); return pQInfo->groupResInfo.numOfDataPages; } @@ -3202,12 +3209,14 @@ int32_t flushFromResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupR return TSDB_CODE_SUCCESS; } -void resetMergeResultBuf(SQuery *pQuery, SQLFunctionCtx *pCtx, SResultInfo *pResultInfo) { +void resetMergeResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx *pCtx, SResultRow *pRow) { + SQuery* pQuery = pRuntimeEnv->pQuery; + for (int32_t k = 0; k < pQuery->numOfOutput; ++k) { pCtx[k].aOutputBuf = pQuery->sdata[k]->data - pCtx[k].outputBytes; pCtx[k].size = 1; pCtx[k].startOffset = 0; - pCtx[k].resultInfo = &pResultInfo[k]; + pCtx[k].resultInfo = getResultCell(pRuntimeEnv, pRow, k); pQuery->sdata[k]->num = 0; } @@ -3253,7 +3262,7 @@ static void disableFuncInReverseScanImpl(SQInfo* pQInfo, SWindowResInfo *pWindow continue; } - SWindowResult *buf = getWindowResult(pWindowResInfo, i); + SResultRow *buf = getWindowResult(pWindowResInfo, i); // open/close the specified query for each group result for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { @@ -3261,9 +3270,9 @@ static void disableFuncInReverseScanImpl(SQInfo* pQInfo, SWindowResInfo *pWindow if (((functId == TSDB_FUNC_FIRST || functId == TSDB_FUNC_FIRST_DST) && order == TSDB_ORDER_ASC) || ((functId == TSDB_FUNC_LAST || functId == TSDB_FUNC_LAST_DST) && order == TSDB_ORDER_DESC)) { - buf->resultInfo[j].complete = false; + buf->pCellInfo[j].complete = false; } else if (functId != TSDB_FUNC_TS && functId != TSDB_FUNC_TAG) { - buf->resultInfo[j].complete = true; + buf->pCellInfo[j].complete = true; } } } @@ -3327,24 +3336,17 @@ void switchCtxOrder(SQueryRuntimeEnv *pRuntimeEnv) { } } -int32_t createQueryResultInfo(SQuery *pQuery, SWindowResult *pResultRow, bool isSTableQuery, size_t interBufSize) { - int32_t numOfCols = pQuery->numOfOutput; - -// size_t size = numOfCols * sizeof(SResultInfo) + interBufSize; - pResultRow->resultInfo = (SResultInfo*)((char*)pResultRow + sizeof(SWindowResult)); - -// pResultRow->resultInfo = calloc(1, size); -// if (pResultRow->resultInfo == NULL) { -// return TSDB_CODE_QRY_OUT_OF_MEMORY; -// } +int32_t createQueryResultInfo(SQuery *pQuery, SResultRow *pResultRow) { +// int32_t numOfCols = pQuery->numOfOutput; + pResultRow->pCellInfo = (SResultRowCellInfo*)((char*)pResultRow + sizeof(SResultRow)); pResultRow->pageId = -1; pResultRow->rowId = -1; - char* buf = (char*) pResultRow->resultInfo + numOfCols * sizeof(SResultInfo); +// char* buf = (char*) pResultRow->pCellInfo + numOfCols * sizeof(SResultRowCellInfo); // set the intermediate result output buffer - setWindowResultInfo(pResultRow->resultInfo, pQuery, isSTableQuery, buf); +// setResultRowCellInfo(pRunimeEnv, pResultRow, buf); return TSDB_CODE_SUCCESS; } @@ -3409,7 +3411,7 @@ void initCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { int32_t functionId = pQuery->pSelectExpr[j].base.functionId; pRuntimeEnv->pCtx[j].currentStage = 0; - SResultInfo* pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); + SResultRowCellInfo* pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); if (pResInfo->initialized) { continue; } @@ -3478,7 +3480,7 @@ bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) { SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; for (int32_t i = 0; i < pWindowResInfo->size; ++i) { - SWindowResult *pResult = getWindowResult(pWindowResInfo, i); + SResultRow *pResult = getWindowResult(pWindowResInfo, i); if (!pResult->closed) { continue; } @@ -3492,7 +3494,7 @@ bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) { } aAggs[functId].xNextStep(&pRuntimeEnv->pCtx[j]); - SResultInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); + SResultRowCellInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); toContinue |= (!pResInfo->complete); } @@ -3505,7 +3507,7 @@ bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) { } aAggs[functId].xNextStep(&pRuntimeEnv->pCtx[j]); - SResultInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); + SResultRowCellInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[j]); toContinue |= (!pResInfo->complete); } @@ -3704,7 +3706,7 @@ void finalizeQueryResult(SQueryRuntimeEnv *pRuntimeEnv) { } for (int32_t i = 0; i < pWindowResInfo->size; ++i) { - SWindowResult *buf = pWindowResInfo->pResult[i]; + SResultRow *buf = pWindowResInfo->pResult[i]; if (!isWindowResClosed(pWindowResInfo, i)) { continue; } @@ -3797,7 +3799,7 @@ void setExecutionContext(SQInfo *pQInfo, int32_t groupIndex, TSKEY nextKey) { } uint64_t uid = 0; // uid is always set to be 0 - SWindowResult *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&groupIndex, + SResultRow *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&groupIndex, sizeof(groupIndex), true, uid); if (pWindowRes == NULL) { return; @@ -3820,7 +3822,7 @@ void setExecutionContext(SQInfo *pQInfo, int32_t groupIndex, TSKEY nextKey) { initCtxOutputBuf(pRuntimeEnv); } -void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult) { +void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult) { SQuery *pQuery = pRuntimeEnv->pQuery; // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group @@ -3839,15 +3841,11 @@ void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult * set the output buffer information and intermediate buffer * not all queries require the interResultBuf, such as COUNT */ - pCtx->resultInfo = &pResult->resultInfo[i]; - - // set super table query flag - SResultInfo *pResInfo = GET_RES_INFO(pCtx); - pResInfo->superTableQ = pRuntimeEnv->stableQuery; + pCtx->resultInfo = &pResult->pCellInfo[i]; } } -void setWindowResOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pResult) { +void setWindowResOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult) { SQuery *pQuery = pRuntimeEnv->pQuery; // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group @@ -3856,7 +3854,7 @@ void setWindowResOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult * for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; - pCtx->resultInfo = &pResult->resultInfo[i]; + pCtx->resultInfo = &pResult->pCellInfo[i]; if (pCtx->resultInfo->initialized && pCtx->resultInfo->complete) { continue; } @@ -3869,12 +3867,6 @@ void setWindowResOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult * pCtx->ptsOutputBuf = pRuntimeEnv->pCtx[0].aOutputBuf; } - /* - * set the output buffer information and intermediate buffer - * not all queries require the interResultBuf, such as COUNT - */ - pCtx->resultInfo->superTableQ = pRuntimeEnv->stableQuery; // set super table query flag - if (!pCtx->resultInfo->initialized) { aAggs[functionId].init(pCtx); } @@ -4015,7 +4007,7 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SWindowResInfo *pResultInfo, int32_ qDebug("QInfo:%p start to copy data from windowResInfo to query buf", pQInfo); int32_t totalSet = numOfClosedTimeWindow(pResultInfo); - SWindowResult** result = pResultInfo->pResult; + SResultRow** result = pResultInfo->pResult; if (orderType == TSDB_ORDER_ASC) { startIdx = pQInfo->groupIndex; @@ -4102,7 +4094,7 @@ static void updateWindowResNumOfRes(SQueryRuntimeEnv *pRuntimeEnv) { } for (int32_t i = 0; i < pRuntimeEnv->windowResInfo.size; ++i) { - SWindowResult *pResult = pRuntimeEnv->windowResInfo.pResult[i]; + SResultRow *pResult = pRuntimeEnv->windowResInfo.pResult[i]; for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { int32_t functionId = pRuntimeEnv->pCtx[j].functionId; @@ -4110,7 +4102,7 @@ static void updateWindowResNumOfRes(SQueryRuntimeEnv *pRuntimeEnv) { continue; } - pResult->numOfRows = (uint16_t)(MAX(pResult->numOfRows, pResult->resultInfo[j].numOfRes)); + pResult->numOfRows = (uint16_t)(MAX(pResult->numOfRows, pResult->pCellInfo[j].numOfRes)); } } } @@ -4678,7 +4670,7 @@ static void enableExecutionForNextTable(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - SResultInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[i]); + SResultRowCellInfo *pResInfo = GET_RES_INFO(&pRuntimeEnv->pCtx[i]); if (pResInfo != NULL) { pResInfo->complete = false; } @@ -5033,9 +5025,9 @@ static void sequentialTableProcess(SQInfo *pQInfo) { for (int32_t i = 0; i < pWindowResInfo->size; ++i) { pWindowResInfo->pResult[i]->closed = true; // enable return all results for group by normal columns - SWindowResult *pResult = pWindowResInfo->pResult[i]; + SResultRow *pResult = pWindowResInfo->pResult[i]; for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - pResult->numOfRows = (uint16_t)(MAX(pResult->numOfRows, pResult->resultInfo[j].numOfRes)); + pResult->numOfRows = (uint16_t)(MAX(pResult->numOfRows, pResult->pCellInfo[j].numOfRes)); } } @@ -6297,7 +6289,7 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou calResultBufSize(pQuery); for (int32_t col = 0; col < pQuery->numOfOutput; ++col) { - assert(pExprs[col].interBytes >= pExprs[col].bytes); + assert(pExprs[col].interBytes >= pExprs[col].bytes || pExprs[col].interBytes == 0); // allocate additional memory for interResults that are usually larger then final results size_t size = (size_t)((pQuery->rec.capacity + 1) * pExprs[col].bytes + pExprs[col].interBytes + sizeof(tFilePage)); diff --git a/src/query/src/qResultbuf.c b/src/query/src/qResultbuf.c index bd7c4694d0..edb2ca687f 100644 --- a/src/query/src/qResultbuf.c +++ b/src/query/src/qResultbuf.c @@ -407,9 +407,9 @@ void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) { } if (pResultBuf->file != NULL) { - qDebug("QInfo:%p res output buffer closed, total:%.2f Kb, inmem size:%dbytes, file size:%"PRId64" bytes", - pResultBuf->handle, pResultBuf->totalBufSize/1024.0, listNEles(pResultBuf->lruList) * pResultBuf->pageSize, - pResultBuf->fileSize); + qDebug("QInfo:%p res output buffer closed, total:%.2f Kb, inmem size:%.2f Kb, file size:%.2f", + pResultBuf->handle, pResultBuf->totalBufSize/1024.0, listNEles(pResultBuf->lruList) * pResultBuf->pageSize / 1024.0, + pResultBuf->fileSize/1024.0); fclose(pResultBuf->file); } else { diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index 7ec39cc0c8..02a81936f3 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -27,7 +27,7 @@ int32_t getOutputInterResultBufSize(SQuery* pQuery) { size += pQuery->pSelectExpr[i].interBytes; } - assert(size > 0); + assert(size >= 0); return size; } @@ -41,26 +41,12 @@ int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, SQueryRuntimeEnv *pRun pWindowResInfo->size = 0; pWindowResInfo->prevSKey = TSKEY_INITIAL_VAL; -// SQueryCostInfo* pSummary = &pRuntimeEnv->summary; - pWindowResInfo->pResult = calloc(pWindowResInfo->capacity, POINTER_BYTES); if (pWindowResInfo->pResult == NULL) { return TSDB_CODE_QRY_OUT_OF_MEMORY; } pWindowResInfo->interval = pRuntimeEnv->pQuery->interval.interval; - -// pSummary->winInfoSize += POINTER_BYTES * pWindowResInfo->capacity; -// pSummary->winInfoSize += (pRuntimeEnv->pQuery->numOfOutput * sizeof(SResultInfo) + pRuntimeEnv->interBufSize) * pWindowResInfo->capacity; -// pSummary->numOfTimeWindows = pWindowResInfo->capacity; - -// for (int32_t i = 0; i < pWindowResInfo->capacity; ++i) { -// int32_t code = createQueryResultInfo(pRuntimeEnv->pQuery, pWindowResInfo->pResult[i], pRuntimeEnv->stableQuery, pRuntimeEnv->interBufSize); -// if (code != TSDB_CODE_SUCCESS) { -// return code; -// } -// } - return TSDB_CODE_SUCCESS; } @@ -82,8 +68,8 @@ void resetTimeWindowInfo(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowR } for (int32_t i = 0; i < pWindowResInfo->size; ++i) { - SWindowResult *pWindowRes = pWindowResInfo->pResult[i]; - clearTimeWindowResBuf(pRuntimeEnv, pWindowRes); + SResultRow *pWindowRes = pWindowResInfo->pResult[i]; + clearResultRow(pRuntimeEnv, pWindowRes); } pWindowResInfo->curIndex = -1; @@ -108,7 +94,7 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { int16_t bytes = -1; for (int32_t i = 0; i < num; ++i) { - SWindowResult *pResult = pWindowResInfo->pResult[i]; + SResultRow *pResult = pWindowResInfo->pResult[i]; if (pResult->closed) { // remove the window slot from hash table // todo refactor @@ -131,19 +117,19 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { // clear all the closed windows from the window list for (int32_t k = 0; k < remain; ++k) { - copyTimeWindowResBuf(pRuntimeEnv, pWindowResInfo->pResult[k], pWindowResInfo->pResult[num + k]); + copyResultRow(pRuntimeEnv, pWindowResInfo->pResult[k], pWindowResInfo->pResult[num + k]); } // move the unclosed window in the front of the window list for (int32_t k = remain; k < pWindowResInfo->size; ++k) { - SWindowResult *pWindowRes = pWindowResInfo->pResult[k]; - clearTimeWindowResBuf(pRuntimeEnv, pWindowRes); + SResultRow *pWindowRes = pWindowResInfo->pResult[k]; + clearResultRow(pRuntimeEnv, pWindowRes); } pWindowResInfo->size = remain; for (int32_t k = 0; k < pWindowResInfo->size; ++k) { - SWindowResult *pResult = pWindowResInfo->pResult[k]; + SResultRow *pResult = pWindowResInfo->pResult[k]; if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { key = varDataVal(pResult->key); @@ -240,7 +226,7 @@ void closeTimeWindow(SWindowResInfo *pWindowResInfo, int32_t slot) { getWindowResult(pWindowResInfo, slot)->closed = true; } -void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindowRes) { +void clearResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pWindowRes) { if (pWindowRes == NULL) { return; } @@ -248,7 +234,7 @@ void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindow tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pageId); for (int32_t i = 0; i < pRuntimeEnv->pQuery->numOfOutput; ++i) { - SResultInfo *pResultInfo = &pWindowRes->resultInfo[i]; + SResultRowCellInfo *pResultInfo = &pWindowRes->pCellInfo[i]; char * s = getPosInResultPage(pRuntimeEnv, i, pWindowRes, page); size_t size = pRuntimeEnv->pQuery->pSelectExpr[i].bytes; @@ -269,7 +255,7 @@ void clearTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *pWindow * since the attribute of "Pos" is bound to each window result when the window result is created in the * disk-based result buffer. */ -void copyTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *dst, const SWindowResult *src) { +void copyResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *dst, const SResultRow *src) { dst->numOfRows = src->numOfRows; dst->win = src->win; dst->closed = src->closed; @@ -277,30 +263,35 @@ void copyTimeWindowResBuf(SQueryRuntimeEnv *pRuntimeEnv, SWindowResult *dst, con int32_t nOutputCols = pRuntimeEnv->pQuery->numOfOutput; for (int32_t i = 0; i < nOutputCols; ++i) { - SResultInfo *pDst = &dst->resultInfo[i]; - SResultInfo *pSrc = &src->resultInfo[i]; + SResultRowCellInfo *pDst = getResultCell(pRuntimeEnv, dst, i); + SResultRowCellInfo *pSrc = getResultCell(pRuntimeEnv, src, i); - char *buf = pDst->interResultBuf; - memcpy(pDst, pSrc, sizeof(SResultInfo)); - pDst->interResultBuf = buf; // restore the allocated buffer +// char *buf = pDst->interResultBuf; + memcpy(pDst, pSrc, sizeof(SResultRowCellInfo) + pRuntimeEnv->pCtx[i].interBufBytes); +// pDst->interResultBuf = buf; // restore the allocated buffer // copy the result info struct - memcpy(pDst->interResultBuf, pSrc->interResultBuf, pDst->bufLen); +// memcpy(pDst->interResultBuf, pSrc->interResultBuf, pRuntimeEnv->pCtx[i].interBufBytes); // copy the output buffer data from src to dst, the position info keep unchanged tFilePage *dstpage = getResBufPage(pRuntimeEnv->pResultBuf, dst->pageId); char * dstBuf = getPosInResultPage(pRuntimeEnv, i, dst, dstpage); tFilePage *srcpage = getResBufPage(pRuntimeEnv->pResultBuf, src->pageId); - char * srcBuf = getPosInResultPage(pRuntimeEnv, i, (SWindowResult *)src, srcpage); + char * srcBuf = getPosInResultPage(pRuntimeEnv, i, (SResultRow *)src, srcpage); size_t s = pRuntimeEnv->pQuery->pSelectExpr[i].bytes; memcpy(dstBuf, srcBuf, s); } } +SResultRowCellInfo* getResultCell(SQueryRuntimeEnv* pRuntimeEnv, const SResultRow* pRow, int32_t index) { + assert(index >= 0 && index < pRuntimeEnv->pQuery->numOfOutput); + return (SResultRowCellInfo*)((char*) pRow->pCellInfo + pRuntimeEnv->rowCellInfoOffset[index]); +} + size_t getWindowResultSize(SQueryRuntimeEnv* pRuntimeEnv) { - return (pRuntimeEnv->pQuery->numOfOutput * sizeof(SResultInfo)) + pRuntimeEnv->interBufSize + sizeof(SWindowResult); + return (pRuntimeEnv->pQuery->numOfOutput * sizeof(SResultRowCellInfo)) + pRuntimeEnv->interBufSize + sizeof(SResultRow); } SWindowResultPool* initWindowResultPool(size_t size) { @@ -320,7 +311,7 @@ SWindowResultPool* initWindowResultPool(size_t size) { return p; } -SWindowResult* getNewWindowResult(SWindowResultPool* p) { +SResultRow* getNewWindowResult(SWindowResultPool* p) { if (p == NULL) { return NULL; } diff --git a/src/util/inc/hash.h b/src/util/inc/hash.h index d4e9b1677c..42bc136584 100644 --- a/src/util/inc/hash.h +++ b/src/util/inc/hash.h @@ -31,13 +31,16 @@ extern "C" { typedef void (*_hash_free_fn_t)(void *param); typedef struct SHashNode { - char *key; +// char *key; struct SHashNode *next; - uint32_t hashVal; // the hash value of key, if hashVal == HASH_VALUE_IN_TRASH, this node is moved to trash + uint32_t hashVal; // the hash value of key uint32_t keyLen; // length of the key - char *data; +// char *data; } SHashNode; +#define GET_HASH_NODE_KEY(_n) ((char*)(_n) + sizeof(SHashNode)) +#define GET_HASH_NODE_DATA(_n) ((char*)(_n) + sizeof(SHashNode) + (_n)->keyLen) + typedef enum SHashLockTypeE { HASH_NO_LOCK = 0, HASH_ENTRY_LOCK = 1, diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 8ffbb1f0f6..10767ee30f 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -22,14 +22,13 @@ #define DO_FREE_HASH_NODE(_n) \ do { \ - taosTFree((_n)->data); \ taosTFree(_n); \ } while (0) #define FREE_HASH_NODE(_h, _n) \ do { \ if ((_h)->freeFp) { \ - (_h)->freeFp((_n)->data); \ + (_h)->freeFp(GET_HASH_NODE_DATA(_n)); \ } \ \ DO_FREE_HASH_NODE(_n); \ @@ -77,7 +76,7 @@ static FORCE_INLINE int32_t taosHashCapacity(int32_t length) { static FORCE_INLINE SHashNode *doSearchInEntryList(SHashEntry *pe, const void *key, size_t keyLen, uint32_t hashVal) { SHashNode *pNode = pe->next; while (pNode) { - if ((pNode->keyLen == keyLen) && (memcmp(pNode->key, key, keyLen) == 0)) { + if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0)) { assert(pNode->hashVal == hashVal); break; } @@ -115,11 +114,13 @@ static SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *p * @param dsize size of actual data * @return hash node */ -static FORCE_INLINE SHashNode *doUpdateHashNode(SHashNode *pNode, SHashNode *pNewNode) { +static FORCE_INLINE SHashNode *doUpdateHashNode(SHashNode* prev, SHashNode *pNode, SHashNode *pNewNode) { assert(pNode->keyLen == pNewNode->keyLen); - SWAP(pNode->key, pNewNode->key, void *); - SWAP(pNode->data, pNewNode->data, void *); + if (prev != NULL) { + prev->next = pNewNode; + } + pNewNode->next = pNode->next; return pNewNode; } @@ -208,12 +209,14 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da assert(pNode == NULL); } + SHashNode* prev = NULL; while (pNode) { - if ((pNode->keyLen == keyLen) && (memcmp(pNode->key, key, keyLen) == 0)) { + if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0)) { assert(pNode->hashVal == hashVal); break; } + prev = pNode; pNode = pNode->next; } @@ -239,7 +242,7 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da } else { // not support the update operation, return error if (pHashObj->enableUpdate) { - doUpdateHashNode(pNode, pNewNode); + doUpdateHashNode(prev, pNode, pNewNode); } if (pHashObj->type == HASH_ENTRY_LOCK) { @@ -293,13 +296,13 @@ void* taosHashGetCB(SHashObj *pHashObj, const void *key, size_t keyLen, void (*f SHashNode *pNode = doSearchInEntryList(pe, key, keyLen, hashVal); if (pNode != NULL) { if (fp != NULL) { - fp(pNode->data); + fp(GET_HASH_NODE_DATA(pNode)); } if (d != NULL) { - memcpy(d, pNode->data, dsize); + memcpy(d, GET_HASH_NODE_DATA(pNode), dsize); } else { - data = pNode->data; + data = GET_HASH_NODE_DATA(pNode); } } @@ -357,13 +360,13 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe SHashNode *pRes = NULL; // remove it - if ((pNode->keyLen == keyLen) && (memcmp(pNode->key, key, keyLen) == 0)) { + if ((pNode->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY(pNode), key, keyLen) == 0)) { pe->num -= 1; pRes = pNode; pe->next = pNode->next; } else { while (pNode->next != NULL) { - if (((pNode->next)->keyLen == keyLen) && (memcmp((pNode->next)->key, key, keyLen) == 0)) { + if (((pNode->next)->keyLen == keyLen) && (memcmp(GET_HASH_NODE_KEY((pNode->next)), key, keyLen) == 0)) { assert((pNode->next)->hashVal == hashVal); break; } @@ -392,7 +395,7 @@ int32_t taosHashRemoveWithData(SHashObj *pHashObj, const void *key, size_t keyLe __rd_unlock(&pHashObj->lock, pHashObj->type); if (data != NULL && pRes != NULL) { - memcpy(data, pRes->data, dsize); + memcpy(data, GET_HASH_NODE_DATA(pRes), dsize); } if (pRes != NULL) { @@ -426,7 +429,7 @@ int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), voi // todo remove the first node SHashNode *pNode = NULL; while((pNode = pEntry->next) != NULL) { - if (fp && (!fp(param, pNode->data))) { + if (fp && (!fp(param, GET_HASH_NODE_DATA(pNode)))) { pEntry->num -= 1; atomic_sub_fetch_64(&pHashObj->size, 1); @@ -451,7 +454,7 @@ int32_t taosHashCondTraverse(SHashObj *pHashObj, bool (*fp)(void *, void *), voi while ((pNext = pNode->next) != NULL) { // not qualified, remove it - if (fp && (!fp(param, pNext->data))) { + if (fp && (!fp(param, GET_HASH_NODE_DATA(pNext)))) { pNode->next = pNext->next; pEntry->num -= 1; atomic_sub_fetch_64(&pHashObj->size, 1); @@ -605,7 +608,7 @@ bool taosHashIterNext(SHashMutableIterator *pIter) { } } -void *taosHashIterGet(SHashMutableIterator *iter) { return (iter == NULL) ? NULL : iter->pCur->data; } +void *taosHashIterGet(SHashMutableIterator *iter) { return (iter == NULL) ? NULL : GET_HASH_NODE_DATA(iter->pCur); } void *taosHashDestroyIter(SHashMutableIterator *iter) { if (iter == NULL) { @@ -743,21 +746,19 @@ void taosHashTableResize(SHashObj *pHashObj) { } SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *pData, size_t dsize, uint32_t hashVal) { - SHashNode *pNewNode = calloc(1, sizeof(SHashNode)); + SHashNode *pNewNode = calloc(1, sizeof(SHashNode) + keyLen + dsize); if (pNewNode == NULL) { uError("failed to allocate memory, reason:%s", strerror(errno)); return NULL; } - pNewNode->data = malloc(dsize + keyLen); - memcpy(pNewNode->data, pData, dsize); - - pNewNode->key = pNewNode->data + dsize; - memcpy(pNewNode->key, key, keyLen); - pNewNode->keyLen = (uint32_t)keyLen; pNewNode->hashVal = hashVal; + + memcpy(GET_HASH_NODE_DATA(pNewNode), pData, dsize); + memcpy(GET_HASH_NODE_KEY(pNewNode), key, keyLen); + return pNewNode; } From 95104645c59417fa6a5db20a3e38d425f9e5e224 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 2 Nov 2020 14:40:36 +0800 Subject: [PATCH 13/35] [TD-225]: optimize the memory consumption for interval query. --- src/query/inc/qExecutor.h | 7 +++---- src/query/inc/qUtil.h | 3 +-- src/query/src/qExecutor.c | 8 +++----- src/query/src/qUtil.c | 4 +--- 4 files changed, 8 insertions(+), 14 deletions(-) diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index 64a0cb1a62..18cc99285e 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -83,14 +83,13 @@ typedef struct SResultRec { typedef struct SWindowResInfo { SResultRow** pResult; // result list - int16_t type; // data type for hash key + int16_t type:8; // data type for hash key + int32_t size:24; // number of result set + int32_t threshold; // threshold to halt query and return the generated results. int32_t capacity; // max capacity int32_t curIndex; // current start active index - int32_t size; // number of result set int64_t startTime; // start time of the first time window for sliding query int64_t prevSKey; // previous (not completed) sliding window start key - int64_t threshold; // threshold to halt query and return the generated results. - int64_t interval; // time window interval } SWindowResInfo; typedef struct SColumnFilterElem { diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h index 698608a689..84ef0d3f2e 100644 --- a/src/query/inc/qUtil.h +++ b/src/query/inc/qUtil.h @@ -30,8 +30,7 @@ void clearResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* pRow); void copyResultRow(SQueryRuntimeEnv* pRuntimeEnv, SResultRow* dst, const SResultRow* src); SResultRowCellInfo* getResultCell(SQueryRuntimeEnv* pRuntimeEnv, const SResultRow* pRow, int32_t index); -int32_t initWindowResInfo(SWindowResInfo* pWindowResInfo, SQueryRuntimeEnv* pRuntimeEnv, int32_t size, - int32_t threshold, int16_t type); +int32_t initWindowResInfo(SWindowResInfo* pWindowResInfo, int32_t size, int32_t threshold, int16_t type); void cleanupTimeWindowInfo(SWindowResInfo* pWindowResInfo); void resetTimeWindowInfo(SQueryRuntimeEnv* pRuntimeEnv, SWindowResInfo* pWindowResInfo); diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index a573431bfc..c98000d73c 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -1136,8 +1136,6 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat pWindowRes->win.ekey = v; } - assert(pRuntimeEnv->windowResInfo.interval == 0); - if (pWindowRes->pageId == -1) { int32_t ret = addNewWindowResultBuf(pWindowRes, pResultBuf, GROUPRESULTID, pRuntimeEnv->numOfRowsPerPage); if (ret != 0) { @@ -3758,7 +3756,7 @@ static STableQueryInfo *createTableQueryInfo(SQueryRuntimeEnv *pRuntimeEnv, void if (QUERY_IS_INTERVAL_QUERY(pQuery) || pRuntimeEnv->groupbyNormalCol) { int32_t initialSize = 16; int32_t initialThreshold = 100; - int32_t code = initWindowResInfo(&pTableQueryInfo->windowResInfo, pRuntimeEnv, initialSize, initialThreshold, TSDB_DATA_TYPE_INT); + int32_t code = initWindowResInfo(&pTableQueryInfo->windowResInfo, initialSize, initialThreshold, TSDB_DATA_TYPE_INT); if (code != TSDB_CODE_SUCCESS) { return NULL; } @@ -4623,7 +4621,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo } } - code = initWindowResInfo(&pRuntimeEnv->windowResInfo, pRuntimeEnv, 8, threshold, type); + code = initWindowResInfo(&pRuntimeEnv->windowResInfo, 8, threshold, type); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -4643,7 +4641,7 @@ int32_t doInitQInfo(SQInfo *pQInfo, STSBuf *pTsBuf, void *tsdb, int32_t vgId, bo type = TSDB_DATA_TYPE_TIMESTAMP; } - code = initWindowResInfo(&pRuntimeEnv->windowResInfo, pRuntimeEnv, numOfResultRows, 4096, type); + code = initWindowResInfo(&pRuntimeEnv->windowResInfo, numOfResultRows, 1024, type); if (code != TSDB_CODE_SUCCESS) { return code; } diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index 02a81936f3..908c9026d2 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -31,8 +31,7 @@ int32_t getOutputInterResultBufSize(SQuery* pQuery) { return size; } -int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, SQueryRuntimeEnv *pRuntimeEnv, int32_t size, - int32_t threshold, int16_t type) { +int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, int32_t size, int32_t threshold, int16_t type) { pWindowResInfo->capacity = size; pWindowResInfo->threshold = threshold; @@ -46,7 +45,6 @@ int32_t initWindowResInfo(SWindowResInfo *pWindowResInfo, SQueryRuntimeEnv *pRun return TSDB_CODE_QRY_OUT_OF_MEMORY; } - pWindowResInfo->interval = pRuntimeEnv->pQuery->interval.interval; return TSDB_CODE_SUCCESS; } From 64ab48d7ead9a3eede2494b913d935d3d7c82dee Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 2 Nov 2020 15:03:02 +0800 Subject: [PATCH 14/35] [TD-1870]: optimize the memory consumption during interval query. --- src/tsdb/src/tsdbRead.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tsdb/src/tsdbRead.c b/src/tsdb/src/tsdbRead.c index 90e16aee53..3daefc1f5f 100644 --- a/src/tsdb/src/tsdbRead.c +++ b/src/tsdb/src/tsdbRead.c @@ -72,8 +72,8 @@ typedef struct STableCheckInfo { STable* pTableObj; SCompInfo* pCompInfo; int32_t compSize; - int32_t numOfBlocks; // number of qualified data blocks not the original blocks - int32_t chosen; // indicate which iterator should move forward + int32_t numOfBlocks:29; // number of qualified data blocks not the original blocks + int8_t chosen:2; // indicate which iterator should move forward bool initBuf; // whether to initialize the in-memory skip list iterator or not SSkipListIterator* iter; // mem buffer skip list iterator SSkipListIterator* iiter; // imem buffer skip list iterator From 2cd975745c7ae815ba524ab3b5e412b584c44bcd Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 2 Nov 2020 15:23:43 +0800 Subject: [PATCH 15/35] [TD-225] refactor codes. --- src/query/inc/qExecutor.h | 8 ++++---- src/query/inc/qUtil.h | 12 ++++++------ src/query/src/qExecutor.c | 24 ++++++++++++------------ src/query/src/qUtil.c | 22 +++++++++++----------- 4 files changed, 33 insertions(+), 33 deletions(-) diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index 18cc99285e..30fc49e74a 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -40,7 +40,7 @@ typedef struct SGroupResInfo { int32_t rowId; } SGroupResInfo; -typedef struct SWindowResultPool { +typedef struct SResultRowPool { int32_t elemSize; int32_t blockSize; int32_t numOfElemPerBlock; @@ -51,7 +51,7 @@ typedef struct SWindowResultPool { } position; SArray* pData; // SArray -} SWindowResultPool; +} SResultRowPool; typedef struct SSqlGroupbyExpr { int16_t tableIndex; @@ -188,9 +188,9 @@ typedef struct SQueryRuntimeEnv { int32_t interBufSize; // intermediate buffer sizse int32_t prevGroupId; // previous executed group id SDiskbasedResultBuf* pResultBuf; // query result buffer based on blocked-wised disk file - SHashObj* pWindowHashTable; // quick locate the window object for each result + SHashObj* pResultRowHashTable; // quick locate the window object for each result char* keyBuf; // window key buffer - SWindowResultPool* pool; // window result object pool + SResultRowPool* pool; // window result object pool int32_t* rowCellInfoOffset;// offset value for each row result cell info } SQueryRuntimeEnv; diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h index 84ef0d3f2e..77ea0dd013 100644 --- a/src/query/inc/qUtil.h +++ b/src/query/inc/qUtil.h @@ -73,12 +73,12 @@ __filter_func_t *getValueFilterFuncArray(int32_t type); size_t getWindowResultSize(SQueryRuntimeEnv* pRuntimeEnv); -SWindowResultPool* initWindowResultPool(size_t size); -SResultRow* getNewWindowResult(SWindowResultPool* p); -int64_t getWindowResultPoolMemSize(SWindowResultPool* p); -void* destroyWindowResultPool(SWindowResultPool* p); -int32_t getNumOfAllocatedWindowResult(SWindowResultPool* p); -int32_t getNumOfUsedWindowResult(SWindowResultPool* p); +SResultRowPool* initResultRowPool(size_t size); +SResultRow* getNewResultRow(SResultRowPool* p); +int64_t getResultRowPoolMemSize(SResultRowPool* p); +void* destroyResultRowPool(SResultRowPool* p); +int32_t getNumOfAllocatedResultRows(SResultRowPool* p); +int32_t getNumOfUsedResultRows(SResultRowPool* p); #endif // TDENGINE_QUERYUTIL_H diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index c98000d73c..462966459e 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -452,7 +452,7 @@ static SResultRow *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWindow SQuery *pQuery = pRuntimeEnv->pQuery; SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, uid); - int32_t *p1 = (int32_t *) taosHashGet(pRuntimeEnv->pWindowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); + int32_t *p1 = (int32_t *) taosHashGet(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); if (p1 != NULL) { pWindowResInfo->curIndex = *p1; } else { @@ -485,7 +485,7 @@ static SResultRow *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWindow pWindowResInfo->capacity = (int32_t)newCapacity; } // pRuntimeEnv->summary.winInfoSize += (pQuery->numOfOutput * sizeof(SResultRowCellInfo) + pRuntimeEnv->interBufSize) * inc; - SResultRow* pResult = getNewWindowResult(pRuntimeEnv->pool); + SResultRow* pResult = getNewResultRow(pRuntimeEnv->pool); pWindowResInfo->pResult[pWindowResInfo->size] = pResult; int32_t ret = createQueryResultInfo(pQuery, pResult); if (ret != TSDB_CODE_SUCCESS) { @@ -494,7 +494,7 @@ static SResultRow *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWindow // add a new result set for a new group pWindowResInfo->curIndex = pWindowResInfo->size++; - taosHashPut(pRuntimeEnv->pWindowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), (char *)&pWindowResInfo->curIndex, sizeof(int32_t)); + taosHashPut(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), (char *)&pWindowResInfo->curIndex, sizeof(int32_t)); } // too many time window in query @@ -1771,10 +1771,10 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { pRuntimeEnv->pTSBuf = tsBufDestroy(pRuntimeEnv->pTSBuf); taosTFree(pRuntimeEnv->keyBuf); - taosHashCleanup(pRuntimeEnv->pWindowHashTable); - pRuntimeEnv->pWindowHashTable = NULL; + taosHashCleanup(pRuntimeEnv->pResultRowHashTable); + pRuntimeEnv->pResultRowHashTable = NULL; - pRuntimeEnv->pool = destroyWindowResultPool(pRuntimeEnv->pool); + pRuntimeEnv->pool = destroyResultRowPool(pRuntimeEnv->pool); } #define IS_QUERY_KILLED(_q) ((_q)->code == TSDB_CODE_TSC_QUERY_CANCELLED) @@ -4246,16 +4246,16 @@ static void queryCostStatis(SQInfo *pQInfo) { SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv; SQueryCostInfo *pSummary = &pRuntimeEnv->summary; - uint64_t hashSize = taosHashGetMemSize(pQInfo->runtimeEnv.pWindowHashTable); + uint64_t hashSize = taosHashGetMemSize(pQInfo->runtimeEnv.pResultRowHashTable); hashSize += taosHashGetMemSize(pQInfo->tableqinfoGroupInfo.map); pSummary->hashSize = hashSize; // add the merge time pSummary->elapsedTime += pSummary->firstStageMergeTime; - SWindowResultPool* p = pQInfo->runtimeEnv.pool; - pSummary->winInfoSize = getWindowResultPoolMemSize(p); - pSummary->numOfTimeWindows = getNumOfAllocatedWindowResult(p); + SResultRowPool* p = pQInfo->runtimeEnv.pool; + pSummary->winInfoSize = getResultRowPoolMemSize(p); + pSummary->numOfTimeWindows = getNumOfAllocatedResultRows(p); qDebug("QInfo:%p :cost summary: elapsed time:%"PRId64" us, first merge:%"PRId64" us, total blocks:%d, " "load block statis:%d, load data block:%d, total rows:%"PRId64 ", check rows:%"PRId64, @@ -6322,9 +6322,9 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou pQInfo->runtimeEnv.interBufSize = getOutputInterResultBufSize(pQuery); pQInfo->runtimeEnv.summary.tableInfoSize += (pTableGroupInfo->numOfTables * sizeof(STableQueryInfo)); - pQInfo->runtimeEnv.pWindowHashTable = taosHashInit(pTableGroupInfo->numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + pQInfo->runtimeEnv.pResultRowHashTable = taosHashInit(pTableGroupInfo->numOfTables, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); pQInfo->runtimeEnv.keyBuf = malloc(TSDB_MAX_BYTES_PER_ROW); - pQInfo->runtimeEnv.pool = initWindowResultPool(getWindowResultSize(&pQInfo->runtimeEnv)); + pQInfo->runtimeEnv.pool = initResultRowPool(getWindowResultSize(&pQInfo->runtimeEnv)); pQInfo->pBuf = calloc(pTableGroupInfo->numOfTables, sizeof(STableQueryInfo)); if (pQInfo->pBuf == NULL) { diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index 908c9026d2..600f48928a 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -105,7 +105,7 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { } SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, uid); - taosHashRemove(pRuntimeEnv->pWindowHashTable, (const char *)pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); + taosHashRemove(pRuntimeEnv->pResultRowHashTable, (const char *)pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); } else { break; } @@ -138,14 +138,14 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { } SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, uid); - int32_t *p = (int32_t *)taosHashGet(pRuntimeEnv->pWindowHashTable, (const char *)pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); + int32_t *p = (int32_t *)taosHashGet(pRuntimeEnv->pResultRowHashTable, (const char *)pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); assert(p != NULL); int32_t v = (*p - num); assert(v >= 0 && v <= pWindowResInfo->size); SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, uid); - taosHashPut(pRuntimeEnv->pWindowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), (char *)&v, sizeof(int32_t)); + taosHashPut(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), (char *)&v, sizeof(int32_t)); } pWindowResInfo->curIndex = -1; @@ -292,8 +292,8 @@ size_t getWindowResultSize(SQueryRuntimeEnv* pRuntimeEnv) { return (pRuntimeEnv->pQuery->numOfOutput * sizeof(SResultRowCellInfo)) + pRuntimeEnv->interBufSize + sizeof(SResultRow); } -SWindowResultPool* initWindowResultPool(size_t size) { - SWindowResultPool* p = calloc(1, sizeof(SWindowResultPool)); +SResultRowPool* initResultRowPool(size_t size) { + SResultRowPool* p = calloc(1, sizeof(SResultRowPool)); if (p == NULL) { return NULL; } @@ -309,7 +309,7 @@ SWindowResultPool* initWindowResultPool(size_t size) { return p; } -SResultRow* getNewWindowResult(SWindowResultPool* p) { +SResultRow* getNewResultRow(SResultRowPool* p) { if (p == NULL) { return NULL; } @@ -330,7 +330,7 @@ SResultRow* getNewWindowResult(SWindowResultPool* p) { return ptr; } -int64_t getWindowResultPoolMemSize(SWindowResultPool* p) { +int64_t getResultRowPoolMemSize(SResultRowPool* p) { if (p == NULL) { return 0; } @@ -338,15 +338,15 @@ int64_t getWindowResultPoolMemSize(SWindowResultPool* p) { return taosArrayGetSize(p->pData) * p->blockSize; } -int32_t getNumOfAllocatedWindowResult(SWindowResultPool* p) { +int32_t getNumOfAllocatedResultRows(SResultRowPool* p) { return taosArrayGetSize(p->pData) * p->numOfElemPerBlock; } -int32_t getNumOfUsedWindowResult(SWindowResultPool* p) { - return getNumOfAllocatedWindowResult(p) - p->numOfElemPerBlock + p->position.pos; +int32_t getNumOfUsedResultRows(SResultRowPool* p) { + return getNumOfAllocatedResultRows(p) - p->numOfElemPerBlock + p->position.pos; } -void* destroyWindowResultPool(SWindowResultPool* p) { +void* destroyResultRowPool(SResultRowPool* p) { if (p == NULL) { return NULL; } From a9f0a30dc5d70ab1cc4dda6af51c12e6bc3e23f6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 2 Nov 2020 23:44:15 +0800 Subject: [PATCH 16/35] [TD-1870]: optimize the memory consumption. --- src/client/inc/tscLocalMerge.h | 5 +- src/client/src/tscFunctionImpl.c | 22 ++-- src/client/src/tscLocalMerge.c | 19 +--- src/query/inc/qExecutor.h | 4 +- src/query/inc/qUtil.h | 4 +- src/query/src/qExecutor.c | 167 +++++++++++-------------------- src/query/src/qUtil.c | 37 ++++--- src/util/src/hash.c | 10 +- 8 files changed, 107 insertions(+), 161 deletions(-) diff --git a/src/client/inc/tscLocalMerge.h b/src/client/inc/tscLocalMerge.h index edd83ad071..0af8c8b576 100644 --- a/src/client/inc/tscLocalMerge.h +++ b/src/client/inc/tscLocalMerge.h @@ -64,9 +64,8 @@ typedef struct SLocalReducer { SColumnModel * resColModel; tExtMemBuffer ** pExtMemBuffer; // disk-based buffer SFillInfo* pFillInfo; // interpolation support structure - char * pFinalRes; // result data after interpo - tFilePage * discardData; - SResultRowCellInfo * pResInfo; + char* pFinalRes; // result data after interpo + tFilePage* discardData; bool discard; int32_t offset; // limit offset value bool orderPrjOnSTable; // projection query on stable diff --git a/src/client/src/tscFunctionImpl.c b/src/client/src/tscFunctionImpl.c index b22c178466..e0bd762333 100644 --- a/src/client/src/tscFunctionImpl.c +++ b/src/client/src/tscFunctionImpl.c @@ -99,7 +99,7 @@ typedef struct SSumInfo { // the attribute of hasResult is not needed since the num attribute would server as this purpose typedef struct SAvgInfo { double sum; - int64_t num; // num servers as the hasResult attribute in other struct + int64_t num; } SAvgInfo; typedef struct SStddevInfo { @@ -167,7 +167,13 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_INTERP) { *type = (int16_t)dataType; *bytes = (int16_t)dataBytes; - *interBytes = 0;//*bytes; + + if (functionId == TSDB_FUNC_INTERP) { + *interBytes = sizeof(SInterpInfoDetail); + } else { + *interBytes = 0; + } + return TSDB_CODE_SUCCESS; } @@ -175,21 +181,21 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI if (functionId == TSDB_FUNC_TID_TAG) { // todo use struct *type = TSDB_DATA_TYPE_BINARY; *bytes = (int16_t)(dataBytes + sizeof(int16_t) + sizeof(int64_t) + sizeof(int32_t) + sizeof(int32_t) + VARSTR_HEADER_SIZE); - *interBytes = 0;//*bytes; + *interBytes = 0; return TSDB_CODE_SUCCESS; } if (functionId == TSDB_FUNC_COUNT) { *type = TSDB_DATA_TYPE_BIGINT; *bytes = sizeof(int64_t); - *interBytes = 0;//*bytes; + *interBytes = 0; return TSDB_CODE_SUCCESS; } if (functionId == TSDB_FUNC_ARITHM) { *type = TSDB_DATA_TYPE_DOUBLE; *bytes = sizeof(double); - *interBytes = 0;//*bytes; + *interBytes = 0; return TSDB_CODE_SUCCESS; } @@ -334,11 +340,6 @@ int32_t getResultDataInfo(int32_t dataType, int32_t dataBytes, int32_t functionI return TSDB_CODE_SUCCESS; } -//void setResultInfoBuf(SResultRowCellInfo *pResInfo, char* buf) { -// assert(GET_ROWCELL_INTERBUF(pResInfo) == NULL); -// GET_ROWCELL_INTERBUF(pResInfo) = buf; -//} - // set the query flag to denote that query is completed static void no_next_step(SQLFunctionCtx *pCtx) { SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx); @@ -3998,7 +3999,6 @@ static void interp_function(SQLFunctionCtx *pCtx) { } } } - } SET_VAL(pCtx, pCtx->size, 1); diff --git a/src/client/src/tscLocalMerge.c b/src/client/src/tscLocalMerge.c index 58c2b76b72..c67edf5b5a 100644 --- a/src/client/src/tscLocalMerge.c +++ b/src/client/src/tscLocalMerge.c @@ -99,11 +99,8 @@ static void tscInitSqlContext(SSqlCmd *pCmd, SLocalReducer *pReducer, tOrderDesc pCtx->param[1].i64Key = pQueryInfo->order.orderColId; } -// SResultRowCellInfo *pResInfo = &pReducer->pResInfo[i]; - pCtx->interBufBytes = pExpr->interBytes; -// pResInfo->interResultBuf = calloc(1, (size_t) pCtx->interBufBytes); - - pCtx->resultInfo = &pReducer->pResInfo[i]; + pCtx->interBufBytes = pExpr->interBytes; + pCtx->resultInfo = calloc(1, pCtx->interBufBytes + sizeof(SResultRowCellInfo)); pCtx->stableQuery = true; } @@ -345,7 +342,6 @@ void tscCreateLocalReducer(tExtMemBuffer **pMemBuffer, int32_t numOfBuffer, tOrd size_t numOfCols = tscSqlExprNumOfExprs(pQueryInfo); pReducer->pTempBuffer->num = 0; - pReducer->pResInfo = calloc(numOfCols, sizeof(SResultRowCellInfo)); tscCreateResPointerInfo(pRes, pQueryInfo); tscInitSqlContext(pCmd, pReducer, pDesc); @@ -496,6 +492,8 @@ void tscDestroyLocalReducer(SSqlObj *pSql) { SQLFunctionCtx *pCtx = &pLocalReducer->pCtx[i]; tVariantDestroy(&pCtx->tag); + taosTFree(pCtx->resultInfo); + if (pCtx->tagInfo.pTagCtxList != NULL) { taosTFree(pCtx->tagInfo.pTagCtxList); } @@ -509,15 +507,6 @@ void tscDestroyLocalReducer(SSqlObj *pSql) { taosTFree(pLocalReducer->pTempBuffer); taosTFree(pLocalReducer->pResultBuf); - if (pLocalReducer->pResInfo != NULL) { - size_t num = tscSqlExprNumOfExprs(pQueryInfo); - for (int32_t i = 0; i < num; ++i) { -// taosTFree(pLocalReducer->pResInfo[i].interResultBuf); - } - - taosTFree(pLocalReducer->pResInfo); - } - if (pLocalReducer->pLoserTree) { taosTFree(pLocalReducer->pLoserTree->param); taosTFree(pLocalReducer->pLoserTree); diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index 30fc49e74a..d7db897cdf 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -168,7 +168,7 @@ typedef struct SQuery { typedef struct SQueryRuntimeEnv { jmp_buf env; - SResultRowCellInfo* resultInfo; // todo refactor to merge with SWindowResInfo + SResultRow* pResultRow; // todo refactor to merge with SWindowResInfo SQuery* pQuery; SQLFunctionCtx* pCtx; int32_t numOfRowsPerPage; @@ -190,7 +190,7 @@ typedef struct SQueryRuntimeEnv { SDiskbasedResultBuf* pResultBuf; // query result buffer based on blocked-wised disk file SHashObj* pResultRowHashTable; // quick locate the window object for each result char* keyBuf; // window key buffer - SResultRowPool* pool; // window result object pool + SResultRowPool* pool; // window result object pool int32_t* rowCellInfoOffset;// offset value for each row result cell info } SQueryRuntimeEnv; diff --git a/src/query/inc/qUtil.h b/src/query/inc/qUtil.h index 77ea0dd013..5d649261a6 100644 --- a/src/query/inc/qUtil.h +++ b/src/query/inc/qUtil.h @@ -42,7 +42,7 @@ void closeTimeWindow(SWindowResInfo* pWindowResInfo, int32_t slot); void closeAllTimeWindow(SWindowResInfo* pWindowResInfo); void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_t order); -static FORCE_INLINE SResultRow *getWindowResult(SWindowResInfo *pWindowResInfo, int32_t slot) { +static FORCE_INLINE SResultRow *getResultRow(SWindowResInfo *pWindowResInfo, int32_t slot) { assert(pWindowResInfo != NULL && slot >= 0 && slot < pWindowResInfo->size); return pWindowResInfo->pResult[slot]; } @@ -52,7 +52,7 @@ static FORCE_INLINE SResultRow *getWindowResult(SWindowResInfo *pWindowResInfo, bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot); -int32_t createQueryResultInfo(SQuery *pQuery, SResultRow *pResultRow); +int32_t initResultRow(SResultRow *pResultRow); static FORCE_INLINE char *getPosInResultPage(SQueryRuntimeEnv *pRuntimeEnv, int32_t columnIndex, SResultRow *pResult, tFilePage* page) { diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 462966459e..c0390b51e4 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -178,7 +178,7 @@ static void getNextTimeWindow(SQuery* pQuery, STimeWindow* tw) { // todo move to utility static int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *group); -static void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult); +static void setResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult); static void setWindowResOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult); static void resetMergeResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx *pCtx, SResultRow *pRow); static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t functionId); @@ -449,10 +449,9 @@ static bool hasNullValue(SColIndex* pColIndex, SDataStatis *pStatis, SDataStatis static SResultRow *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo, char *pData, int16_t bytes, bool masterscan, uint64_t uid) { - SQuery *pQuery = pRuntimeEnv->pQuery; - SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, uid); - int32_t *p1 = (int32_t *) taosHashGet(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); + int32_t *p1 = + (int32_t *)taosHashGet(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); if (p1 != NULL) { pWindowResInfo->curIndex = *p1; } else { @@ -470,9 +469,6 @@ static SResultRow *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWindow } char *t = realloc(pWindowResInfo->pResult, (size_t)(newCapacity * POINTER_BYTES)); - // pRuntimeEnv->summary.winInfoSize += (newCapacity - pWindowResInfo->capacity) * sizeof(SResultRow); - // pRuntimeEnv->summary.numOfTimeWindows += (newCapacity - pWindowResInfo->capacity); - if (t == NULL) { longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -484,17 +480,18 @@ static SResultRow *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWindow pWindowResInfo->capacity = (int32_t)newCapacity; } -// pRuntimeEnv->summary.winInfoSize += (pQuery->numOfOutput * sizeof(SResultRowCellInfo) + pRuntimeEnv->interBufSize) * inc; - SResultRow* pResult = getNewResultRow(pRuntimeEnv->pool); - pWindowResInfo->pResult[pWindowResInfo->size] = pResult; - int32_t ret = createQueryResultInfo(pQuery, pResult); - if (ret != TSDB_CODE_SUCCESS) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } + + SResultRow *pResult = getNewResultRow(pRuntimeEnv->pool); + pWindowResInfo->pResult[pWindowResInfo->size] = pResult; + int32_t ret = initResultRow(pResult); + if (ret != TSDB_CODE_SUCCESS) { + longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } // add a new result set for a new group pWindowResInfo->curIndex = pWindowResInfo->size++; - taosHashPut(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), (char *)&pWindowResInfo->curIndex, sizeof(int32_t)); + taosHashPut(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), + (char *)&pWindowResInfo->curIndex, sizeof(int32_t)); } // too many time window in query @@ -502,7 +499,7 @@ static SResultRow *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWindow longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_TOO_MANY_TIMEWINDOW); } - return getWindowResult(pWindowResInfo, pWindowResInfo->curIndex); + return getResultRow(pWindowResInfo, pWindowResInfo->curIndex); } // get the correct time window according to the handled timestamp @@ -518,7 +515,7 @@ static STimeWindow getActiveTimeWindow(SWindowResInfo *pWindowResInfo, int64_t t } } else { int32_t slot = curTimeWindowIndex(pWindowResInfo); - SResultRow* pWindowRes = getWindowResult(pWindowResInfo, slot); + SResultRow* pWindowRes = getResultRow(pWindowResInfo, slot); w = pWindowRes->win; } @@ -1091,7 +1088,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis * taosTFree(sasArray); } -static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pData, int16_t type, int16_t bytes) { +static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pData, int16_t type, int16_t bytes, int32_t groupIndex) { if (isNull(pData, type)) { // ignore the null value return -1; } @@ -1113,7 +1110,7 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR); } - uint64_t uid = 0; // uid is always set to be 0. + uint64_t uid = groupIndex; // uid is always set to be 0. SResultRow *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, d, len, true, uid); if (pWindowRes == NULL) { return -1; @@ -1143,7 +1140,7 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat } } - setWindowResOutputBuf(pRuntimeEnv, pWindowRes); + setResultOutputBuf(pRuntimeEnv, pWindowRes); initCtxOutputBuf(pRuntimeEnv); return TSDB_CODE_SUCCESS; } @@ -1374,7 +1371,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS if (groupbyColumnValue) { char *val = groupbyColumnData + bytes * offset; - int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, val, type, bytes); + int32_t ret = setGroupResultOutputBuf(pRuntimeEnv, val, type, bytes, item->groupIndex); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code continue; } @@ -1607,29 +1604,15 @@ static int32_t setCtxTagColumnInfo(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx return TSDB_CODE_SUCCESS; } -static FORCE_INLINE void setResultRowCellInfo(SQueryRuntimeEnv* pRuntimeEnv, SResultRow *pRow, char* buf) { -// SQuery* pQuery = pRuntimeEnv->pQuery; -// -// char* p = buf; -// for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { -// int32_t size = pQuery->pSelectExpr[i].interBytes; -// SResultRowCellInfo* pInfo = getResultCell(pRuntimeEnv, pRow, i); -// setResultInfoBuf(pInfo, p); -// p += size; -// } -} - static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order) { qDebug("QInfo:%p setup runtime env", GET_QINFO_ADDR(pRuntimeEnv)); SQuery *pQuery = pRuntimeEnv->pQuery; - size_t size = pRuntimeEnv->interBufSize + pQuery->numOfOutput * sizeof(SResultRowCellInfo); - - pRuntimeEnv->resultInfo = calloc(1, size); pRuntimeEnv->pCtx = (SQLFunctionCtx *)calloc(pQuery->numOfOutput, sizeof(SQLFunctionCtx)); pRuntimeEnv->rowCellInfoOffset = calloc(pQuery->numOfOutput, sizeof(int32_t)); + pRuntimeEnv->pResultRow = getNewResultRow(pRuntimeEnv->pool);//calloc(1, sizeof(SResultRow)); - if (pRuntimeEnv->resultInfo == NULL || pRuntimeEnv->pCtx == NULL || pRuntimeEnv->rowCellInfoOffset == NULL) { + if (pRuntimeEnv->pResultRow == NULL || pRuntimeEnv->pCtx == NULL || pRuntimeEnv->rowCellInfoOffset == NULL) { goto _clean; } @@ -1666,7 +1649,6 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order pCtx->inputType = pQuery->colList[index].type; } - assert(isValidDataType(pCtx->inputType)); pCtx->ptsOutputBuf = NULL; @@ -1711,13 +1693,9 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order } -// char* buf = (char*) pRuntimeEnv->resultInfo + sizeof(SResultRowCellInfo) * pQuery->numOfOutput; - - // set the intermediate result output buffer -// setResultRowCellInfo(pRuntimeEnv, pRuntimeEnv->resultInfo, NULL); - // if it is group by normal column, do not set output buffer, the output buffer is pResult - if (!pRuntimeEnv->groupbyNormalCol && !pRuntimeEnv->stableQuery) { + // fixed output query/multi-output query for normal table + if (!pRuntimeEnv->groupbyNormalCol && !pRuntimeEnv->stableQuery && !QUERY_IS_INTERVAL_QUERY(pRuntimeEnv->pQuery)) { resetCtxOutputBuf(pRuntimeEnv); } @@ -1729,7 +1707,6 @@ static int32_t setupQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv, int16_t order return TSDB_CODE_SUCCESS; _clean: - taosTFree(pRuntimeEnv->resultInfo); taosTFree(pRuntimeEnv->pCtx); return TSDB_CODE_QRY_OUT_OF_MEMORY; @@ -1758,7 +1735,6 @@ static void teardownQueryRuntimeEnv(SQueryRuntimeEnv *pRuntimeEnv) { taosTFree(pCtx->tagInfo.pTagCtxList); } - taosTFree(pRuntimeEnv->resultInfo); taosTFree(pRuntimeEnv->pCtx); } @@ -2833,14 +2809,14 @@ int32_t tableResultComparFn(const void *pLeft, const void *pRight, void *param) } SWindowResInfo *pWindowResInfo1 = &supporter->pTableQueryInfo[left]->windowResInfo; - SResultRow * pWindowRes1 = getWindowResult(pWindowResInfo1, leftPos); + SResultRow * pWindowRes1 = getResultRow(pWindowResInfo1, leftPos); tFilePage *page1 = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes1->pageId); char *b1 = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes1, page1); TSKEY leftTimestamp = GET_INT64_VAL(b1); SWindowResInfo *pWindowResInfo2 = &supporter->pTableQueryInfo[right]->windowResInfo; - SResultRow * pWindowRes2 = getWindowResult(pWindowResInfo2, rightPos); + SResultRow * pWindowRes2 = getResultRow(pWindowResInfo2, rightPos); tFilePage *page2 = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes2->pageId); char *b2 = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes2, page2); @@ -2961,7 +2937,9 @@ void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) { pQuery->rec.rows += offset; } -int64_t getNumOfResultWindowRes(SQuery *pQuery, SResultRow *pResultRow) { +int64_t getNumOfResultWindowRes(SQueryRuntimeEnv* pRuntimeEnv, SResultRow *pResultRow) { + SQuery* pQuery = pRuntimeEnv->pQuery; + for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { int32_t functionId = pQuery->pSelectExpr[j].base.functionId; @@ -2973,7 +2951,7 @@ int64_t getNumOfResultWindowRes(SQuery *pQuery, SResultRow *pResultRow) { continue; } - SResultRowCellInfo *pResultInfo = &pResultRow->pCellInfo[j]; + SResultRowCellInfo *pResultInfo = getResultCell(pRuntimeEnv, pResultRow, j); assert(pResultInfo != NULL); if (pResultInfo->numOfRes > 0) { @@ -3042,18 +3020,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { SLoserTreeInfo *pTree = NULL; tLoserTreeCreate(&pTree, numOfTables, &cs, tableResultComparFn); - SResultRow* pRow = calloc(1, getWindowResultSize(pRuntimeEnv)); - if (pRow == NULL) { - longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - pRow->pCellInfo = (SResultRowCellInfo*) ((char*) pRow + sizeof(SResultRow)); -// char* buf = (char*) pRow + sizeof(SResultRowCellInfo)*; -// if (buf == NULL) { -// longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY); -// } - - setResultRowCellInfo(pRuntimeEnv, pRow, NULL); + SResultRow* pRow = getNewResultRow(pRuntimeEnv->pool); resetMergeResultBuf(pRuntimeEnv, pRuntimeEnv->pCtx, pRow); pQInfo->groupResInfo.groupId = getGroupResultId(pQInfo->groupIndex); @@ -3069,23 +3036,20 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { taosTFree(pTableList); taosTFree(posList); taosTFree(pTree); -// taosTFree(pResultInfo); -// taosTFree(buf); - longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); } int32_t pos = pTree->pNode[0].index; SWindowResInfo *pWindowResInfo = &pTableList[pos]->windowResInfo; - SResultRow *pWindowRes = getWindowResult(pWindowResInfo, cs.position[pos]); + SResultRow *pWindowRes = getResultRow(pWindowResInfo, cs.position[pos]); tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pageId); char *b = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes, page); TSKEY ts = GET_INT64_VAL(b); assert(ts == pWindowRes->win.skey); - int64_t num = getNumOfResultWindowRes(pQuery, pWindowRes); + int64_t num = getNumOfResultWindowRes(pRuntimeEnv, pWindowRes); if (num <= 0) { cs.position[pos] += 1; @@ -3128,7 +3092,7 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { } } else { // current page is not needed anymore - SResultRow *pNextWindowRes = getWindowResult(pWindowResInfo, cs.position[pos]); + SResultRow *pNextWindowRes = getResultRow(pWindowResInfo, cs.position[pos]); if (pNextWindowRes->pageId != currentPageId) { releaseResBufPage(pRuntimeEnv->pResultBuf, page); } @@ -3145,8 +3109,6 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) { taosTFree(pTree); taosTFree(pTableList); taosTFree(posList); -// taosTFree(pResultInfo); - return -1; } } @@ -3251,8 +3213,8 @@ static void updateTableQueryInfoForReverseScan(SQuery *pQuery, STableQueryInfo * pTableQueryInfo->windowResInfo.curIndex = pTableQueryInfo->windowResInfo.size - 1; } -static void disableFuncInReverseScanImpl(SQInfo* pQInfo, SWindowResInfo *pWindowResInfo, int32_t order) { - SQuery* pQuery = pQInfo->runtimeEnv.pQuery; +static void disableFuncInReverseScanImpl(SQueryRuntimeEnv* pRuntimeEnv, SWindowResInfo *pWindowResInfo, int32_t order) { + SQuery* pQuery = pRuntimeEnv->pQuery; for (int32_t i = 0; i < pWindowResInfo->size; ++i) { bool closed = getTimeWindowResStatus(pWindowResInfo, i); @@ -3260,17 +3222,18 @@ static void disableFuncInReverseScanImpl(SQInfo* pQInfo, SWindowResInfo *pWindow continue; } - SResultRow *buf = getWindowResult(pWindowResInfo, i); + SResultRow *pRow = getResultRow(pWindowResInfo, i); // open/close the specified query for each group result for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { int32_t functId = pQuery->pSelectExpr[j].base.functionId; + SResultRowCellInfo* pInfo = getResultCell(pRuntimeEnv, pRow, j); if (((functId == TSDB_FUNC_FIRST || functId == TSDB_FUNC_FIRST_DST) && order == TSDB_ORDER_ASC) || ((functId == TSDB_FUNC_LAST || functId == TSDB_FUNC_LAST_DST) && order == TSDB_ORDER_DESC)) { - buf->pCellInfo[j].complete = false; + pInfo->complete = false; } else if (functId != TSDB_FUNC_TS && functId != TSDB_FUNC_TAG) { - buf->pCellInfo[j].complete = true; + pInfo->complete = true; } } } @@ -3284,7 +3247,7 @@ void disableFuncInReverseScan(SQInfo *pQInfo) { // group by normal columns and interval query on normal table SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; if (pRuntimeEnv->groupbyNormalCol || QUERY_IS_INTERVAL_QUERY(pQuery)) { - disableFuncInReverseScanImpl(pQInfo, pWindowResInfo, order); + disableFuncInReverseScanImpl(pRuntimeEnv, pWindowResInfo, order); } else { // for simple result of table query, for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { // todo refactor int32_t functId = pQuery->pSelectExpr[j].base.functionId; @@ -3334,22 +3297,16 @@ void switchCtxOrder(SQueryRuntimeEnv *pRuntimeEnv) { } } -int32_t createQueryResultInfo(SQuery *pQuery, SResultRow *pResultRow) { -// int32_t numOfCols = pQuery->numOfOutput; - +int32_t initResultRow(SResultRow *pResultRow) { pResultRow->pCellInfo = (SResultRowCellInfo*)((char*)pResultRow + sizeof(SResultRow)); pResultRow->pageId = -1; pResultRow->rowId = -1; - -// char* buf = (char*) pResultRow->pCellInfo + numOfCols * sizeof(SResultRowCellInfo); - - // set the intermediate result output buffer -// setResultRowCellInfo(pRunimeEnv, pResultRow, buf); return TSDB_CODE_SUCCESS; } void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { SQuery *pQuery = pRuntimeEnv->pQuery; + SResultRow* pRow = pRuntimeEnv->pResultRow; for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; @@ -3359,8 +3316,9 @@ void resetCtxOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) { * set the output buffer information and intermediate buffer * not all queries require the interResultBuf, such as COUNT/TAGPRJ/PRJ/TAG etc. */ - RESET_RESULT_INFO(&pRuntimeEnv->resultInfo[i]); - pCtx->resultInfo = &pRuntimeEnv->resultInfo[i]; + SResultRowCellInfo* pCellInfo = getResultCell(pRuntimeEnv, pRow, i); + RESET_RESULT_INFO(pCellInfo); + pCtx->resultInfo = pCellInfo; // set the timestamp output buffer for top/bottom/diff query int32_t functionId = pQuery->pSelectExpr[i].base.functionId; @@ -3478,12 +3436,12 @@ bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) { SWindowResInfo *pWindowResInfo = &pRuntimeEnv->windowResInfo; for (int32_t i = 0; i < pWindowResInfo->size; ++i) { - SResultRow *pResult = getWindowResult(pWindowResInfo, i); + SResultRow *pResult = getResultRow(pWindowResInfo, i); if (!pResult->closed) { continue; } - setWindowResOutputBuf(pRuntimeEnv, pResult); + setResultOutputBuf(pRuntimeEnv, pResult); for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { int16_t functId = pQuery->pSelectExpr[j].base.functionId; @@ -3709,7 +3667,7 @@ void finalizeQueryResult(SQueryRuntimeEnv *pRuntimeEnv) { continue; } - setWindowResOutputBuf(pRuntimeEnv, buf); + setResultOutputBuf(pRuntimeEnv, buf); for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { aAggs[pQuery->pSelectExpr[j].base.functionId].xFinalize(&pRuntimeEnv->pCtx[j]); @@ -3816,11 +3774,11 @@ void setExecutionContext(SQInfo *pQInfo, int32_t groupIndex, TSKEY nextKey) { // record the current active group id pRuntimeEnv->prevGroupId = groupIndex; - setWindowResOutputBuf(pRuntimeEnv, pWindowRes); + setResultOutputBuf(pRuntimeEnv, pWindowRes); initCtxOutputBuf(pRuntimeEnv); } -void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult) { +void setResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult) { SQuery *pQuery = pRuntimeEnv->pQuery; // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group @@ -3836,10 +3794,10 @@ void setWindowResOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult) { } /* - * set the output buffer information and intermediate buffer + * set the output buffer information and intermediate buffer, * not all queries require the interResultBuf, such as COUNT */ - pCtx->resultInfo = &pResult->pCellInfo[i]; + pCtx->resultInfo = getResultCell(pRuntimeEnv, pResult, i); } } @@ -3852,7 +3810,7 @@ void setWindowResOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pRe for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { SQLFunctionCtx *pCtx = &pRuntimeEnv->pCtx[i]; - pCtx->resultInfo = &pResult->pCellInfo[i]; + pCtx->resultInfo = getResultCell(pRuntimeEnv, pResult, i); if (pCtx->resultInfo->initialized && pCtx->resultInfo->complete) { continue; } @@ -4100,7 +4058,8 @@ static void updateWindowResNumOfRes(SQueryRuntimeEnv *pRuntimeEnv) { continue; } - pResult->numOfRows = (uint16_t)(MAX(pResult->numOfRows, pResult->pCellInfo[j].numOfRes)); + SResultRowCellInfo* pCell = getResultCell(pRuntimeEnv, pResult, j); + pResult->numOfRows = (uint16_t)(MAX(pResult->numOfRows, pCell->numOfRes)); } } } @@ -4891,7 +4850,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { size_t numOfGroups = GET_NUM_OF_TABLEGROUP(pQInfo); - if (isPointInterpoQuery(pQuery) || isFirstLastRowQuery(pQuery)) { + if (isPointInterpoQuery(pQuery)) { resetCtxOutputBuf(pRuntimeEnv); assert(pQuery->limit.offset == 0 && pQuery->limit.limit != 0); @@ -4921,11 +4880,7 @@ static void sequentialTableProcess(SQInfo *pQInfo) { pRuntimeEnv->pQueryHandle = NULL; } - if (isFirstLastRowQuery(pQuery)) { - assert(0); // last_row query switch to other routine to handle - } else { - pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(pQInfo->tsdb, &cond, &gp, pQInfo); - } + pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(pQInfo->tsdb, &cond, &gp, pQInfo); taosArrayDestroy(tx); taosArrayDestroy(g1); @@ -4939,10 +4894,6 @@ static void sequentialTableProcess(SQInfo *pQInfo) { assert(taosArrayGetSize(s) >= 1); setTagVal(pRuntimeEnv, taosArrayGetP(s, 0), pQInfo->tsdb); - if (isFirstLastRowQuery(pQuery)) { - assert(taosArrayGetSize(s) == 1); - } - taosArrayDestroy(s); // here we simply set the first table as current table @@ -5025,7 +4976,8 @@ static void sequentialTableProcess(SQInfo *pQInfo) { SResultRow *pResult = pWindowResInfo->pResult[i]; for (int32_t j = 0; j < pQuery->numOfOutput; ++j) { - pResult->numOfRows = (uint16_t)(MAX(pResult->numOfRows, pResult->pCellInfo[j].numOfRes)); + SResultRowCellInfo* pCell = getResultCell(pRuntimeEnv, pResult, j); + pResult->numOfRows = (uint16_t)(MAX(pResult->numOfRows, pCell->numOfRes)); } } @@ -5296,7 +5248,6 @@ static void multiTableQueryProcess(SQInfo *pQInfo) { if (pQInfo->code != TSDB_CODE_SUCCESS || IS_QUERY_KILLED(pQInfo)) { qDebug("QInfo:%p query killed or error occurred, code:%s, abort", pQInfo, tstrerror(pQInfo->code)); //TODO finalizeQueryResult may cause SEGSEV, since the memory may not allocated yet, add a cleanup function instead -// finalizeQueryResult(pRuntimeEnv); // clean up allocated resource during query longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); } @@ -6287,8 +6238,6 @@ static SQInfo *createQInfoImpl(SQueryTableMsg *pQueryMsg, SSqlGroupbyExpr *pGrou calResultBufSize(pQuery); for (int32_t col = 0; col < pQuery->numOfOutput; ++col) { - assert(pExprs[col].interBytes >= pExprs[col].bytes || pExprs[col].interBytes == 0); - // allocate additional memory for interResults that are usually larger then final results size_t size = (size_t)((pQuery->rec.capacity + 1) * pExprs[col].bytes + pExprs[col].interBytes + sizeof(tFilePage)); pQuery->sdata[col] = (tFilePage *)calloc(1, size); diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index 600f48928a..824cae3932 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -87,7 +87,7 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { assert(num >= 0 && num <= numOfClosed); int16_t type = pWindowResInfo->type; - uint64_t uid = 0; // uid is always set to be 0. + STableId* id = TSDB_TABLEID(pRuntimeEnv->pQuery->current->pTable); // uid is always set to be 0. char *key = NULL; int16_t bytes = -1; @@ -104,7 +104,7 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { bytes = tDataTypeDesc[pWindowResInfo->type].nSize; } - SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, uid); + SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, id->uid); taosHashRemove(pRuntimeEnv->pResultRowHashTable, (const char *)pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); } else { break; @@ -137,14 +137,14 @@ void clearFirstNTimeWindow(SQueryRuntimeEnv *pRuntimeEnv, int32_t num) { bytes = tDataTypeDesc[pWindowResInfo->type].nSize; } - SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, uid); + SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, id->uid); int32_t *p = (int32_t *)taosHashGet(pRuntimeEnv->pResultRowHashTable, (const char *)pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); assert(p != NULL); int32_t v = (*p - num); assert(v >= 0 && v <= pWindowResInfo->size); - SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, uid); + SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, key, bytes, id->uid); taosHashPut(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), (char *)&v, sizeof(int32_t)); } @@ -217,11 +217,11 @@ void removeRedundantWindow(SWindowResInfo *pWindowResInfo, TSKEY lastKey, int32_ } bool isWindowResClosed(SWindowResInfo *pWindowResInfo, int32_t slot) { - return (getWindowResult(pWindowResInfo, slot)->closed == true); + return (getResultRow(pWindowResInfo, slot)->closed == true); } void closeTimeWindow(SWindowResInfo *pWindowResInfo, int32_t slot) { - getWindowResult(pWindowResInfo, slot)->closed = true; + getResultRow(pWindowResInfo, slot)->closed = true; } void clearResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pWindowRes) { @@ -229,18 +229,21 @@ void clearResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pWindowRes) { return; } - tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pageId); + // the result does not put into the SDiskbasedResultBuf, ignore it. + if (pWindowRes->pageId >= 0) { + tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pageId); - for (int32_t i = 0; i < pRuntimeEnv->pQuery->numOfOutput; ++i) { - SResultRowCellInfo *pResultInfo = &pWindowRes->pCellInfo[i]; - - char * s = getPosInResultPage(pRuntimeEnv, i, pWindowRes, page); - size_t size = pRuntimeEnv->pQuery->pSelectExpr[i].bytes; - memset(s, 0, size); - - RESET_RESULT_INFO(pResultInfo); + for (int32_t i = 0; i < pRuntimeEnv->pQuery->numOfOutput; ++i) { + SResultRowCellInfo *pResultInfo = &pWindowRes->pCellInfo[i]; + + char * s = getPosInResultPage(pRuntimeEnv, i, pWindowRes, page); + size_t size = pRuntimeEnv->pQuery->pSelectExpr[i].bytes; + memset(s, 0, size); + + RESET_RESULT_INFO(pResultInfo); + } } - + pWindowRes->numOfRows = 0; pWindowRes->pageId = -1; pWindowRes->rowId = -1; @@ -327,6 +330,8 @@ SResultRow* getNewResultRow(SResultRowPool* p) { } p->position.pos = (p->position.pos + 1)%p->numOfElemPerBlock; + initResultRow(ptr); + return ptr; } diff --git a/src/util/src/hash.c b/src/util/src/hash.c index 10767ee30f..22b5da1491 100644 --- a/src/util/src/hash.c +++ b/src/util/src/hash.c @@ -114,10 +114,12 @@ static SHashNode *doCreateHashNode(const void *key, size_t keyLen, const void *p * @param dsize size of actual data * @return hash node */ -static FORCE_INLINE SHashNode *doUpdateHashNode(SHashNode* prev, SHashNode *pNode, SHashNode *pNewNode) { +static FORCE_INLINE SHashNode *doUpdateHashNode(SHashEntry* pe, SHashNode* prev, SHashNode *pNode, SHashNode *pNewNode) { assert(pNode->keyLen == pNewNode->keyLen); if (prev != NULL) { prev->next = pNewNode; + } else { + pe->next = pNewNode; } pNewNode->next = pNode->next; @@ -242,7 +244,10 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da } else { // not support the update operation, return error if (pHashObj->enableUpdate) { - doUpdateHashNode(prev, pNode, pNewNode); + doUpdateHashNode(pe, prev, pNode, pNewNode); + DO_FREE_HASH_NODE(pNode); + } else { + DO_FREE_HASH_NODE(pNewNode); } if (pHashObj->type == HASH_ENTRY_LOCK) { @@ -252,7 +257,6 @@ int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *da // enable resize __rd_unlock(&pHashObj->lock, pHashObj->type); - DO_FREE_HASH_NODE(pNewNode); return pHashObj->enableUpdate ? 0 : -1; } } From e61456b77b963e656655be27fd076808604f9d26 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 2 Nov 2020 23:45:04 +0800 Subject: [PATCH 17/35] [TD-225] fix typo in test script. --- tests/script/general/parser/fill.sim | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/script/general/parser/fill.sim b/tests/script/general/parser/fill.sim index f89c27d71f..92aa6a922c 100644 --- a/tests/script/general/parser/fill.sim +++ b/tests/script/general/parser/fill.sim @@ -116,7 +116,7 @@ if $data81 != 1 then endi # avg_with_fill -print avg_witt_constant_fill +print avg_with_constant_fill sql select avg(c1), avg(c2), avg(c3), avg(c4), avg(c5) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 6, 6, 6, 6, 6) if $rows != 9 then return -1 @@ -371,12 +371,10 @@ if $data11 != 99 then endi sql select * from $tb -#print data08 = $data08 if $data08 != NCHAR then + print expect NCHAR, actual:$data08 return -1 endi -#return -1 - # fill_into_nonarithmetic_fieds sql select first(c6), first(c7), first(c8) from $tb where ts >= $ts0 and ts <= $tsu interval(5m) fill(value, 20000000, 20000000, 20000000) From f3d5df2a91d90e7d8fdbb8979dc3028d29a3a488 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 2 Nov 2020 23:46:02 +0800 Subject: [PATCH 18/35] [TD-1874]: fix bugs in join query. --- src/client/src/tscSQLParser.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 8716b5cb02..87efa26c8c 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -2728,6 +2728,7 @@ static bool functionCompatibleCheck(SQueryInfo* pQueryInfo, bool joinQuery) { int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* pCmd) { const char* msg1 = "too many columns in group by clause"; const char* msg2 = "invalid column name in group by clause"; + const char* msg3 = "columns from one table allowed as group by columns"; const char* msg7 = "not support group by expression"; const char* msg8 = "not allowed column type for group by"; const char* msg9 = "tags not allowed for table query"; @@ -2763,7 +2764,11 @@ int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } - tableIndex = index.tableIndex; + if (tableIndex == COLUMN_INDEX_INITIAL_VAL) { + tableIndex = index.tableIndex; + } else if (tableIndex != index.tableIndex) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg3); + } pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); pTableMeta = pTableMetaInfo->pTableMeta; From d13a4e79d99b56fba9b74991939aa3547befa34c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 3 Nov 2020 10:04:08 +0800 Subject: [PATCH 19/35] [TD-1875]: fix limit/offset error in super table join query. --- src/client/src/tscSubquery.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index d58b44b92f..1299357d84 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -1061,7 +1061,11 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR pRes1->numOfRows, pRes1->numOfTotal); assert(pRes1->row < pRes1->numOfRows); } else { - pRes1->numOfClauseTotal += pRes1->numOfRows; + SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pParentSql->pSubs[i]->cmd, 0); + if (!tscIsTwoStageSTableQuery(pQueryInfo, 0)) { + pRes1->numOfClauseTotal += pRes1->numOfRows; + } + tscDebug("%p sub:%p index:%d numOfRows:%"PRId64" total:%"PRId64, pParentSql, pParentSql->pSubs[i], i, pRes1->numOfRows, pRes1->numOfTotal); } From 5e0fba93a134ddb20eeb8597a85c3a5ada6825df Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 3 Nov 2020 10:06:44 +0800 Subject: [PATCH 20/35] [TD-225] refactor. --- src/client/src/tscSubquery.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 1299357d84..786e237a53 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -1048,6 +1048,7 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR } // update the records for each subquery in parent sql object. + bool isSTableSub = tscIsTwoStageSTableQuery(pQueryInfo, 0); for (int32_t i = 0; i < pState->numOfSub; ++i) { if (pParentSql->pSubs[i] == NULL) { tscDebug("%p %p sub:%d not retrieve data", pParentSql, NULL, i); @@ -1061,8 +1062,7 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR pRes1->numOfRows, pRes1->numOfTotal); assert(pRes1->row < pRes1->numOfRows); } else { - SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pParentSql->pSubs[i]->cmd, 0); - if (!tscIsTwoStageSTableQuery(pQueryInfo, 0)) { + if (!isSTableSub) { pRes1->numOfClauseTotal += pRes1->numOfRows; } From 7bd533bfef3c6be680064cec0499ffc874d25a6e Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 3 Nov 2020 10:50:06 +0800 Subject: [PATCH 21/35] [TD-1876]: fix bugs in arithmetic expression during two super table join query. --- src/client/src/tscSQLParser.c | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 87efa26c8c..55658ac446 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -1211,7 +1211,7 @@ static void tscInsertPrimaryTSSourceColumn(SQueryInfo* pQueryInfo, SColumnIndex* tscColumnListInsert(pQueryInfo->colList, &tsCol); } static int32_t handleArithmeticExpr(SSqlCmd* pCmd, int32_t clauseIndex, int32_t exprIndex, tSQLExprItem* pItem) { - const char* msg1 = "invalid column name, or illegal column type"; + const char* msg1 = "invalid column name, illegal column type, or columns in arithmetic expression from two tables"; const char* msg2 = "invalid arithmetic expression in select clause"; const char* msg3 = "tag columns can not be used in arithmetic expression"; const char* msg4 = "columns from different table mixed up in arithmetic expression"; @@ -3353,7 +3353,8 @@ int32_t doArithmeticExprToString(tSQLExpr* pExpr, char** exprString) { return TSDB_CODE_SUCCESS; } -static int32_t validateSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQueryInfo, SColumnList* pList, int32_t* type) { +static int32_t validateSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQueryInfo, SColumnList* pList, + int32_t* type, uint64_t* uid) { if (pExpr->nSQLOptr == TK_ID) { if (*type == NON_ARITHMEIC_EXPR) { *type = NORMAL_ARITHMETIC; @@ -3402,13 +3403,22 @@ static int32_t validateSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryInfo* pQuer } // Not supported data type in arithmetic expression + uint64_t id = -1; for(int32_t i = 0; i < inc; ++i) { SSqlExpr* p1 = tscSqlExprGet(pQueryInfo, i + outputIndex); int16_t t = p1->resType; if (t == TSDB_DATA_TYPE_BINARY || t == TSDB_DATA_TYPE_NCHAR || t == TSDB_DATA_TYPE_BOOL || t == TSDB_DATA_TYPE_TIMESTAMP) { return TSDB_CODE_TSC_INVALID_SQL; } + + if (i == 0) { + id = p1->uid; + } else if (id != p1->uid){ + return TSDB_CODE_TSC_INVALID_SQL; + } } + + *uid = id; } return TSDB_CODE_SUCCESS; @@ -3420,13 +3430,16 @@ static int32_t validateArithmeticSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryI } tSQLExpr* pLeft = pExpr->pLeft; + uint64_t uidLeft = 0; + uint64_t uidRight = 0; + if (pLeft->nSQLOptr >= TK_PLUS && pLeft->nSQLOptr <= TK_REM) { int32_t ret = validateArithmeticSQLExpr(pCmd, pLeft, pQueryInfo, pList, type); if (ret != TSDB_CODE_SUCCESS) { return ret; } } else { - int32_t ret = validateSQLExpr(pCmd, pLeft, pQueryInfo, pList, type); + int32_t ret = validateSQLExpr(pCmd, pLeft, pQueryInfo, pList, type, &uidLeft); if (ret != TSDB_CODE_SUCCESS) { return ret; } @@ -3439,10 +3452,15 @@ static int32_t validateArithmeticSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryI return ret; } } else { - int32_t ret = validateSQLExpr(pCmd, pRight, pQueryInfo, pList, type); + int32_t ret = validateSQLExpr(pCmd, pRight, pQueryInfo, pList, type, &uidRight); if (ret != TSDB_CODE_SUCCESS) { return ret; } + + // the expression not from the same table, return error + if (uidLeft != uidRight) { + return TSDB_CODE_TSC_INVALID_SQL; + } } return TSDB_CODE_SUCCESS; From 5be04aa8d4165c1b2f139c69db32ab8c552a07e8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 3 Nov 2020 11:02:11 +0800 Subject: [PATCH 22/35] [TD-225]refactors. --- src/query/src/qExecutor.c | 43 +++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index c0390b51e4..8b12af7fa9 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -447,7 +447,7 @@ static bool hasNullValue(SColIndex* pColIndex, SDataStatis *pStatis, SDataStatis return true; } -static SResultRow *doSetTimeWindowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo, char *pData, +static SResultRow *doPrepareResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowResInfo *pWindowResInfo, char *pData, int16_t bytes, bool masterscan, uint64_t uid) { SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, uid); int32_t *p1 = @@ -601,8 +601,8 @@ static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowRes assert(win->skey <= win->ekey); SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; - SResultRow *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&win->skey, TSDB_KEYSIZE, masterscan, pBockInfo->uid); - if (pWindowRes == NULL) { + SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&win->skey, TSDB_KEYSIZE, masterscan, pBockInfo->uid); + if (pResultRow == NULL) { *newWind = false; return masterscan? -1:0; @@ -611,17 +611,16 @@ static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SWindowRes *newWind = true; // not assign result buffer yet, add new result buffer - if (pWindowRes->pageId == -1) { - int32_t ret = addNewWindowResultBuf(pWindowRes, pResultBuf, pBockInfo->tid, pRuntimeEnv->numOfRowsPerPage); + if (pResultRow->pageId == -1) { + int32_t ret = addNewWindowResultBuf(pResultRow, pResultBuf, pBockInfo->tid, pRuntimeEnv->numOfRowsPerPage); if (ret != TSDB_CODE_SUCCESS) { return -1; } } // set time window for current result - pWindowRes->win = (*win); - - setWindowResOutputBufInitCtx(pRuntimeEnv, pWindowRes); + pResultRow->win = (*win); + setWindowResOutputBufInitCtx(pRuntimeEnv, pResultRow); return TSDB_CODE_SUCCESS; } @@ -1111,8 +1110,8 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat } uint64_t uid = groupIndex; // uid is always set to be 0. - SResultRow *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, d, len, true, uid); - if (pWindowRes == NULL) { + SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, d, len, true, uid); + if (pResultRow == NULL) { return -1; } @@ -1126,21 +1125,21 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat } if (type == TSDB_DATA_TYPE_BINARY || type == TSDB_DATA_TYPE_NCHAR) { - pWindowRes->key = malloc(varDataTLen(pData)); - varDataCopy(pWindowRes->key, pData); + pResultRow->key = malloc(varDataTLen(pData)); + varDataCopy(pResultRow->key, pData); } else { - pWindowRes->win.skey = v; - pWindowRes->win.ekey = v; + pResultRow->win.skey = v; + pResultRow->win.ekey = v; } - if (pWindowRes->pageId == -1) { - int32_t ret = addNewWindowResultBuf(pWindowRes, pResultBuf, GROUPRESULTID, pRuntimeEnv->numOfRowsPerPage); + if (pResultRow->pageId == -1) { + int32_t ret = addNewWindowResultBuf(pResultRow, pResultBuf, GROUPRESULTID, pRuntimeEnv->numOfRowsPerPage); if (ret != 0) { return -1; } } - setResultOutputBuf(pRuntimeEnv, pWindowRes); + setResultOutputBuf(pRuntimeEnv, pResultRow); initCtxOutputBuf(pRuntimeEnv); return TSDB_CODE_SUCCESS; } @@ -3755,9 +3754,9 @@ void setExecutionContext(SQInfo *pQInfo, int32_t groupIndex, TSKEY nextKey) { } uint64_t uid = 0; // uid is always set to be 0 - SResultRow *pWindowRes = doSetTimeWindowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&groupIndex, + SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&groupIndex, sizeof(groupIndex), true, uid); - if (pWindowRes == NULL) { + if (pResultRow == NULL) { return; } @@ -3765,8 +3764,8 @@ void setExecutionContext(SQInfo *pQInfo, int32_t groupIndex, TSKEY nextKey) { * not assign result buffer yet, add new result buffer * all group belong to one result set, and each group result has different group id so set the id to be one */ - if (pWindowRes->pageId == -1) { - if (addNewWindowResultBuf(pWindowRes, pRuntimeEnv->pResultBuf, groupIndex, pRuntimeEnv->numOfRowsPerPage) != + if (pResultRow->pageId == -1) { + if (addNewWindowResultBuf(pResultRow, pRuntimeEnv->pResultBuf, groupIndex, pRuntimeEnv->numOfRowsPerPage) != TSDB_CODE_SUCCESS) { return; } @@ -3774,7 +3773,7 @@ void setExecutionContext(SQInfo *pQInfo, int32_t groupIndex, TSKEY nextKey) { // record the current active group id pRuntimeEnv->prevGroupId = groupIndex; - setResultOutputBuf(pRuntimeEnv, pWindowRes); + setResultOutputBuf(pRuntimeEnv, pResultRow); initCtxOutputBuf(pRuntimeEnv); } From f7d0f89e5ed7ad8ab9757c5bd7376e7837743ae8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 3 Nov 2020 11:02:34 +0800 Subject: [PATCH 23/35] [TD-225] add test cases. --- tests/script/general/parser/join_multivnode.sim | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/script/general/parser/join_multivnode.sim b/tests/script/general/parser/join_multivnode.sim index 102d5ba93f..85d5c79d86 100644 --- a/tests/script/general/parser/join_multivnode.sim +++ b/tests/script/general/parser/join_multivnode.sim @@ -315,4 +315,7 @@ if $data03 != 0 then return -1 endi +select count(join_mt0.c1), first(join_mt0.c1)-last(join_mt1.c1), first(join_mt1.c9) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts;", NULL); +select count(join_mt0.c1), first(join_mt0.c1)-last(join_mt0.c1), first(join_mt1.c9) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts;", NULL); + system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file From 0b83e6746ed2d1927585b19bf69f1c176e27d2a2 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 3 Nov 2020 18:02:31 +0800 Subject: [PATCH 24/35] [TD-1876]: fix bugs in arithmetic expression in super table join query processing. --- src/client/src/tscSQLParser.c | 32 +++----- src/client/src/tscServer.c | 28 ++----- src/client/src/tscSubquery.c | 1 - src/query/inc/qTsbuf.h | 2 + src/query/src/qExecutor.c | 3 +- src/query/src/qTsbuf.c | 81 ++++++++++++++++--- src/query/tests/tsBufTest.cpp | 11 ++- .../script/general/parser/join_multivnode.sim | 2 + 8 files changed, 100 insertions(+), 60 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 55658ac446..27246aa7d3 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -65,7 +65,6 @@ static bool validateTagParams(tFieldList* pTagsList, tFieldList* pFieldList, SSq static int32_t setObjFullName(char* fullName, const char* account, SStrToken* pDB, SStrToken* tableName, int32_t* len); static void getColumnName(tSQLExprItem* pItem, char* resultFieldName, int32_t nameLength); -static void getRevisedName(char* resultFieldName, int32_t functionId, int32_t maxLen, char* columnName); static int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExprItem* pItem, bool finalResult); static int32_t insertResultField(SQueryInfo* pQueryInfo, int32_t outputIndex, SColumnList* pIdList, int16_t bytes, @@ -1625,12 +1624,11 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t } static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSchema* pSchema, SConvertFunc cvtFunc, - char* aliasName, int32_t resColIdx, SColumnIndex* pColIndex, bool finalResult) { + tSQLExprItem* item, int32_t resColIdx, SColumnIndex* pColIndex, bool finalResult) { const char* msg1 = "not support column types"; int16_t type = 0; int16_t bytes = 0; - char columnName[TSDB_COL_NAME_LEN] = {0}; int32_t functionID = cvtFunc.execFuncId; if (functionID == TSDB_FUNC_SPREAD) { @@ -1647,14 +1645,13 @@ static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS bytes = pSchema[pColIndex->columnIndex].bytes; } - if (aliasName != NULL) { - tstrncpy(columnName, aliasName, sizeof(columnName)); - } else { - getRevisedName(columnName, cvtFunc.originFuncId, sizeof(columnName) - 1, pSchema[pColIndex->columnIndex].name); - } - SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, functionID, pColIndex, type, bytes, bytes, false); - tstrncpy(pExpr->aliasName, columnName, sizeof(pExpr->aliasName)); + if (item->aliasName != NULL) { + tstrncpy(pExpr->aliasName, item->aliasName, tListLen(pExpr->aliasName)); + } else { + int32_t len = MIN(tListLen(pExpr->aliasName), item->pNode->token.n + 1); + tstrncpy(pExpr->aliasName, item->pNode->token.z, len); + } if (cvtFunc.originFuncId == TSDB_FUNC_LAST_ROW && cvtFunc.originFuncId != functionID) { pExpr->colInfo.flag |= TSDB_COL_NULL; @@ -1674,7 +1671,7 @@ static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS // if it is not in the final result, do not add it SColumnList ids = getColumnList(1, pColIndex->tableIndex, pColIndex->columnIndex); if (finalResult) { - insertResultField(pQueryInfo, resColIdx, &ids, bytes, (int8_t)type, columnName, pExpr); + insertResultField(pQueryInfo, resColIdx, &ids, bytes, (int8_t)type, pExpr->aliasName, pExpr); } else { tscColumnListInsert(pQueryInfo->colList, &(ids.ids[0])); } @@ -1939,8 +1936,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col SColumnIndex index = COLUMN_INDEX_INITIALIZER; - if (pParamElem->pNode->nSQLOptr == TK_ALL) { - // select table.* + if (pParamElem->pNode->nSQLOptr == TK_ALL) { // select table.* SStrToken tmpToken = pParamElem->pNode->colInfo; if (getTableIndexByName(&tmpToken, pQueryInfo, &index) != TSDB_CODE_SUCCESS) { @@ -1952,7 +1948,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col for (int32_t j = 0; j < tscGetNumOfColumns(pTableMetaInfo->pTableMeta); ++j) { index.columnIndex = j; - if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, pItem->aliasName, colIndex++, &index, finalResult) != 0) { + if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, pItem, colIndex++, &index, finalResult) != 0) { return TSDB_CODE_TSC_INVALID_SQL; } } @@ -1970,7 +1966,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); } - if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, pItem->aliasName, colIndex + i, &index, finalResult) != 0) { + if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, pItem, colIndex + i, &index, finalResult) != 0) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -2007,7 +2003,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col for (int32_t i = 0; i < tscGetNumOfColumns(pTableMetaInfo->pTableMeta); ++i) { SColumnIndex index = {.tableIndex = j, .columnIndex = i}; - if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, pItem->aliasName, colIndex, &index, finalResult) != 0) { + if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, pItem, colIndex, &index, finalResult) != 0) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -2240,10 +2236,6 @@ void getColumnName(tSQLExprItem* pItem, char* resultFieldName, int32_t nameLengt } } -void getRevisedName(char* resultFieldName, int32_t functionId, int32_t maxLen, char* columnName) { - snprintf(resultFieldName, maxLen, "%s(%s)", aAggs[functionId].aName, columnName); -} - static bool isTablenameToken(SStrToken* token) { SStrToken tmpToken = *token; SStrToken tableToken = {0}; diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index e678464932..779cee2163 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -878,37 +878,19 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { // compressed ts block pQueryMsg->tsOffset = htonl((int32_t)(pMsg - pCmd->payload)); - int32_t tsLen = 0; - int32_t numOfBlocks = 0; if (pQueryInfo->tsBuf != NULL) { int32_t vnodeId = htonl(pQueryMsg->head.vgId); - STSVnodeBlockInfo *pBlockInfo = tsBufGetVnodeBlockInfo(pQueryInfo->tsBuf, vnodeId); - assert(QUERY_IS_JOIN_QUERY(pQueryInfo->type) && pBlockInfo != NULL); // this query should not be sent - - // todo refactor, extract method and put into tsBuf.c - if (fseek(pQueryInfo->tsBuf->f, pBlockInfo->offset, SEEK_SET) != 0) { - int code = TAOS_SYSTEM_ERROR(ferror(pQueryInfo->tsBuf->f)); - tscError("%p: fseek failed: %s", pSql, tstrerror(code)); + int32_t code = dumpFileBlockByVnodeId(pQueryInfo->tsBuf, vnodeId, pMsg, &pQueryMsg->tsLen, &pQueryMsg->tsNumOfBlocks); + if (code != TSDB_CODE_SUCCESS) { return code; } - size_t s = fread(pMsg, 1, pBlockInfo->compLen, pQueryInfo->tsBuf->f); - if (s != pBlockInfo->compLen) { - int code = TAOS_SYSTEM_ERROR(ferror(pQueryInfo->tsBuf->f)); - tscError("%p: fread didn't return expected data: %s", pSql, tstrerror(code)); - return code; - } + pMsg += pQueryMsg->tsLen; - pMsg += pBlockInfo->compLen; - tsLen = pBlockInfo->compLen; - numOfBlocks = pBlockInfo->numOfBlocks; - } - - pQueryMsg->tsLen = htonl(tsLen); - pQueryMsg->tsNumOfBlocks = htonl(numOfBlocks); - if (pQueryInfo->tsBuf != NULL) { pQueryMsg->tsOrder = htonl(pQueryInfo->tsBuf->tsOrder); + pQueryMsg->tsLen = htonl(pQueryMsg->tsLen); + pQueryMsg->tsNumOfBlocks = htonl(pQueryMsg->tsNumOfBlocks); } int32_t msgLen = (int32_t)(pMsg - pCmd->payload); diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 786e237a53..38b0388d35 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -189,7 +189,6 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ STSElem el1 = tsBufGetElem(pSupporter1->pTSBuf); if (el1.vnode < 0) { // no data exists, abort - completed = true; break; } } diff --git a/src/query/inc/qTsbuf.h b/src/query/inc/qTsbuf.h index 6c2a955f47..fc5efa069d 100644 --- a/src/query/inc/qTsbuf.h +++ b/src/query/inc/qTsbuf.h @@ -135,6 +135,8 @@ int32_t tsBufGetNumOfVnodes(STSBuf* pTSBuf); void tsBufGetVnodeIdList(STSBuf* pTSBuf, int32_t* num, int32_t** vnodeId); +int32_t dumpFileBlockByVnodeId(STSBuf* pTSBuf, int32_t vnodeId, void* buf, int32_t* len, int32_t* numOfBlocks); + #ifdef __cplusplus } #endif diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 8b12af7fa9..d65d01d155 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -6373,12 +6373,13 @@ static int32_t initQInfo(SQueryTableMsg *pQueryMsg, void *tsdb, int32_t vgId, SQ SQuery *pQuery = pQInfo->runtimeEnv.pQuery; STSBuf *pTSBuf = NULL; - if (pQueryMsg->tsLen > 0) { // open new file to save the result + if (pQueryMsg->tsLen > 0) { // open new file to save the result char *tsBlock = (char *) pQueryMsg + pQueryMsg->tsOffset; pTSBuf = tsBufCreateFromCompBlocks(tsBlock, pQueryMsg->tsNumOfBlocks, pQueryMsg->tsLen, pQueryMsg->tsOrder, vgId); tsBufResetPos(pTSBuf); bool ret = tsBufNextPos(pTSBuf); + UNUSED(ret); } diff --git a/src/query/src/qTsbuf.c b/src/query/src/qTsbuf.c index ed7fa16cc4..82f36a8951 100644 --- a/src/query/src/qTsbuf.c +++ b/src/query/src/qTsbuf.c @@ -219,6 +219,15 @@ static void shrinkBuffer(STSList* ptsData) { } } +static int32_t getTagAreaLength(tVariant* pa) { + int32_t t = sizeof(pa->nLen) * 2 + sizeof(pa->nType); + if (pa->nType != TSDB_DATA_TYPE_NULL) { + t += pa->nLen; + } + + return t; +} + static void writeDataToDisk(STSBuf* pTSBuf) { if (pTSBuf->tsData.len == 0) { return; @@ -244,19 +253,27 @@ static void writeDataToDisk(STSBuf* pTSBuf) { */ int32_t metaLen = 0; metaLen += (int32_t)fwrite(&pBlock->tag.nType, 1, sizeof(pBlock->tag.nType), pTSBuf->f); - metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); + int32_t trueLen = pBlock->tag.nLen; if (pBlock->tag.nType == TSDB_DATA_TYPE_BINARY || pBlock->tag.nType == TSDB_DATA_TYPE_NCHAR) { + metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); metaLen += (int32_t)fwrite(pBlock->tag.pz, 1, (size_t)pBlock->tag.nLen, pTSBuf->f); } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { - metaLen += (int32_t)fwrite(&pBlock->tag.i64Key, 1, sizeof(int64_t), pTSBuf->f); + metaLen += (int32_t)fwrite(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); + metaLen += (int32_t)fwrite(&pBlock->tag.i64Key, 1, (size_t) pBlock->tag.nLen, pTSBuf->f); + } else { + trueLen = 0; + metaLen += (int32_t)fwrite(&trueLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); } fwrite(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f); fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); fwrite(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f); fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); - + + metaLen += fwrite(&trueLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); + assert(metaLen == getTagAreaLength(&pBlock->tag)); + int32_t blockSize = metaLen + sizeof(pBlock->numOfElem) + sizeof(pBlock->compLen) * 2 + pBlock->compLen; pTSBuf->fileSize += blockSize; @@ -291,17 +308,22 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { pBlock->padding = 0; pBlock->numOfElem = 0; + int32_t offset = -1; + if (order == TSDB_ORDER_DESC) { /* * set the right position for the reversed traverse, the reversed traverse is started from * the end of each comp data block */ - int32_t ret = fseek(pTSBuf->f, -(int32_t)(sizeof(pBlock->padding)), SEEK_CUR); - size_t sz = fread(&pBlock->padding, sizeof(pBlock->padding), 1, pTSBuf->f); + int32_t prev = -(int32_t) (sizeof(pBlock->padding) + sizeof(pBlock->tag.nLen)); + int32_t ret = fseek(pTSBuf->f, prev, SEEK_CUR); + size_t sz = fread(&pBlock->padding, 1, sizeof(pBlock->padding), pTSBuf->f); + sz = fread(&pBlock->tag.nLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); UNUSED(sz); - + pBlock->compLen = pBlock->padding; - int32_t offset = pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + sizeof(pBlock->tag); + + offset = pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + getTagAreaLength(&pBlock->tag); ret = fseek(pTSBuf->f, -offset, SEEK_CUR); UNUSED(ret); } @@ -320,7 +342,7 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { sz = fread(pBlock->tag.pz, (size_t)pBlock->tag.nLen, 1, pTSBuf->f); } else if (pBlock->tag.nType != TSDB_DATA_TYPE_NULL) { - sz = fread(&pBlock->tag.i64Key, sizeof(int64_t), 1, pTSBuf->f); + sz = fread(&pBlock->tag.i64Key, (size_t) pBlock->tag.nLen, 1, pTSBuf->f); } sz = fread(&pBlock->numOfElem, sizeof(pBlock->numOfElem), 1, pTSBuf->f); @@ -328,8 +350,7 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { sz = fread(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); UNUSED(sz); sz = fread(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f); - UNUSED(sz); - + if (decomp) { pTSBuf->tsData.len = tsDecompressTimestamp(pBlock->payload, pBlock->compLen, pBlock->numOfElem, pTSBuf->tsData.rawBuf, @@ -338,11 +359,20 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { // read the comp length at the length of comp block sz = fread(&pBlock->padding, sizeof(pBlock->padding), 1, pTSBuf->f); + assert(pBlock->padding == pBlock->compLen); + + int32_t n = 0; + sz = fread(&n, sizeof(pBlock->tag.nLen), 1, pTSBuf->f); + if (pBlock->tag.nType == TSDB_DATA_TYPE_NULL) { + assert(n == 0); + } else { + assert(n == pBlock->tag.nLen); + } + UNUSED(sz); // for backwards traverse, set the start position at the end of previous block if (order == TSDB_ORDER_DESC) { - int32_t offset = pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + sizeof(pBlock->tag); int32_t r = fseek(pTSBuf->f, -offset, SEEK_CUR); UNUSED(r); } @@ -479,7 +509,7 @@ static int32_t tsBufFindBlock(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo, int if (pTSBuf->cur.order == TSDB_ORDER_DESC) { STSBlock* pBlock = &pTSBuf->block; int32_t compBlockSize = - pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + sizeof(pBlock->tag); + pBlock->compLen + sizeof(pBlock->compLen) * 2 + sizeof(pBlock->numOfElem) + getTagAreaLength(&pBlock->tag); int32_t ret = fseek(pTSBuf->f, -compBlockSize, SEEK_CUR); UNUSED(ret); } @@ -507,7 +537,7 @@ static int32_t tsBufFindBlockByTag(STSBuf* pTSBuf, STSVnodeBlockInfo* pBlockInfo } if (tVariantCompare(&pTSBuf->block.tag, tag) == 0) { - return i; + return (pTSBuf->cur.order == TSDB_ORDER_ASC)? i: (pBlockInfo->numOfBlocks - (i + 1)); } } @@ -993,4 +1023,29 @@ void tsBufGetVnodeIdList(STSBuf* pTSBuf, int32_t* num, int32_t** vnodeId) { for(int32_t i = 0; i < size; ++i) { (*vnodeId)[i] = pTSBuf->pData[i].info.vnode; } +} + +int32_t dumpFileBlockByVnodeId(STSBuf* pTSBuf, int32_t vnodeId, void* buf, int32_t* len, int32_t* numOfBlocks) { + STSVnodeBlockInfo *pBlockInfo = tsBufGetVnodeBlockInfo(pTSBuf, vnodeId); + + *len = 0; + *numOfBlocks = 0; + + if (fseek(pTSBuf->f, pBlockInfo->offset, SEEK_SET) != 0) { + int code = TAOS_SYSTEM_ERROR(ferror(pTSBuf->f)); +// qError("%p: fseek failed: %s", pSql, tstrerror(code)); + return code; + } + + size_t s = fread(buf, 1, pBlockInfo->compLen, pTSBuf->f); + if (s != pBlockInfo->compLen) { + int code = TAOS_SYSTEM_ERROR(ferror(pTSBuf->f)); +// tscError("%p: fread didn't return expected data: %s", pSql, tstrerror(code)); + return code; + } + + *len = pBlockInfo->compLen; + *numOfBlocks = pBlockInfo->numOfBlocks; + + return TSDB_CODE_SUCCESS; } \ No newline at end of file diff --git a/src/query/tests/tsBufTest.cpp b/src/query/tests/tsBufTest.cpp index 8cd3a9cbef..2892af3979 100644 --- a/src/query/tests/tsBufTest.cpp +++ b/src/query/tests/tsBufTest.cpp @@ -472,13 +472,20 @@ void mergeIdenticalVnodeBufferTest() { tsBufFlush(pTSBuf2); tsBufMerge(pTSBuf1, pTSBuf2); - EXPECT_EQ(pTSBuf1->numOfVnodes, 1); + EXPECT_EQ(pTSBuf1->numOfVnodes, 2); EXPECT_EQ(pTSBuf1->numOfTotal, numOfTags * 2 * num); tsBufResetPos(pTSBuf1); + + int32_t count = 0; while (tsBufNextPos(pTSBuf1)) { STSElem elem = tsBufGetElem(pTSBuf1); - EXPECT_EQ(elem.vnode, 12); + + if (count++ < numOfTags * num) { + EXPECT_EQ(elem.vnode, 12); + } else { + EXPECT_EQ(elem.vnode, 77); + } printf("%d-%" PRIu64 "-%" PRIu64 "\n", elem.vnode, elem.tag->i64Key, elem.ts); } diff --git a/tests/script/general/parser/join_multivnode.sim b/tests/script/general/parser/join_multivnode.sim index 85d5c79d86..dc2c0fcf3e 100644 --- a/tests/script/general/parser/join_multivnode.sim +++ b/tests/script/general/parser/join_multivnode.sim @@ -316,6 +316,8 @@ if $data03 != 0 then endi select count(join_mt0.c1), first(join_mt0.c1)-last(join_mt1.c1), first(join_mt1.c9) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts;", NULL); +select count(join_mt0.c1), first(join_mt0.c1)/count(*), first(join_mt1.c9) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts;", NULL); select count(join_mt0.c1), first(join_mt0.c1)-last(join_mt0.c1), first(join_mt1.c9) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts;", NULL); +select last(join_mt0.c1) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts;", NULL); system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file From f3425fc9bebe475d1a5d0bca351ff8cf92905b1c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 3 Nov 2020 18:48:27 +0800 Subject: [PATCH 25/35] [TD-225] disable sliding in ordinary query. --- src/client/src/tscSQLParser.c | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 27246aa7d3..215b38394b 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -79,9 +79,9 @@ static void setColumnOffsetValueInResultset(SQueryInfo* pQueryInfo); static int32_t parseGroupbyClause(SQueryInfo* pQueryInfo, tVariantList* pList, SSqlCmd* pCmd); -static int32_t parseIntervalClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql); +static int32_t parseIntervalClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql); static int32_t parseOffsetClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql); -static int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql); +static int32_t parseSlidingClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql); static int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExprItem* pItem); @@ -615,10 +615,12 @@ static bool isTopBottomQuery(SQueryInfo* pQueryInfo) { return false; } -int32_t parseIntervalClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { +int32_t parseIntervalClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { const char* msg1 = "invalid query expression"; const char* msg2 = "interval cannot be less than 10 ms"; + SSqlCmd* pCmd = &pSql->cmd; + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); @@ -655,7 +657,7 @@ int32_t parseIntervalClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ return TSDB_CODE_TSC_INVALID_SQL; } - if (parseSlidingClause(pCmd, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) { + if (parseSlidingClause(pSql, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -708,7 +710,7 @@ int32_t parseIntervalClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQ return TSDB_CODE_TSC_INVALID_SQL; } - if (parseSlidingClause(pCmd, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) { + if (parseSlidingClause(pSql, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -766,13 +768,15 @@ int32_t parseOffsetClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQue return TSDB_CODE_SUCCESS; } -int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { +int32_t parseSlidingClause(SSqlObj* pSql, SQueryInfo* pQueryInfo, SQuerySQL* pQuerySql) { const char* msg0 = "sliding value too small"; const char* msg1 = "sliding value no larger than the interval value"; const char* msg2 = "sliding value can not less than 1% of interval value"; - const char* msg3 = "does not support sliding when interval is natual month/year"; + const char* msg3 = "does not support sliding when interval is natural month/year"; + const char* msg4 = "sliding not support yet in ordinary query"; const static int32_t INTERVAL_SLIDING_FACTOR = 100; + SSqlCmd* pCmd = &pSql->cmd; STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); STableComInfo tinfo = tscGetTableInfo(pTableMetaInfo->pTableMeta); @@ -805,6 +809,10 @@ int32_t parseSlidingClause(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SQuerySQL* pQu return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg2); } + if (pQueryInfo->interval.sliding != pQueryInfo->interval.interval && pSql->pStream == NULL) { + return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg4); + } + return TSDB_CODE_SUCCESS; } @@ -6114,7 +6122,7 @@ int32_t doCheckForStream(SSqlObj* pSql, SSqlInfo* pInfo) { } // set interval value - if (parseIntervalClause(pCmd, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) { + if (parseIntervalClause(pSql, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } else { if ((pQueryInfo->interval.interval > 0) && @@ -6322,7 +6330,7 @@ int32_t doCheckForQuery(SSqlObj* pSql, SQuerySQL* pQuerySql, int32_t index) { } // set interval value - if (parseIntervalClause(pCmd, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) { + if (parseIntervalClause(pSql, pQueryInfo, pQuerySql) != TSDB_CODE_SUCCESS) { return TSDB_CODE_TSC_INVALID_SQL; } else { if ((pQueryInfo->interval.interval > 0) && From bc8cffd2a0feafb7f8dd5d8267c8396be6e5c44c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 4 Nov 2020 14:10:11 +0800 Subject: [PATCH 26/35] [TD-1373]: fix bugs in super table join. --- src/client/inc/tscSubquery.h | 2 +- src/client/inc/tscUtil.h | 1 + src/client/inc/tsclient.h | 7 +- src/client/src/tscAsync.c | 4 +- src/client/src/tscSQLParser.c | 50 ++++--- src/client/src/tscServer.c | 5 +- src/client/src/tscSubquery.c | 242 +++++++++++++++++++--------------- src/client/src/tscUtil.c | 19 ++- 8 files changed, 196 insertions(+), 134 deletions(-) diff --git a/src/client/inc/tscSubquery.h b/src/client/inc/tscSubquery.h index bc01de1103..3406dcd858 100644 --- a/src/client/inc/tscSubquery.h +++ b/src/client/inc/tscSubquery.h @@ -23,7 +23,7 @@ extern "C" { #include "tscUtil.h" #include "tsclient.h" -void tscFetchDatablockFromSubquery(SSqlObj* pSql); +void tscFetchDatablockForSubquery(SSqlObj* pSql); void tscSetupOutputColumnIndex(SSqlObj* pSql); void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code); diff --git a/src/client/inc/tscUtil.h b/src/client/inc/tscUtil.h index 39299d3308..0d57dd18c3 100644 --- a/src/client/inc/tscUtil.h +++ b/src/client/inc/tscUtil.h @@ -228,6 +228,7 @@ void tscClearSubqueryInfo(SSqlCmd* pCmd); void tscFreeVgroupTableInfo(SArray* pVgroupTables); SArray* tscVgroupTableInfoClone(SArray* pVgroupTables); void tscRemoveVgroupTableGroup(SArray* pVgroupTable, int32_t index); +void tscVgroupTableCopy(SVgroupTableInfo* info, SVgroupTableInfo* pInfo); int tscGetSTableVgroupInfo(SSqlObj* pSql, int32_t clauseIndex); int tscGetTableMeta(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo); diff --git a/src/client/inc/tsclient.h b/src/client/inc/tsclient.h index d21179df3d..bcc9f8354d 100644 --- a/src/client/inc/tsclient.h +++ b/src/client/inc/tsclient.h @@ -339,9 +339,9 @@ typedef struct STscObj { } STscObj; typedef struct SSubqueryState { - int32_t numOfRemain; // the number of remain unfinished subquery - int32_t numOfSub; // the number of total sub-queries - uint64_t numOfRetrievedRows; // total number of points in this query + int32_t numOfRemain; // the number of remain unfinished subquery + int32_t numOfSub; // the number of total sub-queries + uint64_t numOfRetrievedRows; // total number of points in this query } SSubqueryState; typedef struct SSqlObj { @@ -515,7 +515,6 @@ extern SRpcCorEpSet tscMgmtEpSet; extern int (*tscBuildMsg[TSDB_SQL_MAX])(SSqlObj *pSql, SSqlInfo *pInfo); -int32_t tscCompareTidTags(const void* p1, const void* p2); void tscBuildVgroupTableInfo(SSqlObj* pSql, STableMetaInfo* pTableMetaInfo, SArray* tables); #ifdef __cplusplus diff --git a/src/client/src/tscAsync.c b/src/client/src/tscAsync.c index a85a0ea570..e70cd11fbe 100644 --- a/src/client/src/tscAsync.c +++ b/src/client/src/tscAsync.c @@ -176,7 +176,7 @@ static void tscProcessAsyncRetrieveImpl(void *param, TAOS_RES *tres, int numOfRo } if (pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE) { - tscFetchDatablockFromSubquery(pSql); + tscFetchDatablockForSubquery(pSql); } else { tscProcessSql(pSql); } @@ -226,7 +226,7 @@ void taos_fetch_rows_a(TAOS_RES *taosa, __async_cb_func_t fp, void *param) { // handle the sub queries of join query if (pCmd->command == TSDB_SQL_TABLE_JOIN_RETRIEVE) { - tscFetchDatablockFromSubquery(pSql); + tscFetchDatablockForSubquery(pSql); } else if (pRes->completed) { if(pCmd->command == TSDB_SQL_FETCH || (pCmd->command >= TSDB_SQL_SERV_STATUS && pCmd->command <= TSDB_SQL_CURRENT_USER)) { if (hasMoreVnodesToTry(pSql)) { // sequentially retrieve data from remain vnodes. diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 215b38394b..4a9ee92d83 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -1632,7 +1632,7 @@ int32_t addProjectionExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, t } static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SSchema* pSchema, SConvertFunc cvtFunc, - tSQLExprItem* item, int32_t resColIdx, SColumnIndex* pColIndex, bool finalResult) { + const char* name, int32_t resColIdx, SColumnIndex* pColIndex, bool finalResult) { const char* msg1 = "not support column types"; int16_t type = 0; @@ -1640,7 +1640,7 @@ static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS int32_t functionID = cvtFunc.execFuncId; if (functionID == TSDB_FUNC_SPREAD) { - int32_t t1 = pSchema[pColIndex->columnIndex].type; + int32_t t1 = pSchema->type; if (t1 == TSDB_DATA_TYPE_BINARY || t1 == TSDB_DATA_TYPE_NCHAR || t1 == TSDB_DATA_TYPE_BOOL) { invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1); return -1; @@ -1649,17 +1649,12 @@ static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS bytes = tDataTypeDesc[type].nSize; } } else { - type = pSchema[pColIndex->columnIndex].type; - bytes = pSchema[pColIndex->columnIndex].bytes; + type = pSchema->type; + bytes = pSchema->bytes; } SSqlExpr* pExpr = tscSqlExprAppend(pQueryInfo, functionID, pColIndex, type, bytes, bytes, false); - if (item->aliasName != NULL) { - tstrncpy(pExpr->aliasName, item->aliasName, tListLen(pExpr->aliasName)); - } else { - int32_t len = MIN(tListLen(pExpr->aliasName), item->pNode->token.n + 1); - tstrncpy(pExpr->aliasName, item->pNode->token.z, len); - } + tstrncpy(pExpr->aliasName, name, tListLen(pExpr->aliasName)); if (cvtFunc.originFuncId == TSDB_FUNC_LAST_ROW && cvtFunc.originFuncId != functionID) { pExpr->colInfo.flag |= TSDB_COL_NULL; @@ -1687,6 +1682,18 @@ static int32_t setExprInfoForFunctions(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, SS return TSDB_CODE_SUCCESS; } +void setResultColName(char* name, tSQLExprItem* pItem, int32_t functionId, SStrToken* pToken) { + if (pItem->aliasName != NULL) { + tstrncpy(name, pItem->aliasName, TSDB_COL_NAME_LEN); + } else { + char uname[TSDB_COL_NAME_LEN] = {0}; + int32_t len = MIN(pToken->n + 1, TSDB_COL_NAME_LEN); + tstrncpy(uname, pToken->z, len); + + snprintf(name, TSDB_COL_NAME_LEN - 1, "%s(%s)", aAggs[functionId].aName, uname); + } +} + int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t colIndex, tSQLExprItem* pItem, bool finalResult) { STableMetaInfo* pTableMetaInfo = NULL; int32_t optr = pItem->pNode->nSQLOptr; @@ -1954,9 +1961,13 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); + char name[TSDB_COL_NAME_LEN] = {0}; for (int32_t j = 0; j < tscGetNumOfColumns(pTableMetaInfo->pTableMeta); ++j) { index.columnIndex = j; - if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, pItem, colIndex++, &index, finalResult) != 0) { + SStrToken t = {.z = pSchema[j].name, .n = strnlen(pSchema[j].name, TSDB_COL_NAME_LEN)}; + setResultColName(name, pItem, cvtFunc.originFuncId, &t); + + if (setExprInfoForFunctions(pCmd, pQueryInfo, &pSchema[j], cvtFunc, name, colIndex++, &index, finalResult) != 0) { return TSDB_CODE_TSC_INVALID_SQL; } } @@ -1967,14 +1978,18 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col } pTableMetaInfo = tscGetMetaInfo(pQueryInfo, index.tableIndex); - SSchema* pSchema = tscGetTableSchema(pTableMetaInfo->pTableMeta); // functions can not be applied to tags if ((index.columnIndex >= tscGetNumOfColumns(pTableMetaInfo->pTableMeta)) || (index.columnIndex < 0)) { return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg6); } - if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, pItem, colIndex + i, &index, finalResult) != 0) { + char name[TSDB_COL_NAME_LEN] = {0}; + + SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index.columnIndex); + setResultColName(name, pItem, cvtFunc.originFuncId, &pParamElem->pNode->colInfo); + + if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, name, colIndex + i, &index, finalResult) != 0) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -2011,7 +2026,12 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col for (int32_t i = 0; i < tscGetNumOfColumns(pTableMetaInfo->pTableMeta); ++i) { SColumnIndex index = {.tableIndex = j, .columnIndex = i}; - if (setExprInfoForFunctions(pCmd, pQueryInfo, pSchema, cvtFunc, pItem, colIndex, &index, finalResult) != 0) { + + char name[TSDB_COL_NAME_LEN] = {0}; + SStrToken t = {.z = pSchema->name, .n = strnlen(pSchema->name, TSDB_COL_NAME_LEN)}; + setResultColName(name, pItem, cvtFunc.originFuncId, &t); + + if (setExprInfoForFunctions(pCmd, pQueryInfo, &pSchema[index.columnIndex], cvtFunc, name, colIndex, &index, finalResult) != 0) { return TSDB_CODE_TSC_INVALID_SQL; } @@ -3458,7 +3478,7 @@ static int32_t validateArithmeticSQLExpr(SSqlCmd* pCmd, tSQLExpr* pExpr, SQueryI } // the expression not from the same table, return error - if (uidLeft != uidRight) { + if (uidLeft != uidRight && uidLeft != 0 && uidRight != 0) { return TSDB_CODE_TSC_INVALID_SQL; } } diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index 779cee2163..7f16476d89 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -880,8 +880,9 @@ int tscBuildQueryMsg(SSqlObj *pSql, SSqlInfo *pInfo) { pQueryMsg->tsOffset = htonl((int32_t)(pMsg - pCmd->payload)); if (pQueryInfo->tsBuf != NULL) { - int32_t vnodeId = htonl(pQueryMsg->head.vgId); - int32_t code = dumpFileBlockByVnodeId(pQueryInfo->tsBuf, vnodeId, pMsg, &pQueryMsg->tsLen, &pQueryMsg->tsNumOfBlocks); + // note: here used the index instead of actual vnode id. + int32_t vnodeIndex = pTableMetaInfo->vgroupIndex; + int32_t code = dumpFileBlockByVnodeId(pQueryInfo->tsBuf, vnodeIndex, pMsg, &pQueryMsg->tsLen, &pQueryMsg->tsNumOfBlocks); if (code != TSDB_CODE_SUCCESS) { return code; } diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index 38b0388d35..faf016be23 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -44,6 +44,17 @@ static int32_t tsCompare(int32_t order, int64_t left, int64_t right) { } } +static void skipRemainValue(STSBuf* pTSBuf, tVariant* tag1) { + while (tsBufNextPos(pTSBuf)) { + STSElem el1 = tsBufGetElem(pTSBuf); + + int32_t res = tVariantCompare(el1.tag, tag1); + if (res != 0) { // it is a record with new tag + return; + } + } +} + static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJoinSupporter* pSupporter2, STimeWindow * win) { SQueryInfo* pQueryInfo = tscGetQueryInfoDetail(&pSql->cmd, pSql->cmd.clauseIndex); @@ -92,54 +103,48 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ int64_t numOfInput1 = 1; int64_t numOfInput2 = 1; - int32_t numOfVnodes = 0; - int32_t* idList = NULL; - tsBufGetVnodeIdList(pSupporter2->pTSBuf, &numOfVnodes, &idList); - - bool completed = false; while(1) { STSElem elem = tsBufGetElem(pSupporter1->pTSBuf); // no data in pSupporter1 anymore, jump out of loop - if (elem.vnode < 0 || completed) { + if (!tsBufIsValidElem(&elem)) { break; } - bool f = false; - for (int32_t i = 0; i < numOfVnodes; ++i) { - STSElem el = tsBufGetElemStartPos(pSupporter2->pTSBuf, idList[i], elem.tag); - if (el.vnode == idList[i]) { - f = true; - break; - } - } + // find the data in supporter2 with the same tag value + STSElem e2 = tsBufFindElemStartPosByTag(pSupporter2->pTSBuf, elem.tag); /** * there are elements in pSupporter2 with the same tag, continue */ - if (f) { + tVariant tag1 = {0}; + tVariantAssign(&tag1, elem.tag); + + if (tsBufIsValidElem(&e2)) { while (1) { STSElem elem1 = tsBufGetElem(pSupporter1->pTSBuf); STSElem elem2 = tsBufGetElem(pSupporter2->pTSBuf); + // data with current are exhausted + if (!tsBufIsValidElem(&elem1) || tVariantCompare(elem1.tag, &tag1) != 0) { + break; + } + + if (!tsBufIsValidElem(&elem2) || tVariantCompare(elem2.tag, &tag1) != 0) { // ignore all records with the same tag + skipRemainValue(pSupporter1->pTSBuf, &tag1); + break; + } + /* * in case of stable query, limit/offset is not applied here. the limit/offset is applied to the - * final results which is acquired after the secondry merge of in the client. + * final results which is acquired after the secondary merge of in the client. */ int32_t re = tsCompare(order, elem1.ts, elem2.ts); if (re < 0) { - if (!tsBufNextPos(pSupporter1->pTSBuf)) { - completed = true; - break; - } - + tsBufNextPos(pSupporter1->pTSBuf); numOfInput1++; } else if (re > 0) { - if (!tsBufNextPos(pSupporter2->pTSBuf)) { - completed = true; - break; - } - + tsBufNextPos(pSupporter2->pTSBuf); numOfInput2++; } else { if (pLimit->offset == 0 || pQueryInfo->interval.interval > 0 || QUERY_IS_STABLE_QUERY(pQueryInfo->type)) { @@ -154,43 +159,18 @@ static int64_t doTSBlockIntersect(SSqlObj* pSql, SJoinSupporter* pSupporter1, SJ tsBufAppend(output1, elem1.vnode, elem1.tag, (const char*)&elem1.ts, sizeof(elem1.ts)); tsBufAppend(output2, elem2.vnode, elem2.tag, (const char*)&elem2.ts, sizeof(elem2.ts)); } else { - pLimit->offset -= 1; - } - - if (!tsBufNextPos(pSupporter1->pTSBuf)) { - completed = true; - break; + pLimit->offset -= 1;//offset apply to projection? } + tsBufNextPos(pSupporter1->pTSBuf); numOfInput1++; - if (!tsBufNextPos(pSupporter2->pTSBuf)) { - completed = true; - break; - } - + tsBufNextPos(pSupporter2->pTSBuf); numOfInput2++; } } } else { // no data in pSupporter2, ignore current data in pSupporter2 - tVariant tag = {0}; - tVariantAssign(&tag, elem.tag); - - // ignore all records with the same tag - while (tsBufNextPos(pSupporter1->pTSBuf)) { - STSElem el1 = tsBufGetElem(pSupporter1->pTSBuf); - int32_t res = tVariantCompare(el1.tag, &tag); - - // it is a record with new tag - if (res != 0) { - break; - } - } - - STSElem el1 = tsBufGetElem(pSupporter1->pTSBuf); - if (el1.vnode < 0) { // no data exists, abort - break; - } + skipRemainValue(pSupporter1->pTSBuf, &tag1); } } @@ -299,6 +279,68 @@ static UNUSED_FUNC bool needSecondaryQuery(SQueryInfo* pQueryInfo) { return false; } +static void filterVgroupTables(SQueryInfo* pQueryInfo, SArray* pVgroupTables) { + int32_t num = 0; + int32_t* list = NULL; + tsBufGetVnodeIdList(pQueryInfo->tsBuf, &num, &list); + + // The virtual node, of which all tables are disqualified after the timestamp intersection, + // is removed to avoid next stage query. + // TODO: If tables from some vnodes are not qualified for next stage query, discard them. + for (int32_t k = 0; k < taosArrayGetSize(pVgroupTables);) { + SVgroupTableInfo* p = taosArrayGet(pVgroupTables, k); + + bool found = false; + for (int32_t f = 0; f < num; ++f) { + if (p->vgInfo.vgId == list[f]) { + found = true; + break; + } + } + + if (!found) { + tscRemoveVgroupTableGroup(pVgroupTables, k); + } else { + k++; + } + } + + assert(taosArrayGetSize(pVgroupTables) > 0); + TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_MULTITABLE_QUERY); + + taosTFree(list); +} + +static SArray* buildVgroupTableByResult(SQueryInfo* pQueryInfo, SArray* pVgroupTables) { + int32_t num = 0; + int32_t* list = NULL; + tsBufGetVnodeIdList(pQueryInfo->tsBuf, &num, &list); + + int32_t numOfGroups = taosArrayGetSize(pVgroupTables); + + SArray* pNew = taosArrayInit(num, sizeof(SVgroupTableInfo)); + + SVgroupTableInfo info; + for (int32_t i = 0; i < num; ++i) { + int32_t vnodeId = list[i]; + + for (int32_t j = 0; j < numOfGroups; ++j) { + SVgroupTableInfo* p1 = taosArrayGet(pVgroupTables, j); + if (p1->vgInfo.vgId == vnodeId) { + tscVgroupTableCopy(&info, p1); + break; + } + } + + taosArrayPush(pNew, &info); + } + + taosTFree(list); + TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_MULTITABLE_QUERY); + + return pNew; +} + /* * launch secondary stage query to fetch the result that contains timestamp in set */ @@ -373,12 +415,11 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { pQueryInfo->fieldsInfo = pSupporter->fieldsInfo; pQueryInfo->groupbyExpr = pSupporter->groupInfo; - SQueryInfo *pNewQueryInfo = tscGetQueryInfoDetail(&pNew->cmd, 0); - assert(pNew->subState.numOfSub == 0 && pNew->cmd.numOfClause == 1 && pNewQueryInfo->numOfTables == 1); + assert(pNew->subState.numOfSub == 0 && pNew->cmd.numOfClause == 1 && pQueryInfo->numOfTables == 1); - tscFieldInfoUpdateOffset(pNewQueryInfo); + tscFieldInfoUpdateOffset(pQueryInfo); - STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pNewQueryInfo, 0); + STableMetaInfo *pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); pTableMetaInfo->pVgroupTables = pSupporter->pVgroupTables; pSupporter->exprList = NULL; @@ -392,7 +433,7 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { * during the timestamp intersection. */ pSupporter->limit = pQueryInfo->limit; - pNewQueryInfo->limit = pSupporter->limit; + pQueryInfo->limit = pSupporter->limit; SColumnIndex index = {.tableIndex = 0, .columnIndex = PRIMARYKEY_TIMESTAMP_COL_INDEX}; SSchema* s = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, 0); @@ -407,7 +448,7 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { tscAddSpecialColumnForSelect(pQueryInfo, 0, functionId, &index, s, TSDB_COL_NORMAL); tscPrintSelectClause(pNew, 0); - tscFieldInfoUpdateOffset(pNewQueryInfo); + tscFieldInfoUpdateOffset(pQueryInfo); pExpr = tscSqlExprGet(pQueryInfo, 0); } @@ -422,39 +463,21 @@ static int32_t tscLaunchRealSubqueries(SSqlObj* pSql) { pExpr->numOfParams = 1; } - int32_t num = 0; - int32_t *list = NULL; - tsBufGetVnodeIdList(pNewQueryInfo->tsBuf, &num, &list); - - if (pTableMetaInfo->pVgroupTables != NULL) { - for(int32_t k = 0; k < taosArrayGetSize(pTableMetaInfo->pVgroupTables);) { - SVgroupTableInfo* p = taosArrayGet(pTableMetaInfo->pVgroupTables, k); - - bool found = false; - for(int32_t f = 0; f < num; ++f) { - if (p->vgInfo.vgId == list[f]) { - found = true; - break; - } - } - - if (!found) { - tscRemoveVgroupTableGroup(pTableMetaInfo->pVgroupTables, k); - } else { - k++; - } + if (UTIL_TABLE_IS_SUPER_TABLE(pTableMetaInfo)) { + assert(pTableMetaInfo->pVgroupTables != NULL); + if (tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { + SArray* p = buildVgroupTableByResult(pQueryInfo, pTableMetaInfo->pVgroupTables); + tscFreeVgroupTableInfo(pTableMetaInfo->pVgroupTables); + pTableMetaInfo->pVgroupTables = p; + } else { + filterVgroupTables(pQueryInfo, pTableMetaInfo->pVgroupTables); } - - assert(taosArrayGetSize(pTableMetaInfo->pVgroupTables) > 0); - TSDB_QUERY_SET_TYPE(pQueryInfo->type, TSDB_QUERY_TYPE_MULTITABLE_QUERY); } - taosTFree(list); - - size_t numOfCols = taosArrayGetSize(pNewQueryInfo->colList); + size_t numOfCols = taosArrayGetSize(pQueryInfo->colList); tscDebug("%p subquery:%p tableIndex:%d, vgroupIndex:%d, type:%d, exprInfo:%" PRIzu ", colList:%" PRIzu ", fieldsInfo:%d, name:%s", - pSql, pNew, 0, pTableMetaInfo->vgroupIndex, pNewQueryInfo->type, taosArrayGetSize(pNewQueryInfo->exprList), - numOfCols, pNewQueryInfo->fieldsInfo.numOfOutput, pTableMetaInfo->name); + pSql, pNew, 0, pTableMetaInfo->vgroupIndex, pQueryInfo->type, taosArrayGetSize(pQueryInfo->exprList), + numOfCols, pQueryInfo->fieldsInfo.numOfOutput, pTableMetaInfo->name); } //prepare the subqueries object failed, abort @@ -1034,6 +1057,7 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR } } + assert(pState->numOfRemain > 0); if (atomic_sub_fetch_32(&pState->numOfRemain, 1) > 0) { tscDebug("%p sub:%p completed, remain:%d, total:%d", pParentSql, tres, pState->numOfRemain, pState->numOfSub); return; @@ -1047,7 +1071,7 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR } // update the records for each subquery in parent sql object. - bool isSTableSub = tscIsTwoStageSTableQuery(pQueryInfo, 0); + bool stableQuery = tscIsTwoStageSTableQuery(pQueryInfo, 0); for (int32_t i = 0; i < pState->numOfSub; ++i) { if (pParentSql->pSubs[i] == NULL) { tscDebug("%p %p sub:%d not retrieve data", pParentSql, NULL, i); @@ -1061,7 +1085,7 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR pRes1->numOfRows, pRes1->numOfTotal); assert(pRes1->row < pRes1->numOfRows); } else { - if (!isSTableSub) { + if (!stableQuery) { pRes1->numOfClauseTotal += pRes1->numOfRows; } @@ -1074,7 +1098,7 @@ static void joinRetrieveFinalResCallback(void* param, TAOS_RES* tres, int numOfR tscBuildResFromSubqueries(pParentSql); } -void tscFetchDatablockFromSubquery(SSqlObj* pSql) { +void tscFetchDatablockForSubquery(SSqlObj* pSql) { assert(pSql->subState.numOfSub >= 1); int32_t numOfFetch = 0; @@ -1136,11 +1160,22 @@ void tscFetchDatablockFromSubquery(SSqlObj* pSql) { if (numOfFetch <= 0) { bool tryNextVnode = false; - SSqlObj* pp = pSql->pSubs[0]; - SQueryInfo* pi = tscGetQueryInfoDetail(&pp->cmd, 0); + bool orderedPrjQuery = false; + for(int32_t i = 0; i < pSql->subState.numOfSub; ++i) { + SSqlObj* pSub = pSql->pSubs[i]; + if (pSub == NULL) { + continue; + } + + SQueryInfo* p = tscGetQueryInfoDetail(&pSub->cmd, 0); + orderedPrjQuery = tscNonOrderedProjectionQueryOnSTable(p, 0); + if (orderedPrjQuery) { + break; + } + } // get the number of subquery that need to retrieve the next vnode. - if (tscNonOrderedProjectionQueryOnSTable(pi, 0)) { + if (orderedPrjQuery) { for (int32_t i = 0; i < pSql->subState.numOfSub; ++i) { SSqlObj* pSub = pSql->pSubs[i]; if (pSub != NULL && pSub->res.row >= pSub->res.numOfRows && pSub->res.completed) { @@ -1244,7 +1279,6 @@ void tscSetupOutputColumnIndex(SSqlObj* pSql) { SSqlCmd* pCmd = &pSql->cmd; SSqlRes* pRes = &pSql->res; - tscDebug("%p all subquery response, retrieve data for subclause:%d", pSql, pCmd->clauseIndex); // the column transfer support struct has been built if (pRes->pColumnIndex != NULL) { @@ -1340,21 +1374,23 @@ void tscJoinQueryCallback(void* param, TAOS_RES* tres, int code) { return; } - // wait for the other subqueries response from vnode - if (atomic_sub_fetch_32(&pParentSql->subState.numOfRemain, 1) > 0) { - return; + + STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); + + // In case of consequence query from other vnode, do not wait for other query response here. + if (!(pTableMetaInfo->vgroupIndex > 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0))) { + if (atomic_sub_fetch_32(&pParentSql->subState.numOfRemain, 1) > 0) { + return; + } } tscSetupOutputColumnIndex(pParentSql); - STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, 0); /** * if the query is a continue query (vgroupIndex > 0 for projection query) for next vnode, do the retrieval of * data instead of returning to its invoker */ if (pTableMetaInfo->vgroupIndex > 0 && tscNonOrderedProjectionQueryOnSTable(pQueryInfo, 0)) { -// pParentSql->subState.numOfRemain = pParentSql->subState.numOfSub; // reset the record value - pSql->fp = joinRetrieveFinalResCallback; // continue retrieve data pSql->cmd.command = TSDB_SQL_FETCH; tscProcessSql(pSql); diff --git a/src/client/src/tscUtil.c b/src/client/src/tscUtil.c index dca23e0dcb..effb2db6ac 100644 --- a/src/client/src/tscUtil.c +++ b/src/client/src/tscUtil.c @@ -1696,6 +1696,17 @@ void tscRemoveVgroupTableGroup(SArray* pVgroupTable, int32_t index) { taosArrayRemove(pVgroupTable, index); } +void tscVgroupTableCopy(SVgroupTableInfo* info, SVgroupTableInfo* pInfo) { + memset(info, 0, sizeof(SVgroupTableInfo)); + + info->vgInfo = pInfo->vgInfo; + for(int32_t j = 0; j < pInfo->vgInfo.numOfEps; ++j) { + info->vgInfo.epAddr[j].fqdn = strdup(pInfo->vgInfo.epAddr[j].fqdn); + } + + info->itemList = taosArrayClone(pInfo->itemList); +} + SArray* tscVgroupTableInfoClone(SArray* pVgroupTables) { if (pVgroupTables == NULL) { return NULL; @@ -1707,14 +1718,8 @@ SArray* tscVgroupTableInfoClone(SArray* pVgroupTables) { SVgroupTableInfo info; for (size_t i = 0; i < num; i++) { SVgroupTableInfo* pInfo = taosArrayGet(pVgroupTables, i); - memset(&info, 0, sizeof(SVgroupTableInfo)); + tscVgroupTableCopy(&info, pInfo); - info.vgInfo = pInfo->vgInfo; - for(int32_t j = 0; j < pInfo->vgInfo.numOfEps; ++j) { - info.vgInfo.epAddr[j].fqdn = strdup(pInfo->vgInfo.epAddr[j].fqdn); - } - - info.itemList = taosArrayClone(pInfo->itemList); taosArrayPush(pa, &info); } From 5980b836c9bc8153025bfadd16d2ac253ce08754 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 4 Nov 2020 14:11:24 +0800 Subject: [PATCH 27/35] [TD-225] --- src/query/inc/qTsbuf.h | 4 ++++ src/query/src/qTsbuf.c | 30 ++++++++++++++++++++++++------ 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/src/query/inc/qTsbuf.h b/src/query/inc/qTsbuf.h index fc5efa069d..73b18f8915 100644 --- a/src/query/inc/qTsbuf.h +++ b/src/query/inc/qTsbuf.h @@ -137,6 +137,10 @@ void tsBufGetVnodeIdList(STSBuf* pTSBuf, int32_t* num, int32_t** vnodeId); int32_t dumpFileBlockByVnodeId(STSBuf* pTSBuf, int32_t vnodeId, void* buf, int32_t* len, int32_t* numOfBlocks); +STSElem tsBufFindElemStartPosByTag(STSBuf* pTSBuf, tVariant* pTag); + +bool tsBufIsValidElem(STSElem* pElem); + #ifdef __cplusplus } #endif diff --git a/src/query/src/qTsbuf.c b/src/query/src/qTsbuf.c index 82f36a8951..2514da7231 100644 --- a/src/query/src/qTsbuf.c +++ b/src/query/src/qTsbuf.c @@ -477,7 +477,7 @@ void tsBufFlush(STSBuf* pTSBuf) { fsync(fileno(pTSBuf->f)); } -static int32_t tsBufFindVnodeIndexFromId(STSVnodeBlockInfoEx* pVnodeInfoEx, int32_t numOfVnodes, int32_t vnodeId) { +static int32_t tsBufFindVnodeById(STSVnodeBlockInfoEx* pVnodeInfoEx, int32_t numOfVnodes, int32_t vnodeId) { int32_t j = -1; for (int32_t i = 0; i < numOfVnodes; ++i) { if (pVnodeInfoEx[i].info.vnode == vnodeId) { @@ -606,7 +606,7 @@ static int32_t doUpdateVnodeInfo(STSBuf* pTSBuf, int64_t offset, STSVnodeBlockIn } STSVnodeBlockInfo* tsBufGetVnodeBlockInfo(STSBuf* pTSBuf, int32_t vnodeId) { - int32_t j = tsBufFindVnodeIndexFromId(pTSBuf->pData, pTSBuf->numOfVnodes, vnodeId); + int32_t j = tsBufFindVnodeById(pTSBuf->pData, pTSBuf->numOfVnodes, vnodeId); if (j == -1) { return NULL; } @@ -870,7 +870,7 @@ STSElem tsBufGetElemStartPos(STSBuf* pTSBuf, int32_t vnodeId, tVariant* tag) { return elem; } - int32_t j = tsBufFindVnodeIndexFromId(pTSBuf->pData, pTSBuf->numOfVnodes, vnodeId); + int32_t j = tsBufFindVnodeById(pTSBuf->pData, pTSBuf->numOfVnodes, vnodeId); if (j == -1) { return elem; } @@ -1025,8 +1025,9 @@ void tsBufGetVnodeIdList(STSBuf* pTSBuf, int32_t* num, int32_t** vnodeId) { } } -int32_t dumpFileBlockByVnodeId(STSBuf* pTSBuf, int32_t vnodeId, void* buf, int32_t* len, int32_t* numOfBlocks) { - STSVnodeBlockInfo *pBlockInfo = tsBufGetVnodeBlockInfo(pTSBuf, vnodeId); +int32_t dumpFileBlockByVnodeId(STSBuf* pTSBuf, int32_t vnodeIndex, void* buf, int32_t* len, int32_t* numOfBlocks) { + assert(vnodeIndex >= 0 && vnodeIndex < pTSBuf->numOfVnodes); + STSVnodeBlockInfo *pBlockInfo = &pTSBuf->pData[vnodeIndex].info; *len = 0; *numOfBlocks = 0; @@ -1048,4 +1049,21 @@ int32_t dumpFileBlockByVnodeId(STSBuf* pTSBuf, int32_t vnodeId, void* buf, int32 *numOfBlocks = pBlockInfo->numOfBlocks; return TSDB_CODE_SUCCESS; -} \ No newline at end of file +} + +STSElem tsBufFindElemStartPosByTag(STSBuf* pTSBuf, tVariant* pTag) { + STSElem el = {.vnode = -1}; + + for (int32_t i = 0; i < pTSBuf->numOfVnodes; ++i) { + el = tsBufGetElemStartPos(pTSBuf, pTSBuf->pData[i].info.vnode, pTag); + if (el.vnode == pTSBuf->pData[i].info.vnode) { + return el; + } + } + + return el; +} + +bool tsBufIsValidElem(STSElem* pElem) { + return pElem->vnode >= 0; +} From f78fd9935dda787428f2804ff0433ccac61e7e3b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 4 Nov 2020 14:11:52 +0800 Subject: [PATCH 28/35] [TD-225] update test cases. --- tests/script/general/parser/groupby.sim | 96 +++++++++---------- tests/script/general/parser/join.sim | 3 + .../script/general/parser/join_multivnode.sim | 8 +- tests/script/general/parser/testSuite.sim | 96 +++++++++---------- 4 files changed, 103 insertions(+), 100 deletions(-) diff --git a/tests/script/general/parser/groupby.sim b/tests/script/general/parser/groupby.sim index b70fe88e81..145c4a008f 100644 --- a/tests/script/general/parser/groupby.sim +++ b/tests/script/general/parser/groupby.sim @@ -435,53 +435,53 @@ sql insert into t1 values ('2020-03-27 04:21:16.000', 1)('2020-03-27 04:31:17.00 sql insert into t2 values ('2020-03-27 04:11:16.000', 1)('2020-03-27 04:11:17.000', 2) ('2020-03-27 04:11:18.000', 3) ('2020-03-27 04:11:19.000', 4) ; sql insert into t2 values ('2020-03-27 04:21:16.000', 1)('2020-03-27 04:31:17.000', 2) ('2020-03-27 04:51:18.000', 3) ('2020-03-27 05:10:19.000', 4) ; -sql select irate(c) from st where t1="1" and ts >= '2020-03-27 04:11:17.732' and ts < '2020-03-27 05:11:17.732' interval(1m) sliding(15s) group by tbname,t1,t2; -if $rows != 40 then - return -1 -endi - -if $data01 != 1.000000000 then - return -1 -endi -if $data02 != t1 then - return -1 -endi -if $data03 != 1 then - return -1 -endi -if $data04 != 1 then - return -1 -endi - -if $data11 != 1.000000000 then - return -1 -endi -if $data12 != t1 then - return -1 -endi -if $data13 != 1 then - return -1 -endi -if $data14 != 1 then - return -1 -endi - -sql select irate(c) from st where t1="1" and ts >= '2020-03-27 04:11:17.732' and ts < '2020-03-27 05:11:17.732' interval(1m) sliding(15s) group by tbname,t1,t2 limit 1; -if $rows != 2 then - return -1 -endi - -if $data11 != 1.000000000 then - return -1 -endi -if $data12 != t2 then - return -1 -endi -if $data13 != 1 then - return -1 -endi -if $data14 != 2 then - return -1 -endi +#sql select irate(c) from st where t1="1" and ts >= '2020-03-27 04:11:17.732' and ts < '2020-03-27 05:11:17.732' interval(1m) sliding(15s) group by tbname,t1,t2; +#if $rows != 40 then +# return -1 +#endi +# +#if $data01 != 1.000000000 then +# return -1 +#endi +#if $data02 != t1 then +# return -1 +#endi +#if $data03 != 1 then +# return -1 +#endi +#if $data04 != 1 then +# return -1 +#endi +# +#if $data11 != 1.000000000 then +# return -1 +#endi +#if $data12 != t1 then +# return -1 +#endi +#if $data13 != 1 then +# return -1 +#endi +#if $data14 != 1 then +# return -1 +#endi +# +#sql select irate(c) from st where t1="1" and ts >= '2020-03-27 04:11:17.732' and ts < '2020-03-27 05:11:17.732' interval(1m) sliding(15s) group by tbname,t1,t2 limit 1; +#if $rows != 2 then +# return -1 +#endi +# +#if $data11 != 1.000000000 then +# return -1 +#endi +#if $data12 != t2 then +# return -1 +#endi +#if $data13 != 1 then +# return -1 +#endi +#if $data14 != 2 then +# return -1 +#endi system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/general/parser/join.sim b/tests/script/general/parser/join.sim index 79b30ffe92..624c2e72c4 100644 --- a/tests/script/general/parser/join.sim +++ b/tests/script/general/parser/join.sim @@ -444,6 +444,9 @@ if $rows != $val then return -1 endi +#=============================================================== +sql select first(join_tb0.c8),first(join_tb0.c9) from join_tb1 , join_tb0 where join_tb1.ts = join_tb0.ts and join_tb1.ts <= 100002 and join_tb0.c7 = true + #====================group by========================================= diff --git a/tests/script/general/parser/join_multivnode.sim b/tests/script/general/parser/join_multivnode.sim index dc2c0fcf3e..c5fcf575ae 100644 --- a/tests/script/general/parser/join_multivnode.sim +++ b/tests/script/general/parser/join_multivnode.sim @@ -315,9 +315,9 @@ if $data03 != 0 then return -1 endi -select count(join_mt0.c1), first(join_mt0.c1)-last(join_mt1.c1), first(join_mt1.c9) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts;", NULL); -select count(join_mt0.c1), first(join_mt0.c1)/count(*), first(join_mt1.c9) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts;", NULL); -select count(join_mt0.c1), first(join_mt0.c1)-last(join_mt0.c1), first(join_mt1.c9) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts;", NULL); -select last(join_mt0.c1) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts;", NULL); +sql_error select count(join_mt0.c1), first(join_mt0.c1)-last(join_mt1.c1), first(join_mt1.c9) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts;", NULL); +sql select count(join_mt0.c1), first(join_mt0.c1)/count(*), first(join_mt1.c9) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts;", NULL); +sql select count(join_mt0.c1), first(join_mt0.c1)-last(join_mt0.c1), first(join_mt1.c9) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts;", NULL); +sql select last(join_mt0.c1) from join_mt0, join_mt1 where join_mt0.t1=join_mt1.t1 and join_mt0.ts=join_mt1.ts;", NULL); system sh/exec.sh -n dnode1 -s stop -x SIGINT \ No newline at end of file diff --git a/tests/script/general/parser/testSuite.sim b/tests/script/general/parser/testSuite.sim index b848408925..3dd80b8e38 100644 --- a/tests/script/general/parser/testSuite.sim +++ b/tests/script/general/parser/testSuite.sim @@ -1,51 +1,51 @@ -#sleep 2000 -#run general/parser/alter.sim -#sleep 2000 -#run general/parser/alter1.sim -#sleep 2000 -#run general/parser/alter_stable.sim -#sleep 2000 -#run general/parser/auto_create_tb.sim -#sleep 2000 -#run general/parser/auto_create_tb_drop_tb.sim -#sleep 2000 -#run general/parser/col_arithmetic_operation.sim -#sleep 2000 -#run general/parser/columnValue.sim -#sleep 2000 -#run general/parser/commit.sim -#sleep 2000 -#run general/parser/create_db.sim -#sleep 2000 -#run general/parser/create_mt.sim -#sleep 2000 -#run general/parser/create_tb.sim -#sleep 2000 -#run general/parser/dbtbnameValidate.sim -#sleep 2000 -#run general/parser/fill.sim -#sleep 2000 -#run general/parser/fill_stb.sim -#sleep 2000 -##run general/parser/fill_us.sim # -#sleep 2000 -#run general/parser/first_last.sim -#sleep 2000 -#run general/parser/import_commit1.sim -#sleep 2000 -#run general/parser/import_commit2.sim -#sleep 2000 -#run general/parser/import_commit3.sim -#sleep 2000 -##run general/parser/import_file.sim -#sleep 2000 -#run general/parser/insert_tb.sim -#sleep 2000 -#run general/parser/tags_dynamically_specifiy.sim -#sleep 2000 -#run general/parser/interp.sim -#sleep 2000 -#run general/parser/lastrow.sim +sleep 2000 +run general/parser/alter.sim +sleep 2000 +run general/parser/alter1.sim +sleep 2000 +run general/parser/alter_stable.sim +sleep 2000 +run general/parser/auto_create_tb.sim +sleep 2000 +run general/parser/auto_create_tb_drop_tb.sim +sleep 2000 +run general/parser/col_arithmetic_operation.sim +sleep 2000 +run general/parser/columnValue.sim +sleep 2000 +run general/parser/commit.sim +sleep 2000 +run general/parser/create_db.sim +sleep 2000 +run general/parser/create_mt.sim +sleep 2000 +run general/parser/create_tb.sim +sleep 2000 +run general/parser/dbtbnameValidate.sim +sleep 2000 +run general/parser/fill.sim +sleep 2000 +run general/parser/fill_stb.sim +sleep 2000 +#run general/parser/fill_us.sim # +sleep 2000 +run general/parser/first_last.sim +sleep 2000 +run general/parser/import_commit1.sim +sleep 2000 +run general/parser/import_commit2.sim +sleep 2000 +run general/parser/import_commit3.sim +sleep 2000 +#run general/parser/import_file.sim +sleep 2000 +run general/parser/insert_tb.sim +sleep 2000 +run general/parser/tags_dynamically_specifiy.sim +sleep 2000 +run general/parser/interp.sim +sleep 2000 +run general/parser/lastrow.sim sleep 2000 run general/parser/limit.sim sleep 2000 From a5e08be9531b8268363f84c3768544f42d105305 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 4 Nov 2020 14:22:48 +0800 Subject: [PATCH 29/35] [TD-225] --- src/client/src/tscSQLParser.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 4a9ee92d83..ae5eb57bdf 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -1690,7 +1690,12 @@ void setResultColName(char* name, tSQLExprItem* pItem, int32_t functionId, SStrT int32_t len = MIN(pToken->n + 1, TSDB_COL_NAME_LEN); tstrncpy(uname, pToken->z, len); - snprintf(name, TSDB_COL_NAME_LEN - 1, "%s(%s)", aAggs[functionId].aName, uname); + int32_t size = TSDB_COL_NAME_LEN + tListLen(aAggs[functionId].aName) + 2 + 1; + char tmp[TSDB_COL_NAME_LEN + tListLen(aAggs[functionId].aName) + 2 + 1] = {0}; + + snprintf(tmp, size, "%s(%s)", aAggs[functionId].aName, uname); + + tstrncpy(name, tmp, TSDB_COL_NAME_LEN); } } From 825982443050e00de5ee940185ec7001ff3796d4 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 4 Nov 2020 16:08:49 +0800 Subject: [PATCH 30/35] [TD-225]fix compiler error. --- src/query/src/qTsbuf.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/src/qTsbuf.c b/src/query/src/qTsbuf.c index 691d2aed9c..df9ea1db03 100644 --- a/src/query/src/qTsbuf.c +++ b/src/query/src/qTsbuf.c @@ -271,7 +271,7 @@ static void writeDataToDisk(STSBuf* pTSBuf) { fwrite(pBlock->payload, (size_t)pBlock->compLen, 1, pTSBuf->f); fwrite(&pBlock->compLen, sizeof(pBlock->compLen), 1, pTSBuf->f); - metaLen += fwrite(&trueLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); + metaLen += (int32_t) fwrite(&trueLen, 1, sizeof(pBlock->tag.nLen), pTSBuf->f); assert(metaLen == getTagAreaLength(&pBlock->tag)); int32_t blockSize = metaLen + sizeof(pBlock->numOfElem) + sizeof(pBlock->compLen) * 2 + pBlock->compLen; From c47dbec976507ba806116b43e284ad0a556ef342 Mon Sep 17 00:00:00 2001 From: Ping Xiao Date: Wed, 4 Nov 2020 16:30:47 +0800 Subject: [PATCH 31/35] update TAOS-SQL md --- documentation20/webdocs/markdowndocs/TAOS SQL-ch.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md b/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md index 4082d72f11..191af56bbd 100644 --- a/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md +++ b/documentation20/webdocs/markdowndocs/TAOS SQL-ch.md @@ -844,7 +844,7 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数 - **PERCENTILE** ```mysql - SELECT PERCENTILE(field_name, P) FROM { tb_name | stb_name } [WHERE clause]; + SELECT PERCENTILE(field_name, P) FROM { tb_name } [WHERE clause]; ``` 功能说明：统计表中某列的值百分比分位数。 返回结果数据类型： 双精度浮点数Double。 From 38c01e2992f30a9cf29efe6e024d14a343711779 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 4 Nov 2020 16:44:33 +0800 Subject: [PATCH 32/35] [TD-225] --- src/query/src/qUtil.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index 824cae3932..fec3034572 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -303,9 +303,8 @@ SResultRowPool* initResultRowPool(size_t size) { p->numOfElemPerBlock = 128; - p->elemSize = size; + p->elemSize = (int32_t) size; p->blockSize = p->numOfElemPerBlock * p->elemSize; - p->position.pos = 0; p->pData = taosArrayInit(8, POINTER_BYTES); From adf204ec8fac283dc80f95aaf8d80b02a032bd87 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 4 Nov 2020 16:48:10 +0800 Subject: [PATCH 33/35] [TD-225] --- src/query/src/qUtil.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index fec3034572..dc968bad06 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -325,7 +325,7 @@ SResultRow* getNewResultRow(SResultRowPool* p) { size_t last = taosArrayGetSize(p->pData); void** pBlock = taosArrayGet(p->pData, last - 1); - ptr = (*pBlock) + p->elemSize * p->position.pos; + ptr = ((char*) (*pBlock)) + p->elemSize * p->position.pos; } p->position.pos = (p->position.pos + 1)%p->numOfElemPerBlock; @@ -343,7 +343,7 @@ int64_t getResultRowPoolMemSize(SResultRowPool* p) { } int32_t getNumOfAllocatedResultRows(SResultRowPool* p) { - return taosArrayGetSize(p->pData) * p->numOfElemPerBlock; + return (int32_t) taosArrayGetSize(p->pData) * p->numOfElemPerBlock; } int32_t getNumOfUsedResultRows(SResultRowPool* p) { From 9d224db14fd62ff08a932f2db1a843e801a26d52 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 4 Nov 2020 17:07:02 +0800 Subject: [PATCH 34/35] compile in windows --- src/client/src/tscSQLParser.c | 4 ++-- src/client/src/tscSubquery.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/client/src/tscSQLParser.c b/src/client/src/tscSQLParser.c index 3e7a64c828..f7aaf83066 100644 --- a/src/client/src/tscSQLParser.c +++ b/src/client/src/tscSQLParser.c @@ -1973,7 +1973,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col char name[TSDB_COL_NAME_LEN] = {0}; for (int32_t j = 0; j < tscGetNumOfColumns(pTableMetaInfo->pTableMeta); ++j) { index.columnIndex = j; - SStrToken t = {.z = pSchema[j].name, .n = strnlen(pSchema[j].name, TSDB_COL_NAME_LEN)}; + SStrToken t = {.z = pSchema[j].name, .n = (uint32_t)strnlen(pSchema[j].name, TSDB_COL_NAME_LEN)}; setResultColName(name, pItem, cvtFunc.originFuncId, &t); if (setExprInfoForFunctions(pCmd, pQueryInfo, &pSchema[j], cvtFunc, name, colIndex++, &index, finalResult) != 0) { @@ -2037,7 +2037,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col SColumnIndex index = {.tableIndex = j, .columnIndex = i}; char name[TSDB_COL_NAME_LEN] = {0}; - SStrToken t = {.z = pSchema->name, .n = strnlen(pSchema->name, TSDB_COL_NAME_LEN)}; + SStrToken t = {.z = pSchema->name, .n = (uint32_t)strnlen(pSchema->name, TSDB_COL_NAME_LEN)}; setResultColName(name, pItem, cvtFunc.originFuncId, &t); if (setExprInfoForFunctions(pCmd, pQueryInfo, &pSchema[index.columnIndex], cvtFunc, name, colIndex, &index, finalResult) != 0) { diff --git a/src/client/src/tscSubquery.c b/src/client/src/tscSubquery.c index f328a6c075..de0d899f8c 100644 --- a/src/client/src/tscSubquery.c +++ b/src/client/src/tscSubquery.c @@ -316,7 +316,7 @@ static SArray* buildVgroupTableByResult(SQueryInfo* pQueryInfo, SArray* pVgroupT int32_t* list = NULL; tsBufGetVnodeIdList(pQueryInfo->tsBuf, &num, &list); - int32_t numOfGroups = taosArrayGetSize(pVgroupTables); + size_t numOfGroups = taosArrayGetSize(pVgroupTables); SArray* pNew = taosArrayInit(num, sizeof(SVgroupTableInfo)); From 13dda6b431d4bb5f509dcc2566fe881484bd0fb4 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Wed, 4 Nov 2020 09:17:27 +0000 Subject: [PATCH 35/35] TD-1932 --- src/mnode/src/mnodeDb.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/mnode/src/mnodeDb.c b/src/mnode/src/mnodeDb.c index 812b1c609f..fb4793c056 100644 --- a/src/mnode/src/mnodeDb.c +++ b/src/mnode/src/mnodeDb.c @@ -969,8 +969,13 @@ static SDbCfg mnodeGetAlterDbOption(SDbObj *pDb, SAlterDbMsg *pAlter) { } if (update >= 0 && update != pDb->cfg.update) { +#if 0 mDebug("db:%s, update:%d change to %d", pDb->name, pDb->cfg.update, update); newCfg.update = update; +#else + mError("db:%s, can't alter update option", pDb->name); + terrno = TSDB_CODE_MND_INVALID_DB_OPTION; +#endif } return newCfg;