From 45d78aff8cd1efccc168005555c962d1792da424 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 16 May 2024 11:35:34 +0800 Subject: [PATCH 01/33] enh(query): compress msg before sending. --- include/common/tmsg.h | 2 + source/client/inc/clientInt.h | 3 +- source/client/src/clientEnv.c | 1 + source/client/src/clientImpl.c | 43 +++++++++++++++++---- source/client/src/clientMsgHandler.c | 2 + source/libs/command/src/command.c | 1 + source/libs/executor/src/dataDispatcher.c | 2 + source/libs/executor/src/exchangeoperator.c | 12 ++++++ source/libs/qworker/src/qworker.c | 19 +++++++++ 9 files changed, 76 insertions(+), 9 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index ed23290be4..96001afc91 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2115,6 +2115,7 @@ typedef struct { int8_t precision; int8_t compressed; int8_t streamBlockType; + int32_t payloadLen; int32_t compLen; int32_t numOfBlocks; int64_t numOfRows; // from int32_t change to int64_t @@ -2143,6 +2144,7 @@ typedef struct { int8_t compressed; int32_t compLen; int32_t numOfRows; + int32_t fullLen; char data[]; } SRetrieveMetaTableRsp; diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 9507472df0..e06c7e63ac 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -195,8 +195,9 @@ typedef struct SReqResultInfo { uint64_t current; bool localResultFetched; bool completed; - int32_t precision; bool convertUcs4; + char* decompressBuf; + int32_t precision; int32_t payloadLen; char* convertJson; } SReqResultInfo; diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 439103e5c4..38a895e4fd 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -370,6 +370,7 @@ void doFreeReqResultInfo(SReqResultInfo *pResInfo) { taosMemoryFreeClear(pResInfo->fields); taosMemoryFreeClear(pResInfo->userFields); taosMemoryFreeClear(pResInfo->convertJson); + taosMemoryFreeClear(pResInfo->decompressBuf); if (pResInfo->convertBuf != NULL) { for (int32_t i = 0; i < pResInfo->numOfCols; ++i) { diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 1d5bc9cfb4..6f1c3fccd7 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1711,10 +1711,8 @@ void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4) } SReqResultInfo* pResInfo = &pRequest->body.resInfo; - SSchedulerReq req = { - .syncReq = true, - .pFetchRes = (void**)&pResInfo->pData, - }; + SSchedulerReq req = { .syncReq = true, .pFetchRes = (void**)&pResInfo->pData }; + pRequest->code = schedulerFetchRows(pRequest->body.queryJob, &req); if (pRequest->code != TSDB_CODE_SUCCESS) { pResultInfo->numOfRows = 0; @@ -2195,17 +2193,46 @@ int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableR taosMemoryFreeClear(pResultInfo->pRspMsg); pResultInfo->pRspMsg = (const char*)pRsp; - pResultInfo->pData = (void*)pRsp->data; pResultInfo->numOfRows = htobe64(pRsp->numOfRows); pResultInfo->current = 0; pResultInfo->completed = (pRsp->completed == 1); - pResultInfo->payloadLen = htonl(pRsp->compLen); pResultInfo->precision = pRsp->precision; + // decompress data if needed + if (pRsp->compressed) { + int32_t payloadLen = htonl(pRsp->payloadLen); + + if (pResultInfo->decompressBuf == NULL) { + pResultInfo->decompressBuf = taosMemoryMalloc(payloadLen); + } else { + char* p = taosMemoryRealloc(pResultInfo->decompressBuf, payloadLen); + if (p == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tscError("failed to prepare the decompress buffer, size:%d", payloadLen); + return terrno; + } + + pResultInfo->decompressBuf = p; + } + + int32_t len = tsDecompressString((void*)pRsp->data, htonl(pRsp->compLen), 1, pResultInfo->decompressBuf, payloadLen, + ONE_STAGE_COMP, NULL, 0); + ASSERT(len == payloadLen); + + pResultInfo->pData = pResultInfo->decompressBuf; + pResultInfo->payloadLen = payloadLen; + } else { + pResultInfo->pData = (void*)pRsp->data; + pResultInfo->payloadLen = htonl(pRsp->compLen); + ASSERT(pRsp->compLen == pRsp->payloadLen); + } + // TODO handle the compressed case pResultInfo->totalRows += pResultInfo->numOfRows; - return setResultDataPtr(pResultInfo, pResultInfo->fields, pResultInfo->numOfCols, pResultInfo->numOfRows, - convertUcs4); + + int32_t code = + setResultDataPtr(pResultInfo, pResultInfo->fields, pResultInfo->numOfCols, pResultInfo->numOfRows, convertUcs4); + return code; } TSDB_SERVER_STATUS taos_check_server_status(const char* fqdn, int port, char* details, int maxlen) { diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 360a346fb7..3e178c876f 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -511,6 +511,7 @@ static int32_t buildShowVariablesRsp(SArray* pVars, SRetrieveTableRsp** pRsp) { (*pRsp)->precision = 0; (*pRsp)->compressed = 0; (*pRsp)->compLen = 0; + (*pRsp)->payloadLen = 0; (*pRsp)->numOfRows = htobe64((int64_t)pBlock->info.rows); (*pRsp)->numOfCols = htonl(SHOW_VARIABLES_RESULT_COLS); @@ -623,6 +624,7 @@ static int32_t buildRetriveTableRspForCompactDb(SCompactDbRsp* pCompactDb, SRetr (*pRsp)->precision = 0; (*pRsp)->compressed = 0; (*pRsp)->compLen = 0; + (*pRsp)->payloadLen = 0; (*pRsp)->numOfRows = htobe64((int64_t)pBlock->info.rows); (*pRsp)->numOfCols = htonl(COMPACT_DB_RESULT_COLS); diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index d19b4c6913..8f3263cbeb 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -37,6 +37,7 @@ static int32_t buildRetrieveTableRsp(SSDataBlock* pBlock, int32_t numOfCols, SRe (*pRsp)->precision = 0; (*pRsp)->compressed = 0; (*pRsp)->compLen = 0; + (*pRsp)->payloadLen = 0; (*pRsp)->numOfRows = htobe64((int64_t)pBlock->info.rows); (*pRsp)->numOfCols = htonl(numOfCols); diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index abe566473f..b9a586c52c 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -63,12 +63,14 @@ typedef struct SDataDispatchHandle { static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pInput, SDataDispatchBuf* pBuf) { int32_t numOfCols = 0; SNode* pNode; + FOREACH(pNode, pHandle->pSchema->pSlots) { SSlotDescNode* pSlotDesc = (SSlotDescNode*)pNode; if (pSlotDesc->output) { ++numOfCols; } } + SDataCacheEntry* pEntry = (SDataCacheEntry*)pBuf->pData; pEntry->compressed = 0; pEntry->numOfRows = pInput->pData->info.rows; diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index 06dd43e170..72d0af1726 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -410,6 +410,7 @@ int32_t loadRemoteDataCallback(void* param, SDataBuf* pMsg, int32_t code) { SRetrieveTableRsp* pRsp = pSourceDataInfo->pRsp; pRsp->numOfRows = htobe64(pRsp->numOfRows); pRsp->compLen = htonl(pRsp->compLen); + pRsp->payloadLen = htonl(pRsp->payloadLen); pRsp->numOfCols = htonl(pRsp->numOfCols); pRsp->useconds = htobe64(pRsp->useconds); pRsp->numOfBlocks = htonl(pRsp->numOfBlocks); @@ -674,6 +675,16 @@ int32_t doExtractResultBlocks(SExchangeInfo* pExchangeInfo, SSourceDataInfo* pDa char* pStart = pRetrieveRsp->data; int32_t index = 0; int32_t code = 0; + char* p = NULL; + + if (pRetrieveRsp->compressed) { // decompress the data + p = taosMemoryMalloc(pRetrieveRsp->payloadLen); + int32_t t = tsDecompressString(pRetrieveRsp->data, pRetrieveRsp->compLen, 1, p, pRetrieveRsp->payloadLen, + ONE_STAGE_COMP, NULL, 0); + ASSERT(t == pRetrieveRsp->payloadLen); + pStart = p; + } + while (index++ < pRetrieveRsp->numOfBlocks) { SSDataBlock* pb = NULL; if (taosArrayGetSize(pExchangeInfo->pRecycledBlocks) > 0) { @@ -692,6 +703,7 @@ int32_t doExtractResultBlocks(SExchangeInfo* pExchangeInfo, SSourceDataInfo* pDa taosArrayPush(pExchangeInfo->pResultBlockList, &pb); } + taosMemoryFreeClear(p); return code; } diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 594bb205d8..39fdcdee6f 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -904,6 +904,25 @@ int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg) { bool qComplete = (DS_BUF_EMPTY == sOutput.bufStatus && sOutput.queryEnd); qwBuildFetchRsp(rsp, &sOutput, dataLen, qComplete); + + { + if (dataLen > 8192) { + char* p = taosMemoryMalloc(dataLen); + int32_t len = + tsCompressString(((SRetrieveTableRsp *)rsp)->data, dataLen, 1, p, dataLen, ONE_STAGE_COMP, NULL, 0); + + memcpy(((SRetrieveTableRsp*)rsp)->data, p, len); + ((SRetrieveTableRsp*)rsp)->payloadLen = htonl(dataLen); + ((SRetrieveTableRsp*)rsp)->compLen = htonl(len); + + ((SRetrieveTableRsp*)rsp)->compressed = 1; + taosMemoryFree(p); + } else { + ((SRetrieveTableRsp*)rsp)->payloadLen = ((SRetrieveTableRsp*)rsp)->compLen; + ((SRetrieveTableRsp*)rsp)->compressed = 0; + } + } + if (qComplete) { atomic_store_8((int8_t *)&ctx->queryEnd, true); } From 4aeb3c5c90445859adbeaad55b3e0ba6021ae6ce Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 16 May 2024 13:30:34 +0800 Subject: [PATCH 02/33] fix(query): remove invalid assert. --- source/client/src/clientImpl.c | 1 - 1 file changed, 1 deletion(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 6f1c3fccd7..bb55ae4437 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -2224,7 +2224,6 @@ int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableR } else { pResultInfo->pData = (void*)pRsp->data; pResultInfo->payloadLen = htonl(pRsp->compLen); - ASSERT(pRsp->compLen == pRsp->payloadLen); } // TODO handle the compressed case From 92b247e5b4d6082a92122feacbf3b6eaa517b915 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 16 May 2024 15:55:48 +0800 Subject: [PATCH 03/33] refactor: move compress data in query threads. --- include/libs/executor/dataSinkMgt.h | 2 +- source/client/src/clientImpl.c | 3 +- source/client/test/clientTests.cpp | 57 +++++++++-------------- source/libs/command/src/explain.c | 1 + source/libs/executor/inc/dataSinkInt.h | 2 +- source/libs/executor/src/dataDeleter.c | 5 +- source/libs/executor/src/dataDispatcher.c | 26 +++++++++-- source/libs/executor/src/dataInserter.c | 2 +- source/libs/executor/src/dataSinkMgt.c | 4 +- source/libs/qworker/inc/qwMsg.h | 2 +- source/libs/qworker/src/qwMsg.c | 3 +- source/libs/qworker/src/qworker.c | 48 +++++++++++-------- 12 files changed, 87 insertions(+), 68 deletions(-) diff --git a/include/libs/executor/dataSinkMgt.h b/include/libs/executor/dataSinkMgt.h index 2e2882e35e..a052bc3359 100644 --- a/include/libs/executor/dataSinkMgt.h +++ b/include/libs/executor/dataSinkMgt.h @@ -104,7 +104,7 @@ void dsReset(DataSinkHandle handle); * @param handle * @param pLen data length */ -void dsGetDataLength(DataSinkHandle handle, int64_t* pLen, bool* pQueryEnd); +void dsGetDataLength(DataSinkHandle handle, int64_t* pLen, int64_t* pRawLen, bool* pQueryEnd); /** * Get data, the caller needs to allocate data memory. diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index bb55ae4437..833722d86f 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -2217,13 +2217,14 @@ int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableR int32_t len = tsDecompressString((void*)pRsp->data, htonl(pRsp->compLen), 1, pResultInfo->decompressBuf, payloadLen, ONE_STAGE_COMP, NULL, 0); - ASSERT(len == payloadLen); +// ASSERT(len == payloadLen); pResultInfo->pData = pResultInfo->decompressBuf; pResultInfo->payloadLen = payloadLen; } else { pResultInfo->pData = (void*)pRsp->data; pResultInfo->payloadLen = htonl(pRsp->compLen); + ASSERT(pRsp->compLen == pRsp->payloadLen); } // TODO handle the compressed case diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 1498476634..b5710ffb99 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -821,32 +821,21 @@ TEST(clientCase, projection_query_tables) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(pConn, nullptr); + TAOS_RES* pRes = NULL; + // TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 1"); // if (taos_errno(pRes) != 0) { // printf("error in create db, reason:%s\n", taos_errstr(pRes)); // } // taos_free_result(pRes); - - TAOS_RES* pRes = taos_query(pConn, "alter local 'fqdn 127.0.0.1'"); - if (taos_errno(pRes) != 0) { - printf("failed to exec query, %s\n", taos_errstr(pRes)); - } - - taos_free_result(pRes); - - pRes = taos_query(pConn, "select last(ts), ts from cache_1.t1"); +/* + TAOS_RES* pRes = taos_query(pConn, "select last(ts), ts from cache_1.t1"); // pRes = taos_query(pConn, "select last(ts), ts from cache_1.no_pk_t1"); if (taos_errno(pRes) != 0) { - printf("failed to exec query, %s\n", taos_errstr(pRes)); + printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); } taos_free_result(pRes); -// pRes = taos_query(pConn, "create stream stream_1 trigger at_once fill_history 1 ignore expired 0 into str_res1 as select _wstart as ts, count(*) from stable_1 interval(10s);"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); - pRes = taos_query(pConn, "create table tu using st2 tags(2)"); if (taos_errno(pRes) != 0) { printf("failed to create table tu, reason:%s\n", taos_errstr(pRes)); @@ -876,27 +865,27 @@ TEST(clientCase, projection_query_tables) { for(int32_t j = 0; j < 1; ++j) { start += 20; for (int32_t i = 0; i < 1; ++i) { - createNewTable(pConn, i, 100, start, pstr); + createNewTable(pConn, i, 100000, 0, pstr); } } +*/ - // - // pRes = taos_query(pConn, "select * from tu"); - // if (taos_errno(pRes) != 0) { - // printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); - // taos_free_result(pRes); - // ASSERT_TRUE(false); - // } - // - // TAOS_ROW pRow = NULL; - // TAOS_FIELD* pFields = taos_fetch_fields(pRes); - // int32_t numOfFields = taos_num_fields(pRes); - // - // char str[512] = {0}; - // while ((pRow = taos_fetch_row(pRes)) != NULL) { - // int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - // printf("%s\n", str); - // } + pRes = taos_query(pConn, "select * from abc1.st2"); + if (taos_errno(pRes) != 0) { + printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + ASSERT_TRUE(false); + } + + TAOS_ROW pRow = NULL; + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + char str[512] = {0}; + while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); + } taos_free_result(pRes); taos_close(pConn); diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index 5afc629865..bde0b47489 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -1957,6 +1957,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { int32_t len = blockEncode(pBlock, rsp->data, taosArrayGetSize(pBlock->pDataBlock)); rsp->compLen = htonl(len); + rsp->payloadLen = rsp->compLen; blockDataDestroy(pBlock); diff --git a/source/libs/executor/inc/dataSinkInt.h b/source/libs/executor/inc/dataSinkInt.h index dcebd2c6fd..a294d7e487 100644 --- a/source/libs/executor/inc/dataSinkInt.h +++ b/source/libs/executor/inc/dataSinkInt.h @@ -36,7 +36,7 @@ typedef struct SDataSinkManager { typedef int32_t (*FPutDataBlock)(struct SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue); typedef void (*FEndPut)(struct SDataSinkHandle* pHandle, uint64_t useconds); typedef void (*FReset)(struct SDataSinkHandle* pHandle); -typedef void (*FGetDataLength)(struct SDataSinkHandle* pHandle, int64_t* pLen, bool* pQueryEnd); +typedef void (*FGetDataLength)(struct SDataSinkHandle* pHandle, int64_t* pLen, int64_t* pRowLen, bool* pQueryEnd); typedef int32_t (*FGetDataBlock)(struct SDataSinkHandle* pHandle, SOutputData* pOutput); typedef int32_t (*FDestroyDataSinker)(struct SDataSinkHandle* pHandle); typedef int32_t (*FGetCacheSize)(struct SDataSinkHandle* pHandle, uint64_t* size); diff --git a/source/libs/executor/src/dataDeleter.c b/source/libs/executor/src/dataDeleter.c index 960ae14fcf..15288c4406 100644 --- a/source/libs/executor/src/dataDeleter.c +++ b/source/libs/executor/src/dataDeleter.c @@ -154,7 +154,7 @@ static void endPut(struct SDataSinkHandle* pHandle, uint64_t useconds) { taosThreadMutexUnlock(&pDeleter->mutex); } -static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, bool* pQueryEnd) { +static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, int64_t* pRawLen, bool* pQueryEnd) { SDataDeleterHandle* pDeleter = (SDataDeleterHandle*)pHandle; if (taosQueueEmpty(pDeleter->pDataBlocks)) { *pQueryEnd = pDeleter->queryEnd; @@ -171,6 +171,8 @@ static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, bool* pQueryE SDataCacheEntry* pEntry = (SDataCacheEntry*)pDeleter->nextOutput.pData; *pLen = pEntry->dataLen; + *pRawLen = pEntry->dataLen; + *pQueryEnd = pDeleter->queryEnd; qDebug("got data len %" PRId64 ", row num %d in sink", *pLen, ((SDataCacheEntry*)(pDeleter->nextOutput.pData))->numOfRows); @@ -186,6 +188,7 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { pOutput->queryEnd = pDeleter->queryEnd; return TSDB_CODE_SUCCESS; } + SDataCacheEntry* pEntry = (SDataCacheEntry*)(pDeleter->nextOutput.pData); memcpy(pOutput->pData, pEntry->data, pEntry->dataLen); pDeleter->pParam->pUidList = NULL; diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index b9a586c52c..2c127fba12 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -31,6 +31,7 @@ typedef struct SDataDispatchBuf { } SDataDispatchBuf; typedef struct SDataCacheEntry { + int32_t rawLen; int32_t dataLen; int32_t numOfRows; int32_t numOfCols; @@ -78,10 +79,26 @@ static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pIn pEntry->dataLen = 0; pBuf->useSize = sizeof(SDataCacheEntry); - pEntry->dataLen = blockEncode(pInput->pData, pEntry->data, numOfCols); // ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data + 8)); // ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data + 8 + 4)); + { + if (pBuf->allocSize > 8192) { + char* p = taosMemoryMalloc(pBuf->allocSize); + int32_t dataLen = blockEncode(pInput->pData, p, numOfCols); + + int32_t len = tsCompressString(p, dataLen, 1, pEntry->data, pBuf->allocSize, ONE_STAGE_COMP, NULL, 0); + + pEntry->compressed = 1; + pEntry->dataLen = len; + pEntry->rawLen = dataLen; + taosMemoryFree(p); + } else { + pEntry->dataLen = blockEncode(pInput->pData, pEntry->data, numOfCols); + pEntry->rawLen = pEntry->dataLen; + } + } + pBuf->useSize += pEntry->dataLen; atomic_add_fetch_64(&pHandle->cachedSize, pEntry->dataLen); @@ -165,7 +182,7 @@ static void resetDispatcher(struct SDataSinkHandle* pHandle) { taosThreadMutexUnlock(&pDispatcher->mutex); } -static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, bool* pQueryEnd) { +static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, int64_t* pRowLen, bool* pQueryEnd) { SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; if (taosQueueEmpty(pDispatcher->pDataBlocks)) { *pQueryEnd = pDispatcher->queryEnd; @@ -182,9 +199,7 @@ static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, bool* pQueryE SDataCacheEntry* pEntry = (SDataCacheEntry*)pDispatcher->nextOutput.pData; *pLen = pEntry->dataLen; - - // ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data + 8)); - // ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data + 8 + 4)); + *pRowLen = pEntry->rawLen; *pQueryEnd = pDispatcher->queryEnd; qDebug("got data len %" PRId64 ", row num %d in sink", *pLen, @@ -202,6 +217,7 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { pOutput->queryEnd = pDispatcher->queryEnd; return TSDB_CODE_SUCCESS; } + SDataCacheEntry* pEntry = (SDataCacheEntry*)(pDispatcher->nextOutput.pData); memcpy(pOutput->pData, pEntry->data, pEntry->dataLen); pOutput->numOfRows = pEntry->numOfRows; diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index 39bbc1bc69..5ba2f8bf42 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -370,7 +370,7 @@ static void endPut(struct SDataSinkHandle* pHandle, uint64_t useconds) { taosThreadMutexUnlock(&pInserter->mutex); } -static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, bool* pQueryEnd) { +static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, int64_t* pRawLen, bool* pQueryEnd) { SDataInserterHandle* pDispatcher = (SDataInserterHandle*)pHandle; *pLen = pDispatcher->submitRes.affectedRows; qDebug("got total affectedRows %" PRId64, *pLen); diff --git a/source/libs/executor/src/dataSinkMgt.c b/source/libs/executor/src/dataSinkMgt.c index 2a59bbf1dc..16d0d8f340 100644 --- a/source/libs/executor/src/dataSinkMgt.c +++ b/source/libs/executor/src/dataSinkMgt.c @@ -75,9 +75,9 @@ void dsReset(DataSinkHandle handle) { } } -void dsGetDataLength(DataSinkHandle handle, int64_t* pLen, bool* pQueryEnd) { +void dsGetDataLength(DataSinkHandle handle, int64_t* pLen, int64_t* pRawLen, bool* pQueryEnd) { SDataSinkHandle* pHandleImpl = (SDataSinkHandle*)handle; - pHandleImpl->fGetLen(pHandleImpl, pLen, pQueryEnd); + pHandleImpl->fGetLen(pHandleImpl, pLen, pRawLen, pQueryEnd); } int32_t dsGetDataBlock(DataSinkHandle handle, SOutputData* pOutput) { diff --git a/source/libs/qworker/inc/qwMsg.h b/source/libs/qworker/inc/qwMsg.h index ae68f69802..6d57a3df46 100644 --- a/source/libs/qworker/inc/qwMsg.h +++ b/source/libs/qworker/inc/qwMsg.h @@ -38,7 +38,7 @@ int32_t qwBuildAndSendDropRsp(SRpcHandleInfo *pConn, int32_t code); int32_t qwBuildAndSendCancelRsp(SRpcHandleInfo *pConn, int32_t code); int32_t qwBuildAndSendFetchRsp(int32_t rspType, SRpcHandleInfo *pConn, SRetrieveTableRsp *pRsp, int32_t dataLength, int32_t code); -void qwBuildFetchRsp(void *msg, SOutputData *input, int32_t len, bool qComplete); +void qwBuildFetchRsp(void *msg, SOutputData *input, int32_t len, int32_t rawDataLen, bool qComplete); int32_t qwBuildAndSendCQueryMsg(QW_FPARAMS_DEF, SRpcHandleInfo *pConn); int32_t qwBuildAndSendQueryRsp(int32_t rspType, SRpcHandleInfo *pConn, int32_t code, SQWTaskCtx *ctx); int32_t qwBuildAndSendExplainRsp(SRpcHandleInfo *pConn, SArray *pExecList); diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index faa90dcbf8..e2a99a11e7 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -29,13 +29,14 @@ int32_t qwMallocFetchRsp(int8_t rpcMalloc, int32_t length, SRetrieveTableRsp **r return TSDB_CODE_SUCCESS; } -void qwBuildFetchRsp(void *msg, SOutputData *input, int32_t len, bool qComplete) { +void qwBuildFetchRsp(void *msg, SOutputData *input, int32_t len, int32_t rawDataLen, bool qComplete) { SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)msg; rsp->useconds = htobe64(input->useconds); rsp->completed = qComplete; rsp->precision = input->precision; rsp->compressed = input->compressed; + rsp->payloadLen = htonl(rawDataLen); rsp->compLen = htonl(len); rsp->numOfRows = htobe64(input->numOfRows); rsp->numOfCols = htonl(input->numOfCols); diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 39fdcdee6f..2a24deafd2 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -102,7 +102,7 @@ int32_t qwHandleTaskComplete(QW_FPARAMS_DEF, SQWTaskCtx *ctx) { } if (!ctx->needFetch) { - dsGetDataLength(ctx->sinkHandle, &ctx->affectedRows, NULL); + dsGetDataLength(ctx->sinkHandle, &ctx->affectedRows, NULL, NULL); } } @@ -285,8 +285,10 @@ int32_t qwGenerateSchHbRsp(SQWorker *mgmt, SQWSchStatus *sch, SQWHbInfo *hbInfo) return TSDB_CODE_SUCCESS; } -int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, void **rspMsg, SOutputData *pOutput) { +int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, int32_t *pRawDataLen, void **rspMsg, + SOutputData *pOutput) { int64_t len = 0; + int64_t rawLen = 0; SRetrieveTableRsp *rsp = NULL; bool queryEnd = false; int32_t code = 0; @@ -300,7 +302,7 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, *dataLen = 0; while (true) { - dsGetDataLength(ctx->sinkHandle, &len, &queryEnd); + dsGetDataLength(ctx->sinkHandle, &len, &rawLen, &queryEnd); if (len < 0) { QW_TASK_ELOG("invalid length from dsGetDataLength, length:%" PRId64 "", len); @@ -343,6 +345,7 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, QW_TASK_DLOG("there are data in sink, dataLength:%" PRId64 "", len); *dataLen += len; + *pRawDataLen += rawLen; QW_ERR_RET(qwMallocFetchRsp(!ctx->localExec, *dataLen, &rsp)); @@ -375,11 +378,11 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, break; } - if (pOutput->numOfRows >= QW_MIN_RES_ROWS) { - QW_TASK_DLOG("task fetched blocks %d rows %" PRId64 " reaches the min rows", pOutput->numOfBlocks, - pOutput->numOfRows); +// if (pOutput->numOfRows >= QW_MIN_RES_ROWS) { +// QW_TASK_DLOG("task fetched blocks %d rows %" PRId64 " reaches the min rows", pOutput->numOfBlocks, +// pOutput->numOfRows); break; - } +// } } *rspMsg = rsp; @@ -389,11 +392,12 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, int32_t qwGetDeleteResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, SDeleteRes *pRes) { int64_t len = 0; + int64_t rawLen = 0; bool queryEnd = false; int32_t code = 0; SOutputData output = {0}; - dsGetDataLength(ctx->sinkHandle, &len, &queryEnd); + dsGetDataLength(ctx->sinkHandle, &len, &rawLen, &queryEnd); if (len <= 0 || len != sizeof(SDeleterRes)) { QW_TASK_ELOG("invalid length from dsGetDataLength, length:%" PRId64, len); @@ -433,9 +437,10 @@ int32_t qwQuickRspFetchReq(QW_FPARAMS_DEF, SQWTaskCtx * ctx, SQWMsg *qwMsg, i if (QW_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { void *rsp = NULL; int32_t dataLen = 0; + int32_t rawLen = 0; SOutputData sOutput = {0}; if (TSDB_CODE_SUCCESS == code) { - code = qwGetQueryResFromSink(QW_FPARAMS(), ctx, &dataLen, &rsp, &sOutput); + code = qwGetQueryResFromSink(QW_FPARAMS(), ctx, &dataLen, &rawLen, &rsp, &sOutput); } if (NULL == rsp && TSDB_CODE_SUCCESS == code) { @@ -445,7 +450,7 @@ int32_t qwQuickRspFetchReq(QW_FPARAMS_DEF, SQWTaskCtx * ctx, SQWMsg *qwMsg, i if (NULL != rsp) { bool qComplete = (DS_BUF_EMPTY == sOutput.bufStatus && sOutput.queryEnd); - qwBuildFetchRsp(rsp, &sOutput, dataLen, qComplete); + qwBuildFetchRsp(rsp, &sOutput, dataLen, rawLen, qComplete); if (qComplete) { atomic_store_8((int8_t *)&ctx->queryEnd, true); } @@ -791,6 +796,7 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { SQWPhaseInput input = {0}; void *rsp = NULL; int32_t dataLen = 0; + int32_t rawLen = 0; bool queryStop = false; bool qComplete = false; @@ -810,7 +816,7 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { if (QW_EVENT_RECEIVED(ctx, QW_EVENT_FETCH)) { SOutputData sOutput = {0}; - QW_ERR_JRET(qwGetQueryResFromSink(QW_FPARAMS(), ctx, &dataLen, &rsp, &sOutput)); + QW_ERR_JRET(qwGetQueryResFromSink(QW_FPARAMS(), ctx, &dataLen, &rawLen, &rsp, &sOutput)); if ((!sOutput.queryEnd) && (DS_BUF_LOW == sOutput.bufStatus || DS_BUF_EMPTY == sOutput.bufStatus)) { QW_TASK_DLOG("task not end and buf is %s, need to continue query", qwBufStatusStr(sOutput.bufStatus)); @@ -821,7 +827,7 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { if (rsp) { qComplete = (DS_BUF_EMPTY == sOutput.bufStatus && sOutput.queryEnd); - qwBuildFetchRsp(rsp, &sOutput, dataLen, qComplete); + qwBuildFetchRsp(rsp, &sOutput, dataLen, rawLen, qComplete); if (qComplete) { atomic_store_8((int8_t *)&ctx->queryEnd, true); atomic_store_8((int8_t *)&ctx->queryContinue, 0); @@ -878,6 +884,8 @@ int32_t qwProcessCQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg) { int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg) { int32_t code = 0; int32_t dataLen = 0; + int32_t rawDataLen = 0; + bool locked = false; SQWTaskCtx *ctx = NULL; void *rsp = NULL; @@ -896,17 +904,17 @@ int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg) { } SOutputData sOutput = {0}; - QW_ERR_JRET(qwGetQueryResFromSink(QW_FPARAMS(), ctx, &dataLen, &rsp, &sOutput)); + QW_ERR_JRET(qwGetQueryResFromSink(QW_FPARAMS(), ctx, &dataLen, &rawDataLen, &rsp, &sOutput)); if (NULL == rsp) { QW_SET_EVENT_RECEIVED(ctx, QW_EVENT_FETCH); } else { bool qComplete = (DS_BUF_EMPTY == sOutput.bufStatus && sOutput.queryEnd); - qwBuildFetchRsp(rsp, &sOutput, dataLen, qComplete); + qwBuildFetchRsp(rsp, &sOutput, dataLen, rawDataLen, qComplete); { - if (dataLen > 8192) { + if (/*dataLen > 8192*/ 0) { char* p = taosMemoryMalloc(dataLen); int32_t len = tsCompressString(((SRetrieveTableRsp *)rsp)->data, dataLen, 1, p, dataLen, ONE_STAGE_COMP, NULL, 0); @@ -918,8 +926,8 @@ int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg) { ((SRetrieveTableRsp*)rsp)->compressed = 1; taosMemoryFree(p); } else { - ((SRetrieveTableRsp*)rsp)->payloadLen = ((SRetrieveTableRsp*)rsp)->compLen; - ((SRetrieveTableRsp*)rsp)->compressed = 0; +// ((SRetrieveTableRsp*)rsp)->payloadLen = ((SRetrieveTableRsp*)rsp)->compLen; +// ((SRetrieveTableRsp*)rsp)->compressed = 0; } } @@ -1484,6 +1492,7 @@ int32_t qWorkerProcessLocalFetch(void *pMgmt, uint64_t sId, uint64_t qId, uint64 SQWorker *mgmt = (SQWorker *)pMgmt; int32_t code = 0; int32_t dataLen = 0; + int32_t rawLen = 0; SQWTaskCtx *ctx = NULL; void *rsp = NULL; bool queryStop = false; @@ -1491,7 +1500,6 @@ int32_t qWorkerProcessLocalFetch(void *pMgmt, uint64_t sId, uint64_t qId, uint64 SQWPhaseInput input = {0}; QW_ERR_JRET(qwHandlePrePhaseEvents(QW_FPARAMS(), QW_PHASE_PRE_FETCH, &input, NULL)); - QW_ERR_JRET(qwGetTaskCtx(QW_FPARAMS(), &ctx)); ctx->fetchMsgType = TDMT_SCH_MERGE_FETCH; @@ -1500,7 +1508,7 @@ int32_t qWorkerProcessLocalFetch(void *pMgmt, uint64_t sId, uint64_t qId, uint64 SOutputData sOutput = {0}; while (true) { - QW_ERR_JRET(qwGetQueryResFromSink(QW_FPARAMS(), ctx, &dataLen, &rsp, &sOutput)); + QW_ERR_JRET(qwGetQueryResFromSink(QW_FPARAMS(), ctx, &dataLen, &rawLen, &rsp, &sOutput)); if (NULL == rsp) { QW_ERR_JRET(qwExecTask(QW_FPARAMS(), ctx, &queryStop)); @@ -1509,7 +1517,7 @@ int32_t qWorkerProcessLocalFetch(void *pMgmt, uint64_t sId, uint64_t qId, uint64 } else { bool qComplete = (DS_BUF_EMPTY == sOutput.bufStatus && sOutput.queryEnd); - qwBuildFetchRsp(rsp, &sOutput, dataLen, qComplete); + qwBuildFetchRsp(rsp, &sOutput, dataLen, rawLen, qComplete); if (qComplete) { atomic_store_8((int8_t *)&ctx->queryEnd, true); } From 76a296ca521b2125310af75e3308f62c0b5a5a15 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 16 May 2024 19:04:37 +0800 Subject: [PATCH 04/33] fix(query): enable the assert. --- source/client/src/clientImpl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 833722d86f..6f1c3fccd7 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -2217,7 +2217,7 @@ int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableR int32_t len = tsDecompressString((void*)pRsp->data, htonl(pRsp->compLen), 1, pResultInfo->decompressBuf, payloadLen, ONE_STAGE_COMP, NULL, 0); -// ASSERT(len == payloadLen); + ASSERT(len == payloadLen); pResultInfo->pData = pResultInfo->decompressBuf; pResultInfo->payloadLen = payloadLen; From 66254c96a7c4b4f118eb65ad1c702ac5ade55804 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 16 May 2024 19:26:07 +0800 Subject: [PATCH 05/33] fix(query):remove invalid assert. --- source/client/test/clientTests.cpp | 13 ++++++++----- source/libs/executor/src/exchangeoperator.c | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index b5710ffb99..22eacdc291 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -901,13 +901,13 @@ TEST(clientCase, tsbs_perf_test) { } TEST(clientCase, projection_query_stables) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + TAOS* pConn = taos_connect("192.168.1.53", "root", "taosdata", NULL, 0); ASSERT_NE(pConn, nullptr); - TAOS_RES* pRes = taos_query(pConn, "use test"); + TAOS_RES* pRes = taos_query(pConn, "use qzdata"); taos_free_result(pRes); - pRes = taos_query(pConn, "select * from meters limit 50000000"); + pRes = taos_query(pConn, "select * from `X212MGPH018`"); if (taos_errno(pRes) != 0) { printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); taos_free_result(pRes); @@ -918,10 +918,13 @@ TEST(clientCase, projection_query_stables) { TAOS_FIELD* pFields = taos_fetch_fields(pRes); int32_t numOfFields = taos_num_fields(pRes); - char str[512] = {0}; + int32_t i = 0; + char str[512] = {0}; while ((pRow = taos_fetch_row(pRes)) != NULL) { // int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - // printf("%s\n", str); + if (i++ % 100000 == 0) { + printf("%d\n", i); + } } taos_free_result(pRes); diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index 72d0af1726..a563897122 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -681,7 +681,7 @@ int32_t doExtractResultBlocks(SExchangeInfo* pExchangeInfo, SSourceDataInfo* pDa p = taosMemoryMalloc(pRetrieveRsp->payloadLen); int32_t t = tsDecompressString(pRetrieveRsp->data, pRetrieveRsp->compLen, 1, p, pRetrieveRsp->payloadLen, ONE_STAGE_COMP, NULL, 0); - ASSERT(t == pRetrieveRsp->payloadLen); +// ASSERT(t == pRetrieveRsp->payloadLen); pStart = p; } From 5307f157667db6c021a0c2d99b14658e80da4835 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 16 May 2024 19:54:29 +0800 Subject: [PATCH 06/33] refactor: do some internal refactor. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 4 +--- source/libs/executor/src/dataDispatcher.c | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index c25deb0d66..9d08ff4e04 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -4242,13 +4242,11 @@ int32_t tsdbReaderOpen2(void* pVnode, SQueryTableDataCond* pCond, void* pTableLi SVnodeCfg* pConf = &(((SVnode*)pVnode)->config); int32_t capacity = pConf->tsdbCfg.maxRows; + capacity = 8192; // for debug purpose if (pResBlock != NULL) { blockDataEnsureCapacity(pResBlock, capacity); } - // for debug purpose -// capacity = 7; - int32_t code = tsdbReaderCreate(pVnode, pCond, ppReader, capacity, pResBlock, idstr); if (code != TSDB_CODE_SUCCESS) { goto _err; diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index 2c127fba12..e2126f1fc4 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -83,7 +83,7 @@ static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pIn // ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data + 8 + 4)); { - if (pBuf->allocSize > 8192) { + if (/*pBuf->allocSize > 8192*/ 0) { char* p = taosMemoryMalloc(pBuf->allocSize); int32_t dataLen = blockEncode(pInput->pData, p, numOfCols); From 63d9c82b87aa9fdf383b0e86be57bd37dab111a8 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 16 May 2024 23:25:01 +0800 Subject: [PATCH 07/33] fix(query): remove assert. --- source/client/src/clientImpl.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 6f1c3fccd7..52a776f66b 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -2224,7 +2224,7 @@ int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableR } else { pResultInfo->pData = (void*)pRsp->data; pResultInfo->payloadLen = htonl(pRsp->compLen); - ASSERT(pRsp->compLen == pRsp->payloadLen); +// ASSERT(pRsp->compLen == pRsp->payloadLen); } // TODO handle the compressed case From 6e1fe0ae38cdf5296c2c1819adc9bf72c682d4d6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Thu, 16 May 2024 23:30:35 +0800 Subject: [PATCH 08/33] refactor: do some internal refactor. --- source/libs/executor/src/dataSinkMgt.c | 3 ++- source/libs/executor/src/executor.c | 2 +- source/libs/qworker/inc/qwInt.h | 2 +- source/libs/qworker/src/qworker.c | 8 ++++---- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/source/libs/executor/src/dataSinkMgt.c b/source/libs/executor/src/dataSinkMgt.c index 16d0d8f340..e711ffdf5c 100644 --- a/source/libs/executor/src/dataSinkMgt.c +++ b/source/libs/executor/src/dataSinkMgt.c @@ -18,13 +18,14 @@ #include "planner.h" #include "tarray.h" -SDataSinkStat gDataSinkStat = {0}; +SDataSinkStat gDataSinkStat = {0}; int32_t dsDataSinkMgtInit(SDataSinkMgtCfg* cfg, SStorageAPI* pAPI, void** ppSinkManager) { SDataSinkManager* pSinkManager = taosMemoryMalloc(sizeof(SDataSinkManager)); if (NULL == pSinkManager) { return TSDB_CODE_OUT_OF_MEMORY; } + pSinkManager->cfg = *cfg; pSinkManager->pAPI = pAPI; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 29f667cb66..889a9b62e1 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -537,7 +537,7 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, } if (handle) { - SDataSinkMgtCfg cfg = {.maxDataBlockNum = 500, .maxDataBlockNumPerQuery = 50}; + SDataSinkMgtCfg cfg = {.maxDataBlockNum = 5000, .maxDataBlockNumPerQuery = 500}; void* pSinkManager = NULL; code = dsDataSinkMgtInit(&cfg, &(*pTask)->storageAPI, &pSinkManager); if (code != TSDB_CODE_SUCCESS) { diff --git a/source/libs/qworker/inc/qwInt.h b/source/libs/qworker/inc/qwInt.h index b4bd1943c5..2222f9cb31 100644 --- a/source/libs/qworker/inc/qwInt.h +++ b/source/libs/qworker/inc/qwInt.h @@ -35,7 +35,7 @@ extern "C" { #define QW_DEFAULT_SHORT_RUN_TIMES 2 #define QW_DEFAULT_HEARTBEAT_MSEC 5000 #define QW_SCH_TIMEOUT_MSEC 180000 -#define QW_MIN_RES_ROWS 4096 +#define QW_MIN_RES_ROWS 16384 enum { QW_PHASE_PRE_QUERY = 1, diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 2a24deafd2..3fee7d94b9 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -378,11 +378,11 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, break; } -// if (pOutput->numOfRows >= QW_MIN_RES_ROWS) { -// QW_TASK_DLOG("task fetched blocks %d rows %" PRId64 " reaches the min rows", pOutput->numOfBlocks, -// pOutput->numOfRows); + if (pOutput->numOfRows >= QW_MIN_RES_ROWS) { + QW_TASK_DLOG("task fetched blocks %d rows %" PRId64 " reaches the min rows", pOutput->numOfBlocks, + pOutput->numOfRows); break; -// } + } } *rspMsg = rsp; From b669c7b4640090d3060434c173e9d370a0e295f6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 17 May 2024 00:11:44 +0800 Subject: [PATCH 09/33] fix(test): update test case. --- source/client/test/clientTests.cpp | 17 +++++++++++++---- source/util/test/CMakeLists.txt | 2 +- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 22eacdc291..449e44eb13 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -918,14 +918,23 @@ TEST(clientCase, projection_query_stables) { TAOS_FIELD* pFields = taos_fetch_fields(pRes); int32_t numOfFields = taos_num_fields(pRes); + int32_t numOfRows = 0; int32_t i = 0; char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - // int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - if (i++ % 100000 == 0) { - printf("%d\n", i); + while (1) { + int32_t c = taos_fetch_block_s(pRes, &numOfRows, &pRow); + if (numOfRows <= 0) { + break; } + i += numOfRows; + printf("%d\n", i); } +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// if (i++ % 100000 == 0) { +// printf("%d\n", i); +// } +// } taos_free_result(pRes); taos_close(pConn); diff --git a/source/util/test/CMakeLists.txt b/source/util/test/CMakeLists.txt index e8e3348343..07981483bb 100644 --- a/source/util/test/CMakeLists.txt +++ b/source/util/test/CMakeLists.txt @@ -13,7 +13,7 @@ IF (HEADER_GTEST_INCLUDE_DIR AND (LIB_GTEST_STATIC_DIR OR LIB_GTEST_SHARED_DIR)) LIST(REMOVE_ITEM SOURCE_LIST ${CMAKE_CURRENT_SOURCE_DIR}/trefTest.c) ADD_EXECUTABLE(utilTest ${SOURCE_LIST}) - TARGET_LINK_LIBRARIES(utilTest util common os gtest pthread) + TARGET_LINK_LIBRARIES(utilTest util common os gtest pthread public taos) LIST(REMOVE_ITEM SOURCE_LIST ${CMAKE_CURRENT_SOURCE_DIR}/cacheTest.cpp) LIST(APPEND SOURCE_LIST ${CMAKE_CURRENT_SOURCE_DIR}/hashTest.cpp) From 149eeecd76d1d25f5b20afbba8816137a8def26f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 17 May 2024 00:37:16 +0800 Subject: [PATCH 10/33] fix(test): update test case. --- source/client/test/clientTests.cpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 449e44eb13..d7c1cbb3d9 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -907,7 +907,7 @@ TEST(clientCase, projection_query_stables) { TAOS_RES* pRes = taos_query(pConn, "use qzdata"); taos_free_result(pRes); - pRes = taos_query(pConn, "select * from `X212MGPH018`"); + pRes = taos_query(pConn, "select * from `QZ_212_DYS_02`"); if (taos_errno(pRes) != 0) { printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); taos_free_result(pRes); @@ -920,6 +920,8 @@ TEST(clientCase, projection_query_stables) { int32_t numOfRows = 0; int32_t i = 0; + int32_t prev = 0; + char str[512] = {0}; while (1) { int32_t c = taos_fetch_block_s(pRes, &numOfRows, &pRow); @@ -927,7 +929,12 @@ TEST(clientCase, projection_query_stables) { break; } i += numOfRows; - printf("%d\n", i); + + if ( (i / 1000000) > prev) { + printf("%d\n", i); + prev = i; + } + //printf("%d\n", i); } // while ((pRow = taos_fetch_row(pRes)) != NULL) { // int32_t code = taos_print_row(str, pRow, pFields, numOfFields); From 0e08cc16c52322a65a10149996e61ff72cf0fd6b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 17 May 2024 00:39:36 +0800 Subject: [PATCH 11/33] fix(test): update test case. --- source/client/test/clientTests.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index d7c1cbb3d9..a22716df59 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -932,7 +932,7 @@ TEST(clientCase, projection_query_stables) { if ( (i / 1000000) > prev) { printf("%d\n", i); - prev = i; + prev = i/1000000; } //printf("%d\n", i); } From 15181eb14bbed37cd1fd6db0c7caaa5e064e48c9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 17 May 2024 09:51:56 +0800 Subject: [PATCH 12/33] enh(tsdb): increase the capacity for return data. --- source/dnode/vnode/src/tsdb/tsdbRead2.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 9d08ff4e04..76dbfd8a45 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -4242,7 +4242,7 @@ int32_t tsdbReaderOpen2(void* pVnode, SQueryTableDataCond* pCond, void* pTableLi SVnodeCfg* pConf = &(((SVnode*)pVnode)->config); int32_t capacity = pConf->tsdbCfg.maxRows; - capacity = 8192; // for debug purpose + capacity = 16384; // for debug purpose if (pResBlock != NULL) { blockDataEnsureCapacity(pResBlock, capacity); } From c5b3c04036c93dc2afe250b785ed66d86d9e950f Mon Sep 17 00:00:00 2001 From: Yihao Deng Date: Fri, 17 May 2024 10:27:56 +0800 Subject: [PATCH 13/33] opt read buf --- source/libs/transport/inc/transComm.h | 4 ++-- source/libs/transport/src/transCli.c | 7 ++++++- source/libs/transport/src/transComm.c | 12 +++++++----- source/libs/transport/src/transSvr.c | 24 ++++++++++++++---------- 4 files changed, 29 insertions(+), 18 deletions(-) diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index da6d71e07b..00944115c0 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -302,8 +302,8 @@ int transClearBuffer(SConnBuffer* buf); int transDestroyBuffer(SConnBuffer* buf); int transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf); bool transReadComplete(SConnBuffer* connBuf); -int transResetBuffer(SConnBuffer* connBuf); -int transDumpFromBuffer(SConnBuffer* connBuf, char** buf); +int transResetBuffer(SConnBuffer* connBuf, int8_t resetBuf); +int transDumpFromBuffer(SConnBuffer* connBuf, char** buf, int8_t resetBuf); int transSetConnOption(uv_tcp_t* stream, int keepalive); diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index dfd7630f35..a1b09b9ee8 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -382,13 +382,18 @@ void cliHandleResp(SCliConn* conn) { STransMsgHead* pHead = NULL; - int32_t msgLen = transDumpFromBuffer(&conn->readBuf, (char**)&pHead); + int8_t resetBuf = conn->status == ConnAcquire ? 0 : 1; + int32_t msgLen = transDumpFromBuffer(&conn->readBuf, (char**)&pHead, resetBuf); if (msgLen <= 0) { taosMemoryFree(pHead); tDebug("%s conn %p recv invalid packet ", CONN_GET_INST_LABEL(conn), conn); return; } + if (resetBuf == 0) { + tTrace("%s conn %p not reset read buf", transLabel(pTransInst), conn); + } + if (transDecompressMsg((char**)&pHead, msgLen) < 0) { tDebug("%s conn %p recv invalid packet, failed to decompress", CONN_GET_INST_LABEL(conn), conn); } diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index fff13e7ebb..5598aa39e3 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -126,7 +126,7 @@ int transClearBuffer(SConnBuffer* buf) { return 0; } -int transDumpFromBuffer(SConnBuffer* connBuf, char** buf) { +int transDumpFromBuffer(SConnBuffer* connBuf, char** buf, int8_t resetBuf) { static const int HEADSIZE = sizeof(STransMsgHead); SConnBuffer* p = connBuf; @@ -137,7 +137,7 @@ int transDumpFromBuffer(SConnBuffer* connBuf, char** buf) { if (total >= HEADSIZE && !p->invalid) { *buf = taosMemoryCalloc(1, total); memcpy(*buf, p->buf, total); - if (transResetBuffer(connBuf) < 0) { + if (transResetBuffer(connBuf, resetBuf) < 0) { return -1; } } else { @@ -146,7 +146,7 @@ int transDumpFromBuffer(SConnBuffer* connBuf, char** buf) { return total; } -int transResetBuffer(SConnBuffer* connBuf) { +int transResetBuffer(SConnBuffer* connBuf, int8_t resetBuf) { SConnBuffer* p = connBuf; if (p->total < p->len) { int left = p->len - p->total; @@ -159,8 +159,10 @@ int transResetBuffer(SConnBuffer* connBuf) { p->total = 0; p->len = 0; if (p->cap > BUFFER_CAP) { - p->cap = BUFFER_CAP; - p->buf = taosMemoryRealloc(p->buf, p->cap); + if (resetBuf) { + p->cap = BUFFER_CAP; + p->buf = taosMemoryRealloc(p->buf, p->cap); + } } } else { ASSERTS(0, "invalid read from sock buf"); diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 21ad5be869..c0b2eba90a 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -342,11 +342,15 @@ static bool uvHandleReq(SSvrConn* pConn) { STransMsgHead* pHead = NULL; - int msgLen = transDumpFromBuffer(&pConn->readBuf, (char**)&pHead); + int8_t resetBuf = pConn->status == ConnAcquire ? 0 : 1; + int msgLen = transDumpFromBuffer(&pConn->readBuf, (char**)&pHead, resetBuf); if (msgLen <= 0) { tError("%s conn %p read invalid packet", transLabel(pTransInst), pConn); return false; } + if (resetBuf == 0) { + tTrace("%s conn %p not reset read buf", transLabel(pTransInst), pConn); + } if (transDecompressMsg((char**)&pHead, msgLen) < 0) { tError("%s conn %p recv invalid packet, failed to decompress", transLabel(pTransInst), pConn); @@ -677,17 +681,17 @@ static FORCE_INLINE void destroySmsg(SSvrMsg* smsg) { } static FORCE_INLINE void destroySmsgWrapper(void* smsg, void* param) { destroySmsg((SSvrMsg*)smsg); } static void destroyAllConn(SWorkThrd* pThrd) { - tTrace("thread %p destroy all conn ", pThrd); - while (!QUEUE_IS_EMPTY(&pThrd->conn)) { - queue* h = QUEUE_HEAD(&pThrd->conn); - QUEUE_REMOVE(h); - QUEUE_INIT(h); + tTrace("thread %p destroy all conn ", pThrd); + while (!QUEUE_IS_EMPTY(&pThrd->conn)) { + queue* h = QUEUE_HEAD(&pThrd->conn); + QUEUE_REMOVE(h); + QUEUE_INIT(h); - SSvrConn* c = QUEUE_DATA(h, SSvrConn, queue); - while (T_REF_VAL_GET(c) >= 2) { - transUnrefSrvHandle(c); + SSvrConn* c = QUEUE_DATA(h, SSvrConn, queue); + while (T_REF_VAL_GET(c) >= 2) { + transUnrefSrvHandle(c); } - transUnrefSrvHandle(c); + transUnrefSrvHandle(c); } } void uvWorkerAsyncCb(uv_async_t* handle) { From 27c4e3634cc8cf84f860f96659339e8258f27882 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 17 May 2024 17:04:05 +0800 Subject: [PATCH 14/33] refactor: do some internal refactor. --- source/client/src/clientImpl.c | 2 +- source/dnode/vnode/src/tsdb/tsdbRead2.c | 1 - source/libs/executor/src/exchangeoperator.c | 2 +- source/libs/qworker/src/qworker.c | 20 ++++++++++---------- 4 files changed, 12 insertions(+), 13 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 52a776f66b..6f1c3fccd7 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -2224,7 +2224,7 @@ int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableR } else { pResultInfo->pData = (void*)pRsp->data; pResultInfo->payloadLen = htonl(pRsp->compLen); -// ASSERT(pRsp->compLen == pRsp->payloadLen); + ASSERT(pRsp->compLen == pRsp->payloadLen); } // TODO handle the compressed case diff --git a/source/dnode/vnode/src/tsdb/tsdbRead2.c b/source/dnode/vnode/src/tsdb/tsdbRead2.c index 76dbfd8a45..381e040c5b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead2.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead2.c @@ -4242,7 +4242,6 @@ int32_t tsdbReaderOpen2(void* pVnode, SQueryTableDataCond* pCond, void* pTableLi SVnodeCfg* pConf = &(((SVnode*)pVnode)->config); int32_t capacity = pConf->tsdbCfg.maxRows; - capacity = 16384; // for debug purpose if (pResBlock != NULL) { blockDataEnsureCapacity(pResBlock, capacity); } diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index a563897122..72d0af1726 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -681,7 +681,7 @@ int32_t doExtractResultBlocks(SExchangeInfo* pExchangeInfo, SSourceDataInfo* pDa p = taosMemoryMalloc(pRetrieveRsp->payloadLen); int32_t t = tsDecompressString(pRetrieveRsp->data, pRetrieveRsp->compLen, 1, p, pRetrieveRsp->payloadLen, ONE_STAGE_COMP, NULL, 0); -// ASSERT(t == pRetrieveRsp->payloadLen); + ASSERT(t == pRetrieveRsp->payloadLen); pStart = p; } diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 3fee7d94b9..ca03a61023 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -914,20 +914,21 @@ int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg) { qwBuildFetchRsp(rsp, &sOutput, dataLen, rawDataLen, qComplete); { - if (/*dataLen > 8192*/ 0) { + SRetrieveTableRsp* pRsp = rsp; + + if (dataLen > 8192) { char* p = taosMemoryMalloc(dataLen); - int32_t len = - tsCompressString(((SRetrieveTableRsp *)rsp)->data, dataLen, 1, p, dataLen, ONE_STAGE_COMP, NULL, 0); - memcpy(((SRetrieveTableRsp*)rsp)->data, p, len); - ((SRetrieveTableRsp*)rsp)->payloadLen = htonl(dataLen); - ((SRetrieveTableRsp*)rsp)->compLen = htonl(len); + int32_t len = tsCompressString(pRsp->data, dataLen, 1, p, dataLen, ONE_STAGE_COMP, NULL, 0); + memcpy(pRsp->data, p, len); - ((SRetrieveTableRsp*)rsp)->compressed = 1; + pRsp->payloadLen = htonl(dataLen); + pRsp->compLen = htonl(len); + pRsp->compressed = 1; taosMemoryFree(p); } else { -// ((SRetrieveTableRsp*)rsp)->payloadLen = ((SRetrieveTableRsp*)rsp)->compLen; -// ((SRetrieveTableRsp*)rsp)->compressed = 0; + pRsp->payloadLen = pRsp->compLen; + pRsp->compressed = 0; } } @@ -949,7 +950,6 @@ int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg) { atomic_store_8((int8_t *)&ctx->queryContinue, 1); } else if (0 == atomic_load_8((int8_t *)&ctx->queryInQueue)) { qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_EXEC, ctx->dynamicTask); - atomic_store_8((int8_t *)&ctx->queryInQueue, 1); QW_ERR_JRET(qwBuildAndSendCQueryMsg(QW_FPARAMS(), &qwMsg->connInfo)); From a2692dbba4d10869f3eed710e44435c382e231fd Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 17 May 2024 17:51:41 +0800 Subject: [PATCH 15/33] refactor(query): opt memory prepare. --- source/client/inc/clientInt.h | 3 ++- source/client/src/clientEnv.c | 2 +- source/client/src/clientImpl.c | 26 ++++++++++++--------- source/libs/executor/src/exchangeoperator.c | 26 ++++++++++++++++----- 4 files changed, 38 insertions(+), 19 deletions(-) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index e06c7e63ac..21dd930c8f 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -196,7 +196,8 @@ typedef struct SReqResultInfo { bool localResultFetched; bool completed; bool convertUcs4; - char* decompressBuf; + char* decompBuf; + int32_t decompBufSize; int32_t precision; int32_t payloadLen; char* convertJson; diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 38a895e4fd..1ee771b619 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -370,7 +370,7 @@ void doFreeReqResultInfo(SReqResultInfo *pResInfo) { taosMemoryFreeClear(pResInfo->fields); taosMemoryFreeClear(pResInfo->userFields); taosMemoryFreeClear(pResInfo->convertJson); - taosMemoryFreeClear(pResInfo->decompressBuf); + taosMemoryFreeClear(pResInfo->decompBuf); if (pResInfo->convertBuf != NULL) { for (int32_t i = 0; i < pResInfo->numOfCols; ++i) { diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 6f1c3fccd7..7c5aeeb375 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -2202,24 +2202,28 @@ int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableR if (pRsp->compressed) { int32_t payloadLen = htonl(pRsp->payloadLen); - if (pResultInfo->decompressBuf == NULL) { - pResultInfo->decompressBuf = taosMemoryMalloc(payloadLen); + if (pResultInfo->decompBuf == NULL) { + pResultInfo->decompBuf = taosMemoryMalloc(payloadLen); + pResultInfo->decompBufSize = payloadLen; } else { - char* p = taosMemoryRealloc(pResultInfo->decompressBuf, payloadLen); - if (p == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - tscError("failed to prepare the decompress buffer, size:%d", payloadLen); - return terrno; - } + if (pResultInfo->decompBufSize < payloadLen) { + char* p = taosMemoryRealloc(pResultInfo->decompBuf, payloadLen); + if (p == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tscError("failed to prepare the decompress buffer, size:%d", payloadLen); + return terrno; + } - pResultInfo->decompressBuf = p; + pResultInfo->decompBuf = p; + pResultInfo->decompBufSize = payloadLen; + } } - int32_t len = tsDecompressString((void*)pRsp->data, htonl(pRsp->compLen), 1, pResultInfo->decompressBuf, payloadLen, + int32_t len = tsDecompressString((void*)pRsp->data, htonl(pRsp->compLen), 1, pResultInfo->decompBuf, payloadLen, ONE_STAGE_COMP, NULL, 0); ASSERT(len == payloadLen); - pResultInfo->pData = pResultInfo->decompressBuf; + pResultInfo->pData = pResultInfo->decompBuf; pResultInfo->payloadLen = payloadLen; } else { pResultInfo->pData = (void*)pRsp->data; diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index 72d0af1726..dff92eb52d 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -41,6 +41,8 @@ typedef struct SSourceDataInfo { SArray* pSrcUidList; int32_t srcOpType; bool tableSeq; + char* decompBuf; + int32_t decompBufSize; } SSourceDataInfo; static void destroyExchangeOperatorInfo(void* param); @@ -370,7 +372,10 @@ void freeBlock(void* pParam) { void freeSourceDataInfo(void* p) { SSourceDataInfo* pInfo = (SSourceDataInfo*)p; + taosMemoryFreeClear(pInfo->decompBuf); taosMemoryFreeClear(pInfo->pRsp); + + pInfo->decompBufSize = 0; } void doDestroyExchangeOperatorInfo(void* param) { @@ -675,14 +680,24 @@ int32_t doExtractResultBlocks(SExchangeInfo* pExchangeInfo, SSourceDataInfo* pDa char* pStart = pRetrieveRsp->data; int32_t index = 0; int32_t code = 0; - char* p = NULL; if (pRetrieveRsp->compressed) { // decompress the data - p = taosMemoryMalloc(pRetrieveRsp->payloadLen); - int32_t t = tsDecompressString(pRetrieveRsp->data, pRetrieveRsp->compLen, 1, p, pRetrieveRsp->payloadLen, - ONE_STAGE_COMP, NULL, 0); + if (pDataInfo->decompBuf == NULL) { + pDataInfo->decompBuf = taosMemoryMalloc(pRetrieveRsp->payloadLen); + pDataInfo->decompBufSize = pRetrieveRsp->payloadLen; + } else { + if (pDataInfo->decompBufSize < pRetrieveRsp->payloadLen) { + char* p = taosMemoryRealloc(pDataInfo->decompBuf, pRetrieveRsp->payloadLen); + if (p != NULL) { + pDataInfo->decompBuf = p; + pDataInfo->decompBufSize = pRetrieveRsp->payloadLen; + } + } + } + int32_t t = tsDecompressString(pRetrieveRsp->data, pRetrieveRsp->compLen, 1, pDataInfo->decompBuf, + pRetrieveRsp->payloadLen, ONE_STAGE_COMP, NULL, 0); ASSERT(t == pRetrieveRsp->payloadLen); - pStart = p; + pStart = pDataInfo->decompBuf; } while (index++ < pRetrieveRsp->numOfBlocks) { @@ -703,7 +718,6 @@ int32_t doExtractResultBlocks(SExchangeInfo* pExchangeInfo, SSourceDataInfo* pDa taosArrayPush(pExchangeInfo->pResultBlockList, &pb); } - taosMemoryFreeClear(p); return code; } From e7aa0ca177356e9d6de839e063de791baa4bc294 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 17 May 2024 18:39:34 +0800 Subject: [PATCH 16/33] fix(query): fix error. --- source/libs/executor/src/dataDispatcher.c | 6 +++--- source/libs/qworker/src/qwMsg.c | 1 + source/libs/qworker/src/qworker.c | 21 +-------------------- 3 files changed, 5 insertions(+), 23 deletions(-) diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index e2126f1fc4..eb6a5d9f9e 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -83,10 +83,10 @@ static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pIn // ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data + 8 + 4)); { - if (/*pBuf->allocSize > 8192*/ 0) { - char* p = taosMemoryMalloc(pBuf->allocSize); - int32_t dataLen = blockEncode(pInput->pData, p, numOfCols); + if (pBuf->allocSize > 16384) { + char* p = taosMemoryMalloc(pBuf->allocSize); + int32_t dataLen = blockEncode(pInput->pData, p, numOfCols); int32_t len = tsCompressString(p, dataLen, 1, pEntry->data, pBuf->allocSize, ONE_STAGE_COMP, NULL, 0); pEntry->compressed = 1; diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index e2a99a11e7..ef9ef42c85 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -41,6 +41,7 @@ void qwBuildFetchRsp(void *msg, SOutputData *input, int32_t len, int32_t rawData rsp->numOfRows = htobe64(input->numOfRows); rsp->numOfCols = htonl(input->numOfCols); rsp->numOfBlocks = htonl(input->numOfBlocks); + ASSERT(rawDataLen != 100446); } void qwFreeFetchRsp(void *msg) { diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index ca03a61023..89b42873de 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -300,6 +300,7 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, } *dataLen = 0; + *pRawDataLen = 0; while (true) { dsGetDataLength(ctx->sinkHandle, &len, &rawLen, &queryEnd); @@ -912,26 +913,6 @@ int32_t qwProcessFetch(QW_FPARAMS_DEF, SQWMsg *qwMsg) { bool qComplete = (DS_BUF_EMPTY == sOutput.bufStatus && sOutput.queryEnd); qwBuildFetchRsp(rsp, &sOutput, dataLen, rawDataLen, qComplete); - - { - SRetrieveTableRsp* pRsp = rsp; - - if (dataLen > 8192) { - char* p = taosMemoryMalloc(dataLen); - - int32_t len = tsCompressString(pRsp->data, dataLen, 1, p, dataLen, ONE_STAGE_COMP, NULL, 0); - memcpy(pRsp->data, p, len); - - pRsp->payloadLen = htonl(dataLen); - pRsp->compLen = htonl(len); - pRsp->compressed = 1; - taosMemoryFree(p); - } else { - pRsp->payloadLen = pRsp->compLen; - pRsp->compressed = 0; - } - } - if (qComplete) { atomic_store_8((int8_t *)&ctx->queryEnd, true); } From 43bec6c00f4df76b0c9ff51257325c8161f304d9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 18 May 2024 09:47:31 +0800 Subject: [PATCH 17/33] fix(query): fix error in refactor. --- source/client/src/clientImpl.c | 16 ++++++++---- source/client/test/clientTests.cpp | 10 +++---- source/libs/executor/src/dataDispatcher.c | 3 +-- source/libs/executor/src/exchangeoperator.c | 26 ++++++++++++++---- source/libs/qworker/src/qworker.c | 29 ++++++++++----------- 5 files changed, 52 insertions(+), 32 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 7c5aeeb375..bc5bffdb48 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -2218,15 +2218,21 @@ int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableR pResultInfo->decompBufSize = payloadLen; } } + } - int32_t len = tsDecompressString((void*)pRsp->data, htonl(pRsp->compLen), 1, pResultInfo->decompBuf, payloadLen, - ONE_STAGE_COMP, NULL, 0); - ASSERT(len == payloadLen); + int32_t compLen = *(int32_t*)pRsp->data; + int32_t rawLen = *(int32_t*)(pRsp->data + sizeof(int32_t)); + + char* pStart = (char*)pRsp->data + sizeof(int32_t) * 2; + + if (pRsp->compressed && compLen < rawLen) { + int32_t len = tsDecompressString(pStart, compLen, 1, pResultInfo->decompBuf, rawLen, ONE_STAGE_COMP, NULL, 0); + ASSERT(len == rawLen); pResultInfo->pData = pResultInfo->decompBuf; - pResultInfo->payloadLen = payloadLen; + pResultInfo->payloadLen = rawLen; } else { - pResultInfo->pData = (void*)pRsp->data; + pResultInfo->pData = pStart; pResultInfo->payloadLen = htonl(pRsp->compLen); ASSERT(pRsp->compLen == pRsp->payloadLen); } diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index a22716df59..61a97f2c24 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -901,13 +901,13 @@ TEST(clientCase, tsbs_perf_test) { } TEST(clientCase, projection_query_stables) { - TAOS* pConn = taos_connect("192.168.1.53", "root", "taosdata", NULL, 0); + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(pConn, nullptr); - TAOS_RES* pRes = taos_query(pConn, "use qzdata"); + TAOS_RES* pRes = taos_query(pConn, "use abc1"); taos_free_result(pRes); - pRes = taos_query(pConn, "select * from `QZ_212_DYS_02`"); + pRes = taos_query(pConn, "select * from st2"); if (taos_errno(pRes) != 0) { printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); taos_free_result(pRes); @@ -924,8 +924,8 @@ TEST(clientCase, projection_query_stables) { char str[512] = {0}; while (1) { - int32_t c = taos_fetch_block_s(pRes, &numOfRows, &pRow); - if (numOfRows <= 0) { + pRow = taos_fetch_row(pRes); + if (pRow == NULL) { break; } i += numOfRows; diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index eb6a5d9f9e..ea65db7779 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -77,10 +77,9 @@ static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pIn pEntry->numOfRows = pInput->pData->info.rows; pEntry->numOfCols = numOfCols; pEntry->dataLen = 0; + pEntry->rawLen = 0; pBuf->useSize = sizeof(SDataCacheEntry); - // ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data + 8)); - // ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data + 8 + 4)); { if (pBuf->allocSize > 16384) { diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index dff92eb52d..4c2976ed94 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -677,7 +677,9 @@ int32_t prepareConcurrentlyLoad(SOperatorInfo* pOperator) { int32_t doExtractResultBlocks(SExchangeInfo* pExchangeInfo, SSourceDataInfo* pDataInfo) { SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp; - char* pStart = pRetrieveRsp->data; + char* pNextStart = pRetrieveRsp->data; + char* pStart = pNextStart; + int32_t index = 0; int32_t code = 0; @@ -694,14 +696,13 @@ int32_t doExtractResultBlocks(SExchangeInfo* pExchangeInfo, SSourceDataInfo* pDa } } } - int32_t t = tsDecompressString(pRetrieveRsp->data, pRetrieveRsp->compLen, 1, pDataInfo->decompBuf, - pRetrieveRsp->payloadLen, ONE_STAGE_COMP, NULL, 0); - ASSERT(t == pRetrieveRsp->payloadLen); - pStart = pDataInfo->decompBuf; } + while (index++ < pRetrieveRsp->numOfBlocks) { SSDataBlock* pb = NULL; + pStart = pNextStart; + if (taosArrayGetSize(pExchangeInfo->pRecycledBlocks) > 0) { pb = *(SSDataBlock**)taosArrayPop(pExchangeInfo->pRecycledBlocks); blockDataCleanup(pb); @@ -709,6 +710,21 @@ int32_t doExtractResultBlocks(SExchangeInfo* pExchangeInfo, SSourceDataInfo* pDa pb = createOneDataBlock(pExchangeInfo->pDummyBlock, false); } + int32_t compLen = *(int32_t*) pStart; + pStart += sizeof(int32_t); + + int32_t rawLen = *(int32_t*) pStart; + pStart += sizeof(int32_t); + ASSERT(compLen <= rawLen && compLen != 0); + + if (pRetrieveRsp->compressed && (compLen < rawLen)) { + int32_t t = tsDecompressString(pStart, compLen, 1, pDataInfo->decompBuf, rawLen, ONE_STAGE_COMP, NULL, 0); + ASSERT(t == rawLen); + + pNextStart = pStart + compLen; + pStart = pDataInfo->decompBuf; + } + code = extractDataBlockFromFetchRsp(pb, pStart, NULL, &pStart); if (code != 0) { taosMemoryFreeClear(pDataInfo->pRsp); diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 89b42873de..b799a72cce 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -289,7 +289,7 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, SOutputData *pOutput) { int64_t len = 0; int64_t rawLen = 0; - SRetrieveTableRsp *rsp = NULL; + SRetrieveTableRsp *pRsp = NULL; bool queryEnd = false; int32_t code = 0; SOutputData output = {0}; @@ -325,32 +325,36 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, qwUpdateTaskStatus(QW_FPARAMS(), JOB_TASK_STATUS_SUCC, ctx->dynamicTask); } - if (NULL == rsp) { - QW_ERR_RET(qwMallocFetchRsp(!ctx->localExec, len, &rsp)); + if (NULL == pRsp) { + QW_ERR_RET(qwMallocFetchRsp(!ctx->localExec, len, &pRsp)); *pOutput = output; } else { pOutput->queryEnd = output.queryEnd; pOutput->bufStatus = output.bufStatus; pOutput->useconds = output.useconds; } - break; } pOutput->bufStatus = DS_BUF_EMPTY; - break; } // Got data from sink QW_TASK_DLOG("there are data in sink, dataLength:%" PRId64 "", len); - *dataLen += len; - *pRawDataLen += rawLen; + *dataLen += len + sizeof(int32_t) * 2; + *pRawDataLen += rawLen + sizeof(int32_t) * 2; - QW_ERR_RET(qwMallocFetchRsp(!ctx->localExec, *dataLen, &rsp)); + QW_ERR_RET(qwMallocFetchRsp(!ctx->localExec, *dataLen, &pRsp)); + + // set the serialize start position + output.pData = pRsp->data + *dataLen - (len + sizeof(int32_t) * 2); + + ((int32_t*) output.pData)[0] = len; + ((int32_t*) output.pData)[1] = rawLen; + output.pData += sizeof(int32_t) * 2; - output.pData = rsp->data + *dataLen - len; code = dsGetDataBlock(ctx->sinkHandle, &output); if (code) { QW_TASK_ELOG("dsGetDataBlock failed, code:%x - %s", code, tstrerror(code)); @@ -386,8 +390,7 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, } } - *rspMsg = rsp; - + *rspMsg = pRsp; return TSDB_CODE_SUCCESS; } @@ -1106,7 +1109,6 @@ int32_t qwProcessHb(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) { _return: memcpy(&rsp.epId, &req->epId, sizeof(req->epId)); - qwBuildAndSendHbRsp(&qwMsg->connInfo, &rsp, code); if (code) { @@ -1133,7 +1135,6 @@ void qwProcessHbTimerEvent(void *param, void *tmrId) { } SQWSchStatus *sch = NULL; - int32_t taskNum = 0; SQWHbInfo *rspList = NULL; SArray *pExpiredSch = NULL; int32_t code = 0; @@ -1166,8 +1167,6 @@ void qwProcessHbTimerEvent(void *param, void *tmrId) { return; } - void *key = NULL; - size_t keyLen = 0; int32_t i = 0; int64_t currentMs = taosGetTimestampMs(); From 937ded9f70e2bc11f9d06356a215c42c3369e4ea Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 18 May 2024 09:54:22 +0800 Subject: [PATCH 18/33] refactor(query): tune sink param. --- source/libs/executor/src/dataDispatcher.c | 4 +++- source/libs/executor/src/executor.c | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index ea65db7779..022bd951b0 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -268,6 +268,7 @@ int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pD terrno = TSDB_CODE_OUT_OF_MEMORY; goto _return; } + dispatcher->sink.fPut = putDataBlock; dispatcher->sink.fEndPut = endPut; dispatcher->sink.fReset = resetDispatcher; @@ -275,12 +276,14 @@ int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pD dispatcher->sink.fGetData = getDataBlock; dispatcher->sink.fDestroy = destroyDataSinker; dispatcher->sink.fGetCacheSize = getCacheSize; + dispatcher->pManager = pManager; dispatcher->pSchema = pDataSink->pInputDataBlockDesc; dispatcher->status = DS_BUF_EMPTY; dispatcher->queryEnd = false; dispatcher->pDataBlocks = taosOpenQueue(); taosThreadMutexInit(&dispatcher->mutex, NULL); + if (NULL == dispatcher->pDataBlocks) { taosMemoryFree(dispatcher); terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -290,7 +293,6 @@ int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pD return TSDB_CODE_SUCCESS; _return: - taosMemoryFree(pManager); return terrno; } diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 889a9b62e1..29f667cb66 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -537,7 +537,7 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, } if (handle) { - SDataSinkMgtCfg cfg = {.maxDataBlockNum = 5000, .maxDataBlockNumPerQuery = 500}; + SDataSinkMgtCfg cfg = {.maxDataBlockNum = 500, .maxDataBlockNumPerQuery = 50}; void* pSinkManager = NULL; code = dsDataSinkMgtInit(&cfg, &(*pTask)->storageAPI, &pSinkManager); if (code != TSDB_CODE_SUCCESS) { From cc4018e7eabf8eb5ec53296b065a9ab9ce051d00 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 18 May 2024 10:02:33 +0800 Subject: [PATCH 19/33] other: add missing cmakefile. --- contrib/CMakeLists.txt | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 796b690d01..33179662fd 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -121,7 +121,7 @@ if (${BUILD_CONTRIB}) cat("${TD_SUPPORT_DIR}/rocksdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) add_definitions(-DUSE_ROCKSDB) endif() -else() +else() if (NOT ${TD_LINUX}) if(${BUILD_WITH_ROCKSDB}) cat("${TD_SUPPORT_DIR}/rocksdb_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) @@ -183,6 +183,11 @@ if(${BUILD_GEOS}) cat("${TD_SUPPORT_DIR}/geos_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) endif() +# +if(${BUILD_PCRE2}) + cat("${TD_SUPPORT_DIR}/pcre2_CMakeLists.txt.in" ${CONTRIB_TMP_FILE}) +endif() + # download dependencies configure_file(${CONTRIB_TMP_FILE} "${TD_CONTRIB_DIR}/deps-download/CMakeLists.txt") execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" . @@ -270,8 +275,8 @@ unset(CMAKE_PROJECT_INCLUDE_BEFORE) # add_subdirectory(xz EXCLUDE_FROM_ALL) # target_include_directories( # xz -# PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/xz -# PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/xz +# PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/xz +# PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/xz # ) # leveldb @@ -351,7 +356,7 @@ if (${BUILD_WITH_ROCKSDB}) rocksdb PUBLIC $ ) - else() + else() if (NOT ${TD_LINUX}) MESSAGE(STATUS "ROCKSDB CXX STATUS CONFIG: " ${CMAKE_CXX_FLAGS}) MESSAGE(STATUS "ROCKSDB C STATUS CONFIG: " ${CMAKE_C_FLAGS}) @@ -400,8 +405,8 @@ if (${BUILD_WITH_ROCKSDB}) rocksdb PUBLIC $ ) - endif() - + endif() + endif() endif() @@ -605,6 +610,10 @@ if(${BUILD_GEOS}) ) endif(${BUILD_GEOS}) +if (${BUILD_PCRE2}) + add_subdirectory(pcre2 EXCLUDE_FROM_ALL) +endif(${BUILD_PCRE2}) + # ================================================================================================ # Build test # ================================================================================================ From 6124666488afefd82de7e426a0d8c158dc2158c5 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 18 May 2024 18:40:49 +0800 Subject: [PATCH 20/33] fix(query): fix bugs caused by refactor. --- source/client/src/clientImpl.c | 31 ++++++++++++++++-------------- source/client/test/clientTests.cpp | 16 +++++++-------- source/libs/command/src/explain.c | 9 ++++++--- 3 files changed, 31 insertions(+), 25 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index bc5bffdb48..1057a00725 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -2063,6 +2063,7 @@ int32_t setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32 tscError("setResultDataPtr paras error"); return TSDB_CODE_TSC_INTERNAL_ERROR; } + if (numOfRows == 0) { return TSDB_CODE_SUCCESS; } @@ -2199,9 +2200,9 @@ int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableR pResultInfo->precision = pRsp->precision; // decompress data if needed - if (pRsp->compressed) { - int32_t payloadLen = htonl(pRsp->payloadLen); + int32_t payloadLen = htonl(pRsp->payloadLen); + if (pRsp->compressed) { if (pResultInfo->decompBuf == NULL) { pResultInfo->decompBuf = taosMemoryMalloc(payloadLen); pResultInfo->decompBufSize = payloadLen; @@ -2220,21 +2221,23 @@ int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableR } } - int32_t compLen = *(int32_t*)pRsp->data; - int32_t rawLen = *(int32_t*)(pRsp->data + sizeof(int32_t)); + if (payloadLen > 0) { + int32_t compLen = *(int32_t*)pRsp->data; + int32_t rawLen = *(int32_t*)(pRsp->data + sizeof(int32_t)); - char* pStart = (char*)pRsp->data + sizeof(int32_t) * 2; + char* pStart = (char*)pRsp->data + sizeof(int32_t) * 2; - if (pRsp->compressed && compLen < rawLen) { - int32_t len = tsDecompressString(pStart, compLen, 1, pResultInfo->decompBuf, rawLen, ONE_STAGE_COMP, NULL, 0); - ASSERT(len == rawLen); + if (pRsp->compressed && compLen < rawLen) { + int32_t len = tsDecompressString(pStart, compLen, 1, pResultInfo->decompBuf, rawLen, ONE_STAGE_COMP, NULL, 0); + ASSERT(len == rawLen); - pResultInfo->pData = pResultInfo->decompBuf; - pResultInfo->payloadLen = rawLen; - } else { - pResultInfo->pData = pStart; - pResultInfo->payloadLen = htonl(pRsp->compLen); - ASSERT(pRsp->compLen == pRsp->payloadLen); + pResultInfo->pData = pResultInfo->decompBuf; + pResultInfo->payloadLen = rawLen; + } else { + pResultInfo->pData = pStart; + pResultInfo->payloadLen = htonl(pRsp->compLen); + ASSERT(pRsp->compLen == pRsp->payloadLen); + } } // TODO handle the compressed case diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 61a97f2c24..b5bad92dc4 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -904,15 +904,15 @@ TEST(clientCase, projection_query_stables) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(pConn, nullptr); - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - taos_free_result(pRes); + TAOS_RES* pRes = taos_query(pConn, "explain select * from dbvg.st where tbname='ct1'"); +// taos_free_result(pRes); - pRes = taos_query(pConn, "select * from st2"); - if (taos_errno(pRes) != 0) { - printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); - } +// pRes = taos_query(pConn, "select * from st2"); +// if (taos_errno(pRes) != 0) { +// printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// ASSERT_TRUE(false); +// } TAOS_ROW pRow = NULL; TAOS_FIELD* pFields = taos_fetch_fields(pRes); diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index bde0b47489..3333f22e10 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -70,7 +70,6 @@ char* qExplainGetTimerangeTargetStr(int32_t target) { return targetName[target]; } - void qExplainFreeResNode(SExplainResNode *resNode) { if (NULL == resNode) { return; @@ -1942,7 +1941,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { pBlock->info.rows = rowNum; - int32_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock); + int32_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock) + sizeof(int32_t)*2; SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, rspSize); if (NULL == rsp) { @@ -1954,10 +1953,14 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { rsp->completed = 1; rsp->numOfRows = htobe64((int64_t)rowNum); - int32_t len = blockEncode(pBlock, rsp->data, taosArrayGetSize(pBlock->pDataBlock)); + int32_t len = blockEncode(pBlock, rsp->data + sizeof(int32_t)*2, taosArrayGetSize(pBlock->pDataBlock)); rsp->compLen = htonl(len); rsp->payloadLen = rsp->compLen; + rsp->compressed = 0; + + ((int32_t*)rsp->data)[0] = len; + ((int32_t*)rsp->data)[1] = len; blockDataDestroy(pBlock); From 8f4da7f308efd4bc6d1642c505785940c261e534 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sat, 18 May 2024 23:11:18 +0800 Subject: [PATCH 21/33] fix(query): fix bugs caused by refactor. --- source/libs/executor/src/exchangeoperator.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/source/libs/executor/src/exchangeoperator.c b/source/libs/executor/src/exchangeoperator.c index 4c2976ed94..017332cf2d 100644 --- a/source/libs/executor/src/exchangeoperator.c +++ b/source/libs/executor/src/exchangeoperator.c @@ -717,11 +717,10 @@ int32_t doExtractResultBlocks(SExchangeInfo* pExchangeInfo, SSourceDataInfo* pDa pStart += sizeof(int32_t); ASSERT(compLen <= rawLen && compLen != 0); + pNextStart = pStart + compLen; if (pRetrieveRsp->compressed && (compLen < rawLen)) { int32_t t = tsDecompressString(pStart, compLen, 1, pDataInfo->decompBuf, rawLen, ONE_STAGE_COMP, NULL, 0); ASSERT(t == rawLen); - - pNextStart = pStart + compLen; pStart = pDataInfo->decompBuf; } From 19b17826d4c33cc309f727a8867ab22111cec5c3 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 19 May 2024 00:16:43 +0800 Subject: [PATCH 22/33] fix(query): fix bugs caused by refactor. --- source/libs/qworker/src/qworker.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index b799a72cce..8efe42e9bc 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -365,7 +365,10 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, pOutput->precision = output.precision; pOutput->bufStatus = output.bufStatus; pOutput->useconds = output.useconds; - pOutput->compressed = output.compressed; + + if (output.compressed) { + pOutput->compressed = output.compressed; + } pOutput->numOfCols = output.numOfCols; pOutput->numOfRows += output.numOfRows; pOutput->numOfBlocks++; From ad95b27cf25618ad77f362394297d4a070de70be Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 19 May 2024 10:28:24 +0800 Subject: [PATCH 23/33] fix(query): fix bugs caused by refactor. --- source/libs/command/src/command.c | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 8f3263cbeb..75ce4c9819 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -25,8 +25,16 @@ extern SConfig* tsCfg; +#define PAYLOAD_PREFIX_LEN ((sizeof(int32_t))<<1) + +#define SET_PAYLOAD_LEN(_p, _compLen, _fullLen) \ + do { \ + ((int32_t*)(_p))[0] = (_compLen); \ + ((int32_t*)(_p))[1] = (_fullLen); \ + } while (0); + static int32_t buildRetrieveTableRsp(SSDataBlock* pBlock, int32_t numOfCols, SRetrieveTableRsp** pRsp) { - size_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock); + size_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock) + PAYLOAD_PREFIX_LEN; *pRsp = taosMemoryCalloc(1, rspSize); if (NULL == *pRsp) { return TSDB_CODE_OUT_OF_MEMORY; @@ -36,12 +44,16 @@ static int32_t buildRetrieveTableRsp(SSDataBlock* pBlock, int32_t numOfCols, SRe (*pRsp)->completed = 1; (*pRsp)->precision = 0; (*pRsp)->compressed = 0; - (*pRsp)->compLen = 0; - (*pRsp)->payloadLen = 0; + (*pRsp)->numOfRows = htobe64((int64_t)pBlock->info.rows); (*pRsp)->numOfCols = htonl(numOfCols); - int32_t len = blockEncode(pBlock, (*pRsp)->data, numOfCols); + int32_t len = blockEncode(pBlock, (*pRsp)->data + sizeof(int32_t)*2, numOfCols); + SET_PAYLOAD_LEN((*pRsp)->data, len, len); + + int32_t payloadLen = len + PAYLOAD_PREFIX_LEN; + (*pRsp)->payloadLen = htonl(payloadLen); + (*pRsp)->compLen = htonl(payloadLen); return TSDB_CODE_SUCCESS; } From d69d11a74fe8bb1c1eda5457a79a6789b1d279a6 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 19 May 2024 12:30:18 +0800 Subject: [PATCH 24/33] fix(query): fix bugs caused by refactor. --- source/libs/executor/src/dataDispatcher.c | 14 ++++++++++---- source/util/test/cfgTest.cpp | 2 +- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index 022bd951b0..5058c652ce 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -87,10 +87,16 @@ static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pIn int32_t dataLen = blockEncode(pInput->pData, p, numOfCols); int32_t len = tsCompressString(p, dataLen, 1, pEntry->data, pBuf->allocSize, ONE_STAGE_COMP, NULL, 0); - - pEntry->compressed = 1; - pEntry->dataLen = len; - pEntry->rawLen = dataLen; + if (len < dataLen) { + pEntry->compressed = 1; + pEntry->dataLen = len; + pEntry->rawLen = dataLen; + } else { // no need to compress data + pEntry->compressed = 0; + pEntry->dataLen = dataLen; + pEntry->rawLen = dataLen; + memcpy(pEntry->data, p, dataLen); + } taosMemoryFree(p); } else { pEntry->dataLen = blockEncode(pInput->pData, pEntry->data, numOfCols); diff --git a/source/util/test/cfgTest.cpp b/source/util/test/cfgTest.cpp index 92422b6a80..9f8645b14c 100644 --- a/source/util/test/cfgTest.cpp +++ b/source/util/test/cfgTest.cpp @@ -67,7 +67,7 @@ TEST_F(CfgTest, 02_Basic) { SConfigItem* pItem = NULL; SConfigIter* pIter = cfgCreateIter(pConfig); - while((pItem == cfgNextIter(pIter)) != NULL) { + while((pItem = cfgNextIter(pIter)) != NULL) { switch (pItem->dtype) { case CFG_DTYPE_BOOL: printf("index:%d, cfg:%s value:%d\n", size, pItem->name, pItem->bval); From 217fb7551c6d1ce490f351249f6d154058403a78 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 19 May 2024 13:18:12 +0800 Subject: [PATCH 25/33] fix(query): fix bugs caused by refactor. --- include/libs/command/command.h | 8 ++++++++ source/client/src/clientMsgHandler.c | 28 ++++++++++++++++++-------- source/dnode/mnode/impl/src/mndDnode.c | 1 - source/libs/command/src/command.c | 8 -------- 4 files changed, 28 insertions(+), 17 deletions(-) diff --git a/include/libs/command/command.h b/include/libs/command/command.h index b788b03386..47c106dc51 100644 --- a/include/libs/command/command.h +++ b/include/libs/command/command.h @@ -20,6 +20,14 @@ #include "plannodes.h" #include "tmsg.h" +#define PAYLOAD_PREFIX_LEN ((sizeof(int32_t)) << 1) + +#define SET_PAYLOAD_LEN(_p, _compLen, _fullLen) \ + do { \ + ((int32_t*)(_p))[0] = (_compLen); \ + ((int32_t*)(_p))[1] = (_fullLen); \ + } while (0); + typedef struct SExplainCtx SExplainCtx; int32_t qExecCommand(int64_t* pConnId, bool sysInfoUser, SNode *pStmt, SRetrieveTableRsp **pRsp, int8_t biMode); diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 3e178c876f..f04e5e53c6 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -25,6 +25,7 @@ #include "tglobal.h" #include "tname.h" #include "tversion.h" +#include "command.h" extern SClientHbMgr clientHbMgr; @@ -499,7 +500,7 @@ static int32_t buildShowVariablesRsp(SArray* pVars, SRetrieveTableRsp** pRsp) { return code; } - size_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock); + size_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock) + PAYLOAD_PREFIX_LEN; *pRsp = taosMemoryCalloc(1, rspSize); if (NULL == *pRsp) { blockDataDestroy(pBlock); @@ -510,15 +511,20 @@ static int32_t buildShowVariablesRsp(SArray* pVars, SRetrieveTableRsp** pRsp) { (*pRsp)->completed = 1; (*pRsp)->precision = 0; (*pRsp)->compressed = 0; - (*pRsp)->compLen = 0; - (*pRsp)->payloadLen = 0; + (*pRsp)->numOfRows = htobe64((int64_t)pBlock->info.rows); (*pRsp)->numOfCols = htonl(SHOW_VARIABLES_RESULT_COLS); - int32_t len = blockEncode(pBlock, (*pRsp)->data, SHOW_VARIABLES_RESULT_COLS); + int32_t len = blockEncode(pBlock, (*pRsp)->data + PAYLOAD_PREFIX_LEN, SHOW_VARIABLES_RESULT_COLS); blockDataDestroy(pBlock); - if (len != rspSize - sizeof(SRetrieveTableRsp)) { + SET_PAYLOAD_LEN((*pRsp)->data, len, len); + + int32_t payloadLen = len + PAYLOAD_PREFIX_LEN; + (*pRsp)->payloadLen = htonl(payloadLen); + (*pRsp)->compLen = htonl(payloadLen); + + if (payloadLen != rspSize - sizeof(SRetrieveTableRsp)) { uError("buildShowVariablesRsp error, len:%d != rspSize - sizeof(SRetrieveTableRsp):%" PRIu64, len, (uint64_t)(rspSize - sizeof(SRetrieveTableRsp))); return TSDB_CODE_TSC_INVALID_INPUT; @@ -612,7 +618,7 @@ static int32_t buildRetriveTableRspForCompactDb(SCompactDbRsp* pCompactDb, SRetr return code; } - size_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock); + size_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock) + PAYLOAD_PREFIX_LEN; *pRsp = taosMemoryCalloc(1, rspSize); if (NULL == *pRsp) { blockDataDestroy(pBlock); @@ -628,10 +634,16 @@ static int32_t buildRetriveTableRspForCompactDb(SCompactDbRsp* pCompactDb, SRetr (*pRsp)->numOfRows = htobe64((int64_t)pBlock->info.rows); (*pRsp)->numOfCols = htonl(COMPACT_DB_RESULT_COLS); - int32_t len = blockEncode(pBlock, (*pRsp)->data, COMPACT_DB_RESULT_COLS); + int32_t len = blockEncode(pBlock, (*pRsp)->data + PAYLOAD_PREFIX_LEN, COMPACT_DB_RESULT_COLS); blockDataDestroy(pBlock); - if (len != rspSize - sizeof(SRetrieveTableRsp)) { + SET_PAYLOAD_LEN((*pRsp)->data, len, len); + + int32_t payloadLen = len + PAYLOAD_PREFIX_LEN; + (*pRsp)->payloadLen = htonl(payloadLen); + (*pRsp)->compLen = htonl(payloadLen); + + if (payloadLen != rspSize - sizeof(SRetrieveTableRsp)) { uError("buildRetriveTableRspForCompactDb error, len:%d != rspSize - sizeof(SRetrieveTableRsp):%" PRIu64, len, (uint64_t)(rspSize - sizeof(SRetrieveTableRsp))); return TSDB_CODE_TSC_INVALID_INPUT; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 57ddbb9beb..4de0086f96 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -1148,7 +1148,6 @@ _OVER: } tFreeSShowVariablesRsp(&rsp); - return code; } diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 75ce4c9819..3557dd86ae 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -25,14 +25,6 @@ extern SConfig* tsCfg; -#define PAYLOAD_PREFIX_LEN ((sizeof(int32_t))<<1) - -#define SET_PAYLOAD_LEN(_p, _compLen, _fullLen) \ - do { \ - ((int32_t*)(_p))[0] = (_compLen); \ - ((int32_t*)(_p))[1] = (_fullLen); \ - } while (0); - static int32_t buildRetrieveTableRsp(SSDataBlock* pBlock, int32_t numOfCols, SRetrieveTableRsp** pRsp) { size_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock) + PAYLOAD_PREFIX_LEN; *pRsp = taosMemoryCalloc(1, rspSize); From 1e5098f676713ce50daa80733cd722aec5b66031 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 19 May 2024 13:32:30 +0800 Subject: [PATCH 26/33] fix(query): fix bugs caused by refactor. --- include/common/tmsg.h | 8 ++++++++ include/libs/command/command.h | 8 -------- source/libs/command/src/explain.c | 2 +- source/libs/stream/src/streamDispatch.c | 12 ++++++++++-- 4 files changed, 19 insertions(+), 11 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index e583920357..b5c0a84063 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2129,6 +2129,14 @@ typedef struct { char data[]; } SRetrieveTableRsp; +#define PAYLOAD_PREFIX_LEN ((sizeof(int32_t)) << 1) + +#define SET_PAYLOAD_LEN(_p, _compLen, _fullLen) \ + do { \ + ((int32_t*)(_p))[0] = (_compLen); \ + ((int32_t*)(_p))[1] = (_fullLen); \ + } while (0); + typedef struct { int64_t version; int64_t numOfRows; diff --git a/include/libs/command/command.h b/include/libs/command/command.h index 47c106dc51..b788b03386 100644 --- a/include/libs/command/command.h +++ b/include/libs/command/command.h @@ -20,14 +20,6 @@ #include "plannodes.h" #include "tmsg.h" -#define PAYLOAD_PREFIX_LEN ((sizeof(int32_t)) << 1) - -#define SET_PAYLOAD_LEN(_p, _compLen, _fullLen) \ - do { \ - ((int32_t*)(_p))[0] = (_compLen); \ - ((int32_t*)(_p))[1] = (_fullLen); \ - } while (0); - typedef struct SExplainCtx SExplainCtx; int32_t qExecCommand(int64_t* pConnId, bool sysInfoUser, SNode *pStmt, SRetrieveTableRsp **pRsp, int8_t biMode); diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index 3333f22e10..7081901db4 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -1941,7 +1941,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { pBlock->info.rows = rowNum; - int32_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock) + sizeof(int32_t)*2; + int32_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock) + PAYLOAD_PREFIX_LEN; SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)taosMemoryCalloc(1, rspSize); if (NULL == rsp) { diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 58c6e19581..c9779919a4 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -129,7 +129,8 @@ int32_t streamTaskBroadcastRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* r static int32_t buildStreamRetrieveReq(SStreamTask* pTask, const SSDataBlock* pBlock, SStreamRetrieveReq* req){ SRetrieveTableRsp* pRetrieve = NULL; - int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock); + + int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock) + PAYLOAD_PREFIX_LEN; pRetrieve = taosMemoryCalloc(1, dataStrLen); if (pRetrieve == NULL) return TSDB_CODE_OUT_OF_MEMORY; @@ -146,7 +147,14 @@ static int32_t buildStreamRetrieveReq(SStreamTask* pTask, const SSDataBlock* pBl pRetrieve->ekey = htobe64(pBlock->info.window.ekey); pRetrieve->version = htobe64(pBlock->info.version); - int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols); + int32_t actualLen = blockEncode(pBlock, pRetrieve->data+ PAYLOAD_PREFIX_LEN, numOfCols); + + SET_PAYLOAD_LEN(pRetrieve->data, actualLen, actualLen); + + int32_t payloadLen = actualLen + PAYLOAD_PREFIX_LEN; + pRetrieve->payloadLen = htonl(payloadLen); + pRetrieve->compLen = htonl(payloadLen); + pRetrieve->compressed = 0; req->streamId = pTask->id.streamId; req->srcNodeId = pTask->info.nodeId; From 82c10fd709682538cdd73e6d35c54b6115ae6bfd Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 19 May 2024 13:40:09 +0800 Subject: [PATCH 27/33] fix(query): fix bugs caused by refactor. --- source/libs/stream/src/streamData.c | 3 ++- source/libs/stream/src/streamDispatch.c | 10 +++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/source/libs/stream/src/streamData.c b/source/libs/stream/src/streamData.c index bcda85e7a7..309234179c 100644 --- a/source/libs/stream/src/streamData.c +++ b/source/libs/stream/src/streamData.c @@ -98,7 +98,8 @@ int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock taosArrayPush(pArray, &(SSDataBlock){0}); SRetrieveTableRsp* pRetrieve = pReq->pRetrieve; SSDataBlock* pDataBlock = taosArrayGet(pArray, 0); - blockDecode(pDataBlock, pRetrieve->data); + + blockDecode(pDataBlock, pRetrieve->data + PAYLOAD_PREFIX_LEN); // TODO: refactor pDataBlock->info.window.skey = be64toh(pRetrieve->skey); diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index c9779919a4..28bd05d0c2 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -671,7 +671,7 @@ int32_t streamTaskSendCheckpointSourceRsp(SStreamTask* pTask) { } int32_t streamAddBlockIntoDispatchMsg(const SSDataBlock* pBlock, SStreamDispatchReq* pReq) { - int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock); + int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock) + PAYLOAD_PREFIX_LEN; ASSERT(dataStrLen > 0); void* buf = taosMemoryCalloc(1, dataStrLen); @@ -694,8 +694,12 @@ int32_t streamAddBlockIntoDispatchMsg(const SSDataBlock* pBlock, SStreamDispatch pRetrieve->numOfCols = htonl(numOfCols); int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols); - actualLen += sizeof(SRetrieveTableRsp); - ASSERT(actualLen <= dataStrLen); + SET_PAYLOAD_LEN(pRetrieve->data, actualLen, actualLen); + + int32_t payloadLen = actualLen + PAYLOAD_PREFIX_LEN; + pRetrieve->payloadLen = htonl(payloadLen); + pRetrieve->compLen = htonl(payloadLen); + taosArrayPush(pReq->dataLen, &actualLen); taosArrayPush(pReq->data, &buf); From 7a90e68667080a590d15f8260b6992112b09c6dc Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 19 May 2024 13:55:46 +0800 Subject: [PATCH 28/33] fix(query): fix bugs caused by refactor. --- source/libs/command/src/command.c | 2 +- source/libs/command/src/explain.c | 7 +++---- source/libs/stream/src/streamDispatch.c | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index 3557dd86ae..4cc496e4a0 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -40,7 +40,7 @@ static int32_t buildRetrieveTableRsp(SSDataBlock* pBlock, int32_t numOfCols, SRe (*pRsp)->numOfRows = htobe64((int64_t)pBlock->info.rows); (*pRsp)->numOfCols = htonl(numOfCols); - int32_t len = blockEncode(pBlock, (*pRsp)->data + sizeof(int32_t)*2, numOfCols); + int32_t len = blockEncode(pBlock, (*pRsp)->data + PAYLOAD_PREFIX_LEN, numOfCols); SET_PAYLOAD_LEN((*pRsp)->data, len, len); int32_t payloadLen = len + PAYLOAD_PREFIX_LEN; diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index 7081901db4..2ac7e6ae3b 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -1953,14 +1953,13 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { rsp->completed = 1; rsp->numOfRows = htobe64((int64_t)rowNum); - int32_t len = blockEncode(pBlock, rsp->data + sizeof(int32_t)*2, taosArrayGetSize(pBlock->pDataBlock)); + int32_t len = blockEncode(pBlock, rsp->data + PAYLOAD_PREFIX_LEN, taosArrayGetSize(pBlock->pDataBlock)); rsp->compLen = htonl(len); - rsp->payloadLen = rsp->compLen; + rsp->payloadLen = htonl(len); rsp->compressed = 0; - ((int32_t*)rsp->data)[0] = len; - ((int32_t*)rsp->data)[1] = len; + SET_PAYLOAD_LEN(rsp->data, len, len); blockDataDestroy(pBlock); diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 28bd05d0c2..e688762aee 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -693,7 +693,7 @@ int32_t streamAddBlockIntoDispatchMsg(const SSDataBlock* pBlock, SStreamDispatch int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock); pRetrieve->numOfCols = htonl(numOfCols); - int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols); + int32_t actualLen = blockEncode(pBlock, pRetrieve->data + PAYLOAD_PREFIX_LEN, numOfCols); SET_PAYLOAD_LEN(pRetrieve->data, actualLen, actualLen); int32_t payloadLen = actualLen + PAYLOAD_PREFIX_LEN; From 0fe2686b635412f93b4fa44e73069356a9c7653a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 19 May 2024 14:55:51 +0800 Subject: [PATCH 29/33] fix(query): fix bugs caused by refactor. --- source/libs/qworker/src/qworker.c | 6 +++--- source/libs/stream/src/streamData.c | 18 +++++++++++++++++- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 8efe42e9bc..93c5628b71 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -343,13 +343,13 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, // Got data from sink QW_TASK_DLOG("there are data in sink, dataLength:%" PRId64 "", len); - *dataLen += len + sizeof(int32_t) * 2; - *pRawDataLen += rawLen + sizeof(int32_t) * 2; + *dataLen += len + PAYLOAD_PREFIX_LEN; + *pRawDataLen += rawLen + PAYLOAD_PREFIX_LEN; QW_ERR_RET(qwMallocFetchRsp(!ctx->localExec, *dataLen, &pRsp)); // set the serialize start position - output.pData = pRsp->data + *dataLen - (len + sizeof(int32_t) * 2); + output.pData = pRsp->data + *dataLen - (len + PAYLOAD_PREFIX_LEN); ((int32_t*) output.pData)[0] = len; ((int32_t*) output.pData)[1] = rawLen; diff --git a/source/libs/stream/src/streamData.c b/source/libs/stream/src/streamData.c index 309234179c..5f3c6f671b 100644 --- a/source/libs/stream/src/streamData.c +++ b/source/libs/stream/src/streamData.c @@ -37,7 +37,23 @@ SStreamDataBlock* createStreamBlockFromDispatchMsg(const SStreamDispatchReq* pRe for (int32_t i = 0; i < blockNum; i++) { SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*) taosArrayGetP(pReq->data, i); SSDataBlock* pDataBlock = taosArrayGet(pArray, i); - blockDecode(pDataBlock, pRetrieve->data); + + int32_t compLen = *(int32_t*)pRetrieve->data; + int32_t fullLen = *(int32_t*)(pRetrieve->data + sizeof(int32_t)); + + char* pInput = pRetrieve->data + PAYLOAD_PREFIX_LEN; + if (pRetrieve->compressed && compLen < fullLen) { + char* p = taosMemoryMalloc(fullLen); + int32_t len = tsDecompressString(pInput, compLen, 1, p, fullLen, ONE_STAGE_COMP, NULL, 0); + ASSERT(len == fullLen); + pInput = p; + } + + blockDecode(pDataBlock, pInput); + + if (pRetrieve->compressed && compLen < fullLen) { + taosMemoryFree(pInput); + } // TODO: refactor pDataBlock->info.window.skey = be64toh(pRetrieve->skey); From f74fad7e60d696363a52cb923e6b402cda826f05 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Sun, 19 May 2024 17:17:40 +0800 Subject: [PATCH 30/33] fix(query): fix error in refactor. --- source/libs/stream/src/streamData.c | 1 - source/libs/stream/src/streamDispatch.c | 11 ++++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/source/libs/stream/src/streamData.c b/source/libs/stream/src/streamData.c index 5f3c6f671b..fa4efc3c6e 100644 --- a/source/libs/stream/src/streamData.c +++ b/source/libs/stream/src/streamData.c @@ -33,7 +33,6 @@ SStreamDataBlock* createStreamBlockFromDispatchMsg(const SStreamDispatchReq* pRe } ASSERT((pReq->blockNum == taosArrayGetSize(pReq->data)) && (pReq->blockNum == taosArrayGetSize(pReq->dataLen))); - for (int32_t i = 0; i < blockNum; i++) { SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*) taosArrayGetP(pReq->data, i); SSDataBlock* pDataBlock = taosArrayGet(pArray, i); diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index e688762aee..f9bd0bbfc0 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -130,9 +130,9 @@ int32_t streamTaskBroadcastRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* r static int32_t buildStreamRetrieveReq(SStreamTask* pTask, const SSDataBlock* pBlock, SStreamRetrieveReq* req){ SRetrieveTableRsp* pRetrieve = NULL; - int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock) + PAYLOAD_PREFIX_LEN; + int32_t len = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock) + PAYLOAD_PREFIX_LEN; - pRetrieve = taosMemoryCalloc(1, dataStrLen); + pRetrieve = taosMemoryCalloc(1, len); if (pRetrieve == NULL) return TSDB_CODE_OUT_OF_MEMORY; int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); @@ -148,7 +148,6 @@ static int32_t buildStreamRetrieveReq(SStreamTask* pTask, const SSDataBlock* pBl pRetrieve->version = htobe64(pBlock->info.version); int32_t actualLen = blockEncode(pBlock, pRetrieve->data+ PAYLOAD_PREFIX_LEN, numOfCols); - SET_PAYLOAD_LEN(pRetrieve->data, actualLen, actualLen); int32_t payloadLen = actualLen + PAYLOAD_PREFIX_LEN; @@ -160,7 +159,7 @@ static int32_t buildStreamRetrieveReq(SStreamTask* pTask, const SSDataBlock* pBl req->srcNodeId = pTask->info.nodeId; req->srcTaskId = pTask->id.taskId; req->pRetrieve = pRetrieve; - req->retrieveLen = dataStrLen; + req->retrieveLen = len; return 0; } @@ -700,7 +699,9 @@ int32_t streamAddBlockIntoDispatchMsg(const SSDataBlock* pBlock, SStreamDispatch pRetrieve->payloadLen = htonl(payloadLen); pRetrieve->compLen = htonl(payloadLen); - taosArrayPush(pReq->dataLen, &actualLen); + payloadLen += sizeof(SRetrieveTableRsp); + + taosArrayPush(pReq->dataLen, &payloadLen); taosArrayPush(pReq->data, &buf); pReq->totalLen += dataStrLen; From 5a91039f040330d302a7d982fad3773e55218c46 Mon Sep 17 00:00:00 2001 From: Yihao Deng Date: Thu, 30 May 2024 03:00:56 +0000 Subject: [PATCH 31/33] TD-30340 --- include/libs/transport/trpc.h | 1 + source/libs/transport/src/transCli.c | 13 +++++++++++-- source/libs/transport/src/transSvr.c | 11 ++++++----- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index c3b5beb3e3..b7a459f957 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -62,6 +62,7 @@ typedef struct SRpcHandleInfo { SRpcConnInfo conn; int8_t forbiddenIp; int8_t notFreeAhandle; + int8_t compressed; } SRpcHandleInfo; typedef struct SRpcMsg { diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index d0fb5cbfdb..8969d98228 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -76,6 +76,9 @@ typedef struct SCliConn { SDelayTask* task; + uint32_t clientIp; + uint32_t serverIp; + char* dstAddr; char src[32]; char dst[32]; @@ -1089,7 +1092,7 @@ void cliSendBatch(SCliConn* pConn) { } pHead->timestamp = taosHton64(taosGetTimestampUs()); - if (pHead->comp == 0) { + if (pHead->comp == 0 && pMsg->info.compressed == 0 && pConn->clientIp != pConn->serverIp) { if (pTransInst->compressSize != -1 && pTransInst->compressSize < pMsg->contLen) { msgLen = transCompressMsg(pMsg->pCont, pMsg->contLen) + sizeof(STransMsgHead); pHead->msgLen = (int32_t)htonl((uint32_t)msgLen); @@ -1167,7 +1170,7 @@ void cliSend(SCliConn* pConn) { uv_timer_start((uv_timer_t*)pConn->timer, cliReadTimeoutCb, TRANS_READ_TIMEOUT, 0); } - if (pHead->comp == 0) { + if (pHead->comp == 0 && pMsg->info.compressed == 0 && pConn->clientIp != pConn->serverIp) { if (pTransInst->compressSize != -1 && pTransInst->compressSize < pMsg->contLen) { msgLen = transCompressMsg(pMsg->pCont, pMsg->contLen) + sizeof(STransMsgHead); pHead->msgLen = (int32_t)htonl((uint32_t)msgLen); @@ -1398,6 +1401,12 @@ void cliConnCb(uv_connect_t* req, int status) { uv_tcp_getsockname((uv_tcp_t*)pConn->stream, &sockname, &addrlen); transSockInfo2Str(&sockname, pConn->src); + struct sockaddr_in addr = *(struct sockaddr_in*)&sockname; + struct sockaddr_in saddr = *(struct sockaddr_in*)&peername; + + pConn->clientIp = addr.sin_addr.s_addr; + pConn->serverIp = saddr.sin_addr.s_addr; + tTrace("%s conn %p connect to server successfully", CONN_GET_INST_LABEL(pConn), pConn); if (pConn->pBatch != NULL) { cliSendBatch(pConn); diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 89c5c71eb0..049554a7c9 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -623,7 +623,8 @@ static int uvPrepareSendData(SSvrMsg* smsg, uv_buf_t* wb) { int32_t len = transMsgLenFromCont(pMsg->contLen); STrans* pTransInst = pConn->pTransInst; - if (pTransInst->compressSize != -1 && pTransInst->compressSize < pMsg->contLen) { + if (pMsg->info.compressed == 0 && pConn->clientIp != pConn->serverIp && pTransInst->compressSize != -1 && + pTransInst->compressSize < pMsg->contLen) { len = transCompressMsg(pMsg->pCont, pMsg->contLen) + sizeof(STransMsgHead); pHead->msgLen = (int32_t)htonl((uint32_t)len); } @@ -688,11 +689,11 @@ static void destroyAllConn(SWorkThrd* pThrd) { QUEUE_REMOVE(h); QUEUE_INIT(h); - SSvrConn* c = QUEUE_DATA(h, SSvrConn, queue); - while (T_REF_VAL_GET(c) >= 2) { - transUnrefSrvHandle(c); + SSvrConn* c = QUEUE_DATA(h, SSvrConn, queue); + while (T_REF_VAL_GET(c) >= 2) { + transUnrefSrvHandle(c); } - transUnrefSrvHandle(c); + transUnrefSrvHandle(c); } } void uvWorkerAsyncCb(uv_async_t* handle) { From 6d7e1eb576381457d4c22f36f170818a1663597a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 4 Jun 2024 15:07:02 +0800 Subject: [PATCH 32/33] fix(query): check for the server ep and client ep to set the compress flag. --- include/common/tmsg.h | 1 + include/libs/executor/dataSinkMgt.h | 1 + include/libs/executor/executor.h | 5 +++-- include/libs/qworker/qworker.h | 1 + include/libs/scheduler/scheduler.h | 10 ---------- source/client/inc/clientInt.h | 1 - source/client/src/clientEnv.c | 1 - source/common/src/tmsg.c | 2 ++ source/libs/executor/inc/dataSinkInt.h | 1 - source/libs/executor/src/dataDispatcher.c | 3 ++- source/libs/executor/src/executor.c | 9 +++++---- source/libs/qworker/src/qwMsg.c | 10 ++++++---- source/libs/qworker/src/qworker.c | 20 +++++--------------- source/libs/scheduler/src/schRemote.c | 9 +++++++++ 14 files changed, 35 insertions(+), 39 deletions(-) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index b5c0a84063..29b9e39fb2 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -2498,6 +2498,7 @@ typedef struct SSubQueryMsg { int8_t taskType; int8_t explain; int8_t needFetch; + int8_t compress; uint32_t sqlLen; char* sql; uint32_t msgLen; diff --git a/include/libs/executor/dataSinkMgt.h b/include/libs/executor/dataSinkMgt.h index a052bc3359..130de8c030 100644 --- a/include/libs/executor/dataSinkMgt.h +++ b/include/libs/executor/dataSinkMgt.h @@ -55,6 +55,7 @@ typedef struct SDataSinkStat { } SDataSinkStat; typedef struct SDataSinkMgtCfg { + int8_t compress; uint32_t maxDataBlockNum; // todo: this should be numOfRows? uint32_t maxDataBlockNumPerQuery; } SDataSinkMgtCfg; diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index fe20639c77..2774fce33b 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -139,8 +139,9 @@ void qUpdateOperatorParam(qTaskInfo_t tinfo, void* pParam); * @param qId * @return */ -int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, struct SSubplan* pPlan, - qTaskInfo_t* pTaskInfo, DataSinkHandle* handle, char* sql, EOPTR_EXEC_MODEL model); +int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, struct SSubplan* pSubplan, + qTaskInfo_t* pTaskInfo, DataSinkHandle* handle, int8_t compressResult, char* sql, + EOPTR_EXEC_MODEL model); /** * diff --git a/include/libs/qworker/qworker.h b/include/libs/qworker/qworker.h index bbd2d76b59..ac27534ab0 100644 --- a/include/libs/qworker/qworker.h +++ b/include/libs/qworker/qworker.h @@ -61,6 +61,7 @@ typedef struct SQWMsgInfo { int8_t taskType; int8_t explain; int8_t needFetch; + int8_t compressMsg; } SQWMsgInfo; typedef struct SQWMsg { diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index 952af3c443..1e2ccf8705 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -48,16 +48,6 @@ typedef struct SQueryProfileSummary { uint64_t resultSize; // generated result size in Kb. } SQueryProfileSummary; -typedef struct STaskInfo { - SQueryNodeAddr addr; - SSubQueryMsg* msg; -} STaskInfo; - -typedef struct SSchdFetchParam { - void** pData; - int32_t* code; -} SSchdFetchParam; - typedef void (*schedulerExecFp)(SExecResult* pResult, void* param, int32_t code); typedef void (*schedulerFetchFp)(void* pResult, void* param, int32_t code); typedef bool (*schedulerChkKillFp)(void* param); diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 21dd930c8f..21d9729c48 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -122,7 +122,6 @@ struct SAppInstInfo { typedef struct SAppInfo { int64_t startTime; char appName[TSDB_APP_NAME_LEN]; - char* ep; int32_t pid; int32_t numOfThreads; SHashObj* pInstMap; diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 77294e7a34..0c13265ad2 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -757,7 +757,6 @@ void taos_init_imp(void) { clientConnRefPool = taosOpenRef(200, destroyTscObj); clientReqRefPool = taosOpenRef(40960, doDestroyRequest); - // transDestroyBuffer(&conn->readBuf); taosGetAppName(appInfo.appName, NULL); taosThreadMutexInit(&appInfo.mutex, NULL); diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index b7d1417451..778f149a58 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -6963,6 +6963,7 @@ int32_t tSerializeSSubQueryMsg(void *buf, int32_t bufLen, SSubQueryMsg *pReq) { if (tEncodeI8(&encoder, pReq->taskType) < 0) return -1; if (tEncodeI8(&encoder, pReq->explain) < 0) return -1; if (tEncodeI8(&encoder, pReq->needFetch) < 0) return -1; + if (tEncodeI8(&encoder, pReq->compress) < 0) return -1; if (tEncodeU32(&encoder, pReq->sqlLen) < 0) return -1; if (tEncodeCStrWithLen(&encoder, pReq->sql, pReq->sqlLen) < 0) return -1; if (tEncodeU32(&encoder, pReq->msgLen) < 0) return -1; @@ -7003,6 +7004,7 @@ int32_t tDeserializeSSubQueryMsg(void *buf, int32_t bufLen, SSubQueryMsg *pReq) if (tDecodeI8(&decoder, &pReq->taskType) < 0) return -1; if (tDecodeI8(&decoder, &pReq->explain) < 0) return -1; if (tDecodeI8(&decoder, &pReq->needFetch) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->compress) < 0) return -1; if (tDecodeU32(&decoder, &pReq->sqlLen) < 0) return -1; if (tDecodeCStrAlloc(&decoder, &pReq->sql) < 0) return -1; if (tDecodeU32(&decoder, &pReq->msgLen) < 0) return -1; diff --git a/source/libs/executor/inc/dataSinkInt.h b/source/libs/executor/inc/dataSinkInt.h index a294d7e487..01bc762a1d 100644 --- a/source/libs/executor/inc/dataSinkInt.h +++ b/source/libs/executor/inc/dataSinkInt.h @@ -25,7 +25,6 @@ extern "C" { #include "storageapi.h" #include "tcommon.h" -struct SDataSink; struct SDataSinkHandle; typedef struct SDataSinkManager { diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index 5058c652ce..51784b345b 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -82,7 +82,7 @@ static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pIn pBuf->useSize = sizeof(SDataCacheEntry); { - if (pBuf->allocSize > 16384) { + if ((pBuf->allocSize > tsCompressMsgSize) && (tsCompressMsgSize > 0) && pHandle->pManager->cfg.compress) { char* p = taosMemoryMalloc(pBuf->allocSize); int32_t dataLen = blockEncode(pInput->pData, p, numOfCols); @@ -295,6 +295,7 @@ int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSinkNode* pD terrno = TSDB_CODE_OUT_OF_MEMORY; goto _return; } + *pHandle = dispatcher; return TSDB_CODE_SUCCESS; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 2f360044c9..390eee2203 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -286,7 +286,7 @@ qTaskInfo_t qCreateQueueExecTaskInfo(void* msg, SReadHandle* pReaderHandle, int3 } qTaskInfo_t pTaskInfo = NULL; - code = qCreateExecTask(pReaderHandle, vgId, 0, pPlan, &pTaskInfo, NULL, NULL, OPTR_EXEC_MODEL_QUEUE); + code = qCreateExecTask(pReaderHandle, vgId, 0, pPlan, &pTaskInfo, NULL, 0, NULL, OPTR_EXEC_MODEL_QUEUE); if (code != TSDB_CODE_SUCCESS) { nodesDestroyNode((SNode*)pPlan); qDestroyTask(pTaskInfo); @@ -322,7 +322,7 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers, int32_t v } qTaskInfo_t pTaskInfo = NULL; - code = qCreateExecTask(readers, vgId, taskId, pPlan, &pTaskInfo, NULL, NULL, OPTR_EXEC_MODEL_STREAM); + code = qCreateExecTask(readers, vgId, taskId, pPlan, &pTaskInfo, NULL, 0, NULL, OPTR_EXEC_MODEL_STREAM); if (code != TSDB_CODE_SUCCESS) { nodesDestroyNode((SNode*)pPlan); qDestroyTask(pTaskInfo); @@ -524,7 +524,8 @@ void qUpdateOperatorParam(qTaskInfo_t tinfo, void* pParam) { } int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, SSubplan* pSubplan, - qTaskInfo_t* pTaskInfo, DataSinkHandle* handle, char* sql, EOPTR_EXEC_MODEL model) { + qTaskInfo_t* pTaskInfo, DataSinkHandle* handle, int8_t compressResult, char* sql, + EOPTR_EXEC_MODEL model) { SExecTaskInfo** pTask = (SExecTaskInfo**)pTaskInfo; taosThreadOnce(&initPoolOnce, initRefPool); @@ -537,7 +538,7 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, } if (handle) { - SDataSinkMgtCfg cfg = {.maxDataBlockNum = 500, .maxDataBlockNumPerQuery = 50}; + SDataSinkMgtCfg cfg = {.maxDataBlockNum = 500, .maxDataBlockNumPerQuery = 50, .compress = compressResult}; void* pSinkManager = NULL; code = dsDataSinkMgtInit(&cfg, &(*pTask)->storageAPI, &pSinkManager); if (code != TSDB_CODE_SUCCESS) { diff --git a/source/libs/qworker/src/qwMsg.c b/source/libs/qworker/src/qwMsg.c index ef9ef42c85..527df395cd 100644 --- a/source/libs/qworker/src/qwMsg.c +++ b/source/libs/qworker/src/qwMsg.c @@ -41,7 +41,6 @@ void qwBuildFetchRsp(void *msg, SOutputData *input, int32_t len, int32_t rawData rsp->numOfRows = htobe64(input->numOfRows); rsp->numOfCols = htonl(input->numOfCols); rsp->numOfBlocks = htonl(input->numOfBlocks); - ASSERT(rawDataLen != 100446); } void qwFreeFetchRsp(void *msg) { @@ -156,6 +155,7 @@ int32_t qwBuildAndSendFetchRsp(int32_t rspType, SRpcHandleInfo *pConn, SRetrieve .info = *pConn, }; + rpcRsp.info.compressed = pRsp->compressed; tmsgSendRsp(&rpcRsp); return TSDB_CODE_SUCCESS; @@ -454,13 +454,15 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg, int qwMsg.msgInfo.explain = msg.explain; qwMsg.msgInfo.taskType = msg.taskType; qwMsg.msgInfo.needFetch = msg.needFetch; + qwMsg.msgInfo.compressMsg = msg.compress; + + QW_SCH_TASK_DLOG("processQuery start, node:%p, type:%s, compress:%d, handle:%p, SQL:%s", node, TMSG_INFO(pMsg->msgType), + msg.compress, pMsg->info.handle, msg.sql); - QW_SCH_TASK_DLOG("processQuery start, node:%p, type:%s, handle:%p, SQL:%s", node, TMSG_INFO(pMsg->msgType), - pMsg->info.handle, msg.sql); code = qwProcessQuery(QW_FPARAMS(), &qwMsg, msg.sql); msg.sql = NULL; - QW_SCH_TASK_DLOG("processQuery end, node:%p, code:%x", node, code); + QW_SCH_TASK_DLOG("processQuery end, node:%p, code:%x", node, code); tFreeSSubQueryMsg(&msg); return TSDB_CODE_SUCCESS; diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 93c5628b71..c181997ad2 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -369,6 +369,7 @@ int32_t qwGetQueryResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen, if (output.compressed) { pOutput->compressed = output.compressed; } + pOutput->numOfCols = output.numOfCols; pOutput->numOfRows += output.numOfRows; pOutput->numOfBlocks++; @@ -723,7 +724,6 @@ _return: int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, char *sql) { int32_t code = 0; - bool queryRsped = false; SSubplan *plan = NULL; SQWPhaseInput input = {0}; qTaskInfo_t pTaskInfo = NULL; @@ -740,8 +740,6 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, char *sql) { ctx->queryMsgType = qwMsg->msgType; ctx->localExec = false; - // QW_TASK_DLOGL("subplan json string, len:%d, %s", qwMsg->msgLen, qwMsg->msg); - code = qMsgToSubplan(qwMsg->msg, qwMsg->msgLen, &plan); if (TSDB_CODE_SUCCESS != code) { code = TSDB_CODE_INVALID_MSG; @@ -749,14 +747,7 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, char *sql) { QW_ERR_JRET(code); } -#if 0 - SReadHandle* pReadHandle = qwMsg->node; - int64_t delay = 0; - bool fhFinish = false; - pReadHandle->api.tqReaderFn.tqGetStreamExecProgress(pReadHandle->vnode, 0, &delay, &fhFinish); -#endif - - code = qCreateExecTask(qwMsg->node, mgmt->nodeId, tId, plan, &pTaskInfo, &sinkHandle, sql, OPTR_EXEC_MODEL_BATCH); + code = qCreateExecTask(qwMsg->node, mgmt->nodeId, tId, plan, &pTaskInfo, &sinkHandle, qwMsg->msgInfo.compressMsg, sql, OPTR_EXEC_MODEL_BATCH); sql = NULL; if (code) { QW_TASK_ELOG("qCreateExecTask failed, code:%x - %s", code, tstrerror(code)); @@ -768,8 +759,6 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, char *sql) { QW_ERR_JRET(TSDB_CODE_APP_ERROR); } - //qwSendQueryRsp(QW_FPARAMS(), qwMsg->msgType + 1, ctx, code, true); - ctx->level = plan->level; ctx->dynamicTask = qIsDynamicExecTask(pTaskInfo); atomic_store_ptr(&ctx->taskHandle, pTaskInfo); @@ -964,6 +953,7 @@ _return: qwDbgSimulateRedirect(qwMsg, ctx, &rsped); qwDbgSimulateDead(QW_FPARAMS(), ctx, &rsped); } + if (!rsped) { qwBuildAndSendFetchRsp(qwMsg->msgType + 1, &qwMsg->connInfo, rsp, dataLen, code); QW_TASK_DLOG("fetch rsp send, msgType:%s, handle:%p, code:%x - %s, dataLen:%d", TMSG_INFO(qwMsg->msgType + 1), @@ -1235,7 +1225,7 @@ int32_t qwProcessDelete(QW_FPARAMS_DEF, SQWMsg *qwMsg, SDeleteRes *pRes) { QW_ERR_JRET(code); } - code = qCreateExecTask(qwMsg->node, mgmt->nodeId, tId, plan, &pTaskInfo, &sinkHandle, NULL, OPTR_EXEC_MODEL_BATCH); + code = qCreateExecTask(qwMsg->node, mgmt->nodeId, tId, plan, &pTaskInfo, &sinkHandle, 0, NULL, OPTR_EXEC_MODEL_BATCH); if (code) { QW_TASK_ELOG("qCreateExecTask failed, code:%x - %s", code, tstrerror(code)); QW_ERR_JRET(code); @@ -1437,7 +1427,7 @@ int32_t qWorkerProcessLocalQuery(void *pMgmt, uint64_t sId, uint64_t qId, uint64 rHandle.pMsgCb = taosMemoryCalloc(1, sizeof(SMsgCb)); rHandle.pMsgCb->clientRpc = qwMsg->connInfo.handle; - code = qCreateExecTask(&rHandle, mgmt->nodeId, tId, plan, &pTaskInfo, &sinkHandle, NULL, OPTR_EXEC_MODEL_BATCH); + code = qCreateExecTask(&rHandle, mgmt->nodeId, tId, plan, &pTaskInfo, &sinkHandle, 0, NULL, OPTR_EXEC_MODEL_BATCH); if (code) { QW_TASK_ELOG("qCreateExecTask failed, code:%x - %s", code, tstrerror(code)); QW_ERR_JRET(code); diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index 4b8695e8ae..d0dc04d6b4 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -20,6 +20,9 @@ #include "tmsg.h" #include "tref.h" #include "trpc.h" +#include "tglobal.h" +#include "tmisce.h" + // clang-format off int32_t schValidateRspMsgType(SSchJob *pJob, SSchTask *pTask, int32_t msgType) { int32_t lastMsgType = pTask->lastMsgType; @@ -1127,6 +1130,12 @@ int32_t schBuildAndSendMsg(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *addr, qMsg.msgLen = pTask->msgLen; qMsg.msg = pTask->msg; + if (strcmp(tsLocalFqdn, GET_ACTIVE_EP(&addr->epSet)->fqdn) == 0) { + qMsg.compress = 0; + } else { + qMsg.compress = 1; + } + msgSize = tSerializeSSubQueryMsg(NULL, 0, &qMsg); if (msgSize < 0) { SCH_TASK_ELOG("tSerializeSSubQueryMsg get size, msgSize:%d", msgSize); From 2ff79c81d733e3fc602739a923d0f96d2c2ee5ee Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 4 Jun 2024 17:47:50 +0800 Subject: [PATCH 33/33] enh(query): allocate the compress buffer before compress data. --- source/libs/executor/src/dataDispatcher.c | 32 +++++++++++++++++++---- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index 51784b345b..297c87ab40 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -49,6 +49,8 @@ typedef struct SDataDispatchHandle { bool queryEnd; uint64_t useconds; uint64_t cachedSize; + void* pCompressBuf; + int32_t bufSize; TdThreadMutex mutex; } SDataDispatchHandle; @@ -83,10 +85,26 @@ static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pIn { if ((pBuf->allocSize > tsCompressMsgSize) && (tsCompressMsgSize > 0) && pHandle->pManager->cfg.compress) { - char* p = taosMemoryMalloc(pBuf->allocSize); + if (pHandle->pCompressBuf == NULL) { + // allocate additional 8 bytes to avoid invalid write if compress failed to reduce the size + pHandle->pCompressBuf = taosMemoryMalloc(pBuf->allocSize + 8); + pHandle->bufSize = pBuf->allocSize + 8; + } else { + if (pHandle->bufSize < pBuf->allocSize + 8) { + pHandle->bufSize = pBuf->allocSize + 8; + void* p = taosMemoryRealloc(pHandle->pCompressBuf, pHandle->bufSize); + if (p != NULL) { + pHandle->pCompressBuf = p; + } else { + terrno = TSDB_CODE_OUT_OF_MEMORY; + qError("failed to prepare compress buf:%d, code: out of memory", pHandle->bufSize); + return; + } + } + } - int32_t dataLen = blockEncode(pInput->pData, p, numOfCols); - int32_t len = tsCompressString(p, dataLen, 1, pEntry->data, pBuf->allocSize, ONE_STAGE_COMP, NULL, 0); + int32_t dataLen = blockEncode(pInput->pData, pHandle->pCompressBuf, numOfCols); + int32_t len = tsCompressString(pHandle->pCompressBuf, dataLen, 1, pEntry->data, pBuf->allocSize, ONE_STAGE_COMP, NULL, 0); if (len < dataLen) { pEntry->compressed = 1; pEntry->dataLen = len; @@ -95,9 +113,8 @@ static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pIn pEntry->compressed = 0; pEntry->dataLen = dataLen; pEntry->rawLen = dataLen; - memcpy(pEntry->data, p, dataLen); + memcpy(pEntry->data, pHandle->pCompressBuf, dataLen); } - taosMemoryFree(p); } else { pEntry->dataLen = blockEncode(pInput->pData, pEntry->data, numOfCols); pEntry->rawLen = pEntry->dataLen; @@ -247,6 +264,7 @@ static int32_t destroyDataSinker(SDataSinkHandle* pHandle) { SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pDispatcher->cachedSize); taosMemoryFreeClear(pDispatcher->nextOutput.pData); + while (!taosQueueEmpty(pDispatcher->pDataBlocks)) { SDataDispatchBuf* pBuf = NULL; taosReadQitem(pDispatcher->pDataBlocks, (void**)&pBuf); @@ -255,7 +273,11 @@ static int32_t destroyDataSinker(SDataSinkHandle* pHandle) { taosFreeQitem(pBuf); } } + taosCloseQueue(pDispatcher->pDataBlocks); + taosMemoryFreeClear(pDispatcher->pCompressBuf); + pDispatcher->bufSize = 0; + taosThreadMutexDestroy(&pDispatcher->mutex); taosMemoryFree(pDispatcher->pManager); return TSDB_CODE_SUCCESS;