diff --git a/.clang-format b/.clang-format index f60fd3cb26..e58d518b3b 100644 --- a/.clang-format +++ b/.clang-format @@ -5,6 +5,7 @@ AccessModifierOffset: -1 AlignAfterOpenBracket: Align AlignConsecutiveAssignments: false AlignConsecutiveDeclarations: true +AlignConsecutiveMacros: true AlignEscapedNewlinesLeft: true AlignOperands: true AlignTrailingComments: true @@ -86,6 +87,5 @@ SpacesInSquareBrackets: false Standard: Auto TabWidth: 8 UseTab: Never -AlignConsecutiveDeclarations: true ... diff --git a/2.0/src/query/inc/qExecutor.h b/2.0/src/query/inc/qExecutor.h index 0c0e3363c8..9c738dad98 100644 --- a/2.0/src/query/inc/qExecutor.h +++ b/2.0/src/query/inc/qExecutor.h @@ -582,9 +582,9 @@ typedef struct SOrderOperatorInfo { void appendUpstream(SOperatorInfo* p, SOperatorInfo* pUpstream); -SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime, int32_t reverseTime); +SOperatorInfo* createTableScanOperatorInfo(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime, int32_t reverseTime); SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime); -SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv); +SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv); SOperatorInfo* createAggregateOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); SOperatorInfo* createProjectOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); @@ -622,7 +622,7 @@ void doCompactSDataBlock(SSDataBlock* pBlock, int32_t numOfRows, int8_t* p); SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numOfRows); -void* destroyOutputBuf(SSDataBlock* pBlock); +void* blockDataDestroy(SSDataBlock* pBlock); void* doDestroyFilterInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols); void setInputDataBlock(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order); diff --git a/2.0/src/query/src/qExecutor.c b/2.0/src/query/src/qExecutor.c index fdadf39d4d..ca656b81ff 100644 --- a/2.0/src/query/src/qExecutor.c +++ b/2.0/src/query/src/qExecutor.c @@ -336,7 +336,7 @@ SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numO return res; } -void* destroyOutputBuf(SSDataBlock* pBlock) { +void* blockDataDestroy(SSDataBlock* pBlock) { if (pBlock == NULL) { return NULL; } @@ -4835,11 +4835,11 @@ int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr break; } case OP_TableSeqScan: { - pRuntimeEnv->proot = createTableSeqScanOperator(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv); + pRuntimeEnv->proot = createTableSeqScanOperatorInfo(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv); break; } case OP_DataBlocksOptScan: { - pRuntimeEnv->proot = createDataBlocksOptScanInfo(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr), pQueryAttr->needReverseScan? 1:0); + pRuntimeEnv->proot = createTableScanOperatorInfo(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr), pQueryAttr->needReverseScan? 1:0); break; } case OP_TableScan: { @@ -5162,7 +5162,7 @@ SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* return pOperator; } -SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv) { +SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv) { STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); pInfo->pTsdbReadHandle = pTsdbQueryHandle; @@ -5267,7 +5267,7 @@ void setTableScanFilterOperatorInfo(STableScanInfo* pTableScanInfo, SOperatorInf } } -SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime, int32_t reverseTime) { +SOperatorInfo* createTableScanOperatorInfo(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv, int32_t repeatTime, int32_t reverseTime) { assert(repeatTime > 0); STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); @@ -5278,7 +5278,7 @@ SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, SQueryRuntime pInfo->order = pRuntimeEnv->pQueryAttr->order.order; SOperatorInfo* pOptr = calloc(1, sizeof(SOperatorInfo)); - pOptr->name = "DataBlocksOptimizedScanOperator"; + pOptr->name = "TableScanOperator"; pOptr->operatorType = OP_DataBlocksOptScan; pOptr->pRuntimeEnv = pRuntimeEnv; pOptr->blockingOptr = false; @@ -5373,7 +5373,7 @@ static void destroyGlobalAggOperatorInfo(void* param, int32_t numOfOutput) { static void destroySlimitOperatorInfo(void* param, int32_t numOfOutput) { SSLimitOperatorInfo *pInfo = (SSLimitOperatorInfo*) param; taosArrayDestroy(pInfo->orderColumnList); - pInfo->pRes = destroyOutputBuf(pInfo->pRes); + pInfo->pRes = blockDataDestroy(pInfo->pRes); tfree(pInfo->prevRow); } @@ -6566,7 +6566,7 @@ static void doDestroyBasicInfo(SOptrBasicInfo* pInfo, int32_t numOfOutput) { tfree(pInfo->rowCellInfoOffset); cleanupResultRowInfo(&pInfo->resultRowInfo); - pInfo->pRes = destroyOutputBuf(pInfo->pRes); + pInfo->pRes = blockDataDestroy(pInfo->pRes); } static void destroyBasicOperatorInfo(void* param, int32_t numOfOutput) { @@ -6590,7 +6590,7 @@ static void destroySWindowOperatorInfo(void* param, int32_t numOfOutput) { static void destroySFillOperatorInfo(void* param, int32_t numOfOutput) { SFillOperatorInfo* pInfo = (SFillOperatorInfo*) param; pInfo->pFillInfo = taosDestroyFillInfo(pInfo->pFillInfo); - pInfo->pRes = destroyOutputBuf(pInfo->pRes); + pInfo->pRes = blockDataDestroy(pInfo->pRes); tfree(pInfo->p); } @@ -6607,12 +6607,12 @@ static void destroyProjectOperatorInfo(void* param, int32_t numOfOutput) { static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) { STagScanInfo* pInfo = (STagScanInfo*) param; - pInfo->pRes = destroyOutputBuf(pInfo->pRes); + pInfo->pRes = blockDataDestroy(pInfo->pRes); } static void destroyOrderOperatorInfo(void* param, int32_t numOfOutput) { SOrderOperatorInfo* pInfo = (SOrderOperatorInfo*) param; - pInfo->pDataBlock = destroyOutputBuf(pInfo->pDataBlock); + pInfo->pDataBlock = blockDataDestroy(pInfo->pDataBlock); } static void destroyConditionOperatorInfo(void* param, int32_t numOfOutput) { @@ -6625,7 +6625,7 @@ static void destroyDistinctOperatorInfo(void* param, int32_t numOfOutput) { taosHashCleanup(pInfo->pSet); tfree(pInfo->buf); taosArrayDestroy(pInfo->pDistinctDataInfo); - pInfo->pRes = destroyOutputBuf(pInfo->pRes); + pInfo->pRes = blockDataDestroy(pInfo->pRes); } SOperatorInfo* createMultiTableAggOperatorInfo(SQueryRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput) { diff --git a/2.0/src/query/tests/resultBufferTest.cpp b/2.0/src/query/tests/resultBufferTest.cpp index 9724b98f7c..b7173ece46 100644 --- a/2.0/src/query/tests/resultBufferTest.cpp +++ b/2.0/src/query/tests/resultBufferTest.cpp @@ -13,7 +13,7 @@ namespace { // simple test void simpleTest() { - SDiskbasedResultBuf* pResultBuf = NULL; + SDiskbasedBuf* pResultBuf = NULL; int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1024, 4096, 1); int32_t pageId = 0; @@ -22,40 +22,40 @@ void simpleTest() { tFilePage* pBufPage = getNewDataBuf(pResultBuf, groupId, &pageId); ASSERT_TRUE(pBufPage != NULL); - ASSERT_EQ(getResBufSize(pResultBuf), 1024); + ASSERT_EQ(getTotalBufSize(pResultBuf), 1024); SIDList list = getDataBufPagesIdList(pResultBuf, groupId); ASSERT_EQ(taosArrayGetSize(list), 1); ASSERT_EQ(getNumOfResultBufGroupId(pResultBuf), 1); - releaseResBufPage(pResultBuf, pBufPage); + releaseBufPage(pResultBuf, pBufPage); tFilePage* pBufPage1 = getNewDataBuf(pResultBuf, groupId, &pageId); - tFilePage* t = getResBufPage(pResultBuf, pageId); + tFilePage* t = getBufPage(pResultBuf, pageId); ASSERT_TRUE(t == pBufPage1); tFilePage* pBufPage2 = getNewDataBuf(pResultBuf, groupId, &pageId); - tFilePage* t1 = getResBufPage(pResultBuf, pageId); + tFilePage* t1 = getBufPage(pResultBuf, pageId); ASSERT_TRUE(t1 == pBufPage2); tFilePage* pBufPage3 = getNewDataBuf(pResultBuf, groupId, &pageId); - tFilePage* t2 = getResBufPage(pResultBuf, pageId); + tFilePage* t2 = getBufPage(pResultBuf, pageId); ASSERT_TRUE(t2 == pBufPage3); tFilePage* pBufPage4 = getNewDataBuf(pResultBuf, groupId, &pageId); - tFilePage* t3 = getResBufPage(pResultBuf, pageId); + tFilePage* t3 = getBufPage(pResultBuf, pageId); ASSERT_TRUE(t3 == pBufPage4); tFilePage* pBufPage5 = getNewDataBuf(pResultBuf, groupId, &pageId); - tFilePage* t4 = getResBufPage(pResultBuf, pageId); + tFilePage* t4 = getBufPage(pResultBuf, pageId); ASSERT_TRUE(t4 == pBufPage5); destroyResultBuf(pResultBuf); } void writeDownTest() { - SDiskbasedResultBuf* pResultBuf = NULL; + SDiskbasedBuf* pResultBuf = NULL; int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1024, 4*1024, 1); int32_t pageId = 0; @@ -68,31 +68,31 @@ void writeDownTest() { *(int32_t*)(pBufPage->data) = nx; writePageId = pageId; - releaseResBufPage(pResultBuf, pBufPage); + releaseBufPage(pResultBuf, pBufPage); tFilePage* pBufPage1 = getNewDataBuf(pResultBuf, groupId, &pageId); - tFilePage* t1 = getResBufPage(pResultBuf, pageId); + tFilePage* t1 = getBufPage(pResultBuf, pageId); ASSERT_TRUE(t1 == pBufPage1); ASSERT_TRUE(pageId == 1); tFilePage* pBufPage2 = getNewDataBuf(pResultBuf, groupId, &pageId); - tFilePage* t2 = getResBufPage(pResultBuf, pageId); + tFilePage* t2 = getBufPage(pResultBuf, pageId); ASSERT_TRUE(t2 == pBufPage2); ASSERT_TRUE(pageId == 2); tFilePage* pBufPage3 = getNewDataBuf(pResultBuf, groupId, &pageId); - tFilePage* t3 = getResBufPage(pResultBuf, pageId); + tFilePage* t3 = getBufPage(pResultBuf, pageId); ASSERT_TRUE(t3 == pBufPage3); ASSERT_TRUE(pageId == 3); tFilePage* pBufPage4 = getNewDataBuf(pResultBuf, groupId, &pageId); - tFilePage* t4 = getResBufPage(pResultBuf, pageId); + tFilePage* t4 = getBufPage(pResultBuf, pageId); ASSERT_TRUE(t4 == pBufPage4); ASSERT_TRUE(pageId == 4); - releaseResBufPage(pResultBuf, t4); + releaseBufPage(pResultBuf, t4); // flush the written page to disk, and read it out again - tFilePage* pBufPagex = getResBufPage(pResultBuf, writePageId); + tFilePage* pBufPagex = getBufPage(pResultBuf, writePageId); ASSERT_EQ(*(int32_t*)pBufPagex->data, nx); SArray* pa = getDataBufPagesIdList(pResultBuf, groupId); @@ -102,7 +102,7 @@ void writeDownTest() { } void recyclePageTest() { - SDiskbasedResultBuf* pResultBuf = NULL; + SDiskbasedBuf* pResultBuf = NULL; int32_t ret = createDiskbasedResultBuffer(&pResultBuf, 1024, 4*1024, 1); int32_t pageId = 0; @@ -112,41 +112,41 @@ void recyclePageTest() { tFilePage* pBufPage = getNewDataBuf(pResultBuf, groupId, &pageId); ASSERT_TRUE(pBufPage != NULL); - releaseResBufPage(pResultBuf, pBufPage); + releaseBufPage(pResultBuf, pBufPage); tFilePage* pBufPage1 = getNewDataBuf(pResultBuf, groupId, &pageId); - tFilePage* t1 = getResBufPage(pResultBuf, pageId); + tFilePage* t1 = getBufPage(pResultBuf, pageId); ASSERT_TRUE(t1 == pBufPage1); ASSERT_TRUE(pageId == 1); tFilePage* pBufPage2 = getNewDataBuf(pResultBuf, groupId, &pageId); - tFilePage* t2 = getResBufPage(pResultBuf, pageId); + tFilePage* t2 = getBufPage(pResultBuf, pageId); ASSERT_TRUE(t2 == pBufPage2); ASSERT_TRUE(pageId == 2); tFilePage* pBufPage3 = getNewDataBuf(pResultBuf, groupId, &pageId); - tFilePage* t3 = getResBufPage(pResultBuf, pageId); + tFilePage* t3 = getBufPage(pResultBuf, pageId); ASSERT_TRUE(t3 == pBufPage3); ASSERT_TRUE(pageId == 3); tFilePage* pBufPage4 = getNewDataBuf(pResultBuf, groupId, &pageId); - tFilePage* t4 = getResBufPage(pResultBuf, pageId); + tFilePage* t4 = getBufPage(pResultBuf, pageId); ASSERT_TRUE(t4 == pBufPage4); ASSERT_TRUE(pageId == 4); - releaseResBufPage(pResultBuf, t4); + releaseBufPage(pResultBuf, t4); tFilePage* pBufPage5 = getNewDataBuf(pResultBuf, groupId, &pageId); - tFilePage* t5 = getResBufPage(pResultBuf, pageId); + tFilePage* t5 = getBufPage(pResultBuf, pageId); ASSERT_TRUE(t5 == pBufPage5); ASSERT_TRUE(pageId == 5); // flush the written page to disk, and read it out again - tFilePage* pBufPagex = getResBufPage(pResultBuf, writePageId); + tFilePage* pBufPagex = getBufPage(pResultBuf, writePageId); *(int32_t*)(pBufPagex->data) = nx; writePageId = pageId; // update the data - releaseResBufPage(pResultBuf, pBufPagex); + releaseBufPage(pResultBuf, pBufPagex); - tFilePage* pBufPagex1 = getResBufPage(pResultBuf, 1); + tFilePage* pBufPagex1 = getBufPage(pResultBuf, 1); SArray* pa = getDataBufPagesIdList(pResultBuf, groupId); ASSERT_EQ(taosArrayGetSize(pa), 6); diff --git a/example/src/tmq.c b/example/src/tmq.c index 99e0c443dd..d8c90a0127 100644 --- a/example/src/tmq.c +++ b/example/src/tmq.c @@ -13,16 +13,14 @@ * along with this program. If not, see . */ +#include #include #include -#include #include #include "taos.h" -static int running = 1; -static void msg_process(tmq_message_t* message) { - tmqShowMsg(message); -} +static int running = 1; +static void msg_process(tmq_message_t* message) { tmqShowMsg(message); } int32_t init_env() { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -44,29 +42,28 @@ int32_t init_env() { } taos_free_result(pRes); - /*pRes = taos_query(pConn, "create stable if not exists st1 (ts timestamp, k int) tags(a int)");*/ - /*if (taos_errno(pRes) != 0) {*/ - /*printf("failed to create super table 123_$^), reason:%s\n", taos_errstr(pRes));*/ - /*return -1;*/ - /*}*/ - /*taos_free_result(pRes);*/ + pRes = taos_query(pConn, "create stable if not exists st1 (ts timestamp, k int) tags(a int)"); + if (taos_errno(pRes) != 0) { + printf("failed to create super table 123_$^), reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); - /*pRes = taos_query(pConn, "create table if not exists tu using st1 tags(1)");*/ - /*if (taos_errno(pRes) != 0) {*/ - /*printf("failed to create child table tu, reason:%s\n", taos_errstr(pRes));*/ - /*return -1;*/ - /*}*/ - /*taos_free_result(pRes);*/ + pRes = taos_query(pConn, "create table if not exists tu1 using st1 tags(1)"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table tu1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); - /*pRes = taos_query(pConn, "create table if not exists tu2 using st1 tags(2)");*/ - /*if (taos_errno(pRes) != 0) {*/ - /*printf("failed to create child table tu2, reason:%s\n", taos_errstr(pRes));*/ - /*return -1;*/ - /*}*/ - /*taos_free_result(pRes);*/ + pRes = taos_query(pConn, "create table if not exists tu2 using st1 tags(2)"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table tu2, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); - - const char* sql = "select * from st1"; + const char* sql = "select * from tu1"; pRes = tmq_create_topic(pConn, "test_stb_topic_1", sql, strlen(sql)); if (taos_errno(pRes) != 0) { printf("failed to create topic test_stb_topic_1, reason:%s\n", taos_errstr(pRes)); @@ -91,11 +88,6 @@ tmq_t* build_consumer() { tmq_conf_set(conf, "group.id", "tg2"); tmq_t* tmq = tmq_consumer_new(pConn, conf, NULL, 0); return tmq; - - tmq_list_t* topic_list = tmq_list_new(); - tmq_list_append(topic_list, "test_stb_topic_1"); - tmq_subscribe(tmq, topic_list); - return NULL; } tmq_list_t* build_topic_list() { @@ -104,8 +96,7 @@ tmq_list_t* build_topic_list() { return topic_list; } -void basic_consume_loop(tmq_t *tmq, - tmq_list_t *topics) { +void basic_consume_loop(tmq_t* tmq, tmq_list_t* topics) { tmq_resp_err_t err; if ((err = tmq_subscribe(tmq, topics))) { @@ -113,15 +104,15 @@ void basic_consume_loop(tmq_t *tmq, printf("subscribe err\n"); return; } - int32_t cnt = 0; + /*int32_t cnt = 0;*/ /*clock_t startTime = clock();*/ while (running) { - tmq_message_t *tmqmessage = tmq_consumer_poll(tmq, 500); + tmq_message_t* tmqmessage = tmq_consumer_poll(tmq, 500); if (tmqmessage) { - cnt++; + /*cnt++;*/ msg_process(tmqmessage); tmq_message_destroy(tmqmessage); - /*} else {*/ + /*} else {*/ /*break;*/ } } @@ -135,11 +126,10 @@ void basic_consume_loop(tmq_t *tmq, fprintf(stderr, "%% Consumer closed\n"); } -void sync_consume_loop(tmq_t *tmq, - tmq_list_t *topics) { +void sync_consume_loop(tmq_t* tmq, tmq_list_t* topics) { static const int MIN_COMMIT_COUNT = 1000; - int msg_count = 0; + int msg_count = 0; tmq_resp_err_t err; if ((err = tmq_subscribe(tmq, topics))) { @@ -148,15 +138,14 @@ void sync_consume_loop(tmq_t *tmq, } while (running) { - tmq_message_t *tmqmessage = tmq_consumer_poll(tmq, 500); + tmq_message_t* tmqmessage = tmq_consumer_poll(tmq, 500); if (tmqmessage) { msg_process(tmqmessage); tmq_message_destroy(tmqmessage); - if ((++msg_count % MIN_COMMIT_COUNT) == 0) - tmq_commit(tmq, NULL, 0); + if ((++msg_count % MIN_COMMIT_COUNT) == 0) tmq_commit(tmq, NULL, 0); } - } + } err = tmq_consumer_close(tmq); if (err) @@ -165,11 +154,48 @@ void sync_consume_loop(tmq_t *tmq, fprintf(stderr, "%% Consumer closed\n"); } -int main() { +void perf_loop(tmq_t* tmq, tmq_list_t* topics) { + tmq_resp_err_t err; + + if ((err = tmq_subscribe(tmq, topics))) { + fprintf(stderr, "%% Failed to start consuming topics: %s\n", tmq_err2str(err)); + printf("subscribe err\n"); + return; + } + int32_t batchCnt = 0; + int32_t skipLogNum = 0; + clock_t startTime = clock(); + while (running) { + tmq_message_t* tmqmessage = tmq_consumer_poll(tmq, 500); + if (tmqmessage) { + batchCnt++; + skipLogNum += tmqGetSkipLogNum(tmqmessage); + /*msg_process(tmqmessage);*/ + tmq_message_destroy(tmqmessage); + } else { + break; + } + } + clock_t endTime = clock(); + printf("log batch cnt: %d, skip log cnt: %d, time used:%f s\n", batchCnt, skipLogNum, + (double)(endTime - startTime) / CLOCKS_PER_SEC); + + err = tmq_consumer_close(tmq); + if (err) + fprintf(stderr, "%% Failed to close consumer: %s\n", tmq_err2str(err)); + else + fprintf(stderr, "%% Consumer closed\n"); +} + +int main(int argc, char* argv[]) { int code; - code = init_env(); - tmq_t* tmq = build_consumer(); + if (argc > 1) { + printf("env init\n"); + code = init_env(); + } + tmq_t* tmq = build_consumer(); tmq_list_t* topic_list = build_topic_list(); + /*perf_loop(tmq, topic_list);*/ basic_consume_loop(tmq, topic_list); /*sync_consume_loop(tmq, topic_list);*/ } diff --git a/include/client/taos.h b/include/client/taos.h index a8627a43da..a960d37937 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -16,8 +16,8 @@ #ifndef TDENGINE_TAOS_H #define TDENGINE_TAOS_H -#include #include +#include #ifdef __cplusplus extern "C" { @@ -31,26 +31,26 @@ typedef void TAOS_SUB; typedef void **TAOS_ROW; // Data type definition -#define TSDB_DATA_TYPE_NULL 0 // 1 bytes -#define TSDB_DATA_TYPE_BOOL 1 // 1 bytes -#define TSDB_DATA_TYPE_TINYINT 2 // 1 byte -#define TSDB_DATA_TYPE_SMALLINT 3 // 2 bytes -#define TSDB_DATA_TYPE_INT 4 // 4 bytes -#define TSDB_DATA_TYPE_BIGINT 5 // 8 bytes -#define TSDB_DATA_TYPE_FLOAT 6 // 4 bytes -#define TSDB_DATA_TYPE_DOUBLE 7 // 8 bytes -#define TSDB_DATA_TYPE_BINARY 8 // string, alias for varchar -#define TSDB_DATA_TYPE_TIMESTAMP 9 // 8 bytes -#define TSDB_DATA_TYPE_NCHAR 10 // unicode string -#define TSDB_DATA_TYPE_UTINYINT 11 // 1 byte -#define TSDB_DATA_TYPE_USMALLINT 12 // 2 bytes -#define TSDB_DATA_TYPE_UINT 13 // 4 bytes -#define TSDB_DATA_TYPE_UBIGINT 14 // 8 bytes -#define TSDB_DATA_TYPE_VARCHAR 15 // string -#define TSDB_DATA_TYPE_VARBINARY 16 // binary -#define TSDB_DATA_TYPE_JSON 17 // json -#define TSDB_DATA_TYPE_DECIMAL 18 // decimal -#define TSDB_DATA_TYPE_BLOB 19 // binary +#define TSDB_DATA_TYPE_NULL 0 // 1 bytes +#define TSDB_DATA_TYPE_BOOL 1 // 1 bytes +#define TSDB_DATA_TYPE_TINYINT 2 // 1 byte +#define TSDB_DATA_TYPE_SMALLINT 3 // 2 bytes +#define TSDB_DATA_TYPE_INT 4 // 4 bytes +#define TSDB_DATA_TYPE_BIGINT 5 // 8 bytes +#define TSDB_DATA_TYPE_FLOAT 6 // 4 bytes +#define TSDB_DATA_TYPE_DOUBLE 7 // 8 bytes +#define TSDB_DATA_TYPE_BINARY 8 // string, alias for varchar +#define TSDB_DATA_TYPE_TIMESTAMP 9 // 8 bytes +#define TSDB_DATA_TYPE_NCHAR 10 // unicode string +#define TSDB_DATA_TYPE_UTINYINT 11 // 1 byte +#define TSDB_DATA_TYPE_USMALLINT 12 // 2 bytes +#define TSDB_DATA_TYPE_UINT 13 // 4 bytes +#define TSDB_DATA_TYPE_UBIGINT 14 // 8 bytes +#define TSDB_DATA_TYPE_VARCHAR 15 // string +#define TSDB_DATA_TYPE_VARBINARY 16 // binary +#define TSDB_DATA_TYPE_JSON 17 // json +#define TSDB_DATA_TYPE_DECIMAL 18 // decimal +#define TSDB_DATA_TYPE_BLOB 19 // binary typedef enum { TSDB_OPTION_LOCALE, @@ -63,9 +63,9 @@ typedef enum { typedef enum { TSDB_SML_UNKNOWN_PROTOCOL = 0, - TSDB_SML_LINE_PROTOCOL = 1, - TSDB_SML_TELNET_PROTOCOL = 2, - TSDB_SML_JSON_PROTOCOL = 3, + TSDB_SML_LINE_PROTOCOL = 1, + TSDB_SML_TELNET_PROTOCOL = 2, + TSDB_SML_JSON_PROTOCOL = 3, } TSDB_SML_PROTOCOL_TYPE; typedef enum { @@ -79,28 +79,28 @@ typedef enum { } TSDB_SML_TIMESTAMP_TYPE; typedef struct taosField { - char name[65]; - int8_t type; - int32_t bytes; + char name[65]; + int8_t type; + int32_t bytes; } TAOS_FIELD; #ifdef _TD_GO_DLL_ - #define DLL_EXPORT __declspec(dllexport) +#define DLL_EXPORT __declspec(dllexport) #else - #define DLL_EXPORT +#define DLL_EXPORT #endif typedef void (*__taos_async_fn_t)(void *param, TAOS_RES *, int code); typedef struct TAOS_BIND { - int buffer_type; - void * buffer; - uintptr_t buffer_length; // unused - uintptr_t *length; - int * is_null; + int buffer_type; + void *buffer; + uintptr_t buffer_length; // unused + uintptr_t *length; + int *is_null; - int is_unsigned; // unused - int * error; // unused + int is_unsigned; // unused + int *error; // unused union { int64_t ts; int8_t b; @@ -113,22 +113,23 @@ typedef struct TAOS_BIND { unsigned char *bin; char *nchar; } u; - unsigned int allocated; + unsigned int allocated; } TAOS_BIND; typedef struct TAOS_MULTI_BIND { - int buffer_type; - void *buffer; - uintptr_t buffer_length; - int32_t *length; - char *is_null; - int num; + int buffer_type; + void *buffer; + uintptr_t buffer_length; + int32_t *length; + char *is_null; + int num; } TAOS_MULTI_BIND; DLL_EXPORT void taos_cleanup(void); DLL_EXPORT int taos_options(TSDB_OPTION option, const void *arg, ...); DLL_EXPORT TAOS *taos_connect(const char *ip, const char *user, const char *pass, const char *db, uint16_t port); -DLL_EXPORT TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, const char *pass, int passLen, const char *db, int dbLen, uint16_t port); +DLL_EXPORT TAOS *taos_connect_l(const char *ip, int ipLen, const char *user, int userLen, const char *pass, int passLen, + const char *db, int dbLen, uint16_t port); DLL_EXPORT TAOS *taos_connect_auth(const char *ip, const char *user, const char *auth, const char *db, uint16_t port); DLL_EXPORT void taos_close(TAOS *taos); @@ -136,62 +137,63 @@ const char *taos_data_type(int type); DLL_EXPORT TAOS_STMT *taos_stmt_init(TAOS *taos); DLL_EXPORT int taos_stmt_prepare(TAOS_STMT *stmt, const char *sql, unsigned long length); -DLL_EXPORT int taos_stmt_set_tbname_tags(TAOS_STMT* stmt, const char* name, TAOS_BIND* tags); -DLL_EXPORT int taos_stmt_set_tbname(TAOS_STMT* stmt, const char* name); -DLL_EXPORT int taos_stmt_set_sub_tbname(TAOS_STMT* stmt, const char* name); +DLL_EXPORT int taos_stmt_set_tbname_tags(TAOS_STMT *stmt, const char *name, TAOS_BIND *tags); +DLL_EXPORT int taos_stmt_set_tbname(TAOS_STMT *stmt, const char *name); +DLL_EXPORT int taos_stmt_set_sub_tbname(TAOS_STMT *stmt, const char *name); -DLL_EXPORT int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert); -DLL_EXPORT int taos_stmt_num_params(TAOS_STMT *stmt, int *nums); -DLL_EXPORT int taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes); -DLL_EXPORT int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND *bind); -DLL_EXPORT int taos_stmt_bind_param_batch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind); -DLL_EXPORT int taos_stmt_bind_single_param_batch(TAOS_STMT* stmt, TAOS_MULTI_BIND* bind, int colIdx); -DLL_EXPORT int taos_stmt_add_batch(TAOS_STMT *stmt); -DLL_EXPORT int taos_stmt_execute(TAOS_STMT *stmt); -DLL_EXPORT TAOS_RES * taos_stmt_use_result(TAOS_STMT *stmt); -DLL_EXPORT int taos_stmt_close(TAOS_STMT *stmt); -DLL_EXPORT char * taos_stmt_errstr(TAOS_STMT *stmt); +DLL_EXPORT int taos_stmt_is_insert(TAOS_STMT *stmt, int *insert); +DLL_EXPORT int taos_stmt_num_params(TAOS_STMT *stmt, int *nums); +DLL_EXPORT int taos_stmt_get_param(TAOS_STMT *stmt, int idx, int *type, int *bytes); +DLL_EXPORT int taos_stmt_bind_param(TAOS_STMT *stmt, TAOS_BIND *bind); +DLL_EXPORT int taos_stmt_bind_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind); +DLL_EXPORT int taos_stmt_bind_single_param_batch(TAOS_STMT *stmt, TAOS_MULTI_BIND *bind, int colIdx); +DLL_EXPORT int taos_stmt_add_batch(TAOS_STMT *stmt); +DLL_EXPORT int taos_stmt_execute(TAOS_STMT *stmt); +DLL_EXPORT TAOS_RES *taos_stmt_use_result(TAOS_STMT *stmt); +DLL_EXPORT int taos_stmt_close(TAOS_STMT *stmt); +DLL_EXPORT char *taos_stmt_errstr(TAOS_STMT *stmt); DLL_EXPORT TAOS_RES *taos_query(TAOS *taos, const char *sql); DLL_EXPORT TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen); DLL_EXPORT TAOS_ROW taos_fetch_row(TAOS_RES *res); -DLL_EXPORT int taos_result_precision(TAOS_RES *res); // get the time precision of result -DLL_EXPORT void taos_free_result(TAOS_RES *res); -DLL_EXPORT int taos_field_count(TAOS_RES *res); -DLL_EXPORT int taos_num_fields(TAOS_RES *res); -DLL_EXPORT int taos_affected_rows(TAOS_RES *res); +DLL_EXPORT int taos_result_precision(TAOS_RES *res); // get the time precision of result +DLL_EXPORT void taos_free_result(TAOS_RES *res); +DLL_EXPORT int taos_field_count(TAOS_RES *res); +DLL_EXPORT int taos_num_fields(TAOS_RES *res); +DLL_EXPORT int taos_affected_rows(TAOS_RES *res); DLL_EXPORT TAOS_FIELD *taos_fetch_fields(TAOS_RES *res); -DLL_EXPORT int taos_select_db(TAOS *taos, const char *db); -DLL_EXPORT int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields); -DLL_EXPORT void taos_stop_query(TAOS_RES *res); -DLL_EXPORT bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col); -DLL_EXPORT int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows); -DLL_EXPORT int taos_validate_sql(TAOS *taos, const char *sql); +DLL_EXPORT int taos_select_db(TAOS *taos, const char *db); +DLL_EXPORT int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields); +DLL_EXPORT void taos_stop_query(TAOS_RES *res); +DLL_EXPORT bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col); +DLL_EXPORT int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows); +DLL_EXPORT int taos_validate_sql(TAOS *taos, const char *sql); -DLL_EXPORT int* taos_fetch_lengths(TAOS_RES *res); +DLL_EXPORT int *taos_fetch_lengths(TAOS_RES *res); DLL_EXPORT const char *taos_get_server_info(TAOS *taos); DLL_EXPORT const char *taos_get_client_info(); DLL_EXPORT const char *taos_errstr(TAOS_RES *tres); -DLL_EXPORT int taos_errno(TAOS_RES *tres); +DLL_EXPORT int taos_errno(TAOS_RES *tres); DLL_EXPORT void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param); DLL_EXPORT void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param); -typedef void (*__taos_sub_fn_t)(TAOS_SUB* tsub, TAOS_RES *res, void* param, int code); -DLL_EXPORT TAOS_SUB *taos_subscribe(TAOS* taos, int restart, const char* topic, const char *sql, __taos_sub_fn_t fp, void *param, int interval); +typedef void (*__taos_sub_fn_t)(TAOS_SUB *tsub, TAOS_RES *res, void *param, int code); +DLL_EXPORT TAOS_SUB *taos_subscribe(TAOS *taos, int restart, const char *topic, const char *sql, __taos_sub_fn_t fp, + void *param, int interval); DLL_EXPORT TAOS_RES *taos_consume(TAOS_SUB *tsub); DLL_EXPORT void taos_unsubscribe(TAOS_SUB *tsub, int keepProgress); DLL_EXPORT TAOS_STREAM *taos_open_stream(TAOS *taos, const char *sql, void (*fp)(void *param, TAOS_RES *, TAOS_ROW row), - int64_t stime, void *param, void (*callback)(void *)); -DLL_EXPORT void taos_close_stream(TAOS_STREAM *tstr); + int64_t stime, void *param, void (*callback)(void *)); +DLL_EXPORT void taos_close_stream(TAOS_STREAM *tstr); -DLL_EXPORT int taos_load_table_info(TAOS *taos, const char* tableNameList); -DLL_EXPORT TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision); +DLL_EXPORT int taos_load_table_info(TAOS *taos, const char *tableNameList); +DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision); /* --------------------------TMQ INTERFACE------------------------------- */ @@ -215,10 +217,10 @@ typedef void(tmq_commit_cb(tmq_t *, tmq_resp_err_t, tmq_topic_vgroup_list_t *, v DLL_EXPORT tmq_list_t *tmq_list_new(); DLL_EXPORT int32_t tmq_list_append(tmq_list_t *, const char *); -DLL_EXPORT TAOS_RES *tmq_create_topic(TAOS *taos, const char *name, const char *sql, int sqlLen); -DLL_EXPORT tmq_t *tmq_consumer_new(void *conn, tmq_conf_t *conf, char *errstr, int32_t errstrLen); -DLL_EXPORT void tmq_message_destroy(tmq_message_t* tmq_message); -DLL_EXPORT const char* tmq_err2str(tmq_resp_err_t); +DLL_EXPORT TAOS_RES *tmq_create_topic(TAOS *taos, const char *name, const char *sql, int sqlLen); +DLL_EXPORT tmq_t *tmq_consumer_new(void *conn, tmq_conf_t *conf, char *errstr, int32_t errstrLen); +DLL_EXPORT void tmq_message_destroy(tmq_message_t *tmq_message); +DLL_EXPORT const char *tmq_err2str(tmq_resp_err_t); /* ------------------------TMQ CONSUMER INTERFACE------------------------ */ DLL_EXPORT tmq_resp_err_t tmq_subscribe(tmq_t *tmq, tmq_list_t *topic_list); @@ -227,7 +229,7 @@ DLL_EXPORT tmq_resp_err_t tmq_unsubscribe(tmq_t* tmq); DLL_EXPORT tmq_resp_err_t tmq_subscription(tmq_t* tmq, tmq_topic_vgroup_list_t** topics); #endif DLL_EXPORT tmq_message_t *tmq_consumer_poll(tmq_t *tmq, int64_t blocking_time); -DLL_EXPORT tmq_resp_err_t tmq_consumer_close(tmq_t* tmq); +DLL_EXPORT tmq_resp_err_t tmq_consumer_close(tmq_t *tmq); #if 0 DLL_EXPORT tmq_resp_err_t tmq_assign(tmq_t* tmq, const tmq_topic_vgroup_list_t* vgroups); DLL_EXPORT tmq_resp_err_t tmq_assignment(tmq_t* tmq, tmq_topic_vgroup_list_t** vgroups); @@ -251,8 +253,9 @@ DLL_EXPORT void tmq_conf_destroy(tmq_conf_t *conf); DLL_EXPORT tmq_conf_res_t tmq_conf_set(tmq_conf_t *conf, const char *key, const char *value); DLL_EXPORT void tmq_conf_set_offset_commit_cb(tmq_conf_t *conf, tmq_commit_cb *cb); -//temporary used function for demo only -void tmqShowMsg(tmq_message_t* tmq_message); +// temporary used function for demo only +void tmqShowMsg(tmq_message_t *tmq_message); +int32_t tmqGetSkipLogNum(tmq_message_t *tmq_message); #ifdef __cplusplus } diff --git a/include/common/common.h b/include/common/common.h index 31f905d47f..fd5b6717ab 100644 --- a/include/common/common.h +++ b/include/common/common.h @@ -16,30 +16,35 @@ #ifndef TDENGINE_COMMON_H #define TDENGINE_COMMON_H + +#ifdef __cplusplus +extern "C" { +#endif + #include "taosdef.h" -#include "tmsg.h" #include "tarray.h" +#include "tmsg.h" #include "tvariant.h" -//typedef struct STimeWindow { -// TSKEY skey; -// TSKEY ekey; -//} STimeWindow; +// typedef struct STimeWindow { +// TSKEY skey; +// TSKEY ekey; +// } STimeWindow; -//typedef struct { -// int32_t dataLen; -// char name[TSDB_TABLE_FNAME_LEN]; -// char *data; -//} STagData; +// typedef struct { +// int32_t dataLen; +// char name[TSDB_TABLE_FNAME_LEN]; +// char *data; +// } STagData; -//typedef struct SSchema { -// uint8_t type; -// char name[TSDB_COL_NAME_LEN]; -// int16_t colId; -// int16_t bytes; -//} SSchema; +// typedef struct SSchema { +// uint8_t type; +// char name[TSDB_COL_NAME_LEN]; +// int16_t colId; +// int16_t bytes; +// } SSchema; -#define TMQ_REQ_TYPE_COMMIT_ONLY 0 -#define TMQ_REQ_TYPE_CONSUME_ONLY 1 +#define TMQ_REQ_TYPE_COMMIT_ONLY 0 +#define TMQ_REQ_TYPE_CONSUME_ONLY 1 #define TMQ_REQ_TYPE_CONSUME_AND_COMMIT 2 typedef struct { @@ -67,7 +72,7 @@ typedef struct SDataBlockInfo { typedef struct SConstantItem { SColumnInfo info; - int32_t startRow; // run-length-encoding to save the space for multiple rows + int32_t startRow; // run-length-encoding to save the space for multiple rows int32_t endRow; SVariant value; } SConstantItem; @@ -80,12 +85,22 @@ typedef struct SSDataBlock { SDataBlockInfo info; } SSDataBlock; +typedef struct SVarColAttr { + int32_t *offset; // start position for each entry in the list + uint32_t length; // used buffer size that contain the valid data + uint32_t allocLen; // allocated buffer size +} SVarColAttr; + // pBlockAgg->numOfNull == info.rows, all data are null // pBlockAgg->numOfNull == 0, no data are null. typedef struct SColumnInfoData { - SColumnInfo info; // TODO filter info needs to be removed - char *nullbitmap;// - char *pData; // the corresponding block data in memory + SColumnInfo info; // TODO filter info needs to be removed + bool hasNull;// if current column data has null value. + char *pData; // the corresponding block data in memory + union { + char *nullbitmap; // bitmap, one bit for each item in the list + SVarColAttr varmeta; + }; } SColumnInfoData; static FORCE_INLINE int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) { @@ -110,7 +125,7 @@ static FORCE_INLINE int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlo return tlen; } -static FORCE_INLINE void* tDecodeDataBlock(void* buf, SSDataBlock* pBlock) { +static FORCE_INLINE void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock) { int32_t sz; buf = taosDecodeFixedI64(buf, &pBlock->info.uid); @@ -127,7 +142,7 @@ static FORCE_INLINE void* tDecodeDataBlock(void* buf, SSDataBlock* pBlock) { buf = taosDecodeBinary(buf, (void**)&data.pData, colSz); taosArrayPush(pBlock->pDataBlock, &data); } - return buf; + return (void*)buf; } static FORCE_INLINE int32_t tEncodeSMqConsumeRsp(void** buf, const SMqConsumeRsp* pRsp) { @@ -146,7 +161,7 @@ static FORCE_INLINE int32_t tEncodeSMqConsumeRsp(void** buf, const SMqConsumeRsp } tlen += taosEncodeFixedI32(buf, sz); for (int32_t i = 0; i < sz; i++) { - SSDataBlock* pBlock = (SSDataBlock*) taosArrayGet(pRsp->pBlockData, i); + SSDataBlock* pBlock = (SSDataBlock*)taosArrayGet(pRsp->pBlockData, i); tlen += tEncodeDataBlock(buf, pBlock); } return tlen; @@ -179,19 +194,18 @@ static FORCE_INLINE void tDeleteSSDataBlock(SSDataBlock* pBlock) { return; } - //int32_t numOfOutput = pBlock->info.numOfCols; + // int32_t numOfOutput = pBlock->info.numOfCols; int32_t sz = taosArrayGetSize(pBlock->pDataBlock); - for(int32_t i = 0; i < sz; ++i) { + for (int32_t i = 0; i < sz; ++i) { SColumnInfoData* pColInfoData = (SColumnInfoData*)taosArrayGet(pBlock->pDataBlock, i); tfree(pColInfoData->pData); } taosArrayDestroy(pBlock->pDataBlock); tfree(pBlock->pBlockAgg); - //tfree(pBlock); + // tfree(pBlock); } - static FORCE_INLINE void tDeleteSMqConsumeRsp(SMqConsumeRsp* pRsp) { if (pRsp->schemas) { if (pRsp->schemas->nCols) { @@ -199,42 +213,42 @@ static FORCE_INLINE void tDeleteSMqConsumeRsp(SMqConsumeRsp* pRsp) { } free(pRsp->schemas); } - taosArrayDestroyEx(pRsp->pBlockData, (void(*)(void*))tDeleteSSDataBlock); - pRsp->pBlockData = NULL; - //for (int i = 0; i < taosArrayGetSize(pRsp->pBlockData); i++) { - //SSDataBlock* pDataBlock = (SSDataBlock*)taosArrayGet(pRsp->pBlockData, i); - //tDeleteSSDataBlock(pDataBlock); + taosArrayDestroyEx(pRsp->pBlockData, (void (*)(void*))tDeleteSSDataBlock); + pRsp->pBlockData = NULL; + // for (int i = 0; i < taosArrayGetSize(pRsp->pBlockData); i++) { + // SSDataBlock* pDataBlock = (SSDataBlock*)taosArrayGet(pRsp->pBlockData, i); + // tDeleteSSDataBlock(pDataBlock); //} } //====================================================================================================================== // the following structure shared by parser and executor typedef struct SColumn { - uint64_t uid; - char name[TSDB_COL_NAME_LEN]; - int8_t flag; // column type: normal column, tag, or user-input column (integer/float/string) - SColumnInfo info; + uint64_t uid; + char name[TSDB_COL_NAME_LEN]; + int8_t flag; // column type: normal column, tag, or user-input column (integer/float/string) + SColumnInfo info; } SColumn; typedef struct SLimit { - int64_t limit; - int64_t offset; + int64_t limit; + int64_t offset; } SLimit; typedef struct SOrder { - uint32_t order; - SColumn col; + uint32_t order; + SColumn col; } SOrder; typedef struct SGroupbyExpr { - SArray* columnInfo; // SArray, group by columns information - bool groupbyTag; // group by tag or column + SArray* columnInfo; // SArray, group by columns information + bool groupbyTag; // group by tag or column } SGroupbyExpr; // the structure for sql function in select clause typedef struct SSqlExpr { - char token[TSDB_COL_NAME_LEN]; // original token - SSchema resSchema; + char token[TSDB_COL_NAME_LEN]; // original token + SSchema resSchema; int32_t numOfCols; SColumn* pColumns; // data columns that are required by query @@ -244,8 +258,8 @@ typedef struct SSqlExpr { } SSqlExpr; typedef struct SExprInfo { - struct SSqlExpr base; - struct tExprNode *pExpr; + struct SSqlExpr base; + struct tExprNode* pExpr; } SExprInfo; typedef struct SStateWindow { @@ -253,13 +267,17 @@ typedef struct SStateWindow { } SStateWindow; typedef struct SSessionWindow { - int64_t gap; // gap between two session window(in microseconds) + int64_t gap; // gap between two session window(in microseconds) SColumn col; } SSessionWindow; -#define QUERY_ASC_FORWARD_STEP 1 +#define QUERY_ASC_FORWARD_STEP 1 #define QUERY_DESC_FORWARD_STEP -1 #define GET_FORWARD_DIRECTION_FACTOR(ord) (((ord) == TSDB_ORDER_ASC) ? QUERY_ASC_FORWARD_STEP : QUERY_DESC_FORWARD_STEP) +#ifdef __cplusplus +} +#endif + #endif // TDENGINE_COMMON_H diff --git a/include/common/taosdef.h b/include/common/taosdef.h index 46c0c98ff0..e5a7a3563e 100644 --- a/include/common/taosdef.h +++ b/include/common/taosdef.h @@ -23,7 +23,7 @@ extern "C" { #include "taos.h" #include "tdef.h" -typedef uint64_t tb_uid_t; +typedef int64_t tb_uid_t; #define TSWINDOW_INITIALIZER ((STimeWindow){INT64_MIN, INT64_MAX}) #define TSWINDOW_DESC_INITIALIZER ((STimeWindow){INT64_MAX, INT64_MIN}) @@ -38,12 +38,12 @@ typedef enum { } EQType; typedef enum { - TSDB_SUPER_TABLE = 1, // super table - TSDB_CHILD_TABLE = 2, // table created from super table - TSDB_NORMAL_TABLE = 3, // ordinary table - TSDB_STREAM_TABLE = 4, // table created from stream computing - TSDB_TEMP_TABLE = 5, // temp table created by nest query - TSDB_TABLE_MAX = 6 + TSDB_SUPER_TABLE = 1, // super table + TSDB_CHILD_TABLE = 2, // table created from super table + TSDB_NORMAL_TABLE = 3, // ordinary table + TSDB_STREAM_TABLE = 4, // table created from stream computing + TSDB_TEMP_TABLE = 5, // temp table created by nest query + TSDB_TABLE_MAX = 6 } ETableType; typedef enum { diff --git a/include/common/tep.h b/include/common/tep.h index 69dd385a37..584b8a5a71 100644 --- a/include/common/tep.h +++ b/include/common/tep.h @@ -7,12 +7,22 @@ extern "C" { #include "os.h" #include "tmsg.h" +#include "common.h" typedef struct SCorEpSet { int32_t version; SEpSet epSet; } SCorEpSet; +typedef struct SBlockOrderInfo { + int32_t order; + int32_t colIndex; + SColumnInfoData *pColData; +// int32_t type; +// int32_t bytes; +// bool hasNull; +} SBlockOrderInfo; + int taosGetFqdnPortFromEp(const char *ep, SEp *pEp); void addEpIntoEpSet(SEpSet *pEpSet, const char *fqdn, uint16_t port); @@ -21,6 +31,77 @@ bool isEpsetEqual(const SEpSet *s1, const SEpSet *s2); void updateEpSet_s(SCorEpSet *pEpSet, SEpSet *pNewEpSet); SEpSet getEpSet_s(SCorEpSet *pEpSet); +#define NBIT (3u) +#define BitPos(_n) ((_n) & ((1 << NBIT) - 1)) +#define BMCharPos(bm_, r_) ((bm_)[(r_) >> NBIT]) +#define colDataIsNull_f(bm_, r_) ((BMCharPos(bm_, r_) & (1u << (7u - BitPos(r_)))) == (1u << (7u - BitPos(r_)))) + +#define colDataSetNull_f(bm_, r_) \ + do { \ + BMCharPos(bm_, r_) |= (1u << (7u - BitPos(r_))); \ + } while (0) + +static FORCE_INLINE bool colDataIsNull(const SColumnInfoData* pColumnInfoData, uint32_t totalRows, uint32_t row, SColumnDataAgg* pColAgg) { + if (!pColumnInfoData->hasNull) { + return false; + } + + if (pColAgg != NULL) { + if (pColAgg->numOfNull == totalRows) { + ASSERT(pColumnInfoData->nullbitmap == NULL); + return true; + } else if (pColAgg->numOfNull == 0) { + ASSERT(pColumnInfoData->nullbitmap == NULL); + return false; + } + } + + if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { + return pColumnInfoData->varmeta.offset[row] == -1; + } else { + if (pColumnInfoData->nullbitmap == NULL) { + return false; + } + + return colDataIsNull_f(pColumnInfoData->nullbitmap, row); + } +} + +#define colDataGet(p1_, r_) \ + ((IS_VAR_DATA_TYPE((p1_)->info.type)) ? (p1_)->pData + (p1_)->varmeta.offset[(r_)] \ + : (p1_)->pData + ((r_) * (p1_)->info.bytes)); + +int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull); +int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, const SColumnInfoData* pSource, uint32_t numOfRow2); +int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock); + +int32_t colDataGetSize(const SColumnInfoData* pColumnInfoData, int32_t numOfRows); +void colDataTrim(SColumnInfoData* pColumnInfoData); + +size_t colDataGetNumOfCols(const SSDataBlock* pBlock); +size_t colDataGetNumOfRows(const SSDataBlock* pBlock); + +int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc); +int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startIndex, int32_t* stopIndex, int32_t pageSize); +SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int32_t rowCount); + +int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock); +int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf); + +size_t blockDataGetSize(const SSDataBlock* pBlock); +size_t blockDataGetRowSize(const SSDataBlock* pBlock); +double blockDataGetSerialRowSize(const SSDataBlock* pBlock); +size_t blockDataGetSerialMetaSize(const SSDataBlock* pBlock); + +size_t blockDataNumOfRowsForSerialize(const SSDataBlock* pBlock, int32_t blockSize); + +int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst); +int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst); + +int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows); +void blockDataClearup(SSDataBlock* pDataBlock, bool hasVarCol); +void *blockDataDestroy(SSDataBlock *pBlock); + #ifdef __cplusplus } #endif diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 4a56c4b079..7337f1afb8 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -25,9 +25,9 @@ extern "C" { #include "taoserror.h" #include "tarray.h" #include "tcoding.h" -#include "trow.h" #include "thash.h" #include "tlist.h" +#include "trow.h" /* ------------------------ MESSAGE DEFINITIONS ------------------------ */ #define TD_MSG_NUMBER_ @@ -69,7 +69,7 @@ typedef uint16_t tmsg_t; #define TSDB_IE_TYPE_DNODE_STATE 7 typedef enum { - HEARTBEAT_TYPE_MQ = 0, + HEARTBEAT_TYPE_MQ = 0, HEARTBEAT_TYPE_QUERY = 1, // types can be added here // @@ -82,7 +82,6 @@ enum { HEARTBEAT_KEY_MQ_TMP, }; - typedef enum _mgmt_table { TSDB_MGMT_TABLE_START, TSDB_MGMT_TABLE_ACCT, @@ -192,28 +191,16 @@ typedef struct { // Submit message for one table typedef struct SSubmitBlk { - uint64_t uid; // table unique id - int32_t tid; // table id - int32_t padding; // TODO just for padding here - int32_t sversion; // data schema version - int32_t dataLen; // data part length, not including the SSubmitBlk head - int32_t schemaLen; // schema length, if length is 0, no schema exists - int16_t numOfRows; // total number of rows in current submit block - char data[]; + int64_t uid; // table unique id + int32_t tid; // table id + int32_t padding; // TODO just for padding here + int32_t sversion; // data schema version + int32_t dataLen; // data part length, not including the SSubmitBlk head + int32_t schemaLen; // schema length, if length is 0, no schema exists + int16_t numOfRows; // total number of rows in current submit block + char data[]; } SSubmitBlk; -typedef struct { - /* data */ -} SSubmitReq; - -typedef struct { - /* data */ -} SSubmitRsp; - -typedef struct { - /* data */ -} SSubmitReqReader; - // Submit message for this TSDB typedef struct { SMsgHead header; @@ -221,12 +208,12 @@ typedef struct { int32_t length; int32_t numOfBlocks; char blocks[]; -} SSubmitMsg; +} SSubmitReq; typedef struct { int32_t totalLen; int32_t len; - STSRow *row; + STSRow* row; } SSubmitBlkIter; typedef struct { @@ -235,9 +222,9 @@ typedef struct { void* pMsg; } SSubmitMsgIter; -int tInitSubmitMsgIter(SSubmitMsg* pMsg, SSubmitMsgIter* pIter); -int tGetSubmitMsgNext(SSubmitMsgIter* pIter, SSubmitBlk** pPBlock); -int tInitSubmitBlkIter(SSubmitBlk* pBlock, SSubmitBlkIter* pIter); +int32_t tInitSubmitMsgIter(SSubmitReq* pMsg, SSubmitMsgIter* pIter); +int32_t tGetSubmitMsgNext(SSubmitMsgIter* pIter, SSubmitBlk** pPBlock); +int32_t tInitSubmitBlkIter(SSubmitBlk* pBlock, SSubmitBlkIter* pIter); STSRow* tGetSubmitBlkNext(SSubmitBlkIter* pIter); typedef struct { @@ -245,16 +232,16 @@ typedef struct { int32_t vnode; // vnode index of failed block int32_t sid; // table index of failed block int32_t code; // errorcode while write data to vnode, such as not created, dropped, no space, invalid table -} SShellSubmitRspBlock; +} SSubmitRspBlock; typedef struct { - int32_t code; // 0-success, > 0 error code - int32_t numOfRows; // number of records the client is trying to write - int32_t affectedRows; // number of records actually written - int32_t failedRows; // number of failed records (exclude duplicate records) - int32_t numOfFailedBlocks; - SShellSubmitRspBlock failedBlocks[]; -} SShellSubmitRsp; + int32_t code; // 0-success, > 0 error code + int32_t numOfRows; // number of records the client is trying to write + int32_t affectedRows; // number of records actually written + int32_t failedRows; // number of failed records (exclude duplicate records) + int32_t numOfFailedBlocks; + SSubmitRspBlock failedBlocks[]; +} SSubmitRsp; typedef struct SSchema { int8_t type; @@ -273,16 +260,17 @@ typedef struct { char comment[TSDB_STB_COMMENT_LEN]; } SMCreateStbReq; -int32_t tSerializeSMCreateStbReq(void** buf, SMCreateStbReq* pReq); -void* tDeserializeSMCreateStbReq(void* buf, SMCreateStbReq* pReq); +int32_t tSerializeSMCreateStbReq(void* buf, int32_t bufLen, SMCreateStbReq* pReq); +int32_t tDeserializeSMCreateStbReq(void* buf, int32_t bufLen, SMCreateStbReq* pReq); +void tFreeSMCreateStbReq(SMCreateStbReq* pReq); typedef struct { char name[TSDB_TABLE_FNAME_LEN]; int8_t igNotExists; } SMDropStbReq; -int32_t tSerializeSMDropStbReq(void** buf, SMDropStbReq* pReq); -void* tDeserializeSMDropStbReq(void* buf, SMDropStbReq* pReq); +int32_t tSerializeSMDropStbReq(void* buf, int32_t bufLen, SMDropStbReq* pReq); +int32_t tDeserializeSMDropStbReq(void* buf, int32_t bufLen, SMDropStbReq* pReq); typedef struct { char name[TSDB_TABLE_FNAME_LEN]; @@ -291,8 +279,20 @@ typedef struct { SArray* pFields; } SMAltertbReq; -int32_t tSerializeSMAlterStbReq(void** buf, SMAltertbReq* pReq); -void* tDeserializeSMAlterStbReq(void* buf, SMAltertbReq* pReq); +int32_t tSerializeSMAlterStbReq(void* buf, int32_t bufLen, SMAltertbReq* pReq); +int32_t tDeserializeSMAlterStbReq(void* buf, int32_t bufLen, SMAltertbReq* pReq); +void tFreeSMAltertbReq(SMAltertbReq* pReq); + +typedef struct SEpSet { + int8_t inUse; + int8_t numOfEps; + SEp eps[TSDB_MAX_REPLICA]; +} SEpSet; + +int32_t tEncodeSEpSet(SCoder* pEncoder, const SEpSet* pEp); +int32_t tDecodeSEpSet(SCoder* pDecoder, SEpSet* pEp); +int32_t taosEncodeSEpSet(void** buf, const SEpSet* pEp); +void* taosDecodeSEpSet(void* buf, SEpSet* pEp); typedef struct { int32_t pid; @@ -301,43 +301,21 @@ typedef struct { int64_t startTime; } SConnectReq; -typedef struct SEpSet { - int8_t inUse; - int8_t numOfEps; - SEp eps[TSDB_MAX_REPLICA]; -} SEpSet; - -static FORCE_INLINE int taosEncodeSEpSet(void** buf, const SEpSet* pEp) { - int tlen = 0; - tlen += taosEncodeFixedI8(buf, pEp->inUse); - tlen += taosEncodeFixedI8(buf, pEp->numOfEps); - for (int i = 0; i < TSDB_MAX_REPLICA; i++) { - tlen += taosEncodeFixedU16(buf, pEp->eps[i].port); - tlen += taosEncodeString(buf, pEp->eps[i].fqdn); - } - return tlen; -} - -static FORCE_INLINE void* taosDecodeSEpSet(void* buf, SEpSet* pEp) { - buf = taosDecodeFixedI8(buf, &pEp->inUse); - buf = taosDecodeFixedI8(buf, &pEp->numOfEps); - for (int i = 0; i < TSDB_MAX_REPLICA; i++) { - buf = taosDecodeFixedU16(buf, &pEp->eps[i].port); - buf = taosDecodeStringTo(buf, pEp->eps[i].fqdn); - } - return buf; -} +int32_t tSerializeSConnectReq(void* buf, int32_t bufLen, SConnectReq* pReq); +int32_t tDeserializeSConnectReq(void* buf, int32_t bufLen, SConnectReq* pReq); typedef struct { int32_t acctId; int64_t clusterId; int32_t connId; int8_t superUser; - int8_t align[3]; SEpSet epSet; char sVersion[128]; } SConnectRsp; +int32_t tSerializeSConnectRsp(void* buf, int32_t bufLen, SConnectRsp* pRsp); +int32_t tDeserializeSConnectRsp(void* buf, int32_t bufLen, SConnectRsp* pRsp); + typedef struct { char user[TSDB_USER_LEN]; char pass[TSDB_PASSWORD_LEN]; @@ -525,28 +503,6 @@ typedef struct { int32_t code; } SQueryTableRsp; -// todo: the show handle should be replaced with id -typedef struct { - SMsgHead header; - union { - int64_t showId; - int64_t qhandle; - int64_t qId; - }; // query handle - int8_t free; -} SRetrieveTableReq; - -typedef struct { - int64_t useconds; - int8_t completed; // all results are returned to client - int8_t precision; - int8_t compressed; - int32_t compLen; - - int32_t numOfRows; - char data[]; -} SRetrieveTableRsp; - typedef struct { char db[TSDB_DB_FNAME_LEN]; int32_t numOfVgroups; @@ -612,6 +568,27 @@ typedef struct { int32_t tSerializeSUseDbReq(void* buf, int32_t bufLen, SUseDbReq* pReq); int32_t tDeserializeSUseDbReq(void* buf, int32_t bufLen, SUseDbReq* pReq); +typedef struct { + char db[TSDB_DB_FNAME_LEN]; + uint64_t uid; + int32_t vgVersion; + int32_t vgNum; + int8_t hashMethod; + SArray* pVgroupInfos; // Array of SVgroupInfo +} SUseDbRsp; + +int32_t tSerializeSUseDbRsp(void* buf, int32_t bufLen, SUseDbRsp* pRsp); +int32_t tDeserializeSUseDbRsp(void* buf, int32_t bufLen, SUseDbRsp* pRsp); +void tFreeSUsedbRsp(SUseDbRsp* pRsp); + +typedef struct { + SArray* pArray; // Array of SUseDbRsp +} SUseDbBatchRsp; + +int32_t tSerializeSUseDbBatchRsp(void* buf, int32_t bufLen, SUseDbBatchRsp* pRsp); +int32_t tDeserializeSUseDbBatchRsp(void* buf, int32_t bufLen, SUseDbBatchRsp* pRsp); +void tFreeSUseDbBatchRsp(SUseDbBatchRsp* pRsp); + typedef struct { char db[TSDB_DB_FNAME_LEN]; } SSyncDbReq, SCompactDbReq; @@ -707,8 +684,8 @@ typedef struct { SArray* pVloads; // array of SVnodeLoad } SStatusReq; -int32_t tSerializeSStatusReq(void** buf, SStatusReq* pReq); -void* tDeserializeSStatusReq(void* buf, SStatusReq* pReq); +int32_t tSerializeSStatusReq(void* buf, int32_t bufLen, SStatusReq* pReq); +int32_t tDeserializeSStatusReq(void* buf, int32_t bufLen, SStatusReq* pReq); typedef struct { int32_t dnodeId; @@ -716,9 +693,9 @@ typedef struct { } SDnodeCfg; typedef struct { - int32_t id; - int8_t isMnode; - SEp ep; + int32_t id; + int8_t isMnode; + SEp ep; } SDnodeEp; typedef struct { @@ -727,12 +704,15 @@ typedef struct { SArray* pDnodeEps; // Array of SDnodeEp } SStatusRsp; -int32_t tSerializeSStatusRsp(void** buf, SStatusRsp* pRsp); -void* tDeserializeSStatusRsp(void* buf, SStatusRsp* pRsp); +int32_t tSerializeSStatusRsp(void* buf, int32_t bufLen, SStatusRsp* pRsp); +int32_t tDeserializeSStatusRsp(void* buf, int32_t bufLen, SStatusRsp* pRsp); typedef struct { - int32_t reserve; -} STransReq; + int32_t reserved; +} SMTimerReq; + +int32_t tSerializeSMTimerMsg(void* buf, int32_t bufLen, SMTimerReq* pReq); +int32_t tDeserializeSMTimerMsg(void* buf, int32_t bufLen, SMTimerReq* pReq); typedef struct { int32_t id; @@ -767,6 +747,9 @@ typedef struct { SReplica replicas[TSDB_MAX_REPLICA]; } SCreateVnodeReq, SAlterVnodeReq; +int32_t tSerializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq* pReq); +int32_t tDeserializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq* pReq); + typedef struct { int32_t vgId; int32_t dnodeId; @@ -774,10 +757,8 @@ typedef struct { char db[TSDB_DB_FNAME_LEN]; } SDropVnodeReq, SSyncVnodeReq, SCompactVnodeReq; -typedef struct { - int32_t vgId; - int8_t accessState; -} SAuthVnodeReq; +int32_t tSerializeSDropVnodeReq(void* buf, int32_t bufLen, SDropVnodeReq* pReq); +int32_t tDeserializeSDropVnodeReq(void* buf, int32_t bufLen, SDropVnodeReq* pReq); typedef struct { SMsgHead header; @@ -785,6 +766,9 @@ typedef struct { char tbName[TSDB_TABLE_NAME_LEN]; } STableInfoReq; +int32_t tSerializeSTableInfoReq(void* buf, int32_t bufLen, STableInfoReq* pReq); +int32_t tDeserializeSTableInfoReq(void* buf, int32_t bufLen, STableInfoReq* pReq); + typedef struct { int8_t metaClone; // create local clone of the cached table meta int32_t numOfVgroups; @@ -795,10 +779,10 @@ typedef struct { // todo refactor typedef struct SVgroupInfo { - int32_t vgId; - uint32_t hashBegin; - uint32_t hashEnd; - SEpSet epset; + int32_t vgId; + uint32_t hashBegin; + uint32_t hashEnd; + SEpSet epset; } SVgroupInfo; typedef struct { @@ -821,9 +805,21 @@ typedef struct { uint64_t suid; uint64_t tuid; int32_t vgId; - SSchema pSchema[]; + SSchema* pSchemas; } STableMetaRsp; +int32_t tSerializeSTableMetaRsp(void* buf, int32_t bufLen, STableMetaRsp* pRsp); +int32_t tDeserializeSTableMetaRsp(void* buf, int32_t bufLen, STableMetaRsp* pRsp); +void tFreeSTableMetaRsp(STableMetaRsp* pRsp); + +typedef struct { + SArray* pArray; // Array of STableMetaRsp +} STableMetaBatchRsp; + +int32_t tSerializeSTableMetaBatchRsp(void* buf, int32_t bufLen, STableMetaBatchRsp* pRsp); +int32_t tDeserializeSTableMetaBatchRsp(void* buf, int32_t bufLen, STableMetaBatchRsp* pRsp); +void tFreeSTableMetaBatchRsp(STableMetaBatchRsp* pRsp); + typedef struct { int32_t numOfTables; int32_t numOfVgroup; @@ -841,37 +837,48 @@ typedef struct { char* data; } STagData; -typedef struct { - char db[TSDB_DB_FNAME_LEN]; - uint64_t uid; - int32_t vgVersion; - int32_t vgNum; - int8_t hashMethod; - SVgroupInfo vgroupInfo[]; -} SUseDbRsp; - /* * sql: show tables like '%a_%' * payload is the query condition, e.g., '%a_%' * payloadLen is the length of payload */ typedef struct { - int8_t type; + int32_t type; char db[TSDB_DB_FNAME_LEN]; - int16_t payloadLen; - char payload[]; + int32_t payloadLen; + char* payload; } SShowReq; -typedef struct { - char db[TSDB_DB_FNAME_LEN]; - int32_t numOfVgroup; - int32_t vgid[]; -} SCompactReq; +int32_t tSerializeSShowReq(void* buf, int32_t bufLen, SShowReq* pReq); +int32_t tDeserializeSShowReq(void* buf, int32_t bufLen, SShowReq* pReq); +void tFreeSShowReq(SShowReq* pReq); typedef struct { int64_t showId; STableMetaRsp tableMeta; -} SShowRsp; +} SShowRsp, SVShowTablesRsp; + +int32_t tSerializeSShowRsp(void* buf, int32_t bufLen, SShowRsp* pRsp); +int32_t tDeserializeSShowRsp(void* buf, int32_t bufLen, SShowRsp* pRsp); +void tFreeSShowRsp(SShowRsp* pRsp); + +typedef struct { + int64_t showId; + int8_t free; +} SRetrieveTableReq; + +int32_t tSerializeSRetrieveTableReq(void* buf, int32_t bufLen, SRetrieveTableReq* pReq); +int32_t tDeserializeSRetrieveTableReq(void* buf, int32_t bufLen, SRetrieveTableReq* pReq); + +typedef struct { + int64_t useconds; + int8_t completed; // all results are returned to client + int8_t precision; + int8_t compressed; + int32_t compLen; + int32_t numOfRows; + char data[]; +} SRetrieveTableRsp; typedef struct { char fqdn[TSDB_FQDN_LEN]; // end point, hostname:port @@ -910,6 +917,9 @@ typedef struct { SReplica replicas[TSDB_MAX_REPLICA]; } SDCreateMnodeReq, SDAlterMnodeReq; +int32_t tSerializeSDCreateMnodeReq(void* buf, int32_t bufLen, SDCreateMnodeReq* pReq); +int32_t tDeserializeSDCreateMnodeReq(void* buf, int32_t bufLen, SDCreateMnodeReq* pReq); + typedef struct { int32_t dnodeId; } SMCreateQnodeReq, SMDropQnodeReq, SDCreateQnodeReq, SDDropQnodeReq, SMCreateSnodeReq, SMDropSnodeReq, @@ -957,10 +967,16 @@ typedef struct { int32_t queryId; } SKillQueryReq; +int32_t tSerializeSKillQueryReq(void* buf, int32_t bufLen, SKillQueryReq* pReq); +int32_t tDeserializeSKillQueryReq(void* buf, int32_t bufLen, SKillQueryReq* pReq); + typedef struct { int32_t connId; } SKillConnReq; +int32_t tSerializeSKillConnReq(void* buf, int32_t bufLen, SKillConnReq* pReq); +int32_t tDeserializeSKillConnReq(void* buf, int32_t bufLen, SKillConnReq* pReq); + typedef struct { char user[TSDB_USER_LEN]; char spi; @@ -969,9 +985,11 @@ typedef struct { char ckey[TSDB_PASSWORD_LEN]; } SAuthReq, SAuthRsp; +int32_t tSerializeSAuthReq(void* buf, int32_t bufLen, SAuthReq* pReq); +int32_t tDeserializeSAuthReq(void* buf, int32_t bufLen, SAuthReq* pReq); + typedef struct { int8_t finished; - int8_t align[7]; char name[TSDB_STEP_NAME_LEN]; char desc[TSDB_STEP_DESC_LEN]; } SStartupReq; @@ -989,7 +1007,7 @@ typedef struct SSubQueryMsg { uint64_t queryId; uint64_t taskId; int8_t taskType; - uint32_t sqlLen; // the query sql, + uint32_t sqlLen; // the query sql, uint32_t phyLen; char msg[]; } SSubQueryMsg; @@ -1008,7 +1026,6 @@ typedef struct { uint64_t taskId; } SQueryContinueReq; - typedef struct { SMsgHead header; uint64_t sId; @@ -1066,46 +1083,23 @@ typedef struct { } STaskDropRsp; typedef struct { - int8_t igExists; - char* name; - char* sql; - char* physicalPlan; - char* logicalPlan; -} SCMCreateTopicReq; + char name[TSDB_TOPIC_FNAME_LEN]; + int8_t igExists; + char* sql; + char* physicalPlan; + char* logicalPlan; +} SMCreateTopicReq; -static FORCE_INLINE int tSerializeSCMCreateTopicReq(void** buf, const SCMCreateTopicReq* pReq) { - int tlen = 0; - tlen += taosEncodeFixedI8(buf, pReq->igExists); - tlen += taosEncodeString(buf, pReq->name); - tlen += taosEncodeString(buf, pReq->sql); - tlen += taosEncodeString(buf, pReq->physicalPlan); - tlen += taosEncodeString(buf, pReq->logicalPlan); - return tlen; -} - -static FORCE_INLINE void* tDeserializeSCMCreateTopicReq(void* buf, SCMCreateTopicReq* pReq) { - buf = taosDecodeFixedI8(buf, &(pReq->igExists)); - buf = taosDecodeString(buf, &(pReq->name)); - buf = taosDecodeString(buf, &(pReq->sql)); - buf = taosDecodeString(buf, &(pReq->physicalPlan)); - buf = taosDecodeString(buf, &(pReq->logicalPlan)); - return buf; -} +int32_t tSerializeMCreateTopicReq(void* buf, int32_t bufLen, const SMCreateTopicReq* pReq); +int32_t tDeserializeSMCreateTopicReq(void* buf, int32_t bufLen, SMCreateTopicReq* pReq); +void tFreeSMCreateTopicReq(SMCreateTopicReq* pReq); typedef struct { int64_t topicId; -} SCMCreateTopicRsp; +} SMCreateTopicRsp; -static FORCE_INLINE int tSerializeSCMCreateTopicRsp(void** buf, const SCMCreateTopicRsp* pRsp) { - int tlen = 0; - tlen += taosEncodeFixedI64(buf, pRsp->topicId); - return tlen; -} - -static FORCE_INLINE void* tDeserializeSCMCreateTopicRsp(void* buf, SCMCreateTopicRsp* pRsp) { - buf = taosDecodeFixedI64(buf, &pRsp->topicId); - return buf; -} +int32_t tSerializeSMCreateTopicRsp(void* buf, int32_t bufLen, const SMCreateTopicRsp* pRsp); +int32_t tDeserializeSMCreateTopicRsp(void* buf, int32_t bufLen, SMCreateTopicRsp* pRsp); typedef struct { int32_t topicNum; @@ -1205,14 +1199,10 @@ static FORCE_INLINE void* tDeserializeSMVSubscribeReq(void* buf, SMVSubscribeReq } typedef struct { - int32_t reserved; -} SMqTmrMsg; - -typedef struct { - const char* key; - SArray* lostConsumers; //SArray - SArray* removedConsumers; //SArray - SArray* newConsumers; //SArray + const char* key; + SArray* lostConsumers; // SArray + SArray* removedConsumers; // SArray + SArray* newConsumers; // SArray } SMqRebSubscribe; static FORCE_INLINE SMqRebSubscribe* tNewSMqRebSubscribe(const char* key) { @@ -1242,44 +1232,23 @@ _err: return NULL; } -// this message is sent from mnode to mnode(read thread to write thread), so there is no need for serialization / deserialization +// this message is sent from mnode to mnode(read thread to write thread), so there is no need for serialization or +// deserialization typedef struct { - //SArray* rebSubscribes; //SArray - SHashObj* rebSubHash; // SHashObj + SHashObj* rebSubHash; // SHashObj } SMqDoRebalanceMsg; -#if 0 -static FORCE_INLINE SMqDoRebalanceMsg* tNewSMqDoRebalanceMsg() { - SMqDoRebalanceMsg *pMsg = malloc(sizeof(SMqDoRebalanceMsg)); - if (pMsg == NULL) { - return NULL; - } - pMsg->rebSubscribes = taosArrayInit(0, sizeof(SMqRebSubscribe)); - if (pMsg->rebSubscribes == NULL) { - free(pMsg); - return NULL; - } - return pMsg; -} -#endif - typedef struct { int64_t status; } SMVSubscribeRsp; -typedef struct { - char name[TSDB_TOPIC_NAME_LEN]; - int8_t igExists; - int32_t execLen; - void* executor; - int32_t sqlLen; - char* sql; -} SCreateTopicReq; - typedef struct { char name[TSDB_TABLE_FNAME_LEN]; int8_t igNotExists; -} SDropTopicReq; +} SMDropTopicReq; + +int32_t tSerializeSMDropTopicReq(void* buf, int32_t bufLen, SMDropTopicReq* pReq); +int32_t tDeserializeSMDropTopicReq(void* buf, int32_t bufLen, SMDropTopicReq* pReq); typedef struct { char name[TSDB_TABLE_FNAME_LEN]; @@ -1381,11 +1350,6 @@ typedef struct { SMsgHead head; } SVShowTablesReq; -typedef struct { - int64_t id; - STableMetaRsp metaInfo; -} SVShowTablesRsp; - typedef struct { SMsgHead head; int32_t id; @@ -1397,7 +1361,6 @@ typedef struct { int8_t precision; int8_t compressed; int32_t compLen; - int32_t numOfRows; char data[]; } SVShowTablesFetchRsp; @@ -1418,9 +1381,9 @@ static FORCE_INLINE int32_t tEncodeSMsgHead(void** buf, const SMsgHead* pMsg) { } typedef struct SMqHbRsp { - int8_t status; //idle or not + int8_t status; // idle or not int8_t vnodeChanged; - int8_t epChanged; // should use new epset + int8_t epChanged; // should use new epset int8_t reserved; SEpSet epSet; } SMqHbRsp; @@ -1443,7 +1406,7 @@ static FORCE_INLINE void* taosDecodeSMqHbRsp(void* buf, SMqHbRsp* pRsp) { } typedef struct SMqHbOneTopicBatchRsp { - char topicName[TSDB_TOPIC_FNAME_LEN]; + char topicName[TSDB_TOPIC_FNAME_LEN]; SArray* rsps; // SArray } SMqHbOneTopicBatchRsp; @@ -1473,8 +1436,8 @@ static FORCE_INLINE void* taosDecodeSMqHbOneTopicBatchRsp(void* buf, SMqHbOneTop } typedef struct SMqHbBatchRsp { - int64_t consumerId; - SArray* batchRsps; // SArray + int64_t consumerId; + SArray* batchRsps; // SArray } SMqHbBatchRsp; static FORCE_INLINE int taosEncodeSMqHbBatchRsp(void** buf, const SMqHbBatchRsp* pBatchRsp) { @@ -1483,7 +1446,7 @@ static FORCE_INLINE int taosEncodeSMqHbBatchRsp(void** buf, const SMqHbBatchRsp* int32_t sz; tlen += taosEncodeFixedI32(buf, sz); for (int32_t i = 0; i < sz; i++) { - SMqHbOneTopicBatchRsp* pRsp = (SMqHbOneTopicBatchRsp*) taosArrayGet(pBatchRsp->batchRsps, i); + SMqHbOneTopicBatchRsp* pRsp = (SMqHbOneTopicBatchRsp*)taosArrayGet(pBatchRsp->batchRsps, i); tlen += taosEncodeSMqHbOneTopicBatchRsp(buf, pRsp); } return tlen; @@ -1495,7 +1458,7 @@ static FORCE_INLINE void* taosDecodeSMqHbBatchRsp(void* buf, SMqHbBatchRsp* pBat buf = taosDecodeFixedI32(buf, &sz); pBatchRsp->batchRsps = taosArrayInit(sz, sizeof(SMqHbOneTopicBatchRsp)); for (int32_t i = 0; i < sz; i++) { - SMqHbOneTopicBatchRsp rsp; + SMqHbOneTopicBatchRsp rsp; buf = taosDecodeSMqHbOneTopicBatchRsp(buf, &rsp); buf = taosArrayPush(pBatchRsp->batchRsps, &rsp); } @@ -1535,43 +1498,30 @@ typedef struct { SArray* rsps; // SArray } SClientHbBatchRsp; -static FORCE_INLINE uint32_t hbKeyHashFunc(const char* key, uint32_t keyLen) { - return taosIntHash_64(key, keyLen); -} +static FORCE_INLINE uint32_t hbKeyHashFunc(const char* key, uint32_t keyLen) { return taosIntHash_64(key, keyLen); } -int tSerializeSClientHbReq(void** buf, const SClientHbReq* pReq); -void* tDeserializeSClientHbReq(void* buf, SClientHbReq* pReq); - -int tSerializeSClientHbRsp(void** buf, const SClientHbRsp* pRsp); -void* tDeserializeSClientHbRsp(void* buf, SClientHbRsp* pRsp); - - -static FORCE_INLINE void tFreeReqKvHash(SHashObj* info) { - void *pIter = taosHashIterate(info, NULL); +static FORCE_INLINE void tFreeReqKvHash(SHashObj* info) { + void* pIter = taosHashIterate(info, NULL); while (pIter != NULL) { SKv* kv = (SKv*)pIter; - tfree(kv->value); - pIter = taosHashIterate(info, pIter); } } - -static FORCE_INLINE void tFreeClientHbReq(void *pReq) { +static FORCE_INLINE void tFreeClientHbReq(void* pReq) { SClientHbReq* req = (SClientHbReq*)pReq; if (req->info) { tFreeReqKvHash(req->info); - taosHashCleanup(req->info); } } -int tSerializeSClientHbBatchReq(void** buf, const SClientHbBatchReq* pReq); -void* tDeserializeSClientHbBatchReq(void* buf, SClientHbBatchReq* pReq); +int32_t tSerializeSClientHbBatchReq(void* buf, int32_t bufLen, const SClientHbBatchReq* pReq); +int32_t tDeserializeSClientHbBatchReq(void* buf, int32_t bufLen, SClientHbBatchReq* pReq); static FORCE_INLINE void tFreeClientHbBatchReq(void* pReq, bool deep) { - SClientHbBatchReq *req = (SClientHbBatchReq*)pReq; + SClientHbBatchReq* req = (SClientHbBatchReq*)pReq; if (deep) { taosArrayDestroyEx(req->reqs, tFreeClientHbReq); } else { @@ -1580,54 +1530,52 @@ static FORCE_INLINE void tFreeClientHbBatchReq(void* pReq, bool deep) { free(pReq); } -static FORCE_INLINE void tFreeClientKv(void *pKv) { - SKv *kv = (SKv *)pKv; +static FORCE_INLINE void tFreeClientKv(void* pKv) { + SKv* kv = (SKv*)pKv; if (kv) { tfree(kv->value); } } -static FORCE_INLINE void tFreeClientHbRsp(void *pRsp) { +static FORCE_INLINE void tFreeClientHbRsp(void* pRsp) { SClientHbRsp* rsp = (SClientHbRsp*)pRsp; if (rsp->info) taosArrayDestroyEx(rsp->info, tFreeClientKv); } - static FORCE_INLINE void tFreeClientHbBatchRsp(void* pRsp) { - SClientHbBatchRsp *rsp = (SClientHbBatchRsp*)pRsp; + SClientHbBatchRsp* rsp = (SClientHbBatchRsp*)pRsp; taosArrayDestroyEx(rsp->rsps, tFreeClientHbRsp); } +int32_t tSerializeSClientHbBatchRsp(void* buf, int32_t bufLen, const SClientHbBatchRsp* pBatchRsp); +int32_t tDeserializeSClientHbBatchRsp(void* buf, int32_t bufLen, SClientHbBatchRsp* pBatchRsp); -int tSerializeSClientHbBatchRsp(void** buf, const SClientHbBatchRsp* pBatchRsp); -void* tDeserializeSClientHbBatchRsp(void* buf, SClientHbBatchRsp* pBatchRsp); - -static FORCE_INLINE int taosEncodeSKv(void** buf, const SKv* pKv) { - int tlen = 0; - tlen += taosEncodeFixedI32(buf, pKv->key); - tlen += taosEncodeFixedI32(buf, pKv->valueLen); - tlen += taosEncodeBinary(buf, pKv->value, pKv->valueLen); - return tlen; +static FORCE_INLINE int32_t tEncodeSKv(SCoder* pEncoder, const SKv* pKv) { + if (tEncodeI32(pEncoder, pKv->key) < 0) return -1; + if (tEncodeI32(pEncoder, pKv->valueLen) < 0) return -1; + if (tEncodeBinary(pEncoder, (const char*)pKv->value, pKv->valueLen) < 0) return -1; + return 0; } -static FORCE_INLINE void* taosDecodeSKv(void* buf, SKv* pKv) { - buf = taosDecodeFixedI32(buf, &pKv->key); - buf = taosDecodeFixedI32(buf, &pKv->valueLen); - buf = taosDecodeBinary(buf, &pKv->value, pKv->valueLen); - return buf; +static FORCE_INLINE int32_t tDecodeSKv(SCoder* pDecoder, SKv* pKv) { + if (tDecodeI32(pDecoder, &pKv->key) < 0) return -1; + if (tDecodeI32(pDecoder, &pKv->valueLen) < 0) return -1; + pKv->value = malloc(pKv->valueLen + 1); + if (pKv->value == NULL) return -1; + if (tDecodeCStrTo(pDecoder, (char*)pKv->value) < 0) return -1; + return 0; } -static FORCE_INLINE int taosEncodeSClientHbKey(void** buf, const SClientHbKey* pKey) { - int tlen = 0; - tlen += taosEncodeFixedI32(buf, pKey->connId); - tlen += taosEncodeFixedI32(buf, pKey->hbType); - return tlen; +static FORCE_INLINE int32_t tEncodeSClientHbKey(SCoder* pEncoder, const SClientHbKey* pKey) { + if (tEncodeI32(pEncoder, pKey->connId) < 0) return -1; + if (tEncodeI32(pEncoder, pKey->hbType) < 0) return -1; + return 0; } -static FORCE_INLINE void* taosDecodeSClientHbKey(void* buf, SClientHbKey* pKey) { - buf = taosDecodeFixedI32(buf, &pKey->connId); - buf = taosDecodeFixedI32(buf, &pKey->hbType); - return buf; +static FORCE_INLINE int32_t tDecodeSClientHbKey(SCoder* pDecoder, SClientHbKey* pKey) { + if (tDecodeI32(pDecoder, &pKey->connId) < 0) return -1; + if (tDecodeI32(pDecoder, &pKey->hbType) < 0) return -1; + return 0; } typedef struct SMqHbVgInfo { @@ -1682,10 +1630,10 @@ static FORCE_INLINE void* taosDecodeSMqHbTopicInfoMsg(void* buf, SMqHbTopicInfo* } typedef struct SMqHbMsg { - int32_t status; // ask hb endpoint - int32_t epoch; - int64_t consumerId; - SArray* pTopics; // SArray + int32_t status; // ask hb endpoint + int32_t epoch; + int64_t consumerId; + SArray* pTopics; // SArray } SMqHbMsg; static FORCE_INLINE int taosEncodeSMqMsg(void** buf, const SMqHbMsg* pMsg) { @@ -1718,15 +1666,15 @@ static FORCE_INLINE void* taosDecodeSMqMsg(void* buf, SMqHbMsg* pMsg) { } typedef struct { - int64_t leftForVer; - int32_t vgId; - int64_t consumerId; - char topicName[TSDB_TOPIC_FNAME_LEN]; - char cgroup[TSDB_CONSUMER_GROUP_LEN]; - char* sql; - char* logicalPlan; - char* physicalPlan; - char* qmsg; + int64_t leftForVer; + int32_t vgId; + int64_t consumerId; + char topicName[TSDB_TOPIC_FNAME_LEN]; + char cgroup[TSDB_CONSUMER_GROUP_LEN]; + char* sql; + char* logicalPlan; + char* physicalPlan; + char* qmsg; } SMqSetCVgReq; static FORCE_INLINE int32_t tEncodeSMqSetCVgReq(void** buf, const SMqSetCVgReq* pReq) { @@ -1757,16 +1705,10 @@ static FORCE_INLINE void* tDecodeSMqSetCVgReq(void* buf, SMqSetCVgReq* pReq) { } typedef struct { - int64_t leftForVer; - int32_t vgId; - int64_t oldConsumerId; - int64_t newConsumerId; - //char topicName[TSDB_TOPIC_FNAME_LEN]; - //char cgroup[TSDB_CONSUMER_GROUP_LEN]; - //char* sql; - //char* logicalPlan; - //char* physicalPlan; - //char* qmsg; + int64_t leftForVer; + int32_t vgId; + int64_t oldConsumerId; + int64_t newConsumerId; } SMqMVRebReq; static FORCE_INLINE int32_t tEncodeSMqMVRebReq(void** buf, const SMqMVRebReq* pReq) { @@ -1775,13 +1717,6 @@ static FORCE_INLINE int32_t tEncodeSMqMVRebReq(void** buf, const SMqMVRebReq* pR tlen += taosEncodeFixedI32(buf, pReq->vgId); tlen += taosEncodeFixedI64(buf, pReq->oldConsumerId); tlen += taosEncodeFixedI64(buf, pReq->newConsumerId); - //tlen += taosEncodeString(buf, pReq->topicName); - //tlen += taosEncodeString(buf, pReq->cgroup); - //tlen += taosEncodeString(buf, pReq->sql); - //tlen += taosEncodeString(buf, pReq->logicalPlan); - //tlen += taosEncodeString(buf, pReq->physicalPlan); - //tlen += taosEncodeString(buf, pReq->qmsg); - //tlen += tEncodeSSubQueryMsg(buf, &pReq->msg); return tlen; } @@ -1790,13 +1725,6 @@ static FORCE_INLINE void* tDecodeSMqMVRebReq(void* buf, SMqMVRebReq* pReq) { buf = taosDecodeFixedI32(buf, &pReq->vgId); buf = taosDecodeFixedI64(buf, &pReq->oldConsumerId); buf = taosDecodeFixedI64(buf, &pReq->newConsumerId); - //buf = taosDecodeStringTo(buf, pReq->topicName); - //buf = taosDecodeStringTo(buf, pReq->cgroup); - //buf = taosDecodeString(buf, &pReq->sql); - //buf = taosDecodeString(buf, &pReq->logicalPlan); - //buf = taosDecodeString(buf, &pReq->physicalPlan); - //buf = taosDecodeString(buf, &pReq->qmsg); - //buf = tDecodeSSubQueryMsg(buf, &pReq->msg); return buf; } @@ -1818,10 +1746,10 @@ typedef struct { typedef struct { uint32_t nCols; - SSchema *pSchema; + SSchema* pSchema; } SSchemaWrapper; -static FORCE_INLINE int32_t tEncodeSSchema(void** buf, const SSchema* pSchema) { +static FORCE_INLINE int32_t taosEncodeSSchema(void** buf, const SSchema* pSchema) { int32_t tlen = 0; tlen += taosEncodeFixedI8(buf, pSchema->type); tlen += taosEncodeFixedI32(buf, pSchema->bytes); @@ -1830,7 +1758,7 @@ static FORCE_INLINE int32_t tEncodeSSchema(void** buf, const SSchema* pSchema) { return tlen; } -static FORCE_INLINE void* tDecodeSSchema(void* buf, SSchema* pSchema) { +static FORCE_INLINE void* taosDecodeSSchema(void* buf, SSchema* pSchema) { buf = taosDecodeFixedI8(buf, &pSchema->type); buf = taosDecodeFixedI32(buf, &pSchema->bytes); buf = taosDecodeFixedI32(buf, &pSchema->colId); @@ -1838,31 +1766,48 @@ static FORCE_INLINE void* tDecodeSSchema(void* buf, SSchema* pSchema) { return buf; } +static FORCE_INLINE int32_t tEncodeSSchema(SCoder* pEncoder, const SSchema* pSchema) { + if (tEncodeI8(pEncoder, pSchema->type) < 0) return -1; + if (tEncodeI32(pEncoder, pSchema->bytes) < 0) return -1; + if (tEncodeI32(pEncoder, pSchema->colId) < 0) return -1; + if (tEncodeCStr(pEncoder, pSchema->name) < 0) return -1; + return 0; +} + +static FORCE_INLINE int32_t tDecodeSSchema(SCoder* pDecoder, SSchema* pSchema) { + if (tDecodeI8(pDecoder, &pSchema->type) < 0) return -1; + if (tDecodeI32(pDecoder, &pSchema->bytes) < 0) return -1; + if (tDecodeI32(pDecoder, &pSchema->colId) < 0) return -1; + if (tDecodeCStrTo(pDecoder, pSchema->name) < 0) return -1; + return 0; +} + static FORCE_INLINE int32_t tEncodeSSchemaWrapper(void** buf, const SSchemaWrapper* pSW) { int32_t tlen = 0; tlen += taosEncodeFixedU32(buf, pSW->nCols); - for (int32_t i = 0; i < pSW->nCols; i ++) { - tlen += tEncodeSSchema(buf, &pSW->pSchema[i]); + for (int32_t i = 0; i < pSW->nCols; i++) { + tlen += taosEncodeSSchema(buf, &pSW->pSchema[i]); } return tlen; } static FORCE_INLINE void* tDecodeSSchemaWrapper(void* buf, SSchemaWrapper* pSW) { buf = taosDecodeFixedU32(buf, &pSW->nCols); - pSW->pSchema = (SSchema*) calloc(pSW->nCols, sizeof(SSchema)); + pSW->pSchema = (SSchema*)calloc(pSW->nCols, sizeof(SSchema)); if (pSW->pSchema == NULL) { return NULL; } - for (int32_t i = 0; i < pSW->nCols; i ++) { - buf = tDecodeSSchema(buf, &pSW->pSchema[i]); + + for (int32_t i = 0; i < pSW->nCols; i++) { + buf = taosDecodeSSchema(buf, &pSW->pSchema[i]); } return buf; } typedef struct { - int64_t uid; - int32_t numOfRows; - char* colData; + int64_t uid; + int32_t numOfRows; + char* colData; } SMqTbData; typedef struct { @@ -1884,24 +1829,24 @@ typedef struct { int64_t rspOffset; int32_t skipLogNum; int32_t numOfTopics; - SArray* pBlockData; //SArray + SArray* pBlockData; // SArray } SMqConsumeRsp; // one req for one vg+topic typedef struct { - SMsgHead head; - //0: commit only, current offset - //1: consume only, poll next offset - //2: commit current and consume next offset - int32_t reqType; + SMsgHead head; + // 0: commit only, current offset + // 1: consume only, poll next offset + // 2: commit current and consume next offset + int32_t reqType; - int64_t reqId; - int64_t consumerId; - int64_t blockingTime; - char cgroup[TSDB_CONSUMER_GROUP_LEN]; + int64_t reqId; + int64_t consumerId; + int64_t blockingTime; + char cgroup[TSDB_CONSUMER_GROUP_LEN]; - int64_t offset; - char topic[TSDB_TOPIC_FNAME_LEN]; + int64_t offset; + char topic[TSDB_TOPIC_FNAME_LEN]; } SMqConsumeReq; typedef struct { @@ -1911,7 +1856,7 @@ typedef struct { typedef struct { char topic[TSDB_TOPIC_FNAME_LEN]; - SArray* vgs; // SArray + SArray* vgs; // SArray } SMqSubTopicEp; typedef struct { @@ -1921,9 +1866,7 @@ typedef struct { SArray* topics; // SArray } SMqCMGetSubEpRsp; -static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) { - taosArrayDestroy(pSubTopicEp->vgs); -} +static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) { taosArrayDestroy(pSubTopicEp->vgs); } static FORCE_INLINE int32_t tEncodeSMqSubVgEp(void** buf, const SMqSubVgEp* pVgEp) { int32_t tlen = 0; @@ -1939,7 +1882,7 @@ static FORCE_INLINE void* tDecodeSMqSubVgEp(void* buf, SMqSubVgEp* pVgEp) { } static FORCE_INLINE void tDeleteSMqCMGetSubEpRsp(SMqCMGetSubEpRsp* pRsp) { - taosArrayDestroyEx(pRsp->topics, (void (*)(void*)) tDeleteSMqSubTopicEp); + taosArrayDestroyEx(pRsp->topics, (void (*)(void*))tDeleteSMqSubTopicEp); } static FORCE_INLINE int32_t tEncodeSMqSubTopicEp(void** buf, const SMqSubTopicEp* pTopicEp) { @@ -2006,4 +1949,4 @@ static FORCE_INLINE void* tDecodeSMqCMGetSubEpRsp(void* buf, SMqCMGetSubEpRsp* p } #endif -#endif /*_TD_COMMON_TAOS_MSG_H_*/ \ No newline at end of file +#endif /*_TD_COMMON_TAOS_MSG_H_*/ diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index cf890f5cbb..ca30e682bf 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -13,6 +13,8 @@ * along with this program. If not, see . */ +// clang-format off + #if 0 #undef TD_MSG_INFO_ #undef TD_MSG_NUMBER_ @@ -82,7 +84,6 @@ enum { TD_DEF_MSG_TYPE(TDMT_DND_CREATE_VNODE, "dnode-create-vnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_ALTER_VNODE, "dnode-alter-vnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_DROP_VNODE, "dnode-drop-vnode", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_DND_AUTH_VNODE, "dnode-auth-vnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_SYNC_VNODE, "dnode-sync-vnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_COMPACT_VNODE, "dnode-compact-vnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_CONFIG_DNODE, "dnode-config-dnode", NULL, NULL) @@ -137,12 +138,12 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_TRANS, "mnode-trans", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_GRANT, "mnode-grant", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_AUTH, "mnode-auth", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_CREATE_TOPIC, "mnode-create-topic", SCMCreateTopicReq, SCMCreateTopicRsp) + TD_DEF_MSG_TYPE(TDMT_MND_CREATE_TOPIC, "mnode-create-topic", SMCreateTopicReq, SMCreateTopicRsp) TD_DEF_MSG_TYPE(TDMT_MND_ALTER_TOPIC, "mnode-alter-topic", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_TOPIC, "mnode-drop-topic", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_SUBSCRIBE, "mnode-subscribe", SCMSubscribeReq, SCMSubscribeRsp) TD_DEF_MSG_TYPE(TDMT_MND_GET_SUB_EP, "mnode-get-sub-ep", SMqCMGetSubEpReq, SMqCMGetSubEpRsp) - TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-timer", SMqTmrMsg, SMqTmrMsg) + TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mnode-mq-timer", SMTimerReq, SMTimerReq) TD_DEF_MSG_TYPE(TDMT_MND_MQ_DO_REBALANCE, "mnode-mq-do-rebalance", SMqDoRebalanceMsg, SMqDoRebalanceMsg) // Requests handled by VNODE @@ -191,3 +192,4 @@ enum { TDMT_MAX #endif }; +// clang-format on diff --git a/include/common/trow.h b/include/common/trow.h index 4bb44e31f0..fad74a359d 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -473,7 +473,7 @@ static int32_t tdSRowResetBuf(SRowBuilder *pBuilder, void *pBuf) { terrno = TSDB_CODE_INVALID_PARA; return terrno; } - + TD_ROW_SET_TYPE(pBuilder->pBuf, pBuilder->rowType); uint32_t len = 0; diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index bdf9a925a8..5078b00c8c 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -105,7 +105,6 @@ typedef struct SFuncExecEnv { int32_t calcMemSize; } SFuncExecEnv; -typedef void* FuncMgtHandle; typedef bool (*FExecGetEnv)(SFunctionNode* pFunc, SFuncExecEnv* pEnv); typedef bool (*FExecInit)(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); typedef void (*FExecProcess)(struct SqlFunctionCtx *pCtx); @@ -127,9 +126,7 @@ typedef struct SScalarFuncExecFuncs { int32_t fmFuncMgtInit(); -int32_t fmGetHandle(FuncMgtHandle* pHandle); - -int32_t fmGetFuncInfo(FuncMgtHandle handle, const char* pFuncName, int32_t* pFuncId, int32_t* pFuncType); +int32_t fmGetFuncInfo(const char* pFuncName, int32_t* pFuncId, int32_t* pFuncType); int32_t fmGetFuncResultType(SFunctionNode* pFunc); diff --git a/include/nodes/nodes.h b/include/libs/nodes/nodes.h similarity index 95% rename from include/nodes/nodes.h rename to include/libs/nodes/nodes.h index 223bc6d7c1..8a8e230359 100644 --- a/include/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -49,7 +49,6 @@ typedef enum ENodeType { QUERY_NODE_VALUE, QUERY_NODE_OPERATOR, QUERY_NODE_LOGIC_CONDITION, - QUERY_NODE_IS_NULL_CONDITION, QUERY_NODE_FUNCTION, QUERY_NODE_REAL_TABLE, QUERY_NODE_TEMP_TABLE, @@ -62,6 +61,7 @@ typedef enum ENodeType { QUERY_NODE_INTERVAL_WINDOW, QUERY_NODE_NODE_LIST, QUERY_NODE_FILL, + QUERY_NODE_COLUMN_REF, // Only be used in parser module. QUERY_NODE_RAW_EXPR, @@ -69,7 +69,11 @@ typedef enum ENodeType { // Statement nodes are used in parser and planner module. QUERY_NODE_SET_OPERATOR, QUERY_NODE_SELECT_STMT, - QUERY_NODE_SHOW_STMT + QUERY_NODE_SHOW_STMT, + + QUERY_NODE_LOGIC_PLAN_SCAN, + QUERY_NODE_LOGIC_PLAN_FILTER, + QUERY_NODE_LOGIC_PLAN_AGG } ENodeType; /** @@ -122,7 +126,8 @@ void nodesRewriteListPostOrder(SNodeList* pList, FNodeRewriter rewriter, void* p bool nodesEqualNode(const SNode* a, const SNode* b); -void nodesCloneNode(const SNode* pNode); +SNode* nodesCloneNode(const SNode* pNode); +SNodeList* nodesCloneList(const SNodeList* pList); int32_t nodesNodeToString(const SNode* pNode, char** pStr, int32_t* pLen); int32_t nodesStringToNode(const char* pStr, SNode** pNode); diff --git a/include/nodes/nodesShowStmts.h b/include/libs/nodes/nodesShowStmts.h similarity index 100% rename from include/nodes/nodesShowStmts.h rename to include/libs/nodes/nodesShowStmts.h diff --git a/include/nodes/querynodes.h b/include/libs/nodes/querynodes.h similarity index 96% rename from include/nodes/querynodes.h rename to include/libs/nodes/querynodes.h index f41a5faec2..b6439acdbe 100644 --- a/include/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -37,7 +37,7 @@ typedef struct SDataType { } SDataType; typedef struct SExprNode { - ENodeType nodeType; + ENodeType type; SDataType resType; char aliasName[TSDB_COL_NAME_LEN]; SNodeList* pAssociationList; @@ -59,6 +59,11 @@ typedef struct SColumnNode { SNode* pProjectRef; } SColumnNode; +typedef struct SColumnRef { + ENodeType type; + int32_t slotId; +} SColumnRef; + typedef struct SValueNode { SExprNode node; // QUERY_NODE_VALUE char* literal; @@ -80,6 +85,10 @@ typedef enum EOperatorType { OP_TYPE_DIV, OP_TYPE_MOD, + // bit operator + OP_TYPE_BIT_AND, + OP_TYPE_BIT_OR, + // comparison operator OP_TYPE_GREATER_THAN, OP_TYPE_GREATER_EQUAL, @@ -93,8 +102,8 @@ typedef enum EOperatorType { OP_TYPE_NOT_LIKE, OP_TYPE_MATCH, OP_TYPE_NMATCH, - OP_TYPE_ISNULL, - OP_TYPE_NOTNULL, + OP_TYPE_IS_NULL, + OP_TYPE_IS_NOT_NULL, OP_TYPE_BIT_AND, OP_TYPE_BIT_OR, @@ -117,17 +126,11 @@ typedef enum ELogicConditionType { } ELogicConditionType; typedef struct SLogicConditionNode { - ENodeType type; // QUERY_NODE_LOGIC_CONDITION + SExprNode node; // QUERY_NODE_LOGIC_CONDITION ELogicConditionType condType; SNodeList* pParameterList; } SLogicConditionNode; -typedef struct SIsNullCondNode { - ENodeType type; // QUERY_NODE_IS_NULL_CONDITION - SNode* pExpr; - bool isNull; -} SIsNullCondNode; - typedef struct SNodeListNode { ENodeType type; // QUERY_NODE_NODE_LIST SNodeList* pNodeList; @@ -142,7 +145,7 @@ typedef struct SFunctionNode { } SFunctionNode; typedef struct STableNode { - ENodeType type; + SExprNode node; char dbName[TSDB_DB_NAME_LEN]; char tableName[TSDB_TABLE_NAME_LEN]; char tableAlias[TSDB_TABLE_NAME_LEN]; diff --git a/include/libs/parser/parsenodes.h b/include/libs/parser/parsenodes.h index 292b4ab2c9..6cd5feca24 100644 --- a/include/libs/parser/parsenodes.h +++ b/include/libs/parser/parsenodes.h @@ -135,7 +135,7 @@ typedef struct SVgDataBlocks { SVgroupInfo vg; int32_t numOfTables; // number of tables in current submit block uint32_t size; - char *pData; // SMsgDesc + SSubmitMsg + SSubmitBlk + ... + char *pData; // SMsgDesc + SSubmitReq + SSubmitBlk + ... } SVgDataBlocks; typedef struct SVnodeModifOpStmtInfo { diff --git a/include/libs/planner/plannerOp.h b/include/libs/planner/plannerOp.h index 31f5457c90..9030ffc946 100644 --- a/include/libs/planner/plannerOp.h +++ b/include/libs/planner/plannerOp.h @@ -24,7 +24,7 @@ #endif OP_ENUM_MACRO(StreamScan) -OP_ENUM_MACRO(DataBlocksOptScan) +OP_ENUM_MACRO(TableScan) OP_ENUM_MACRO(TableSeqScan) OP_ENUM_MACRO(TagScan) OP_ENUM_MACRO(SystemTableScan) diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 0a822927ad..5c12d29b50 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -459,6 +459,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION TAOS_DEF_ERROR_CODE(0, 0x260A) //Not a GROUP BY expression #define TSDB_CODE_PAR_NOT_SELECTED_EXPRESSION TAOS_DEF_ERROR_CODE(0, 0x260B) //Not SELECTed expression #define TSDB_CODE_PAR_NOT_SINGLE_GROUP TAOS_DEF_ERROR_CODE(0, 0x260C) //Not a single-group group function +#define TSDB_CODE_PAR_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x260D) //Out of memory #ifdef __cplusplus } diff --git a/include/util/tcoding.h b/include/util/tcoding.h index c105ce1ab9..fed9d12cee 100644 --- a/include/util/tcoding.h +++ b/include/util/tcoding.h @@ -37,7 +37,7 @@ static FORCE_INLINE int taosEncodeFixedU8(void **buf, uint8_t value) { return (int)sizeof(value); } -static FORCE_INLINE void *taosDecodeFixedU8(void *buf, uint8_t *value) { +static FORCE_INLINE void *taosDecodeFixedU8(const void *buf, uint8_t *value) { *value = ((uint8_t *)buf)[0]; return POINTER_SHIFT(buf, sizeof(*value)); } @@ -51,7 +51,7 @@ static FORCE_INLINE int taosEncodeFixedI8(void **buf, int8_t value) { return (int)sizeof(value); } -static FORCE_INLINE void *taosDecodeFixedI8(void *buf, int8_t *value) { +static FORCE_INLINE void *taosDecodeFixedI8(const void *buf, int8_t *value) { *value = ((int8_t *)buf)[0]; return POINTER_SHIFT(buf, sizeof(*value)); } @@ -71,7 +71,7 @@ static FORCE_INLINE int taosEncodeFixedU16(void **buf, uint16_t value) { return (int)sizeof(value); } -static FORCE_INLINE void *taosDecodeFixedU16(void *buf, uint16_t *value) { +static FORCE_INLINE void *taosDecodeFixedU16(const void *buf, uint16_t *value) { if (IS_LITTLE_ENDIAN()) { memcpy(value, buf, sizeof(*value)); } else { @@ -87,9 +87,9 @@ static FORCE_INLINE int taosEncodeFixedI16(void **buf, int16_t value) { return taosEncodeFixedU16(buf, ZIGZAGE(int16_t, value)); } -static FORCE_INLINE void *taosDecodeFixedI16(void *buf, int16_t *value) { +static FORCE_INLINE void *taosDecodeFixedI16(const void *buf, int16_t *value) { uint16_t tvalue = 0; - void * ret = taosDecodeFixedU16(buf, &tvalue); + void *ret = taosDecodeFixedU16(buf, &tvalue); *value = ZIGZAGD(int16_t, tvalue); return ret; } @@ -111,7 +111,7 @@ static FORCE_INLINE int taosEncodeFixedU32(void **buf, uint32_t value) { return (int)sizeof(value); } -static FORCE_INLINE void *taosDecodeFixedU32(void *buf, uint32_t *value) { +static FORCE_INLINE void *taosDecodeFixedU32(const void *buf, uint32_t *value) { if (IS_LITTLE_ENDIAN()) { memcpy(value, buf, sizeof(*value)); } else { @@ -129,9 +129,9 @@ static FORCE_INLINE int taosEncodeFixedI32(void **buf, int32_t value) { return taosEncodeFixedU32(buf, ZIGZAGE(int32_t, value)); } -static FORCE_INLINE void *taosDecodeFixedI32(void *buf, int32_t *value) { +static FORCE_INLINE void *taosDecodeFixedI32(const void *buf, int32_t *value) { uint32_t tvalue = 0; - void * ret = taosDecodeFixedU32(buf, &tvalue); + void *ret = taosDecodeFixedU32(buf, &tvalue); *value = ZIGZAGD(int32_t, tvalue); return ret; } @@ -158,7 +158,7 @@ static FORCE_INLINE int taosEncodeFixedU64(void **buf, uint64_t value) { return (int)sizeof(value); } -static FORCE_INLINE void *taosDecodeFixedU64(void *buf, uint64_t *value) { +static FORCE_INLINE void *taosDecodeFixedU64(const void *buf, uint64_t *value) { if (IS_LITTLE_ENDIAN()) { memcpy(value, buf, sizeof(*value)); } else { @@ -180,9 +180,9 @@ static FORCE_INLINE int taosEncodeFixedI64(void **buf, int64_t value) { return taosEncodeFixedU64(buf, ZIGZAGE(int64_t, value)); } -static FORCE_INLINE void *taosDecodeFixedI64(void *buf, int64_t *value) { +static FORCE_INLINE void *taosDecodeFixedI64(const void *buf, int64_t *value) { uint64_t tvalue = 0; - void * ret = taosDecodeFixedU64(buf, &tvalue); + void *ret = taosDecodeFixedU64(buf, &tvalue); *value = ZIGZAGD(int64_t, tvalue); return ret; } @@ -205,7 +205,7 @@ static FORCE_INLINE int taosEncodeVariantU16(void **buf, uint16_t value) { return i + 1; } -static FORCE_INLINE void *taosDecodeVariantU16(void *buf, uint16_t *value) { +static FORCE_INLINE void *taosDecodeVariantU16(const void *buf, uint16_t *value) { int i = 0; uint16_t tval = 0; *value = 0; @@ -228,9 +228,9 @@ static FORCE_INLINE int taosEncodeVariantI16(void **buf, int16_t value) { return taosEncodeVariantU16(buf, ZIGZAGE(int16_t, value)); } -static FORCE_INLINE void *taosDecodeVariantI16(void *buf, int16_t *value) { +static FORCE_INLINE void *taosDecodeVariantI16(const void *buf, int16_t *value) { uint16_t tvalue = 0; - void * ret = taosDecodeVariantU16(buf, &tvalue); + void *ret = taosDecodeVariantU16(buf, &tvalue); *value = ZIGZAGD(int16_t, tvalue); return ret; } @@ -253,7 +253,7 @@ static FORCE_INLINE int taosEncodeVariantU32(void **buf, uint32_t value) { return i + 1; } -static FORCE_INLINE void *taosDecodeVariantU32(void *buf, uint32_t *value) { +static FORCE_INLINE void *taosDecodeVariantU32(const void *buf, uint32_t *value) { int i = 0; uint32_t tval = 0; *value = 0; @@ -276,9 +276,9 @@ static FORCE_INLINE int taosEncodeVariantI32(void **buf, int32_t value) { return taosEncodeVariantU32(buf, ZIGZAGE(int32_t, value)); } -static FORCE_INLINE void *taosDecodeVariantI32(void *buf, int32_t *value) { +static FORCE_INLINE void *taosDecodeVariantI32(const void *buf, int32_t *value) { uint32_t tvalue = 0; - void * ret = taosDecodeVariantU32(buf, &tvalue); + void *ret = taosDecodeVariantU32(buf, &tvalue); *value = ZIGZAGD(int32_t, tvalue); return ret; } @@ -301,7 +301,7 @@ static FORCE_INLINE int taosEncodeVariantU64(void **buf, uint64_t value) { return i + 1; } -static FORCE_INLINE void *taosDecodeVariantU64(void *buf, uint64_t *value) { +static FORCE_INLINE void *taosDecodeVariantU64(const void *buf, uint64_t *value) { int i = 0; uint64_t tval = 0; *value = 0; @@ -324,9 +324,9 @@ static FORCE_INLINE int taosEncodeVariantI64(void **buf, int64_t value) { return taosEncodeVariantU64(buf, ZIGZAGE(int64_t, value)); } -static FORCE_INLINE void *taosDecodeVariantI64(void *buf, int64_t *value) { +static FORCE_INLINE void *taosDecodeVariantI64(const void *buf, int64_t *value) { uint64_t tvalue = 0; - void * ret = taosDecodeVariantU64(buf, &tvalue); + void *ret = taosDecodeVariantU64(buf, &tvalue); *value = ZIGZAGD(int64_t, tvalue); return ret; } @@ -346,7 +346,7 @@ static FORCE_INLINE int taosEncodeString(void **buf, const char *value) { return tlen; } -static FORCE_INLINE void *taosDecodeString(void *buf, char **value) { +static FORCE_INLINE void *taosDecodeString(const void *buf, char **value) { uint64_t size = 0; buf = taosDecodeVariantU64(buf, &size); @@ -360,7 +360,7 @@ static FORCE_INLINE void *taosDecodeString(void *buf, char **value) { return POINTER_SHIFT(buf, size); } -static FORCE_INLINE void *taosDecodeStringTo(void *buf, char *value) { +static FORCE_INLINE void *taosDecodeStringTo(const void *buf, char *value) { uint64_t size = 0; buf = taosDecodeVariantU64(buf, &size); @@ -373,7 +373,7 @@ static FORCE_INLINE void *taosDecodeStringTo(void *buf, char *value) { // ---- binary static FORCE_INLINE int taosEncodeBinary(void **buf, const void *value, int32_t valueLen) { - int tlen = 0; + int tlen = 0; if (buf != NULL) { memcpy(*buf, value, valueLen); @@ -384,8 +384,7 @@ static FORCE_INLINE int taosEncodeBinary(void **buf, const void *value, int32_t return tlen; } -static FORCE_INLINE void *taosDecodeBinary(void *buf, void **value, int32_t valueLen) { - +static FORCE_INLINE void *taosDecodeBinary(const void *buf, void **value, int32_t valueLen) { *value = malloc((size_t)valueLen); if (*value == NULL) return NULL; memcpy(*value, buf, (size_t)valueLen); @@ -393,8 +392,7 @@ static FORCE_INLINE void *taosDecodeBinary(void *buf, void **value, int32_t valu return POINTER_SHIFT(buf, valueLen); } -static FORCE_INLINE void *taosDecodeBinaryTo(void *buf, void *value, int32_t valueLen) { - +static FORCE_INLINE void *taosDecodeBinaryTo(const void *buf, void *value, int32_t valueLen) { memcpy(value, buf, (size_t)valueLen); return POINTER_SHIFT(buf, valueLen); } diff --git a/include/util/tlosertree.h b/include/util/tlosertree.h index d6ffde82ca..241647ba1e 100644 --- a/include/util/tlosertree.h +++ b/include/util/tlosertree.h @@ -22,28 +22,31 @@ extern "C" { typedef int (*__merge_compare_fn_t)(const void *, const void *, void *param); -typedef struct SLoserTreeNode { +typedef struct STreeNode { int32_t index; - void *pData; -} SLoserTreeNode; + void *pData; // TODO remove it? +} STreeNode; -typedef struct SLoserTreeInfo { - int32_t numOfEntries; - int32_t totalEntries; +typedef struct SMultiwayMergeTreeInfo { + int32_t numOfSources; + int32_t totalSources; __merge_compare_fn_t comparFn; void * param; - SLoserTreeNode *pNode; -} SLoserTreeInfo; + struct STreeNode *pNode; +} SMultiwayMergeTreeInfo; -uint32_t tLoserTreeCreate(SLoserTreeInfo **pTree, int32_t numOfEntries, void *param, __merge_compare_fn_t compareFn); +#define tMergeTreeGetChosenIndex(t_) ((t_)->pNode[0].index) +#define tMergeTreeGetAdjustIndex(t_) (tMergeTreeGetChosenIndex(t_) + (t_)->numOfSources) -void tLoserTreeInit(SLoserTreeInfo *pTree); +int32_t tMergeTreeCreate(SMultiwayMergeTreeInfo **pTree, uint32_t numOfEntries, void *param, __merge_compare_fn_t compareFn); -void tLoserTreeAdjust(SLoserTreeInfo *pTree, int32_t idx); +void tMergeTreeDestroy(SMultiwayMergeTreeInfo* pTree); -void tLoserTreeRebuild(SLoserTreeInfo *pTree); +void tMergeTreeAdjust(SMultiwayMergeTreeInfo *pTree, int32_t idx); -void tLoserTreeDisplay(SLoserTreeInfo *pTree); +void tMergeTreeRebuild(SMultiwayMergeTreeInfo *pTree); + +void tMergeTreePrint(const SMultiwayMergeTreeInfo *pTree); #ifdef __cplusplus } diff --git a/include/util/tpagedbuf.h b/include/util/tpagedbuf.h new file mode 100644 index 0000000000..e989c31cd6 --- /dev/null +++ b/include/util/tpagedbuf.h @@ -0,0 +1,171 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_TPAGEDBUF_H +#define TDENGINE_TPAGEDBUF_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "tlist.h" +#include "thash.h" +#include "os.h" +#include "tlockfree.h" + +typedef struct SArray* SIDList; +typedef struct SPageInfo SPageInfo; +typedef struct SDiskbasedBuf SDiskbasedBuf; + +#define DEFAULT_INTERN_BUF_PAGE_SIZE (1024L) // in bytes +#define DEFAULT_PAGE_SIZE (16384L) + +typedef struct SFilePage { + int64_t num; + char data[]; +} SFilePage; + +typedef struct SDiskbasedBufStatis { + int64_t flushBytes; + int64_t loadBytes; + int32_t loadPages; + int32_t getPages; + int32_t releasePages; + int32_t flushPages; +} SDiskbasedBufStatis; + +/** + * create disk-based result buffer + * @param pBuf + * @param rowSize + * @param pagesize + * @param inMemPages + * @param handle + * @return + */ +int32_t createDiskbasedBuffer(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId, const char* dir); + +/** + * + * @param pBuf + * @param groupId + * @param pageId + * @return + */ +SFilePage* getNewDataBuf(SDiskbasedBuf* pBuf, int32_t groupId, int32_t* pageId); + +/** + * + * @param pBuf + * @param groupId + * @return + */ +SIDList getDataBufPagesIdList(SDiskbasedBuf* pBuf, int32_t groupId); + +/** + * get the specified buffer page by id + * @param pBuf + * @param id + * @return + */ +SFilePage* getBufPage(SDiskbasedBuf* pBuf, int32_t id); + +/** + * release the referenced buf pages + * @param pBuf + * @param page + */ +void releaseBufPage(SDiskbasedBuf* pBuf, void* page); + +/** + * + * @param pBuf + * @param pi + */ +void releaseBufPageInfo(SDiskbasedBuf* pBuf, struct SPageInfo* pi); + +/** + * get the total buffer size in the format of disk file + * @param pBuf + * @return + */ +size_t getTotalBufSize(const SDiskbasedBuf* pBuf); + +/** + * get the number of groups in the result buffer + * @param pBuf + * @return + */ +size_t getNumOfResultBufGroupId(const SDiskbasedBuf* pBuf); + +/** + * destroy result buffer + * @param pBuf + */ +void destroyResultBuf(SDiskbasedBuf* pBuf); + +/** + * + * @param pList + * @return + */ +SPageInfo* getLastPageInfo(SIDList pList); + +/** + * + * @param pPgInfo + * @return + */ +int32_t getPageId(const SPageInfo* pPgInfo); + +/** + * Return the buffer page size. + * @param pBuf + * @return + */ +int32_t getBufPageSize(const SDiskbasedBuf* pBuf); + +int32_t getNumOfInMemBufPages(const SDiskbasedBuf* pBuf); + +/** + * + * @param pBuf + * @return + */ +bool isAllDataInMemBuf(const SDiskbasedBuf* pBuf); + +/** + * Set the buffer page is dirty, and needs to be flushed to disk when swap out. + * @param pPageInfo + * @param dirty + */ +void setBufPageDirty(SFilePage* pPageInfo, bool dirty); + +/** + * Print the statistics when closing this buffer + * @param pBuf + */ +void printStatisBeforeClose(SDiskbasedBuf* pBuf); + +/** + * return buf statistics. + */ +SDiskbasedBufStatis getDBufStatis(const SDiskbasedBuf* pBuf); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_TPAGEDBUF_H diff --git a/include/util/tpagedfile.h b/include/util/tpagedfile.h deleted file mode 100644 index 5bc4dc92a0..0000000000 --- a/include/util/tpagedfile.h +++ /dev/null @@ -1,169 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef TDENGINE_TPAGEDFILE_H -#define TDENGINE_TPAGEDFILE_H - -#ifdef __cplusplus -extern "C" { -#endif - -#include "tlist.h" -#include "thash.h" -#include "os.h" -#include "tlockfree.h" - -typedef struct SArray* SIDList; - -typedef struct SPageDiskInfo { - int32_t offset; - int32_t length; -} SPageDiskInfo; - -typedef struct SPageInfo { - SListNode* pn; // point to list node - int32_t pageId; - SPageDiskInfo info; - void* pData; - bool used; // set current page is in used -} SPageInfo; - -typedef struct SFreeListItem { - int32_t offset; - int32_t len; -} SFreeListItem; - -typedef struct SResultBufStatis { - int32_t flushBytes; - int32_t loadBytes; - int32_t getPages; - int32_t releasePages; - int32_t flushPages; -} SResultBufStatis; - -typedef struct SDiskbasedResultBuf { - int32_t numOfPages; - int64_t totalBufSize; - int64_t fileSize; // disk file size - FILE* file; - int32_t allocateId; // allocated page id - char* path; // file path - int32_t pageSize; // current used page size - int32_t inMemPages; // numOfPages that are allocated in memory - SHashObj* groupSet; // id hash table - SHashObj* all; - SList* lruList; - void* emptyDummyIdList; // dummy id list - void* assistBuf; // assistant buffer for compress/decompress data - SArray* pFree; // free area in file - bool comp; // compressed before flushed to disk - int32_t nextPos; // next page flush position - - uint64_t qId; // for debug purpose - SResultBufStatis statis; -} SDiskbasedResultBuf; - -#define DEFAULT_INTERN_BUF_PAGE_SIZE (1024L) // in bytes -#define PAGE_INFO_INITIALIZER (SPageDiskInfo){-1, -1} -#define DEFAULT_PAGE_SIZE (16384L) - -typedef struct SFilePage { - int64_t num; - char data[]; -} SFilePage; - -/** - * create disk-based result buffer - * @param pResultBuf - * @param rowSize - * @param pagesize - * @param inMemPages - * @param handle - * @return - */ -int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId, const char* dir); - -/** - * - * @param pResultBuf - * @param groupId - * @param pageId - * @return - */ -SFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t* pageId); - -/** - * - * @param pResultBuf - * @param groupId - * @return - */ -SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId); - -/** - * get the specified buffer page by id - * @param pResultBuf - * @param id - * @return - */ -SFilePage* getResBufPage(SDiskbasedResultBuf* pResultBuf, int32_t id); - -/** - * release the referenced buf pages - * @param pResultBuf - * @param page - */ -void releaseResBufPage(SDiskbasedResultBuf* pResultBuf, void* page); - -/** - * - * @param pResultBuf - * @param pi - */ -void releaseResBufPageInfo(SDiskbasedResultBuf* pResultBuf, SPageInfo* pi); - - -/** - * get the total buffer size in the format of disk file - * @param pResultBuf - * @return - */ -size_t getResBufSize(const SDiskbasedResultBuf* pResultBuf); - -/** - * get the number of groups in the result buffer - * @param pResultBuf - * @return - */ -size_t getNumOfResultBufGroupId(const SDiskbasedResultBuf* pResultBuf); - -/** - * destroy result buffer - * @param pResultBuf - */ -void destroyResultBuf(SDiskbasedResultBuf* pResultBuf); - -/** - * - * @param pList - * @return - */ -SPageInfo* getLastPageInfo(SIDList pList); - -#ifdef __cplusplus -} -#endif - -#endif // TDENGINE_TPAGEDFILE_H diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index fbf045b99c..2833b329a7 100644 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -3,5 +3,4 @@ add_subdirectory(util) add_subdirectory(common) add_subdirectory(libs) add_subdirectory(client) -add_subdirectory(dnode) -add_subdirectory(nodes) \ No newline at end of file +add_subdirectory(dnode) \ No newline at end of file diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 3f60c5c6e1..73a09f473e 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -28,16 +28,17 @@ static int32_t hbMqHbRspHandle(struct SAppHbMgr *pAppHbMgr, SClientHbRsp* pRsp) } static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) { - int32_t msgLen = 0; int32_t code = 0; - - while (msgLen < valueLen) { - SUseDbRsp *rsp = (SUseDbRsp *)((char *)value + msgLen); - rsp->vgVersion = ntohl(rsp->vgVersion); - rsp->vgNum = ntohl(rsp->vgNum); - rsp->uid = be64toh(rsp->uid); + SUseDbBatchRsp batchUseRsp = {0}; + if (tDeserializeSUseDbBatchRsp(value, valueLen, &batchUseRsp) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + int32_t numOfBatchs = taosArrayGetSize(batchUseRsp.pArray); + for (int32_t i = 0; i < numOfBatchs; ++i) { + SUseDbRsp *rsp = taosArrayGet(batchUseRsp.pArray, i); tscDebug("hb db rsp, db:%s, vgVersion:%d, uid:%"PRIx64, rsp->db, rsp->vgVersion, rsp->uid); if (rsp->vgVersion < 0) { @@ -52,16 +53,9 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog return TSDB_CODE_TSC_OUT_OF_MEMORY; } - for (int32_t i = 0; i < rsp->vgNum; ++i) { - rsp->vgroupInfo[i].vgId = ntohl(rsp->vgroupInfo[i].vgId); - rsp->vgroupInfo[i].hashBegin = ntohl(rsp->vgroupInfo[i].hashBegin); - rsp->vgroupInfo[i].hashEnd = ntohl(rsp->vgroupInfo[i].hashEnd); - - for (int32_t n = 0; n < rsp->vgroupInfo[i].epset.numOfEps; ++n) { - rsp->vgroupInfo[i].epset.eps[n].port = ntohs(rsp->vgroupInfo[i].epset.eps[n].port); - } - - if (0 != taosHashPut(vgInfo.vgHash, &rsp->vgroupInfo[i].vgId, sizeof(rsp->vgroupInfo[i].vgId), &rsp->vgroupInfo[i], sizeof(rsp->vgroupInfo[i]))) { + for (int32_t j = 0; j < rsp->vgNum; ++j) { + SVgroupInfo *pInfo = taosArrayGet(rsp->pVgroupInfos, j); + if (taosHashPut(vgInfo.vgHash, &pInfo->vgId, sizeof(int32_t), pInfo, sizeof(SVgroupInfo)) != 0) { tscError("hash push failed, errno:%d", errno); taosHashCleanup(vgInfo.vgHash); return TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -74,66 +68,44 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog if (code) { return code; } - - msgLen += sizeof(SUseDbRsp) + rsp->vgNum * sizeof(SVgroupInfo); } + tFreeSUseDbBatchRsp(&batchUseRsp); return TSDB_CODE_SUCCESS; } static int32_t hbProcessStbInfoRsp(void *value, int32_t valueLen, struct SCatalog *pCatalog) { - int32_t msgLen = 0; int32_t code = 0; - int32_t schemaNum = 0; - - while (msgLen < valueLen) { - STableMetaRsp *rsp = (STableMetaRsp *)((char *)value + msgLen); - rsp->numOfColumns = ntohl(rsp->numOfColumns); - rsp->suid = be64toh(rsp->suid); - rsp->dbId = be64toh(rsp->dbId); - + STableMetaBatchRsp batchMetaRsp = {0}; + if (tDeserializeSTableMetaBatchRsp(value, valueLen, &batchMetaRsp) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + int32_t numOfBatchs = taosArrayGetSize(batchMetaRsp.pArray); + for (int32_t i = 0; i < numOfBatchs; ++i) { + STableMetaRsp *rsp = taosArrayGet(batchMetaRsp.pArray, i); + if (rsp->numOfColumns < 0) { - schemaNum = 0; - tscDebug("hb remove stb, db:%s, stb:%s", rsp->dbFName, rsp->stbName); - catalogRemoveStbMeta(pCatalog, rsp->dbFName, rsp->dbId, rsp->stbName, rsp->suid); } else { tscDebug("hb update stb, db:%s, stb:%s", rsp->dbFName, rsp->stbName); - - rsp->numOfTags = ntohl(rsp->numOfTags); - rsp->sversion = ntohl(rsp->sversion); - rsp->tversion = ntohl(rsp->tversion); - rsp->tuid = be64toh(rsp->tuid); - rsp->vgId = ntohl(rsp->vgId); - - SSchema* pSchema = rsp->pSchema; - - schemaNum = rsp->numOfColumns + rsp->numOfTags; - - for (int i = 0; i < schemaNum; ++i) { - pSchema->bytes = ntohl(pSchema->bytes); - pSchema->colId = ntohl(pSchema->colId); - - pSchema++; - } - - if (rsp->pSchema[0].colId != PRIMARYKEY_TIMESTAMP_COL_ID) { - tscError("invalid colId[%d] for the first column in table meta rsp msg", rsp->pSchema[0].colId); + if (rsp->pSchemas[0].colId != PRIMARYKEY_TIMESTAMP_COL_ID) { + tscError("invalid colId[%d] for the first column in table meta rsp msg", rsp->pSchemas[0].colId); + tFreeSTableMetaBatchRsp(&batchMetaRsp); return TSDB_CODE_TSC_INVALID_VALUE; - } + } catalogUpdateSTableMeta(pCatalog, rsp); } - - msgLen += sizeof(STableMetaRsp) + schemaNum * sizeof(SSchema); } + tFreeSTableMetaBatchRsp(&batchMetaRsp); return TSDB_CODE_SUCCESS; } - static int32_t hbQueryHbRspHandle(struct SAppHbMgr *pAppHbMgr, SClientHbRsp* pRsp) { SHbConnInfo * info = taosHashGet(pAppHbMgr->connInfo, &pRsp->connKey, sizeof(SClientHbKey)); if (NULL == info) { @@ -199,9 +171,10 @@ static int32_t hbMqAsyncCallBack(void* param, const SDataBuf* pMsg, int32_t code tfree(param); return -1; } + char *key = (char *)param; SClientHbBatchRsp pRsp = {0}; - tDeserializeSClientHbBatchRsp(pMsg->pData, &pRsp); + tDeserializeSClientHbBatchRsp(pMsg->pData, pMsg->len, &pRsp); int32_t rspNum = taosArrayGetSize(pRsp.rsps); @@ -342,7 +315,7 @@ void hbFreeReq(void *req) { SClientHbBatchReq* hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { - SClientHbBatchReq* pBatchReq = malloc(sizeof(SClientHbBatchReq)); + SClientHbBatchReq* pBatchReq = calloc(1, sizeof(SClientHbBatchReq)); if (pBatchReq == NULL) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; return NULL; @@ -414,7 +387,7 @@ static void* hbThreadFunc(void* param) { if (pReq == NULL) { continue; } - int tlen = tSerializeSClientHbBatchReq(NULL, pReq); + int tlen = tSerializeSClientHbBatchReq(NULL, 0, pReq); void *buf = malloc(tlen); if (buf == NULL) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -422,8 +395,7 @@ static void* hbThreadFunc(void* param) { hbClearReqInfo(pAppHbMgr); break; } - void *abuf = buf; - tSerializeSClientHbBatchReq(&abuf, pReq); + tSerializeSClientHbBatchReq(buf, tlen, pReq); SMsgSendInfo *pInfo = malloc(sizeof(SMsgSendInfo)); if (pInfo == NULL) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index dfe7b12ce4..f3bbb9481b 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -9,7 +9,7 @@ #include "tglobal.h" #include "tmsgtype.h" #include "tnote.h" -#include "tpagedfile.h" +#include "tpagedbuf.h" #include "tref.h" static int32_t initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet); @@ -357,40 +357,38 @@ STscObj* taosConnectImpl(const char *user, const char *auth, const char *db, __t return pTscObj; } -static SMsgSendInfo* buildConnectMsg(SRequestObj *pRequest) { - SMsgSendInfo *pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo)); +static SMsgSendInfo* buildConnectMsg(SRequestObj* pRequest) { + SMsgSendInfo* pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo)); if (pMsgSendInfo == NULL) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; return NULL; } - pMsgSendInfo->msgType = TDMT_MND_CONNECT; - pMsgSendInfo->msgInfo.len = sizeof(SConnectReq); + pMsgSendInfo->msgType = TDMT_MND_CONNECT; pMsgSendInfo->requestObjRefId = pRequest->self; - pMsgSendInfo->requestId = pRequest->requestId; - pMsgSendInfo->fp = handleRequestRspFp[TMSG_INDEX(pMsgSendInfo->msgType)]; - pMsgSendInfo->param = pRequest; + pMsgSendInfo->requestId = pRequest->requestId; + pMsgSendInfo->fp = handleRequestRspFp[TMSG_INDEX(pMsgSendInfo->msgType)]; + pMsgSendInfo->param = pRequest; - SConnectReq *pConnect = calloc(1, sizeof(SConnectReq)); - if (pConnect == NULL) { - tfree(pMsgSendInfo); - terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; - return NULL; - } - - STscObj *pObj = pRequest->pTscObj; + SConnectReq connectReq = {0}; + STscObj* pObj = pRequest->pTscObj; char* db = getDbOfConnection(pObj); if (db != NULL) { - tstrncpy(pConnect->db, db, sizeof(pConnect->db)); + tstrncpy(connectReq.db, db, sizeof(connectReq.db)); } tfree(db); - pConnect->pid = htonl(appInfo.pid); - pConnect->startTime = htobe64(appInfo.startTime); - tstrncpy(pConnect->app, appInfo.appName, tListLen(pConnect->app)); + connectReq.pid = htonl(appInfo.pid); + connectReq.startTime = htobe64(appInfo.startTime); + tstrncpy(connectReq.app, appInfo.appName, sizeof(connectReq.app)); - pMsgSendInfo->msgInfo.pData = pConnect; + int32_t contLen = tSerializeSConnectReq(NULL, 0, &connectReq); + void* pReq = malloc(contLen); + tSerializeSConnectReq(pReq, contLen, &connectReq); + + pMsgSendInfo->msgInfo.len = contLen; + pMsgSendInfo->msgInfo.pData = pReq; return pMsgSendInfo; } diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 8ab1880069..48c9920c0c 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -20,14 +20,14 @@ #include "clientLog.h" #include "catalog.h" -int (*handleRequestRspFp[TDMT_MAX])(void*, const SDataBuf* pMsg, int32_t code); +int32_t (*handleRequestRspFp[TDMT_MAX])(void*, const SDataBuf* pMsg, int32_t code); static void setErrno(SRequestObj* pRequest, int32_t code) { pRequest->code = code; terrno = code; } -int genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code) { +int32_t genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code) { SRequestObj* pRequest = param; setErrno(pRequest, code); @@ -36,7 +36,7 @@ int genericRspCallback(void* param, const SDataBuf* pMsg, int32_t code) { return code; } -int processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) { +int32_t processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) { SRequestObj* pRequest = param; if (code != TSDB_CODE_SUCCESS) { free(pMsg->pData); @@ -45,41 +45,35 @@ int processConnectRsp(void* param, const SDataBuf* pMsg, int32_t code) { return code; } - STscObj *pTscObj = pRequest->pTscObj; + STscObj* pTscObj = pRequest->pTscObj; - SConnectRsp *pConnect = (SConnectRsp *)pMsg->pData; - pConnect->acctId = htonl(pConnect->acctId); - pConnect->connId = htonl(pConnect->connId); - pConnect->clusterId = htobe64(pConnect->clusterId); + SConnectRsp connectRsp = {0}; + tDeserializeSConnectRsp(pMsg->pData, pMsg->len, &connectRsp); + assert(connectRsp.epSet.numOfEps > 0); - assert(pConnect->epSet.numOfEps > 0); - for(int32_t i = 0; i < pConnect->epSet.numOfEps; ++i) { - pConnect->epSet.eps[i].port = htons(pConnect->epSet.eps[i].port); + if (!isEpsetEqual(&pTscObj->pAppInfo->mgmtEp.epSet, &connectRsp.epSet)) { + updateEpSet_s(&pTscObj->pAppInfo->mgmtEp, &connectRsp.epSet); } - if (!isEpsetEqual(&pTscObj->pAppInfo->mgmtEp.epSet, &pConnect->epSet)) { - updateEpSet_s(&pTscObj->pAppInfo->mgmtEp, &pConnect->epSet); + for (int32_t i = 0; i < connectRsp.epSet.numOfEps; ++i) { + tscDebug("0x%" PRIx64 " epSet.fqdn[%d]:%s port:%d, connObj:0x%" PRIx64, pRequest->requestId, i, + connectRsp.epSet.eps[i].fqdn, connectRsp.epSet.eps[i].port, pTscObj->id); } - for (int i = 0; i < pConnect->epSet.numOfEps; ++i) { - tscDebug("0x%" PRIx64 " epSet.fqdn[%d]:%s port:%d, connObj:0x%"PRIx64, pRequest->requestId, i, pConnect->epSet.eps[i].fqdn, - pConnect->epSet.eps[i].port, pTscObj->id); - } - - pTscObj->connId = pConnect->connId; - pTscObj->acctId = pConnect->acctId; - tstrncpy(pTscObj->ver, pConnect->sVersion, tListLen(pTscObj->ver)); + pTscObj->connId = connectRsp.connId; + pTscObj->acctId = connectRsp.acctId; + tstrncpy(pTscObj->ver, connectRsp.sVersion, tListLen(pTscObj->ver)); // update the appInstInfo - pTscObj->pAppInfo->clusterId = pConnect->clusterId; + pTscObj->pAppInfo->clusterId = connectRsp.clusterId; atomic_add_fetch_64(&pTscObj->pAppInfo->numOfConns, 1); pTscObj->connType = HEARTBEAT_TYPE_QUERY; - hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, pConnect->connId, pConnect->clusterId, HEARTBEAT_TYPE_QUERY); + hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, connectRsp.connId, connectRsp.clusterId, HEARTBEAT_TYPE_QUERY); // pRequest->body.resInfo.pRspMsg = pMsg->pData; - tscDebug("0x%" PRIx64 " clusterId:%" PRId64 ", totalConn:%" PRId64, pRequest->requestId, pConnect->clusterId, + tscDebug("0x%" PRIx64 " clusterId:%" PRId64 ", totalConn:%" PRId64, pRequest->requestId, connectRsp.clusterId, pTscObj->pAppInfo->numOfConns); free(pMsg->pData); @@ -97,14 +91,15 @@ SMsgSendInfo* buildMsgInfoImpl(SRequestObj *pRequest) { if (pRequest->type == TDMT_MND_SHOW_RETRIEVE || pRequest->type == TDMT_VND_SHOW_TABLES_FETCH) { if (pRequest->type == TDMT_MND_SHOW_RETRIEVE) { - SRetrieveTableReq* pRetrieveMsg = calloc(1, sizeof(SRetrieveTableReq)); - if (pRetrieveMsg == NULL) { - return NULL; - } + SRetrieveTableReq retrieveReq = {0}; + retrieveReq.showId = pRequest->body.showInfo.execId; - pRetrieveMsg->showId = htobe64(pRequest->body.showInfo.execId); - pMsgSendInfo->msgInfo.pData = pRetrieveMsg; - pMsgSendInfo->msgInfo.len = sizeof(SRetrieveTableReq); + int32_t contLen = tSerializeSRetrieveTableReq(NULL, 0, &retrieveReq); + void* pReq = malloc(contLen); + tSerializeSRetrieveTableReq(pReq, contLen, &retrieveReq); + + pMsgSendInfo->msgInfo.pData = pReq; + pMsgSendInfo->msgInfo.len = contLen; } else { SVShowTablesFetchReq* pFetchMsg = calloc(1, sizeof(SVShowTablesFetchReq)); if (pFetchMsg == NULL) { @@ -134,39 +129,29 @@ int32_t processShowRsp(void* param, const SDataBuf* pMsg, int32_t code) { return code; } - SShowRsp* pShow = (SShowRsp *)pMsg->pData; - pShow->showId = htobe64(pShow->showId); + SShowRsp showRsp = {0}; + tDeserializeSShowRsp(pMsg->pData, pMsg->len, &showRsp); + STableMetaRsp *pMetaMsg = &showRsp.tableMeta; - STableMetaRsp *pMetaMsg = &(pShow->tableMeta); - pMetaMsg->numOfColumns = htonl(pMetaMsg->numOfColumns); - - SSchema* pSchema = pMetaMsg->pSchema; - pMetaMsg->tuid = htobe64(pMetaMsg->tuid); - for (int i = 0; i < pMetaMsg->numOfColumns; ++i) { - pSchema->bytes = htonl(pSchema->bytes); - pSchema->colId = htonl(pSchema->colId); - pSchema++; - } - - pSchema = pMetaMsg->pSchema; tfree(pRequest->body.resInfo.pRspMsg); - pRequest->body.resInfo.pRspMsg = pMsg->pData; SReqResultInfo* pResInfo = &pRequest->body.resInfo; if (pResInfo->fields == NULL) { TAOS_FIELD* pFields = calloc(pMetaMsg->numOfColumns, sizeof(TAOS_FIELD)); for (int32_t i = 0; i < pMetaMsg->numOfColumns; ++i) { - tstrncpy(pFields[i].name, pSchema[i].name, tListLen(pFields[i].name)); - pFields[i].type = pSchema[i].type; - pFields[i].bytes = pSchema[i].bytes; + SSchema* pSchema = &pMetaMsg->pSchemas[i]; + tstrncpy(pFields[i].name, pSchema->name, tListLen(pFields[i].name)); + pFields[i].type = pSchema->type; + pFields[i].bytes = pSchema->bytes; } pResInfo->fields = pFields; } pResInfo->numOfCols = pMetaMsg->numOfColumns; - pRequest->body.showInfo.execId = pShow->showId; + pRequest->body.showInfo.execId = showRsp.showId; + tFreeSShowRsp(&showRsp); // todo if (pRequest->type == TDMT_VND_SHOW_TABLES) { @@ -264,9 +249,13 @@ int32_t processUseDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { return code; } - SUseDbRsp* pUseDbRsp = (SUseDbRsp*) pMsg->pData; + SUseDbRsp usedbRsp = {0}; + tDeserializeSUseDbRsp(pMsg->pData, pMsg->len, &usedbRsp); + SName name = {0}; - tNameFromString(&name, pUseDbRsp->db, T_NAME_ACCT|T_NAME_DB); + tNameFromString(&name, usedbRsp.db, T_NAME_ACCT|T_NAME_DB); + + tFreeSUsedbRsp(&usedbRsp); char db[TSDB_DB_NAME_LEN] = {0}; tNameGetDbName(&name, db); @@ -300,14 +289,12 @@ int32_t processDropDbRsp(void* param, const SDataBuf* pMsg, int32_t code) { return code; } - SDropDbRsp *rsp = (SDropDbRsp *)pMsg->pData; - - struct SCatalog *pCatalog = NULL; - rsp->uid = be64toh(rsp->uid); + SDropDbRsp dropdbRsp = {0}; + tDeserializeSDropDbRsp(pMsg->pData, pMsg->len, &dropdbRsp); + struct SCatalog* pCatalog = NULL; catalogGetHandle(pRequest->pTscObj->pAppInfo->clusterId, &pCatalog); - - catalogRemoveDB(pCatalog, rsp->db, rsp->uid); + catalogRemoveDB(pCatalog, dropdbRsp.db, dropdbRsp.uid); tsem_post(&pRequest->body.rspSem); return code; diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 8b1faaef4e..419d6294a7 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -25,7 +25,7 @@ #include "tglobal.h" #include "tmsgtype.h" #include "tnote.h" -#include "tpagedfile.h" +#include "tpagedbuf.h" #include "tref.h" struct tmq_list_t { @@ -33,6 +33,7 @@ struct tmq_list_t { int32_t tot; char* elems[]; }; + struct tmq_topic_vgroup_t { char* topic; int32_t vgId; @@ -48,14 +49,17 @@ struct tmq_topic_vgroup_list_t { struct tmq_conf_t { char clientId[256]; char groupId[256]; + bool auto_commit; /*char* ip;*/ /*uint16_t port;*/ tmq_commit_cb* commit_cb; }; struct tmq_t { + // conf char groupId[256]; char clientId[256]; + bool autoCommit; SRWLatch lock; int64_t consumerId; int32_t epoch; @@ -94,25 +98,25 @@ typedef struct SMqClientTopic { SArray* vgs; // SArray } SMqClientTopic; -typedef struct SMqSubscribeCbParam { +typedef struct { tmq_t* tmq; tsem_t rspSem; tmq_resp_err_t rspErr; } SMqSubscribeCbParam; -typedef struct SMqAskEpCbParam { +typedef struct { tmq_t* tmq; int32_t wait; } SMqAskEpCbParam; -typedef struct SMqConsumeCbParam { +typedef struct { tmq_t* tmq; SMqClientVg* pVg; tmq_message_t** retMsg; tsem_t rspSem; } SMqConsumeCbParam; -typedef struct SMqCommitCbParam { +typedef struct { tmq_t* tmq; SMqClientVg* pVg; int32_t async; @@ -121,6 +125,7 @@ typedef struct SMqCommitCbParam { tmq_conf_t* tmq_conf_new() { tmq_conf_t* conf = calloc(1, sizeof(tmq_conf_t)); + conf->auto_commit = false; return conf; } @@ -131,11 +136,24 @@ void tmq_conf_destroy(tmq_conf_t* conf) { tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value) { if (strcmp(key, "group.id") == 0) { strcpy(conf->groupId, value); + return TMQ_CONF_OK; } if (strcmp(key, "client.id") == 0) { strcpy(conf->clientId, value); + return TMQ_CONF_OK; } - return TMQ_CONF_OK; + if (strcmp(key, "enable.auto.commit") == 0) { + if (strcmp(value, "true") == 0) { + conf->auto_commit = true; + return TMQ_CONF_OK; + } else if (strcmp(value, "false") == 0) { + conf->auto_commit = false; + return TMQ_CONF_OK; + } else { + return TMQ_CONF_INVALID; + } + } + return TMQ_CONF_UNKNOWN; } tmq_list_t* tmq_list_new() { @@ -155,6 +173,12 @@ int32_t tmq_list_append(tmq_list_t* ptr, const char* src) { return 0; } +tmq_resp_err_t tmq_reset_offset(tmq_t* tmq, const tmq_topic_vgroup_list_t* offsets) { + // build msg + // send to mnode + return TMQ_RESP_ERR__SUCCESS; +} + int32_t tmqSubscribeCb(void* param, const SDataBuf* pMsg, int32_t code) { SMqSubscribeCbParam* pParam = (SMqSubscribeCbParam*)param; pParam->rspErr = code; @@ -182,11 +206,14 @@ tmq_t* tmq_consumer_new(void* conn, tmq_conf_t* conf, char* errstr, int32_t errs pTmq->pollCnt = 0; pTmq->epoch = 0; taosInitRWLatch(&pTmq->lock); + // set conf strcpy(pTmq->clientId, conf->clientId); strcpy(pTmq->groupId, conf->groupId); + pTmq->autoCommit = conf->auto_commit; pTmq->commit_cb = conf->commit_cb; + tsem_init(&pTmq->rspSem, 0, 0); - pTmq->consumerId = generateRequestId() & ((uint64_t)-1 >> 1); + pTmq->consumerId = generateRequestId() & (((uint64_t)-1) >> 1); pTmq->clientTopics = taosArrayInit(0, sizeof(SMqClientTopic)); return pTmq; } @@ -357,25 +384,21 @@ TAOS_RES* tmq_create_topic(TAOS* taos, const char* topicName, const char* sql, i tNameFromString(&name, dbName, T_NAME_ACCT | T_NAME_DB); tNameFromString(&name, topicName, T_NAME_TABLE); - char topicFname[TSDB_TOPIC_FNAME_LEN] = {0}; - tNameExtractFullName(&name, topicFname); - - SCMCreateTopicReq req = { - .name = (char*)topicFname, + SMCreateTopicReq req = { .igExists = 1, .physicalPlan = (char*)pStr, .sql = (char*)sql, .logicalPlan = (char*)"no logic plan", }; + tNameExtractFullName(&name, req.name); - int tlen = tSerializeSCMCreateTopicReq(NULL, &req); + int tlen = tSerializeMCreateTopicReq(NULL, 0, &req); void* buf = malloc(tlen); if (buf == NULL) { goto _return; } - void* abuf = buf; - tSerializeSCMCreateTopicReq(&abuf, &req); + tSerializeMCreateTopicReq(buf, tlen, &req); /*printf("formatted: %s\n", dagStr);*/ pRequest->body.requestMsg = (SDataBuf){.pData = buf, .len = tlen}; @@ -452,6 +475,12 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) { return buf; } +int32_t tmqGetSkipLogNum(tmq_message_t* tmq_message) { + if (tmq_message == NULL) return 0; + SMqConsumeRsp* pRsp = (SMqConsumeRsp*)tmq_message; + return pRsp->skipLogNum; +} + void tmqShowMsg(tmq_message_t* tmq_message) { if (tmq_message == NULL) return; @@ -557,8 +586,15 @@ int32_t tmqAskEpCb(void* param, const SDataBuf* pMsg, int32_t code) { topic.vgs = taosArrayInit(vgSz, sizeof(SMqClientVg)); for (int32_t j = 0; j < vgSz; j++) { SMqSubVgEp* pVgEp = taosArrayGet(pTopicEp->vgs, j); + // clang-format off SMqClientVg clientVg = { - .pollCnt = 0, .committedOffset = -1, .currentOffset = -1, .vgId = pVgEp->vgId, .epSet = pVgEp->epSet}; + .pollCnt = 0, + .committedOffset = -1, + .currentOffset = -1, + .vgId = pVgEp->vgId, + .epSet = pVgEp->epSet + }; + // clang-format on taosArrayPush(topic.vgs, &clientVg); set = true; } @@ -649,17 +685,19 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { int64_t status = atomic_load_64(&tmq->status); tmqAskEp(tmq, status == 0); - if (blocking_time < 0) blocking_time = 1; + if (blocking_time <= 0) blocking_time = 1; if (blocking_time > 1000) blocking_time = 1000; /*blocking_time = 1;*/ if (taosArrayGetSize(tmq->clientTopics) == 0) { tscDebug("consumer:%ld poll but not assigned", tmq->consumerId); + printf("over1\n"); usleep(blocking_time * 1000); return NULL; } SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, tmq->nextTopicIdx); if (taosArrayGetSize(pTopic->vgs) == 0) { + printf("over2\n"); usleep(blocking_time * 1000); return NULL; } @@ -670,7 +708,8 @@ tmq_message_t* tmq_consumer_poll(tmq_t* tmq, int64_t blocking_time) { pTopic->nextVgIdx = (pTopic->nextVgIdx + 1) % taosArrayGetSize(pTopic->vgs); SMqClientVg* pVg = taosArrayGet(pTopic->vgs, pTopic->nextVgIdx); /*printf("consume vg %d, offset %ld\n", pVg->vgId, pVg->currentOffset);*/ - SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blocking_time, TMQ_REQ_TYPE_CONSUME_ONLY, pTopic, pVg); + int32_t reqType = tmq->autoCommit ? TMQ_REQ_TYPE_CONSUME_AND_COMMIT : TMQ_REQ_TYPE_CONSUME_ONLY; + SMqConsumeReq* pReq = tmqBuildConsumeReqImpl(tmq, blocking_time, reqType, pTopic, pVg); if (pReq == NULL) { ASSERT(false); usleep(blocking_time * 1000); diff --git a/source/common/src/tep.c b/source/common/src/tep.c index 45587a8856..970b6d954f 100644 --- a/source/common/src/tep.c +++ b/source/common/src/tep.c @@ -1,4 +1,5 @@ #include "tep.h" +#include #include "common.h" #include "tglobal.h" #include "tlockfree.h" @@ -60,68 +61,168 @@ SEpSet getEpSet_s(SCorEpSet *pEpSet) { return ep; } -bool colDataIsNull(const SColumnInfoData* pColumnInfoData, uint32_t totalRows, uint32_t row, SColumnDataAgg* pColAgg) { - if (pColAgg != NULL) { - if (pColAgg->numOfNull == totalRows) { - ASSERT(pColumnInfoData->nullbitmap == NULL); - return true; - } else if (pColAgg->numOfNull == 0) { - ASSERT(pColumnInfoData->nullbitmap == NULL); - return false; - } - } +#define BitmapLen(_n) (((_n) + ((1<> NBIT) - if (pColumnInfoData->nullbitmap == NULL) { - return false; - } - - uint8_t v = (pColumnInfoData->nullbitmap[row>>3] & (1<<(8 - (row&0x07)))); - return (v == 1); -} - -bool colDataIsNull_f(const char* bitmap, uint32_t row) { - return (bitmap[row>>3] & (1<<(8 - (row&0x07)))); -} - -void colDataSetNull_f(char* bitmap, uint32_t row) { // TODO - return; -} - -void* colDataGet(const SColumnInfoData* pColumnInfoData, uint32_t row) { +int32_t colDataGetSize(const SColumnInfoData* pColumnInfoData, int32_t numOfRows) { + ASSERT(pColumnInfoData != NULL); if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { - uint32_t offset = ((uint32_t*)pColumnInfoData->pData)[row]; - return (char*)(pColumnInfoData->pData) + offset; // the first part is the pointer to the true binary data + return pColumnInfoData->varmeta.length; } else { - return (char*)(pColumnInfoData->pData) + (row * pColumnInfoData->info.bytes); + return pColumnInfoData->info.bytes * numOfRows; } } +void colDataTrim(SColumnInfoData* pColumnInfoData) { + // TODO +} + int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull) { ASSERT(pColumnInfoData != NULL); if (isNull) { - // TODO set null value in the nullbitmap + // There is a placehold for each NULL value of binary or nchar type. + if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { + pColumnInfoData->varmeta.offset[currentRow] = -1; // it is a null value of VAR type. + } else { + colDataSetNull_f(pColumnInfoData->nullbitmap, currentRow); + } + + pColumnInfoData->hasNull = true; return 0; } int32_t type = pColumnInfoData->info.type; if (IS_VAR_DATA_TYPE(type)) { - // TODO continue append var_type + SVarColAttr* pAttr = &pColumnInfoData->varmeta; + if (pAttr->allocLen < pAttr->length + varDataTLen(pData)) { + uint32_t newSize = pAttr->allocLen; + if (newSize == 0) { + newSize = 8; + } + + while(newSize < pAttr->length + varDataTLen(pData)) { + newSize = newSize * 1.5; + } + + char* buf = realloc(pColumnInfoData->pData, newSize); + if (buf == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pColumnInfoData->pData = buf; + pAttr->allocLen = newSize; + } + + uint32_t len = pColumnInfoData->varmeta.length; + pColumnInfoData->varmeta.offset[currentRow] = len; + + memcpy(pColumnInfoData->pData + len, pData, varDataTLen(pData)); + pColumnInfoData->varmeta.length += varDataTLen(pData); } else { char* p = pColumnInfoData->pData + pColumnInfoData->info.bytes * currentRow; switch(type) { case TSDB_DATA_TYPE_TINYINT: case TSDB_DATA_TYPE_UTINYINT: {*(int8_t*) p = *(int8_t*) pData;break;} + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: {*(int16_t*) p = *(int16_t*) pData;break;} + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: {*(int32_t*) p = *(int32_t*) pData;break;} + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: {*(int64_t*) p = *(int64_t*) pData;break;} default: assert(0); } - } return 0; } -size_t colDataGetCols(const SSDataBlock* pBlock) { +static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, const SColumnInfoData* pSource, int32_t numOfRow2) { + uint32_t total = numOfRow1 + numOfRow2; + + if (BitmapLen(numOfRow1) < BitmapLen(total)) { + char* tmp = realloc(pColumnInfoData->nullbitmap, BitmapLen(total)); + uint32_t extend = BitmapLen(total) - BitmapLen(numOfRow1); + memset(tmp + BitmapLen(numOfRow1), 0, extend); + pColumnInfoData->nullbitmap = tmp; + } + + uint32_t remindBits = BitPos(numOfRow1); + uint32_t shiftBits = 8 - remindBits; + + if (remindBits == 0) { // no need to shift bits of bitmap + memcpy(pColumnInfoData->nullbitmap + BitmapLen(numOfRow1), pSource->nullbitmap, BitmapLen(numOfRow2)); + } else { + int32_t len = BitmapLen(numOfRow2); + int32_t i = 0; + + uint8_t* p = (uint8_t*)pSource->nullbitmap; + pColumnInfoData->nullbitmap[BitmapLen(numOfRow1) - 1] |= (p[0] >> remindBits); + + uint8_t* start = (uint8_t*)&pColumnInfoData->nullbitmap[BitmapLen(numOfRow1)]; + while (i < len) { + start[i] |= (p[i] << shiftBits); + i += 1; + + if (i > 1) { + start[i - 1] |= (p[i] >> remindBits); + } + } + } +} + +int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, const SColumnInfoData* pSource, uint32_t numOfRow2) { + ASSERT(pColumnInfoData != NULL && pSource != NULL && pColumnInfoData->info.type == pSource->info.type); + + if (numOfRow2 == 0) { + return numOfRow1; + } + + if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { + // Handle the bitmap + char* p = realloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * (numOfRow1 + numOfRow2)); + if (p == NULL) { + // TODO + } + + pColumnInfoData->varmeta.offset = (int32_t*) p; + for(int32_t i = 0; i < numOfRow2; ++i) { + pColumnInfoData->varmeta.offset[i + numOfRow1] = pSource->varmeta.offset[i] + pColumnInfoData->varmeta.length; + } + + // copy data + uint32_t len = pSource->varmeta.length; + uint32_t oldLen = pColumnInfoData->varmeta.length; + if (pColumnInfoData->varmeta.allocLen < len + oldLen) { + char* tmp = realloc(pColumnInfoData->pData, len + oldLen); + if (tmp == NULL) { + return TSDB_CODE_VND_OUT_OF_MEMORY; + } + + pColumnInfoData->pData = tmp; + pColumnInfoData->varmeta.allocLen = len + oldLen; + } + + memcpy(pColumnInfoData->pData + oldLen, pSource->pData, len); + pColumnInfoData->varmeta.length = len + oldLen; + } else { + doBitmapMerge(pColumnInfoData, numOfRow1, pSource, numOfRow2); + + int32_t newSize = (numOfRow1 + numOfRow2) * pColumnInfoData->info.bytes; + char* tmp = realloc(pColumnInfoData->pData, newSize); + if (tmp == NULL) { + return TSDB_CODE_VND_OUT_OF_MEMORY; + } + + pColumnInfoData->pData = tmp; + int32_t offset = pColumnInfoData->info.bytes * numOfRow1; + memcpy(pColumnInfoData->pData + offset, pSource->pData, pSource->info.bytes * numOfRow2); + } + + return numOfRow1 + numOfRow2; +} + +size_t colDataGetNumOfCols(const SSDataBlock* pBlock) { ASSERT(pBlock); size_t constantCols = (pBlock->pConstantList != NULL)? taosArrayGetSize(pBlock->pConstantList):0; @@ -129,11 +230,11 @@ size_t colDataGetCols(const SSDataBlock* pBlock) { return pBlock->info.numOfCols; } -size_t colDataGetRows(const SSDataBlock* pBlock) { +size_t colDataGetNumOfRows(const SSDataBlock* pBlock) { return pBlock->info.rows; } -int32_t colDataUpdateTsWindow(SSDataBlock* pDataBlock) { +int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock) { if (pDataBlock == NULL || pDataBlock->info.rows <= 0) { return 0; } @@ -153,6 +254,848 @@ int32_t colDataUpdateTsWindow(SSDataBlock* pDataBlock) { return 0; } +int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc) { + assert(pSrc != NULL && pDest != NULL && pDest->info.numOfCols == pSrc->info.numOfCols); + + int32_t numOfCols = pSrc->info.numOfCols; + for(int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pCol2 = taosArrayGet(pDest->pDataBlock, i); + SColumnInfoData* pCol1 = taosArrayGet(pSrc->pDataBlock, i); + + uint32_t oldLen = colDataGetSize(pCol2, pDest->info.rows); + uint32_t newLen = colDataGetSize(pCol1, pSrc->info.rows); + + int32_t newSize = oldLen + newLen; + char* tmp = realloc(pCol2->pData, newSize); + if (tmp != NULL) { + pCol2->pData = tmp; + colDataMergeCol(pCol2, pDest->info.rows, pCol1, pSrc->info.rows); + } else { + return TSDB_CODE_VND_OUT_OF_MEMORY; + } + } + + pDest->info.rows += pSrc->info.rows; + return TSDB_CODE_SUCCESS; +} + +size_t blockDataGetSize(const SSDataBlock* pBlock) { + assert(pBlock != NULL); + + size_t total = 0; + int32_t numOfCols = pBlock->info.numOfCols; + + for(int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); + total += colDataGetSize(pColInfoData, pBlock->info.rows); + + if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { + total += sizeof(int32_t) * pBlock->info.rows; + } else { + total += BitmapLen(pBlock->info.rows); + } + } + + return total; +} + +// the number of tuples can be fit in one page. +// Actual data rows pluses the corresponding meta data must fit in one memory buffer of the given page size. +int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startIndex, int32_t* stopIndex, int32_t pageSize) { + ASSERT(pBlock != NULL && stopIndex != NULL); + + int32_t numOfCols = pBlock->info.numOfCols; + int32_t numOfRows = pBlock->info.rows; + + int32_t bitmapChar = 1; + + size_t headerSize = sizeof(int32_t); + size_t colHeaderSize = sizeof(int32_t) * numOfCols; + size_t payloadSize = pageSize - (headerSize + colHeaderSize); + + // TODO speedup by checking if the whole page can fit in firstly. + if (!hasVarCol) { + size_t rowSize = blockDataGetRowSize(pBlock); + int32_t capacity = (payloadSize / (rowSize * 8 + bitmapChar * numOfCols)) * 8; + + *stopIndex = startIndex + capacity; + if (*stopIndex >= numOfRows) { + *stopIndex = numOfRows - 1; + } + + return TSDB_CODE_SUCCESS; + } else { + // iterate the rows that can be fit in this buffer page + int32_t size = (headerSize + colHeaderSize); + + for(int32_t j = startIndex; j < numOfRows; ++j) { + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pColInfoData = TARRAY_GET_ELEM(pBlock->pDataBlock, i); + if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { + bool isNull = colDataIsNull(pColInfoData, numOfRows, j, NULL); + if (isNull) { + // do nothing + } else { + char* p = colDataGet(pColInfoData, j); + size += varDataTLen(p); + } + + size += sizeof(pColInfoData->varmeta.offset[0]); + } else { + size += pColInfoData->info.bytes; + + if (((j - startIndex) & 0x07) == 0) { + size += 1; // the space for null bitmap + } + } + } + + if (size > pageSize) { + *stopIndex = j - 1; + ASSERT(*stopIndex > startIndex); + + return TSDB_CODE_SUCCESS; + } + } + + // all fit in + *stopIndex = numOfRows - 1; + return TSDB_CODE_SUCCESS; + } +} + +SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int32_t rowCount) { + if (pBlock == NULL || startIndex < 0 || rowCount > pBlock->info.rows || rowCount + startIndex > pBlock->info.rows) { + return NULL; + } + + SSDataBlock* pDst = calloc(1, sizeof(SSDataBlock)); + if (pDst == NULL) { + return NULL; + } + + pDst->info = pBlock->info; + + pDst->info.rows = 0; + pDst->pDataBlock = taosArrayInit(pBlock->info.numOfCols, sizeof(SColumnInfoData)); + + for(int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + SColumnInfoData colInfo = {0}; + SColumnInfoData* pSrcCol = taosArrayGet(pBlock->pDataBlock, i); + colInfo.info = pSrcCol->info; + + if (IS_VAR_DATA_TYPE(pSrcCol->info.type)) { + SVarColAttr* pAttr = &colInfo.varmeta; + pAttr->offset = calloc(rowCount, sizeof(int32_t)); + } else { + colInfo.nullbitmap = calloc(1, BitmapLen(rowCount)); + colInfo.pData = calloc(rowCount, colInfo.info.bytes); + } + + taosArrayPush(pDst->pDataBlock, &colInfo); + } + + for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i); + SColumnInfoData* pDstCol = taosArrayGet(pDst->pDataBlock, i); + + for (int32_t j = startIndex; j < (startIndex + rowCount); ++j) { + bool isNull = colDataIsNull(pColData, pBlock->info.rows, j, pBlock->pBlockAgg); + char* p = colDataGet(pColData, j); + + colDataAppend(pDstCol, j - startIndex, p, isNull); + } + } + + pDst->info.rows = rowCount; + return pDst; +} +/** + * + * +------------------+---------------+--------------------+ + * |the number of rows| column length | column #1 | + * | (4 bytes) | (4 bytes) |--------------------+ + * | | | null bitmap| values| + * +------------------+---------------+--------------------+ + * @param buf + * @param pBlock + * @return + */ +int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock) { + ASSERT(pBlock != NULL); + // write the number of rows + *(uint32_t*) buf = pBlock->info.rows; + + int32_t numOfCols = pBlock->info.numOfCols; + int32_t numOfRows = pBlock->info.rows; + + char* pStart = buf + sizeof(uint32_t); + + for(int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i); + if (IS_VAR_DATA_TYPE(pCol->info.type)) { + memcpy(pStart, pCol->varmeta.offset, numOfRows * sizeof(int32_t)); + pStart += numOfRows * sizeof(int32_t); + } else { + memcpy(pStart, pCol->nullbitmap, BitmapLen(numOfRows)); + pStart += BitmapLen(pBlock->info.rows); + } + + uint32_t dataSize = colDataGetSize(pCol, numOfRows); + + *(int32_t*) pStart = dataSize; + pStart += sizeof(int32_t); + + memcpy(pStart, pCol->pData, dataSize); + pStart += dataSize; + } + + return 0; +} + +int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) { + pBlock->info.rows = *(int32_t*) buf; + + int32_t numOfCols = pBlock->info.numOfCols; + const char* pStart = buf + sizeof(uint32_t); + + for(int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i); + + size_t metaSize = pBlock->info.rows * sizeof(int32_t); + if (IS_VAR_DATA_TYPE(pCol->info.type)) { + memcpy(pCol->varmeta.offset, pStart, metaSize); + pStart += metaSize; + } else { + memcpy(pCol->nullbitmap, pStart, BitmapLen(pBlock->info.rows)); + pStart += BitmapLen(pBlock->info.rows); + } + + int32_t colLength = *(int32_t*) pStart; + pStart += sizeof(int32_t); + + if (IS_VAR_DATA_TYPE(pCol->info.type)) { + if (pCol->varmeta.allocLen < colLength) { + char* tmp = realloc(pCol->pData, colLength); + if (tmp == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pCol->pData = tmp; + pCol->varmeta.allocLen = colLength; + } + + pCol->varmeta.length = colLength; + ASSERT(pCol->varmeta.length <= pCol->varmeta.allocLen); + } + + memcpy(pCol->pData, pStart, colLength); + pStart += colLength; + } + + return TSDB_CODE_SUCCESS; +} + +size_t blockDataGetRowSize(const SSDataBlock* pBlock) { + ASSERT(pBlock != NULL); + size_t rowSize = 0; + + size_t numOfCols = pBlock->info.numOfCols; + for(int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); + rowSize += pColInfo->info.bytes; + } + + return rowSize; +} + +/** + * @refitem blockDataToBuf for the meta size + * + * @param pBlock + * @return + */ +size_t blockDataGetSerialMetaSize(const SSDataBlock* pBlock) { + return sizeof(int32_t) + pBlock->info.numOfCols * sizeof(int32_t); +} + +double blockDataGetSerialRowSize(const SSDataBlock* pBlock) { + ASSERT(pBlock != NULL); + double rowSize = 0; + + size_t numOfCols = pBlock->info.numOfCols; + for(int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); + rowSize += pColInfo->info.bytes; + + if (IS_VAR_DATA_TYPE(pColInfo->info.type)) { + rowSize += sizeof(int32_t); + } else { + rowSize += 1/8.0; + } + } + + return rowSize; +} + +typedef struct SSDataBlockSortHelper { + SArray *orderInfo; // SArray + SSDataBlock *pDataBlock; + bool nullFirst; +} SSDataBlockSortHelper; + +int32_t dataBlockCompar(const void* p1, const void* p2, const void* param) { + const SSDataBlockSortHelper* pHelper = (const SSDataBlockSortHelper*) param; + + SSDataBlock* pDataBlock = pHelper->pDataBlock; + + int32_t left = *(int32_t*) p1; + int32_t right = *(int32_t*) p2; + + SArray* pInfo = pHelper->orderInfo; + + for(int32_t i = 0; i < pInfo->size; ++i) { + SBlockOrderInfo* pOrder = TARRAY_GET_ELEM(pInfo, i); + SColumnInfoData* pColInfoData = pOrder->pColData;//TARRAY_GET_ELEM(pDataBlock->pDataBlock, pOrder->colIndex); + + if (pColInfoData->hasNull) { + bool leftNull = colDataIsNull(pColInfoData, pDataBlock->info.rows, left, pDataBlock->pBlockAgg); + bool rightNull = colDataIsNull(pColInfoData, pDataBlock->info.rows, right, pDataBlock->pBlockAgg); + if (leftNull && rightNull) { + continue; // continue to next slot + } + + if (rightNull) { + return pHelper->nullFirst? 1:-1; + } + + if (leftNull) { + return pHelper->nullFirst? -1:1; + } + } + + void* left1 = colDataGet(pColInfoData, left); + void* right1 = colDataGet(pColInfoData, right); + + switch(pColInfoData->info.type) { + case TSDB_DATA_TYPE_INT: { + int32_t leftx = *(int32_t*) left1; + int32_t rightx = *(int32_t*) right1; + + if (leftx == rightx) { + break; + } else { + if (pOrder->order == TSDB_ORDER_ASC) { + return (leftx < rightx)? -1:1; + } else { + return (leftx < rightx)? 1:-1; + } + } + } + default: + assert(0); + } + } + + return 0; +} + +static int32_t doAssignOneTuple(SColumnInfoData* pDstCols, int32_t numOfRows, const SSDataBlock* pSrcBlock, int32_t tupleIndex) { + int32_t code = 0; + int32_t numOfCols = pSrcBlock->info.numOfCols; + + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pDst = &pDstCols[i]; + SColumnInfoData* pSrc = taosArrayGet(pSrcBlock->pDataBlock, i); + + if (pSrc->hasNull && colDataIsNull(pSrc, pSrcBlock->info.rows, tupleIndex, pSrcBlock->pBlockAgg)) { + code = colDataAppend(pDst, numOfRows, NULL, true); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } else { + char* p = colDataGet(pSrc, tupleIndex); + code = colDataAppend(pDst, numOfRows, p, false); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t blockDataAssign(SColumnInfoData* pCols, const SSDataBlock* pDataBlock, int32_t* index) { +#if 0 + for (int32_t i = 0; i < pDataBlock->info.rows; ++i) { + int32_t code = doAssignOneTuple(pCols, i, pDataBlock, index[i]); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + } +#else + for(int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { + SColumnInfoData* pDst = &pCols[i]; + SColumnInfoData* pSrc = taosArrayGet(pDataBlock->pDataBlock, i); + + if (IS_VAR_DATA_TYPE(pSrc->info.type)) { + memcpy(pDst->pData, pSrc->pData, pSrc->varmeta.length); + pDst->varmeta.length = pSrc->varmeta.length; + + for(int32_t j = 0; j < pDataBlock->info.rows; ++j) { + pDst->varmeta.offset[j] = pSrc->varmeta.offset[index[j]]; + } + } else { + switch (pSrc->info.type) { + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_INT: { + for (int32_t j = 0; j < pDataBlock->info.rows; ++j) { + int32_t* p = (int32_t*)pDst->pData; + int32_t* srclist = (int32_t*)pSrc->pData; + + p[j] = srclist[index[j]]; + if (colDataIsNull_f(pSrc->nullbitmap, index[j])) { + colDataSetNull_f(pDst->nullbitmap, j); + } + } + break; + } + case TSDB_DATA_TYPE_UTINYINT: + case TSDB_DATA_TYPE_TINYINT: { + for (int32_t j = 0; j < pDataBlock->info.rows; ++j) { + int32_t* p = (int32_t*)pDst->pData; + int32_t* srclist = (int32_t*)pSrc->pData; + + p[j] = srclist[index[j]]; + if (colDataIsNull_f(pSrc->nullbitmap, index[j])) { + colDataSetNull_f(pDst->nullbitmap, j); + } + } + break; + } + case TSDB_DATA_TYPE_USMALLINT: + case TSDB_DATA_TYPE_SMALLINT: { + for (int32_t j = 0; j < pDataBlock->info.rows; ++j) { + int32_t* p = (int32_t*)pDst->pData; + int32_t* srclist = (int32_t*)pSrc->pData; + + p[j] = srclist[index[j]]; + if (colDataIsNull_f(pSrc->nullbitmap, index[j])) { + colDataSetNull_f(pDst->nullbitmap, j); + } + } + break; + } + case TSDB_DATA_TYPE_UBIGINT: + case TSDB_DATA_TYPE_BIGINT: { + for (int32_t j = 0; j < pDataBlock->info.rows; ++j) { + int32_t* p = (int32_t*)pDst->pData; + int32_t* srclist = (int32_t*)pSrc->pData; + + p[j] = srclist[index[j]]; + if (colDataIsNull_f(pSrc->nullbitmap, index[j])) { + colDataSetNull_f(pDst->nullbitmap, j); + } + } + break; + } + default: + assert(0); + } + } + } +#endif + return TSDB_CODE_SUCCESS; +} + +static SColumnInfoData* createHelpColInfoData(const SSDataBlock* pDataBlock) { + int32_t rows = pDataBlock->info.rows; + int32_t numOfCols = pDataBlock->info.numOfCols; + + SColumnInfoData* pCols = calloc(numOfCols, sizeof(SColumnInfoData)); + if (pCols == NULL) { + return NULL; + } + + for(int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, i); + pCols[i].info = pColInfoData->info; + + if (IS_VAR_DATA_TYPE(pCols[i].info.type)) { + pCols[i].varmeta.offset = calloc(rows, sizeof(int32_t)); + pCols[i].pData = calloc(1, pColInfoData->varmeta.length); + + pCols[i].varmeta.length = pColInfoData->varmeta.length; + pCols[i].varmeta.allocLen = pCols[i].varmeta.length; + } else { + pCols[i].nullbitmap = calloc(1, BitmapLen(rows)); + pCols[i].pData = calloc(rows, pCols[i].info.bytes); + } + } + + return pCols; +} + +static void copyBackToBlock(SSDataBlock* pDataBlock, SColumnInfoData* pCols) { + int32_t numOfCols = pDataBlock->info.numOfCols; + + for(int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, i); + pColInfoData->info = pCols[i].info; + + if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { + tfree(pColInfoData->varmeta.offset); + pColInfoData->varmeta = pCols[i].varmeta; + } else { + tfree(pColInfoData->nullbitmap); + pColInfoData->nullbitmap = pCols[i].nullbitmap; + } + + tfree(pColInfoData->pData); + pColInfoData->pData = pCols[i].pData; + } + + tfree(pCols); +} + +static int32_t* createTupleIndex(size_t rows) { + int32_t* index = calloc(rows, sizeof(int32_t)); + if (index == NULL) { + return NULL; + } + + for(int32_t i = 0; i < rows; ++i) { + index[i] = i; + } + + return index; +} + +static void destroyTupleIndex(int32_t* index) { + tfree(index); +} + +static __compar_fn_t getComparFn(int32_t type, int32_t order) { + switch(type) { + case TSDB_DATA_TYPE_TINYINT: return order == TSDB_ORDER_ASC? compareInt8Val:compareInt8ValDesc; + case TSDB_DATA_TYPE_SMALLINT: return order == TSDB_ORDER_ASC? compareInt16Val:compareInt16ValDesc; + case TSDB_DATA_TYPE_INT: return order == TSDB_ORDER_ASC? compareInt32Val:compareInt32ValDesc; + case TSDB_DATA_TYPE_BIGINT: return order == TSDB_ORDER_ASC? compareInt64Val:compareInt64ValDesc; + case TSDB_DATA_TYPE_FLOAT: return order == TSDB_ORDER_ASC? compareFloatVal:compareFloatValDesc; + case TSDB_DATA_TYPE_DOUBLE: return order == TSDB_ORDER_ASC? compareDoubleVal:compareDoubleValDesc; + case TSDB_DATA_TYPE_UTINYINT: return order == TSDB_ORDER_ASC? compareUint8Val:compareUint8ValDesc; + case TSDB_DATA_TYPE_USMALLINT:return order == TSDB_ORDER_ASC? compareUint16Val:compareUint16ValDesc; + case TSDB_DATA_TYPE_UINT: return order == TSDB_ORDER_ASC? compareUint32Val:compareUint32ValDesc; + case TSDB_DATA_TYPE_UBIGINT: return order == TSDB_ORDER_ASC? compareUint64Val:compareUint64ValDesc; + default: + return order == TSDB_ORDER_ASC? compareInt32Val:compareInt32ValDesc; + } +} + +int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst) { + ASSERT(pDataBlock != NULL && pOrderInfo != NULL); + if (pDataBlock->info.rows <= 1) { + return TSDB_CODE_SUCCESS; + } + + // Allocate the additional buffer. + uint32_t rows = pDataBlock->info.rows; + + bool sortColumnHasNull = false; + bool varTypeSort = false; + + for (int32_t i = 0; i < taosArrayGetSize(pOrderInfo); ++i) { + SBlockOrderInfo* pInfo = taosArrayGet(pOrderInfo, i); + + SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, pInfo->colIndex); + if (pColInfoData->hasNull) { + sortColumnHasNull = true; + } + + if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { + varTypeSort = true; + } + } + + if (taosArrayGetSize(pOrderInfo) == 1 && (!sortColumnHasNull)) { + if (pDataBlock->info.numOfCols == 1) { + if (!varTypeSort) { + SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, 0); + SBlockOrderInfo* pOrder = taosArrayGet(pOrderInfo, 0); + + int64_t p0 = taosGetTimestampUs(); + + __compar_fn_t fn = getComparFn(pColInfoData->info.type, pOrder->order); + qsort(pColInfoData->pData, pDataBlock->info.rows, pColInfoData->info.bytes, fn); + + int64_t p1 = taosGetTimestampUs(); + printf("sort:%ld, rows:%d\n", p1 - p0, pDataBlock->info.rows); + + return TSDB_CODE_SUCCESS; + } else { // var data type + + } + } else if (pDataBlock->info.numOfCols == 2) { + + } + } + + int32_t* index = createTupleIndex(rows); + if (index == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return terrno; + } + + int64_t p0 = taosGetTimestampUs(); + + SSDataBlockSortHelper helper = {.nullFirst = nullFirst, .pDataBlock = pDataBlock, .orderInfo = pOrderInfo}; + for(int32_t i = 0; i < taosArrayGetSize(helper.orderInfo); ++i) { + struct SBlockOrderInfo* pInfo = taosArrayGet(helper.orderInfo, i); + pInfo->pColData = taosArrayGet(pDataBlock->pDataBlock, pInfo->colIndex); + } + + taosqsort(index, rows, sizeof(int32_t), &helper, dataBlockCompar); + + int64_t p1 = taosGetTimestampUs(); + + SColumnInfoData* pCols = createHelpColInfoData(pDataBlock); + if (pCols == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return terrno; + } + + int64_t p2 = taosGetTimestampUs(); + + int32_t code = blockDataAssign(pCols, pDataBlock, index); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + return code; + } + + int64_t p3 = taosGetTimestampUs(); + + copyBackToBlock(pDataBlock, pCols); + int64_t p4 = taosGetTimestampUs(); + + printf("sort:%ld, create:%ld, assign:%ld, copyback:%ld, rows:%d\n", p1-p0, p2 - p1, p3 - p2, p4-p3, rows); + destroyTupleIndex(index); + + return TSDB_CODE_SUCCESS; +} + +typedef struct SHelper { + int32_t index; + union {char *pData; int64_t i64; double d64;}; +} SHelper; + +SHelper* createTupleIndex_rv(int32_t numOfRows, SArray* pOrderInfo, SSDataBlock* pBlock) { + int32_t sortValLengthPerRow = 0; + int32_t numOfCols = taosArrayGetSize(pOrderInfo); + + for(int32_t i = 0; i < numOfCols; ++i) { + SBlockOrderInfo* pInfo = taosArrayGet(pOrderInfo, i); + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, pInfo->colIndex); + pInfo->pColData = pColInfo; + sortValLengthPerRow += pColInfo->info.bytes; + } + + size_t len = sortValLengthPerRow * pBlock->info.rows; + + char* buf = calloc(1, len); + SHelper* phelper = calloc(numOfRows, sizeof(SHelper)); + for(int32_t i = 0; i < numOfRows; ++i) { + phelper[i].index = i; + phelper[i].pData = buf + sortValLengthPerRow * i; + } + + int32_t offset = 0; + for(int32_t i = 0; i < numOfCols; ++i) { + SBlockOrderInfo* pInfo = taosArrayGet(pOrderInfo, i); + for(int32_t j = 0; j < numOfRows; ++j) { + phelper[j].i64 = *(int32_t*) pInfo->pColData->pData + pInfo->pColData->info.bytes * j; +// memcpy(phelper[j].pData + offset, pInfo->pColData->pData + pInfo->pColData->info.bytes * j, pInfo->pColData->info.bytes); + } + + offset += pInfo->pColData->info.bytes; + } + + return phelper; +} + +int32_t dataBlockCompar_rv(const void* p1, const void* p2, const void* param) { + const SSDataBlockSortHelper* pHelper = (const SSDataBlockSortHelper*) param; + +// SSDataBlock* pDataBlock = pHelper->pDataBlock; + + SHelper* left = (SHelper*) p1; + SHelper* right = (SHelper*) p2; + + SArray* pInfo = pHelper->orderInfo; + + int32_t offset = 0; +// for(int32_t i = 0; i < pInfo->size; ++i) { +// SBlockOrderInfo* pOrder = TARRAY_GET_ELEM(pInfo, 0); +// SColumnInfoData* pColInfoData = pOrder->pColData;//TARRAY_GET_ELEM(pDataBlock->pDataBlock, pOrder->colIndex); + +// if (pColInfoData->hasNull) { +// bool leftNull = colDataIsNull(pColInfoData, pDataBlock->info.rows, left, pDataBlock->pBlockAgg); +// bool rightNull = colDataIsNull(pColInfoData, pDataBlock->info.rows, right, pDataBlock->pBlockAgg); +// if (leftNull && rightNull) { +// continue; // continue to next slot +// } +// +// if (rightNull) { +// return pHelper->nullFirst? 1:-1; +// } +// +// if (leftNull) { +// return pHelper->nullFirst? -1:1; +// } +// } + +// void* left1 = colDataGet(pColInfoData, left); +// void* right1 = colDataGet(pColInfoData, right); + +// switch(pColInfoData->info.type) { +// case TSDB_DATA_TYPE_INT: { + int32_t leftx = *(int32_t*)left->pData;//*(int32_t*)(left->pData + offset); + int32_t rightx = *(int32_t*)right->pData;//*(int32_t*)(right->pData + offset); + +// offset += pColInfoData->info.bytes; + if (leftx == rightx) { +// break; + return 0; + } else { +// if (pOrder->order == TSDB_ORDER_ASC) { + return (leftx < rightx)? -1:1; +// } else { +// return (leftx < rightx)? 1:-1; +// } + } +// } +// default: +// assert(0); +// } +// } + + return 0; +} + +int32_t varColSort(SColumnInfoData* pColumnInfoData, SBlockOrderInfo* pOrder) { + +} + +int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst) { +// Allocate the additional buffer. + int64_t p0 = taosGetTimestampUs(); + + SSDataBlockSortHelper helper = {.nullFirst = nullFirst, .pDataBlock = pDataBlock, .orderInfo = pOrderInfo}; + + uint32_t rows = pDataBlock->info.rows; + SHelper* index = createTupleIndex_rv(rows, helper.orderInfo, pDataBlock); + if (index == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return terrno; + } + + taosqsort(index, rows, sizeof(SHelper), &helper, dataBlockCompar_rv); + + int64_t p1 = taosGetTimestampUs(); + SColumnInfoData* pCols = createHelpColInfoData(pDataBlock); + if (pCols == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return terrno; + } + + int64_t p2 = taosGetTimestampUs(); + + // int32_t code = blockDataAssign(pCols, pDataBlock, index); + // if (code != TSDB_CODE_SUCCESS) { + // terrno = code; + // return code; + // } + + int64_t p3 = taosGetTimestampUs(); + + copyBackToBlock(pDataBlock, pCols); + int64_t p4 = taosGetTimestampUs(); + + printf("sort:%ld, create:%ld, assign:%ld, copyback:%ld, rows:%d\n", p1 - p0, p2 - p1, p3 - p2, p4 - p3, rows); + // destroyTupleIndex(index); +} + +void blockDataClearup(SSDataBlock* pDataBlock, bool hasVarCol) { + pDataBlock->info.rows = 0; + + if (hasVarCol) { + for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { + SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); + + if (IS_VAR_DATA_TYPE(p->info.type)) { + p->varmeta.length = 0; + } + } + } +} + +int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) { + for(int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { + SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); + if (IS_VAR_DATA_TYPE(p->info.type)) { + char* tmp = realloc(p->varmeta.offset, sizeof(int32_t) * numOfRows); + if (tmp == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + p->varmeta.offset = (int32_t*)tmp; + memset(p->varmeta.offset, 0, sizeof(int32_t) * numOfRows); + + p->varmeta.length = 0; + p->varmeta.allocLen = 0; + tfree(p->pData); + } else { + char* tmp = realloc(p->nullbitmap, BitmapLen(numOfRows)); + if (tmp == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + p->nullbitmap = tmp; + memset(p->nullbitmap, 0, BitmapLen(numOfRows)); + + tmp = realloc(p->pData, numOfRows * p->info.bytes); + if (tmp == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + p->pData = tmp; + } + } + + return TSDB_CODE_SUCCESS; +} + +void* blockDataDestroy(SSDataBlock* pBlock) { + if (pBlock == NULL) { + return NULL; + } + + int32_t numOfOutput = pBlock->info.numOfCols; + for(int32_t i = 0; i < numOfOutput; ++i) { + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); + if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { + tfree(pColInfoData->varmeta.offset); + } else { + tfree(pColInfoData->nullbitmap); + } + + tfree(pColInfoData->pData); + } + + taosArrayDestroy(pBlock->pDataBlock); + tfree(pBlock->pBlockAgg); + tfree(pBlock); + return NULL; +} \ No newline at end of file diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 20e1da9d07..3806227803 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -27,7 +27,7 @@ #undef TD_MSG_SEG_CODE_ #include "tmsgdef.h" -int32_t tInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter) { +int32_t tInitSubmitMsgIter(SSubmitReq *pMsg, SSubmitMsgIter *pIter) { if (pMsg == NULL) { terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP; return -1; @@ -36,7 +36,7 @@ int32_t tInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter) { pIter->totalLen = pMsg->length; pIter->len = 0; pIter->pMsg = pMsg; - if (pMsg->length <= sizeof(SSubmitMsg)) { + if (pMsg->length <= sizeof(SSubmitReq)) { terrno = TSDB_CODE_TDB_SUBMIT_MSG_MSSED_UP; return -1; } @@ -46,7 +46,7 @@ int32_t tInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter) { int32_t tGetSubmitMsgNext(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock) { if (pIter->len == 0) { - pIter->len += sizeof(SSubmitMsg); + pIter->len += sizeof(SSubmitReq); } else { SSubmitBlk *pSubmitBlk = (SSubmitBlk *)POINTER_SHIFT(pIter->pMsg, pIter->len); pIter->len += (sizeof(SSubmitBlk) + pSubmitBlk->dataLen + pSubmitBlk->schemaLen); @@ -85,118 +85,197 @@ STSRow *tGetSubmitBlkNext(SSubmitBlkIter *pIter) { } } -int32_t tSerializeSClientHbReq(void **buf, const SClientHbReq *pReq) { +int32_t tEncodeSEpSet(SCoder *pEncoder, const SEpSet *pEp) { + if (tEncodeI8(pEncoder, pEp->inUse) < 0) return -1; + if (tEncodeI8(pEncoder, pEp->numOfEps) < 0) return -1; + for (int32_t i = 0; i < TSDB_MAX_REPLICA; i++) { + if (tEncodeU16(pEncoder, pEp->eps[i].port) < 0) return -1; + if (tEncodeCStr(pEncoder, pEp->eps[i].fqdn) < 0) return -1; + } + return 0; +} + +int32_t tDecodeSEpSet(SCoder *pDecoder, SEpSet *pEp) { + if (tDecodeI8(pDecoder, &pEp->inUse) < 0) return -1; + if (tDecodeI8(pDecoder, &pEp->numOfEps) < 0) return -1; + for (int32_t i = 0; i < TSDB_MAX_REPLICA; i++) { + if (tDecodeU16(pDecoder, &pEp->eps[i].port) < 0) return -1; + if (tDecodeCStrTo(pDecoder, pEp->eps[i].fqdn) < 0) return -1; + } + return 0; +} + +int32_t taosEncodeSEpSet(void **buf, const SEpSet *pEp) { int32_t tlen = 0; - tlen += taosEncodeSClientHbKey(buf, &pReq->connKey); - - int32_t kvNum = taosHashGetSize(pReq->info); - tlen += taosEncodeFixedI32(buf, kvNum); - SKv *kv; - void *pIter = taosHashIterate(pReq->info, NULL); - while (pIter != NULL) { - kv = pIter; - tlen += taosEncodeSKv(buf, kv); - - pIter = taosHashIterate(pReq->info, pIter); + tlen += taosEncodeFixedI8(buf, pEp->inUse); + tlen += taosEncodeFixedI8(buf, pEp->numOfEps); + for (int32_t i = 0; i < TSDB_MAX_REPLICA; i++) { + tlen += taosEncodeFixedU16(buf, pEp->eps[i].port); + tlen += taosEncodeString(buf, pEp->eps[i].fqdn); } return tlen; } -void *tDeserializeSClientHbReq(void *buf, SClientHbReq *pReq) { - buf = taosDecodeSClientHbKey(buf, &pReq->connKey); +void *taosDecodeSEpSet(void *buf, SEpSet *pEp) { + buf = taosDecodeFixedI8(buf, &pEp->inUse); + buf = taosDecodeFixedI8(buf, &pEp->numOfEps); + for (int32_t i = 0; i < TSDB_MAX_REPLICA; i++) { + buf = taosDecodeFixedU16(buf, &pEp->eps[i].port); + buf = taosDecodeStringTo(buf, pEp->eps[i].fqdn); + } + return buf; +} - // TODO: error handling - int32_t kvNum; - buf = taosDecodeFixedI32(buf, &kvNum); +static int32_t tSerializeSClientHbReq(SCoder *pEncoder, const SClientHbReq *pReq) { + if (tEncodeSClientHbKey(pEncoder, &pReq->connKey) < 0) return -1; + + int32_t kvNum = taosHashGetSize(pReq->info); + if (tEncodeI32(pEncoder, kvNum) < 0) return -1; + void *pIter = taosHashIterate(pReq->info, NULL); + while (pIter != NULL) { + SKv *kv = pIter; + if (tEncodeSKv(pEncoder, kv) < 0) return -1; + pIter = taosHashIterate(pReq->info, pIter); + } + + return 0; +} + +static int32_t tDeserializeSClientHbReq(SCoder *pDecoder, SClientHbReq *pReq) { + if (tDecodeSClientHbKey(pDecoder, &pReq->connKey) < 0) return -1; + + int32_t kvNum = 0; + if (tDecodeI32(pDecoder, &kvNum) < 0) return -1; if (pReq->info == NULL) { pReq->info = taosHashInit(kvNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); } + if (pReq->info == NULL) return -1; for (int32_t i = 0; i < kvNum; i++) { - SKv kv; - buf = taosDecodeSKv(buf, &kv); + SKv kv = {0}; + if (tDecodeSKv(pDecoder, &kv) < 0) return -1; taosHashPut(pReq->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv)); } - return buf; + return 0; } -int32_t tSerializeSClientHbRsp(void **buf, const SClientHbRsp *pRsp) { - int32_t tlen = 0; +static int32_t tSerializeSClientHbRsp(SCoder *pEncoder, const SClientHbRsp *pRsp) { + if (tEncodeSClientHbKey(pEncoder, &pRsp->connKey) < 0) return -1; + if (tEncodeI32(pEncoder, pRsp->status) < 0) return -1; + int32_t kvNum = taosArrayGetSize(pRsp->info); - tlen += taosEncodeSClientHbKey(buf, &pRsp->connKey); - tlen += taosEncodeFixedI32(buf, pRsp->status); - tlen += taosEncodeFixedI32(buf, kvNum); + if (tEncodeI32(pEncoder, kvNum) < 0) return -1; for (int32_t i = 0; i < kvNum; i++) { - SKv *kv = (SKv *)taosArrayGet(pRsp->info, i); - tlen += taosEncodeSKv(buf, kv); + SKv *kv = taosArrayGet(pRsp->info, i); + if (tEncodeSKv(pEncoder, kv) < 0) return -1; } - return tlen; + + return 0; } -void *tDeserializeSClientHbRsp(void *buf, SClientHbRsp *pRsp) { +static int32_t tDeserializeSClientHbRsp(SCoder *pDecoder, SClientHbRsp *pRsp) { + if (tDecodeSClientHbKey(pDecoder, &pRsp->connKey) < 0) return -1; + if (tDecodeI32(pDecoder, &pRsp->status) < 0) return -1; + int32_t kvNum = 0; - buf = taosDecodeSClientHbKey(buf, &pRsp->connKey); - buf = taosDecodeFixedI32(buf, &pRsp->status); - buf = taosDecodeFixedI32(buf, &kvNum); + if (tDecodeI32(pDecoder, &kvNum) < 0) return -1; pRsp->info = taosArrayInit(kvNum, sizeof(SKv)); + if (pRsp->info == NULL) return -1; for (int32_t i = 0; i < kvNum; i++) { SKv kv = {0}; - buf = taosDecodeSKv(buf, &kv); + tDecodeSKv(pDecoder, &kv); taosArrayPush(pRsp->info, &kv); } - return buf; + return 0; } -int32_t tSerializeSClientHbBatchReq(void **buf, const SClientHbBatchReq *pBatchReq) { - int32_t tlen = 0; - tlen += taosEncodeFixedI64(buf, pBatchReq->reqId); +int32_t tSerializeSClientHbBatchReq(void *buf, int32_t bufLen, const SClientHbBatchReq *pBatchReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI64(&encoder, pBatchReq->reqId) < 0) return -1; + int32_t reqNum = taosArrayGetSize(pBatchReq->reqs); - tlen += taosEncodeFixedI32(buf, reqNum); + if (tEncodeI32(&encoder, reqNum) < 0) return -1; for (int32_t i = 0; i < reqNum; i++) { SClientHbReq *pReq = taosArrayGet(pBatchReq->reqs, i); - tlen += tSerializeSClientHbReq(buf, pReq); + if (tSerializeSClientHbReq(&encoder, pReq) < 0) return -1; } + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); return tlen; } -void *tDeserializeSClientHbBatchReq(void *buf, SClientHbBatchReq *pBatchReq) { - buf = taosDecodeFixedI64(buf, &pBatchReq->reqId); - if (pBatchReq->reqs == NULL) { - pBatchReq->reqs = taosArrayInit(0, sizeof(SClientHbReq)); - } +int32_t tDeserializeSClientHbBatchReq(void *buf, int32_t bufLen, SClientHbBatchReq *pBatchReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); - int32_t reqNum; - buf = taosDecodeFixedI32(buf, &reqNum); + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI64(&decoder, &pBatchReq->reqId) < 0) return -1; + + int32_t reqNum = 0; + if (tDecodeI32(&decoder, &reqNum) < 0) return -1; + if (pBatchReq->reqs == NULL) { + pBatchReq->reqs = taosArrayInit(reqNum, sizeof(SClientHbReq)); + } for (int32_t i = 0; i < reqNum; i++) { SClientHbReq req = {0}; - buf = tDeserializeSClientHbReq(buf, &req); + tDeserializeSClientHbReq(&decoder, &req); taosArrayPush(pBatchReq->reqs, &req); } - return buf; + + tEndDecode(&decoder); + tCoderClear(&decoder); + return 0; } -int32_t tSerializeSClientHbBatchRsp(void **buf, const SClientHbBatchRsp *pBatchRsp) { - int32_t tlen = 0; - int32_t sz = taosArrayGetSize(pBatchRsp->rsps); - tlen += taosEncodeFixedI32(buf, sz); - for (int32_t i = 0; i < sz; i++) { +int32_t tSerializeSClientHbBatchRsp(void *buf, int32_t bufLen, const SClientHbBatchRsp *pBatchRsp) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI64(&encoder, pBatchRsp->reqId) < 0) return -1; + if (tEncodeI64(&encoder, pBatchRsp->rspId) < 0) return -1; + + int32_t rspNum = taosArrayGetSize(pBatchRsp->rsps); + if (tEncodeI32(&encoder, rspNum) < 0) return -1; + for (int32_t i = 0; i < rspNum; i++) { SClientHbRsp *pRsp = taosArrayGet(pBatchRsp->rsps, i); - tlen += tSerializeSClientHbRsp(buf, pRsp); + if (tSerializeSClientHbRsp(&encoder, pRsp) < 0) return -1; } + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); return tlen; } -void *tDeserializeSClientHbBatchRsp(void *buf, SClientHbBatchRsp *pBatchRsp) { - int32_t sz; - buf = taosDecodeFixedI32(buf, &sz); - pBatchRsp->rsps = taosArrayInit(sz, sizeof(SClientHbRsp)); - for (int32_t i = 0; i < sz; i++) { +int32_t tDeserializeSClientHbBatchRsp(void *buf, int32_t bufLen, SClientHbBatchRsp *pBatchRsp) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI64(&decoder, &pBatchRsp->reqId) < 0) return -1; + if (tDecodeI64(&decoder, &pBatchRsp->rspId) < 0) return -1; + + int32_t rspNum = 0; + if (tDecodeI32(&decoder, &rspNum) < 0) return -1; + if (pBatchRsp->rsps == NULL) { + pBatchRsp->rsps = taosArrayInit(rspNum, sizeof(SClientHbReq)); + } + for (int32_t i = 0; i < rspNum; i++) { SClientHbRsp rsp = {0}; - buf = tDeserializeSClientHbRsp(buf, &rsp); + tDeserializeSClientHbRsp(&decoder, &rsp); taosArrayPush(pBatchRsp->rsps, &rsp); } - return buf; + + tEndDecode(&decoder); + tCoderClear(&decoder); + return 0; } int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) { @@ -210,7 +289,7 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) { switch (pReq->type) { case TD_SUPER_TABLE: - tlen += taosEncodeFixedU64(buf, pReq->stbCfg.suid); + tlen += taosEncodeFixedI64(buf, pReq->stbCfg.suid); tlen += taosEncodeFixedU32(buf, pReq->stbCfg.nCols); for (uint32_t i = 0; i < pReq->stbCfg.nCols; i++) { tlen += taosEncodeFixedI8(buf, pReq->stbCfg.pSchema[i].type); @@ -227,7 +306,7 @@ int32_t tSerializeSVCreateTbReq(void **buf, SVCreateTbReq *pReq) { } break; case TD_CHILD_TABLE: - tlen += taosEncodeFixedU64(buf, pReq->ctbCfg.suid); + tlen += taosEncodeFixedI64(buf, pReq->ctbCfg.suid); tlen += tdEncodeKVRow(buf, pReq->ctbCfg.pTag); break; case TD_NORMAL_TABLE: @@ -255,7 +334,7 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { switch (pReq->type) { case TD_SUPER_TABLE: - buf = taosDecodeFixedU64(buf, &(pReq->stbCfg.suid)); + buf = taosDecodeFixedI64(buf, &(pReq->stbCfg.suid)); buf = taosDecodeFixedU32(buf, &(pReq->stbCfg.nCols)); pReq->stbCfg.pSchema = (SSchema *)malloc(pReq->stbCfg.nCols * sizeof(SSchema)); for (uint32_t i = 0; i < pReq->stbCfg.nCols; i++) { @@ -274,7 +353,7 @@ void *tDeserializeSVCreateTbReq(void *buf, SVCreateTbReq *pReq) { } break; case TD_CHILD_TABLE: - buf = taosDecodeFixedU64(buf, &pReq->ctbCfg.suid); + buf = taosDecodeFixedI64(buf, &pReq->ctbCfg.suid); buf = tdDecodeKVRow(buf, &pReq->ctbCfg.pTag); break; case TD_NORMAL_TABLE: @@ -337,267 +416,340 @@ void *tDeserializeSVDropTbReq(void *buf, SVDropTbReq *pReq) { return buf; } -int32_t tSerializeSMCreateStbReq(void **buf, SMCreateStbReq *pReq) { - int32_t tlen = 0; +int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); - tlen += taosEncodeString(buf, pReq->name); - tlen += taosEncodeFixedI8(buf, pReq->igExists); - tlen += taosEncodeFixedI32(buf, pReq->numOfColumns); - tlen += taosEncodeFixedI32(buf, pReq->numOfTags); + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; + if (tEncodeI8(&encoder, pReq->igExists) < 0) return -1; + if (tEncodeI32(&encoder, pReq->numOfColumns) < 0) return -1; + if (tEncodeI32(&encoder, pReq->numOfTags) < 0) return -1; for (int32_t i = 0; i < pReq->numOfColumns; ++i) { SField *pField = taosArrayGet(pReq->pColumns, i); - tlen += taosEncodeFixedI8(buf, pField->type); - tlen += taosEncodeFixedI32(buf, pField->bytes); - tlen += taosEncodeString(buf, pField->name); + if (tEncodeI8(&encoder, pField->type) < 0) return -1; + if (tEncodeI32(&encoder, pField->bytes) < 0) return -1; + if (tEncodeCStr(&encoder, pField->name) < 0) return -1; } for (int32_t i = 0; i < pReq->numOfTags; ++i) { SField *pField = taosArrayGet(pReq->pTags, i); - tlen += taosEncodeFixedI8(buf, pField->type); - tlen += taosEncodeFixedI32(buf, pField->bytes); - tlen += taosEncodeString(buf, pField->name); + if (tEncodeI8(&encoder, pField->type) < 0) return -1; + if (tEncodeI32(&encoder, pField->bytes) < 0) return -1; + if (tEncodeCStr(&encoder, pField->name) < 0) return -1; } - tlen += taosEncodeString(buf, pReq->comment); + if (tEncodeCStr(&encoder, pReq->comment) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); return tlen; } -void *tDeserializeSMCreateStbReq(void *buf, SMCreateStbReq *pReq) { - buf = taosDecodeStringTo(buf, pReq->name); - buf = taosDecodeFixedI8(buf, &pReq->igExists); - buf = taosDecodeFixedI32(buf, &pReq->numOfColumns); - buf = taosDecodeFixedI32(buf, &pReq->numOfTags); +int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->igExists) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->numOfColumns) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->numOfTags) < 0) return -1; pReq->pColumns = taosArrayInit(pReq->numOfColumns, sizeof(SField)); pReq->pTags = taosArrayInit(pReq->numOfTags, sizeof(SField)); if (pReq->pColumns == NULL || pReq->pTags == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; + return -1; } for (int32_t i = 0; i < pReq->numOfColumns; ++i) { SField field = {0}; - buf = taosDecodeFixedI8(buf, &field.type); - buf = taosDecodeFixedI32(buf, &field.bytes); - buf = taosDecodeStringTo(buf, field.name); + if (tDecodeI8(&decoder, &field.type) < 0) return -1; + if (tDecodeI32(&decoder, &field.bytes) < 0) return -1; + if (tDecodeCStrTo(&decoder, field.name) < 0) return -1; if (taosArrayPush(pReq->pColumns, &field) == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; + return -1; } } for (int32_t i = 0; i < pReq->numOfTags; ++i) { SField field = {0}; - buf = taosDecodeFixedI8(buf, &field.type); - buf = taosDecodeFixedI32(buf, &field.bytes); - buf = taosDecodeStringTo(buf, field.name); + if (tDecodeI8(&decoder, &field.type) < 0) return -1; + if (tDecodeI32(&decoder, &field.bytes) < 0) return -1; + if (tDecodeCStrTo(&decoder, field.name) < 0) return -1; if (taosArrayPush(pReq->pTags, &field) == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; + return -1; } } - buf = taosDecodeStringTo(buf, pReq->comment); - return buf; + if (tDecodeCStrTo(&decoder, pReq->comment) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; } -int32_t tSerializeSMDropStbReq(void **buf, SMDropStbReq *pReq) { - int32_t tlen = 0; +void tFreeSMCreateStbReq(SMCreateStbReq *pReq) { + taosArrayDestroy(pReq->pColumns); + taosArrayDestroy(pReq->pTags); + pReq->pColumns = NULL; + pReq->pTags = NULL; +} - tlen += taosEncodeString(buf, pReq->name); - tlen += taosEncodeFixedI8(buf, pReq->igNotExists); +int32_t tSerializeSMDropStbReq(void *buf, int32_t bufLen, SMDropStbReq *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; + if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); return tlen; } -void *tDeserializeSMDropStbReq(void *buf, SMDropStbReq *pReq) { - buf = taosDecodeStringTo(buf, pReq->name); - buf = taosDecodeFixedI8(buf, &pReq->igNotExists); +int32_t tDeserializeSMDropStbReq(void *buf, int32_t bufLen, SMDropStbReq *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); - return buf; + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; } -int32_t tSerializeSMAlterStbReq(void **buf, SMAltertbReq *pReq) { - int32_t tlen = 0; - - tlen += taosEncodeString(buf, pReq->name); - tlen += taosEncodeFixedI8(buf, pReq->alterType); - tlen += taosEncodeFixedI32(buf, pReq->numOfFields); +int32_t tSerializeSMAlterStbReq(void *buf, int32_t bufLen, SMAltertbReq *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; + if (tEncodeI8(&encoder, pReq->alterType) < 0) return -1; + if (tEncodeI32(&encoder, pReq->numOfFields) < 0) return -1; for (int32_t i = 0; i < pReq->numOfFields; ++i) { SField *pField = taosArrayGet(pReq->pFields, i); - tlen += taosEncodeFixedU8(buf, pField->type); - tlen += taosEncodeFixedI32(buf, pField->bytes); - tlen += taosEncodeString(buf, pField->name); + if (tEncodeI8(&encoder, pField->type) < 0) return -1; + if (tEncodeI32(&encoder, pField->bytes) < 0) return -1; + if (tEncodeCStr(&encoder, pField->name) < 0) return -1; } + tEndEncode(&encoder); + int32_t tlen = encoder.pos; + tCoderClear(&encoder); return tlen; } -void *tDeserializeSMAlterStbReq(void *buf, SMAltertbReq *pReq) { - buf = taosDecodeStringTo(buf, pReq->name); - buf = taosDecodeFixedI8(buf, &pReq->alterType); - buf = taosDecodeFixedI32(buf, &pReq->numOfFields); +int32_t tDeserializeSMAlterStbReq(void *buf, int32_t bufLen, SMAltertbReq *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->alterType) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->numOfFields) < 0) return -1; pReq->pFields = taosArrayInit(pReq->numOfFields, sizeof(SField)); if (pReq->pFields == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; + return -1; } for (int32_t i = 0; i < pReq->numOfFields; ++i) { SField field = {0}; - buf = taosDecodeFixedU8(buf, &field.type); - buf = taosDecodeFixedI32(buf, &field.bytes); - buf = taosDecodeStringTo(buf, field.name); + if (tDecodeI8(&decoder, &field.type) < 0) return -1; + if (tDecodeI32(&decoder, &field.bytes) < 0) return -1; + if (tDecodeCStrTo(&decoder, field.name) < 0) return -1; if (taosArrayPush(pReq->pFields, &field) == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; + return -1; } } - return buf; + tEndDecode(&decoder); + tCoderClear(&decoder); + return 0; } -int32_t tSerializeSStatusReq(void **buf, SStatusReq *pReq) { - int32_t tlen = 0; +void tFreeSMAltertbReq(SMAltertbReq *pReq) { + taosArrayDestroy(pReq->pFields); + pReq->pFields = NULL; +} + +int32_t tSerializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; // status - tlen += taosEncodeFixedI32(buf, pReq->sver); - tlen += taosEncodeFixedI64(buf, pReq->dver); - tlen += taosEncodeFixedI32(buf, pReq->dnodeId); - tlen += taosEncodeFixedI64(buf, pReq->clusterId); - tlen += taosEncodeFixedI64(buf, pReq->rebootTime); - tlen += taosEncodeFixedI64(buf, pReq->updateTime); - tlen += taosEncodeFixedI32(buf, pReq->numOfCores); - tlen += taosEncodeFixedI32(buf, pReq->numOfSupportVnodes); - tlen += taosEncodeString(buf, pReq->dnodeEp); + if (tEncodeI32(&encoder, pReq->sver) < 0) return -1; + if (tEncodeI64(&encoder, pReq->dver) < 0) return -1; + if (tEncodeI32(&encoder, pReq->dnodeId) < 0) return -1; + if (tEncodeI64(&encoder, pReq->clusterId) < 0) return -1; + if (tEncodeI64(&encoder, pReq->rebootTime) < 0) return -1; + if (tEncodeI64(&encoder, pReq->updateTime) < 0) return -1; + if (tEncodeI32(&encoder, pReq->numOfCores) < 0) return -1; + if (tEncodeI32(&encoder, pReq->numOfSupportVnodes) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->dnodeEp) < 0) return -1; // cluster cfg - tlen += taosEncodeFixedI32(buf, pReq->clusterCfg.statusInterval); - tlen += taosEncodeFixedI64(buf, pReq->clusterCfg.checkTime); - tlen += taosEncodeString(buf, pReq->clusterCfg.timezone); - tlen += taosEncodeString(buf, pReq->clusterCfg.locale); - tlen += taosEncodeString(buf, pReq->clusterCfg.charset); + if (tEncodeI32(&encoder, pReq->clusterCfg.statusInterval) < 0) return -1; + if (tEncodeI64(&encoder, pReq->clusterCfg.checkTime) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->clusterCfg.timezone) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->clusterCfg.locale) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->clusterCfg.charset) < 0) return -1; // vnode loads int32_t vlen = (int32_t)taosArrayGetSize(pReq->pVloads); - tlen += taosEncodeFixedI32(buf, vlen); + if (tEncodeI32(&encoder, vlen) < 0) return -1; for (int32_t i = 0; i < vlen; ++i) { SVnodeLoad *pload = taosArrayGet(pReq->pVloads, i); - tlen += taosEncodeFixedI32(buf, pload->vgId); - tlen += taosEncodeFixedI8(buf, pload->role); - tlen += taosEncodeFixedI64(buf, pload->numOfTables); - tlen += taosEncodeFixedI64(buf, pload->numOfTimeSeries); - tlen += taosEncodeFixedI64(buf, pload->totalStorage); - tlen += taosEncodeFixedI64(buf, pload->compStorage); - tlen += taosEncodeFixedI64(buf, pload->pointsWritten); + if (tEncodeI32(&encoder, pload->vgId) < 0) return -1; + if (tEncodeI8(&encoder, pload->role) < 0) return -1; + if (tEncodeI64(&encoder, pload->numOfTables) < 0) return -1; + if (tEncodeI64(&encoder, pload->numOfTimeSeries) < 0) return -1; + if (tEncodeI64(&encoder, pload->totalStorage) < 0) return -1; + if (tEncodeI64(&encoder, pload->compStorage) < 0) return -1; + if (tEncodeI64(&encoder, pload->pointsWritten) < 0) return -1; } + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); return tlen; } -void *tDeserializeSStatusReq(void *buf, SStatusReq *pReq) { +int32_t tDeserializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + // status - buf = taosDecodeFixedI32(buf, &pReq->sver); - buf = taosDecodeFixedI64(buf, &pReq->dver); - buf = taosDecodeFixedI32(buf, &pReq->dnodeId); - buf = taosDecodeFixedI64(buf, &pReq->clusterId); - buf = taosDecodeFixedI64(buf, &pReq->rebootTime); - buf = taosDecodeFixedI64(buf, &pReq->updateTime); - buf = taosDecodeFixedI32(buf, &pReq->numOfCores); - buf = taosDecodeFixedI32(buf, &pReq->numOfSupportVnodes); - buf = taosDecodeStringTo(buf, pReq->dnodeEp); + if (tDecodeI32(&decoder, &pReq->sver) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->dver) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->clusterId) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->rebootTime) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->updateTime) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->numOfCores) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->numOfSupportVnodes) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->dnodeEp) < 0) return -1; // cluster cfg - buf = taosDecodeFixedI32(buf, &pReq->clusterCfg.statusInterval); - buf = taosDecodeFixedI64(buf, &pReq->clusterCfg.checkTime); - buf = taosDecodeStringTo(buf, pReq->clusterCfg.timezone); - buf = taosDecodeStringTo(buf, pReq->clusterCfg.locale); - buf = taosDecodeStringTo(buf, pReq->clusterCfg.charset); + if (tDecodeI32(&decoder, &pReq->clusterCfg.statusInterval) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->clusterCfg.checkTime) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->clusterCfg.timezone) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->clusterCfg.locale) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->clusterCfg.charset) < 0) return -1; // vnode loads int32_t vlen = 0; - buf = taosDecodeFixedI32(buf, &vlen); + if (tDecodeI32(&decoder, &vlen) < 0) return -1; pReq->pVloads = taosArrayInit(vlen, sizeof(SVnodeLoad)); if (pReq->pVloads == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; + return -1; } for (int32_t i = 0; i < vlen; ++i) { SVnodeLoad vload = {0}; - buf = taosDecodeFixedI32(buf, &vload.vgId); - buf = taosDecodeFixedI8(buf, &vload.role); - buf = taosDecodeFixedI64(buf, &vload.numOfTables); - buf = taosDecodeFixedI64(buf, &vload.numOfTimeSeries); - buf = taosDecodeFixedI64(buf, &vload.totalStorage); - buf = taosDecodeFixedI64(buf, &vload.compStorage); - buf = taosDecodeFixedI64(buf, &vload.pointsWritten); + if (tDecodeI32(&decoder, &vload.vgId) < 0) return -1; + if (tDecodeI8(&decoder, &vload.role) < 0) return -1; + if (tDecodeI64(&decoder, &vload.numOfTables) < 0) return -1; + if (tDecodeI64(&decoder, &vload.numOfTimeSeries) < 0) return -1; + if (tDecodeI64(&decoder, &vload.totalStorage) < 0) return -1; + if (tDecodeI64(&decoder, &vload.compStorage) < 0) return -1; + if (tDecodeI64(&decoder, &vload.pointsWritten) < 0) return -1; if (taosArrayPush(pReq->pVloads, &vload) == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; + return -1; } } - return buf; + tEndDecode(&decoder); + tCoderClear(&decoder); + return 0; } -int32_t tSerializeSStatusRsp(void **buf, SStatusRsp *pRsp) { - int32_t tlen = 0; +int32_t tSerializeSStatusRsp(void *buf, int32_t bufLen, SStatusRsp *pRsp) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); - // status; - tlen += taosEncodeFixedI64(buf, pRsp->dver); + if (tStartEncode(&encoder) < 0) return -1; + + // status + if (tEncodeI64(&encoder, pRsp->dver) < 0) return -1; // dnode cfg - tlen += taosEncodeFixedI32(buf, pRsp->dnodeCfg.dnodeId); - tlen += taosEncodeFixedI64(buf, pRsp->dnodeCfg.clusterId); + if (tEncodeI32(&encoder, pRsp->dnodeCfg.dnodeId) < 0) return -1; + if (tEncodeI64(&encoder, pRsp->dnodeCfg.clusterId) < 0) return -1; // dnode eps int32_t dlen = (int32_t)taosArrayGetSize(pRsp->pDnodeEps); - tlen += taosEncodeFixedI32(buf, dlen); + if (tEncodeI32(&encoder, dlen) < 0) return -1; for (int32_t i = 0; i < dlen; ++i) { SDnodeEp *pDnodeEp = taosArrayGet(pRsp->pDnodeEps, i); - tlen += taosEncodeFixedI32(buf, pDnodeEp->id); - tlen += taosEncodeFixedI8(buf, pDnodeEp->isMnode); - tlen += taosEncodeString(buf, pDnodeEp->ep.fqdn); - tlen += taosEncodeFixedU16(buf, pDnodeEp->ep.port); + if (tEncodeI32(&encoder, pDnodeEp->id) < 0) return -1; + if (tEncodeI8(&encoder, pDnodeEp->isMnode) < 0) return -1; + if (tEncodeCStr(&encoder, pDnodeEp->ep.fqdn) < 0) return -1; + if (tEncodeU16(&encoder, pDnodeEp->ep.port) < 0) return -1; } + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); return tlen; } -void *tDeserializeSStatusRsp(void *buf, SStatusRsp *pRsp) { +int32_t tDeserializeSStatusRsp(void *buf, int32_t bufLen, SStatusRsp *pRsp) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + // status - buf = taosDecodeFixedI64(buf, &pRsp->dver); + if (tDecodeI64(&decoder, &pRsp->dver) < 0) return -1; // cluster cfg - buf = taosDecodeFixedI32(buf, &pRsp->dnodeCfg.dnodeId); - buf = taosDecodeFixedI64(buf, &pRsp->dnodeCfg.clusterId); + if (tDecodeI32(&decoder, &pRsp->dnodeCfg.dnodeId) < 0) return -1; + if (tDecodeI64(&decoder, &pRsp->dnodeCfg.clusterId) < 0) return -1; // dnode eps int32_t dlen = 0; - buf = taosDecodeFixedI32(buf, &dlen); + if (tDecodeI32(&decoder, &dlen) < 0) return -1; pRsp->pDnodeEps = taosArrayInit(dlen, sizeof(SDnodeEp)); if (pRsp->pDnodeEps == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; + return -1; } for (int32_t i = 0; i < dlen; ++i) { SDnodeEp dnodeEp = {0}; - buf = taosDecodeFixedI32(buf, &dnodeEp.id); - buf = taosDecodeFixedI8(buf, &dnodeEp.isMnode); - buf = taosDecodeStringTo(buf, dnodeEp.ep.fqdn); - buf = taosDecodeFixedU16(buf, &dnodeEp.ep.port); + if (tDecodeI32(&decoder, &dnodeEp.id) < 0) return -1; + if (tDecodeI8(&decoder, &dnodeEp.isMnode) < 0) return -1; + if (tDecodeCStrTo(&decoder, dnodeEp.ep.fqdn) < 0) return -1; + if (tDecodeU16(&decoder, &dnodeEp.ep.port) < 0) return -1; if (taosArrayPush(pRsp->pDnodeEps, &dnodeEp) == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; + return -1; } } - return buf; + tEndDecode(&decoder); + tCoderClear(&decoder); + return 0; } int32_t tSerializeSCreateAcctReq(void *buf, int32_t bufLen, SCreateAcctReq *pReq) { @@ -1306,4 +1458,851 @@ int32_t tDeserializeSSyncDbReq(void *buf, int32_t bufLen, SSyncDbReq *pReq) { tCoderClear(&decoder); return 0; -} \ No newline at end of file +} + +static int32_t tSerializeSUseDbRspImp(SCoder *pEncoder, SUseDbRsp *pRsp) { + if (tEncodeCStr(pEncoder, pRsp->db) < 0) return -1; + if (tEncodeU64(pEncoder, pRsp->uid) < 0) return -1; + if (tEncodeI32(pEncoder, pRsp->vgVersion) < 0) return -1; + if (tEncodeI32(pEncoder, pRsp->vgNum) < 0) return -1; + if (tEncodeI8(pEncoder, pRsp->hashMethod) < 0) return -1; + + for (int32_t i = 0; i < pRsp->vgNum; ++i) { + SVgroupInfo *pVgInfo = taosArrayGet(pRsp->pVgroupInfos, i); + if (tEncodeI32(pEncoder, pVgInfo->vgId) < 0) return -1; + if (tEncodeU32(pEncoder, pVgInfo->hashBegin) < 0) return -1; + if (tEncodeU32(pEncoder, pVgInfo->hashEnd) < 0) return -1; + if (tEncodeSEpSet(pEncoder, &pVgInfo->epset) < 0) return -1; + } + + return 0; +} + +int32_t tSerializeSUseDbRsp(void *buf, int32_t bufLen, SUseDbRsp *pRsp) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tSerializeSUseDbRspImp(&encoder, pRsp) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tSerializeSUseDbBatchRsp(void *buf, int32_t bufLen, SUseDbBatchRsp *pRsp) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + + int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); + if (tEncodeI32(&encoder, numOfBatch) < 0) return -1; + for (int32_t i = 0; i < numOfBatch; ++i) { + SUseDbRsp *pUsedbRsp = taosArrayGet(pRsp->pArray, i); + if (tSerializeSUseDbRspImp(&encoder, pUsedbRsp) < 0) return -1; + } + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSUseDbRspImp(SCoder *pDecoder, SUseDbRsp *pRsp) { + if (tDecodeCStrTo(pDecoder, pRsp->db) < 0) return -1; + if (tDecodeU64(pDecoder, &pRsp->uid) < 0) return -1; + if (tDecodeI32(pDecoder, &pRsp->vgVersion) < 0) return -1; + if (tDecodeI32(pDecoder, &pRsp->vgNum) < 0) return -1; + if (tDecodeI8(pDecoder, &pRsp->hashMethod) < 0) return -1; + + pRsp->pVgroupInfos = taosArrayInit(pRsp->vgNum, sizeof(SVgroupInfo)); + if (pRsp->pVgroupInfos == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + for (int32_t i = 0; i < pRsp->vgNum; ++i) { + SVgroupInfo vgInfo = {0}; + if (tDecodeI32(pDecoder, &vgInfo.vgId) < 0) return -1; + if (tDecodeU32(pDecoder, &vgInfo.hashBegin) < 0) return -1; + if (tDecodeU32(pDecoder, &vgInfo.hashEnd) < 0) return -1; + if (tDecodeSEpSet(pDecoder, &vgInfo.epset) < 0) return -1; + taosArrayPush(pRsp->pVgroupInfos, &vgInfo); + } + + return 0; +} + +int32_t tDeserializeSUseDbRsp(void *buf, int32_t bufLen, SUseDbRsp *pRsp) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDeserializeSUseDbRspImp(&decoder, pRsp) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +int32_t tDeserializeSUseDbBatchRsp(void *buf, int32_t bufLen, SUseDbBatchRsp *pRsp) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + + int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); + if (tDecodeI32(&decoder, &numOfBatch) < 0) return -1; + + pRsp->pArray = taosArrayInit(numOfBatch, sizeof(SUseDbBatchRsp)); + if (pRsp->pArray == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + for (int32_t i = 0; i < numOfBatch; ++i) { + SUseDbRsp usedbRsp = {0}; + if (tDeserializeSUseDbRspImp(&decoder, &usedbRsp) < 0) return -1; + taosArrayPush(pRsp->pArray, &usedbRsp); + } + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +void tFreeSUsedbRsp(SUseDbRsp *pRsp) { taosArrayDestroy(pRsp->pVgroupInfos); } + +void tFreeSUseDbBatchRsp(SUseDbBatchRsp *pRsp) { + int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); + for (int32_t i = 0; i < numOfBatch; ++i) { + SUseDbRsp *pUsedbRsp = taosArrayGet(pRsp->pArray, i); + tFreeSUsedbRsp(pUsedbRsp); + } + + taosArrayDestroy(pRsp->pArray); +} + +int32_t tSerializeSShowReq(void *buf, int32_t bufLen, SShowReq *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI32(&encoder, pReq->type) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->db) < 0) return -1; + if (tEncodeI32(&encoder, pReq->payloadLen) < 0) return -1; + if (pReq->payloadLen > 0) { + if (tEncodeBinary(&encoder, pReq->payload, pReq->payloadLen) < 0) return -1; + } + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSShowReq(void *buf, int32_t bufLen, SShowReq *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->type) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->payloadLen) < 0) return -1; + if (pReq->payloadLen > 0) { + pReq->payload = malloc(pReq->payloadLen); + if (pReq->payload == NULL) return -1; + if (tDecodeCStrTo(&decoder, pReq->payload) < 0) return -1; + } + + tEndDecode(&decoder); + tCoderClear(&decoder); + return 0; +} + +void tFreeSShowReq(SShowReq *pReq) { tfree(pReq->payload); } + +int32_t tSerializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableReq *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI64(&encoder, pReq->showId) < 0) return -1; + if (tEncodeI8(&encoder, pReq->free) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSRetrieveTableReq(void *buf, int32_t bufLen, SRetrieveTableReq *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->showId) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->free) < 0) return -1; + + tEndDecode(&decoder); + tCoderClear(&decoder); + return 0; +} + +static int32_t tEncodeSTableMetaRsp(SCoder *pEncoder, STableMetaRsp *pRsp) { + if (tEncodeCStr(pEncoder, pRsp->tbName) < 0) return -1; + if (tEncodeCStr(pEncoder, pRsp->stbName) < 0) return -1; + if (tEncodeCStr(pEncoder, pRsp->dbFName) < 0) return -1; + if (tEncodeU64(pEncoder, pRsp->dbId) < 0) return -1; + if (tEncodeI32(pEncoder, pRsp->numOfTags) < 0) return -1; + if (tEncodeI32(pEncoder, pRsp->numOfColumns) < 0) return -1; + if (tEncodeI8(pEncoder, pRsp->precision) < 0) return -1; + if (tEncodeI8(pEncoder, pRsp->tableType) < 0) return -1; + if (tEncodeI8(pEncoder, pRsp->update) < 0) return -1; + if (tEncodeI32(pEncoder, pRsp->sversion) < 0) return -1; + if (tEncodeI32(pEncoder, pRsp->tversion) < 0) return -1; + if (tEncodeU64(pEncoder, pRsp->suid) < 0) return -1; + if (tEncodeU64(pEncoder, pRsp->tuid) < 0) return -1; + if (tEncodeI32(pEncoder, pRsp->vgId) < 0) return -1; + for (int32_t i = 0; i < pRsp->numOfColumns + pRsp->numOfTags; ++i) { + SSchema *pSchema = &pRsp->pSchemas[i]; + if (tEncodeSSchema(pEncoder, pSchema) < 0) return -1; + } + + return 0; +} + +static int32_t tDecodeSTableMetaRsp(SCoder *pDecoder, STableMetaRsp *pRsp) { + if (tDecodeCStrTo(pDecoder, pRsp->tbName) < 0) return -1; + if (tDecodeCStrTo(pDecoder, pRsp->stbName) < 0) return -1; + if (tDecodeCStrTo(pDecoder, pRsp->dbFName) < 0) return -1; + if (tDecodeU64(pDecoder, &pRsp->dbId) < 0) return -1; + if (tDecodeI32(pDecoder, &pRsp->numOfTags) < 0) return -1; + if (tDecodeI32(pDecoder, &pRsp->numOfColumns) < 0) return -1; + if (tDecodeI8(pDecoder, &pRsp->precision) < 0) return -1; + if (tDecodeI8(pDecoder, &pRsp->tableType) < 0) return -1; + if (tDecodeI8(pDecoder, &pRsp->update) < 0) return -1; + if (tDecodeI32(pDecoder, &pRsp->sversion) < 0) return -1; + if (tDecodeI32(pDecoder, &pRsp->tversion) < 0) return -1; + if (tDecodeU64(pDecoder, &pRsp->suid) < 0) return -1; + if (tDecodeU64(pDecoder, &pRsp->tuid) < 0) return -1; + if (tDecodeI32(pDecoder, &pRsp->vgId) < 0) return -1; + + int32_t totalCols = pRsp->numOfTags + pRsp->numOfColumns; + pRsp->pSchemas = malloc(sizeof(SSchema) * totalCols); + if (pRsp->pSchemas == NULL) return -1; + + for (int32_t i = 0; i < totalCols; ++i) { + SSchema *pSchema = &pRsp->pSchemas[i]; + if (tDecodeSSchema(pDecoder, pSchema) < 0) return -1; + } + + return 0; +} + +int32_t tSerializeSTableMetaRsp(void *buf, int32_t bufLen, STableMetaRsp *pRsp) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeSTableMetaRsp(&encoder, pRsp) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tSerializeSTableMetaBatchRsp(void *buf, int32_t bufLen, STableMetaBatchRsp *pRsp) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + + int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); + if (tEncodeI32(&encoder, numOfBatch) < 0) return -1; + for (int32_t i = 0; i < numOfBatch; ++i) { + STableMetaRsp *pMetaRsp = taosArrayGet(pRsp->pArray, i); + if (tEncodeSTableMetaRsp(&encoder, pMetaRsp) < 0) return -1; + } + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSTableMetaRsp(void *buf, int32_t bufLen, STableMetaRsp *pRsp) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeSTableMetaRsp(&decoder, pRsp) < 0) return -1; + + tEndDecode(&decoder); + tCoderClear(&decoder); + return 0; +} + +int32_t tDeserializeSTableMetaBatchRsp(void *buf, int32_t bufLen, STableMetaBatchRsp *pRsp) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + + int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); + if (tDecodeI32(&decoder, &numOfBatch) < 0) return -1; + + pRsp->pArray = taosArrayInit(numOfBatch, sizeof(STableMetaRsp)); + if (pRsp->pArray == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + for (int32_t i = 0; i < numOfBatch; ++i) { + STableMetaRsp tableMetaRsp = {0}; + if (tDecodeSTableMetaRsp(&decoder, &tableMetaRsp) < 0) return -1; + taosArrayPush(pRsp->pArray, &tableMetaRsp); + } + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +void tFreeSTableMetaRsp(STableMetaRsp *pRsp) { tfree(pRsp->pSchemas); } + +void tFreeSTableMetaBatchRsp(STableMetaBatchRsp *pRsp) { + int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); + for (int32_t i = 0; i < numOfBatch; ++i) { + STableMetaRsp *pMetaRsp = taosArrayGet(pRsp->pArray, i); + tFreeSTableMetaRsp(pMetaRsp); + } + + taosArrayDestroy(pRsp->pArray); +} + +int32_t tSerializeSShowRsp(void *buf, int32_t bufLen, SShowRsp *pRsp) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI64(&encoder, pRsp->showId) < 0) return -1; + if (tEncodeSTableMetaRsp(&encoder, &pRsp->tableMeta) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSShowRsp(void *buf, int32_t bufLen, SShowRsp *pRsp) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI64(&decoder, &pRsp->showId) < 0) return -1; + if (tDecodeSTableMetaRsp(&decoder, &pRsp->tableMeta) < 0) return -1; + + tEndDecode(&decoder); + tCoderClear(&decoder); + return 0; +} + +void tFreeSShowRsp(SShowRsp *pRsp) { tFreeSTableMetaRsp(&pRsp->tableMeta); } + +int32_t tSerializeSTableInfoReq(void *buf, int32_t bufLen, STableInfoReq *pReq) { + int32_t headLen = sizeof(SMsgHead); + if (buf != NULL) { + buf = (char *)buf + headLen; + bufLen -= headLen; + } + + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->dbFName) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->tbName) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + + if (buf != NULL) { + SMsgHead *pHead = (SMsgHead *)((char *)buf - headLen); + pHead->vgId = htonl(pReq->header.vgId); + pHead->contLen = htonl(tlen + headLen); + } + + return tlen + headLen; +} + +int32_t tDeserializeSTableInfoReq(void *buf, int32_t bufLen, STableInfoReq *pReq) { + int32_t headLen = sizeof(SMsgHead); + + SMsgHead *pHead = buf; + pHead->vgId = pReq->header.vgId; + pHead->contLen = pReq->header.contLen; + + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, (char *)buf + headLen, bufLen - headLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->dbFName) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->tbName) < 0) return -1; + + tEndDecode(&decoder); + tCoderClear(&decoder); + return 0; +} + +int32_t tSerializeSMDropTopicReq(void *buf, int32_t bufLen, SMDropTopicReq *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; + if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSMDropTopicReq(void *buf, int32_t bufLen, SMDropTopicReq *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +int32_t tSerializeMCreateTopicReq(void *buf, int32_t bufLen, const SMCreateTopicReq *pReq) { + int32_t sqlLen = 0; + int32_t physicalPlanLen = 0; + int32_t logicalPlanLen = 0; + if (pReq->sql != NULL) sqlLen = (int32_t)strlen(pReq->sql); + if (pReq->physicalPlan != NULL) physicalPlanLen = (int32_t)strlen(pReq->physicalPlan); + if (pReq->logicalPlan != NULL) logicalPlanLen = (int32_t)strlen(pReq->logicalPlan); + + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; + if (tEncodeI8(&encoder, pReq->igExists) < 0) return -1; + if (tEncodeI32(&encoder, sqlLen) < 0) return -1; + if (tEncodeI32(&encoder, physicalPlanLen) < 0) return -1; + if (tEncodeI32(&encoder, logicalPlanLen) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->sql) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->physicalPlan) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->logicalPlan) < 0) return -1; + + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSMCreateTopicReq(void *buf, int32_t bufLen, SMCreateTopicReq *pReq) { + int32_t sqlLen = 0; + int32_t physicalPlanLen = 0; + int32_t logicalPlanLen = 0; + + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->igExists) < 0) return -1; + if (tDecodeI32(&decoder, &sqlLen) < 0) return -1; + if (tDecodeI32(&decoder, &physicalPlanLen) < 0) return -1; + if (tDecodeI32(&decoder, &logicalPlanLen) < 0) return -1; + + pReq->sql = calloc(1, sqlLen + 1); + pReq->physicalPlan = calloc(1, physicalPlanLen + 1); + pReq->logicalPlan = calloc(1, logicalPlanLen + 1); + if (pReq->sql == NULL || pReq->physicalPlan == NULL || pReq->logicalPlan == NULL) return -1; + + if (tDecodeCStrTo(&decoder, pReq->sql) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->physicalPlan) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->logicalPlan) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +void tFreeSMCreateTopicReq(SMCreateTopicReq *pReq) { + tfree(pReq->sql); + tfree(pReq->physicalPlan); + tfree(pReq->logicalPlan); +} + +int32_t tSerializeSMCreateTopicRsp(void *buf, int32_t bufLen, const SMCreateTopicRsp *pRsp) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI64(&encoder, pRsp->topicId) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSMCreateTopicRsp(void *buf, int32_t bufLen, SMCreateTopicRsp *pRsp) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI64(&decoder, &pRsp->topicId) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +int32_t tSerializeSConnectReq(void *buf, int32_t bufLen, SConnectReq *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI32(&encoder, pReq->pid) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->app) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->db) < 0) return -1; + if (tEncodeI64(&encoder, pReq->startTime) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSConnectReq(void *buf, int32_t bufLen, SConnectReq *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->pid) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->app) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; + if (tDecodeI64(&decoder, &pReq->startTime) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +int32_t tSerializeSConnectRsp(void *buf, int32_t bufLen, SConnectRsp *pRsp) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI32(&encoder, pRsp->acctId) < 0) return -1; + if (tEncodeI64(&encoder, pRsp->clusterId) < 0) return -1; + if (tEncodeI32(&encoder, pRsp->connId) < 0) return -1; + if (tEncodeI8(&encoder, pRsp->superUser) < 0) return -1; + if (tEncodeSEpSet(&encoder, &pRsp->epSet) < 0) return -1; + if (tEncodeCStr(&encoder, pRsp->sVersion) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSConnectRsp(void *buf, int32_t bufLen, SConnectRsp *pRsp) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI32(&decoder, &pRsp->acctId) < 0) return -1; + if (tDecodeI64(&decoder, &pRsp->clusterId) < 0) return -1; + if (tDecodeI32(&decoder, &pRsp->connId) < 0) return -1; + if (tDecodeI8(&decoder, &pRsp->superUser) < 0) return -1; + if (tDecodeSEpSet(&decoder, &pRsp->epSet) < 0) return -1; + if (tDecodeCStrTo(&decoder, pRsp->sVersion) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +int32_t tSerializeSMTimerMsg(void *buf, int32_t bufLen, SMTimerReq *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI32(&encoder, pReq->reserved) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSMTimerMsg(void *buf, int32_t bufLen, SMTimerReq *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->reserved) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +int32_t tEncodeSReplica(SCoder *pEncoder, SReplica *pReplica) { + if (tEncodeI32(pEncoder, pReplica->id) < 0) return -1; + if (tEncodeU16(pEncoder, pReplica->port) < 0) return -1; + if (tEncodeCStr(pEncoder, pReplica->fqdn) < 0) return -1; + return 0; +} + +int32_t tDecodeSReplica(SCoder *pDecoder, SReplica *pReplica) { + if (tDecodeI32(pDecoder, &pReplica->id) < 0) return -1; + if (tDecodeU16(pDecoder, &pReplica->port) < 0) return -1; + if (tDecodeCStrTo(pDecoder, pReplica->fqdn) < 0) return -1; + return 0; +} + +int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI32(&encoder, pReq->vgId) < 0) return -1; + if (tEncodeI32(&encoder, pReq->dnodeId) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->db) < 0) return -1; + if (tEncodeU64(&encoder, pReq->dbUid) < 0) return -1; + if (tEncodeI32(&encoder, pReq->vgVersion) < 0) return -1; + if (tEncodeI32(&encoder, pReq->cacheBlockSize) < 0) return -1; + if (tEncodeI32(&encoder, pReq->totalBlocks) < 0) return -1; + if (tEncodeI32(&encoder, pReq->daysPerFile) < 0) return -1; + if (tEncodeI32(&encoder, pReq->daysToKeep0) < 0) return -1; + if (tEncodeI32(&encoder, pReq->daysToKeep1) < 0) return -1; + if (tEncodeI32(&encoder, pReq->daysToKeep2) < 0) return -1; + if (tEncodeI32(&encoder, pReq->minRows) < 0) return -1; + if (tEncodeI32(&encoder, pReq->maxRows) < 0) return -1; + if (tEncodeI32(&encoder, pReq->commitTime) < 0) return -1; + if (tEncodeI32(&encoder, pReq->fsyncPeriod) < 0) return -1; + if (tEncodeI8(&encoder, pReq->walLevel) < 0) return -1; + if (tEncodeI8(&encoder, pReq->precision) < 0) return -1; + if (tEncodeI8(&encoder, pReq->compression) < 0) return -1; + if (tEncodeI8(&encoder, pReq->quorum) < 0) return -1; + if (tEncodeI8(&encoder, pReq->update) < 0) return -1; + if (tEncodeI8(&encoder, pReq->cacheLastRow) < 0) return -1; + if (tEncodeI8(&encoder, pReq->replica) < 0) return -1; + if (tEncodeI8(&encoder, pReq->selfIndex) < 0) return -1; + for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { + SReplica *pReplica = &pReq->replicas[i]; + if (tEncodeSReplica(&encoder, pReplica) < 0) return -1; + } + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; + if (tDecodeU64(&decoder, &pReq->dbUid) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->vgVersion) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->cacheBlockSize) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->totalBlocks) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->daysPerFile) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->daysToKeep0) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->daysToKeep1) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->daysToKeep2) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->minRows) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->maxRows) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->commitTime) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->fsyncPeriod) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->walLevel) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->precision) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->compression) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->quorum) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->update) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->cacheLastRow) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->replica) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->selfIndex) < 0) return -1; + for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { + SReplica *pReplica = &pReq->replicas[i]; + if (tDecodeSReplica(&decoder, pReplica) < 0) return -1; + } + + tEndDecode(&decoder); + tCoderClear(&decoder); + return 0; +} + +int32_t tSerializeSDropVnodeReq(void *buf, int32_t bufLen, SDropVnodeReq *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI32(&encoder, pReq->vgId) < 0) return -1; + if (tEncodeI32(&encoder, pReq->dnodeId) < 0) return -1; + if (tEncodeU64(&encoder, pReq->dbUid) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->db) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSDropVnodeReq(void *buf, int32_t bufLen, SDropVnodeReq *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->vgId) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -1; + if (tDecodeU64(&decoder, &pReq->dbUid) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->db) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +int32_t tSerializeSKillQueryReq(void *buf, int32_t bufLen, SKillQueryReq *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI32(&encoder, pReq->connId) < 0) return -1; + if (tEncodeI32(&encoder, pReq->queryId) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSKillQueryReq(void *buf, int32_t bufLen, SKillQueryReq *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->connId) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->queryId) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +int32_t tSerializeSKillConnReq(void *buf, int32_t bufLen, SKillConnReq *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI32(&encoder, pReq->connId) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSKillConnReq(void *buf, int32_t bufLen, SKillConnReq *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->connId) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +int32_t tSerializeSDCreateMnodeReq(void *buf, int32_t bufLen, SDCreateMnodeReq *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI32(&encoder, pReq->dnodeId) < 0) return -1; + if (tEncodeI8(&encoder, pReq->replica) < 0) return -1; + for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { + SReplica *pReplica = &pReq->replicas[i]; + if (tEncodeSReplica(&encoder, pReplica) < 0) return -1; + } + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSDCreateMnodeReq(void *buf, int32_t bufLen, SDCreateMnodeReq *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->dnodeId) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->replica) < 0) return -1; + for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { + SReplica *pReplica = &pReq->replicas[i]; + if (tDecodeSReplica(&decoder, pReplica) < 0) return -1; + } + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} + +int32_t tSerializeSAuthReq(void *buf, int32_t bufLen, SAuthReq *pReq) { + SCoder encoder = {0}; + tCoderInit(&encoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_ENCODER); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->user) < 0) return -1; + if (tEncodeI8(&encoder, pReq->spi) < 0) return -1; + if (tEncodeI8(&encoder, pReq->encrypt) < 0) return -1; + if (tEncodeBinary(&encoder, pReq->secret, TSDB_PASSWORD_LEN) < 0) return -1; + if (tEncodeBinary(&encoder, pReq->ckey, TSDB_PASSWORD_LEN) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tCoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSAuthReq(void *buf, int32_t bufLen, SAuthReq *pReq) { + SCoder decoder = {0}; + tCoderInit(&decoder, TD_LITTLE_ENDIAN, buf, bufLen, TD_DECODER); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->spi) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->encrypt) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->secret) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->ckey) < 0) return -1; + tEndDecode(&decoder); + + tCoderClear(&decoder); + return 0; +} diff --git a/source/common/src/tname.c b/source/common/src/tname.c index f8ef9f0979..f3deb84ccf 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -1,3 +1,4 @@ +#include #include "os.h" #include "tutil.h" @@ -268,4 +269,5 @@ SSchema createSchema(uint8_t type, int32_t bytes, int32_t colId, const char* nam tstrncpy(s.name, name, tListLen(s.name)); return s; -} \ No newline at end of file +} + diff --git a/source/common/src/ttszip.c b/source/common/src/ttszip.c index 41eebc5da4..6d57992c35 100644 --- a/source/common/src/ttszip.c +++ b/source/common/src/ttszip.c @@ -344,8 +344,8 @@ STSBlock* readDataFromDisk(STSBuf* pTSBuf, int32_t order, bool decomp) { UNUSED(ret); } - fread(&pBlock->tag.nType, sizeof(pBlock->tag.nType), 1, pTSBuf->f); - fread(&pBlock->tag.nLen, sizeof(pBlock->tag.nLen), 1, pTSBuf->f); + int32_t ret = fread(&pBlock->tag.nType, sizeof(pBlock->tag.nType), 1, pTSBuf->f); + ret = fread(&pBlock->tag.nLen, sizeof(pBlock->tag.nLen), 1, pTSBuf->f); // NOTE: mix types tags are not supported size_t sz = 0; diff --git a/source/common/test/commonTests.cpp b/source/common/test/commonTests.cpp index b91b6b06f2..e9e8d086b3 100644 --- a/source/common/test/commonTests.cpp +++ b/source/common/test/commonTests.cpp @@ -1,11 +1,12 @@ +#include #include +#include #include #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wwrite-strings" #pragma GCC diagnostic ignored "-Wunused-function" #pragma GCC diagnostic ignored "-Wunused-variable" -#pragma GCC diagnostic ignored "-Wunused-but-set-variable" #pragma GCC diagnostic ignored "-Wsign-compare" #include "os.h" @@ -96,4 +97,199 @@ TEST(testCase, toInteger_test) { ASSERT_EQ(ret, -1); } +TEST(testCase, Datablock_test) { + SSDataBlock* b = static_cast(calloc(1, sizeof(SSDataBlock))); + b->info.numOfCols = 2; + b->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); + + SColumnInfoData infoData = {0}; + infoData.info.bytes = 4; + infoData.info.type = TSDB_DATA_TYPE_INT; + infoData.info.colId = 1; + + infoData.pData = (char*) calloc(40, infoData.info.bytes); + infoData.nullbitmap = (char*) calloc(1, sizeof(char) * (40/8)); + taosArrayPush(b->pDataBlock, &infoData); + + SColumnInfoData infoData1 = {0}; + infoData1.info.bytes = 40; + infoData1.info.type = TSDB_DATA_TYPE_BINARY; + infoData1.info.colId = 2; + + infoData1.varmeta.offset = (int32_t*) calloc(40, sizeof(uint32_t)); + taosArrayPush(b->pDataBlock, &infoData1); + + char* str = "the value of: %d"; + char buf[128] = {0}; + char varbuf[128] = {0}; + + for(int32_t i = 0; i < 40; ++i) { + SColumnInfoData* p0 = (SColumnInfoData *) taosArrayGet(b->pDataBlock, 0); + SColumnInfoData* p1 = (SColumnInfoData *) taosArrayGet(b->pDataBlock, 1); + + if (i&0x01) { + int32_t len = sprintf(buf, str, i); + STR_TO_VARSTR(varbuf, buf) + colDataAppend(p0, i, (const char*) &i, false); + colDataAppend(p1, i, (const char*) varbuf, false); + + memset(varbuf, 0, sizeof(varbuf)); + memset(buf, 0, sizeof(buf)); + } else { + colDataAppend(p0, i, (const char*) &i, true); + colDataAppend(p1, i, (const char*) varbuf, true); + } + + b->info.rows++; + } + + SColumnInfoData* p0 = (SColumnInfoData *) taosArrayGet(b->pDataBlock, 0); + SColumnInfoData* p1 = (SColumnInfoData *) taosArrayGet(b->pDataBlock, 1); + for(int32_t i = 0; i < 40; ++i) { + if (i & 0x01) { + ASSERT_EQ(colDataIsNull_f(p0->nullbitmap, i), false); + ASSERT_EQ(colDataIsNull(p1, b->info.rows, i, nullptr), false); + } else { + ASSERT_EQ(colDataIsNull_f(p0->nullbitmap, i), true); + + ASSERT_EQ(colDataIsNull(p0, b->info.rows, i, nullptr), true); + ASSERT_EQ(colDataIsNull(p1, b->info.rows, i, nullptr), true); + } + } + + printf("binary column length:%d\n", *(int32_t*) p1->pData); + + ASSERT_EQ(colDataGetNumOfCols(b), 2); + ASSERT_EQ(colDataGetNumOfRows(b), 40); + + char* pData = colDataGet(p1, 3); + printf("the second row of binary:%s, length:%d\n", (char*)varDataVal(pData), varDataLen(pData)); + + SArray* pOrderInfo = taosArrayInit(3, sizeof(SBlockOrderInfo)); + SBlockOrderInfo order = {.order = TSDB_ORDER_ASC, .colIndex = 0}; + taosArrayPush(pOrderInfo, &order); + + blockDataSort(b, pOrderInfo, true); + blockDataDestroy(b); + + taosArrayDestroy(pOrderInfo); +} + +#if 0 +TEST(testCase, non_var_dataBlock_split_test) { + SSDataBlock* b = static_cast(calloc(1, sizeof(SSDataBlock))); + b->info.numOfCols = 2; + b->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); + + SColumnInfoData infoData = {0}; + infoData.info.bytes = 4; + infoData.info.type = TSDB_DATA_TYPE_INT; + infoData.info.colId = 1; + + int32_t numOfRows = 1000000; + + infoData.pData = (char*) calloc(numOfRows, infoData.info.bytes); + infoData.nullbitmap = (char*) calloc(1, sizeof(char) * (numOfRows/8)); + taosArrayPush(b->pDataBlock, &infoData); + + SColumnInfoData infoData1 = {0}; + infoData1.info.bytes = 1; + infoData1.info.type = TSDB_DATA_TYPE_TINYINT; + infoData1.info.colId = 2; + + infoData1.pData = (char*) calloc(numOfRows, infoData.info.bytes); + infoData1.nullbitmap = (char*) calloc(1, sizeof(char) * (numOfRows/8)); + taosArrayPush(b->pDataBlock, &infoData1); + + for(int32_t i = 0; i < numOfRows; ++i) { + SColumnInfoData* p0 = (SColumnInfoData*)taosArrayGet(b->pDataBlock, 0); + SColumnInfoData* p1 = (SColumnInfoData*)taosArrayGet(b->pDataBlock, 1); + + int8_t v = i; + colDataAppend(p0, i, (const char*)&i, false); + colDataAppend(p1, i, (const char*)&v, false); + b->info.rows++; + } + + int32_t pageSize = 64 * 1024; + + int32_t startIndex= 0; + int32_t stopIndex = 0; + int32_t count = 1; + while(1) { + blockDataSplitRows(b, false, startIndex, &stopIndex, pageSize); + printf("the %d split, from: %d to %d\n", count++, startIndex, stopIndex); + + if (stopIndex == numOfRows - 1) { + break; + } + + startIndex = stopIndex + 1; + } + +} + +#endif + +TEST(testCase, var_dataBlock_split_test) { + SSDataBlock* b = static_cast(calloc(1, sizeof(SSDataBlock))); + b->info.numOfCols = 2; + b->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); + + int32_t numOfRows = 1000000; + + SColumnInfoData infoData = {0}; + infoData.info.bytes = 4; + infoData.info.type = TSDB_DATA_TYPE_INT; + infoData.info.colId = 1; + + infoData.pData = (char*) calloc(numOfRows, infoData.info.bytes); + infoData.nullbitmap = (char*) calloc(1, sizeof(char) * (numOfRows/8)); + taosArrayPush(b->pDataBlock, &infoData); + + SColumnInfoData infoData1 = {0}; + infoData1.info.bytes = 40; + infoData1.info.type = TSDB_DATA_TYPE_BINARY; + infoData1.info.colId = 2; + + infoData1.varmeta.offset = (int32_t*) calloc(numOfRows, sizeof(uint32_t)); + taosArrayPush(b->pDataBlock, &infoData1); + + char buf[41] = {0}; + char buf1[100] = {0}; + + for(int32_t i = 0; i < numOfRows; ++i) { + SColumnInfoData* p0 = (SColumnInfoData*)taosArrayGet(b->pDataBlock, 0); + SColumnInfoData* p1 = (SColumnInfoData*)taosArrayGet(b->pDataBlock, 1); + + int8_t v = i; + colDataAppend(p0, i, (const char*)&i, false); + + sprintf(buf, "the number of row:%d", i); + int32_t len = sprintf(buf1, buf, i); + STR_TO_VARSTR(buf1, buf) + colDataAppend(p1, i, buf1, false); + b->info.rows++; + + memset(buf, 0, sizeof(buf)); + memset(buf1, 0, sizeof(buf1)); + } + + int32_t pageSize = 64 * 1024; + + int32_t startIndex= 0; + int32_t stopIndex = 0; + int32_t count = 1; + while(1) { + blockDataSplitRows(b, true, startIndex, &stopIndex, pageSize); + printf("the %d split, from: %d to %d\n", count++, startIndex, stopIndex); + + if (stopIndex == numOfRows - 1) { + break; + } + + startIndex = stopIndex + 1; + } +} + #pragma GCC diagnostic pop \ No newline at end of file diff --git a/source/dnode/mgmt/impl/src/dndBnode.c b/source/dnode/mgmt/impl/src/dndBnode.c index f26ec72f1d..e37a164660 100644 --- a/source/dnode/mgmt/impl/src/dndBnode.c +++ b/source/dnode/mgmt/impl/src/dndBnode.c @@ -258,11 +258,14 @@ static int32_t dndDropBnode(SDnode *pDnode) { return 0; } -int32_t dndProcessCreateBnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { - SDCreateBnodeReq *pMsg = pRpcMsg->pCont; - pMsg->dnodeId = htonl(pMsg->dnodeId); +int32_t dndProcessCreateBnodeReq(SDnode *pDnode, SRpcMsg *pReq) { + SDCreateBnodeReq createReq = {0}; + if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } - if (pMsg->dnodeId != dndGetDnodeId(pDnode)) { + if (createReq.dnodeId != dndGetDnodeId(pDnode)) { terrno = TSDB_CODE_DND_BNODE_INVALID_OPTION; dError("failed to create bnode since %s", terrstr()); return -1; @@ -271,11 +274,14 @@ int32_t dndProcessCreateBnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { } } -int32_t dndProcessDropBnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { - SDDropBnodeReq *pMsg = pRpcMsg->pCont; - pMsg->dnodeId = htonl(pMsg->dnodeId); +int32_t dndProcessDropBnodeReq(SDnode *pDnode, SRpcMsg *pReq) { + SDDropBnodeReq dropReq = {0}; + if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } - if (pMsg->dnodeId != dndGetDnodeId(pDnode)) { + if (dropReq.dnodeId != dndGetDnodeId(pDnode)) { terrno = TSDB_CODE_DND_BNODE_INVALID_OPTION; dError("failed to drop bnode since %s", terrstr()); return -1; diff --git a/source/dnode/mgmt/impl/src/dndMgmt.c b/source/dnode/mgmt/impl/src/dndMgmt.c index 41dd54a0ff..3d149a3e60 100644 --- a/source/dnode/mgmt/impl/src/dndMgmt.c +++ b/source/dnode/mgmt/impl/src/dndMgmt.c @@ -379,10 +379,9 @@ void dndSendStatusReq(SDnode *pDnode) { req.pVloads = taosArrayInit(TSDB_MAX_VNODES, sizeof(SVnodeLoad)); dndGetVnodeLoads(pDnode, req.pVloads); - int32_t contLen = tSerializeSStatusReq(NULL, &req); + int32_t contLen = tSerializeSStatusReq(NULL, 0, &req); void *pHead = rpcMallocCont(contLen); - void *pBuf = pHead; - tSerializeSStatusReq(&pBuf, &req); + tSerializeSStatusReq(pHead, contLen, &req); taosArrayDestroy(req.pVloads); SRpcMsg rpcMsg = {.pCont = pHead, .contLen = contLen, .msgType = TDMT_MND_STATUS, .ahandle = (void *)9527}; @@ -395,7 +394,7 @@ void dndSendStatusReq(SDnode *pDnode) { static void dndUpdateDnodeCfg(SDnode *pDnode, SDnodeCfg *pCfg) { SDnodeMgmt *pMgmt = &pDnode->dmgmt; if (pMgmt->dnodeId == 0) { - dInfo("set dnodeId:%d clusterId:0x%" PRId64, pCfg->dnodeId, pCfg->clusterId); + dInfo("set dnodeId:%d clusterId:%" PRId64, pCfg->dnodeId, pCfg->clusterId); taosWLockLatch(&pMgmt->latch); pMgmt->dnodeId = pCfg->dnodeId; pMgmt->clusterId = pCfg->clusterId; @@ -437,7 +436,8 @@ static void dndProcessStatusRsp(SDnode *pDnode, SRpcMsg *pRsp) { } } else { SStatusRsp statusRsp = {0}; - if (pRsp->pCont != NULL && pRsp->contLen != 0 && tDeserializeSStatusRsp(pRsp->pCont, &statusRsp) != NULL) { + if (pRsp->pCont != NULL && pRsp->contLen != 0 && + tDeserializeSStatusRsp(pRsp->pCont, pRsp->contLen, &statusRsp) == 0) { pMgmt->dver = statusRsp.dver; dndUpdateDnodeCfg(pDnode, &statusRsp.dnodeCfg); dndUpdateDnodeEps(pDnode, statusRsp.pDnodeEps); @@ -652,9 +652,6 @@ static void dndProcessMgmtQueue(SDnode *pDnode, SRpcMsg *pMsg) { case TDMT_DND_DROP_VNODE: code = dndProcessDropVnodeReq(pDnode, pMsg); break; - case TDMT_DND_AUTH_VNODE: - code = dndProcessAuthVnodeReq(pDnode, pMsg); - break; case TDMT_DND_SYNC_VNODE: code = dndProcessSyncVnodeReq(pDnode, pMsg); break; diff --git a/source/dnode/mgmt/impl/src/dndMnode.c b/source/dnode/mgmt/impl/src/dndMnode.c index 0874c633bf..c6db75c057 100644 --- a/source/dnode/mgmt/impl/src/dndMnode.c +++ b/source/dnode/mgmt/impl/src/dndMnode.c @@ -424,28 +424,22 @@ static int32_t dndDropMnode(SDnode *pDnode) { return 0; } -static SDCreateMnodeReq *dndParseCreateMnodeReq(SRpcMsg *pReq) { - SDCreateMnodeReq *pCreate = pReq->pCont; - pCreate->dnodeId = htonl(pCreate->dnodeId); - for (int32_t i = 0; i < pCreate->replica; ++i) { - pCreate->replicas[i].id = htonl(pCreate->replicas[i].id); - pCreate->replicas[i].port = htons(pCreate->replicas[i].port); - } - - return pCreate; -} int32_t dndProcessCreateMnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - SDCreateMnodeReq *pCreate = dndParseCreateMnodeReq(pReq); + SDCreateMnodeReq createReq = {0}; + if (tDeserializeSDCreateMnodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } - if (pCreate->replica <= 1 || pCreate->dnodeId != dndGetDnodeId(pDnode)) { + if (createReq.replica <= 1 || createReq.dnodeId != dndGetDnodeId(pDnode)) { terrno = TSDB_CODE_DND_MNODE_INVALID_OPTION; dError("failed to create mnode since %s", terrstr()); return -1; } SMnodeOpt option = {0}; - if (dndBuildMnodeOptionFromReq(pDnode, &option, pCreate) != 0) { + if (dndBuildMnodeOptionFromReq(pDnode, &option, &createReq) != 0) { terrno = TSDB_CODE_DND_MNODE_INVALID_OPTION; dError("failed to create mnode since %s", terrstr()); return -1; @@ -464,16 +458,20 @@ int32_t dndProcessCreateMnodeReq(SDnode *pDnode, SRpcMsg *pReq) { } int32_t dndProcessAlterMnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - SDAlterMnodeReq *pAlter = dndParseCreateMnodeReq(pReq); + SDAlterMnodeReq alterReq = {0}; + if (tDeserializeSDCreateMnodeReq(pReq->pCont, pReq->contLen, &alterReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } - if (pAlter->dnodeId != dndGetDnodeId(pDnode)) { + if (alterReq.dnodeId != dndGetDnodeId(pDnode)) { terrno = TSDB_CODE_DND_MNODE_INVALID_OPTION; dError("failed to alter mnode since %s", terrstr()); return -1; } SMnodeOpt option = {0}; - if (dndBuildMnodeOptionFromReq(pDnode, &option, pAlter) != 0) { + if (dndBuildMnodeOptionFromReq(pDnode, &option, &alterReq) != 0) { terrno = TSDB_CODE_DND_MNODE_INVALID_OPTION; dError("failed to alter mnode since %s", terrstr()); return -1; @@ -494,10 +492,13 @@ int32_t dndProcessAlterMnodeReq(SDnode *pDnode, SRpcMsg *pReq) { } int32_t dndProcessDropMnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - SDDropMnodeReq *pDrop = pReq->pCont; - pDrop->dnodeId = htonl(pDrop->dnodeId); + SDDropMnodeReq dropReq = {0}; + if (tDeserializeSMCreateDropMnodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } - if (pDrop->dnodeId != dndGetDnodeId(pDnode)) { + if (dropReq.dnodeId != dndGetDnodeId(pDnode)) { terrno = TSDB_CODE_DND_MNODE_INVALID_OPTION; dError("failed to drop mnode since %s", terrstr()); return -1; diff --git a/source/dnode/mgmt/impl/src/dndQnode.c b/source/dnode/mgmt/impl/src/dndQnode.c index fa53375381..64545ec09f 100644 --- a/source/dnode/mgmt/impl/src/dndQnode.c +++ b/source/dnode/mgmt/impl/src/dndQnode.c @@ -264,11 +264,14 @@ static int32_t dndDropQnode(SDnode *pDnode) { return 0; } -int32_t dndProcessCreateQnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { - SDCreateQnodeReq *pMsg = pRpcMsg->pCont; - pMsg->dnodeId = htonl(pMsg->dnodeId); +int32_t dndProcessCreateQnodeReq(SDnode *pDnode, SRpcMsg *pReq) { + SDCreateQnodeReq createReq = {0}; + if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } - if (pMsg->dnodeId != dndGetDnodeId(pDnode)) { + if (createReq.dnodeId != dndGetDnodeId(pDnode)) { terrno = TSDB_CODE_DND_QNODE_INVALID_OPTION; dError("failed to create qnode since %s", terrstr()); return -1; @@ -277,11 +280,14 @@ int32_t dndProcessCreateQnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { } } -int32_t dndProcessDropQnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { - SDDropQnodeReq *pMsg = pRpcMsg->pCont; - pMsg->dnodeId = htonl(pMsg->dnodeId); +int32_t dndProcessDropQnodeReq(SDnode *pDnode, SRpcMsg *pReq) { + SDDropQnodeReq dropReq = {0}; + if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } - if (pMsg->dnodeId != dndGetDnodeId(pDnode)) { + if (dropReq.dnodeId != dndGetDnodeId(pDnode)) { terrno = TSDB_CODE_DND_QNODE_INVALID_OPTION; dError("failed to drop qnode since %s", terrstr()); return -1; diff --git a/source/dnode/mgmt/impl/src/dndSnode.c b/source/dnode/mgmt/impl/src/dndSnode.c index 6f22e5b00c..77686a6027 100644 --- a/source/dnode/mgmt/impl/src/dndSnode.c +++ b/source/dnode/mgmt/impl/src/dndSnode.c @@ -258,11 +258,14 @@ static int32_t dndDropSnode(SDnode *pDnode) { return 0; } -int32_t dndProcessCreateSnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { - SDCreateSnodeReq *pMsg = pRpcMsg->pCont; - pMsg->dnodeId = htonl(pMsg->dnodeId); +int32_t dndProcessCreateSnodeReq(SDnode *pDnode, SRpcMsg *pReq) { + SDCreateSnodeReq createReq = {0}; + if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } - if (pMsg->dnodeId != dndGetDnodeId(pDnode)) { + if (createReq.dnodeId != dndGetDnodeId(pDnode)) { terrno = TSDB_CODE_DND_SNODE_INVALID_OPTION; dError("failed to create snode since %s", terrstr()); return -1; @@ -271,11 +274,14 @@ int32_t dndProcessCreateSnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { } } -int32_t dndProcessDropSnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) { - SDDropSnodeReq *pMsg = pRpcMsg->pCont; - pMsg->dnodeId = htonl(pMsg->dnodeId); +int32_t dndProcessDropSnodeReq(SDnode *pDnode, SRpcMsg *pReq) { + SDDropSnodeReq dropReq = {0}; + if (tDeserializeSMCreateDropQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } - if (pMsg->dnodeId != dndGetDnodeId(pDnode)) { + if (dropReq.dnodeId != dndGetDnodeId(pDnode)) { terrno = TSDB_CODE_DND_SNODE_INVALID_OPTION; dError("failed to drop snode since %s", terrstr()); return -1; diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index 239fe9ca4d..931cda475c 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -57,8 +57,6 @@ static void dndInitMsgFp(STransMgmt *pMgmt) { pMgmt->msgFp[TMSG_INDEX(TDMT_DND_DROP_VNODE_RSP)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_DND_SYNC_VNODE)] = dndProcessMgmtMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_DND_SYNC_VNODE_RSP)] = dndProcessMnodeWriteMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_AUTH_VNODE)] = dndProcessMgmtMsg; - pMgmt->msgFp[TMSG_INDEX(TDMT_DND_AUTH_VNODE_RSP)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_DND_COMPACT_VNODE)] = dndProcessMgmtMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_DND_COMPACT_VNODE_RSP)] = dndProcessMnodeWriteMsg; pMgmt->msgFp[TMSG_INDEX(TDMT_DND_CONFIG_DNODE)] = dndProcessMgmtMsg; @@ -310,24 +308,29 @@ static int32_t dndRetrieveUserAuthInfo(void *parent, char *user, char *spi, char return -1; } - SAuthReq *pReq = rpcMallocCont(sizeof(SAuthReq)); - tstrncpy(pReq->user, user, TSDB_USER_LEN); + SAuthReq authReq = {0}; + tstrncpy(authReq.user, user, TSDB_USER_LEN); + int32_t contLen = tSerializeSAuthReq(NULL, 0, &authReq); + void *pReq = rpcMallocCont(contLen); + tSerializeSAuthReq(pReq, contLen, &authReq); - SRpcMsg rpcMsg = {.pCont = pReq, .contLen = sizeof(SAuthReq), .msgType = TDMT_MND_AUTH, .ahandle = (void *)9528}; + SRpcMsg rpcMsg = {.pCont = pReq, .contLen = contLen, .msgType = TDMT_MND_AUTH, .ahandle = (void *)9528}; SRpcMsg rpcRsp = {0}; - dTrace("user:%s, send user auth req to other mnodes, spi:%d encrypt:%d", user, pReq->spi, pReq->encrypt); + dTrace("user:%s, send user auth req to other mnodes, spi:%d encrypt:%d", user, authReq.spi, authReq.encrypt); dndSendMsgToMnodeRecv(pDnode, &rpcMsg, &rpcRsp); if (rpcRsp.code != 0) { terrno = rpcRsp.code; dError("user:%s, failed to get user auth from other mnodes since %s", user, terrstr()); } else { - SAuthRsp *pRsp = rpcRsp.pCont; - memcpy(secret, pRsp->secret, TSDB_PASSWORD_LEN); - memcpy(ckey, pRsp->ckey, TSDB_PASSWORD_LEN); - *spi = pRsp->spi; - *encrypt = pRsp->encrypt; - dTrace("user:%s, success to get user auth from other mnodes, spi:%d encrypt:%d", user, pRsp->spi, pRsp->encrypt); + SAuthRsp authRsp = {0}; + tDeserializeSAuthReq(rpcRsp.pCont, rpcRsp.contLen, &authRsp); + memcpy(secret, authRsp.secret, TSDB_PASSWORD_LEN); + memcpy(ckey, authRsp.ckey, TSDB_PASSWORD_LEN); + *spi = authRsp.spi; + *encrypt = authRsp.encrypt; + dTrace("user:%s, success to get user auth from other mnodes, spi:%d encrypt:%d", user, authRsp.spi, + authRsp.encrypt); } rpcFreeCont(rpcRsp.pCont); diff --git a/source/dnode/mgmt/impl/src/dndVnodes.c b/source/dnode/mgmt/impl/src/dndVnodes.c index 3eeb28e532..ebb2d1b4f0 100644 --- a/source/dnode/mgmt/impl/src/dndVnodes.c +++ b/source/dnode/mgmt/impl/src/dndVnodes.c @@ -15,8 +15,8 @@ #define _DEFAULT_SOURCE #include "dndVnodes.h" -#include "dndTransport.h" #include "dndMgmt.h" +#include "dndTransport.h" typedef struct { int32_t vgId; @@ -34,9 +34,9 @@ typedef struct { int8_t dropped; int8_t accessState; uint64_t dbUid; - char * db; - char * path; - SVnode * pImpl; + char *db; + char *path; + SVnode *pImpl; STaosQueue *pWriteQ; STaosQueue *pSyncQ; STaosQueue *pApplyQ; @@ -50,7 +50,7 @@ typedef struct { int32_t failed; int32_t threadIndex; pthread_t thread; - SDnode * pDnode; + SDnode *pDnode; SWrapperCfg *pCfgs; } SVnodeThread; @@ -68,7 +68,7 @@ void dndProcessVnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pE void dndProcessVnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet); static int32_t dndPutMsgIntoVnodeApplyQueue(SDnode *pDnode, int32_t vgId, SRpcMsg *pMsg); -static SVnodeObj * dndAcquireVnode(SDnode *pDnode, int32_t vgId); +static SVnodeObj *dndAcquireVnode(SDnode *pDnode, int32_t vgId); static void dndReleaseVnode(SDnode *pDnode, SVnodeObj *pVnode); static int32_t dndOpenVnode(SDnode *pDnode, SWrapperCfg *pCfg, SVnode *pImpl); static void dndCloseVnode(SDnode *pDnode, SVnodeObj *pVnode); @@ -81,7 +81,7 @@ static void dndCloseVnodes(SDnode *pDnode); static SVnodeObj *dndAcquireVnode(SDnode *pDnode, int32_t vgId) { SVnodesMgmt *pMgmt = &pDnode->vmgmt; - SVnodeObj * pVnode = NULL; + SVnodeObj *pVnode = NULL; int32_t refCount = 0; taosRLockLatch(&pMgmt->latch); @@ -112,7 +112,7 @@ static void dndReleaseVnode(SDnode *pDnode, SVnodeObj *pVnode) { static int32_t dndOpenVnode(SDnode *pDnode, SWrapperCfg *pCfg, SVnode *pImpl) { SVnodesMgmt *pMgmt = &pDnode->vmgmt; - SVnodeObj * pVnode = calloc(1, sizeof(SVnodeObj)); + SVnodeObj *pVnode = calloc(1, sizeof(SVnodeObj)); if (pVnode == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -189,7 +189,7 @@ static SVnodeObj **dndGetVnodesFromHash(SDnode *pDnode, int32_t *numOfVnodes) { void *pIter = taosHashIterate(pMgmt->hash, NULL); while (pIter) { SVnodeObj **ppVnode = pIter; - SVnodeObj * pVnode = *ppVnode; + SVnodeObj *pVnode = *ppVnode; if (pVnode && num < size) { int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1); dTrace("vgId:%d, acquire vnode, refCount:%d", pVnode->vgId, refCount); @@ -211,9 +211,9 @@ static int32_t dndGetVnodesFromFile(SDnode *pDnode, SWrapperCfg **ppCfgs, int32_ int32_t code = TSDB_CODE_DND_VNODE_READ_FILE_ERROR; int32_t len = 0; int32_t maxLen = 30000; - char * content = calloc(1, maxLen + 1); - cJSON * root = NULL; - FILE * fp = NULL; + char *content = calloc(1, maxLen + 1); + cJSON *root = NULL; + FILE *fp = NULL; char file[PATH_MAX + 20] = {0}; SWrapperCfg *pCfgs = NULL; @@ -254,7 +254,7 @@ static int32_t dndGetVnodesFromFile(SDnode *pDnode, SWrapperCfg **ppCfgs, int32_ } for (int32_t i = 0; i < vnodesNum; ++i) { - cJSON * vnode = cJSON_GetArrayItem(vnodes, i); + cJSON *vnode = cJSON_GetArrayItem(vnodes, i); SWrapperCfg *pCfg = &pCfgs[i]; cJSON *vgId = cJSON_GetObjectItem(vnode, "vgId"); @@ -326,7 +326,7 @@ static int32_t dndWriteVnodesToFile(SDnode *pDnode) { int32_t len = 0; int32_t maxLen = 65536; - char * content = calloc(1, maxLen + 1); + char *content = calloc(1, maxLen + 1); len += snprintf(content + len, maxLen - len, "{\n"); len += snprintf(content + len, maxLen - len, " \"vnodes\": [\n"); @@ -368,8 +368,8 @@ static int32_t dndWriteVnodesToFile(SDnode *pDnode) { static void *dnodeOpenVnodeFunc(void *param) { SVnodeThread *pThread = param; - SDnode * pDnode = pThread->pDnode; - SVnodesMgmt * pMgmt = &pDnode->vmgmt; + SDnode *pDnode = pThread->pDnode; + SVnodesMgmt *pMgmt = &pDnode->vmgmt; dDebug("thread:%d, start to open %d vnodes", pThread->threadIndex, pThread->vnodeNum); setThreadName("open-vnodes"); @@ -383,7 +383,7 @@ static void *dnodeOpenVnodeFunc(void *param) { dndReportStartup(pDnode, "open-vnodes", stepDesc); SVnodeCfg cfg = {.pDnode = pDnode, .pTfs = pDnode->pTfs, .vgId = pCfg->vgId, .dbId = pCfg->dbUid}; - SVnode * pImpl = vnodeOpen(pCfg->path, &cfg); + SVnode *pImpl = vnodeOpen(pCfg->path, &cfg); if (pImpl == NULL) { dError("vgId:%d, failed to open vnode by thread:%d", pCfg->vgId, pThread->threadIndex); pThread->failed++; @@ -499,31 +499,6 @@ static void dndCloseVnodes(SDnode *pDnode) { dInfo("total vnodes:%d are all closed", numOfVnodes); } -static SCreateVnodeReq *dndParseCreateVnodeReq(SRpcMsg *pReq) { - SCreateVnodeReq *pCreate = pReq->pCont; - pCreate->vgId = htonl(pCreate->vgId); - pCreate->dnodeId = htonl(pCreate->dnodeId); - pCreate->dbUid = htobe64(pCreate->dbUid); - pCreate->vgVersion = htonl(pCreate->vgVersion); - pCreate->cacheBlockSize = htonl(pCreate->cacheBlockSize); - pCreate->totalBlocks = htonl(pCreate->totalBlocks); - pCreate->daysPerFile = htonl(pCreate->daysPerFile); - pCreate->daysToKeep0 = htonl(pCreate->daysToKeep0); - pCreate->daysToKeep1 = htonl(pCreate->daysToKeep1); - pCreate->daysToKeep2 = htonl(pCreate->daysToKeep2); - pCreate->minRows = htonl(pCreate->minRows); - pCreate->maxRows = htonl(pCreate->maxRows); - pCreate->commitTime = htonl(pCreate->commitTime); - pCreate->fsyncPeriod = htonl(pCreate->fsyncPeriod); - for (int r = 0; r < pCreate->replica; ++r) { - SReplica *pReplica = &pCreate->replicas[r]; - pReplica->id = htonl(pReplica->id); - pReplica->port = htons(pReplica->port); - } - - return pCreate; -} - static void dndGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { pCfg->vgId = pCreate->vgId; pCfg->wsize = pCreate->cacheBlockSize; @@ -556,37 +531,30 @@ static void dndGenerateWrapperCfg(SDnode *pDnode, SCreateVnodeReq *pCreate, SWra pCfg->vgVersion = pCreate->vgVersion; } -static SDropVnodeReq *dndParseDropVnodeReq(SRpcMsg *pReq) { - SDropVnodeReq *pDrop = pReq->pCont; - pDrop->vgId = htonl(pDrop->vgId); - return pDrop; -} - -static SAuthVnodeReq *dndParseAuthVnodeReq(SRpcMsg *pReq) { - SAuthVnodeReq *pAuth = pReq->pCont; - pAuth->vgId = htonl(pAuth->vgId); - return pAuth; -} - int32_t dndProcessCreateVnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - SCreateVnodeReq *pCreate = dndParseCreateVnodeReq(pReq); - dDebug("vgId:%d, create vnode req is received", pCreate->vgId); - - SVnodeCfg vnodeCfg = {0}; - dndGenerateVnodeCfg(pCreate, &vnodeCfg); - - SWrapperCfg wrapperCfg = {0}; - dndGenerateWrapperCfg(pDnode, pCreate, &wrapperCfg); - - if (pCreate->dnodeId != dndGetDnodeId(pDnode)) { - terrno = TSDB_CODE_DND_VNODE_INVALID_OPTION; - dDebug("vgId:%d, failed to create vnode since %s", pCreate->vgId, terrstr()); + SCreateVnodeReq createReq = {0}; + if (tDeserializeSCreateVnodeReq(pReq->pCont, pReq->contLen, &createReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; return -1; } - SVnodeObj *pVnode = dndAcquireVnode(pDnode, pCreate->vgId); + dDebug("vgId:%d, create vnode req is received", createReq.vgId); + + SVnodeCfg vnodeCfg = {0}; + dndGenerateVnodeCfg(&createReq, &vnodeCfg); + + SWrapperCfg wrapperCfg = {0}; + dndGenerateWrapperCfg(pDnode, &createReq, &wrapperCfg); + + if (createReq.dnodeId != dndGetDnodeId(pDnode)) { + terrno = TSDB_CODE_DND_VNODE_INVALID_OPTION; + dDebug("vgId:%d, failed to create vnode since %s", createReq.vgId, terrstr()); + return -1; + } + + SVnodeObj *pVnode = dndAcquireVnode(pDnode, createReq.vgId); if (pVnode != NULL) { - dDebug("vgId:%d, already exist", pCreate->vgId); + dDebug("vgId:%d, already exist", createReq.vgId); dndReleaseVnode(pDnode, pVnode); terrno = TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED; return -1; @@ -597,13 +565,13 @@ int32_t dndProcessCreateVnodeReq(SDnode *pDnode, SRpcMsg *pReq) { vnodeCfg.dbId = wrapperCfg.dbUid; SVnode *pImpl = vnodeOpen(wrapperCfg.path, &vnodeCfg); if (pImpl == NULL) { - dError("vgId:%d, failed to create vnode since %s", pCreate->vgId, terrstr()); + dError("vgId:%d, failed to create vnode since %s", createReq.vgId, terrstr()); return -1; } int32_t code = dndOpenVnode(pDnode, &wrapperCfg, pImpl); if (code != 0) { - dError("vgId:%d, failed to open vnode since %s", pCreate->vgId, terrstr()); + dError("vgId:%d, failed to open vnode since %s", createReq.vgId, terrstr()); vnodeClose(pImpl); vnodeDestroy(wrapperCfg.path); terrno = code; @@ -622,32 +590,37 @@ int32_t dndProcessCreateVnodeReq(SDnode *pDnode, SRpcMsg *pReq) { } int32_t dndProcessAlterVnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - SAlterVnodeReq *pAlter = (SAlterVnodeReq *)dndParseCreateVnodeReq(pReq); - dDebug("vgId:%d, alter vnode req is received", pAlter->vgId); - - SVnodeCfg vnodeCfg = {0}; - dndGenerateVnodeCfg(pAlter, &vnodeCfg); - - SVnodeObj *pVnode = dndAcquireVnode(pDnode, pAlter->vgId); - if (pVnode == NULL) { - dDebug("vgId:%d, failed to alter vnode since %s", pAlter->vgId, terrstr()); + SAlterVnodeReq alterReq = {0}; + if (tDeserializeSCreateVnodeReq(pReq->pCont, pReq->contLen, &alterReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; return -1; } - if (pAlter->vgVersion == pVnode->vgVersion) { + dDebug("vgId:%d, alter vnode req is received", alterReq.vgId); + + SVnodeCfg vnodeCfg = {0}; + dndGenerateVnodeCfg(&alterReq, &vnodeCfg); + + SVnodeObj *pVnode = dndAcquireVnode(pDnode, alterReq.vgId); + if (pVnode == NULL) { + dDebug("vgId:%d, failed to alter vnode since %s", alterReq.vgId, terrstr()); + return -1; + } + + if (alterReq.vgVersion == pVnode->vgVersion) { dndReleaseVnode(pDnode, pVnode); - dDebug("vgId:%d, no need to alter vnode cfg for version unchanged ", pAlter->vgId); + dDebug("vgId:%d, no need to alter vnode cfg for version unchanged ", alterReq.vgId); return 0; } if (vnodeAlter(pVnode->pImpl, &vnodeCfg) != 0) { - dError("vgId:%d, failed to alter vnode since %s", pAlter->vgId, terrstr()); + dError("vgId:%d, failed to alter vnode since %s", alterReq.vgId, terrstr()); dndReleaseVnode(pDnode, pVnode); return -1; } int32_t oldVersion = pVnode->vgVersion; - pVnode->vgVersion = pAlter->vgVersion; + pVnode->vgVersion = alterReq.vgVersion; int32_t code = dndWriteVnodesToFile(pDnode); if (code != 0) { pVnode->vgVersion = oldVersion; @@ -658,9 +631,13 @@ int32_t dndProcessAlterVnodeReq(SDnode *pDnode, SRpcMsg *pReq) { } int32_t dndProcessDropVnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - SDropVnodeReq *pDrop = dndParseDropVnodeReq(pReq); + SDropVnodeReq dropReq = {0}; + if (tDeserializeSDropVnodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } - int32_t vgId = pDrop->vgId; + int32_t vgId = dropReq.vgId; dDebug("vgId:%d, drop vnode req is received", vgId); SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId); @@ -683,27 +660,11 @@ int32_t dndProcessDropVnodeReq(SDnode *pDnode, SRpcMsg *pReq) { return 0; } -int32_t dndProcessAuthVnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - SAuthVnodeReq *pAuth = (SAuthVnodeReq *)dndParseAuthVnodeReq(pReq); - - int32_t vgId = pAuth->vgId; - dDebug("vgId:%d, auth vnode req is received", vgId); - - SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId); - if (pVnode == NULL) { - dDebug("vgId:%d, failed to auth since %s", vgId, terrstr()); - return -1; - } - - pVnode->accessState = pAuth->accessState; - dndReleaseVnode(pDnode, pVnode); - return 0; -} - int32_t dndProcessSyncVnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - SSyncVnodeReq *pSync = (SSyncVnodeReq *)dndParseDropVnodeReq(pReq); + SSyncVnodeReq syncReq = {0}; + tDeserializeSDropVnodeReq(pReq->pCont, pReq->contLen, &syncReq); - int32_t vgId = pSync->vgId; + int32_t vgId = syncReq.vgId; dDebug("vgId:%d, sync vnode req is received", vgId); SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId); @@ -723,9 +684,10 @@ int32_t dndProcessSyncVnodeReq(SDnode *pDnode, SRpcMsg *pReq) { } int32_t dndProcessCompactVnodeReq(SDnode *pDnode, SRpcMsg *pReq) { - SCompactVnodeReq *pCompact = (SCompactVnodeReq *)dndParseDropVnodeReq(pReq); + SCompactVnodeReq compatcReq = {0}; + tDeserializeSDropVnodeReq(pReq->pCont, pReq->contLen, &compatcReq); - int32_t vgId = pCompact->vgId; + int32_t vgId = compatcReq.vgId; dDebug("vgId:%d, compact vnode req is received", vgId); SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId); diff --git a/source/dnode/mgmt/impl/test/bnode/dbnode.cpp b/source/dnode/mgmt/impl/test/bnode/dbnode.cpp index 398d530648..75f111587f 100644 --- a/source/dnode/mgmt/impl/test/bnode/dbnode.cpp +++ b/source/dnode/mgmt/impl/test/bnode/dbnode.cpp @@ -27,10 +27,12 @@ Testbase DndTestBnode::test; TEST_F(DndTestBnode, 01_Create_Bnode) { { - int32_t contLen = sizeof(SDCreateBnodeReq); + SDCreateBnodeReq createReq = {0}; + createReq.dnodeId = 2; - SDCreateBnodeReq* pReq = (SDCreateBnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_BNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -38,10 +40,12 @@ TEST_F(DndTestBnode, 01_Create_Bnode) { } { - int32_t contLen = sizeof(SDCreateBnodeReq); + SDCreateBnodeReq createReq = {0}; + createReq.dnodeId = 1; - SDCreateBnodeReq* pReq = (SDCreateBnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(1); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_BNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -49,10 +53,12 @@ TEST_F(DndTestBnode, 01_Create_Bnode) { } { - int32_t contLen = sizeof(SDCreateBnodeReq); + SDCreateBnodeReq createReq = {0}; + createReq.dnodeId = 1; - SDCreateBnodeReq* pReq = (SDCreateBnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(1); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_BNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -62,11 +68,12 @@ TEST_F(DndTestBnode, 01_Create_Bnode) { test.Restart(); { - int32_t contLen = sizeof(SDCreateBnodeReq); - - SDCreateBnodeReq* pReq = (SDCreateBnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(1); + SDCreateBnodeReq createReq = {0}; + createReq.dnodeId = 1; + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_BNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_DND_BNODE_ALREADY_DEPLOYED); @@ -75,10 +82,12 @@ TEST_F(DndTestBnode, 01_Create_Bnode) { TEST_F(DndTestBnode, 01_Drop_Bnode) { { - int32_t contLen = sizeof(SDDropBnodeReq); + SDDropBnodeReq dropReq = {0}; + dropReq.dnodeId = 2; - SDDropBnodeReq* pReq = (SDDropBnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_BNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -86,10 +95,12 @@ TEST_F(DndTestBnode, 01_Drop_Bnode) { } { - int32_t contLen = sizeof(SDDropBnodeReq); + SDDropBnodeReq dropReq = {0}; + dropReq.dnodeId = 1; - SDDropBnodeReq* pReq = (SDDropBnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(1); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_BNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -97,10 +108,12 @@ TEST_F(DndTestBnode, 01_Drop_Bnode) { } { - int32_t contLen = sizeof(SDDropBnodeReq); + SDDropBnodeReq dropReq = {0}; + dropReq.dnodeId = 1; - SDDropBnodeReq* pReq = (SDDropBnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(1); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_BNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -110,10 +123,12 @@ TEST_F(DndTestBnode, 01_Drop_Bnode) { test.Restart(); { - int32_t contLen = sizeof(SDDropBnodeReq); + SDDropBnodeReq dropReq = {0}; + dropReq.dnodeId = 1; - SDDropBnodeReq* pReq = (SDDropBnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(1); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_BNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -121,10 +136,12 @@ TEST_F(DndTestBnode, 01_Drop_Bnode) { } { - int32_t contLen = sizeof(SDCreateBnodeReq); + SDCreateBnodeReq createReq = {0}; + createReq.dnodeId = 1; - SDCreateBnodeReq* pReq = (SDCreateBnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(1); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_BNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); diff --git a/source/dnode/mgmt/impl/test/mnode/dmnode.cpp b/source/dnode/mgmt/impl/test/mnode/dmnode.cpp index edaecef49e..8655bcb774 100644 --- a/source/dnode/mgmt/impl/test/mnode/dmnode.cpp +++ b/source/dnode/mgmt/impl/test/mnode/dmnode.cpp @@ -16,25 +16,27 @@ class DndTestMnode : public ::testing::Test { static void SetUpTestSuite() { test.Init("/tmp/dnode_test_mnode", 9114); } static void TearDownTestSuite() { test.Cleanup(); } - static Testbase test; + static Testbase test; public: void SetUp() override {} void TearDown() override {} }; -Testbase DndTestMnode::test; +Testbase DndTestMnode::test; TEST_F(DndTestMnode, 01_Create_Mnode) { { - int32_t contLen = sizeof(SDCreateMnodeReq); + SDCreateMnodeReq createReq = {0}; + createReq.dnodeId = 2; + createReq.replica = 1; + createReq.replicas[0].id = 1; + createReq.replicas[0].port = 9113; + strcpy(createReq.replicas[0].fqdn, "localhost"); - SDCreateMnodeReq* pReq = (SDCreateMnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); - pReq->replica = 1; - pReq->replicas[0].id = htonl(1); - pReq->replicas[0].port = htonl(9113); - strcpy(pReq->replicas[0].fqdn, "localhost"); + int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSDCreateMnodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -42,14 +44,16 @@ TEST_F(DndTestMnode, 01_Create_Mnode) { } { - int32_t contLen = sizeof(SDCreateMnodeReq); + SDCreateMnodeReq createReq = {0}; + createReq.dnodeId = 1; + createReq.replica = 1; + createReq.replicas[0].id = 2; + createReq.replicas[0].port = 9113; + strcpy(createReq.replicas[0].fqdn, "localhost"); - SDCreateMnodeReq* pReq = (SDCreateMnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(1); - pReq->replica = 1; - pReq->replicas[0].id = htonl(2); - pReq->replicas[0].port = htonl(9113); - strcpy(pReq->replicas[0].fqdn, "localhost"); + int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSDCreateMnodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -57,17 +61,19 @@ TEST_F(DndTestMnode, 01_Create_Mnode) { } { - int32_t contLen = sizeof(SDCreateMnodeReq); + SDCreateMnodeReq createReq = {0}; + createReq.dnodeId = 1; + createReq.replica = 2; + createReq.replicas[0].id = 1; + createReq.replicas[0].port = 9113; + strcpy(createReq.replicas[0].fqdn, "localhost"); + createReq.replicas[1].id = 1; + createReq.replicas[1].port = 9114; + strcpy(createReq.replicas[1].fqdn, "localhost"); - SDCreateMnodeReq* pReq = (SDCreateMnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(1); - pReq->replica = 2; - pReq->replicas[0].id = htonl(1); - pReq->replicas[0].port = htonl(9113); - strcpy(pReq->replicas[0].fqdn, "localhost"); - pReq->replicas[1].id = htonl(1); - pReq->replicas[1].port = htonl(9114); - strcpy(pReq->replicas[1].fqdn, "localhost"); + int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSDCreateMnodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -76,15 +82,17 @@ TEST_F(DndTestMnode, 01_Create_Mnode) { } TEST_F(DndTestMnode, 02_Alter_Mnode) { - { - int32_t contLen = sizeof(SDAlterMnodeReq); + { + SDAlterMnodeReq alterReq = {0}; + alterReq.dnodeId = 2; + alterReq.replica = 1; + alterReq.replicas[0].id = 1; + alterReq.replicas[0].port = 9113; + strcpy(alterReq.replicas[0].fqdn, "localhost"); - SDAlterMnodeReq* pReq = (SDAlterMnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); - pReq->replica = 1; - pReq->replicas[0].id = htonl(1); - pReq->replicas[0].port = htonl(9113); - strcpy(pReq->replicas[0].fqdn, "localhost"); + int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, &alterReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSDCreateMnodeReq(pReq, contLen, &alterReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_ALTER_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -92,14 +100,16 @@ TEST_F(DndTestMnode, 02_Alter_Mnode) { } { - int32_t contLen = sizeof(SDAlterMnodeReq); + SDAlterMnodeReq alterReq = {0}; + alterReq.dnodeId = 1; + alterReq.replica = 1; + alterReq.replicas[0].id = 2; + alterReq.replicas[0].port = 9113; + strcpy(alterReq.replicas[0].fqdn, "localhost"); - SDAlterMnodeReq* pReq = (SDAlterMnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(1); - pReq->replica = 1; - pReq->replicas[0].id = htonl(2); - pReq->replicas[0].port = htonl(9113); - strcpy(pReq->replicas[0].fqdn, "localhost"); + int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, &alterReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSDCreateMnodeReq(pReq, contLen, &alterReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_ALTER_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -107,14 +117,16 @@ TEST_F(DndTestMnode, 02_Alter_Mnode) { } { - int32_t contLen = sizeof(SDAlterMnodeReq); + SDAlterMnodeReq alterReq = {0}; + alterReq.dnodeId = 1; + alterReq.replica = 1; + alterReq.replicas[0].id = 1; + alterReq.replicas[0].port = 9113; + strcpy(alterReq.replicas[0].fqdn, "localhost"); - SDAlterMnodeReq* pReq = (SDAlterMnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(1); - pReq->replica = 1; - pReq->replicas[0].id = htonl(1); - pReq->replicas[0].port = htonl(9113); - strcpy(pReq->replicas[0].fqdn, "localhost"); + int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, &alterReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSDCreateMnodeReq(pReq, contLen, &alterReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_ALTER_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -124,10 +136,12 @@ TEST_F(DndTestMnode, 02_Alter_Mnode) { TEST_F(DndTestMnode, 03_Drop_Mnode) { { - int32_t contLen = sizeof(SDDropMnodeReq); + SDDropMnodeReq dropReq = {0}; + dropReq.dnodeId = 2; - SDDropMnodeReq* pReq = (SDDropMnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropMnodeReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropMnodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -135,10 +149,12 @@ TEST_F(DndTestMnode, 03_Drop_Mnode) { } { - int32_t contLen = sizeof(SDDropMnodeReq); + SDDropMnodeReq dropReq = {0}; + dropReq.dnodeId = 1; - SDDropMnodeReq* pReq = (SDDropMnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(1); + int32_t contLen = tSerializeSMCreateDropMnodeReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropMnodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -146,10 +162,12 @@ TEST_F(DndTestMnode, 03_Drop_Mnode) { } { - int32_t contLen = sizeof(SDDropMnodeReq); + SDDropMnodeReq dropReq = {0}; + dropReq.dnodeId = 1; - SDDropMnodeReq* pReq = (SDDropMnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(1); + int32_t contLen = tSerializeSMCreateDropMnodeReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropMnodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -157,30 +175,33 @@ TEST_F(DndTestMnode, 03_Drop_Mnode) { } { - int32_t contLen = sizeof(SDAlterMnodeReq); + SDAlterMnodeReq alterReq = {0}; + alterReq.dnodeId = 1; + alterReq.replica = 1; + alterReq.replicas[0].id = 1; + alterReq.replicas[0].port = 9113; + strcpy(alterReq.replicas[0].fqdn, "localhost"); - SDAlterMnodeReq* pReq = (SDAlterMnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(1); - pReq->replica = 1; - pReq->replicas[0].id = htonl(1); - pReq->replicas[0].port = htonl(9113); - strcpy(pReq->replicas[0].fqdn, "localhost"); + int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, &alterReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSDCreateMnodeReq(pReq, contLen, &alterReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_ALTER_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_DND_MNODE_NOT_DEPLOYED); } - { - int32_t contLen = sizeof(SDCreateMnodeReq); + SDCreateMnodeReq createReq = {0}; + createReq.dnodeId = 1; + createReq.replica = 2; + createReq.replicas[0].id = 1; + createReq.replicas[0].port = 9113; + strcpy(createReq.replicas[0].fqdn, "localhost"); - SDCreateMnodeReq* pReq = (SDCreateMnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(1); - pReq->replica = 2; - pReq->replicas[0].id = htonl(1); - pReq->replicas[0].port = htonl(9113); - strcpy(pReq->replicas[0].fqdn, "localhost"); + int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSDCreateMnodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_MNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); diff --git a/source/dnode/mgmt/impl/test/qnode/dqnode.cpp b/source/dnode/mgmt/impl/test/qnode/dqnode.cpp index 19fd6b4b12..46c31539a2 100644 --- a/source/dnode/mgmt/impl/test/qnode/dqnode.cpp +++ b/source/dnode/mgmt/impl/test/qnode/dqnode.cpp @@ -27,10 +27,12 @@ Testbase DndTestQnode::test; TEST_F(DndTestQnode, 01_Create_Qnode) { { - int32_t contLen = sizeof(SDCreateQnodeReq); + SDCreateQnodeReq createReq = {0}; + createReq.dnodeId = 2; - SDCreateQnodeReq* pReq = (SDCreateQnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -38,10 +40,12 @@ TEST_F(DndTestQnode, 01_Create_Qnode) { } { - int32_t contLen = sizeof(SDCreateQnodeReq); + SDCreateQnodeReq createReq = {0}; + createReq.dnodeId = 1; - SDCreateQnodeReq* pReq = (SDCreateQnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(1); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -49,10 +53,12 @@ TEST_F(DndTestQnode, 01_Create_Qnode) { } { - int32_t contLen = sizeof(SDCreateQnodeReq); + SDCreateQnodeReq createReq = {0}; + createReq.dnodeId = 1; - SDCreateQnodeReq* pReq = (SDCreateQnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(1); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -62,10 +68,12 @@ TEST_F(DndTestQnode, 01_Create_Qnode) { test.Restart(); { - int32_t contLen = sizeof(SDCreateQnodeReq); + SDCreateQnodeReq createReq = {0}; + createReq.dnodeId = 1; - SDCreateQnodeReq* pReq = (SDCreateQnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(1); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -75,10 +83,12 @@ TEST_F(DndTestQnode, 01_Create_Qnode) { TEST_F(DndTestQnode, 02_Drop_Qnode) { { - int32_t contLen = sizeof(SDDropQnodeReq); + SDDropQnodeReq dropReq = {0}; + dropReq.dnodeId = 2; - SDDropQnodeReq* pReq = (SDDropQnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -86,10 +96,12 @@ TEST_F(DndTestQnode, 02_Drop_Qnode) { } { - int32_t contLen = sizeof(SDDropQnodeReq); + SDDropQnodeReq dropReq = {0}; + dropReq.dnodeId = 1; - SDDropQnodeReq* pReq = (SDDropQnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(1); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -97,10 +109,12 @@ TEST_F(DndTestQnode, 02_Drop_Qnode) { } { - int32_t contLen = sizeof(SDDropQnodeReq); + SDDropQnodeReq dropReq = {0}; + dropReq.dnodeId = 1; - SDDropQnodeReq* pReq = (SDDropQnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(1); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -110,10 +124,12 @@ TEST_F(DndTestQnode, 02_Drop_Qnode) { test.Restart(); { - int32_t contLen = sizeof(SDDropQnodeReq); + SDDropQnodeReq dropReq = {0}; + dropReq.dnodeId = 1; - SDDropQnodeReq* pReq = (SDDropQnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(1); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -121,10 +137,12 @@ TEST_F(DndTestQnode, 02_Drop_Qnode) { } { - int32_t contLen = sizeof(SDCreateQnodeReq); + SDCreateQnodeReq createReq = {0}; + createReq.dnodeId = 1; - SDCreateQnodeReq* pReq = (SDCreateQnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(1); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_QNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); diff --git a/source/dnode/mgmt/impl/test/snode/dsnode.cpp b/source/dnode/mgmt/impl/test/snode/dsnode.cpp index 019aa1cbbc..ea98dfdd95 100644 --- a/source/dnode/mgmt/impl/test/snode/dsnode.cpp +++ b/source/dnode/mgmt/impl/test/snode/dsnode.cpp @@ -27,10 +27,12 @@ Testbase DndTestSnode::test; TEST_F(DndTestSnode, 01_Create_Snode) { { - int32_t contLen = sizeof(SDCreateSnodeReq); + SDCreateSnodeReq createReq = {0}; + createReq.dnodeId = 2; - SDCreateSnodeReq* pReq = (SDCreateSnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -38,10 +40,12 @@ TEST_F(DndTestSnode, 01_Create_Snode) { } { - int32_t contLen = sizeof(SDCreateSnodeReq); + SDCreateSnodeReq createReq = {0}; + createReq.dnodeId = 1; - SDCreateSnodeReq* pReq = (SDCreateSnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(1); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -49,10 +53,12 @@ TEST_F(DndTestSnode, 01_Create_Snode) { } { - int32_t contLen = sizeof(SDCreateSnodeReq); + SDCreateSnodeReq createReq = {0}; + createReq.dnodeId = 1; - SDCreateSnodeReq* pReq = (SDCreateSnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(1); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -62,10 +68,12 @@ TEST_F(DndTestSnode, 01_Create_Snode) { test.Restart(); { - int32_t contLen = sizeof(SDCreateSnodeReq); + SDCreateSnodeReq createReq = {0}; + createReq.dnodeId = 1; - SDCreateSnodeReq* pReq = (SDCreateSnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(1); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -75,10 +83,12 @@ TEST_F(DndTestSnode, 01_Create_Snode) { TEST_F(DndTestSnode, 01_Drop_Snode) { { - int32_t contLen = sizeof(SDDropSnodeReq); + SDDropSnodeReq dropReq = {0}; + dropReq.dnodeId = 2; - SDDropSnodeReq* pReq = (SDDropSnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(2); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -86,10 +96,12 @@ TEST_F(DndTestSnode, 01_Drop_Snode) { } { - int32_t contLen = sizeof(SDDropSnodeReq); + SDDropSnodeReq dropReq = {0}; + dropReq.dnodeId = 1; - SDDropSnodeReq* pReq = (SDDropSnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(1); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -97,10 +109,12 @@ TEST_F(DndTestSnode, 01_Drop_Snode) { } { - int32_t contLen = sizeof(SDDropSnodeReq); + SDDropSnodeReq dropReq = {0}; + dropReq.dnodeId = 1; - SDDropSnodeReq* pReq = (SDDropSnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(1); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -110,10 +124,12 @@ TEST_F(DndTestSnode, 01_Drop_Snode) { test.Restart(); { - int32_t contLen = sizeof(SDDropSnodeReq); + SDDropSnodeReq dropReq = {0}; + dropReq.dnodeId = 1; - SDDropSnodeReq* pReq = (SDDropSnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(1); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -121,10 +137,12 @@ TEST_F(DndTestSnode, 01_Drop_Snode) { } { - int32_t contLen = sizeof(SDCreateSnodeReq); + SDCreateSnodeReq createReq = {0}; + createReq.dnodeId = 1; - SDCreateSnodeReq* pReq = (SDCreateSnodeReq*)rpcMallocCont(contLen); - pReq->dnodeId = htonl(1); + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_SNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); diff --git a/source/dnode/mgmt/impl/test/sut/inc/sut.h b/source/dnode/mgmt/impl/test/sut/inc/sut.h index 955f62610a..23913b0531 100644 --- a/source/dnode/mgmt/impl/test/sut/inc/sut.h +++ b/source/dnode/mgmt/impl/test/sut/inc/sut.h @@ -74,7 +74,7 @@ class Testbase { private: int64_t showId; - STableMetaRsp* pMeta; + STableMetaRsp metaRsp; SRetrieveTableRsp* pRetrieveRsp; char* pData; int32_t pos; diff --git a/source/dnode/mgmt/impl/test/sut/src/sut.cpp b/source/dnode/mgmt/impl/test/sut/src/sut.cpp index dc8390fba3..09a738be3b 100644 --- a/source/dnode/mgmt/impl/test/sut/src/sut.cpp +++ b/source/dnode/mgmt/impl/test/sut/src/sut.cpp @@ -56,9 +56,16 @@ void Testbase::Init(const char* path, int16_t port) { server.Start(path, fqdn, port, firstEp); client.Init("root", "taosdata", fqdn, port); taosMsleep(1100); + + tFreeSTableMetaRsp(&metaRsp); + showId = 0; + pData = 0; + pos = 0; + pRetrieveRsp = NULL; } void Testbase::Cleanup() { + tFreeSTableMetaRsp(&metaRsp); server.Stop(); client.Cleanup(); dndCleanup(); @@ -80,61 +87,61 @@ SRpcMsg* Testbase::SendReq(tmsg_t msgType, void* pCont, int32_t contLen) { } void Testbase::SendShowMetaReq(int8_t showType, const char* db) { - int32_t contLen = sizeof(SShowReq); - SShowReq* pShow = (SShowReq*)rpcMallocCont(contLen); - pShow->type = showType; - strcpy(pShow->db, db); + SShowReq showReq = {0}; + showReq.type = showType; + strcpy(showReq.db, db); - SRpcMsg* pRsp = SendReq(TDMT_MND_SHOW, pShow, contLen); - SShowRsp* pShowRsp = (SShowRsp*)pRsp->pCont; + int32_t contLen = tSerializeSShowReq(NULL, 0, &showReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSShowReq(pReq, contLen, &showReq); + tFreeSShowReq(&showReq); - ASSERT(pShowRsp != nullptr); - pShowRsp->showId = htobe64(pShowRsp->showId); - pMeta = &pShowRsp->tableMeta; - pMeta->numOfTags = htonl(pMeta->numOfTags); - pMeta->numOfColumns = htonl(pMeta->numOfColumns); - pMeta->sversion = htonl(pMeta->sversion); - pMeta->tversion = htonl(pMeta->tversion); - pMeta->tuid = htobe64(pMeta->tuid); - pMeta->suid = htobe64(pMeta->suid); + SRpcMsg* pRsp = SendReq(TDMT_MND_SHOW, pReq, contLen); + ASSERT(pRsp->pCont != nullptr); - showId = pShowRsp->showId; + if (pRsp->contLen == 0) return; + + SShowRsp showRsp = {0}; + tDeserializeSShowRsp(pRsp->pCont, pRsp->contLen, &showRsp); + tFreeSTableMetaRsp(&metaRsp); + metaRsp = showRsp.tableMeta; + showId = showRsp.showId; } int32_t Testbase::GetMetaColId(int32_t index) { - SSchema* pSchema = &pMeta->pSchema[index]; - pSchema->colId = htonl(pSchema->colId); + SSchema* pSchema = &metaRsp.pSchemas[index]; return pSchema->colId; } int8_t Testbase::GetMetaType(int32_t index) { - SSchema* pSchema = &pMeta->pSchema[index]; + SSchema* pSchema = &metaRsp.pSchemas[index]; return pSchema->type; } int32_t Testbase::GetMetaBytes(int32_t index) { - SSchema* pSchema = &pMeta->pSchema[index]; - pSchema->bytes = htonl(pSchema->bytes); + SSchema* pSchema = &metaRsp.pSchemas[index]; return pSchema->bytes; } const char* Testbase::GetMetaName(int32_t index) { - SSchema* pSchema = &pMeta->pSchema[index]; + SSchema* pSchema = &metaRsp.pSchemas[index]; return pSchema->name; } -int32_t Testbase::GetMetaNum() { return pMeta->numOfColumns; } +int32_t Testbase::GetMetaNum() { return metaRsp.numOfColumns; } -const char* Testbase::GetMetaTbName() { return pMeta->tbName; } +const char* Testbase::GetMetaTbName() { return metaRsp.tbName; } void Testbase::SendShowRetrieveReq() { - int32_t contLen = sizeof(SRetrieveTableReq); + SRetrieveTableReq retrieveReq = {0}; + retrieveReq.showId = showId; + retrieveReq.free = 0; - SRetrieveTableReq* pRetrieve = (SRetrieveTableReq*)rpcMallocCont(contLen); - pRetrieve->showId = htobe64(showId); - pRetrieve->free = 0; + int32_t contLen = tSerializeSRetrieveTableReq(NULL, 0, &retrieveReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSRetrieveTableReq(pReq, contLen, &retrieveReq); - SRpcMsg* pRsp = SendReq(TDMT_MND_SHOW_RETRIEVE, pRetrieve, contLen); + SRpcMsg* pRsp = SendReq(TDMT_MND_SHOW_RETRIEVE, pReq, contLen); pRetrieveRsp = (SRetrieveTableRsp*)pRsp->pCont; pRetrieveRsp->numOfRows = htonl(pRetrieveRsp->numOfRows); pRetrieveRsp->useconds = htobe64(pRetrieveRsp->useconds); @@ -144,7 +151,7 @@ void Testbase::SendShowRetrieveReq() { pos = 0; } -const char* Testbase::GetShowName() { return pMeta->tbName; } +const char* Testbase::GetShowName() { return metaRsp.tbName; } int8_t Testbase::GetShowInt8() { int8_t data = *((int8_t*)(pData + pos)); @@ -185,6 +192,6 @@ const char* Testbase::GetShowBinary(int32_t len) { int32_t Testbase::GetShowRows() { return pRetrieveRsp->numOfRows; } -STableMetaRsp* Testbase::GetShowMeta() { return pMeta; } +STableMetaRsp* Testbase::GetShowMeta() { return &metaRsp; } SRetrieveTableRsp* Testbase::GetRetrieveRsp() { return pRetrieveRsp; } \ No newline at end of file diff --git a/source/dnode/mgmt/impl/test/vnode/vnode.cpp b/source/dnode/mgmt/impl/test/vnode/vnode.cpp index 380a837c58..4457faf7b1 100644 --- a/source/dnode/mgmt/impl/test/vnode/vnode.cpp +++ b/source/dnode/mgmt/impl/test/vnode/vnode.cpp @@ -27,38 +27,40 @@ Testbase DndTestVnode::test; TEST_F(DndTestVnode, 01_Create_Vnode) { for (int i = 0; i < 3; ++i) { - int32_t contLen = sizeof(SCreateVnodeReq); - - SCreateVnodeReq* pReq = (SCreateVnodeReq*)rpcMallocCont(contLen); - pReq->vgId = htonl(2); - pReq->dnodeId = htonl(1); - strcpy(pReq->db, "1.d1"); - pReq->dbUid = htobe64(9527); - pReq->vgVersion = htonl(1); - pReq->cacheBlockSize = htonl(16); - pReq->totalBlocks = htonl(10); - pReq->daysPerFile = htonl(10); - pReq->daysToKeep0 = htonl(3650); - pReq->daysToKeep1 = htonl(3650); - pReq->daysToKeep2 = htonl(3650); - pReq->minRows = htonl(100); - pReq->minRows = htonl(4096); - pReq->commitTime = htonl(3600); - pReq->fsyncPeriod = htonl(3000); - pReq->walLevel = 1; - pReq->precision = 0; - pReq->compression = 2; - pReq->replica = 1; - pReq->quorum = 1; - pReq->update = 0; - pReq->cacheLastRow = 0; - pReq->selfIndex = 0; - for (int r = 0; r < pReq->replica; ++r) { - SReplica* pReplica = &pReq->replicas[r]; - pReplica->id = htonl(1); - pReplica->port = htons(9527); + SCreateVnodeReq createReq = {0}; + createReq.vgId = 2; + createReq.dnodeId = 1; + strcpy(createReq.db, "1.d1"); + createReq.dbUid = 9527; + createReq.vgVersion = 1; + createReq.cacheBlockSize = 16; + createReq.totalBlocks = 10; + createReq.daysPerFile = 10; + createReq.daysToKeep0 = 3650; + createReq.daysToKeep1 = 3650; + createReq.daysToKeep2 = 3650; + createReq.minRows = 100; + createReq.minRows = 4096; + createReq.commitTime = 3600; + createReq.fsyncPeriod = 3000; + createReq.walLevel = 1; + createReq.precision = 0; + createReq.compression = 2; + createReq.replica = 1; + createReq.quorum = 1; + createReq.update = 0; + createReq.cacheLastRow = 0; + createReq.selfIndex = 0; + for (int r = 0; r < createReq.replica; ++r) { + SReplica* pReplica = &createReq.replicas[r]; + pReplica->id = 1; + pReplica->port = 9527; } + int32_t contLen = tSerializeSCreateVnodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateVnodeReq(pReq, contLen, &createReq); + SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_VNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); if (i == 0) { @@ -70,38 +72,40 @@ TEST_F(DndTestVnode, 01_Create_Vnode) { } { - int32_t contLen = sizeof(SCreateVnodeReq); - - SCreateVnodeReq* pReq = (SCreateVnodeReq*)rpcMallocCont(contLen); - pReq->vgId = htonl(2); - pReq->dnodeId = htonl(3); - strcpy(pReq->db, "1.d1"); - pReq->dbUid = htobe64(9527); - pReq->vgVersion = htonl(1); - pReq->cacheBlockSize = htonl(16); - pReq->totalBlocks = htonl(10); - pReq->daysPerFile = htonl(10); - pReq->daysToKeep0 = htonl(3650); - pReq->daysToKeep1 = htonl(3650); - pReq->daysToKeep2 = htonl(3650); - pReq->minRows = htonl(100); - pReq->minRows = htonl(4096); - pReq->commitTime = htonl(3600); - pReq->fsyncPeriod = htonl(3000); - pReq->walLevel = 1; - pReq->precision = 0; - pReq->compression = 2; - pReq->replica = 1; - pReq->quorum = 1; - pReq->update = 0; - pReq->cacheLastRow = 0; - pReq->selfIndex = 0; - for (int r = 0; r < pReq->replica; ++r) { - SReplica* pReplica = &pReq->replicas[r]; - pReplica->id = htonl(1); - pReplica->port = htons(9527); + SCreateVnodeReq createReq = {0}; + createReq.vgId = 2; + createReq.dnodeId = 3; + strcpy(createReq.db, "1.d1"); + createReq.dbUid = 9527; + createReq.vgVersion = 1; + createReq.cacheBlockSize = 16; + createReq.totalBlocks = 10; + createReq.daysPerFile = 10; + createReq.daysToKeep0 = 3650; + createReq.daysToKeep1 = 3650; + createReq.daysToKeep2 = 3650; + createReq.minRows = 100; + createReq.minRows = 4096; + createReq.commitTime = 3600; + createReq.fsyncPeriod = 3000; + createReq.walLevel = 1; + createReq.precision = 0; + createReq.compression = 2; + createReq.replica = 1; + createReq.quorum = 1; + createReq.update = 0; + createReq.cacheLastRow = 0; + createReq.selfIndex = 0; + for (int r = 0; r < createReq.replica; ++r) { + SReplica* pReplica = &createReq.replicas[r]; + pReplica->id = 1; + pReplica->port = 9527; } + int32_t contLen = tSerializeSCreateVnodeReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateVnodeReq(pReq, contLen, &createReq); + SRpcMsg* pRsp = test.SendReq(TDMT_DND_CREATE_VNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, TSDB_CODE_DND_VNODE_INVALID_OPTION); @@ -110,38 +114,40 @@ TEST_F(DndTestVnode, 01_Create_Vnode) { TEST_F(DndTestVnode, 02_Alter_Vnode) { for (int i = 0; i < 3; ++i) { - int32_t contLen = sizeof(SAlterVnodeReq); - - SAlterVnodeReq* pReq = (SAlterVnodeReq*)rpcMallocCont(contLen); - pReq->vgId = htonl(2); - pReq->dnodeId = htonl(1); - strcpy(pReq->db, "1.d1"); - pReq->dbUid = htobe64(9527); - pReq->vgVersion = htonl(2); - pReq->cacheBlockSize = htonl(16); - pReq->totalBlocks = htonl(10); - pReq->daysPerFile = htonl(10); - pReq->daysToKeep0 = htonl(3650); - pReq->daysToKeep1 = htonl(3650); - pReq->daysToKeep2 = htonl(3650); - pReq->minRows = htonl(100); - pReq->minRows = htonl(4096); - pReq->commitTime = htonl(3600); - pReq->fsyncPeriod = htonl(3000); - pReq->walLevel = 1; - pReq->precision = 0; - pReq->compression = 2; - pReq->replica = 1; - pReq->quorum = 1; - pReq->update = 0; - pReq->cacheLastRow = 0; - pReq->selfIndex = 0; - for (int r = 0; r < pReq->replica; ++r) { - SReplica* pReplica = &pReq->replicas[r]; - pReplica->id = htonl(1); - pReplica->port = htons(9527); + SAlterVnodeReq alterReq = {0}; + alterReq.vgId = 2; + alterReq.dnodeId = 1; + strcpy(alterReq.db, "1.d1"); + alterReq.dbUid = 9527; + alterReq.vgVersion = 2; + alterReq.cacheBlockSize = 16; + alterReq.totalBlocks = 10; + alterReq.daysPerFile = 10; + alterReq.daysToKeep0 = 3650; + alterReq.daysToKeep1 = 3650; + alterReq.daysToKeep2 = 3650; + alterReq.minRows = 100; + alterReq.minRows = 4096; + alterReq.commitTime = 3600; + alterReq.fsyncPeriod = 3000; + alterReq.walLevel = 1; + alterReq.precision = 0; + alterReq.compression = 2; + alterReq.replica = 1; + alterReq.quorum = 1; + alterReq.update = 0; + alterReq.cacheLastRow = 0; + alterReq.selfIndex = 0; + for (int r = 0; r < alterReq.replica; ++r) { + SReplica* pReplica = &alterReq.replicas[r]; + pReplica->id = 1; + pReplica->port = 9527; } + int32_t contLen = tSerializeSCreateVnodeReq(NULL, 0, &alterReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateVnodeReq(pReq, contLen, &alterReq); + SRpcMsg* pRsp = test.SendReq(TDMT_DND_ALTER_VNODE, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, 0); @@ -312,17 +318,19 @@ TEST_F(DndTestVnode, 05_DROP_Stb) { TEST_F(DndTestVnode, 06_Drop_Vnode) { for (int i = 0; i < 3; ++i) { - int32_t contLen = sizeof(SDropVnodeReq); + SDropVnodeReq dropReq = {0}; + dropReq.vgId = 2; + dropReq.dnodeId = 1; + strcpy(dropReq.db, "1.d1"); + dropReq.dbUid = 9527; - SDropVnodeReq* pReq = (SDropVnodeReq*)rpcMallocCont(contLen); - pReq->vgId = htonl(2); - pReq->dnodeId = htonl(1); - strcpy(pReq->db, "1.d1"); - pReq->dbUid = htobe64(9527); + int32_t contLen = tSerializeSDropVnodeReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSDropVnodeReq(pReq, contLen, &dropReq); SRpcMsg rpcMsg = {0}; rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SDropVnodeReq); + rpcMsg.contLen = contLen; rpcMsg.msgType = TDMT_DND_DROP_VNODE; SRpcMsg* pRsp = test.SendReq(TDMT_DND_DROP_VNODE, pReq, contLen); diff --git a/source/dnode/mnode/impl/inc/mndAuth.h b/source/dnode/mnode/impl/inc/mndAuth.h index c2c69f000b..7dcb518f9b 100644 --- a/source/dnode/mnode/impl/inc/mndAuth.h +++ b/source/dnode/mnode/impl/inc/mndAuth.h @@ -36,6 +36,9 @@ int32_t mndCheckCreateDbAuth(SUserObj *pOperUser); int32_t mndCheckAlterDropCompactSyncDbAuth(SUserObj *pOperUser, SDbObj *pDb); int32_t mndCheckUseDbAuth(SUserObj *pOperUser, SDbObj *pDb); +int32_t mndCheckWriteAuth(SUserObj *pOperUser, SDbObj *pDb); +int32_t mndCheckReadAuth(SUserObj *pOperUser, SDbObj *pDb); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mnode/impl/inc/mndDb.h b/source/dnode/mnode/impl/inc/mndDb.h index 5a3e6ed26e..b6264e6db9 100644 --- a/source/dnode/mnode/impl/inc/mndDb.h +++ b/source/dnode/mnode/impl/inc/mndDb.h @@ -24,9 +24,9 @@ extern "C" { int32_t mndInitDb(SMnode *pMnode); void mndCleanupDb(SMnode *pMnode); -SDbObj *mndAcquireDb(SMnode *pMnode, char *db); +SDbObj *mndAcquireDb(SMnode *pMnode, const char *db); void mndReleaseDb(SMnode *pMnode, SDbObj *pDb); -int32_t mndValidateDBInfo(SMnode *pMnode, SDbVgVersion *dbs, int32_t num, void **rsp, int32_t *rspLen); +int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs, void **ppRsp, int32_t *pRspLen); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndStb.h b/source/dnode/mnode/impl/inc/mndStb.h index 0855b5bd4d..82d93a4b9b 100644 --- a/source/dnode/mnode/impl/inc/mndStb.h +++ b/source/dnode/mnode/impl/inc/mndStb.h @@ -22,14 +22,13 @@ extern "C" { #endif -int32_t mndInitStb(SMnode *pMnode); -void mndCleanupStb(SMnode *pMnode); - +int32_t mndInitStb(SMnode *pMnode); +void mndCleanupStb(SMnode *pMnode); SStbObj *mndAcquireStb(SMnode *pMnode, char *stbName); -void mndReleaseStb(SMnode *pMnode, SStbObj *pStb); - -int32_t mndValidateStbInfo(SMnode *pMnode, SSTableMetaVersion *stbs, int32_t num, void **rsp, int32_t *rspLen); - +void mndReleaseStb(SMnode *pMnode, SStbObj *pStb); +SSdbRaw *mndStbActionEncode(SStbObj *pStb); +int32_t mndValidateStbInfo(SMnode *pMnode, SSTableMetaVersion *pStbs, int32_t numOfStbs, void **ppRsp, + int32_t *pRspLen); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndVgroup.h b/source/dnode/mnode/impl/inc/mndVgroup.h index 6ab11aa1b4..85e9a15bd4 100644 --- a/source/dnode/mnode/impl/inc/mndVgroup.h +++ b/source/dnode/mnode/impl/inc/mndVgroup.h @@ -31,8 +31,8 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups); SEpSet mndGetVgroupEpset(SMnode *pMnode, SVgObj *pVgroup); int32_t mndGetVnodesNum(SMnode *pMnode, int32_t dnodeId); -SCreateVnodeReq *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup); -SDropVnodeReq *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup); +void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); +void *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndAuth.c b/source/dnode/mnode/impl/src/mndAuth.c index 84c8b7476c..ab8b9350b0 100644 --- a/source/dnode/mnode/impl/src/mndAuth.c +++ b/source/dnode/mnode/impl/src/mndAuth.c @@ -46,15 +46,25 @@ int32_t mndRetriveAuth(SMnode *pMnode, char *user, char *spi, char *encrypt, cha } static int32_t mndProcessAuthReq(SMnodeMsg *pReq) { - SAuthReq *pAuth = pReq->rpcMsg.pCont; + SAuthReq authReq = {0}; + if (tDeserializeSAuthReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &authReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } - int32_t contLen = sizeof(SAuthRsp); - SAuthRsp *pRsp = rpcMallocCont(contLen); + SAuthReq authRsp = {0}; + memcpy(authRsp.user, authReq.user, TSDB_USER_LEN); + + int32_t code = + mndRetriveAuth(pReq->pMnode, authRsp.user, &authRsp.spi, &authRsp.encrypt, authRsp.secret, authRsp.ckey); + mTrace("user:%s, auth req received, spi:%d encrypt:%d ruser:%s", pReq->user, authRsp.spi, authRsp.encrypt, + authRsp.user); + + int32_t contLen = tSerializeSAuthReq(NULL, 0, &authRsp); + void *pRsp = rpcMallocCont(contLen); + tSerializeSAuthReq(pRsp, contLen, &authRsp); pReq->pCont = pRsp; pReq->contLen = contLen; - - int32_t code = mndRetriveAuth(pReq->pMnode, pAuth->user, &pRsp->spi, &pRsp->encrypt, pRsp->secret, pRsp->ckey); - mTrace("user:%s, auth req received, spi:%d encrypt:%d ruser:%s", pReq->user, pAuth->spi, pAuth->encrypt, pAuth->user); return code; } @@ -141,3 +151,29 @@ int32_t mndCheckAlterDropCompactSyncDbAuth(SUserObj *pOperUser, SDbObj *pDb) { } int32_t mndCheckUseDbAuth(SUserObj *pOperUser, SDbObj *pDb) { return 0; } + +int32_t mndCheckWriteAuth(SUserObj *pOperUser, SDbObj *pDb) { + if (pOperUser->superUser || strcmp(pOperUser->user, pDb->createUser) == 0) { + return 0; + } + + if (taosHashGet(pOperUser->writeDbs, pDb->name, strlen(pDb->name) + 1) != NULL) { + return 0; + } + + terrno = TSDB_CODE_MND_NO_RIGHTS; + return -1; +} + +int32_t mndCheckReadAuth(SUserObj *pOperUser, SDbObj *pDb) { + if (pOperUser->superUser || strcmp(pOperUser->user, pDb->createUser) == 0) { + return 0; + } + + if (taosHashGet(pOperUser->readDbs, pDb->name, strlen(pDb->name) + 1) != NULL) { + return 0; + } + + terrno = TSDB_CODE_MND_NO_RIGHTS; + return -1; +} \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndBnode.c b/source/dnode/mnode/impl/src/mndBnode.c index eb9af7b358..fd029212f5 100644 --- a/source/dnode/mnode/impl/src/mndBnode.c +++ b/source/dnode/mnode/impl/src/mndBnode.c @@ -187,17 +187,21 @@ static int32_t mndSetCreateBnodeCommitLogs(STrans *pTrans, SBnodeObj *pObj) { } static int32_t mndSetCreateBnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SBnodeObj *pObj) { - SDCreateBnodeReq *pReq = malloc(sizeof(SDCreateBnodeReq)); + SDCreateBnodeReq createReq = {0}; + createReq.dnodeId = pDnode->id; + + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); + void *pReq = malloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pReq->dnodeId = htonl(pDnode->id); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); STransAction action = {0}; action.epSet = mndGetDnodeEpset(pDnode); action.pCont = pReq; - action.contLen = sizeof(SDCreateBnodeReq); + action.contLen = contLen; action.msgType = TDMT_DND_CREATE_BNODE; action.acceptableCode = TSDB_CODE_DND_BNODE_ALREADY_DEPLOYED; @@ -210,17 +214,21 @@ static int32_t mndSetCreateBnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S } static int32_t mndSetCreateBnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, SBnodeObj *pObj) { - SDDropBnodeReq *pReq = malloc(sizeof(SDDropBnodeReq)); + SDDropBnodeReq dropReq = {0}; + dropReq.dnodeId = pDnode->id; + + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); + void *pReq = malloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pReq->dnodeId = htonl(pDnode->id); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &dropReq); STransAction action = {0}; action.epSet = mndGetDnodeEpset(pDnode); action.pCont = pReq; - action.contLen = sizeof(SDDropBnodeReq); + action.contLen = contLen; action.msgType = TDMT_DND_DROP_BNODE; action.acceptableCode = TSDB_CODE_DND_BNODE_NOT_DEPLOYED; @@ -329,17 +337,21 @@ static int32_t mndSetDropBnodeCommitLogs(STrans *pTrans, SBnodeObj *pObj) { } static int32_t mndSetDropBnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SBnodeObj *pObj) { - SDDropBnodeReq *pReq = malloc(sizeof(SDDropBnodeReq)); + SDDropBnodeReq dropReq = {0}; + dropReq.dnodeId = pDnode->id; + + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); + void *pReq = malloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pReq->dnodeId = htonl(pDnode->id); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &dropReq); STransAction action = {0}; action.epSet = mndGetDnodeEpset(pDnode); action.pCont = pReq; - action.contLen = sizeof(SDDropBnodeReq); + action.contLen = contLen; action.msgType = TDMT_DND_DROP_BNODE; action.acceptableCode = TSDB_CODE_DND_BNODE_NOT_DEPLOYED; @@ -433,27 +445,27 @@ static int32_t mndGetBnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; - SSchema *pSchema = pMeta->pSchema; + SSchema *pSchema = pMeta->pSchemas; pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; strcpy(pSchema[cols].name, "id"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = TSDB_EP_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "endpoint"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; strcpy(pSchema[cols].name, "create_time"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; - pMeta->numOfColumns = htonl(cols); + pMeta->numOfColumns = cols; pShow->numOfColumns = cols; pShow->offset[0] = 0; diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index a53298efb3..a41ccd9896 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -165,27 +165,27 @@ static int32_t mndCreateDefaultCluster(SMnode *pMnode) { static int32_t mndGetClusterMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta) { int32_t cols = 0; - SSchema *pSchema = pMeta->pSchema; + SSchema *pSchema = pMeta->pSchemas; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_BIGINT; strcpy(pSchema[cols].name, "id"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = TSDB_CLUSTER_ID_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "name"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; strcpy(pSchema[cols].name, "create_time"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; - pMeta->numOfColumns = htonl(cols); + pMeta->numOfColumns = cols; strcpy(pMeta->tbName, mndShowStr(pShow->type)); pShow->numOfColumns = cols; diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 21ebea258d..0f4b538cde 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -14,8 +14,8 @@ */ #define _DEFAULT_SOURCE - #include "mndConsumer.h" +#include "mndAuth.h" #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" @@ -53,13 +53,14 @@ int32_t mndInitConsumer(SMnode *pMnode) { void mndCleanupConsumer(SMnode *pMnode) {} -SMqConsumerObj* mndCreateConsumer(int64_t consumerId, const char* cgroup) { - SMqConsumerObj* pConsumer = malloc(sizeof(SMqConsumerObj)); +SMqConsumerObj *mndCreateConsumer(int64_t consumerId, const char *cgroup) { + SMqConsumerObj *pConsumer = calloc(1, sizeof(SMqConsumerObj)); if (pConsumer == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - pConsumer->recentRemovedTopics = taosArrayInit(0, sizeof(char*)); + + pConsumer->recentRemovedTopics = taosArrayInit(1, sizeof(char *)); pConsumer->epoch = 1; pConsumer->consumerId = consumerId; atomic_store_32(&pConsumer->status, MQ_CONSUMER_STATUS__INIT); @@ -70,7 +71,8 @@ SMqConsumerObj* mndCreateConsumer(int64_t consumerId, const char* cgroup) { SSdbRaw *mndConsumerActionEncode(SMqConsumerObj *pConsumer) { terrno = TSDB_CODE_OUT_OF_MEMORY; - void* buf = NULL; + + void *buf = NULL; int32_t tlen = tEncodeSMqConsumerObj(NULL, pConsumer); int32_t size = sizeof(int32_t) + tlen + MND_CONSUMER_RESERVE_SIZE; @@ -105,7 +107,7 @@ CM_ENCODE_OVER: SSdbRow *mndConsumerActionDecode(SSdbRaw *pRaw) { terrno = TSDB_CODE_OUT_OF_MEMORY; - void* buf = NULL; + void *buf = NULL; int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto CM_DECODE_OVER; diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 5ed3e9e8df..e772ff326c 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -15,8 +15,10 @@ #define _DEFAULT_SOURCE #include "mndDb.h" +#include "mndAuth.h" #include "mndDnode.h" #include "mndShow.h" +#include "mndStb.h" #include "mndTrans.h" #include "mndUser.h" #include "mndVgroup.h" @@ -193,7 +195,7 @@ static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew) { return 0; } -SDbObj *mndAcquireDb(SMnode *pMnode, char *db) { +SDbObj *mndAcquireDb(SMnode *pMnode, const char *db) { SSdb *pSdb = pMnode->pSdb; SDbObj *pDb = sdbAcquire(pSdb, SDB_DB, db); if (pDb == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { @@ -333,11 +335,12 @@ static int32_t mndSetCreateDbRedoActions(SMnode *pMnode, STrans *pTrans, SDbObj action.epSet = mndGetDnodeEpset(pDnode); mndReleaseDnode(pMnode, pDnode); - SCreateVnodeReq *pReq = mndBuildCreateVnodeReq(pMnode, pDnode, pDb, pVgroup); + int32_t contLen = 0; + void *pReq = mndBuildCreateVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); if (pReq == NULL) return -1; action.pCont = pReq; - action.contLen = sizeof(SCreateVnodeReq); + action.contLen = contLen; action.msgType = TDMT_DND_CREATE_VNODE; action.acceptableCode = TSDB_CODE_DND_VNODE_ALREADY_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { @@ -363,13 +366,14 @@ static int32_t mndSetCreateDbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj action.epSet = mndGetDnodeEpset(pDnode); mndReleaseDnode(pMnode, pDnode); - SDropVnodeReq *pReq = mndBuildDropVnodeReq(pMnode, pDnode, pDb, pVgroup); + int32_t contLen = 0; + void *pReq = mndBuildDropVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); if (pReq == NULL) return -1; action.pCont = pReq; - action.contLen = sizeof(SDropVnodeReq); + action.contLen = contLen; action.msgType = TDMT_DND_DROP_VNODE; - action.acceptableCode = TSDB_CODE_DND_VNODE_NOT_DEPLOYED; + action.acceptableCode = TSDB_CODE_DND_VNODE_NOT_DEPLOYED; if (mndTransAppendUndoAction(pTrans, &action) != 0) { free(pReq); return -1; @@ -451,54 +455,54 @@ CREATE_DB_OVER: } static int32_t mndProcessCreateDbReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; - SCreateDbReq *pCreate = pReq->rpcMsg.pCont; + SMnode *pMnode = pReq->pMnode; + int32_t code = -1; + SDbObj *pDb = NULL; + SUserObj *pUser = NULL; + SCreateDbReq createReq = {0}; - pCreate->numOfVgroups = htonl(pCreate->numOfVgroups); - pCreate->cacheBlockSize = htonl(pCreate->cacheBlockSize); - pCreate->totalBlocks = htonl(pCreate->totalBlocks); - pCreate->daysPerFile = htonl(pCreate->daysPerFile); - pCreate->daysToKeep0 = htonl(pCreate->daysToKeep0); - pCreate->daysToKeep1 = htonl(pCreate->daysToKeep1); - pCreate->daysToKeep2 = htonl(pCreate->daysToKeep2); - pCreate->minRows = htonl(pCreate->minRows); - pCreate->maxRows = htonl(pCreate->maxRows); - pCreate->commitTime = htonl(pCreate->commitTime); - pCreate->fsyncPeriod = htonl(pCreate->fsyncPeriod); + if (tDeserializeSCreateDbReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &createReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto CREATE_DB_OVER; + } - mDebug("db:%s, start to create, vgroups:%d", pCreate->db, pCreate->numOfVgroups); + mDebug("db:%s, start to create, vgroups:%d", createReq.db, createReq.numOfVgroups); - SDbObj *pDb = mndAcquireDb(pMnode, pCreate->db); + pDb = mndAcquireDb(pMnode, createReq.db); if (pDb != NULL) { - mndReleaseDb(pMnode, pDb); - if (pCreate->ignoreExist) { - mDebug("db:%s, already exist, ignore exist is set", pCreate->db); - return 0; + if (createReq.ignoreExist) { + mDebug("db:%s, already exist, ignore exist is set", createReq.db); + code = 0; + goto CREATE_DB_OVER; } else { terrno = TSDB_CODE_MND_DB_ALREADY_EXIST; - mError("db:%s, failed to create since %s", pCreate->db, terrstr()); - return -1; + goto CREATE_DB_OVER; } } else if (terrno != TSDB_CODE_MND_DB_NOT_EXIST) { - mError("db:%s, failed to create since %s", pCreate->db, terrstr()); - return -1; + goto CREATE_DB_OVER; } - SUserObj *pOperUser = mndAcquireUser(pMnode, pReq->user); - if (pOperUser == NULL) { - mError("db:%s, failed to create since %s", pCreate->db, terrstr()); - return -1; + pUser = mndAcquireUser(pMnode, pReq->user); + if (pUser == NULL) { + goto CREATE_DB_OVER; } - int32_t code = mndCreateDb(pMnode, pReq, pCreate, pOperUser); - mndReleaseUser(pMnode, pOperUser); - - if (code != 0) { - mError("db:%s, failed to create since %s", pCreate->db, terrstr()); - return -1; + if (mndCheckCreateDbAuth(pUser) != 0) { + goto CREATE_DB_OVER; } - return TSDB_CODE_MND_ACTION_IN_PROGRESS; + code = mndCreateDb(pMnode, pReq, &createReq, pUser); + if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + +CREATE_DB_OVER: + if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + mError("db:%s, failed to create since %s", createReq.db, terrstr()); + } + + mndReleaseDb(pMnode, pDb); + mndReleaseUser(pMnode, pUser); + + return code; } static int32_t mndSetDbCfgFromAlterDbReq(SDbObj *pDb, SAlterDbReq *pAlter) { @@ -575,11 +579,12 @@ static int32_t mndBuildUpdateVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj action.epSet = mndGetDnodeEpset(pDnode); mndReleaseDnode(pMnode, pDnode); - SAlterVnodeReq *pReq = (SAlterVnodeReq *)mndBuildCreateVnodeReq(pMnode, pDnode, pDb, pVgroup); + int32_t contLen = 0; + void *pReq = mndBuildCreateVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); if (pReq == NULL) return -1; action.pCont = pReq; - action.contLen = sizeof(SAlterVnodeReq); + action.contLen = contLen; action.msgType = TDMT_DND_ALTER_VNODE; if (mndTransAppendRedoAction(pTrans, &action) != 0) { free(pReq); @@ -633,43 +638,56 @@ UPDATE_DB_OVER: } static int32_t mndProcessAlterDbReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; - SAlterDbReq *pAlter = pReq->rpcMsg.pCont; - pAlter->totalBlocks = htonl(pAlter->totalBlocks); - pAlter->daysToKeep0 = htonl(pAlter->daysToKeep0); - pAlter->daysToKeep1 = htonl(pAlter->daysToKeep1); - pAlter->daysToKeep2 = htonl(pAlter->daysToKeep2); - pAlter->fsyncPeriod = htonl(pAlter->fsyncPeriod); + SMnode *pMnode = pReq->pMnode; + int32_t code = -1; + SDbObj *pDb = NULL; + SUserObj *pUser = NULL; + SAlterDbReq alterReq = {0}; - mDebug("db:%s, start to alter", pAlter->db); + if (tDeserializeSAlterDbReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &alterReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto ALTER_DB_OVER; + } - SDbObj *pDb = mndAcquireDb(pMnode, pAlter->db); + mDebug("db:%s, start to alter", alterReq.db); + + pDb = mndAcquireDb(pMnode, alterReq.db); if (pDb == NULL) { - mError("db:%s, failed to alter since %s", pAlter->db, terrstr()); - return TSDB_CODE_MND_DB_NOT_EXIST; + terrno = TSDB_CODE_MND_DB_NOT_EXIST; + goto ALTER_DB_OVER; + } + + pUser = mndAcquireUser(pMnode, pReq->user); + if (pUser == NULL) { + goto ALTER_DB_OVER; + } + + if (mndCheckAlterDropCompactSyncDbAuth(pUser, pDb) != 0) { + goto ALTER_DB_OVER; } SDbObj dbObj = {0}; memcpy(&dbObj, pDb, sizeof(SDbObj)); - int32_t code = mndSetDbCfgFromAlterDbReq(&dbObj, pAlter); + code = mndSetDbCfgFromAlterDbReq(&dbObj, &alterReq); if (code != 0) { - mndReleaseDb(pMnode, pDb); - mError("db:%s, failed to alter since %s", pAlter->db, tstrerror(code)); - return code; + goto ALTER_DB_OVER; } dbObj.cfgVersion++; dbObj.updateTime = taosGetTimestampMs(); code = mndUpdateDb(pMnode, pReq, pDb, &dbObj); - mndReleaseDb(pMnode, pDb); + if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; - if (code != 0) { - mError("db:%s, failed to alter since %s", pAlter->db, tstrerror(code)); - return code; +ALTER_DB_OVER: + if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + mError("db:%s, failed to alter since %s", alterReq.db, terrstr()); } - return TSDB_CODE_MND_ACTION_IN_PROGRESS; + mndReleaseDb(pMnode, pDb); + mndReleaseUser(pMnode, pUser); + + return code; } static int32_t mndSetDropDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { @@ -708,6 +726,24 @@ static int32_t mndSetDropDbCommitLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pD sdbRelease(pSdb, pVgroup); } + while (1) { + SStbObj *pStb = NULL; + pIter = sdbFetch(pSdb, SDB_STB, pIter, (void **)&pStb); + if (pIter == NULL) break; + + if (pStb->dbUid == pDb->uid) { + SSdbRaw *pStbRaw = mndStbActionEncode(pStb); + if (pStbRaw == NULL || mndTransAppendCommitlog(pTrans, pStbRaw) != 0) { + sdbCancelFetch(pSdb, pIter); + sdbRelease(pSdb, pStbRaw); + return -1; + } + sdbSetRawStatus(pStbRaw, SDB_STATUS_DROPPED); + } + + sdbRelease(pSdb, pStb); + } + return 0; } @@ -721,11 +757,12 @@ static int32_t mndBuildDropVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj * action.epSet = mndGetDnodeEpset(pDnode); mndReleaseDnode(pMnode, pDnode); - SDropVnodeReq *pReq = mndBuildDropVnodeReq(pMnode, pDnode, pDb, pVgroup); + int32_t contLen = 0; + void *pReq = mndBuildDropVnodeReq(pMnode, pDnode, pDb, pVgroup, &contLen); if (pReq == NULL) return -1; action.pCont = pReq; - action.contLen = sizeof(SDropVnodeReq); + action.contLen = contLen; action.msgType = TDMT_DND_DROP_VNODE; action.acceptableCode = TSDB_CODE_DND_VNODE_NOT_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { @@ -771,11 +808,18 @@ static int32_t mndDropDb(SMnode *pMnode, SMnodeMsg *pReq, SDbObj *pDb) { if (mndSetDropDbCommitLogs(pMnode, pTrans, pDb) != 0) goto DROP_DB_OVER; if (mndSetDropDbRedoActions(pMnode, pTrans, pDb) != 0) goto DROP_DB_OVER; - int32_t rspLen = sizeof(SDropDbRsp); - SDropDbRsp *pRsp = rpcMallocCont(rspLen); - if (pRsp == NULL) goto DROP_DB_OVER; - memcpy(pRsp->db, pDb->name, TSDB_DB_FNAME_LEN); - pRsp->uid = htobe64(pDb->uid); + SDropDbRsp dropRsp = {0}; + memcpy(dropRsp.db, pDb->name, TSDB_DB_FNAME_LEN); + dropRsp.uid = pDb->uid; + + int32_t rspLen = tSerializeSDropDbRsp(NULL, 0, &dropRsp); + void *pRsp = malloc(rspLen); + if (pRsp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto DROP_DB_OVER; + } + tSerializeSDropDbRsp(pRsp, rspLen, &dropRsp); + mndTransSetRpcRsp(pTrans, pRsp, rspLen); if (mndTransPrepare(pMnode, pTrans) != 0) goto DROP_DB_OVER; @@ -788,37 +832,56 @@ DROP_DB_OVER: } static int32_t mndProcessDropDbReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; - SDropDbReq *pDrop = pReq->rpcMsg.pCont; + SMnode *pMnode = pReq->pMnode; + int32_t code = -1; + SDbObj *pDb = NULL; + SUserObj *pUser = NULL; + SDropDbReq dropReq = {0}; - mDebug("db:%s, start to drop", pDrop->db); + if (tDeserializeSDropDbReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &dropReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto DROP_DB_OVER; + } - SDbObj *pDb = mndAcquireDb(pMnode, pDrop->db); + mDebug("db:%s, start to drop", dropReq.db); + + pDb = mndAcquireDb(pMnode, dropReq.db); if (pDb == NULL) { - if (pDrop->ignoreNotExists) { - mDebug("db:%s, not exist, ignore not exist is set", pDrop->db); - return TSDB_CODE_SUCCESS; + if (dropReq.ignoreNotExists) { + code = 0; + goto DROP_DB_OVER; } else { terrno = TSDB_CODE_MND_DB_NOT_EXIST; - mError("db:%s, failed to drop since %s", pDrop->db, terrstr()); - return -1; + goto DROP_DB_OVER; } } - int32_t code = mndDropDb(pMnode, pReq, pDb); - mndReleaseDb(pMnode, pDb); - - if (code != 0) { - mError("db:%s, failed to drop since %s", pDrop->db, terrstr()); - return code; + pUser = mndAcquireUser(pMnode, pReq->user); + if (pUser == NULL) { + goto DROP_DB_OVER; } - return TSDB_CODE_MND_ACTION_IN_PROGRESS; + if (mndCheckAlterDropCompactSyncDbAuth(pUser, pDb) != 0) { + goto DROP_DB_OVER; + } + + code = mndDropDb(pMnode, pReq, pDb); + if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + +DROP_DB_OVER: + if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + mError("db:%s, failed to drop since %s", dropReq.db, terrstr()); + } + + mndReleaseDb(pMnode, pDb); + mndReleaseUser(pMnode, pUser); + + return code; } -static void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SVgroupInfo *vgList, int32_t *vgNum) { +static void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SArray *pVgList) { int32_t vindex = 0; - SSdb *pSdb = pMnode->pSdb; + SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; while (vindex < pDb->cfg.numOfVgroups) { @@ -827,168 +890,243 @@ static void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SVgroupInfo *vgLis if (pIter == NULL) break; if (pVgroup->dbUid == pDb->uid) { - SVgroupInfo *pInfo = &vgList[vindex]; - pInfo->vgId = htonl(pVgroup->vgId); - pInfo->hashBegin = htonl(pVgroup->hashBegin); - pInfo->hashEnd = htonl(pVgroup->hashEnd); - pInfo->epset.numOfEps = pVgroup->replica; + SVgroupInfo vgInfo = {0}; + vgInfo.vgId = pVgroup->vgId; + vgInfo.hashBegin = pVgroup->hashBegin; + vgInfo.hashEnd = pVgroup->hashEnd; + vgInfo.epset.numOfEps = pVgroup->replica; for (int32_t gid = 0; gid < pVgroup->replica; ++gid) { - SVnodeGid *pVgid = &pVgroup->vnodeGid[gid]; - SEp * pEp = &pInfo->epset.eps[gid]; - SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); + SVnodeGid *pVgid = &pVgroup->vnodeGid[gid]; + SEp *pEp = &vgInfo.epset.eps[gid]; + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); if (pDnode != NULL) { memcpy(pEp->fqdn, pDnode->fqdn, TSDB_FQDN_LEN); - pEp->port = htons(pDnode->port); + pEp->port = pDnode->port; } mndReleaseDnode(pMnode, pDnode); if (pVgid->role == TAOS_SYNC_STATE_LEADER) { - pInfo->epset.inUse = gid; + vgInfo.epset.inUse = gid; } } vindex++; + taosArrayPush(pVgList, &vgInfo); } sdbRelease(pSdb, pVgroup); } - - *vgNum = vindex; } static int32_t mndProcessUseDbReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; - SSdb *pSdb = pMnode->pSdb; - SUseDbReq *pUse = pReq->rpcMsg.pCont; - pUse->vgVersion = htonl(pUse->vgVersion); + SMnode *pMnode = pReq->pMnode; + int32_t code = -1; + SDbObj *pDb = NULL; + SUserObj *pUser = NULL; + SUseDbReq usedbReq = {0}; + SUseDbRsp usedbRsp = {0}; - SDbObj *pDb = mndAcquireDb(pMnode, pUse->db); - if (pDb == NULL) { - terrno = TSDB_CODE_MND_DB_NOT_EXIST; - mError("db:%s, failed to process use db req since %s", pUse->db, terrstr()); - return -1; + if (tDeserializeSUseDbReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &usedbReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto USE_DB_OVER; } - int32_t contLen = sizeof(SUseDbRsp) + pDb->cfg.numOfVgroups * sizeof(SVgroupInfo); - SUseDbRsp *pRsp = rpcMallocCont(contLen); + pDb = mndAcquireDb(pMnode, usedbReq.db); + if (pDb == NULL) { + terrno = TSDB_CODE_MND_DB_NOT_EXIST; + goto USE_DB_OVER; + } + + pUser = mndAcquireUser(pMnode, pReq->user); + if (pUser == NULL) { + goto USE_DB_OVER; + } + + if (mndCheckUseDbAuth(pUser, pDb) != 0) { + goto USE_DB_OVER; + } + + usedbRsp.pVgroupInfos = taosArrayInit(pDb->cfg.numOfVgroups, sizeof(SVgroupInfo)); + if (usedbRsp.pVgroupInfos == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto USE_DB_OVER; + } + + if (usedbReq.vgVersion < pDb->vgVersion) { + mndBuildDBVgroupInfo(pDb, pMnode, usedbRsp.pVgroupInfos); + } + + memcpy(usedbRsp.db, pDb->name, TSDB_DB_FNAME_LEN); + usedbRsp.uid = pDb->uid; + usedbRsp.vgVersion = pDb->vgVersion; + usedbRsp.vgNum = taosArrayGetSize(usedbRsp.pVgroupInfos); + usedbRsp.hashMethod = pDb->hashMethod; + + int32_t contLen = tSerializeSUseDbRsp(NULL, 0, &usedbRsp); + void *pRsp = rpcMallocCont(contLen); if (pRsp == NULL) { - mndReleaseDb(pMnode, pDb); + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto USE_DB_OVER; + } + + tSerializeSUseDbRsp(pRsp, contLen, &usedbRsp); + + pReq->pCont = pRsp; + pReq->contLen = contLen; + code = 0; + +USE_DB_OVER: + if (code != 0) { + mError("db:%s, failed to process use db req since %s", usedbReq.db, terrstr()); + } + + mndReleaseDb(pMnode, pDb); + mndReleaseUser(pMnode, pUser); + tFreeSUsedbRsp(&usedbRsp); + + return code; +} + +int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs, void **ppRsp, int32_t *pRspLen) { + SUseDbBatchRsp batchUseRsp = {0}; + batchUseRsp.pArray = taosArrayInit(numOfDbs, sizeof(SUseDbRsp)); + if (batchUseRsp.pArray == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - int32_t vgNum = 0; + for (int32_t i = 0; i < numOfDbs; ++i) { + SDbVgVersion *pDbVgVersion = &pDbs[i]; + pDbVgVersion->dbId = htobe64(pDbVgVersion->dbId); + pDbVgVersion->vgVersion = htonl(pDbVgVersion->vgVersion); - if (pUse->vgVersion < pDb->vgVersion) { - mndBuildDBVgroupInfo(pDb, pMnode, pRsp->vgroupInfo, &vgNum); - } + SUseDbRsp usedbRsp = {0}; - memcpy(pRsp->db, pDb->name, TSDB_DB_FNAME_LEN); - pRsp->uid = htobe64(pDb->uid); - pRsp->vgVersion = htonl(pDb->vgVersion); - pRsp->vgNum = htonl(vgNum); - pRsp->hashMethod = pDb->hashMethod; - - pReq->pCont = pRsp; - pReq->contLen = contLen; - mndReleaseDb(pMnode, pDb); - - return 0; -} - -int32_t mndValidateDBInfo(SMnode *pMnode, SDbVgVersion *dbs, int32_t num, void **rsp, int32_t *rspLen) { - SSdb *pSdb = pMnode->pSdb; - int32_t bufSize = num * (sizeof(SUseDbRsp) + TSDB_DEFAULT_VN_PER_DB * sizeof(SVgroupInfo)); - void *buf = malloc(bufSize); - int32_t len = 0; - int32_t contLen = 0; - int32_t bufOffset = 0; - SUseDbRsp *pRsp = NULL; - - for (int32_t i = 0; i < num; ++i) { - SDbVgVersion *db = &dbs[i]; - db->dbId = be64toh(db->dbId); - db->vgVersion = ntohl(db->vgVersion); - - len = 0; - - SDbObj *pDb = mndAcquireDb(pMnode, db->dbFName); + SDbObj *pDb = mndAcquireDb(pMnode, pDbVgVersion->dbFName); if (pDb == NULL) { - mInfo("db %s not exist", db->dbFName); - - len = sizeof(SUseDbRsp); - } else if (pDb->uid != db->dbId || db->vgVersion < pDb->vgVersion) { - len = sizeof(SUseDbRsp) + pDb->cfg.numOfVgroups * sizeof(SVgroupInfo); - } - - if (0 == len) { + mDebug("db:%s, no exist", pDbVgVersion->dbFName); + memcpy(usedbRsp.db, pDbVgVersion->dbFName, TSDB_DB_FNAME_LEN); + usedbRsp.uid = pDbVgVersion->dbId; + usedbRsp.vgVersion = -1; + taosArrayPush(batchUseRsp.pArray, &usedbRsp); + } else if (pDbVgVersion->vgVersion >= pDb->vgVersion) { + mDebug("db:%s, version not changed", pDbVgVersion->dbFName); mndReleaseDb(pMnode, pDb); - continue; - } - - contLen += len; - - if (contLen > bufSize) { - buf = realloc(buf, contLen); - } - - pRsp = (SUseDbRsp *)((char *)buf + bufOffset); - memcpy(pRsp->db, db->dbFName, TSDB_DB_FNAME_LEN); - if (pDb) { - int32_t vgNum = 0; - mndBuildDBVgroupInfo(pDb, pMnode, pRsp->vgroupInfo, &vgNum); - - pRsp->uid = htobe64(pDb->uid); - pRsp->vgVersion = htonl(pDb->vgVersion); - pRsp->vgNum = htonl(vgNum); - pRsp->hashMethod = pDb->hashMethod; } else { - pRsp->uid = htobe64(db->dbId); - pRsp->vgNum = htonl(0); - pRsp->hashMethod = 0; - pRsp->vgVersion = htonl(-1); + usedbRsp.pVgroupInfos = taosArrayInit(pDb->cfg.numOfVgroups, sizeof(SVgroupInfo)); + if (usedbRsp.pVgroupInfos == NULL) { + mndReleaseDb(pMnode, pDb); + mError("db:%s, failed to malloc usedb response", pDb->name); + continue; + } + + mndBuildDBVgroupInfo(pDb, pMnode, usedbRsp.pVgroupInfos); + memcpy(usedbRsp.db, pDb->name, TSDB_DB_FNAME_LEN); + usedbRsp.uid = pDb->uid; + usedbRsp.vgVersion = pDb->vgVersion; + usedbRsp.vgNum = (int32_t)taosArrayGetSize(usedbRsp.pVgroupInfos); + usedbRsp.hashMethod = pDb->hashMethod; + + taosArrayPush(batchUseRsp.pArray, &usedbRsp); + mndReleaseDb(pMnode, pDb); } - - bufOffset += len; - - mndReleaseDb(pMnode, pDb); } - if (contLen > 0) { - *rsp = buf; - *rspLen = contLen; - } else { - *rsp = NULL; - tfree(buf); - *rspLen = 0; + int32_t rspLen = tSerializeSUseDbBatchRsp(NULL, 0, &batchUseRsp); + void *pRsp = malloc(rspLen); + if (pRsp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tFreeSUseDbBatchRsp(&batchUseRsp); + return -1; } + tSerializeSUseDbBatchRsp(pRsp, rspLen, &batchUseRsp); + *ppRsp = pRsp; + *pRspLen = rspLen; + + tFreeSUseDbBatchRsp(&batchUseRsp); return 0; } static int32_t mndProcessSyncDbReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; - SSyncDbReq *pSync = pReq->rpcMsg.pCont; - SDbObj *pDb = mndAcquireDb(pMnode, pSync->db); + SMnode *pMnode = pReq->pMnode; + int32_t code = -1; + SDbObj *pDb = NULL; + SUserObj *pUser = NULL; + SSyncDbReq syncReq = {0}; + + if (tDeserializeSSyncDbReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &syncReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto SYNC_DB_OVER; + } + + mDebug("db:%s, start to sync", syncReq.db); + + pDb = mndAcquireDb(pMnode, syncReq.db); if (pDb == NULL) { - mError("db:%s, failed to process sync db req since %s", pSync->db, terrstr()); - return -1; + goto SYNC_DB_OVER; + } + + pUser = mndAcquireUser(pMnode, pReq->user); + if (pUser == NULL) { + goto SYNC_DB_OVER; + } + + if (mndCheckAlterDropCompactSyncDbAuth(pUser, pDb) != 0) { + goto SYNC_DB_OVER; + } + + // code = mndSyncDb(); + +SYNC_DB_OVER: + if (code != 0) { + mError("db:%s, failed to process sync db req since %s", syncReq.db, terrstr()); } mndReleaseDb(pMnode, pDb); - return 0; + mndReleaseUser(pMnode, pUser); + + return code; } static int32_t mndProcessCompactDbReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; - SCompactDbReq *pCompact = pReq->rpcMsg.pCont; - SDbObj *pDb = mndAcquireDb(pMnode, pCompact->db); + SMnode *pMnode = pReq->pMnode; + int32_t code = -1; + SDbObj *pDb = NULL; + SUserObj *pUser = NULL; + SCompactDbReq compactReq = {0}; + + if (tDeserializeSSyncDbReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &compactReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto SYNC_DB_OVER; + } + + mDebug("db:%s, start to sync", compactReq.db); + + pDb = mndAcquireDb(pMnode, compactReq.db); if (pDb == NULL) { - mError("db:%s, failed to process compact db req since %s", pCompact->db, terrstr()); - return -1; + goto SYNC_DB_OVER; + } + + pUser = mndAcquireUser(pMnode, pReq->user); + if (pUser == NULL) { + goto SYNC_DB_OVER; + } + + if (mndCheckAlterDropCompactSyncDbAuth(pUser, pDb) != 0) { + goto SYNC_DB_OVER; + } + + // code = mndSyncDb(); + +SYNC_DB_OVER: + if (code != 0) { + mError("db:%s, failed to process compact db req since %s", compactReq.db, terrstr()); } mndReleaseDb(pMnode, pDb); - return 0; + mndReleaseUser(pMnode, pUser); + + return code; } static int32_t mndGetDbMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { @@ -996,117 +1134,117 @@ static int32_t mndGetDbMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMe SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; - SSchema *pSchema = pMeta->pSchema; + SSchema *pSchema = pMeta->pSchemas; pShow->bytes[cols] = (TSDB_DB_NAME_LEN - 1) + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "name"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; strcpy(pSchema[cols].name, "create_time"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; strcpy(pSchema[cols].name, "vgroups"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "ntables"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; strcpy(pSchema[cols].name, "replica"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; strcpy(pSchema[cols].name, "quorum"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; strcpy(pSchema[cols].name, "days"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = 24 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "keep0,keep1,keep2"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "cache"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "blocks"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "minrows"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "maxrows"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = 1; pSchema[cols].type = TSDB_DATA_TYPE_TINYINT; strcpy(pSchema[cols].name, "wallevel"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "fsync"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = 1; pSchema[cols].type = TSDB_DATA_TYPE_TINYINT; strcpy(pSchema[cols].name, "comp"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = 1; pSchema[cols].type = TSDB_DATA_TYPE_TINYINT; strcpy(pSchema[cols].name, "cachelast"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = 3 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "precision"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = 1; pSchema[cols].type = TSDB_DATA_TYPE_TINYINT; strcpy(pSchema[cols].name, "update"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; - pMeta->numOfColumns = htonl(cols); + pMeta->numOfColumns = cols; pShow->numOfColumns = cols; pShow->offset[0] = 0; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 5126ca9db3..6d0b8e7c8d 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -302,7 +302,10 @@ static int32_t mndProcessStatusReq(SMnodeMsg *pReq) { SDnodeObj *pDnode = NULL; int32_t code = -1; - if (tDeserializeSStatusReq(pReq->rpcMsg.pCont, &statusReq) == NULL) goto PROCESS_STATUS_MSG_OVER; + if (tDeserializeSStatusReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &statusReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto PROCESS_STATUS_MSG_OVER; + } if (statusReq.dnodeId == 0) { pDnode = mndAcquireDnodeByEp(pMnode, statusReq.dnodeEp); @@ -410,10 +413,9 @@ static int32_t mndProcessStatusReq(SMnodeMsg *pReq) { mndGetDnodeData(pMnode, statusRsp.pDnodeEps); - int32_t contLen = tSerializeSStatusRsp(NULL, &statusRsp); + int32_t contLen = tSerializeSStatusRsp(NULL, 0, &statusRsp); void *pHead = rpcMallocCont(contLen); - void *pBuf = pHead; - tSerializeSStatusRsp(&pBuf, &statusRsp); + tSerializeSStatusRsp(pHead, contLen, &statusRsp); taosArrayDestroy(statusRsp.pDnodeEps); pReq->contLen = contLen; @@ -607,14 +609,12 @@ static int32_t mndProcessConfigDnodeReq(SMnodeMsg *pReq) { SEpSet epSet = mndGetDnodeEpset(pDnode); mndReleaseDnode(pMnode, pDnode); - SDCfgDnodeReq *pCfgDnode = rpcMallocCont(sizeof(SDCfgDnodeReq)); - pCfgDnode->dnodeId = htonl(cfgReq.dnodeId); - memcpy(pCfgDnode->config, cfgReq.config, TSDB_DNODE_CONFIG_LEN); + int32_t bufLen = tSerializeSMCfgDnodeReq(NULL, 0, &cfgReq); + void *pBuf = rpcMallocCont(bufLen); + tSerializeSMCfgDnodeReq(pBuf, bufLen, &cfgReq); - SRpcMsg rpcMsg = {.msgType = TDMT_DND_CONFIG_DNODE, - .pCont = pCfgDnode, - .contLen = sizeof(SDCfgDnodeReq), - .ahandle = pReq->rpcMsg.ahandle}; + SRpcMsg rpcMsg = { + .msgType = TDMT_DND_CONFIG_DNODE, .pCont = pBuf, .contLen = bufLen, .ahandle = pReq->rpcMsg.ahandle}; mInfo("dnode:%d, app:%p config:%s req send to dnode", cfgReq.dnodeId, rpcMsg.ahandle, cfgReq.config); mndSendReqToDnode(pMnode, &epSet, &rpcMsg); @@ -628,21 +628,21 @@ static int32_t mndProcessConfigDnodeRsp(SMnodeMsg *pRsp) { static int32_t mndGetConfigMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { int32_t cols = 0; - SSchema *pSchema = pMeta->pSchema; + SSchema *pSchema = pMeta->pSchemas; pShow->bytes[cols] = TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; tstrncpy(pSchema[cols].name, "name", sizeof(pSchema[cols].name)); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = TSDB_CONIIG_VALUE_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; tstrncpy(pSchema[cols].name, "value", sizeof(pSchema[cols].name)); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; - pMeta->numOfColumns = htonl(cols); + pMeta->numOfColumns = cols; pShow->numOfColumns = cols; pShow->offset[0] = 0; @@ -705,51 +705,51 @@ static int32_t mndGetDnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; - SSchema *pSchema = pMeta->pSchema; + SSchema *pSchema = pMeta->pSchemas; pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; strcpy(pSchema[cols].name, "id"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = TSDB_EP_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "endpoint"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; strcpy(pSchema[cols].name, "vnodes"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; strcpy(pSchema[cols].name, "support_vnodes"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = 10 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "status"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; strcpy(pSchema[cols].name, "create_time"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = 24 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "offline_reason"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; - pMeta->numOfColumns = htonl(cols); + pMeta->numOfColumns = cols; pShow->numOfColumns = cols; pShow->offset[0] = 0; diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index 3527f103db..9dbb0fc2d0 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -468,51 +468,51 @@ static int32_t mndGetFuncMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *p SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; - SSchema *pSchema = pMeta->pSchema; + SSchema *pSchema = pMeta->pSchemas; pShow->bytes[cols] = TSDB_FUNC_NAME_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "name"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = PATH_MAX + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "comment"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "aggregate"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = TSDB_TYPE_STR_MAX_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "outputtype"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; strcpy(pSchema[cols].name, "create_time"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "code_len"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "bufsize"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; - pMeta->numOfColumns = htonl(cols); + pMeta->numOfColumns = cols; pShow->numOfColumns = cols; pShow->offset[0] = 0; diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index eb9ba49dd2..720dc1b535 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -243,7 +243,7 @@ void mndGetMnodeEpSet(SMnode *pMnode, SEpSet *pEpSet) { pEpSet->inUse = pEpSet->numOfEps; } - addEpIntoEpSet(pEpSet, pObj->pDnode->fqdn, htons(pObj->pDnode->port)); + addEpIntoEpSet(pEpSet, pObj->pDnode->fqdn, pObj->pDnode->port); sdbRelease(pSdb, pObj); } } @@ -284,8 +284,8 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno if (pIter == NULL) break; SReplica *pReplica = &createReq.replicas[numOfReplicas]; - pReplica->id = htonl(pMObj->id); - pReplica->port = htons(pMObj->pDnode->port); + pReplica->id = pMObj->id; + pReplica->port = pMObj->pDnode->port; memcpy(pReplica->fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN); numOfReplicas++; @@ -293,8 +293,8 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno } SReplica *pReplica = &createReq.replicas[numOfReplicas]; - pReplica->id = htonl(pDnode->id); - pReplica->port = htons(pDnode->port); + pReplica->id = pDnode->id; + pReplica->port = pDnode->port; memcpy(pReplica->fqdn, pDnode->fqdn, TSDB_FQDN_LEN); numOfReplicas++; @@ -307,18 +307,14 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno STransAction action = {0}; - SDAlterMnodeReq *pReq = malloc(sizeof(SDAlterMnodeReq)); - if (pReq == NULL) { - sdbCancelFetch(pSdb, pIter); - sdbRelease(pSdb, pMObj); - return -1; - } - memcpy(pReq, &createReq, sizeof(SDAlterMnodeReq)); + createReq.dnodeId = pMObj->id; + int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, &createReq); + void *pReq = malloc(contLen); + tSerializeSDCreateMnodeReq(pReq, contLen, &createReq); - pReq->dnodeId = htonl(pMObj->id); action.epSet = mndGetDnodeEpset(pMObj->pDnode); action.pCont = pReq; - action.contLen = sizeof(SDAlterMnodeReq); + action.contLen = contLen; action.msgType = TDMT_DND_ALTER_MNODE; action.acceptableCode = TSDB_CODE_DND_MNODE_ALREADY_DEPLOYED; @@ -336,14 +332,14 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno STransAction action = {0}; action.epSet = mndGetDnodeEpset(pDnode); - SDCreateMnodeReq *pReq = malloc(sizeof(SDCreateMnodeReq)); - if (pReq == NULL) return -1; - memcpy(pReq, &createReq, sizeof(SDAlterMnodeReq)); - pReq->dnodeId = htonl(pObj->id); + createReq.dnodeId = pObj->id; + int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, &createReq); + void *pReq = malloc(contLen); + tSerializeSDCreateMnodeReq(pReq, contLen, &createReq); action.epSet = mndGetDnodeEpset(pDnode); action.pCont = pReq; - action.contLen = sizeof(SDCreateMnodeReq); + action.contLen = contLen; action.msgType = TDMT_DND_CREATE_MNODE; action.acceptableCode = TSDB_CODE_DND_MNODE_ALREADY_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { @@ -463,8 +459,8 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode if (pMObj->id != pObj->id) { SReplica *pReplica = &alterReq.replicas[numOfReplicas]; - pReplica->id = htonl(pMObj->id); - pReplica->port = htons(pMObj->pDnode->port); + pReplica->id = pMObj->id; + pReplica->port = pMObj->pDnode->port; memcpy(pReplica->fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN); numOfReplicas++; } @@ -481,18 +477,14 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode if (pMObj->id != pObj->id) { STransAction action = {0}; - SDAlterMnodeReq *pReq = malloc(sizeof(SDAlterMnodeReq)); - if (pReq == NULL) { - sdbCancelFetch(pSdb, pIter); - sdbRelease(pSdb, pMObj); - return -1; - } - memcpy(pReq, &alterReq, sizeof(SDAlterMnodeReq)); + alterReq.dnodeId = pMObj->id; + int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, &alterReq); + void *pReq = malloc(contLen); + tSerializeSDCreateMnodeReq(pReq, contLen, &alterReq); - pReq->dnodeId = htonl(pMObj->id); action.epSet = mndGetDnodeEpset(pMObj->pDnode); action.pCont = pReq; - action.contLen = sizeof(SDAlterMnodeReq); + action.contLen = contLen; action.msgType = TDMT_DND_ALTER_MNODE; action.acceptableCode = TSDB_CODE_DND_MNODE_ALREADY_DEPLOYED; @@ -511,16 +503,15 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode STransAction action = {0}; action.epSet = mndGetDnodeEpset(pDnode); - SDDropMnodeReq *pReq = malloc(sizeof(SDDropMnodeReq)); - if (pReq == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - pReq->dnodeId = htonl(pObj->id); + SDDropMnodeReq dropReq = {0}; + dropReq.dnodeId = pObj->id; + int32_t contLen = tSerializeSMCreateDropMnodeReq(NULL, 0, &dropReq); + void *pReq = malloc(contLen); + tSerializeSMCreateDropMnodeReq(pReq, contLen, &dropReq); action.epSet = mndGetDnodeEpset(pDnode); action.pCont = pReq; - action.contLen = sizeof(SDDropMnodeReq); + action.contLen = contLen; action.msgType = TDMT_DND_DROP_MNODE; action.acceptableCode = TSDB_CODE_DND_MNODE_NOT_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { @@ -620,39 +611,39 @@ static int32_t mndGetMnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; - SSchema *pSchema = pMeta->pSchema; + SSchema *pSchema = pMeta->pSchemas; pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; strcpy(pSchema[cols].name, "id"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = TSDB_EP_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "endpoint"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = 12 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "role"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; strcpy(pSchema[cols].name, "create_time"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; strcpy(pSchema[cols].name, "role_time"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; - pMeta->numOfColumns = htonl(cols); + pMeta->numOfColumns = cols; pShow->numOfColumns = cols; pShow->offset[0] = 0; diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index d63ade4320..df892e2242 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -14,16 +14,13 @@ */ #define _DEFAULT_SOURCE -#include "tglobal.h" #include "mndProfile.h" -//#include "mndConsumer.h" #include "mndDb.h" -#include "mndStb.h" #include "mndMnode.h" #include "mndShow.h" -//#include "mndTopic.h" +#include "mndStb.h" #include "mndUser.h" -//#include "mndVgroup.h" +#include "tglobal.h" #define QUERY_ID_SIZE 20 #define QUERY_OBJ_ID_SIZE 18 @@ -184,15 +181,17 @@ static void mndCancelGetNextConn(SMnode *pMnode, void *pIter) { } static int32_t mndProcessConnectReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; - SUserObj *pUser = NULL; - SDbObj *pDb = NULL; - SConnObj *pConn = NULL; - int32_t code = -1; + SMnode *pMnode = pReq->pMnode; + SUserObj *pUser = NULL; + SDbObj *pDb = NULL; + SConnObj *pConn = NULL; + int32_t code = -1; + SConnectReq connReq = {0}; - SConnectReq *pConnReq = pReq->rpcMsg.pCont; - pConnReq->pid = htonl(pConnReq->pid); - pConnReq->startTime = htobe64(pConnReq->startTime); + if (tDeserializeSConnectReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &connReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto CONN_OVER; + } SRpcConnInfo info = {0}; if (rpcGetConnInfo(pReq->rpcMsg.handle, &info) != 0) { @@ -209,41 +208,42 @@ static int32_t mndProcessConnectReq(SMnodeMsg *pReq) { goto CONN_OVER; } - if (pConnReq->db[0]) { - snprintf(pReq->db, TSDB_DB_FNAME_LEN, "%d%s%s", pUser->acctId, TS_PATH_DELIMITER, pConnReq->db); + if (connReq.db[0]) { + snprintf(pReq->db, TSDB_DB_FNAME_LEN, "%d%s%s", pUser->acctId, TS_PATH_DELIMITER, connReq.db); pDb = mndAcquireDb(pMnode, pReq->db); if (pDb == NULL) { terrno = TSDB_CODE_MND_INVALID_DB; - mError("user:%s, failed to login from %s while use db:%s since %s", pReq->user, ip, pConnReq->db, terrstr()); + mError("user:%s, failed to login from %s while use db:%s since %s", pReq->user, ip, connReq.db, terrstr()); goto CONN_OVER; } } - pConn = mndCreateConn(pMnode, &info, pConnReq->pid, pConnReq->app, pConnReq->startTime); + pConn = mndCreateConn(pMnode, &info, connReq.pid, connReq.app, connReq.startTime); if (pConn == NULL) { mError("user:%s, failed to login from %s while create connection since %s", pReq->user, ip, terrstr()); goto CONN_OVER; } - SConnectRsp *pRsp = rpcMallocCont(sizeof(SConnectRsp)); - if (pRsp == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - mError("user:%s, failed to login from %s while create rsp since %s", pReq->user, ip, terrstr()); - goto CONN_OVER; - } + SConnectRsp connectRsp = {0}; + connectRsp.acctId = pUser->acctId; + connectRsp.superUser = pUser->superUser; + connectRsp.clusterId = pMnode->clusterId; + connectRsp.connId = pConn->id; - pRsp->acctId = htonl(pUser->acctId); - pRsp->superUser = pUser->superUser; - pRsp->clusterId = htobe64(pMnode->clusterId); - pRsp->connId = htonl(pConn->id); + snprintf(connectRsp.sVersion, sizeof(connectRsp.sVersion), "ver:%s\nbuild:%s\ngitinfo:%s", version, buildinfo, + gitinfo); + mndGetMnodeEpSet(pMnode, &connectRsp.epSet); - snprintf(pRsp->sVersion, tListLen(pRsp->sVersion), "ver:%s\nbuild:%s\ngitinfo:%s", version, buildinfo, gitinfo); - mndGetMnodeEpSet(pMnode, &pRsp->epSet); + int32_t contLen = tSerializeSConnectRsp(NULL, 0, &connectRsp); + if (contLen < 0) goto CONN_OVER; + void *pRsp = rpcMallocCont(contLen); + if (pRsp == NULL) goto CONN_OVER; + tSerializeSConnectRsp(pRsp, contLen, &connectRsp); - pReq->contLen = sizeof(SConnectRsp); + pReq->contLen = contLen; pReq->pCont = pRsp; - mDebug("user:%s, login from %s, conn:%d, app:%s", info.user, ip, pConn->id, pConnReq->app); + mDebug("user:%s, login from %s, conn:%d, app:%s", info.user, ip, pConn->id, connReq.app); code = 0; @@ -276,7 +276,7 @@ static int32_t mndSaveQueryStreamList(SConnObj *pConn, SHeartBeatReq *pReq) { return TSDB_CODE_SUCCESS; } -static SClientHbRsp* mndMqHbBuildRsp(SMnode* pMnode, SClientHbReq* pReq) { +static SClientHbRsp *mndMqHbBuildRsp(SMnode *pMnode, SClientHbReq *pReq) { #if 0 SClientHbRsp* pRsp = malloc(sizeof(SClientHbRsp)); if (pRsp == NULL) { @@ -343,17 +343,20 @@ static SClientHbRsp* mndMqHbBuildRsp(SMnode* pMnode, SClientHbReq* pReq) { static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) { SMnode *pMnode = pReq->pMnode; - char *batchReqStr = pReq->rpcMsg.pCont; - SClientHbBatchReq batchReq = {0}; - tDeserializeSClientHbBatchReq(batchReqStr, &batchReq); - SArray *pArray = batchReq.reqs; - int sz = taosArrayGetSize(pArray); + SClientHbBatchReq batchReq = {0}; + if (tDeserializeSClientHbBatchReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &batchReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + SClientHbBatchRsp batchRsp = {0}; batchRsp.rsps = taosArrayInit(0, sizeof(SClientHbRsp)); + int32_t sz = taosArrayGetSize(batchReq.reqs); for (int i = 0; i < sz; i++) { - SClientHbReq* pHbReq = taosArrayGet(pArray, i); + SClientHbReq *pHbReq = taosArrayGet(batchReq.reqs, i); if (pHbReq->connKey.hbType == HEARTBEAT_TYPE_QUERY) { int32_t kvNum = taosHashGetSize(pHbReq->info); if (NULL == pHbReq->info || kvNum <= 0) { @@ -364,13 +367,13 @@ static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) { void *pIter = taosHashIterate(pHbReq->info, NULL); while (pIter != NULL) { - SKv* kv = pIter; - + SKv *kv = pIter; + switch (kv->key) { case HEARTBEAT_KEY_DBINFO: { - void *rspMsg = NULL; + void *rspMsg = NULL; int32_t rspLen = 0; - mndValidateDBInfo(pMnode, (SDbVgVersion *)kv->value, kv->valueLen/sizeof(SDbVgVersion), &rspMsg, &rspLen); + mndValidateDbInfo(pMnode, kv->value, kv->valueLen / sizeof(SDbVgVersion), &rspMsg, &rspLen); if (rspMsg && rspLen > 0) { SKv kv = {.key = HEARTBEAT_KEY_DBINFO, .valueLen = rspLen, .value = rspMsg}; taosArrayPush(hbRsp.info, &kv); @@ -378,9 +381,9 @@ static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) { break; } case HEARTBEAT_KEY_STBINFO: { - void *rspMsg = NULL; + void *rspMsg = NULL; int32_t rspLen = 0; - mndValidateStbInfo(pMnode, (SSTableMetaVersion *)kv->value, kv->valueLen/sizeof(SSTableMetaVersion), &rspMsg, &rspLen); + mndValidateStbInfo(pMnode, kv->value, kv->valueLen / sizeof(SSTableMetaVersion), &rspMsg, &rspLen); if (rspMsg && rspLen > 0) { SKv kv = {.key = HEARTBEAT_KEY_STBINFO, .valueLen = rspLen, .value = rspMsg}; taosArrayPush(hbRsp.info, &kv); @@ -392,7 +395,7 @@ static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) { hbRsp.status = TSDB_CODE_MND_APP_ERROR; break; } - + pIter = taosHashIterate(pHbReq->info, pIter); } @@ -405,17 +408,16 @@ static int32_t mndProcessHeartBeatReq(SMnodeMsg *pReq) { } } } - taosArrayDestroyEx(pArray, tFreeClientHbReq); + taosArrayDestroyEx(batchReq.reqs, tFreeClientHbReq); + + int32_t tlen = tSerializeSClientHbBatchRsp(NULL, 0, &batchRsp); + void *buf = rpcMallocCont(tlen); + tSerializeSClientHbBatchRsp(buf, tlen, &batchRsp); - int32_t tlen = tSerializeSClientHbBatchRsp(NULL, &batchRsp); - void* buf = rpcMallocCont(tlen); - void* abuf = buf; - tSerializeSClientHbBatchRsp(&abuf, &batchRsp); - int32_t rspNum = (int32_t)taosArrayGetSize(batchRsp.rsps); for (int32_t i = 0; i < rspNum; ++i) { SClientHbRsp *rsp = taosArrayGet(batchRsp.rsps, i); - int32_t kvNum = (rsp->info) ? taosArrayGetSize(rsp->info): 0; + int32_t kvNum = (rsp->info) ? taosArrayGetSize(rsp->info) : 0; for (int32_t n = 0; n < kvNum; ++n) { SKv *kv = taosArrayGet(rsp->info, n); tfree(kv->value); @@ -514,19 +516,22 @@ static int32_t mndProcessKillQueryReq(SMnodeMsg *pReq) { } mndReleaseUser(pMnode, pUser); - SKillQueryReq *pKill = pReq->rpcMsg.pCont; - int32_t connId = htonl(pKill->connId); - int32_t queryId = htonl(pKill->queryId); - mInfo("kill query msg is received, queryId:%d", pKill->queryId); + SKillQueryReq killReq = {0}; + if (tDeserializeSKillQueryReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &killReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } - SConnObj *pConn = taosCacheAcquireByKey(pMgmt->cache, &connId, sizeof(int32_t)); + mInfo("kill query msg is received, queryId:%d", killReq.queryId); + + SConnObj *pConn = taosCacheAcquireByKey(pMgmt->cache, &killReq.connId, sizeof(int32_t)); if (pConn == NULL) { - mError("connId:%d, failed to kill queryId:%d, conn not exist", connId, queryId); + mError("connId:%d, failed to kill queryId:%d, conn not exist", killReq.connId, killReq.queryId); terrno = TSDB_CODE_MND_INVALID_CONN_ID; return -1; } else { - mInfo("connId:%d, queryId:%d is killed by user:%s", connId, queryId, pReq->user); - pConn->queryId = queryId; + mInfo("connId:%d, queryId:%d is killed by user:%s", killReq.connId, killReq.queryId, pReq->user); + pConn->queryId = killReq.queryId; taosCacheRelease(pMgmt->cache, (void **)&pConn, false); return 0; } @@ -545,16 +550,19 @@ static int32_t mndProcessKillConnReq(SMnodeMsg *pReq) { } mndReleaseUser(pMnode, pUser); - SKillConnReq *pKill = pReq->rpcMsg.pCont; - int32_t connId = htonl(pKill->connId); + SKillConnReq killReq = {0}; + if (tDeserializeSKillConnReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &killReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } - SConnObj *pConn = taosCacheAcquireByKey(pMgmt->cache, &connId, sizeof(int32_t)); + SConnObj *pConn = taosCacheAcquireByKey(pMgmt->cache, &killReq.connId, sizeof(int32_t)); if (pConn == NULL) { - mError("connId:%d, failed to kill connection, conn not exist", connId); + mError("connId:%d, failed to kill connection, conn not exist", killReq.connId); terrno = TSDB_CODE_MND_INVALID_CONN_ID; return -1; } else { - mInfo("connId:%d, is killed by user:%s", connId, pReq->user); + mInfo("connId:%d, is killed by user:%s", killReq.connId, pReq->user); pConn->killed = 1; taosCacheRelease(pMgmt->cache, (void **)&pConn, false); return TSDB_CODE_SUCCESS; @@ -575,53 +583,53 @@ static int32_t mndGetConnsMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * mndReleaseUser(pMnode, pUser); int32_t cols = 0; - SSchema *pSchema = pMeta->pSchema; + SSchema *pSchema = pMeta->pSchemas; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "connId"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = TSDB_USER_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "user"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; // app name pShow->bytes[cols] = TSDB_APP_NAME_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "program"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; // app pid pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "pid"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "ip:port"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; strcpy(pSchema[cols].name, "login_time"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; strcpy(pSchema[cols].name, "last_access"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; - pMeta->numOfColumns = htonl(cols); + pMeta->numOfColumns = cols; pShow->numOfColumns = cols; pShow->offset[0] = 0; @@ -704,93 +712,93 @@ static int32_t mndGetQueryMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * mndReleaseUser(pMnode, pUser); int32_t cols = 0; - SSchema *pSchema = pMeta->pSchema; + SSchema *pSchema = pMeta->pSchemas; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "queryId"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "connId"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = TSDB_USER_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "user"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = TSDB_IPv4ADDR_LEN + 6 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "ip:port"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = 22 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "qid"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; strcpy(pSchema[cols].name, "created_time"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_BIGINT; strcpy(pSchema[cols].name, "time"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = QUERY_OBJ_ID_SIZE + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "sql_obj_id"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "pid"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = TSDB_EP_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "ep"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = 1; pSchema[cols].type = TSDB_DATA_TYPE_BOOL; strcpy(pSchema[cols].name, "stable_query"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "sub_queries"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = TSDB_SHOW_SUBQUERY_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "sub_query_info"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "sql"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; - pMeta->numOfColumns = htonl(cols); + pMeta->numOfColumns = cols; pShow->numOfColumns = cols; pShow->offset[0] = 0; diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index db1ea2ab17..f1335ad7e4 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -187,17 +187,21 @@ static int32_t mndSetCreateQnodeCommitLogs(STrans *pTrans, SQnodeObj *pObj) { } static int32_t mndSetCreateQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQnodeObj *pObj) { - SDCreateQnodeReq *pReq = malloc(sizeof(SDCreateQnodeReq)); + SDCreateQnodeReq createReq = {0}; + createReq.dnodeId = pDnode->id; + + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); + void *pReq = malloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pReq->dnodeId = htonl(pDnode->id); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); STransAction action = {0}; action.epSet = mndGetDnodeEpset(pDnode); action.pCont = pReq; - action.contLen = sizeof(SDCreateQnodeReq); + action.contLen = contLen; action.msgType = TDMT_DND_CREATE_QNODE; action.acceptableCode = TSDB_CODE_DND_QNODE_ALREADY_DEPLOYED; @@ -210,17 +214,21 @@ static int32_t mndSetCreateQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S } static int32_t mndSetCreateQnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, SQnodeObj *pObj) { - SDDropQnodeReq *pReq = malloc(sizeof(SDDropQnodeReq)); + SDDropQnodeReq dropReq = {0}; + dropReq.dnodeId = pDnode->id; + + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); + void *pReq = malloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pReq->dnodeId = htonl(pDnode->id); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &dropReq); STransAction action = {0}; action.epSet = mndGetDnodeEpset(pDnode); action.pCont = pReq; - action.contLen = sizeof(SDDropQnodeReq); + action.contLen = contLen; action.msgType = TDMT_DND_DROP_QNODE; action.acceptableCode = TSDB_CODE_DND_QNODE_NOT_DEPLOYED; @@ -329,17 +337,21 @@ static int32_t mndSetDropQnodeCommitLogs(STrans *pTrans, SQnodeObj *pObj) { } static int32_t mndSetDropQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQnodeObj *pObj) { - SDDropQnodeReq *pReq = malloc(sizeof(SDDropQnodeReq)); + SDDropQnodeReq dropReq = {0}; + dropReq.dnodeId = pDnode->id; + + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); + void *pReq = malloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pReq->dnodeId = htonl(pDnode->id); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &dropReq); STransAction action = {0}; action.epSet = mndGetDnodeEpset(pDnode); action.pCont = pReq; - action.contLen = sizeof(SDDropQnodeReq); + action.contLen = contLen; action.msgType = TDMT_DND_DROP_QNODE; action.acceptableCode = TSDB_CODE_DND_QNODE_NOT_DEPLOYED; @@ -433,27 +445,27 @@ static int32_t mndGetQnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; - SSchema *pSchema = pMeta->pSchema; + SSchema *pSchema = pMeta->pSchemas; pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; strcpy(pSchema[cols].name, "id"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = TSDB_EP_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "endpoint"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; strcpy(pSchema[cols].name, "create_time"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; - pMeta->numOfColumns = htonl(cols); + pMeta->numOfColumns = cols; pShow->numOfColumns = cols; pShow->offset[0] = 0; diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index e2ddcee0e9..3198ded37e 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -118,54 +118,62 @@ static void mndReleaseShowObj(SShowObj *pShow, bool forceRemove) { static int32_t mndProcessShowReq(SMnodeMsg *pReq) { SMnode *pMnode = pReq->pMnode; SShowMgmt *pMgmt = &pMnode->showMgmt; - SShowReq *pShowReq = pReq->rpcMsg.pCont; - int8_t type = pShowReq->type; - int16_t payloadLen = htonl(pShowReq->payloadLen); + int32_t code = -1; + SShowReq showReq = {0}; + SShowRsp showRsp = {0}; - if (type <= TSDB_MGMT_TABLE_START || type >= TSDB_MGMT_TABLE_MAX) { - terrno = TSDB_CODE_MND_INVALID_MSG_TYPE; - mError("failed to process show-meta req since %s", terrstr()); - return -1; + if (tDeserializeSShowReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &showReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto SHOW_OVER; } - ShowMetaFp metaFp = pMgmt->metaFps[type]; + if (showReq.type <= TSDB_MGMT_TABLE_START || showReq.type >= TSDB_MGMT_TABLE_MAX) { + terrno = TSDB_CODE_MND_INVALID_MSG_TYPE; + goto SHOW_OVER; + } + + ShowMetaFp metaFp = pMgmt->metaFps[showReq.type]; if (metaFp == NULL) { terrno = TSDB_CODE_MND_INVALID_MSG_TYPE; - mError("failed to process show-meta req:%s since %s", mndShowStr(type), terrstr()); - return -1; + goto SHOW_OVER; } - SShowObj *pShow = mndCreateShowObj(pMnode, pShowReq); + SShowObj *pShow = mndCreateShowObj(pMnode, &showReq); if (pShow == NULL) { - mError("failed to process show-meta req:%s since %s", mndShowStr(type), terrstr()); - return -1; + goto SHOW_OVER; } - int32_t size = sizeof(SShowRsp) + sizeof(SSchema) * TSDB_MAX_COLUMNS + TSDB_EXTRA_PAYLOAD_SIZE; - SShowRsp *pRsp = rpcMallocCont(size); - if (pRsp == NULL) { + showRsp.showId = pShow->id; + showRsp.tableMeta.pSchemas = calloc(TSDB_MAX_COLUMNS, sizeof(SSchema)); + if (showRsp.tableMeta.pSchemas == NULL) { mndReleaseShowObj(pShow, true); terrno = TSDB_CODE_OUT_OF_MEMORY; - mError("show:0x%" PRIx64 ", failed to process show-meta req:%s since malloc rsp error", pShow->id, - mndShowStr(type)); - return -1; + goto SHOW_OVER; } - int32_t code = (*metaFp)(pReq, pShow, &pRsp->tableMeta); - mDebug("show:0x%" PRIx64 ", get meta finished, numOfRows:%d cols:%d type:%s, result:%s", pShow->id, pShow->numOfRows, - pShow->numOfColumns, mndShowStr(type), tstrerror(code)); + code = (*metaFp)(pReq, pShow, &showRsp.tableMeta); + mDebug("show:0x%" PRIx64 ", get meta finished, numOfRows:%d cols:%d showReq.type:%s, result:%s", pShow->id, + pShow->numOfRows, pShow->numOfColumns, mndShowStr(showReq.type), tstrerror(code)); - if (code == TSDB_CODE_SUCCESS) { - pReq->contLen = sizeof(SShowRsp) + sizeof(SSchema) * pShow->numOfColumns; - pReq->pCont = pRsp; - pRsp->showId = htobe64(pShow->id); + if (code == 0) { + int32_t bufLen = tSerializeSShowRsp(NULL, 0, &showRsp); + void *pBuf = rpcMallocCont(bufLen); + tSerializeSShowRsp(pBuf, bufLen, &showRsp); + pReq->contLen = bufLen; + pReq->pCont = pBuf; mndReleaseShowObj(pShow, false); - return TSDB_CODE_SUCCESS; } else { - rpcFreeCont(pRsp); mndReleaseShowObj(pShow, true); - return code; } + +SHOW_OVER: + if (code != 0) { + mError("failed to process show-meta req since %s", terrstr()); + } + + tFreeSShowReq(&showReq); + tFreeSShowRsp(&showRsp); + return code; } static int32_t mndProcessRetrieveReq(SMnodeMsg *pReq) { @@ -175,10 +183,13 @@ static int32_t mndProcessRetrieveReq(SMnodeMsg *pReq) { int32_t size = 0; int32_t rowsRead = 0; - SRetrieveTableReq *pRetrieve = pReq->rpcMsg.pCont; - int64_t showId = htobe64(pRetrieve->showId); + SRetrieveTableReq retrieveReq = {0}; + if (tDeserializeSRetrieveTableReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &retrieveReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } - SShowObj *pShow = mndAcquireShowObj(pMnode, showId); + SShowObj *pShow = mndAcquireShowObj(pMnode, retrieveReq.showId); if (pShow == NULL) { terrno = TSDB_CODE_MND_INVALID_SHOWOBJ; mError("failed to process show-retrieve req:%p since %s", pShow, terrstr()); @@ -202,7 +213,7 @@ static int32_t mndProcessRetrieveReq(SMnodeMsg *pReq) { pShow->numOfReads = pShow->numOfRows; } - if ((pRetrieve->free & TSDB_QUERY_TYPE_FREE_RESOURCE) != TSDB_QUERY_TYPE_FREE_RESOURCE) { + if ((retrieveReq.free & TSDB_QUERY_TYPE_FREE_RESOURCE) != TSDB_QUERY_TYPE_FREE_RESOURCE) { rowsToRead = pShow->numOfRows - pShow->numOfReads; } @@ -226,7 +237,7 @@ static int32_t mndProcessRetrieveReq(SMnodeMsg *pReq) { } // if free flag is set, client wants to clean the resources - if ((pRetrieve->free & TSDB_QUERY_TYPE_FREE_RESOURCE) != TSDB_QUERY_TYPE_FREE_RESOURCE) { + if ((retrieveReq.free & TSDB_QUERY_TYPE_FREE_RESOURCE) != TSDB_QUERY_TYPE_FREE_RESOURCE) { rowsRead = (*retrieveFp)(pReq, pShow, pRsp->data, rowsToRead); } diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index dd699364c0..5904ca0502 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -187,17 +187,21 @@ static int32_t mndSetCreateSnodeCommitLogs(STrans *pTrans, SSnodeObj *pObj) { } static int32_t mndSetCreateSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSnodeObj *pObj) { - SDCreateSnodeReq *pReq = malloc(sizeof(SDCreateSnodeReq)); + SDCreateSnodeReq createReq = {0}; + createReq.dnodeId = pDnode->id; + + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &createReq); + void *pReq = malloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pReq->dnodeId = htonl(pDnode->id); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &createReq); STransAction action = {0}; action.epSet = mndGetDnodeEpset(pDnode); action.pCont = pReq; - action.contLen = sizeof(SDCreateSnodeReq); + action.contLen = contLen; action.msgType = TDMT_DND_CREATE_SNODE; action.acceptableCode = TSDB_CODE_DND_SNODE_ALREADY_DEPLOYED; @@ -210,17 +214,21 @@ static int32_t mndSetCreateSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, S } static int32_t mndSetCreateSnodeUndoActions(STrans *pTrans, SDnodeObj *pDnode, SSnodeObj *pObj) { - SDDropSnodeReq *pReq = malloc(sizeof(SDDropSnodeReq)); + SDDropSnodeReq dropReq = {0}; + dropReq.dnodeId = pDnode->id; + + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); + void *pReq = malloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pReq->dnodeId = htonl(pDnode->id); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &dropReq); STransAction action = {0}; action.epSet = mndGetDnodeEpset(pDnode); action.pCont = pReq; - action.contLen = sizeof(SDDropSnodeReq); + action.contLen = contLen; action.msgType = TDMT_DND_DROP_SNODE; action.acceptableCode = TSDB_CODE_DND_SNODE_NOT_DEPLOYED; @@ -302,7 +310,7 @@ static int32_t mndProcessCreateSnodeReq(SMnodeMsg *pReq) { if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; CREATE_SNODE_OVER: - if(code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { mError("snode:%d, failed to create since %s", createReq.dnodeId, terrstr()); return -1; } @@ -331,17 +339,21 @@ static int32_t mndSetDropSnodeCommitLogs(STrans *pTrans, SSnodeObj *pObj) { } static int32_t mndSetDropSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSnodeObj *pObj) { - SDDropSnodeReq *pReq = malloc(sizeof(SDDropSnodeReq)); + SDDropSnodeReq dropReq = {0}; + dropReq.dnodeId = pDnode->id; + + int32_t contLen = tSerializeSMCreateDropQSBNodeReq(NULL, 0, &dropReq); + void *pReq = malloc(contLen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; } - pReq->dnodeId = htonl(pDnode->id); + tSerializeSMCreateDropQSBNodeReq(pReq, contLen, &dropReq); STransAction action = {0}; action.epSet = mndGetDnodeEpset(pDnode); action.pCont = pReq; - action.contLen = sizeof(SDDropSnodeReq); + action.contLen = contLen; action.msgType = TDMT_DND_DROP_SNODE; action.acceptableCode = TSDB_CODE_DND_SNODE_NOT_DEPLOYED; @@ -436,27 +448,27 @@ static int32_t mndGetSnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; - SSchema *pSchema = pMeta->pSchema; + SSchema *pSchema = pMeta->pSchemas; pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; strcpy(pSchema[cols].name, "id"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = TSDB_EP_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "endpoint"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; strcpy(pSchema[cols].name, "create_time"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; - pMeta->numOfColumns = htonl(cols); + pMeta->numOfColumns = cols; pShow->numOfColumns = cols; pShow->offset[0] = 0; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 344eab38b9..53ffd8698f 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "mndStb.h" +#include "mndAuth.h" #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" @@ -27,7 +28,6 @@ #define TSDB_STB_VER_NUMBER 1 #define TSDB_STB_RESERVE_SIZE 64 -static SSdbRaw *mndStbActionEncode(SStbObj *pStb); static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw); static int32_t mndStbActionInsert(SSdb *pSdb, SStbObj *pStb); static int32_t mndStbActionDelete(SSdb *pSdb, SStbObj *pStb); @@ -69,7 +69,7 @@ int32_t mndInitStb(SMnode *pMnode) { void mndCleanupStb(SMnode *pMnode) {} -static SSdbRaw *mndStbActionEncode(SStbObj *pStb) { +SSdbRaw *mndStbActionEncode(SStbObj *pStb) { terrno = TSDB_CODE_OUT_OF_MEMORY; int32_t size = sizeof(SStbObj) + (pStb->numOfColumns + pStb->numOfTags) * sizeof(SSchema) + TSDB_STB_RESERVE_SIZE; @@ -343,7 +343,7 @@ static int32_t mndCheckCreateStbReq(SMCreateStbReq *pCreate) { return -1; } - SField *pField = taosArrayGet(pCreate->pColumns, 0) ; + SField *pField = taosArrayGet(pCreate->pColumns, 0); if (pField->type != TSDB_DATA_TYPE_TIMESTAMP) { terrno = TSDB_CODE_MND_INVALID_STB_OPTION; return -1; @@ -549,12 +549,19 @@ static int32_t mndProcessMCreateStbReq(SMnodeMsg *pReq) { SStbObj *pTopicStb = NULL; SStbObj *pStb = NULL; SDbObj *pDb = NULL; + SUserObj *pUser = NULL; SMCreateStbReq createReq = {0}; - if (tDeserializeSMCreateStbReq(pReq->rpcMsg.pCont, &createReq) == NULL) goto CREATE_STB_OVER; + if (tDeserializeSMCreateStbReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &createReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto CREATE_STB_OVER; + } mDebug("stb:%s, start to create", createReq.name); - if (mndCheckCreateStbReq(&createReq) != 0) goto CREATE_STB_OVER; + if (mndCheckCreateStbReq(&createReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto CREATE_STB_OVER; + } pStb = mndAcquireStb(pMnode, createReq.name); if (pStb != NULL) { @@ -582,6 +589,15 @@ static int32_t mndProcessMCreateStbReq(SMnodeMsg *pReq) { goto CREATE_STB_OVER; } + pUser = mndAcquireUser(pMnode, pReq->user); + if (pUser == NULL) { + goto CREATE_STB_OVER; + } + + if (mndCheckWriteAuth(pUser, pDb) != 0) { + goto CREATE_STB_OVER; + } + code = mndCreateStb(pMnode, pReq, &createReq, pDb); if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; @@ -593,8 +609,8 @@ CREATE_STB_OVER: mndReleaseStb(pMnode, pStb); mndReleaseStb(pMnode, pTopicStb); mndReleaseDb(pMnode, pDb); - taosArrayDestroy(createReq.pColumns); - taosArrayDestroy(createReq.pTags); + mndReleaseUser(pMnode, pUser); + tFreeSMCreateStbReq(&createReq); return code; } @@ -965,7 +981,7 @@ static int32_t mndAlterStb(SMnode *pMnode, SMnodeMsg *pReq, const SMAltertbReq * int32_t code = -1; STrans *pTrans = NULL; SField *pField0 = taosArrayGet(pAlter->pFields, 0); - + switch (pAlter->alterType) { case TSDB_ALTER_TABLE_ADD_TAG: code = mndAddSuperTableTag(pOld, &stbObj, pAlter->pFields, pAlter->numOfFields); @@ -1020,9 +1036,13 @@ static int32_t mndProcessMAlterStbReq(SMnodeMsg *pReq) { int32_t code = -1; SDbObj *pDb = NULL; SStbObj *pStb = NULL; + SUserObj *pUser = NULL; SMAltertbReq alterReq = {0}; - if (tDeserializeSMAlterStbReq(pReq->rpcMsg.pCont, &alterReq) == NULL) goto ALTER_STB_OVER; + if (tDeserializeSMAlterStbReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &alterReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto ALTER_STB_OVER; + } mDebug("stb:%s, start to alter", alterReq.name); if (mndCheckAlterStbReq(&alterReq) != 0) goto ALTER_STB_OVER; @@ -1039,6 +1059,15 @@ static int32_t mndProcessMAlterStbReq(SMnodeMsg *pReq) { goto ALTER_STB_OVER; } + pUser = mndAcquireUser(pMnode, pReq->user); + if (pUser == NULL) { + goto ALTER_STB_OVER; + } + + if (mndCheckWriteAuth(pUser, pDb) != 0) { + goto ALTER_STB_OVER; + } + code = mndAlterStb(pMnode, pReq, &alterReq, pDb, pStb); if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; @@ -1049,6 +1078,7 @@ ALTER_STB_OVER: mndReleaseStb(pMnode, pStb); mndReleaseDb(pMnode, pDb); + mndReleaseUser(pMnode, pUser); taosArrayDestroy(alterReq.pFields); return code; @@ -1135,43 +1165,60 @@ DROP_STB_OVER: } static int32_t mndProcessMDropStbReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; - + SMnode *pMnode = pReq->pMnode; + int32_t code = -1; + SUserObj *pUser = NULL; + SDbObj *pDb = NULL; + SStbObj *pStb = NULL; SMDropStbReq dropReq = {0}; - tDeserializeSMDropStbReq(pReq->rpcMsg.pCont, &dropReq); + + if (tDeserializeSMDropStbReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &dropReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto DROP_STB_OVER; + } mDebug("stb:%s, start to drop", dropReq.name); - SStbObj *pStb = mndAcquireStb(pMnode, dropReq.name); + pStb = mndAcquireStb(pMnode, dropReq.name); if (pStb == NULL) { if (dropReq.igNotExists) { mDebug("stb:%s, not exist, ignore not exist is set", dropReq.name); - return 0; + code = 0; + goto DROP_STB_OVER; } else { terrno = TSDB_CODE_MND_STB_NOT_EXIST; - mError("stb:%s, failed to drop since %s", dropReq.name, terrstr()); - return -1; + goto DROP_STB_OVER; } } - SDbObj *pDb = mndAcquireDbByStb(pMnode, dropReq.name); + pDb = mndAcquireDbByStb(pMnode, dropReq.name); if (pDb == NULL) { - mndReleaseStb(pMnode, pStb); terrno = TSDB_CODE_MND_DB_NOT_SELECTED; - mError("stb:%s, failed to drop since %s", dropReq.name, terrstr()); - return -1; + goto DROP_STB_OVER; + } + + pUser = mndAcquireUser(pMnode, pReq->user); + if (pUser == NULL) { + goto DROP_STB_OVER; + } + + if (mndCheckWriteAuth(pUser, pDb) != 0) { + goto DROP_STB_OVER; + } + + code = mndDropStb(pMnode, pReq, pDb, pStb); + if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + +DROP_STB_OVER: + if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + mError("stb:%s, failed to drop since %s", dropReq.name, terrstr()); } - int32_t code = mndDropStb(pMnode, pReq, pDb, pStb); mndReleaseDb(pMnode, pDb); mndReleaseStb(pMnode, pStb); + mndReleaseUser(pMnode, pUser); - if (code != 0) { - mError("stb:%s, failed to drop since %s", dropReq.name, terrstr()); - return -1; - } - - return TSDB_CODE_MND_ACTION_IN_PROGRESS; + return code; } static int32_t mndProcessVDropStbRsp(SMnodeMsg *pRsp) { @@ -1179,19 +1226,59 @@ static int32_t mndProcessVDropStbRsp(SMnodeMsg *pRsp) { return 0; } -static int32_t mndProcessStbMetaReq(SMnodeMsg *pReq) { - SMnode *pMnode = pReq->pMnode; - STableInfoReq *pInfo = pReq->rpcMsg.pCont; +static int32_t mndBuildStbSchemaImp(SDbObj *pDb, SStbObj *pStb, const char *tbName, STableMetaRsp *pRsp) { + taosRLockLatch(&pStb->lock); + int32_t totalCols = pStb->numOfColumns + pStb->numOfTags; + pRsp->pSchemas = calloc(totalCols, sizeof(SSchema)); + if (pRsp->pSchemas == NULL) { + taosRUnLockLatch(&pStb->lock); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + strcpy(pRsp->dbFName, pStb->db); + strcpy(pRsp->tbName, tbName); + strcpy(pRsp->stbName, tbName); + pRsp->dbId = pDb->uid; + pRsp->numOfTags = pStb->numOfTags; + pRsp->numOfColumns = pStb->numOfColumns; + pRsp->precision = pDb->cfg.precision; + pRsp->tableType = TSDB_SUPER_TABLE; + pRsp->update = pDb->cfg.update; + pRsp->sversion = pStb->version; + pRsp->suid = pStb->uid; + pRsp->tuid = pStb->uid; + + for (int32_t i = 0; i < pStb->numOfColumns; ++i) { + SSchema *pSchema = &pRsp->pSchemas[i]; + SSchema *pSrcSchema = &pStb->pColumns[i]; + memcpy(pSchema->name, pSrcSchema->name, TSDB_COL_NAME_LEN); + pSchema->type = pSrcSchema->type; + pSchema->colId = pSrcSchema->colId; + pSchema->bytes = pSrcSchema->bytes; + } + + for (int32_t i = 0; i < pStb->numOfTags; ++i) { + SSchema *pSchema = &pRsp->pSchemas[i + pStb->numOfColumns]; + SSchema *pSrcSchema = &pStb->pTags[i]; + memcpy(pSchema->name, pSrcSchema->name, TSDB_COL_NAME_LEN); + pSchema->type = pSrcSchema->type; + pSchema->colId = pSrcSchema->colId; + pSchema->bytes = pSrcSchema->bytes; + } + + taosRUnLockLatch(&pStb->lock); + return 0; +} + +static int32_t mndBuildStbSchema(SMnode *pMnode, const char *dbFName, const char *tbName, STableMetaRsp *pRsp) { char tbFName[TSDB_TABLE_FNAME_LEN] = {0}; - snprintf(tbFName, sizeof(tbFName), "%s.%s", pInfo->dbFName, pInfo->tbName); + snprintf(tbFName, sizeof(tbFName), "%s.%s", dbFName, tbName); - mDebug("stb:%s, start to retrieve meta", tbFName); - - SDbObj *pDb = mndAcquireDb(pMnode, pInfo->dbFName); + SDbObj *pDb = mndAcquireDb(pMnode, dbFName); if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_SELECTED; - mError("stb:%s, failed to retrieve meta since %s", tbFName, terrstr()); return -1; } @@ -1199,181 +1286,108 @@ static int32_t mndProcessStbMetaReq(SMnodeMsg *pReq) { if (pStb == NULL) { mndReleaseDb(pMnode, pDb); terrno = TSDB_CODE_MND_INVALID_STB; - mError("stb:%s, failed to get meta since %s", tbFName, terrstr()); return -1; } - taosRLockLatch(&pStb->lock); - int32_t totalCols = pStb->numOfColumns + pStb->numOfTags; - int32_t contLen = sizeof(STableMetaRsp) + totalCols * sizeof(SSchema); - - STableMetaRsp *pMeta = rpcMallocCont(contLen); - if (pMeta == NULL) { - taosRUnLockLatch(&pStb->lock); - mndReleaseDb(pMnode, pDb); - mndReleaseStb(pMnode, pStb); - terrno = TSDB_CODE_OUT_OF_MEMORY; - mError("stb:%s, failed to get meta since %s", tbFName, terrstr()); - return -1; - } - - strcpy(pMeta->dbFName, pStb->db); - strcpy(pMeta->tbName, pInfo->tbName); - strcpy(pMeta->stbName, pInfo->tbName); - pMeta->dbId = htobe64(pDb->uid); - pMeta->numOfTags = htonl(pStb->numOfTags); - pMeta->numOfColumns = htonl(pStb->numOfColumns); - pMeta->precision = pDb->cfg.precision; - pMeta->tableType = TSDB_SUPER_TABLE; - pMeta->update = pDb->cfg.update; - pMeta->sversion = htonl(pStb->version); - pMeta->suid = htobe64(pStb->uid); - pMeta->tuid = htobe64(pStb->uid); - - for (int32_t i = 0; i < pStb->numOfColumns; ++i) { - SSchema *pSchema = &pMeta->pSchema[i]; - SSchema *pSrcSchema = &pStb->pColumns[i]; - memcpy(pSchema->name, pSrcSchema->name, TSDB_COL_NAME_LEN); - pSchema->type = pSrcSchema->type; - pSchema->colId = htonl(pSrcSchema->colId); - pSchema->bytes = htonl(pSrcSchema->bytes); - } - - for (int32_t i = 0; i < pStb->numOfTags; ++i) { - SSchema *pSchema = &pMeta->pSchema[i + pStb->numOfColumns]; - SSchema *pSrcSchema = &pStb->pTags[i]; - memcpy(pSchema->name, pSrcSchema->name, TSDB_COL_NAME_LEN); - pSchema->type = pSrcSchema->type; - pSchema->colId = htonl(pSrcSchema->colId); - pSchema->bytes = htonl(pSrcSchema->bytes); - } - - taosRUnLockLatch(&pStb->lock); + int32_t code = mndBuildStbSchemaImp(pDb, pStb, tbName, pRsp); mndReleaseDb(pMnode, pDb); mndReleaseStb(pMnode, pStb); - - pReq->pCont = pMeta; - pReq->contLen = contLen; - - mDebug("stb:%s, meta is retrieved, cols:%d tags:%d", tbFName, pStb->numOfColumns, pStb->numOfTags); - return 0; + return code; } -int32_t mndValidateStbInfo(SMnode *pMnode, SSTableMetaVersion *stbs, int32_t num, void **rsp, int32_t *rspLen) { - SSdb *pSdb = pMnode->pSdb; - int32_t bufSize = num * (sizeof(STableMetaRsp) + 4 * sizeof(SSchema)); - void *buf = malloc(bufSize); - int32_t len = 0; - int32_t contLen = 0; - STableMetaRsp *pRsp = NULL; +static int32_t mndProcessStbMetaReq(SMnodeMsg *pReq) { + SMnode *pMnode = pReq->pMnode; + int32_t code = -1; + STableInfoReq infoReq = {0}; + STableMetaRsp metaRsp = {0}; - for (int32_t i = 0; i < num; ++i) { - SSTableMetaVersion *stb = &stbs[i]; - stb->suid = be64toh(stb->suid); - stb->sversion = ntohs(stb->sversion); - stb->tversion = ntohs(stb->tversion); - - if ((contLen + sizeof(STableMetaRsp)) > bufSize) { - bufSize = contLen + (num - i) * (sizeof(STableMetaRsp) + 4 * sizeof(SSchema)); - buf = realloc(buf, bufSize); - } - - pRsp = (STableMetaRsp *)((char *)buf + contLen); - - strcpy(pRsp->dbFName, stb->dbFName); - strcpy(pRsp->tbName, stb->stbName); - strcpy(pRsp->stbName, stb->stbName); - - mDebug("start to retrieve meta, db:%s, stb:%s", stb->dbFName, stb->stbName); - - SDbObj *pDb = mndAcquireDb(pMnode, stb->dbFName); - if (pDb == NULL) { - pRsp->numOfColumns = -1; - pRsp->suid = htobe64(stb->suid); - contLen += sizeof(STableMetaRsp); - mWarn("db:%s, failed to require db since %s", stb->dbFName, terrstr()); - continue; - } - - char tbFName[TSDB_TABLE_FNAME_LEN] = {0}; - snprintf(tbFName, sizeof(tbFName), "%s.%s", stb->dbFName, stb->stbName); - - SStbObj *pStb = mndAcquireStb(pMnode, tbFName); - if (pStb == NULL) { - mndReleaseDb(pMnode, pDb); - pRsp->numOfColumns = -1; - pRsp->suid = htobe64(stb->suid); - contLen += sizeof(STableMetaRsp); - mWarn("stb:%s, failed to get meta since %s", tbFName, terrstr()); - continue; - } - - taosRLockLatch(&pStb->lock); - - if (stb->suid == pStb->uid && stb->sversion == pStb->version) { - taosRUnLockLatch(&pStb->lock); - mndReleaseDb(pMnode, pDb); - mndReleaseStb(pMnode, pStb); - continue; - } - - int32_t totalCols = pStb->numOfColumns + pStb->numOfTags; - int32_t len = totalCols * sizeof(SSchema); - - contLen += sizeof(STableMetaRsp) + len; - - if (contLen > bufSize) { - bufSize = contLen + (num - i - 1) * (sizeof(STableMetaRsp) + 4 * sizeof(SSchema)); - buf = realloc(buf, bufSize); - } - - pRsp->numOfTags = htonl(pStb->numOfTags); - pRsp->numOfColumns = htonl(pStb->numOfColumns); - pRsp->precision = pDb->cfg.precision; - pRsp->tableType = TSDB_SUPER_TABLE; - pRsp->update = pDb->cfg.update; - pRsp->sversion = htonl(pStb->version); - pRsp->suid = htobe64(pStb->uid); - pRsp->tuid = htobe64(pStb->uid); - - for (int32_t i = 0; i < pStb->numOfColumns; ++i) { - SSchema *pSchema = &pRsp->pSchema[i]; - SSchema *pSrcSchema = &pStb->pColumns[i]; - memcpy(pSchema->name, pSrcSchema->name, TSDB_COL_NAME_LEN); - pSchema->type = pSrcSchema->type; - pSchema->colId = htonl(pSrcSchema->colId); - pSchema->bytes = htonl(pSrcSchema->bytes); - } - - for (int32_t i = 0; i < pStb->numOfTags; ++i) { - SSchema *pSchema = &pRsp->pSchema[i + pStb->numOfColumns]; - SSchema *pSrcSchema = &pStb->pTags[i]; - memcpy(pSchema->name, pSrcSchema->name, TSDB_COL_NAME_LEN); - pSchema->type = pSrcSchema->type; - pSchema->colId = htonl(pSrcSchema->colId); - pSchema->bytes = htonl(pSrcSchema->bytes); - } - - taosRUnLockLatch(&pStb->lock); - mndReleaseDb(pMnode, pDb); - mndReleaseStb(pMnode, pStb); + if (tDeserializeSTableInfoReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &infoReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto RETRIEVE_META_OVER; } - if (contLen > 0) { - *rsp = buf; - *rspLen = contLen; - } else { - *rsp = NULL; - tfree(buf); - *rspLen = 0; + mDebug("stb:%s.%s, start to retrieve meta", infoReq.dbFName, infoReq.tbName); + if (mndBuildStbSchema(pMnode, infoReq.dbFName, infoReq.tbName, &metaRsp) != 0) { + goto RETRIEVE_META_OVER; } + int32_t rspLen = tSerializeSTableMetaRsp(NULL, 0, &metaRsp); + if (rspLen < 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto RETRIEVE_META_OVER; + } + + void *pRsp = rpcMallocCont(rspLen); + if (pRsp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto RETRIEVE_META_OVER; + } + + tSerializeSTableMetaRsp(pRsp, rspLen, &metaRsp); + pReq->pCont = pRsp; + pReq->contLen = rspLen; + code = 0; + + mDebug("stb:%s.%s, meta is retrieved", infoReq.dbFName, infoReq.tbName); + +RETRIEVE_META_OVER: + if (code != 0) { + mError("stb:%s.%s, failed to retrieve meta since %s", infoReq.dbFName, infoReq.tbName, terrstr()); + } + + tFreeSTableMetaRsp(&metaRsp); + return code; +} + +int32_t mndValidateStbInfo(SMnode *pMnode, SSTableMetaVersion *pStbVersions, int32_t numOfStbs, void **ppRsp, + int32_t *pRspLen) { + STableMetaBatchRsp batchMetaRsp = {0}; + batchMetaRsp.pArray = taosArrayInit(numOfStbs, sizeof(STableMetaRsp)); + if (batchMetaRsp.pArray == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + for (int32_t i = 0; i < numOfStbs; ++i) { + SSTableMetaVersion *pStbVersion = &pStbVersions[i]; + pStbVersion->suid = be64toh(pStbVersion->suid); + pStbVersion->sversion = ntohs(pStbVersion->sversion); + pStbVersion->tversion = ntohs(pStbVersion->tversion); + + STableMetaRsp metaRsp = {0}; + mDebug("stb:%s.%s, start to retrieve meta", pStbVersion->dbFName, pStbVersion->stbName); + if (mndBuildStbSchema(pMnode, pStbVersion->dbFName, pStbVersion->stbName, &metaRsp) != 0) { + metaRsp.numOfColumns = -1; + metaRsp.suid = pStbVersion->suid; + } + + if (pStbVersion->sversion != metaRsp.sversion) { + taosArrayPush(batchMetaRsp.pArray, &metaRsp); + } + } + + int32_t rspLen = tSerializeSTableMetaBatchRsp(NULL, 0, &batchMetaRsp); + if (rspLen < 0) { + tFreeSTableMetaBatchRsp(&batchMetaRsp); + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + void *pRsp = malloc(rspLen); + if (pRsp == NULL) { + tFreeSTableMetaBatchRsp(&batchMetaRsp); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + tSerializeSTableMetaBatchRsp(pRsp, rspLen, &batchMetaRsp); + *ppRsp = pRsp; + *pRspLen = rspLen; return 0; } static int32_t mndGetNumOfStbs(SMnode *pMnode, char *dbName, int32_t *pNumOfStbs) { - SSdb *pSdb = pMnode->pSdb; - + SSdb *pSdb = pMnode->pSdb; SDbObj *pDb = mndAcquireDb(pMnode, dbName); if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_SELECTED; @@ -1387,7 +1401,7 @@ static int32_t mndGetNumOfStbs(SMnode *pMnode, char *dbName, int32_t *pNumOfStbs pIter = sdbFetch(pSdb, SDB_STB, pIter, (void **)&pStb); if (pIter == NULL) break; - if (strcmp(pStb->db, dbName) == 0) { + if (pStb->dbUid == pDb->uid) { numOfStbs++; } @@ -1395,6 +1409,7 @@ static int32_t mndGetNumOfStbs(SMnode *pMnode, char *dbName, int32_t *pNumOfStbs } *pNumOfStbs = numOfStbs; + mndReleaseDb(pMnode, pDb); return 0; } @@ -1407,33 +1422,33 @@ static int32_t mndGetStbMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pM } int32_t cols = 0; - SSchema *pSchema = pMeta->pSchema; + SSchema *pSchema = pMeta->pSchemas; pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "name"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; strcpy(pSchema[cols].name, "create_time"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "columns"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "tags"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; - pMeta->numOfColumns = htonl(cols); + pMeta->numOfColumns = cols; pShow->numOfColumns = cols; pShow->offset[0] = 0; @@ -1482,7 +1497,7 @@ static int32_t mndRetrieveStb(SMnodeMsg *pReq, SShowObj *pShow, char *data, int3 if (pShow->pIter == NULL) break; if (pStb->dbUid != pDb->uid) { - if (strncmp(pStb->db, pDb->name, tListLen(pStb->db)) == 0) { + if (strncmp(pStb->db, pDb->name, prefixLen) == 0) { mError("Inconsistent table data, name:%s, db:%s, dbUid:%" PRIu64, pStb->name, pDb->name, pDb->uid); } diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 2a3e0008a2..840326d318 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -12,8 +12,8 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#define _DEFAULT_SOURCE +#define _DEFAULT_SOURCE #include "mndSubscribe.h" #include "mndConsumer.h" #include "mndDb.h" @@ -28,7 +28,7 @@ #include "tcompare.h" #include "tname.h" -#define MND_SUBSCRIBE_VER_NUMBER 1 +#define MND_SUBSCRIBE_VER_NUMBER 1 #define MND_SUBSCRIBE_RESERVE_SIZE 64 #define MND_SUBSCRIBE_REBALANCE_CNT 3 @@ -54,12 +54,12 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg); static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg); static int32_t mndProcessDoRebalanceMsg(SMnodeMsg *pMsg); -static int mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqTopicObj *pTopic, const char *cgroup, - const SMqConsumerEp *pConsumerEp); +static int32_t mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqTopicObj *pTopic, const char *cgroup, + const SMqConsumerEp *pConsumerEp); static int32_t mndPersistRebalanceMsg(SMnode *pMnode, STrans *pTrans, const SMqConsumerEp *pConsumerEp); -static int mndInitUnassignedVg(SMnode *pMnode, const SMqTopicObj *pTopic, SMqSubscribeObj *pSub); +static int32_t mndInitUnassignedVg(SMnode *pMnode, const SMqTopicObj *pTopic, SMqSubscribeObj *pSub); int32_t mndInitSubscribe(SMnode *pMnode) { SSdbTable table = {.sdbType = SDB_SUBSCRIBE, @@ -232,22 +232,22 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) { if (epoch != rsp.epoch) { mInfo("send new assignment to consumer, consumer epoch %d, server epoch %d", epoch, rsp.epoch); SArray *pTopics = pConsumer->currentTopics; - int sz = taosArrayGetSize(pTopics); + int32_t sz = taosArrayGetSize(pTopics); rsp.topics = taosArrayInit(sz, sizeof(SMqSubTopicEp)); - for (int i = 0; i < sz; i++) { + for (int32_t i = 0; i < sz; i++) { char *topicName = taosArrayGetP(pTopics, i); SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, pConsumer->cgroup, topicName); ASSERT(pSub); - int csz = taosArrayGetSize(pSub->consumers); + int32_t csz = taosArrayGetSize(pSub->consumers); // TODO: change to bsearch - for (int j = 0; j < csz; j++) { + for (int32_t j = 0; j < csz; j++) { SMqSubConsumer *pSubConsumer = taosArrayGet(pSub->consumers, j); if (consumerId == pSubConsumer->consumerId) { - int vgsz = taosArrayGetSize(pSubConsumer->vgInfo); + int32_t vgsz = taosArrayGetSize(pSubConsumer->vgInfo); SMqSubTopicEp topicEp; strcpy(topicEp.topic, topicName); topicEp.vgs = taosArrayInit(vgsz, sizeof(SMqSubVgEp)); - for (int k = 0; k < vgsz; k++) { + for (int32_t k = 0; k < vgsz; k++) { SMqConsumerEp *pConsumerEp = taosArrayGet(pSubConsumer->vgInfo, k); SMqSubVgEp vgEp = {.epSet = pConsumerEp->epSet, .vgId = pConsumerEp->vgId}; @@ -276,7 +276,7 @@ static int32_t mndProcessGetSubEpReq(SMnodeMsg *pMsg) { } static int32_t mndSplitSubscribeKey(char *key, char **topic, char **cgroup) { - int i = 0; + int32_t i = 0; while (key[i] != ':') { i++; } @@ -317,8 +317,8 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { atomic_val_compare_exchange_32(&pConsumer->status, MQ_CONSUMER_STATUS__ACTIVE, MQ_CONSUMER_STATUS__LOST); if (old == MQ_CONSUMER_STATUS__ACTIVE) { // get all topics of that topic - int sz = taosArrayGetSize(pConsumer->currentTopics); - for (int i = 0; i < sz; i++) { + int32_t sz = taosArrayGetSize(pConsumer->currentTopics); + for (int32_t i = 0; i < sz; i++) { char *topic = taosArrayGetP(pConsumer->currentTopics, i); char *key = mndMakeSubscribeKey(pConsumer->cgroup, topic); SMqRebSubscribe *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key); @@ -334,8 +334,8 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) { } else { rebSubs = pConsumer->recentRemovedTopics; } - int sz = taosArrayGetSize(rebSubs); - for (int i = 0; i < sz; i++) { + int32_t sz = taosArrayGetSize(rebSubs); + for (int32_t i = 0; i < sz; i++) { char *topic = taosArrayGetP(rebSubs, i); char *key = mndMakeSubscribeKey(pConsumer->cgroup, topic); SMqRebSubscribe *pRebSub = mndGetOrCreateRebSub(pRebMsg->rebSubHash, key); @@ -375,12 +375,12 @@ static int32_t mndProcessDoRebalanceMsg(SMnodeMsg *pMsg) { mInfo("mq rebalance subscription: %s", pSub->key); // remove lost consumer - for (int i = 0; i < taosArrayGetSize(pRebSub->lostConsumers); i++) { + for (int32_t i = 0; i < taosArrayGetSize(pRebSub->lostConsumers); i++) { int64_t lostConsumerId = *(int64_t *)taosArrayGet(pRebSub->lostConsumers, i); mInfo("mq remove lost consumer %ld", lostConsumerId); - for (int j = 0; j < taosArrayGetSize(pSub->consumers); j++) { + for (int32_t j = 0; j < taosArrayGetSize(pSub->consumers); j++) { SMqSubConsumer *pSubConsumer = taosArrayGet(pSub->consumers, j); if (pSubConsumer->consumerId == lostConsumerId) { taosArrayAddAll(pSub->unassignedVg, pSubConsumer->vgInfo); @@ -400,10 +400,10 @@ static int32_t mndProcessDoRebalanceMsg(SMnodeMsg *pMsg) { int32_t imbalanceSolved = 0; // iterate all consumers, set unassignedVgStash - for (int i = 0; i < consumerNum; i++) { + for (int32_t i = 0; i < consumerNum; i++) { SMqSubConsumer *pSubConsumer = taosArrayGet(pSub->consumers, i); - int vgThisConsumerBeforeRb = taosArrayGetSize(pSubConsumer->vgInfo); - int vgThisConsumerAfterRb; + int32_t vgThisConsumerBeforeRb = taosArrayGetSize(pSubConsumer->vgInfo); + int32_t vgThisConsumerAfterRb; if (i < imbalanceVg) vgThisConsumerAfterRb = vgEachConsumer + 1; else @@ -424,7 +424,9 @@ static int32_t mndProcessDoRebalanceMsg(SMnodeMsg *pMsg) { if (vgThisConsumerAfterRb != vgThisConsumerBeforeRb || (vgThisConsumerAfterRb != 0 && status != MQ_CONSUMER_STATUS__ACTIVE) || (vgThisConsumerAfterRb == 0 && status != MQ_CONSUMER_STATUS__LOST)) { - pRebConsumer->epoch++; + if (vgThisConsumerAfterRb != vgThisConsumerBeforeRb) { + pRebConsumer->epoch++; + } if (vgThisConsumerAfterRb != 0) { atomic_store_32(&pRebConsumer->status, MQ_CONSUMER_STATUS__ACTIVE); } else { @@ -442,10 +444,10 @@ static int32_t mndProcessDoRebalanceMsg(SMnodeMsg *pMsg) { // assign to vgroup if (taosArrayGetSize(pSub->unassignedVg) != 0) { - for (int i = 0; i < consumerNum; i++) { + for (int32_t i = 0; i < consumerNum; i++) { SMqSubConsumer *pSubConsumer = taosArrayGet(pSub->consumers, i); - int vgThisConsumerBeforeRb = taosArrayGetSize(pSubConsumer->vgInfo); - int vgThisConsumerAfterRb; + int32_t vgThisConsumerBeforeRb = taosArrayGetSize(pSubConsumer->vgInfo); + int32_t vgThisConsumerAfterRb; if (i < imbalanceVg) vgThisConsumerAfterRb = vgEachConsumer + 1; else @@ -602,7 +604,7 @@ static int32_t mndProcessDoRebalanceMsg(SMnodeMsg *pMsg) { #if 0 //update consumer status for the subscribption - for (int i = 0; i < taosArrayGetSize(pSub->assigned); i++) { + for (int32_t i = 0; i < taosArrayGetSize(pSub->assigned); i++) { SMqConsumerEp *pCEp = taosArrayGet(pSub->assigned, i); int64_t consumerId = pCEp->consumerId; if (pCEp->status != -1) { @@ -619,7 +621,7 @@ static int32_t mndProcessDoRebalanceMsg(SMnodeMsg *pMsg) { // TODO: swap with last one, reduce size and reset i taosArrayRemove(pSub->assigned, i); // remove from available consumer - for (int j = 0; j < taosArrayGetSize(pSub->availConsumer); j++) { + for (int32_t j = 0; j < taosArrayGetSize(pSub->availConsumer); j++) { if (*(int64_t *)taosArrayGet(pSub->availConsumer, i) == pCEp->consumerId) { taosArrayRemove(pSub->availConsumer, j); break; @@ -699,7 +701,7 @@ static int32_t mndProcessDoRebalanceMsg(SMnodeMsg *pMsg) { } #endif -static int mndInitUnassignedVg(SMnode *pMnode, const SMqTopicObj *pTopic, SMqSubscribeObj *pSub) { +static int32_t mndInitUnassignedVg(SMnode *pMnode, const SMqTopicObj *pTopic, SMqSubscribeObj *pSub) { SSdb *pSdb = pMnode->pSdb; SVgObj *pVgroup = NULL; SQueryDag *pDag = qStringToDag(pTopic->physicalPlan); @@ -742,8 +744,8 @@ static int mndInitUnassignedVg(SMnode *pMnode, const SMqTopicObj *pTopic, SMqSub return 0; } -static int mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqTopicObj *pTopic, const char *cgroup, - const SMqConsumerEp *pConsumerEp) { +static int32_t mndPersistMqSetConnReq(SMnode *pMnode, STrans *pTrans, const SMqTopicObj *pTopic, const char *cgroup, + const SMqConsumerEp *pConsumerEp) { ASSERT(pConsumerEp->oldConsumerId == -1); int32_t vgId = pConsumerEp->vgId; SVgObj *pVgObj = mndAcquireVgroup(pMnode, vgId); @@ -890,7 +892,7 @@ static char *mndMakeSubscribeKey(const char *cgroup, const char *topicName) { if (key == NULL) { return NULL; } - int tlen = strlen(cgroup); + int32_t tlen = strlen(cgroup); memcpy(key, cgroup, tlen); key[tlen] = ':'; strcpy(key + tlen + 1, topicName); @@ -931,12 +933,12 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { char *cgroup = subscribe.consumerGroup; SArray *newSub = subscribe.topicNames; - int newTopicNum = subscribe.topicNum; + int32_t newTopicNum = subscribe.topicNum; taosArraySortString(newSub, taosArrayCompareString); SArray *oldSub = NULL; - int oldTopicNum = 0; + int32_t oldTopicNum = 0; bool createConsumer = false; // create consumer if not exist SMqConsumerObj *pConsumer = mndAcquireConsumer(pMnode, consumerId); @@ -960,7 +962,7 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { return -1; } - int i = 0, j = 0; + int32_t i = 0, j = 0; while (i < newTopicNum || j < oldTopicNum) { char *newTopicName = NULL; char *oldTopicName = NULL; @@ -975,7 +977,7 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { newTopicName = taosArrayGetP(newSub, i); oldTopicName = taosArrayGetP(oldSub, j); - int comp = compareLenPrefixedStr(newTopicName, oldTopicName); + int32_t comp = compareLenPrefixedStr(newTopicName, oldTopicName); if (comp == 0) { // do nothing oldTopicName = newTopicName = NULL; @@ -997,12 +999,12 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { // cancel subscribe of old topic SMqSubscribeObj *pSub = mndAcquireSubscribe(pMnode, cgroup, oldTopicName); ASSERT(pSub); - int csz = taosArrayGetSize(pSub->consumers); - for (int ci = 0; ci < csz; ci++) { + int32_t csz = taosArrayGetSize(pSub->consumers); + for (int32_t ci = 0; ci < csz; ci++) { SMqSubConsumer *pSubConsumer = taosArrayGet(pSub->consumers, ci); if (pSubConsumer->consumerId == consumerId) { - int vgsz = taosArrayGetSize(pSubConsumer->vgInfo); - for (int vgi = 0; vgi < vgsz; vgi++) { + int32_t vgsz = taosArrayGetSize(pSubConsumer->vgInfo); + for (int32_t vgi = 0; vgi < vgsz; vgi++) { SMqConsumerEp *pConsumerEp = taosArrayGet(pSubConsumer->vgInfo, vgi); mndPersistCancelConnReq(pMnode, pTrans, pConsumerEp); taosArrayPush(pSub->unassignedVg, pConsumerEp); @@ -1046,7 +1048,7 @@ static int32_t mndProcessSubscribeReq(SMnodeMsg *pMsg) { } else { mndPersistRebalanceMsg(pMnode, pTrans, pConsumerEp); } - // do not set status active to trigger rebalance + // to trigger rebalance at once, do not set status active /*atomic_store_32(&pConsumer->status, MQ_CONSUMER_STATUS__ACTIVE);*/ } diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index ea3cc7c2c6..deac89d68a 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "mndTopic.h" +#include "mndAuth.h" #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" @@ -25,18 +26,18 @@ #include "mndVgroup.h" #include "tname.h" -#define MND_TOPIC_VER_NUMBER 1 +#define MND_TOPIC_VER_NUMBER 1 #define MND_TOPIC_RESERVE_SIZE 64 static int32_t mndTopicActionInsert(SSdb *pSdb, SMqTopicObj *pTopic); static int32_t mndTopicActionDelete(SSdb *pSdb, SMqTopicObj *pTopic); static int32_t mndTopicActionUpdate(SSdb *pSdb, SMqTopicObj *pTopic, SMqTopicObj *pNewTopic); -static int32_t mndProcessCreateTopicMsg(SMnodeMsg *pMsg); -static int32_t mndProcessDropTopicMsg(SMnodeMsg *pMsg); -static int32_t mndProcessDropTopicInRsp(SMnodeMsg *pMsg); -static int32_t mndProcessTopicMetaMsg(SMnodeMsg *pMsg); -static int32_t mndGetTopicMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta); -static int32_t mndRetrieveTopic(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows); +static int32_t mndProcessCreateTopicReq(SMnodeMsg *pReq); +static int32_t mndProcessDropTopicReq(SMnodeMsg *pReq); +static int32_t mndProcessDropTopicInRsp(SMnodeMsg *pRsp); +static int32_t mndProcessTopicMetaReq(SMnodeMsg *pReq); +static int32_t mndGetTopicMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); +static int32_t mndRetrieveTopic(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows); static void mndCancelGetNextTopic(SMnode *pMnode, void *pIter); int32_t mndInitTopic(SMnode *pMnode) { @@ -48,10 +49,14 @@ int32_t mndInitTopic(SMnode *pMnode) { .updateFp = (SdbUpdateFp)mndTopicActionUpdate, .deleteFp = (SdbDeleteFp)mndTopicActionDelete}; - mndSetMsgHandle(pMnode, TDMT_MND_CREATE_TOPIC, mndProcessCreateTopicMsg); - mndSetMsgHandle(pMnode, TDMT_MND_DROP_TOPIC, mndProcessDropTopicMsg); + mndSetMsgHandle(pMnode, TDMT_MND_CREATE_TOPIC, mndProcessCreateTopicReq); + mndSetMsgHandle(pMnode, TDMT_MND_DROP_TOPIC, mndProcessDropTopicReq); mndSetMsgHandle(pMnode, TDMT_VND_DROP_TOPIC_RSP, mndProcessDropTopicInRsp); + mndAddShowMetaHandle(pMnode, TSDB_MGMT_TABLE_TP, mndGetTopicMeta); + mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_TP, mndRetrieveTopic); + mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_TP, mndCancelGetNextTopic); + return sdbSetTable(pMnode->pSdb, table); } @@ -60,8 +65,8 @@ void mndCleanupTopic(SMnode *pMnode) {} SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) { terrno = TSDB_CODE_OUT_OF_MEMORY; - int32_t logicalPlanLen = strlen(pTopic->logicalPlan) + 1; - int32_t physicalPlanLen = strlen(pTopic->physicalPlan) + 1; + int32_t logicalPlanLen = strlen(pTopic->logicalPlan) + 1; + int32_t physicalPlanLen = strlen(pTopic->physicalPlan) + 1; int32_t size = sizeof(SMqTopicObj) + logicalPlanLen + physicalPlanLen + pTopic->sqlLen + MND_TOPIC_RESERVE_SIZE; SSdbRaw *pRaw = sdbAllocRaw(SDB_TOPIC, MND_TOPIC_VER_NUMBER, size); if (pRaw == NULL) goto TOPIC_ENCODE_OVER; @@ -78,7 +83,6 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) { SDB_SET_BINARY(pRaw, dataPos, pTopic->sql, pTopic->sqlLen, TOPIC_ENCODE_OVER); SDB_SET_INT32(pRaw, dataPos, logicalPlanLen, TOPIC_ENCODE_OVER); SDB_SET_BINARY(pRaw, dataPos, pTopic->logicalPlan, logicalPlanLen, TOPIC_ENCODE_OVER); - SDB_SET_INT32(pRaw, dataPos, physicalPlanLen, TOPIC_ENCODE_OVER); SDB_SET_BINARY(pRaw, dataPos, pTopic->physicalPlan, physicalPlanLen, TOPIC_ENCODE_OVER); @@ -127,7 +131,7 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &pTopic->sqlLen, TOPIC_DECODE_OVER); pTopic->sql = calloc(pTopic->sqlLen + 1, sizeof(char)); - SDB_GET_BINARY(pRaw, dataPos, pTopic->sql, pTopic->sqlLen, TOPIC_DECODE_OVER); + SDB_GET_BINARY(pRaw, dataPos, pTopic->sql, pTopic->sqlLen, TOPIC_DECODE_OVER); SDB_GET_INT32(pRaw, dataPos, &len, TOPIC_DECODE_OVER); pTopic->logicalPlan = calloc(len + 1, sizeof(char)); @@ -187,7 +191,7 @@ static int32_t mndTopicActionUpdate(SSdb *pSdb, SMqTopicObj *pOldTopic, SMqTopic SMqTopicObj *mndAcquireTopic(SMnode *pMnode, char *topicName) { SSdb *pSdb = pMnode->pSdb; SMqTopicObj *pTopic = sdbAcquire(pSdb, SDB_TOPIC, topicName); - if (pTopic == NULL) { + if (pTopic == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { terrno = TSDB_CODE_MND_TOPIC_NOT_EXIST; } return pTopic; @@ -225,12 +229,15 @@ static SDDropTopicReq *mndBuildDropTopicMsg(SMnode *pMnode, SVgObj *pVgroup, SMq return pDrop; } -static int32_t mndCheckCreateTopicMsg(SCMCreateTopicReq *creattopReq) { - // deserialize and other stuff +static int32_t mndCheckCreateTopicReq(SMCreateTopicReq *pCreate) { + if (pCreate->name[0] == 0 || pCreate->sql == NULL || pCreate->sql[0] == 0) { + terrno = TSDB_CODE_MND_INVALID_TOPIC_OPTION; + return -1; + } return 0; } -static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pMsg, SCMCreateTopicReq *pCreate, SDbObj *pDb) { +static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pReq, SMCreateTopicReq *pCreate, SDbObj *pDb) { mDebug("topic:%s to create", pCreate->name); SMqTopicObj topicObj = {0}; tstrncpy(topicObj.name, pCreate->name, TSDB_TOPIC_FNAME_LEN); @@ -245,100 +252,172 @@ static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pMsg, SCMCreateTopicReq topicObj.logicalPlan = pCreate->logicalPlan; topicObj.sqlLen = strlen(pCreate->sql); - SSdbRaw *pTopicRaw = mndTopicActionEncode(&topicObj); - if (pTopicRaw == NULL) return -1; - if (sdbSetRawStatus(pTopicRaw, SDB_STATUS_READY) != 0) return -1; - return sdbWrite(pMnode->pSdb, pTopicRaw); + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pReq->rpcMsg); + if (pTrans == NULL) { + mError("topic:%s, failed to create since %s", pCreate->name, terrstr()); + return -1; + } + mDebug("trans:%d, used to create topic:%s", pTrans->id, pCreate->name); + + SSdbRaw *pRedoRaw = mndTopicActionEncode(&topicObj); + if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { + mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY); + + if (mndTransPrepare(pMnode, pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + + mndTransDrop(pTrans); + return 0; } -static int32_t mndProcessCreateTopicMsg(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; - char *msgStr = pMsg->rpcMsg.pCont; +static int32_t mndProcessCreateTopicReq(SMnodeMsg *pReq) { + SMnode *pMnode = pReq->pMnode; + int32_t code = -1; + SMqTopicObj *pTopic = NULL; + SDbObj *pDb = NULL; + SUserObj *pUser = NULL; + SMCreateTopicReq createTopicReq = {0}; - SCMCreateTopicReq createTopicReq = {0}; - tDeserializeSCMCreateTopicReq(msgStr, &createTopicReq); + if (tDeserializeSMCreateTopicReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &createTopicReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto CREATE_TOPIC_OVER; + } mDebug("topic:%s, start to create, sql:%s", createTopicReq.name, createTopicReq.sql); - if (mndCheckCreateTopicMsg(&createTopicReq) != 0) { + if (mndCheckCreateTopicReq(&createTopicReq) != 0) { mError("topic:%s, failed to create since %s", createTopicReq.name, terrstr()); - return -1; + goto CREATE_TOPIC_OVER; } - SMqTopicObj *pTopic = mndAcquireTopic(pMnode, createTopicReq.name); + pTopic = mndAcquireTopic(pMnode, createTopicReq.name); if (pTopic != NULL) { - sdbRelease(pMnode->pSdb, pTopic); if (createTopicReq.igExists) { mDebug("topic:%s, already exist, ignore exist is set", createTopicReq.name); - return 0; + code = 0; + goto CREATE_TOPIC_OVER; } else { terrno = TSDB_CODE_MND_TOPIC_ALREADY_EXIST; - mError("topic:%s, failed to create since already exists", createTopicReq.name); - return -1; + goto CREATE_TOPIC_OVER; } + } else if (terrno != TSDB_CODE_MND_TOPIC_NOT_EXIST) { + goto CREATE_TOPIC_OVER; } - SDbObj *pDb = mndAcquireDbByTopic(pMnode, createTopicReq.name); + pDb = mndAcquireDbByTopic(pMnode, createTopicReq.name); if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_SELECTED; - mError("topic:%s, failed to create since %s", createTopicReq.name, terrstr()); - return -1; + goto CREATE_TOPIC_OVER; } - int32_t code = mndCreateTopic(pMnode, pMsg, &createTopicReq, pDb); + pUser = mndAcquireUser(pMnode, pReq->user); + if (pUser == NULL) { + goto CREATE_TOPIC_OVER; + } + + if (mndCheckWriteAuth(pUser, pDb) != 0) { + goto CREATE_TOPIC_OVER; + } + + code = mndCreateTopic(pMnode, pReq, &createTopicReq, pDb); + if (code == 0) code = TSDB_CODE_MND_ACTION_IN_PROGRESS; + +CREATE_TOPIC_OVER: + if (code != 0 && code != TSDB_CODE_MND_ACTION_IN_PROGRESS) { + mError("topic:%s, failed to create since %s", createTopicReq.name, terrstr()); + } + + mndReleaseTopic(pMnode, pTopic); mndReleaseDb(pMnode, pDb); + mndReleaseUser(pMnode, pUser); - if (code != 0) { - terrno = code; - mError("topic:%s, failed to create since %s", createTopicReq.name, terrstr()); - return -1; - } - - return TSDB_CODE_SUCCESS; + tFreeSMCreateTopicReq(&createTopicReq); + return code; } -static int32_t mndDropTopic(SMnode *pMnode, SMnodeMsg *pMsg, SMqTopicObj *pTopic) { return 0; } +static int32_t mndDropTopic(SMnode *pMnode, SMnodeMsg *pReq, SMqTopicObj *pTopic) { + STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pReq->rpcMsg); + if (pTrans == NULL) { + mError("topic:%s, failed to drop since %s", pTopic->name, terrstr()); + return -1; + } + mDebug("trans:%d, used to drop topic:%s", pTrans->id, pTopic->name); -static int32_t mndProcessDropTopicMsg(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; - SDropTopicReq *pDrop = pMsg->rpcMsg.pCont; + SSdbRaw *pRedoRaw = mndTopicActionEncode(pTopic); + if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) { + mError("trans:%d, failed to append redo log since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } + sdbSetRawStatus(pRedoRaw, SDB_STATUS_DROPPED); - mDebug("topic:%s, start to drop", pDrop->name); + if (mndTransPrepare(pMnode, pTrans) != 0) { + mError("trans:%d, failed to prepare since %s", pTrans->id, terrstr()); + mndTransDrop(pTrans); + return -1; + } - SMqTopicObj *pTopic = mndAcquireTopic(pMnode, pDrop->name); + mndTransDrop(pTrans); + return 0; +} + +static int32_t mndProcessDropTopicReq(SMnodeMsg *pReq) { + SMnode *pMnode = pReq->pMnode; + SMDropTopicReq dropReq = {0}; + + if (tDeserializeSMDropTopicReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &dropReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + mDebug("topic:%s, start to drop", dropReq.name); + + SMqTopicObj *pTopic = mndAcquireTopic(pMnode, dropReq.name); if (pTopic == NULL) { - if (pDrop->igNotExists) { - mDebug("topic:%s, not exist, ignore not exist is set", pDrop->name); + if (dropReq.igNotExists) { + mDebug("topic:%s, not exist, ignore not exist is set", dropReq.name); return 0; } else { terrno = TSDB_CODE_MND_TOPIC_NOT_EXIST; - mError("topic:%s, failed to drop since %s", pDrop->name, terrstr()); + mError("topic:%s, failed to drop since %s", dropReq.name, terrstr()); return -1; } } - int32_t code = mndDropTopic(pMnode, pMsg, pTopic); + int32_t code = mndDropTopic(pMnode, pReq, pTopic); mndReleaseTopic(pMnode, pTopic); if (code != 0) { terrno = code; - mError("topic:%s, failed to drop since %s", pDrop->name, terrstr()); + mError("topic:%s, failed to drop since %s", dropReq.name, terrstr()); return -1; } return TSDB_CODE_MND_ACTION_IN_PROGRESS; } -static int32_t mndProcessDropTopicInRsp(SMnodeMsg *pMsg) { - mndTransProcessRsp(pMsg); +static int32_t mndProcessDropTopicInRsp(SMnodeMsg *pRsp) { + mndTransProcessRsp(pRsp); return 0; } -static int32_t mndProcessTopicMetaMsg(SMnodeMsg *pMsg) { - SMnode *pMnode = pMsg->pMnode; - STableInfoReq *pInfo = pMsg->rpcMsg.pCont; +static int32_t mndProcessTopicMetaReq(SMnodeMsg *pReq) { + SMnode *pMnode = pReq->pMnode; + STableInfoReq infoReq = {0}; - mDebug("topic:%s, start to retrieve meta", pInfo->tbName); + if (tSerializeSTableInfoReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &infoReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + mDebug("topic:%s, start to retrieve meta", infoReq.tbName); #if 0 SDbObj *pDb = mndAcquireDbByTopic(pMnode, pInfo->tableFname); @@ -380,7 +459,7 @@ static int32_t mndProcessTopicMetaMsg(SMnodeMsg *pMsg) { pMeta->tuid = htonl(pTopic->uid); for (int32_t i = 0; i < totalCols; ++i) { - SSchema *pSchema = &pMeta->pSchema[i]; + SSchema *pSchema = &pMeta->pSchemas[i]; SSchema *pSrcSchema = &pTopic->pSchema[i]; memcpy(pSchema->name, pSrcSchema->name, TSDB_COL_NAME_LEN); pSchema->type = pSrcSchema->type; @@ -391,8 +470,8 @@ static int32_t mndProcessTopicMetaMsg(SMnodeMsg *pMsg) { mndReleaseDb(pMnode, pDb); mndReleaseTopic(pMnode, pTopic); - pMsg->pCont = pMeta; - pMsg->contLen = contLen; + pReq->pCont = pMeta; + pReq->contLen = contLen; mDebug("topic:%s, meta is retrieved, cols:%d tags:%d", pInfo->tableFname, pTopic->numOfColumns, pTopic->numOfTags); #endif @@ -400,8 +479,7 @@ static int32_t mndProcessTopicMetaMsg(SMnodeMsg *pMsg) { } static int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTopics) { - SSdb *pSdb = pMnode->pSdb; - + SSdb *pSdb = pMnode->pSdb; SDbObj *pDb = mndAcquireDb(pMnode, dbName); if (pDb == NULL) { terrno = TSDB_CODE_MND_DB_NOT_SELECTED; @@ -415,17 +493,20 @@ static int32_t mndGetNumOfTopics(SMnode *pMnode, char *dbName, int32_t *pNumOfTo pIter = sdbFetch(pSdb, SDB_TOPIC, pIter, (void **)&pTopic); if (pIter == NULL) break; - numOfTopics++; + if (pTopic->dbUid == pDb->uid) { + numOfTopics++; + } sdbRelease(pSdb, pTopic); } *pNumOfTopics = numOfTopics; + mndReleaseDb(pMnode, pDb); return 0; } -static int32_t mndGetTopicMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp *pMeta) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndGetTopicMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta) { + SMnode *pMnode = pReq->pMnode; SSdb *pSdb = pMnode->pSdb; if (mndGetNumOfTopics(pMnode, pShow->db, &pShow->numOfRows) != 0) { @@ -433,33 +514,27 @@ static int32_t mndGetTopicMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp * } int32_t cols = 0; - SSchema *pSchema = pMeta->pSchema; + SSchema *pSchema = pMeta->pSchemas; pShow->bytes[cols] = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "name"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; strcpy(pSchema[cols].name, "create_time"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; - pShow->bytes[cols] = 4; - pSchema[cols].type = TSDB_DATA_TYPE_INT; - strcpy(pSchema[cols].name, "columns"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pShow->bytes[cols] = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE; + pSchema[cols].type = TSDB_DATA_TYPE_BINARY; + strcpy(pSchema[cols].name, "sql"); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; - pShow->bytes[cols] = 4; - pSchema[cols].type = TSDB_DATA_TYPE_INT; - strcpy(pSchema[cols].name, "tags"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); - cols++; - - pMeta->numOfColumns = htonl(cols); + pMeta->numOfColumns = cols; pShow->numOfColumns = cols; pShow->offset[0] = 0; @@ -474,29 +549,19 @@ static int32_t mndGetTopicMeta(SMnodeMsg *pMsg, SShowObj *pShow, STableMetaRsp * return 0; } -static void mndExtractTableName(char *tableId, char *name) { - int32_t pos = -1; - int32_t num = 0; - for (pos = 0; tableId[pos] != 0; ++pos) { - if (tableId[pos] == '.') num++; - if (num == 2) break; - } - - if (num == 2) { - strcpy(name, tableId + pos + 1); - } -} - -static int32_t mndRetrieveTopic(SMnodeMsg *pMsg, SShowObj *pShow, char *data, int32_t rows) { - SMnode *pMnode = pMsg->pMnode; +static int32_t mndRetrieveTopic(SMnodeMsg *pReq, SShowObj *pShow, char *data, int32_t rows) { + SMnode *pMnode = pReq->pMnode; SSdb *pSdb = pMnode->pSdb; int32_t numOfRows = 0; SMqTopicObj *pTopic = NULL; int32_t cols = 0; char *pWrite; - char prefix[64] = {0}; + char prefix[TSDB_DB_FNAME_LEN] = {0}; - tstrncpy(prefix, pShow->db, 64); + SDbObj *pDb = mndAcquireDb(pMnode, pShow->db); + if (pDb == NULL) return 0; + + tstrncpy(prefix, pShow->db, TSDB_DB_FNAME_LEN); strcat(prefix, TS_PATH_DELIMITER); int32_t prefixLen = (int32_t)strlen(prefix); @@ -504,7 +569,11 @@ static int32_t mndRetrieveTopic(SMnodeMsg *pMsg, SShowObj *pShow, char *data, in pShow->pIter = sdbFetch(pSdb, SDB_TOPIC, pShow->pIter, (void **)&pTopic); if (pShow->pIter == NULL) break; - if (strncmp(pTopic->name, prefix, prefixLen) != 0) { + if (pTopic->dbUid != pDb->uid) { + if (strncmp(pTopic->name, prefix, prefixLen) != 0) { + mError("Inconsistent table data, name:%s, db:%s, dbUid:%" PRIu64, pTopic->name, pDb->name, pDb->uid); + } + sdbRelease(pSdb, pTopic); continue; } @@ -521,18 +590,15 @@ static int32_t mndRetrieveTopic(SMnodeMsg *pMsg, SShowObj *pShow, char *data, in *(int64_t *)pWrite = pTopic->createTime; cols++; - /*pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;*/ - /**(int32_t *)pWrite = pTopic->numOfColumns;*/ - /*cols++;*/ - - /*pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;*/ - /**(int32_t *)pWrite = pTopic->numOfTags;*/ - /*cols++;*/ + pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows; + STR_WITH_MAXSIZE_TO_VARSTR(pWrite, pTopic->sql, pShow->bytes[cols]); + cols++; numOfRows++; sdbRelease(pSdb, pTopic); } + mndReleaseDb(pMnode, pDb); pShow->numOfReads += numOfRows; mndVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow); return numOfRows; diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 6686c0887f..3230074add 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -542,12 +542,18 @@ static void mndTransSendRpcRsp(STrans *pTrans) { } if (sendRsp && pTrans->rpcHandle != NULL) { + void *rpcCont = rpcMallocCont(pTrans->rpcRspLen); + if (rpcCont != NULL) { + memcpy(rpcCont, pTrans->rpcRsp, pTrans->rpcRspLen); + } + free(pTrans->rpcRsp); + mDebug("trans:%d, send rsp, code:0x%x stage:%d app:%p", pTrans->id, pTrans->code & 0xFFFF, pTrans->stage, pTrans->rpcAHandle); SRpcMsg rspMsg = {.handle = pTrans->rpcHandle, .code = pTrans->code, .ahandle = pTrans->rpcAHandle, - .pCont = pTrans->rpcRsp, + .pCont = rpcCont, .contLen = pTrans->rpcRspLen}; rpcSendResponse(&rspMsg); pTrans->rpcHandle = NULL; diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 5c290c9c4f..e40f76daa1 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -302,7 +302,10 @@ static int32_t mndProcessCreateUserReq(SMnodeMsg *pReq) { SUserObj *pOperUser = NULL; SCreateUserReq createReq = {0}; - if (tDeserializeSCreateUserReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &createReq) != 0) goto CREATE_USER_OVER; + if (tDeserializeSCreateUserReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &createReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto CREATE_USER_OVER; + } mDebug("user:%s, start to create", createReq.user); @@ -402,7 +405,10 @@ static int32_t mndProcessAlterUserReq(SMnodeMsg *pReq) { SUserObj newUser = {0}; SAlterUserReq alterReq = {0}; - if (tDeserializeSAlterUserReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &alterReq) != 0) goto ALTER_USER_OVER; + if (tDeserializeSAlterUserReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &alterReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto ALTER_USER_OVER; + } mDebug("user:%s, start to alter", alterReq.user); @@ -537,7 +543,10 @@ static int32_t mndProcessDropUserReq(SMnodeMsg *pReq) { SUserObj *pOperUser = NULL; SDropUserReq dropReq = {0}; - if (tDeserializeSDropUserReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &dropReq) != 0) goto DROP_USER_OVER; + if (tDeserializeSDropUserReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &dropReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto DROP_USER_OVER; + } mDebug("user:%s, start to drop", dropReq.user); @@ -583,7 +592,10 @@ static int32_t mndProcessGetUserAuthReq(SMnodeMsg *pReq) { SGetUserAuthReq authReq = {0}; SGetUserAuthRsp authRsp = {0}; - if (tDeserializeSGetUserAuthReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &authReq) != 0) goto GET_AUTH_OVER; + if (tDeserializeSGetUserAuthReq(pReq->rpcMsg.pCont, pReq->rpcMsg.contLen, &authReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto GET_AUTH_OVER; + } mTrace("user:%s, start to get auth", authReq.user); @@ -640,33 +652,33 @@ static int32_t mndGetUserMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp *p SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; - SSchema *pSchema = pMeta->pSchema; + SSchema *pSchema = pMeta->pSchemas; pShow->bytes[cols] = TSDB_USER_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "name"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = 10 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "privilege"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = 8; pSchema[cols].type = TSDB_DATA_TYPE_TIMESTAMP; strcpy(pSchema[cols].name, "create_time"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = TSDB_USER_LEN + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "account"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; - pMeta->numOfColumns = htonl(cols); + pMeta->numOfColumns = cols; pShow->numOfColumns = cols; pShow->offset[0] = 0; diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 33aa4ce07c..a2438a3b8e 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -189,78 +189,95 @@ void mndReleaseVgroup(SMnode *pMnode, SVgObj *pVgroup) { sdbRelease(pSdb, pVgroup); } -SCreateVnodeReq *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup) { - SCreateVnodeReq *pCreate = calloc(1, sizeof(SCreateVnodeReq)); - if (pCreate == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return NULL; - } - - pCreate->vgId = htonl(pVgroup->vgId); - pCreate->dnodeId = htonl(pDnode->id); - memcpy(pCreate->db, pDb->name, TSDB_DB_FNAME_LEN); - pCreate->dbUid = htobe64(pDb->uid); - pCreate->vgVersion = htonl(pVgroup->version); - pCreate->cacheBlockSize = htonl(pDb->cfg.cacheBlockSize); - pCreate->totalBlocks = htonl(pDb->cfg.totalBlocks); - pCreate->daysPerFile = htonl(pDb->cfg.daysPerFile); - pCreate->daysToKeep0 = htonl(pDb->cfg.daysToKeep0); - pCreate->daysToKeep1 = htonl(pDb->cfg.daysToKeep1); - pCreate->daysToKeep2 = htonl(pDb->cfg.daysToKeep2); - pCreate->minRows = htonl(pDb->cfg.minRows); - pCreate->maxRows = htonl(pDb->cfg.maxRows); - pCreate->commitTime = htonl(pDb->cfg.commitTime); - pCreate->fsyncPeriod = htonl(pDb->cfg.fsyncPeriod); - pCreate->walLevel = pDb->cfg.walLevel; - pCreate->precision = pDb->cfg.precision; - pCreate->compression = pDb->cfg.compression; - pCreate->quorum = pDb->cfg.quorum; - pCreate->update = pDb->cfg.update; - pCreate->cacheLastRow = pDb->cfg.cacheLastRow; - pCreate->replica = pVgroup->replica; - pCreate->selfIndex = -1; +void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, int32_t *pContLen) { + SCreateVnodeReq createReq = {0}; + createReq.vgId = pVgroup->vgId; + createReq.dnodeId = pDnode->id; + memcpy(createReq.db, pDb->name, TSDB_DB_FNAME_LEN); + createReq.dbUid = pDb->uid; + createReq.vgVersion = pVgroup->version; + createReq.cacheBlockSize = pDb->cfg.cacheBlockSize; + createReq.totalBlocks = pDb->cfg.totalBlocks; + createReq.daysPerFile = pDb->cfg.daysPerFile; + createReq.daysToKeep0 = pDb->cfg.daysToKeep0; + createReq.daysToKeep1 = pDb->cfg.daysToKeep1; + createReq.daysToKeep2 = pDb->cfg.daysToKeep2; + createReq.minRows = pDb->cfg.minRows; + createReq.maxRows = pDb->cfg.maxRows; + createReq.commitTime = pDb->cfg.commitTime; + createReq.fsyncPeriod = pDb->cfg.fsyncPeriod; + createReq.walLevel = pDb->cfg.walLevel; + createReq.precision = pDb->cfg.precision; + createReq.compression = pDb->cfg.compression; + createReq.quorum = pDb->cfg.quorum; + createReq.update = pDb->cfg.update; + createReq.cacheLastRow = pDb->cfg.cacheLastRow; + createReq.replica = pVgroup->replica; + createReq.selfIndex = -1; for (int32_t v = 0; v < pVgroup->replica; ++v) { - SReplica *pReplica = &pCreate->replicas[v]; + SReplica *pReplica = &createReq.replicas[v]; SVnodeGid *pVgid = &pVgroup->vnodeGid[v]; SDnodeObj *pVgidDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); if (pVgidDnode == NULL) { - free(pCreate); return NULL; } - pReplica->id = htonl(pVgidDnode->id); - pReplica->port = htons(pVgidDnode->port); + pReplica->id = pVgidDnode->id; + pReplica->port = pVgidDnode->port; memcpy(pReplica->fqdn, pVgidDnode->fqdn, TSDB_FQDN_LEN); mndReleaseDnode(pMnode, pVgidDnode); if (pDnode->id == pVgid->dnodeId) { - pCreate->selfIndex = v; + createReq.selfIndex = v; } } - if (pCreate->selfIndex == -1) { - free(pCreate); + if (createReq.selfIndex == -1) { terrno = TSDB_CODE_MND_APP_ERROR; return NULL; } - return pCreate; -} - -SDropVnodeReq *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup) { - SDropVnodeReq *pDrop = calloc(1, sizeof(SDropVnodeReq)); - if (pDrop == NULL) { + int32_t contLen = tSerializeSCreateVnodeReq(NULL, 0, &createReq); + if (contLen < 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - pDrop->dnodeId = htonl(pDnode->id); - pDrop->vgId = htonl(pVgroup->vgId); - memcpy(pDrop->db, pDb->name, TSDB_DB_FNAME_LEN); - pDrop->dbUid = htobe64(pDb->uid); + void *pReq = malloc(contLen); + if (pReq == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } - return pDrop; + tSerializeSCreateVnodeReq(pReq, contLen, &createReq); + *pContLen = contLen; + return pReq; +} + +void *mndBuildDropVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVgObj *pVgroup, + int32_t *pContLen) { + SDropVnodeReq dropReq = {0}; + dropReq.dnodeId = pDnode->id; + dropReq.vgId = pVgroup->vgId; + memcpy(dropReq.db, pDb->name, TSDB_DB_FNAME_LEN); + dropReq.dbUid = pDb->uid; + + int32_t contLen = tSerializeSDropVnodeReq(NULL, 0, &dropReq); + if (contLen < 0) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + void *pReq = malloc(contLen); + if (pReq == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + tSerializeSDropVnodeReq(pReq, contLen, &dropReq); + *pContLen = contLen; + return pReq; } static bool mndResetDnodesArrayFp(SMnode *pMnode, void *pObj, void *p1, void *p2, void *p3) { @@ -488,35 +505,35 @@ static int32_t mndGetVgroupMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp } int32_t cols = 0; - SSchema *pSchema = pMeta->pSchema; + SSchema *pSchema = pMeta->pSchemas; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "vgId"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "tables"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; for (int32_t i = 0; i < pShow->replica; ++i) { pShow->bytes[cols] = 2; pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT; snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%d_dnode", i + 1); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = 9 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%d_status", i + 1); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; } - pMeta->numOfColumns = htonl(cols); + pMeta->numOfColumns = cols; pShow->numOfColumns = cols; pShow->offset[0] = 0; @@ -608,21 +625,21 @@ static int32_t mndGetVnodeMeta(SMnodeMsg *pReq, SShowObj *pShow, STableMetaRsp * SSdb *pSdb = pMnode->pSdb; int32_t cols = 0; - SSchema *pSchema = pMeta->pSchema; + SSchema *pSchema = pMeta->pSchemas; pShow->bytes[cols] = 4; pSchema[cols].type = TSDB_DATA_TYPE_INT; strcpy(pSchema[cols].name, "vgId"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; pShow->bytes[cols] = 12 + VARSTR_HEADER_SIZE; pSchema[cols].type = TSDB_DATA_TYPE_BINARY; strcpy(pSchema[cols].name, "status"); - pSchema[cols].bytes = htonl(pShow->bytes[cols]); + pSchema[cols].bytes = pShow->bytes[cols]; cols++; - pMeta->numOfColumns = htonl(cols); + pMeta->numOfColumns = cols; pShow->numOfColumns = cols; pShow->offset[0] = 0; diff --git a/source/dnode/mnode/impl/src/mnode.c b/source/dnode/mnode/impl/src/mnode.c index 211cfbd1f0..699ccab92c 100644 --- a/source/dnode/mnode/impl/src/mnode.c +++ b/source/dnode/mnode/impl/src/mnode.c @@ -60,11 +60,25 @@ void mndSendRedirectRsp(SMnode *pMnode, SRpcMsg *pMsg) { } } +static void *mndBuildTimerMsg(int32_t *pContLen) { + SMTimerReq timerReq = {0}; + + int32_t contLen = tSerializeSMTimerMsg(NULL, 0, &timerReq); + if (contLen <= 0) return NULL; + void *pReq = rpcMallocCont(contLen); + if (pReq == NULL) return NULL; + + tSerializeSMTimerMsg(pReq, contLen, &timerReq); + *pContLen = contLen; + return pReq; +} + static void mndTransReExecute(void *param, void *tmrId) { SMnode *pMnode = param; if (mndIsMaster(pMnode)) { - STransReq *pMsg = rpcMallocCont(sizeof(STransReq)); - SRpcMsg rpcMsg = {.msgType = TDMT_MND_TRANS, .pCont = pMsg, .contLen = sizeof(STransReq)}; + int32_t contLen = 0; + void *pReq = mndBuildTimerMsg(&contLen); + SRpcMsg rpcMsg = {.msgType = TDMT_MND_TRANS, .pCont = pReq, .contLen = contLen}; pMnode->putReqToMWriteQFp(pMnode->pDnode, &rpcMsg); } @@ -74,8 +88,9 @@ static void mndTransReExecute(void *param, void *tmrId) { static void mndCalMqRebalance(void *param, void *tmrId) { SMnode *pMnode = param; if (mndIsMaster(pMnode)) { - SMqTmrMsg *pMsg = rpcMallocCont(sizeof(SMqTmrMsg)); - SRpcMsg rpcMsg = {.msgType = TDMT_MND_MQ_TIMER, .pCont = pMsg, .contLen = sizeof(SMqTmrMsg)}; + int32_t contLen = 0; + void *pReq = mndBuildTimerMsg(&contLen); + SRpcMsg rpcMsg = {.msgType = TDMT_MND_MQ_TIMER, .pCont = pReq, .contLen = contLen}; pMnode->putReqToMReadQFp(pMnode->pDnode, &rpcMsg); } diff --git a/source/dnode/mnode/impl/test/CMakeLists.txt b/source/dnode/mnode/impl/test/CMakeLists.txt index 5f6f2f3b4f..df0510f783 100644 --- a/source/dnode/mnode/impl/test/CMakeLists.txt +++ b/source/dnode/mnode/impl/test/CMakeLists.txt @@ -13,3 +13,4 @@ add_subdirectory(mnode) add_subdirectory(db) add_subdirectory(stb) add_subdirectory(func) +add_subdirectory(topic) diff --git a/source/dnode/mnode/impl/test/acct/acct.cpp b/source/dnode/mnode/impl/test/acct/acct.cpp index 315f23f798..2bc4f40614 100644 --- a/source/dnode/mnode/impl/test/acct/acct.cpp +++ b/source/dnode/mnode/impl/test/acct/acct.cpp @@ -56,10 +56,13 @@ TEST_F(MndTestAcct, 03_Drop_Acct) { } TEST_F(MndTestAcct, 04_Show_Acct) { - int32_t contLen = sizeof(SShowReq); + SShowReq showReq = {0}; + showReq.type = TSDB_MGMT_TABLE_ACCT; - SShowReq* pReq = (SShowReq*)rpcMallocCont(contLen); - pReq->type = TSDB_MGMT_TABLE_ACCT; + int32_t contLen = tSerializeSShowReq(NULL, 0, &showReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSShowReq(pReq, contLen, &showReq); + tFreeSShowReq(&showReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_SHOW, pReq, contLen); ASSERT_NE(pRsp, nullptr); diff --git a/source/dnode/mnode/impl/test/db/db.cpp b/source/dnode/mnode/impl/test/db/db.cpp index 2d1574467e..a3efad8aa2 100644 --- a/source/dnode/mnode/impl/test/db/db.cpp +++ b/source/dnode/mnode/impl/test/db/db.cpp @@ -53,29 +53,31 @@ TEST_F(MndTestDb, 01_ShowDb) { TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) { { - int32_t contLen = sizeof(SCreateDbReq); + SCreateDbReq createReq = {0}; + strcpy(createReq.db, "1.d1"); + createReq.numOfVgroups = 2; + createReq.cacheBlockSize = 16; + createReq.totalBlocks = 10; + createReq.daysPerFile = 10; + createReq.daysToKeep0 = 3650; + createReq.daysToKeep1 = 3650; + createReq.daysToKeep2 = 3650; + createReq.minRows = 100; + createReq.maxRows = 4096; + createReq.commitTime = 3600; + createReq.fsyncPeriod = 3000; + createReq.walLevel = 1; + createReq.precision = 0; + createReq.compression = 2; + createReq.replications = 1; + createReq.quorum = 1; + createReq.update = 0; + createReq.cacheLastRow = 0; + createReq.ignoreExist = 1; - SCreateDbReq* pReq = (SCreateDbReq*)rpcMallocCont(contLen); - strcpy(pReq->db, "1.d1"); - pReq->numOfVgroups = htonl(2); - pReq->cacheBlockSize = htonl(16); - pReq->totalBlocks = htonl(10); - pReq->daysPerFile = htonl(10); - pReq->daysToKeep0 = htonl(3650); - pReq->daysToKeep1 = htonl(3650); - pReq->daysToKeep2 = htonl(3650); - pReq->minRows = htonl(100); - pReq->maxRows = htonl(4096); - pReq->commitTime = htonl(3600); - pReq->fsyncPeriod = htonl(3000); - pReq->walLevel = 1; - pReq->precision = 0; - pReq->compression = 2; - pReq->replications = 1; - pReq->quorum = 1; - pReq->update = 0; - pReq->cacheLastRow = 0; - pReq->ignoreExist = 1; + int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateDbReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -125,18 +127,20 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) { CheckBinary("master", 9); { - int32_t contLen = sizeof(SAlterDbReq); + SAlterDbReq alterdbReq = {0}; + strcpy(alterdbReq.db, "1.d1"); + alterdbReq.totalBlocks = 12; + alterdbReq.daysToKeep0 = 300; + alterdbReq.daysToKeep1 = 400; + alterdbReq.daysToKeep2 = 500; + alterdbReq.fsyncPeriod = 4000; + alterdbReq.walLevel = 2; + alterdbReq.quorum = 2; + alterdbReq.cacheLastRow = 1; - SAlterDbReq* pReq = (SAlterDbReq*)rpcMallocCont(contLen); - strcpy(pReq->db, "1.d1"); - pReq->totalBlocks = htonl(12); - pReq->daysToKeep0 = htonl(300); - pReq->daysToKeep1 = htonl(400); - pReq->daysToKeep2 = htonl(500); - pReq->fsyncPeriod = htonl(4000); - pReq->walLevel = 2; - pReq->quorum = 2; - pReq->cacheLastRow = 1; + int32_t contLen = tSerializeSAlterDbReq(NULL, 0, &alterdbReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSAlterDbReq(pReq, contLen, &alterdbReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_ALTER_DB, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -194,18 +198,20 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) { CheckInt8(0); // update { - int32_t contLen = sizeof(SDropDbReq); + SDropDbReq dropdbReq = {0}; + strcpy(dropdbReq.db, "1.d1"); - SDropDbReq* pReq = (SDropDbReq*)rpcMallocCont(contLen); - strcpy(pReq->db, "1.d1"); + int32_t contLen = tSerializeSDropDbReq(NULL, 0, &dropdbReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSDropDbReq(pReq, contLen, &dropdbReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DB, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, 0); - SDropDbRsp* pDrop = (SDropDbRsp*)pRsp->pCont; - pDrop->uid = htobe64(pDrop->uid); - EXPECT_STREQ(pDrop->db, "1.d1"); + SDropDbRsp dropdbRsp = {0}; + tDeserializeSDropDbRsp(pRsp->pCont, pRsp->contLen, &dropdbRsp); + EXPECT_STREQ(dropdbRsp.db, "1.d1"); } test.SendShowMetaReq(TSDB_MGMT_TABLE_DB, ""); @@ -217,29 +223,31 @@ TEST_F(MndTestDb, 02_Create_Alter_Drop_Db) { TEST_F(MndTestDb, 03_Create_Use_Restart_Use_Db) { { - int32_t contLen = sizeof(SCreateDbReq); + SCreateDbReq createReq = {0}; + strcpy(createReq.db, "1.d2"); + createReq.numOfVgroups = 2; + createReq.cacheBlockSize = 16; + createReq.totalBlocks = 10; + createReq.daysPerFile = 10; + createReq.daysToKeep0 = 3650; + createReq.daysToKeep1 = 3650; + createReq.daysToKeep2 = 3650; + createReq.minRows = 100; + createReq.maxRows = 4096; + createReq.commitTime = 3600; + createReq.fsyncPeriod = 3000; + createReq.walLevel = 1; + createReq.precision = 0; + createReq.compression = 2; + createReq.replications = 1; + createReq.quorum = 1; + createReq.update = 0; + createReq.cacheLastRow = 0; + createReq.ignoreExist = 1; - SCreateDbReq* pReq = (SCreateDbReq*)rpcMallocCont(contLen); - strcpy(pReq->db, "1.d2"); - pReq->numOfVgroups = htonl(2); - pReq->cacheBlockSize = htonl(16); - pReq->totalBlocks = htonl(10); - pReq->daysPerFile = htonl(10); - pReq->daysToKeep0 = htonl(3650); - pReq->daysToKeep1 = htonl(3650); - pReq->daysToKeep2 = htonl(3650); - pReq->minRows = htonl(100); - pReq->maxRows = htonl(4096); - pReq->commitTime = htonl(3600); - pReq->fsyncPeriod = htonl(3000); - pReq->walLevel = 1; - pReq->precision = 0; - pReq->compression = 2; - pReq->replications = 1; - pReq->quorum = 1; - pReq->update = 0; - pReq->cacheLastRow = 0; - pReq->ignoreExist = 1; + int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateDbReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -256,73 +264,74 @@ TEST_F(MndTestDb, 03_Create_Use_Restart_Use_Db) { uint64_t d2_uid = 0; { - int32_t contLen = sizeof(SUseDbReq); + SUseDbReq usedbReq = {0}; + strcpy(usedbReq.db, "1.d2"); + usedbReq.vgVersion = -1; - SUseDbReq* pReq = (SUseDbReq*)rpcMallocCont(contLen); - strcpy(pReq->db, "1.d2"); - pReq->vgVersion = htonl(-1); + int32_t contLen = tSerializeSUseDbReq(NULL, 0, &usedbReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSUseDbReq(pReq, contLen, &usedbReq); SRpcMsg* pMsg = test.SendReq(TDMT_MND_USE_DB, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); - SUseDbRsp* pRsp = (SUseDbRsp*)pMsg->pCont; - EXPECT_STREQ(pRsp->db, "1.d2"); - pRsp->uid = htobe64(pRsp->uid); - d2_uid = pRsp->uid; - pRsp->vgVersion = htonl(pRsp->vgVersion); - pRsp->vgNum = htonl(pRsp->vgNum); - pRsp->hashMethod = pRsp->hashMethod; - EXPECT_EQ(pRsp->vgVersion, 1); - EXPECT_EQ(pRsp->vgNum, 2); - EXPECT_EQ(pRsp->hashMethod, 1); + SUseDbRsp usedbRsp = {0}; + tDeserializeSUseDbRsp(pMsg->pCont, pMsg->contLen, &usedbRsp); + EXPECT_STREQ(usedbRsp.db, "1.d2"); + EXPECT_EQ(usedbRsp.vgVersion, 1); + EXPECT_EQ(usedbRsp.vgNum, 2); + EXPECT_EQ(usedbRsp.hashMethod, 1); + d2_uid = usedbRsp.uid; { - SVgroupInfo* pInfo = &pRsp->vgroupInfo[0]; - pInfo->vgId = htonl(pInfo->vgId); - pInfo->hashBegin = htonl(pInfo->hashBegin); - pInfo->hashEnd = htonl(pInfo->hashEnd); + SVgroupInfo* pInfo = (SVgroupInfo*)taosArrayGet(usedbRsp.pVgroupInfos, 0); + pInfo->vgId = pInfo->vgId; + pInfo->hashBegin = pInfo->hashBegin; + pInfo->hashEnd = pInfo->hashEnd; EXPECT_GT(pInfo->vgId, 0); EXPECT_EQ(pInfo->hashBegin, 0); EXPECT_EQ(pInfo->hashEnd, UINT32_MAX / 2 - 1); EXPECT_EQ(pInfo->epset.inUse, 0); EXPECT_EQ(pInfo->epset.numOfEps, 1); SEp* pAddr = &pInfo->epset.eps[0]; - pAddr->port = htons(pAddr->port); EXPECT_EQ(pAddr->port, 9030); EXPECT_STREQ(pAddr->fqdn, "localhost"); } { - SVgroupInfo* pInfo = &pRsp->vgroupInfo[1]; - pInfo->vgId = htonl(pInfo->vgId); - pInfo->hashBegin = htonl(pInfo->hashBegin); - pInfo->hashEnd = htonl(pInfo->hashEnd); + SVgroupInfo* pInfo = (SVgroupInfo*)taosArrayGet(usedbRsp.pVgroupInfos, 1); + pInfo->vgId = pInfo->vgId; + pInfo->hashBegin = pInfo->hashBegin; + pInfo->hashEnd = pInfo->hashEnd; EXPECT_GT(pInfo->vgId, 0); EXPECT_EQ(pInfo->hashBegin, UINT32_MAX / 2); EXPECT_EQ(pInfo->hashEnd, UINT32_MAX); EXPECT_EQ(pInfo->epset.inUse, 0); EXPECT_EQ(pInfo->epset.numOfEps, 1); SEp* pAddr = &pInfo->epset.eps[0]; - pAddr->port = htons(pAddr->port); EXPECT_EQ(pAddr->port, 9030); EXPECT_STREQ(pAddr->fqdn, "localhost"); } + + tFreeSUsedbRsp(&usedbRsp); } { - int32_t contLen = sizeof(SDropDbReq); + SDropDbReq dropdbReq = {0}; + strcpy(dropdbReq.db, "1.d2"); - SDropDbReq* pReq = (SDropDbReq*)rpcMallocCont(contLen); - strcpy(pReq->db, "1.d2"); + int32_t contLen = tSerializeSDropDbReq(NULL, 0, &dropdbReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSDropDbReq(pReq, contLen, &dropdbReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_DB, pReq, contLen); ASSERT_NE(pRsp, nullptr); ASSERT_EQ(pRsp->code, 0); - SDropDbRsp* pDrop = (SDropDbRsp*)pRsp->pCont; - pDrop->uid = htobe64(pDrop->uid); - EXPECT_STREQ(pDrop->db, "1.d2"); - EXPECT_EQ(pDrop->uid, d2_uid); + SDropDbRsp dropdbRsp = {0}; + tDeserializeSDropDbRsp(pRsp->pCont, pRsp->contLen, &dropdbRsp); + EXPECT_STREQ(dropdbRsp.db, "1.d2"); + EXPECT_EQ(dropdbRsp.uid, d2_uid); } } diff --git a/source/dnode/mnode/impl/test/profile/profile.cpp b/source/dnode/mnode/impl/test/profile/profile.cpp index ccf13d5d66..14b31e5282 100644 --- a/source/dnode/mnode/impl/test/profile/profile.cpp +++ b/source/dnode/mnode/impl/test/profile/profile.cpp @@ -28,44 +28,44 @@ Testbase MndTestProfile::test; int32_t MndTestProfile::connId; TEST_F(MndTestProfile, 01_ConnectMsg) { - int32_t contLen = sizeof(SConnectReq); + SConnectReq connectReq = {0}; + connectReq.pid = 1234; + strcpy(connectReq.app, "mnode_test_profile"); + strcpy(connectReq.db, ""); - SConnectReq* pReq = (SConnectReq*)rpcMallocCont(contLen); - pReq->pid = htonl(1234); - strcpy(pReq->app, "mnode_test_profile"); - strcpy(pReq->db, ""); + int32_t contLen = tSerializeSConnectReq(NULL, 0, &connectReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSConnectReq(pReq, contLen, &connectReq); SRpcMsg* pMsg = test.SendReq(TDMT_MND_CONNECT, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); - SConnectRsp* pRsp = (SConnectRsp*)pMsg->pCont; - ASSERT_NE(pRsp, nullptr); - pRsp->acctId = htonl(pRsp->acctId); - pRsp->clusterId = htobe64(pRsp->clusterId); - pRsp->connId = htonl(pRsp->connId); - pRsp->epSet.eps[0].port = htons(pRsp->epSet.eps[0].port); + SConnectRsp connectRsp = {0}; + tDeserializeSConnectRsp(pMsg->pCont, pMsg->contLen, &connectRsp); - EXPECT_EQ(pRsp->acctId, 1); - EXPECT_GT(pRsp->clusterId, 0); - EXPECT_EQ(pRsp->connId, 1); - EXPECT_EQ(pRsp->superUser, 1); + EXPECT_EQ(connectRsp.acctId, 1); + EXPECT_GT(connectRsp.clusterId, 0); + EXPECT_EQ(connectRsp.connId, 1); + EXPECT_EQ(connectRsp.superUser, 1); - EXPECT_EQ(pRsp->epSet.inUse, 0); - EXPECT_EQ(pRsp->epSet.numOfEps, 1); - EXPECT_EQ(pRsp->epSet.eps[0].port, 9031); - EXPECT_STREQ(pRsp->epSet.eps[0].fqdn, "localhost"); + EXPECT_EQ(connectRsp.epSet.inUse, 0); + EXPECT_EQ(connectRsp.epSet.numOfEps, 1); + EXPECT_EQ(connectRsp.epSet.eps[0].port, 9031); + EXPECT_STREQ(connectRsp.epSet.eps[0].fqdn, "localhost"); - connId = pRsp->connId; + connId = connectRsp.connId; } TEST_F(MndTestProfile, 02_ConnectMsg_InvalidDB) { - int32_t contLen = sizeof(SConnectReq); + SConnectReq connectReq = {0}; + connectReq.pid = 1234; + strcpy(connectReq.app, "mnode_test_profile"); + strcpy(connectReq.db, "invalid_db"); - SConnectReq* pReq = (SConnectReq*)rpcMallocCont(contLen); - pReq->pid = htonl(1234); - strcpy(pReq->app, "mnode_test_profile"); - strcpy(pReq->db, "invalid_db"); + int32_t contLen = tSerializeSConnectReq(NULL, 0, &connectReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSConnectReq(pReq, contLen, &connectReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CONNECT, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -96,35 +96,35 @@ TEST_F(MndTestProfile, 03_ConnectMsg_Show) { } TEST_F(MndTestProfile, 04_HeartBeatMsg) { - SClientHbBatchReq batchReq; + SClientHbBatchReq batchReq = {0}; batchReq.reqs = taosArrayInit(0, sizeof(SClientHbReq)); SClientHbReq req = {0}; req.connKey = {.connId = 123, .hbType = HEARTBEAT_TYPE_MQ}; req.info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); - SKv kv; + SKv kv = {0}; kv.key = 123; kv.value = (void*)"bcd"; kv.valueLen = 4; taosHashPut(req.info, &kv.key, sizeof(kv.key), &kv, sizeof(kv)); taosArrayPush(batchReq.reqs, &req); - int32_t tlen = tSerializeSClientHbBatchReq(NULL, &batchReq); - - void* buf = (SClientHbBatchReq*)rpcMallocCont(tlen); - void* bufCopy = buf; - tSerializeSClientHbBatchReq(&bufCopy, &batchReq); + int32_t tlen = tSerializeSClientHbBatchReq(NULL, 0, &batchReq); + void* buf = (SClientHbBatchReq*)rpcMallocCont(tlen); + tSerializeSClientHbBatchReq(buf, tlen, &batchReq); + SRpcMsg* pMsg = test.SendReq(TDMT_MND_HEARTBEAT, buf, tlen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); - char* pRspChar = (char*)pMsg->pCont; + SClientHbBatchRsp rsp = {0}; - tDeserializeSClientHbBatchRsp(pRspChar, &rsp); + tDeserializeSClientHbBatchRsp(pMsg->pCont, pMsg->contLen, &rsp); int sz = taosArrayGetSize(rsp.rsps); ASSERT_EQ(sz, 0); - //SClientHbRsp* pRsp = (SClientHbRsp*) taosArrayGet(rsp.rsps, 0); - //EXPECT_EQ(pRsp->connKey.connId, 123); - //EXPECT_EQ(pRsp->connKey.hbType, HEARTBEAT_TYPE_MQ); - //EXPECT_EQ(pRsp->status, 0); + + // SClientHbRsp* pRsp = (SClientHbRsp*) taosArrayGet(rsp.rsps, 0); + // EXPECT_EQ(pRsp->connKey.connId, 123); + // EXPECT_EQ(pRsp->connKey.hbType, HEARTBEAT_TYPE_MQ); + // EXPECT_EQ(pRsp->status, 0); #if 0 int32_t contLen = sizeof(SHeartBeatReq); @@ -167,10 +167,12 @@ TEST_F(MndTestProfile, 05_KillConnMsg) { // temporary remove since kill will use new heartbeat msg #if 0 { - int32_t contLen = sizeof(SKillConnReq); + SKillConnReq killReq = {0}; + killReq.connId = connId; - SKillConnReq* pReq = (SKillConnReq*)rpcMallocCont(contLen); - pReq->connId = htonl(connId); + int32_t contLen = tSerializeSKillConnReq(NULL, 0, &killReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSKillConnReq(pReq, contLen, &killReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_KILL_CONN, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -194,44 +196,44 @@ TEST_F(MndTestProfile, 05_KillConnMsg) { } { - int32_t contLen = sizeof(SConnectReq); + SConnectReq connectReq = {0}; + connectReq.pid = 1234; + strcpy(connectReq.app, "mnode_test_profile"); + strcpy(connectReq.db, "invalid_db"); - SConnectReq* pReq = (SConnectReq*)rpcMallocCont(contLen); - pReq->pid = htonl(1234); - strcpy(pReq->app, "mnode_test_profile"); - strcpy(pReq->db, ""); + int32_t contLen = tSerializeSConnectReq(NULL, 0, &connectReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSConnectReq(pReq, contLen, &connectReq); SRpcMsg* pMsg = test.SendReq(TDMT_MND_CONNECT, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); - SConnectRsp* pRsp = (SConnectRsp*)pMsg->pCont; - ASSERT_NE(pRsp, nullptr); - pRsp->acctId = htonl(pRsp->acctId); - pRsp->clusterId = htobe64(pRsp->clusterId); - pRsp->connId = htonl(pRsp->connId); - pRsp->epSet.port[0] = htons(pRsp->epSet.port[0]); + SConnectRsp connectRsp = {0}; + tDeserializeSConnectRsp(pMsg->pCont, pMsg->contLen, &connectRsp); - EXPECT_EQ(pRsp->acctId, 1); - EXPECT_GT(pRsp->clusterId, 0); - EXPECT_GT(pRsp->connId, connId); - EXPECT_EQ(pRsp->superUser, 1); + EXPECT_EQ(connectRsp.acctId, 1); + EXPECT_GT(connectRsp.clusterId, 0); + EXPECT_GT(connectRsp.connId, connId); + EXPECT_EQ(connectRsp.superUser, 1); - EXPECT_EQ(pRsp->epSet.inUse, 0); - EXPECT_EQ(pRsp->epSet.numOfEps, 1); - EXPECT_EQ(pRsp->epSet.port[0], 9031); - EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost"); + EXPECT_EQ(connectRsp.epSet.inUse, 0); + EXPECT_EQ(connectRsp.epSet.numOfEps, 1); + EXPECT_EQ(connectRsp.epSet.port[0], 9031); + EXPECT_STREQ(connectRsp.epSet.fqdn[0], "localhost"); - connId = pRsp->connId; + connId = connectRsp.connId; } #endif } TEST_F(MndTestProfile, 06_KillConnMsg_InvalidConn) { - int32_t contLen = sizeof(SKillConnReq); + SKillConnReq killReq = {0}; + killReq.connId = 2345; - SKillConnReq* pReq = (SKillConnReq*)rpcMallocCont(contLen); - pReq->connId = htonl(2345); + int32_t contLen = tSerializeSKillConnReq(NULL, 0, &killReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSKillConnReq(pReq, contLen, &killReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_KILL_CONN, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -242,11 +244,13 @@ TEST_F(MndTestProfile, 07_KillQueryMsg) { // temporary remove since kill will use new heartbeat msg #if 0 { - int32_t contLen = sizeof(SKillQueryReq); + SKillQueryReq killReq = {0}; + killReq.connId = connId; + killReq.queryId = 1234; - SKillQueryReq* pReq = (SKillQueryReq*)rpcMallocCont(contLen); - pReq->connId = htonl(connId); - pReq->queryId = htonl(1234); + int32_t contLen = tSerializeSKillQueryReq(NULL, 0, &killReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSKillQueryReq(pReq, contLen, &killReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_KILL_QUERY, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -293,11 +297,13 @@ TEST_F(MndTestProfile, 07_KillQueryMsg) { } TEST_F(MndTestProfile, 08_KillQueryMsg_InvalidConn) { - int32_t contLen = sizeof(SKillQueryReq); + SKillQueryReq killReq = {0}; + killReq.connId = 2345; + killReq.queryId = 2345; - SKillQueryReq* pReq = (SKillQueryReq*)rpcMallocCont(contLen); - pReq->connId = htonl(2345); - pReq->queryId = htonl(1234); + int32_t contLen = tSerializeSKillQueryReq(NULL, 0, &killReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSKillQueryReq(pReq, contLen, &killReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_KILL_QUERY, pReq, contLen); ASSERT_NE(pRsp, nullptr); diff --git a/source/dnode/mnode/impl/test/show/show.cpp b/source/dnode/mnode/impl/test/show/show.cpp index bc31630ffe..a57d99a257 100644 --- a/source/dnode/mnode/impl/test/show/show.cpp +++ b/source/dnode/mnode/impl/test/show/show.cpp @@ -26,11 +26,13 @@ class MndTestShow : public ::testing::Test { Testbase MndTestShow::test; TEST_F(MndTestShow, 01_ShowMsg_InvalidMsgMax) { - int32_t contLen = sizeof(SShowReq); + SShowReq showReq = {0}; + showReq.type = TSDB_MGMT_TABLE_MAX; - SShowReq* pReq = (SShowReq*)rpcMallocCont(contLen); - pReq->type = TSDB_MGMT_TABLE_MAX; - strcpy(pReq->db, ""); + int32_t contLen = tSerializeSShowReq(NULL, 0, &showReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSShowReq(pReq, contLen, &showReq); + tFreeSShowReq(&showReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_SHOW, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -38,11 +40,13 @@ TEST_F(MndTestShow, 01_ShowMsg_InvalidMsgMax) { } TEST_F(MndTestShow, 02_ShowMsg_InvalidMsgStart) { - int32_t contLen = sizeof(SShowReq); + SShowReq showReq = {0}; + showReq.type = TSDB_MGMT_TABLE_START; - SShowReq* pReq = (SShowReq*)rpcMallocCont(sizeof(SShowReq)); - pReq->type = TSDB_MGMT_TABLE_START; - strcpy(pReq->db, ""); + int32_t contLen = tSerializeSShowReq(NULL, 0, &showReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSShowReq(pReq, contLen, &showReq); + tFreeSShowReq(&showReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_SHOW, pReq, contLen); ASSERT_NE(pRsp, nullptr); @@ -50,12 +54,14 @@ TEST_F(MndTestShow, 02_ShowMsg_InvalidMsgStart) { } TEST_F(MndTestShow, 03_ShowMsg_Conn) { - int32_t contLen = sizeof(SConnectReq); + SConnectReq connectReq = {0}; + connectReq.pid = 1234; + strcpy(connectReq.app, "mnode_test_show"); + strcpy(connectReq.db, ""); - SConnectReq* pReq = (SConnectReq*)rpcMallocCont(contLen); - pReq->pid = htonl(1234); - strcpy(pReq->app, "mnode_test_show"); - strcpy(pReq->db, ""); + int32_t contLen = tSerializeSConnectReq(NULL, 0, &connectReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSConnectReq(pReq, contLen, &connectReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CONNECT, pReq, contLen); ASSERT_NE(pRsp, nullptr); diff --git a/source/dnode/mnode/impl/test/stb/stb.cpp b/source/dnode/mnode/impl/test/stb/stb.cpp index c5ab44a014..c551097e72 100644 --- a/source/dnode/mnode/impl/test/stb/stb.cpp +++ b/source/dnode/mnode/impl/test/stb/stb.cpp @@ -38,39 +38,43 @@ class MndTestStb : public ::testing::Test { Testbase MndTestStb::test; void* MndTestStb::BuildCreateDbReq(const char* dbname, int32_t* pContLen) { - int32_t contLen = sizeof(SCreateDbReq); + SCreateDbReq createReq = {0}; + strcpy(createReq.db, dbname); + createReq.numOfVgroups = 2; + createReq.cacheBlockSize = 16; + createReq.totalBlocks = 10; + createReq.daysPerFile = 10; + createReq.daysToKeep0 = 3650; + createReq.daysToKeep1 = 3650; + createReq.daysToKeep2 = 3650; + createReq.minRows = 100; + createReq.maxRows = 4096; + createReq.commitTime = 3600; + createReq.fsyncPeriod = 3000; + createReq.walLevel = 1; + createReq.precision = 0; + createReq.compression = 2; + createReq.replications = 1; + createReq.quorum = 1; + createReq.update = 0; + createReq.cacheLastRow = 0; + createReq.ignoreExist = 1; - SCreateDbReq* pReq = (SCreateDbReq*)rpcMallocCont(contLen); - strcpy(pReq->db, dbname); - pReq->numOfVgroups = htonl(2); - pReq->cacheBlockSize = htonl(16); - pReq->totalBlocks = htonl(10); - pReq->daysPerFile = htonl(10); - pReq->daysToKeep0 = htonl(3650); - pReq->daysToKeep1 = htonl(3650); - pReq->daysToKeep2 = htonl(3650); - pReq->minRows = htonl(100); - pReq->maxRows = htonl(4096); - pReq->commitTime = htonl(3600); - pReq->fsyncPeriod = htonl(3000); - pReq->walLevel = 1; - pReq->precision = 0; - pReq->compression = 2; - pReq->replications = 1; - pReq->quorum = 1; - pReq->update = 0; - pReq->cacheLastRow = 0; - pReq->ignoreExist = 1; + int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateDbReq(pReq, contLen, &createReq); *pContLen = contLen; return pReq; } void* MndTestStb::BuildDropDbReq(const char* dbname, int32_t* pContLen) { - int32_t contLen = sizeof(SDropDbReq); + SDropDbReq dropdbReq = {0}; + strcpy(dropdbReq.db, dbname); - SDropDbReq* pReq = (SDropDbReq*)rpcMallocCont(contLen); - strcpy(pReq->db, dbname); + int32_t contLen = tSerializeSDropDbReq(NULL, 0, &dropdbReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSDropDbReq(pReq, contLen, &dropdbReq); *pContLen = contLen; return pReq; @@ -125,11 +129,10 @@ void* MndTestStb::BuildCreateStbReq(const char* stbname, int32_t* pContLen) { taosArrayPush(createReq.pTags, &field); } - int32_t tlen = tSerializeSMCreateStbReq(NULL, &createReq); + int32_t tlen = tSerializeSMCreateStbReq(NULL, 0, &createReq); void* pHead = rpcMallocCont(tlen); - - void* pBuf = pHead; - tSerializeSMCreateStbReq(&pBuf, &createReq); + tSerializeSMCreateStbReq(pHead, tlen, &createReq); + tFreeSMCreateStbReq(&createReq); *pContLen = tlen; return pHead; } @@ -147,10 +150,9 @@ void* MndTestStb::BuildAlterStbAddTagReq(const char* stbname, const char* tagnam strcpy(field.name, tagname); taosArrayPush(req.pFields, &field); - int32_t contLen = tSerializeSMAlterStbReq(NULL, &req); + int32_t contLen = tSerializeSMAlterStbReq(NULL, 0, &req); void* pHead = rpcMallocCont(contLen); - void* pBuf = pHead; - tSerializeSMAlterStbReq(&pBuf, &req); + tSerializeSMAlterStbReq(pHead, contLen, &req); *pContLen = contLen; return pHead; @@ -169,10 +171,9 @@ void* MndTestStb::BuildAlterStbDropTagReq(const char* stbname, const char* tagna strcpy(field.name, tagname); taosArrayPush(req.pFields, &field); - int32_t contLen = tSerializeSMAlterStbReq(NULL, &req); + int32_t contLen = tSerializeSMAlterStbReq(NULL, 0, &req); void* pHead = rpcMallocCont(contLen); - void* pBuf = pHead; - tSerializeSMAlterStbReq(&pBuf, &req); + tSerializeSMAlterStbReq(pHead, contLen, &req); *pContLen = contLen; return pHead; @@ -198,10 +199,9 @@ void* MndTestStb::BuildAlterStbUpdateTagNameReq(const char* stbname, const char* strcpy(field2.name, newtagname); taosArrayPush(req.pFields, &field2); - int32_t contLen = tSerializeSMAlterStbReq(NULL, &req); + int32_t contLen = tSerializeSMAlterStbReq(NULL, 0, &req); void* pHead = rpcMallocCont(contLen); - void* pBuf = pHead; - tSerializeSMAlterStbReq(&pBuf, &req); + tSerializeSMAlterStbReq(pHead, contLen, &req); *pContLen = contLen; return pHead; @@ -221,10 +221,9 @@ void* MndTestStb::BuildAlterStbUpdateTagBytesReq(const char* stbname, const char strcpy(field.name, tagname); taosArrayPush(req.pFields, &field); - int32_t contLen = tSerializeSMAlterStbReq(NULL, &req); + int32_t contLen = tSerializeSMAlterStbReq(NULL, 0, &req); void* pHead = rpcMallocCont(contLen); - void* pBuf = pHead; - tSerializeSMAlterStbReq(&pBuf, &req); + tSerializeSMAlterStbReq(pHead, contLen, &req); *pContLen = contLen; return pHead; @@ -243,10 +242,9 @@ void* MndTestStb::BuildAlterStbAddColumnReq(const char* stbname, const char* col strcpy(field.name, colname); taosArrayPush(req.pFields, &field); - int32_t contLen = tSerializeSMAlterStbReq(NULL, &req); + int32_t contLen = tSerializeSMAlterStbReq(NULL, 0, &req); void* pHead = rpcMallocCont(contLen); - void* pBuf = pHead; - tSerializeSMAlterStbReq(&pBuf, &req); + tSerializeSMAlterStbReq(pHead, contLen, &req); *pContLen = contLen; return pHead; @@ -265,10 +263,9 @@ void* MndTestStb::BuildAlterStbDropColumnReq(const char* stbname, const char* co strcpy(field.name, colname); taosArrayPush(req.pFields, &field); - int32_t contLen = tSerializeSMAlterStbReq(NULL, &req); + int32_t contLen = tSerializeSMAlterStbReq(NULL, 0, &req); void* pHead = rpcMallocCont(contLen); - void* pBuf = pHead; - tSerializeSMAlterStbReq(&pBuf, &req); + tSerializeSMAlterStbReq(pHead, contLen, &req); *pContLen = contLen; return pHead; @@ -288,10 +285,9 @@ void* MndTestStb::BuildAlterStbUpdateColumnBytesReq(const char* stbname, const c strcpy(field.name, colname); taosArrayPush(req.pFields, &field); - int32_t contLen = tSerializeSMAlterStbReq(NULL, &req); + int32_t contLen = tSerializeSMAlterStbReq(NULL, 0, &req); void* pHead = rpcMallocCont(contLen); - void* pBuf = pHead; - tSerializeSMAlterStbReq(&pBuf, &req); + tSerializeSMAlterStbReq(pHead, contLen, &req); *pContLen = contLen; return pHead; @@ -335,45 +331,37 @@ TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { // ----- meta ------ { - int32_t contLen = sizeof(STableInfoReq); - STableInfoReq* pReq = (STableInfoReq*)rpcMallocCont(contLen); - strcpy(pReq->dbFName, dbname); - strcpy(pReq->tbName, "stb"); + STableInfoReq infoReq = {0}; + strcpy(infoReq.dbFName, dbname); + strcpy(infoReq.tbName, "stb"); + + int32_t contLen = tSerializeSTableInfoReq(NULL, 0, &infoReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSTableInfoReq(pReq, contLen, &infoReq); SRpcMsg* pMsg = test.SendReq(TDMT_MND_STB_META, pReq, contLen); ASSERT_NE(pMsg, nullptr); ASSERT_EQ(pMsg->code, 0); - STableMetaRsp* pRsp = (STableMetaRsp*)pMsg->pCont; - pRsp->numOfTags = htonl(pRsp->numOfTags); - pRsp->numOfColumns = htonl(pRsp->numOfColumns); - pRsp->sversion = htonl(pRsp->sversion); - pRsp->tversion = htonl(pRsp->tversion); - pRsp->suid = be64toh(pRsp->suid); - pRsp->tuid = be64toh(pRsp->tuid); - pRsp->vgId = be64toh(pRsp->vgId); - for (int32_t i = 0; i < pRsp->numOfTags + pRsp->numOfColumns; ++i) { - SSchema* pSchema = &pRsp->pSchema[i]; - pSchema->colId = htonl(pSchema->colId); - pSchema->bytes = htonl(pSchema->bytes); - } + STableMetaRsp metaRsp = {0}; + tDeserializeSTableMetaRsp(pMsg->pCont, pMsg->contLen, &metaRsp); - EXPECT_STREQ(pRsp->dbFName, dbname); - EXPECT_STREQ(pRsp->tbName, "stb"); - EXPECT_STREQ(pRsp->stbName, "stb"); - EXPECT_EQ(pRsp->numOfColumns, 2); - EXPECT_EQ(pRsp->numOfTags, 3); - EXPECT_EQ(pRsp->precision, TSDB_TIME_PRECISION_MILLI); - EXPECT_EQ(pRsp->tableType, TSDB_SUPER_TABLE); - EXPECT_EQ(pRsp->update, 0); - EXPECT_EQ(pRsp->sversion, 1); - EXPECT_EQ(pRsp->tversion, 0); - EXPECT_GT(pRsp->suid, 0); - EXPECT_GT(pRsp->tuid, 0); - EXPECT_EQ(pRsp->vgId, 0); + EXPECT_STREQ(metaRsp.dbFName, dbname); + EXPECT_STREQ(metaRsp.tbName, "stb"); + EXPECT_STREQ(metaRsp.stbName, "stb"); + EXPECT_EQ(metaRsp.numOfColumns, 2); + EXPECT_EQ(metaRsp.numOfTags, 3); + EXPECT_EQ(metaRsp.precision, TSDB_TIME_PRECISION_MILLI); + EXPECT_EQ(metaRsp.tableType, TSDB_SUPER_TABLE); + EXPECT_EQ(metaRsp.update, 0); + EXPECT_EQ(metaRsp.sversion, 1); + EXPECT_EQ(metaRsp.tversion, 0); + EXPECT_GT(metaRsp.suid, 0); + EXPECT_GT(metaRsp.tuid, 0); + EXPECT_EQ(metaRsp.vgId, 0); { - SSchema* pSchema = &pRsp->pSchema[0]; + SSchema* pSchema = &metaRsp.pSchemas[0]; EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_TIMESTAMP); EXPECT_EQ(pSchema->colId, 1); EXPECT_EQ(pSchema->bytes, 8); @@ -381,7 +369,7 @@ TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { } { - SSchema* pSchema = &pRsp->pSchema[1]; + SSchema* pSchema = &metaRsp.pSchemas[1]; EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); EXPECT_EQ(pSchema->colId, 2); EXPECT_EQ(pSchema->bytes, 12); @@ -389,7 +377,7 @@ TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { } { - SSchema* pSchema = &pRsp->pSchema[2]; + SSchema* pSchema = &metaRsp.pSchemas[2]; EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_TINYINT); EXPECT_EQ(pSchema->colId, 3); EXPECT_EQ(pSchema->bytes, 2); @@ -397,7 +385,7 @@ TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { } { - SSchema* pSchema = &pRsp->pSchema[3]; + SSchema* pSchema = &metaRsp.pSchemas[3]; EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BIGINT); EXPECT_EQ(pSchema->colId, 4); EXPECT_EQ(pSchema->bytes, 8); @@ -405,12 +393,14 @@ TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { } { - SSchema* pSchema = &pRsp->pSchema[4]; + SSchema* pSchema = &metaRsp.pSchemas[4]; EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY); EXPECT_EQ(pSchema->colId, 5); EXPECT_EQ(pSchema->bytes, 16); EXPECT_STREQ(pSchema->name, "tag3"); } + + tFreeSTableMetaRsp(&metaRsp); } // restart @@ -432,10 +422,9 @@ TEST_F(MndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) { SMDropStbReq dropReq = {0}; strcpy(dropReq.name, stbname); - int32_t contLen = tSerializeSMDropStbReq(NULL, &dropReq); + int32_t contLen = tSerializeSMDropStbReq(NULL, 0, &dropReq); void* pHead = rpcMallocCont(contLen); - void* pBuf = pHead; - tSerializeSMDropStbReq(&pBuf, &dropReq); + tSerializeSMDropStbReq(pHead, contLen, &dropReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_STB, pHead, contLen); ASSERT_NE(pRsp, nullptr); diff --git a/source/dnode/mnode/impl/test/topic/CMakeLists.txt b/source/dnode/mnode/impl/test/topic/CMakeLists.txt new file mode 100644 index 0000000000..63a77713d6 --- /dev/null +++ b/source/dnode/mnode/impl/test/topic/CMakeLists.txt @@ -0,0 +1,11 @@ +aux_source_directory(. TOPIC_SRC) +add_executable(mnode_test_topic ${TOPIC_SRC}) +target_link_libraries( + mnode_test_topic + PUBLIC sut +) + +add_test( + NAME mnode_test_topic + COMMAND mnode_test_topic +) diff --git a/source/dnode/mnode/impl/test/topic/topic.cpp b/source/dnode/mnode/impl/test/topic/topic.cpp new file mode 100644 index 0000000000..8a4e17d054 --- /dev/null +++ b/source/dnode/mnode/impl/test/topic/topic.cpp @@ -0,0 +1,177 @@ +/** + * @file topic.cpp + * @author slguan (slguan@taosdata.com) + * @brief MNODE module topic tests + * @version 1.0 + * @date 2022-02-16 + * + * @copyright Copyright (c) 2022 + * + */ + +#include "sut.h" + +class MndTestTopic : public ::testing::Test { + protected: + static void SetUpTestSuite() { test.Init("/tmp/mnode_test_topic", 9039); } + static void TearDownTestSuite() { test.Cleanup(); } + + static Testbase test; + + public: + void SetUp() override {} + void TearDown() override {} + + void* BuildCreateDbReq(const char* dbname, int32_t* pContLen); + void* BuildCreateTopicReq(const char* topicName, const char* sql, int32_t* pContLen); + void* BuildDropTopicReq(const char* topicName, int32_t* pContLen); +}; + +Testbase MndTestTopic::test; + +void* MndTestTopic::BuildCreateDbReq(const char* dbname, int32_t* pContLen) { + SCreateDbReq createReq = {0}; + strcpy(createReq.db, dbname); + createReq.numOfVgroups = 2; + createReq.cacheBlockSize = 16; + createReq.totalBlocks = 10; + createReq.daysPerFile = 10; + createReq.daysToKeep0 = 3650; + createReq.daysToKeep1 = 3650; + createReq.daysToKeep2 = 3650; + createReq.minRows = 100; + createReq.maxRows = 4096; + createReq.commitTime = 3600; + createReq.fsyncPeriod = 3000; + createReq.walLevel = 1; + createReq.precision = 0; + createReq.compression = 2; + createReq.replications = 1; + createReq.quorum = 1; + createReq.update = 0; + createReq.cacheLastRow = 0; + createReq.ignoreExist = 1; + + int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateDbReq(pReq, contLen, &createReq); + + *pContLen = contLen; + return pReq; +} + +void* MndTestTopic::BuildCreateTopicReq(const char* topicName, const char* sql, int32_t* pContLen) { + SMCreateTopicReq createReq = {0}; + strcpy(createReq.name, topicName); + createReq.igExists = 0; + createReq.sql = (char*)sql; + createReq.physicalPlan = (char*)"physicalPlan"; + createReq.logicalPlan = (char*)"logicalPlan"; + + int32_t contLen = tSerializeMCreateTopicReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeMCreateTopicReq(pReq, contLen, &createReq); + + *pContLen = contLen; + return pReq; +} + +void* MndTestTopic::BuildDropTopicReq(const char* topicName, int32_t* pContLen) { + SMDropTopicReq dropReq = {0}; + strcpy(dropReq.name, topicName); + + int32_t contLen = tSerializeSMDropTopicReq(NULL, 0, &dropReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSMDropTopicReq(pReq, contLen, &dropReq); + + *pContLen = contLen; + return pReq; +} + +TEST_F(MndTestTopic, 01_Create_Topic) { + const char* dbname = "1.d1"; + const char* topicName = "1.d1.t1"; + + { + int32_t contLen = 0; + void* pReq = BuildCreateDbReq(dbname, &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + } + + { + test.SendShowMetaReq(TSDB_MGMT_TABLE_TP, ""); + } + + { + int32_t contLen = 0; + void* pReq = BuildCreateTopicReq("t1", "sql", &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_TOPIC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_DB_NOT_SELECTED); + } + + { + int32_t contLen = 0; + void* pReq = BuildCreateTopicReq(topicName, "sql", &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_TOPIC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + } + + { + int32_t contLen = 0; + void* pReq = BuildCreateTopicReq(topicName, "sql", &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_TOPIC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_TOPIC_ALREADY_EXIST); + } + + { + test.SendShowMetaReq(TSDB_MGMT_TABLE_TP, dbname); + CHECK_META("show topics", 3); + + CHECK_SCHEMA(0, TSDB_DATA_TYPE_BINARY, TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE, "name"); + CHECK_SCHEMA(1, TSDB_DATA_TYPE_TIMESTAMP, 8, "create_time"); + CHECK_SCHEMA(2, TSDB_DATA_TYPE_BINARY, TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE, "sql"); + + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 1); + + CheckBinary("t1", TSDB_TABLE_NAME_LEN); + CheckTimestamp(); + CheckBinary("sql", TSDB_SHOW_SQL_LEN); + + // restart + test.Restart(); + + test.SendShowMetaReq(TSDB_MGMT_TABLE_TP, dbname); + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 1); + + CheckBinary("t1", TSDB_TABLE_NAME_LEN); + CheckTimestamp(); + CheckBinary("sql", TSDB_SHOW_SQL_LEN); + } + + { + int32_t contLen = 0; + void* pReq = BuildDropTopicReq(topicName, &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_TOPIC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, 0); + } + + { + int32_t contLen = 0; + void* pReq = BuildDropTopicReq(topicName, &contLen); + SRpcMsg* pRsp = test.SendReq(TDMT_MND_DROP_TOPIC, pReq, contLen); + ASSERT_NE(pRsp, nullptr); + ASSERT_EQ(pRsp->code, TSDB_CODE_MND_TOPIC_NOT_EXIST); + + test.SendShowMetaReq(TSDB_MGMT_TABLE_TP, dbname); + test.SendShowRetrieveReq(); + EXPECT_EQ(test.GetShowRows(), 0); + } +} diff --git a/source/dnode/mnode/impl/test/user/user.cpp b/source/dnode/mnode/impl/test/user/user.cpp index 8f9edfab09..d8ce599be1 100644 --- a/source/dnode/mnode/impl/test/user/user.cpp +++ b/source/dnode/mnode/impl/test/user/user.cpp @@ -319,29 +319,31 @@ TEST_F(MndTestUser, 03_Alter_User) { } { - int32_t contLen = sizeof(SCreateDbReq); + SCreateDbReq createReq = {0}; + strcpy(createReq.db, "1.d2"); + createReq.numOfVgroups = 2; + createReq.cacheBlockSize = 16; + createReq.totalBlocks = 10; + createReq.daysPerFile = 10; + createReq.daysToKeep0 = 3650; + createReq.daysToKeep1 = 3650; + createReq.daysToKeep2 = 3650; + createReq.minRows = 100; + createReq.maxRows = 4096; + createReq.commitTime = 3600; + createReq.fsyncPeriod = 3000; + createReq.walLevel = 1; + createReq.precision = 0; + createReq.compression = 2; + createReq.replications = 1; + createReq.quorum = 1; + createReq.update = 0; + createReq.cacheLastRow = 0; + createReq.ignoreExist = 1; - SCreateDbReq* pReq = (SCreateDbReq*)rpcMallocCont(contLen); - strcpy(pReq->db, "1.d2"); - pReq->numOfVgroups = htonl(2); - pReq->cacheBlockSize = htonl(16); - pReq->totalBlocks = htonl(10); - pReq->daysPerFile = htonl(10); - pReq->daysToKeep0 = htonl(3650); - pReq->daysToKeep1 = htonl(3650); - pReq->daysToKeep2 = htonl(3650); - pReq->minRows = htonl(100); - pReq->maxRows = htonl(4096); - pReq->commitTime = htonl(3600); - pReq->fsyncPeriod = htonl(3000); - pReq->walLevel = 1; - pReq->precision = 0; - pReq->compression = 2; - pReq->replications = 1; - pReq->quorum = 1; - pReq->update = 0; - pReq->cacheLastRow = 0; - pReq->ignoreExist = 1; + int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq); + void* pReq = rpcMallocCont(contLen); + tSerializeSCreateDbReq(pReq, contLen, &createReq); SRpcMsg* pRsp = test.SendReq(TDMT_MND_CREATE_DB, pReq, contLen); ASSERT_NE(pRsp, nullptr); diff --git a/source/dnode/vnode/inc/tq.h b/source/dnode/vnode/inc/tq.h index 9d396e0771..a516f423bb 100644 --- a/source/dnode/vnode/inc/tq.h +++ b/source/dnode/vnode/inc/tq.h @@ -38,7 +38,7 @@ extern "C" { typedef struct STQ STQ; // memory allocator provided by vnode -typedef struct STqMemRef { +typedef struct { SMemAllocatorFactory* pAllocatorFactory; SMemAllocator* pAllocator; } STqMemRef; diff --git a/source/dnode/vnode/inc/tsdb.h b/source/dnode/vnode/inc/tsdb.h index a2f935683e..7bdd8fc266 100644 --- a/source/dnode/vnode/inc/tsdb.h +++ b/source/dnode/vnode/inc/tsdb.h @@ -83,7 +83,7 @@ typedef struct { STsdb *tsdbOpen(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF, SMeta *pMeta, STfs *pTfs); void tsdbClose(STsdb *); void tsdbRemove(const char *path); -int tsdbInsertData(STsdb *pTsdb, SSubmitMsg *pMsg, SSubmitRsp *pRsp); +int tsdbInsertData(STsdb *pTsdb, SSubmitReq *pMsg, SSubmitRsp *pRsp); int tsdbPrepareCommit(STsdb *pTsdb); int tsdbCommit(STsdb *pTsdb); diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 520a4d026a..eaf30da1bb 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -35,12 +35,12 @@ typedef struct SDnode SDnode; typedef int32_t (*PutReqToVQueryQFp)(SDnode *pDnode, struct SRpcMsg *pReq); typedef int32_t (*SendReqToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *rpcMsg); -typedef struct STqCfg { +typedef struct { // TODO int32_t reserved; } STqCfg; -typedef struct SVnodeCfg { +typedef struct { int32_t vgId; uint64_t dbId; SDnode *pDnode; @@ -68,11 +68,11 @@ typedef struct { SendReqToDnodeFp sendReqToDnodeFp; } SVnodeOpt; -typedef struct STqReadHandle { +typedef struct { int64_t ver; - uint64_t tbUid; + int64_t tbUid; SHashObj *tbIdHash; - const SSubmitMsg *pMsg; + const SSubmitReq *pMsg; SSubmitBlk *pBlock; SSubmitMsgIter msgIter; SSubmitBlkIter blkIter; @@ -200,7 +200,7 @@ int32_t vnodeCompact(SVnode *pVnode); int32_t vnodeSync(SVnode *pVnode); int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad); -/* ------------------------- TQ QUERY -------------------------- */ +/* ------------------------- TQ READ --------------------------- */ STqReadHandle *tqInitSubmitMsgScanner(SMeta *pMeta); @@ -208,12 +208,12 @@ static FORCE_INLINE void tqReadHandleSetColIdList(STqReadHandle *pReadHandle, SA pReadHandle->pColIdList = pColIdList; } -// static FORCE_INLINE void tqReadHandleSetTbUid(STqReadHandle* pHandle, const SArray* pTableIdList) { -// pHandle->tbUid = pTableIdList; +// static FORCE_INLINE void tqReadHandleSetTbUid(STqReadHandle* pHandle, int64_t tbUid) { +// pHandle->tbUid = tbUid; //} static FORCE_INLINE int tqReadHandleSetTbUidList(STqReadHandle *pHandle, const SArray *tbUidList) { - pHandle->tbIdHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_NO_LOCK); + pHandle->tbIdHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); if (pHandle->tbIdHash == NULL) { return -1; } @@ -225,7 +225,7 @@ static FORCE_INLINE int tqReadHandleSetTbUidList(STqReadHandle *pHandle, const S return 0; } -void tqReadHandleSetMsg(STqReadHandle *pHandle, SSubmitMsg *pMsg, int64_t ver); +void tqReadHandleSetMsg(STqReadHandle *pHandle, SSubmitReq *pMsg, int64_t ver); bool tqNextDataBlock(STqReadHandle *pHandle); int tqRetrieveDataBlockInfo(STqReadHandle *pHandle, SDataBlockInfo *pBlockInfo); // return SArray diff --git a/source/dnode/vnode/src/inc/tqInt.h b/source/dnode/vnode/src/inc/tqInt.h index 13c0150cd2..344ad992f0 100644 --- a/source/dnode/vnode/src/inc/tqInt.h +++ b/source/dnode/vnode/src/inc/tqInt.h @@ -20,6 +20,7 @@ #include "tlog.h" #include "tq.h" #include "trpc.h" + #ifdef __cplusplus extern "C" { #endif @@ -110,10 +111,11 @@ typedef struct { char content[]; } STqSerializedHead; -typedef int (*FTqSerialize)(const void* pObj, STqSerializedHead** ppHead); -typedef const void* (*FTqDeserialize)(const STqSerializedHead* pHead, void** ppObj); +typedef int32_t (*FTqSerialize)(const void* pObj, STqSerializedHead** ppHead); +typedef int32_t (*FTqDeserialize)(void* self, const STqSerializedHead* pHead, void** ppObj); typedef void (*FTqDelete)(void*); -typedef struct STqMetaHandle { + +typedef struct { int64_t key; int64_t offset; int64_t serializedSize; @@ -131,6 +133,7 @@ typedef struct STqMetaList { } STqMetaList; typedef struct { + STQ* pTq; STqMetaList* bucket[TQ_BUCKET_SIZE]; // a table head STqMetaList* unpersistHead; @@ -187,21 +190,22 @@ typedef struct { char* logicalPlan; char* physicalPlan; char* qmsg; + int64_t persistedOffset; int64_t committedOffset; int64_t currentOffset; STqBuffer buffer; SWalReadHandle* pReadhandle; -} STqTopicHandle; +} STqTopic; typedef struct { int64_t consumerId; int64_t epoch; char cgroup[TSDB_TOPIC_FNAME_LEN]; - SArray* topics; // SArray -} STqConsumerHandle; + SArray* topics; // SArray +} STqConsumer; -int tqSerializeConsumer(const STqConsumerHandle*, STqSerializedHead**); -const void* tqDeserializeConsumer(const STqSerializedHead* pHead, STqConsumerHandle**); +int32_t tqSerializeConsumer(const STqConsumer*, STqSerializedHead**); +int32_t tqDeserializeConsumer(STQ*, const STqSerializedHead*, STqConsumer**); static int FORCE_INLINE tqQueryExecuting(int32_t status) { return status; } diff --git a/source/dnode/vnode/src/inc/tqMetaStore.h b/source/dnode/vnode/src/inc/tqMetaStore.h index 3bf9bb7138..eb203b7117 100644 --- a/source/dnode/vnode/src/inc/tqMetaStore.h +++ b/source/dnode/vnode/src/inc/tqMetaStore.h @@ -23,8 +23,8 @@ extern "C" { #endif -STqMetaStore* tqStoreOpen(const char* path, FTqSerialize pSerializer, FTqDeserialize pDeserializer, FTqDelete pDeleter, - int32_t tqConfigFlag); +STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize pSerializer, FTqDeserialize pDeserializer, + FTqDelete pDeleter, int32_t tqConfigFlag); int32_t tqStoreClose(STqMetaStore*); // int32_t tqStoreDelete(TqMetaStore*); // int32_t tqStoreCommitAll(TqMetaStore*); diff --git a/source/dnode/vnode/src/inc/tqOffset.h b/source/dnode/vnode/src/inc/tqOffset.h new file mode 100644 index 0000000000..b58de26f68 --- /dev/null +++ b/source/dnode/vnode/src/inc/tqOffset.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TQ_OFFSET_H_ +#define _TD_TQ_OFFSET_H_ + +#include "tqInt.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct STqOffsetCfg STqOffsetCfg; +typedef struct STqOffsetStore STqOffsetStore; + +STqOffsetStore* STqOffsetOpen(STqOffsetCfg*); +void STqOffsetClose(STqOffsetStore*); + +int64_t tqOffsetFetch(STqOffsetStore* pStore, const char* subscribeKey); +int32_t tqOffsetCommit(STqOffsetStore* pStore, const char* subscribeKey, int64_t offset); +int32_t tqOffsetPersist(STqOffsetStore* pStore, const char* subscribeKey); +int32_t tqOffsetPersistAll(STqOffsetStore* pStore); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_TQ_OFFSET_H_*/ diff --git a/source/dnode/vnode/src/inc/tsdbMemTable.h b/source/dnode/vnode/src/inc/tsdbMemTable.h index c6fbdb407c..0b9d153239 100644 --- a/source/dnode/vnode/src/inc/tsdbMemTable.h +++ b/source/dnode/vnode/src/inc/tsdbMemTable.h @@ -54,7 +54,7 @@ typedef struct STsdbMemTable { STsdbMemTable *tsdbNewMemTable(STsdb *pTsdb); void tsdbFreeMemTable(STsdb *pTsdb, STsdbMemTable *pMemTable); -int tsdbMemTableInsert(STsdb *pTsdb, STsdbMemTable *pMemTable, SSubmitMsg *pMsg, SShellSubmitRsp *pRsp); +int tsdbMemTableInsert(STsdb *pTsdb, STsdbMemTable *pMemTable, SSubmitReq *pMsg, SSubmitRsp *pRsp); int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols, TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 8ffedfebe4..2f70ab5080 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -50,7 +50,8 @@ STQ* tqOpen(const char* path, SWal* pWal, SMeta* pMeta, STqCfg* tqConfig, SMemAl // TODO: error code of buffer pool } #endif - pTq->tqMeta = tqStoreOpen(path, (FTqSerialize)tqSerializeConsumer, (FTqDeserialize)tqDeserializeConsumer, free, 0); + pTq->tqMeta = + tqStoreOpen(pTq, path, (FTqSerialize)tqSerializeConsumer, (FTqDeserialize)tqDeserializeConsumer, free, 0); if (pTq->tqMeta == NULL) { free(pTq); #if 0 @@ -76,19 +77,89 @@ int tqPushMsg(STQ* pTq, void* p, int64_t version) { return 0; } -int tqCommit(STQ* pTq) { - // do nothing - return 0; +int tqCommit(STQ* pTq) { return tqStorePersist(pTq->tqMeta); } + +int32_t tqGetTopicHandleSize(const STqTopic* pTopic) { + return strlen(pTopic->topicName) + strlen(pTopic->sql) + strlen(pTopic->logicalPlan) + strlen(pTopic->physicalPlan) + + strlen(pTopic->qmsg) + sizeof(int64_t) * 3; } -int tqSerializeConsumer(const STqConsumerHandle* pConsumer, STqSerializedHead** ppHead) { - int32_t num = taosArrayGetSize(pConsumer->topics); - int32_t sz = sizeof(STqSerializedHead) + sizeof(int64_t) * 2 + TSDB_TOPIC_FNAME_LEN + - num * (sizeof(int64_t) + TSDB_TOPIC_FNAME_LEN); +int32_t tqGetConsumerHandleSize(const STqConsumer* pConsumer) { + int num = taosArrayGetSize(pConsumer->topics); + int32_t sz = 0; + for (int i = 0; i < num; i++) { + STqTopic* pTopic = taosArrayGet(pConsumer->topics, i); + sz += tqGetTopicHandleSize(pTopic); + } + return sz; +} + +static FORCE_INLINE int32_t tEncodeSTqTopic(void** buf, const STqTopic* pTopic) { + int32_t tlen = 0; + tlen += taosEncodeString(buf, pTopic->topicName); + /*tlen += taosEncodeString(buf, pTopic->sql);*/ + /*tlen += taosEncodeString(buf, pTopic->logicalPlan);*/ + /*tlen += taosEncodeString(buf, pTopic->physicalPlan);*/ + tlen += taosEncodeString(buf, pTopic->qmsg); + tlen += taosEncodeFixedI64(buf, pTopic->persistedOffset); + tlen += taosEncodeFixedI64(buf, pTopic->committedOffset); + tlen += taosEncodeFixedI64(buf, pTopic->currentOffset); + return tlen; +} + +static FORCE_INLINE const void* tDecodeSTqTopic(const void* buf, STqTopic* pTopic) { + buf = taosDecodeStringTo(buf, pTopic->topicName); + /*buf = taosDecodeString(buf, &pTopic->sql);*/ + /*buf = taosDecodeString(buf, &pTopic->logicalPlan);*/ + /*buf = taosDecodeString(buf, &pTopic->physicalPlan);*/ + buf = taosDecodeString(buf, &pTopic->qmsg); + buf = taosDecodeFixedI64(buf, &pTopic->persistedOffset); + buf = taosDecodeFixedI64(buf, &pTopic->committedOffset); + buf = taosDecodeFixedI64(buf, &pTopic->currentOffset); + return buf; +} + +static FORCE_INLINE int32_t tEncodeSTqConsumer(void** buf, const STqConsumer* pConsumer) { + int32_t sz; + + int32_t tlen = 0; + tlen += taosEncodeFixedI64(buf, pConsumer->consumerId); + tlen += taosEncodeFixedI64(buf, pConsumer->epoch); + tlen += taosEncodeString(buf, pConsumer->cgroup); + sz = taosArrayGetSize(pConsumer->topics); + tlen += taosEncodeFixedI32(buf, sz); + for (int32_t i = 0; i < sz; i++) { + STqTopic* pTopic = taosArrayGet(pConsumer->topics, i); + tlen += tEncodeSTqTopic(buf, pTopic); + } + return tlen; +} + +static FORCE_INLINE const void* tDecodeSTqConsumer(const void* buf, STqConsumer* pConsumer) { + int32_t sz; + + buf = taosDecodeFixedI64(buf, &pConsumer->consumerId); + buf = taosDecodeFixedI64(buf, &pConsumer->epoch); + buf = taosDecodeStringTo(buf, pConsumer->cgroup); + buf = taosDecodeFixedI32(buf, &sz); + pConsumer->topics = taosArrayInit(sz, sizeof(STqTopic)); + if (pConsumer->topics == NULL) return NULL; + for (int32_t i = 0; i < sz; i++) { + STqTopic pTopic; + buf = tDecodeSTqTopic(buf, &pTopic); + taosArrayPush(pConsumer->topics, &pTopic); + } + return buf; +} + +int tqSerializeConsumer(const STqConsumer* pConsumer, STqSerializedHead** ppHead) { + int32_t sz = tEncodeSTqConsumer(NULL, pConsumer); + if (sz > (*ppHead)->ssize) { - void* tmpPtr = realloc(*ppHead, sz); + void* tmpPtr = realloc(*ppHead, sizeof(STqSerializedHead) + sz); if (tmpPtr == NULL) { free(*ppHead); + terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return -1; } *ppHead = tmpPtr; @@ -96,45 +167,167 @@ int tqSerializeConsumer(const STqConsumerHandle* pConsumer, STqSerializedHead** } void* ptr = (*ppHead)->content; - *(int64_t*)ptr = pConsumer->consumerId; - ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); - *(int64_t*)ptr = pConsumer->epoch; - ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); - memcpy(ptr, pConsumer->topics, TSDB_TOPIC_FNAME_LEN); - ptr = POINTER_SHIFT(ptr, TSDB_TOPIC_FNAME_LEN); - *(int32_t*)ptr = num; - ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); - for (int32_t i = 0; i < num; i++) { - STqTopicHandle* pTopic = taosArrayGet(pConsumer->topics, i); - memcpy(ptr, pTopic->topicName, TSDB_TOPIC_FNAME_LEN); - ptr = POINTER_SHIFT(ptr, TSDB_TOPIC_FNAME_LEN); - *(int64_t*)ptr = pTopic->committedOffset; - POINTER_SHIFT(ptr, sizeof(int64_t)); + void* abuf = ptr; + tEncodeSTqConsumer(&abuf, pConsumer); + + return 0; +} + +int32_t tqDeserializeConsumer(STQ* pTq, const STqSerializedHead* pHead, STqConsumer** ppConsumer) { + const void* str = pHead->content; + *ppConsumer = calloc(1, sizeof(STqConsumer)); + if (*ppConsumer == NULL) { + terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; + return -1; + } + if (tDecodeSTqConsumer(str, *ppConsumer) == NULL) { + terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; + return -1; + } + STqConsumer* pConsumer = *ppConsumer; + int32_t sz = taosArrayGetSize(pConsumer->topics); + for (int32_t i = 0; i < sz; i++) { + STqTopic* pTopic = taosArrayGet(pConsumer->topics, i); + pTopic->pReadhandle = walOpenReadHandle(pTq->pWal); + if (pTopic->pReadhandle == NULL) { + ASSERT(false); + } + for (int i = 0; i < TQ_BUFFER_SIZE; i++) { + pTopic->buffer.output[i].status = 0; + STqReadHandle* pReadHandle = tqInitSubmitMsgScanner(pTq->pMeta); + SReadHandle handle = {.reader = pReadHandle, .meta = pTq->pMeta}; + pTopic->buffer.output[i].pReadHandle = pReadHandle; + pTopic->buffer.output[i].task = qCreateStreamExecTaskInfo(pTopic->qmsg, &handle); + } } return 0; } -const void* tqDeserializeConsumer(const STqSerializedHead* pHead, STqConsumerHandle** ppConsumer) { - STqConsumerHandle* pConsumer = *ppConsumer; - const void* ptr = pHead->content; - pConsumer->consumerId = *(int64_t*)ptr; - ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); - pConsumer->epoch = *(int64_t*)ptr; - ptr = POINTER_SHIFT(ptr, sizeof(int64_t)); - memcpy(pConsumer->cgroup, ptr, TSDB_TOPIC_FNAME_LEN); - ptr = POINTER_SHIFT(ptr, TSDB_TOPIC_FNAME_LEN); - int32_t sz = *(int32_t*)ptr; - ptr = POINTER_SHIFT(ptr, sizeof(int32_t)); - pConsumer->topics = taosArrayInit(sz, sizeof(STqTopicHandle)); - for (int32_t i = 0; i < sz; i++) { - /*STqTopicHandle* topicHandle = */ - /*taosArrayPush(pConsumer->topics, );*/ +int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { + SMqConsumeReq* pReq = pMsg->pCont; + int64_t consumerId = pReq->consumerId; + int64_t fetchOffset = pReq->offset; + /*int64_t blockingTime = pReq->blockingTime;*/ + + SMqConsumeRsp rsp = {.consumerId = consumerId, .numOfTopics = 0, .pBlockData = NULL}; + + STqConsumer* pConsumer = tqHandleGet(pTq->tqMeta, consumerId); + if (pConsumer == NULL) { + pMsg->pCont = NULL; + pMsg->contLen = 0; + pMsg->code = -1; + rpcSendResponse(pMsg); + return 0; } - return NULL; + int sz = taosArrayGetSize(pConsumer->topics); + ASSERT(sz == 1); + STqTopic* pTopic = taosArrayGet(pConsumer->topics, 0); + ASSERT(strcmp(pTopic->topicName, pReq->topic) == 0); + ASSERT(pConsumer->consumerId == consumerId); + + if (pReq->reqType == TMQ_REQ_TYPE_COMMIT_ONLY) { + pTopic->committedOffset = pReq->offset; + /*printf("offset %ld committed\n", pTopic->committedOffset);*/ + pMsg->pCont = NULL; + pMsg->contLen = 0; + pMsg->code = 0; + rpcSendResponse(pMsg); + return 0; + } + + if (pReq->reqType == TMQ_REQ_TYPE_CONSUME_AND_COMMIT) { + if (pTopic->committedOffset < pReq->offset - 1) { + pTopic->committedOffset = pReq->offset - 1; + /*printf("offset %ld committed\n", pTopic->committedOffset);*/ + } + } + + rsp.committedOffset = pTopic->committedOffset; + rsp.reqOffset = pReq->offset; + rsp.skipLogNum = 0; + + if (fetchOffset <= pTopic->committedOffset) { + fetchOffset = pTopic->committedOffset + 1; + } + + SWalHead* pHead; + while (1) { + int8_t pos = fetchOffset % TQ_BUFFER_SIZE; + if (walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) { + // TODO: no more log, set timer to wait blocking time + // if data inserted during waiting, launch query and + // rsponse to user + break; + } + pHead = pTopic->pReadhandle->pHead; + if (pHead->head.msgType == TDMT_VND_SUBMIT) { + SSubmitReq* pCont = (SSubmitReq*)&pHead->head.body; + qTaskInfo_t task = pTopic->buffer.output[pos].task; + qSetStreamInput(task, pCont); + SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); + while (1) { + SSDataBlock* pDataBlock; + uint64_t ts; + if (qExecTask(task, &pDataBlock, &ts) < 0) { + ASSERT(false); + } + if (pDataBlock == NULL) { + fetchOffset++; + pos = fetchOffset % TQ_BUFFER_SIZE; + rsp.skipLogNum++; + break; + } + + taosArrayPush(pRes, pDataBlock); + rsp.schemas = pTopic->buffer.output[pos].pReadHandle->pSchemaWrapper; + rsp.rspOffset = fetchOffset; + pTopic->currentOffset = fetchOffset; + + rsp.numOfTopics = 1; + rsp.pBlockData = pRes; + + int32_t tlen = tEncodeSMqConsumeRsp(NULL, &rsp); + void* buf = rpcMallocCont(tlen); + if (buf == NULL) { + pMsg->code = -1; + return -1; + } + + void* abuf = buf; + tEncodeSMqConsumeRsp(&abuf, &rsp); + taosArrayDestroyEx(rsp.pBlockData, (void (*)(void*))tDeleteSSDataBlock); + pMsg->pCont = buf; + pMsg->contLen = tlen; + pMsg->code = 0; + rpcSendResponse(pMsg); + return 0; + } + } else { + fetchOffset++; + rsp.skipLogNum++; + } + } + + int32_t tlen = tEncodeSMqConsumeRsp(NULL, &rsp); + void* buf = rpcMallocCont(tlen); + if (buf == NULL) { + pMsg->code = -1; + return -1; + } + + void* abuf = buf; + tEncodeSMqConsumeRsp(&abuf, &rsp); + rsp.pBlockData = NULL; + pMsg->pCont = buf; + pMsg->contLen = tlen; + pMsg->code = 0; + rpcSendResponse(pMsg); + return 0; } -int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { +#if 0 +int32_t tqProcessConsumeReqV0(STQ* pTq, SRpcMsg* pMsg) { SMqConsumeReq* pReq = pMsg->pCont; int64_t reqId = pReq->reqId; int64_t consumerId = pReq->consumerId; @@ -145,7 +338,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { /*printf("vg %d get consume req\n", pReq->head.vgId);*/ - STqConsumerHandle* pConsumer = tqHandleGet(pTq->tqMeta, consumerId); + STqConsumer* pConsumer = tqHandleGet(pTq->tqMeta, consumerId); if (pConsumer == NULL) { pMsg->pCont = NULL; pMsg->contLen = 0; @@ -156,10 +349,10 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { int sz = taosArrayGetSize(pConsumer->topics); for (int i = 0; i < sz; i++) { - STqTopicHandle* pTopic = taosArrayGet(pConsumer->topics, i); + STqTopic* pTopic = taosArrayGet(pConsumer->topics, i); // TODO: support multiple topic in one req if (strcmp(pTopic->topicName, pReq->topic) != 0) { - /*ASSERT(false);*/ + ASSERT(false); continue; } @@ -195,6 +388,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { break; } if (walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) { + printf("read offset %ld\n", fetchOffset); // check err atomic_store_8(&pTopic->buffer.output[pos].status, 0); skip = 1; @@ -203,10 +397,10 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { // read until find TDMT_VND_SUBMIT pHead = pTopic->pReadhandle->pHead; if (pHead->head.msgType == TDMT_VND_SUBMIT) { - break; } rsp.skipLogNum++; if (walReadWithHandle(pTopic->pReadhandle, fetchOffset) < 0) { + printf("read offset %ld\n", fetchOffset); atomic_store_8(&pTopic->buffer.output[pos].status, 0); skip = 1; break; @@ -215,9 +409,10 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { fetchOffset++; } if (skip == 1) continue; - SSubmitMsg* pCont = (SSubmitMsg*)&pHead->head.body; + SSubmitReq* pCont = (SSubmitReq*)&pHead->head.body; qTaskInfo_t task = pTopic->buffer.output[pos].task; + printf("current fetch offset %ld\n", fetchOffset); qSetStreamInput(task, pCont); // SArray @@ -237,6 +432,7 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { // TODO copy rsp.schemas = pTopic->buffer.output[pos].pReadHandle->pSchemaWrapper; rsp.rspOffset = fetchOffset; + pTopic->currentOffset = fetchOffset; atomic_store_8(&pTopic->buffer.output[pos].status, 0); @@ -268,28 +464,27 @@ int32_t tqProcessConsumeReq(STQ* pTq, SRpcMsg* pMsg) { } void* abuf = buf; tEncodeSMqConsumeRsp(&abuf, &rsp); + if (rsp.pBlockData) { taosArrayDestroyEx(rsp.pBlockData, (void (*)(void*))tDeleteSSDataBlock); rsp.pBlockData = NULL; - /*for (int i = 0; i < taosArrayGetSize(rsp.pBlockData); i++) {*/ - /*SSDataBlock* pBlock = taosArrayGet(rsp.pBlockData, i);*/ - /*tDeleteSSDataBlock(pBlock);*/ - /*}*/ - /*taosArrayDestroy(rsp.pBlockData);*/ } + pMsg->pCont = buf; pMsg->contLen = tlen; pMsg->code = 0; rpcSendResponse(pMsg); return 0; } +#endif int32_t tqProcessRebReq(STQ* pTq, char* msg) { SMqMVRebReq req = {0}; tDecodeSMqMVRebReq(msg, &req); - STqConsumerHandle* pConsumer = tqHandleGet(pTq->tqMeta, req.oldConsumerId); + STqConsumer* pConsumer = tqHandleGet(pTq->tqMeta, req.oldConsumerId); ASSERT(pConsumer); + pConsumer->consumerId = req.newConsumerId; tqHandleMovePut(pTq->tqMeta, req.newConsumerId, pConsumer); tqHandleCommit(pTq->tqMeta, req.newConsumerId); tqHandlePurge(pTq->tqMeta, req.oldConsumerId); @@ -302,19 +497,20 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) { tDecodeSMqSetCVgReq(msg, &req); /*printf("vg %d set to consumer from %ld to %ld\n", req.vgId, req.oldConsumerId, req.newConsumerId);*/ - STqConsumerHandle* pConsumer = calloc(1, sizeof(STqConsumerHandle)); + STqConsumer* pConsumer = calloc(1, sizeof(STqConsumer)); if (pConsumer == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return -1; } strcpy(pConsumer->cgroup, req.cgroup); - pConsumer->topics = taosArrayInit(0, sizeof(STqTopicHandle)); + pConsumer->topics = taosArrayInit(0, sizeof(STqTopic)); pConsumer->consumerId = req.consumerId; pConsumer->epoch = 0; - STqTopicHandle* pTopic = calloc(1, sizeof(STqTopicHandle)); + STqTopic* pTopic = calloc(1, sizeof(STqTopic)); if (pTopic == NULL) { + taosArrayDestroy(pConsumer->topics); free(pConsumer); return -1; } @@ -330,6 +526,7 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) { pTopic->buffer.lastOffset = -1; pTopic->pReadhandle = walOpenReadHandle(pTq->pWal); if (pTopic->pReadhandle == NULL) { + ASSERT(false); } for (int i = 0; i < TQ_BUFFER_SIZE; i++) { pTopic->buffer.output[i].status = 0; @@ -344,143 +541,3 @@ int32_t tqProcessSetConnReq(STQ* pTq, char* msg) { terrno = TSDB_CODE_SUCCESS; return 0; } - -STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta) { - STqReadHandle* pReadHandle = malloc(sizeof(STqReadHandle)); - if (pReadHandle == NULL) { - return NULL; - } - pReadHandle->pVnodeMeta = pMeta; - pReadHandle->pMsg = NULL; - pReadHandle->ver = -1; - pReadHandle->pColIdList = NULL; - pReadHandle->sver = -1; - pReadHandle->pSchema = NULL; - pReadHandle->pSchemaWrapper = NULL; - return pReadHandle; -} - -void tqReadHandleSetMsg(STqReadHandle* pReadHandle, SSubmitMsg* pMsg, int64_t ver) { - pReadHandle->pMsg = pMsg; - pMsg->length = htonl(pMsg->length); - pMsg->numOfBlocks = htonl(pMsg->numOfBlocks); - tInitSubmitMsgIter(pMsg, &pReadHandle->msgIter); - pReadHandle->ver = ver; - memset(&pReadHandle->blkIter, 0, sizeof(SSubmitBlkIter)); -} - -bool tqNextDataBlock(STqReadHandle* pHandle) { - while (1) { - if (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) < 0) { - return false; - } - if (pHandle->pBlock == NULL) return false; - - pHandle->pBlock->uid = htobe64(pHandle->pBlock->uid); - /*if (pHandle->tbUid == pHandle->pBlock->uid) {*/ - ASSERT(pHandle->tbIdHash); - void* ret = taosHashGet(pHandle->tbIdHash, &pHandle->pBlock->uid, sizeof(int64_t)); - if (ret != NULL) { - pHandle->pBlock->tid = htonl(pHandle->pBlock->tid); - pHandle->pBlock->sversion = htonl(pHandle->pBlock->sversion); - pHandle->pBlock->dataLen = htonl(pHandle->pBlock->dataLen); - pHandle->pBlock->schemaLen = htonl(pHandle->pBlock->schemaLen); - pHandle->pBlock->numOfRows = htons(pHandle->pBlock->numOfRows); - return true; - } - } - return false; -} - -int tqRetrieveDataBlockInfo(STqReadHandle* pHandle, SDataBlockInfo* pBlockInfo) { - /*int32_t sversion = pHandle->pBlock->sversion;*/ - /*SSchemaWrapper* pSchema = metaGetTableSchema(pHandle->pMeta, pHandle->pBlock->uid, sversion, false);*/ - pBlockInfo->numOfCols = taosArrayGetSize(pHandle->pColIdList); - pBlockInfo->rows = pHandle->pBlock->numOfRows; - pBlockInfo->uid = pHandle->pBlock->uid; - return 0; -} - -SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { - /*int32_t sversion = pHandle->pBlock->sversion;*/ - // TODO set to real sversion - int32_t sversion = 0; - if (pHandle->sver != sversion) { - pHandle->pSchema = metaGetTbTSchema(pHandle->pVnodeMeta, pHandle->pBlock->uid, sversion); - - tb_uid_t quid; - STbCfg* pTbCfg = metaGetTbInfoByUid(pHandle->pVnodeMeta, pHandle->pBlock->uid); - if (pTbCfg->type == META_CHILD_TABLE) { - quid = pTbCfg->ctbCfg.suid; - } else { - quid = pHandle->pBlock->uid; - } - pHandle->pSchemaWrapper = metaGetTableSchema(pHandle->pVnodeMeta, quid, sversion, true); - pHandle->sver = sversion; - } - - STSchema* pTschema = pHandle->pSchema; - SSchemaWrapper* pSchemaWrapper = pHandle->pSchemaWrapper; - - int32_t numOfRows = pHandle->pBlock->numOfRows; - int32_t numOfCols = pHandle->pSchema->numOfCols; - int32_t colNumNeed = taosArrayGetSize(pHandle->pColIdList); - - // TODO: stable case - if (colNumNeed > pSchemaWrapper->nCols) { - colNumNeed = pSchemaWrapper->nCols; - } - - SArray* pArray = taosArrayInit(colNumNeed, sizeof(SColumnInfoData)); - if (pArray == NULL) { - return NULL; - } - - int j = 0; - for (int32_t i = 0; i < colNumNeed; i++) { - int32_t colId = *(int32_t*)taosArrayGet(pHandle->pColIdList, i); - while (j < pSchemaWrapper->nCols && pSchemaWrapper->pSchema[j].colId < colId) { - j++; - } - SSchema* pColSchema = &pSchemaWrapper->pSchema[j]; - SColumnInfoData colInfo = {0}; - int sz = numOfRows * pColSchema->bytes; - colInfo.info.bytes = pColSchema->bytes; - colInfo.info.colId = colId; - colInfo.info.type = pColSchema->type; - - colInfo.pData = calloc(1, sz); - if (colInfo.pData == NULL) { - // TODO free - taosArrayDestroy(pArray); - return NULL; - } - taosArrayPush(pArray, &colInfo); - } - - STSRowIter iter = {0}; - tdSTSRowIterInit(&iter, pTschema); - STSRow* row; - // int32_t kvIdx = 0; - int32_t curRow = 0; - tInitSubmitBlkIter(pHandle->pBlock, &pHandle->blkIter); - while ((row = tGetSubmitBlkNext(&pHandle->blkIter)) != NULL) { - tdSTSRowIterReset(&iter, row); - // get all wanted col of that block - for (int32_t i = 0; i < colNumNeed; i++) { - SColumnInfoData* pColData = taosArrayGet(pArray, i); - STColumn* pCol = schemaColAt(pTschema, i); - // TODO - ASSERT(pCol->colId == pColData->info.colId); - // void* val = tdGetMemRowDataOfColEx(row, pCol->colId, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset, &kvIdx); - SCellVal sVal = {0}; - if (!tdSTSRowIterNext(&iter, pCol->colId, pCol->type, &sVal)) { - // TODO: reach end - break; - } - memcpy(POINTER_SHIFT(pColData->pData, curRow * pCol->bytes), sVal.val, pCol->bytes); - } - curRow++; - } - return pArray; -} diff --git a/source/dnode/vnode/src/tq/tqMetaStore.c b/source/dnode/vnode/src/tq/tqMetaStore.c index 121be98572..d40cc2294f 100644 --- a/source/dnode/vnode/src/tq/tqMetaStore.c +++ b/source/dnode/vnode/src/tq/tqMetaStore.c @@ -68,20 +68,21 @@ static inline int tqReadLastPage(int fd, STqIdxPageBuf* pBuf) { return lseek(fd, offset, SEEK_SET); } -STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserialize deserializer, FTqDelete deleter, - int32_t tqConfigFlag) { - STqMetaStore* pMeta = malloc(sizeof(STqMetaStore)); +STqMetaStore* tqStoreOpen(STQ* pTq, const char* path, FTqSerialize serializer, FTqDeserialize deserializer, + FTqDelete deleter, int32_t tqConfigFlag) { + STqMetaStore* pMeta = calloc(1, sizeof(STqMetaStore)); if (pMeta == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return NULL; } - memset(pMeta, 0, sizeof(STqMetaStore)); + pMeta->pTq = pTq; // concat data file name and index file name size_t pathLen = strlen(path); pMeta->dirPath = malloc(pathLen + 1); if (pMeta->dirPath == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; + free(pMeta); return NULL; } strcpy(pMeta->dirPath, path); @@ -103,12 +104,11 @@ STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserial } pMeta->idxFd = idxFd; - pMeta->unpersistHead = malloc(sizeof(STqMetaList)); + pMeta->unpersistHead = calloc(1, sizeof(STqMetaList)); if (pMeta->unpersistHead == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return NULL; } - memset(pMeta->unpersistHead, 0, sizeof(STqMetaList)); pMeta->unpersistHead->unpersistNext = pMeta->unpersistHead->unpersistPrev = pMeta->unpersistHead; strcpy(name, path); @@ -145,12 +145,11 @@ STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserial ASSERT(idxBuf.head.writeOffset == idxRead); // loop read every entry for (int i = 0; i < idxBuf.head.writeOffset - TQ_IDX_PAGE_HEAD_SIZE; i += TQ_IDX_SIZE) { - STqMetaList* pNode = malloc(sizeof(STqMetaList)); + STqMetaList* pNode = calloc(1, sizeof(STqMetaList)); if (pNode == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; // TODO: free memory } - memset(pNode, 0, sizeof(STqMetaList)); memcpy(&pNode->handle, &idxBuf.buffer[i], TQ_IDX_SIZE); lseek(fileFd, pNode->handle.offset, SEEK_SET); @@ -169,25 +168,25 @@ STqMetaStore* tqStoreOpen(const char* path, FTqSerialize serializer, FTqDeserial } if (serializedObj->action == TQ_ACTION_INUSE) { if (serializedObj->ssize != sizeof(STqSerializedHead)) { - pMeta->pDeserializer(serializedObj, &pNode->handle.valueInUse); + pMeta->pDeserializer(pTq, serializedObj, &pNode->handle.valueInUse); } else { pNode->handle.valueInUse = TQ_DELETE_TOKEN; } } else if (serializedObj->action == TQ_ACTION_INTXN) { if (serializedObj->ssize != sizeof(STqSerializedHead)) { - pMeta->pDeserializer(serializedObj, &pNode->handle.valueInTxn); + pMeta->pDeserializer(pTq, serializedObj, &pNode->handle.valueInTxn); } else { pNode->handle.valueInTxn = TQ_DELETE_TOKEN; } } else if (serializedObj->action == TQ_ACTION_INUSE_CONT) { if (serializedObj->ssize != sizeof(STqSerializedHead)) { - pMeta->pDeserializer(serializedObj, &pNode->handle.valueInUse); + pMeta->pDeserializer(pTq, serializedObj, &pNode->handle.valueInUse); } else { pNode->handle.valueInUse = TQ_DELETE_TOKEN; } STqSerializedHead* ptr = POINTER_SHIFT(serializedObj, serializedObj->ssize); if (ptr->ssize != sizeof(STqSerializedHead)) { - pMeta->pDeserializer(ptr, &pNode->handle.valueInTxn); + pMeta->pDeserializer(pTq, ptr, &pNode->handle.valueInTxn); } else { pNode->handle.valueInTxn = TQ_DELETE_TOKEN; } @@ -302,7 +301,7 @@ int32_t tqStorePersist(STqMetaStore* pMeta) { pSHead->ver = TQ_SVER; pSHead->checksum = 0; pSHead->ssize = sizeof(STqSerializedHead); - int allocatedSize = sizeof(STqSerializedHead); + /*int allocatedSize = sizeof(STqSerializedHead);*/ int offset = lseek(pMeta->fileFd, 0, SEEK_CUR); tqReadLastPage(pMeta->idxFd, &idxBuf); @@ -417,14 +416,14 @@ static int32_t tqHandlePutCommitted(STqMetaStore* pMeta, int64_t key, void* valu pNode = pNode->next; } } - STqMetaList* pNewNode = malloc(sizeof(STqMetaList)); + STqMetaList* pNewNode = calloc(1, sizeof(STqMetaList)); if (pNewNode == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return -1; } - memset(pNewNode, 0, sizeof(STqMetaList)); pNewNode->handle.key = key; pNewNode->handle.valueInUse = value; + pNewNode->next = pMeta->bucket[bucketKey]; // put into unpersist list pNewNode->unpersistPrev = pMeta->unpersistHead; pNewNode->unpersistNext = pMeta->unpersistHead->unpersistNext; @@ -489,12 +488,11 @@ static inline int32_t tqHandlePutImpl(STqMetaStore* pMeta, int64_t key, void* va pNode = pNode->next; } } - STqMetaList* pNewNode = malloc(sizeof(STqMetaList)); + STqMetaList* pNewNode = calloc(1, sizeof(STqMetaList)); if (pNewNode == NULL) { terrno = TSDB_CODE_TQ_OUT_OF_MEMORY; return -1; } - memset(pNewNode, 0, sizeof(STqMetaList)); pNewNode->handle.key = key; pNewNode->handle.valueInTxn = value; pNewNode->next = pMeta->bucket[bucketKey]; diff --git a/source/nodes/src/nodesCloneFuncs.c b/source/dnode/vnode/src/tq/tqOffset.c similarity index 52% rename from source/nodes/src/nodesCloneFuncs.c rename to source/dnode/vnode/src/tq/tqOffset.c index 2d0a6483ae..4115cb7313 100644 --- a/source/nodes/src/nodesCloneFuncs.c +++ b/source/dnode/vnode/src/tq/tqOffset.c @@ -12,9 +12,31 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ +#define _DEFAULT_SOURCE -#include "nodes.h" +#include "tqOffset.h" -void nodesCloneNode(const SNode* pNode) { +enum ETqOffsetPersist { + TQ_OFFSET_PERSIST__LAZY = 1, + TQ_OFFSET_PERSIST__EAGER, +}; +struct STqOffsetCfg { + int8_t persistPolicy; +}; + +struct STqOffsetStore { + STqOffsetCfg cfg; + SHashObj* pHash; // SHashObj +}; + +STqOffsetStore* STqOffsetOpen(STqOffsetCfg* pCfg) { + STqOffsetStore* pStore = malloc(sizeof(STqOffsetStore)); + if (pStore == NULL) { + return NULL; + } + memcpy(&pStore->cfg, pCfg, sizeof(STqOffsetCfg)); + pStore->pHash = taosHashInit(64, MurmurHash3_32, true, HASH_NO_LOCK); + return pStore; } + diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c new file mode 100644 index 0000000000..e76d43becd --- /dev/null +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -0,0 +1,160 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#define _DEFAULT_SOURCE + +#include "vnode.h" + +STqReadHandle* tqInitSubmitMsgScanner(SMeta* pMeta) { + STqReadHandle* pReadHandle = malloc(sizeof(STqReadHandle)); + if (pReadHandle == NULL) { + return NULL; + } + pReadHandle->pVnodeMeta = pMeta; + pReadHandle->pMsg = NULL; + pReadHandle->ver = -1; + pReadHandle->pColIdList = NULL; + pReadHandle->sver = -1; + pReadHandle->pSchema = NULL; + pReadHandle->pSchemaWrapper = NULL; + return pReadHandle; +} + +void tqReadHandleSetMsg(STqReadHandle* pReadHandle, SSubmitReq* pMsg, int64_t ver) { + pReadHandle->pMsg = pMsg; + pMsg->length = htonl(pMsg->length); + pMsg->numOfBlocks = htonl(pMsg->numOfBlocks); + tInitSubmitMsgIter(pMsg, &pReadHandle->msgIter); + pReadHandle->ver = ver; + memset(&pReadHandle->blkIter, 0, sizeof(SSubmitBlkIter)); +} + +bool tqNextDataBlock(STqReadHandle* pHandle) { + while (1) { + if (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) < 0) { + return false; + } + if (pHandle->pBlock == NULL) return false; + + pHandle->pBlock->uid = htobe64(pHandle->pBlock->uid); + /*if (pHandle->tbUid == pHandle->pBlock->uid) {*/ + ASSERT(pHandle->tbIdHash); + void* ret = taosHashGet(pHandle->tbIdHash, &pHandle->pBlock->uid, sizeof(int64_t)); + if (ret != NULL) { + /*printf("retrieve one tb %ld\n", pHandle->pBlock->uid);*/ + pHandle->pBlock->tid = htonl(pHandle->pBlock->tid); + pHandle->pBlock->sversion = htonl(pHandle->pBlock->sversion); + pHandle->pBlock->dataLen = htonl(pHandle->pBlock->dataLen); + pHandle->pBlock->schemaLen = htonl(pHandle->pBlock->schemaLen); + pHandle->pBlock->numOfRows = htons(pHandle->pBlock->numOfRows); + return true; + } else { + /*printf("skip one tb %ld\n", pHandle->pBlock->uid);*/ + } + } + return false; +} + +int tqRetrieveDataBlockInfo(STqReadHandle* pHandle, SDataBlockInfo* pBlockInfo) { + /*int32_t sversion = pHandle->pBlock->sversion;*/ + /*SSchemaWrapper* pSchema = metaGetTableSchema(pHandle->pMeta, pHandle->pBlock->uid, sversion, false);*/ + pBlockInfo->numOfCols = taosArrayGetSize(pHandle->pColIdList); + pBlockInfo->rows = pHandle->pBlock->numOfRows; + pBlockInfo->uid = pHandle->pBlock->uid; + return 0; +} + +SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) { + /*int32_t sversion = pHandle->pBlock->sversion;*/ + // TODO set to real sversion + int32_t sversion = 0; + if (pHandle->sver != sversion) { + pHandle->pSchema = metaGetTbTSchema(pHandle->pVnodeMeta, pHandle->pBlock->uid, sversion); + + tb_uid_t quid; + STbCfg* pTbCfg = metaGetTbInfoByUid(pHandle->pVnodeMeta, pHandle->pBlock->uid); + if (pTbCfg->type == META_CHILD_TABLE) { + quid = pTbCfg->ctbCfg.suid; + } else { + quid = pHandle->pBlock->uid; + } + pHandle->pSchemaWrapper = metaGetTableSchema(pHandle->pVnodeMeta, quid, sversion, true); + pHandle->sver = sversion; + } + + STSchema* pTschema = pHandle->pSchema; + SSchemaWrapper* pSchemaWrapper = pHandle->pSchemaWrapper; + + int32_t numOfRows = pHandle->pBlock->numOfRows; + int32_t numOfCols = pHandle->pSchema->numOfCols; + int32_t colNumNeed = taosArrayGetSize(pHandle->pColIdList); + + // TODO: stable case + if (colNumNeed > pSchemaWrapper->nCols) { + colNumNeed = pSchemaWrapper->nCols; + } + + SArray* pArray = taosArrayInit(colNumNeed, sizeof(SColumnInfoData)); + if (pArray == NULL) { + return NULL; + } + + int j = 0; + for (int32_t i = 0; i < colNumNeed; i++) { + int32_t colId = *(int32_t*)taosArrayGet(pHandle->pColIdList, i); + while (j < pSchemaWrapper->nCols && pSchemaWrapper->pSchema[j].colId < colId) { + j++; + } + SSchema* pColSchema = &pSchemaWrapper->pSchema[j]; + SColumnInfoData colInfo = {0}; + int sz = numOfRows * pColSchema->bytes; + colInfo.info.bytes = pColSchema->bytes; + colInfo.info.colId = colId; + colInfo.info.type = pColSchema->type; + + colInfo.pData = calloc(1, sz); + if (colInfo.pData == NULL) { + // TODO free + taosArrayDestroy(pArray); + return NULL; + } + taosArrayPush(pArray, &colInfo); + } + + STSRowIter iter = {0}; + tdSTSRowIterInit(&iter, pTschema); + STSRow* row; + // int32_t kvIdx = 0; + int32_t curRow = 0; + tInitSubmitBlkIter(pHandle->pBlock, &pHandle->blkIter); + while ((row = tGetSubmitBlkNext(&pHandle->blkIter)) != NULL) { + tdSTSRowIterReset(&iter, row); + // get all wanted col of that block + for (int32_t i = 0; i < colNumNeed; i++) { + SColumnInfoData* pColData = taosArrayGet(pArray, i); + STColumn* pCol = schemaColAt(pTschema, i); + // TODO + ASSERT(pCol->colId == pColData->info.colId); + // void* val = tdGetMemRowDataOfColEx(row, pCol->colId, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset, &kvIdx); + SCellVal sVal = {0}; + if (!tdSTSRowIterNext(&iter, pCol->colId, pCol->type, &sVal)) { + // TODO: reach end + break; + } + memcpy(POINTER_SHIFT(pColData->pData, curRow * pCol->bytes), sVal.val, pCol->bytes); + } + curRow++; + } + return pArray; +} diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 9b44e14c9b..26c313f421 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -18,7 +18,7 @@ #define TSDB_MAX_SUBBLOCKS 8 typedef struct { - STable * pTable; + STable *pTable; SSkipListIterator *pIter; } SCommitIter; @@ -34,11 +34,11 @@ typedef struct { bool isLFileSame; TSKEY minKey; TSKEY maxKey; - SArray * aBlkIdx; // SBlockIdx array - STable * pTable; - SArray * aSupBlk; // Table super-block array - SArray * aSubBlk; // table sub-block array - SDataCols * pDataCols; + SArray *aBlkIdx; // SBlockIdx array + STable *pTable; + SArray *aSupBlk; // Table super-block array + SArray *aSubBlk; // table sub-block array + SDataCols *pDataCols; } SCommitH; #define TSDB_DEFAULT_BLOCK_ROWS(maxRows) ((maxRows)*4 / 5) @@ -90,7 +90,7 @@ int tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf); int tsdbApplyRtnOnFSet(STsdb *pRepo, SDFileSet *pSet, SRtn *pRtn) { SDiskID did; SDFileSet nSet = {0}; - STsdbFS * pfs = REPO_FS(pRepo); + STsdbFS *pfs = REPO_FS(pRepo); int level; ASSERT(pSet->fid >= pRtn->minFid); @@ -135,12 +135,13 @@ int tsdbPrepareCommit(STsdb *pTsdb) { pTsdb->imem = pTsdb->mem; pTsdb->mem = NULL; + return 0; } int tsdbCommit(STsdb *pRepo) { STsdbMemTable *pMem = pRepo->imem; SCommitH commith = {0}; - SDFileSet * pSet = NULL; + SDFileSet *pSet = NULL; int fid; if (pRepo->imem == NULL) return 0; @@ -303,7 +304,7 @@ static void tsdbSeekCommitIter(SCommitH *pCommith, TSKEY key) { } static int tsdbNextCommitFid(SCommitH *pCommith) { - STsdb * pRepo = TSDB_COMMIT_REPO(pCommith); + STsdb *pRepo = TSDB_COMMIT_REPO(pCommith); STsdbCfg *pCfg = REPO_CFG(pRepo); int fid = TSDB_IVLD_FID; @@ -336,7 +337,7 @@ static void tsdbDestroyCommitH(SCommitH *pCommith) { } static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid) { - STsdb * pRepo = TSDB_COMMIT_REPO(pCommith); + STsdb *pRepo = TSDB_COMMIT_REPO(pCommith); STsdbCfg *pCfg = REPO_CFG(pRepo); ASSERT(pSet == NULL || pSet->fid == fid); @@ -391,12 +392,12 @@ static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid) { } static int tsdbCreateCommitIters(SCommitH *pCommith) { - STsdb * pRepo = TSDB_COMMIT_REPO(pCommith); - STsdbMemTable * pMem = pRepo->imem; + STsdb *pRepo = TSDB_COMMIT_REPO(pCommith); + STsdbMemTable *pMem = pRepo->imem; SSkipListIterator *pSlIter; - SCommitIter * pCommitIter; - SSkipListNode * pNode; - STbData * pTbData; + SCommitIter *pCommitIter; + SSkipListNode *pNode; + STbData *pTbData; pCommith->niters = SL_SIZE(pMem->pSlIdx); pCommith->iters = (SCommitIter *)calloc(pCommith->niters, sizeof(SCommitIter)); @@ -452,7 +453,7 @@ static void tsdbResetCommitFile(SCommitH *pCommith) { static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid) { SDiskID did; - STsdb * pRepo = TSDB_COMMIT_REPO(pCommith); + STsdb *pRepo = TSDB_COMMIT_REPO(pCommith); SDFileSet *pWSet = TSDB_COMMIT_WRITE_FSET(pCommith); if (tfsAllocDisk(pRepo->pTfs, tsdbGetFidLevel(fid, &(pCommith->rtn)), &did) < 0) { @@ -583,7 +584,7 @@ int tsdbWriteBlockInfoImpl(SDFile *pHeadf, STable *pTable, SArray *pSupA, SArray uint32_t tlen; SBlockInfo *pBlkInfo; int64_t offset; - SBlock * pBlock; + SBlock *pBlock; memset(pIdx, 0, sizeof(*pIdx)); @@ -1130,7 +1131,7 @@ static int tsdbComparKeyBlock(const void *arg1, const void *arg2) { int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDataCols *pDataCols, SBlock *pBlock, bool isLast, bool isSuper, void **ppBuf, void **ppCBuf) { - STsdbCfg * pCfg = REPO_CFG(pRepo); + STsdbCfg *pCfg = REPO_CFG(pRepo); SBlockData *pBlockData; int64_t offset = 0; int rowsToWrite = pDataCols->numOfRows; @@ -1147,7 +1148,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDataCols * // Get # of cols not all NULL(not including key column) int nColsNotAllNull = 0; for (int ncol = 1; ncol < pDataCols->numOfCols; ncol++) { // ncol from 1, we skip the timestamp column - SDataCol * pDataCol = pDataCols->cols + ncol; + SDataCol *pDataCol = pDataCols->cols + ncol; SBlockCol *pBlockCol = pBlockData->cols + nColsNotAllNull; if (isAllRowsNull(pDataCol)) { // all data to commit are NULL, just ignore it @@ -1188,7 +1189,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDataCols * // All not NULL columns finish if (ncol != 0 && tcol >= nColsNotAllNull) break; - SDataCol * pDataCol = pDataCols->cols + ncol; + SDataCol *pDataCol = pDataCols->cols + ncol; SBlockCol *pBlockCol = pBlockData->cols + tcol; if (ncol != 0 && (pDataCol->colId != pBlockCol->colId)) continue; @@ -1212,7 +1213,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDataCols * } #endif - void * tptr; + void *tptr; // Make room if (tsdbMakeRoom(ppBuf, lsize + tlen + COMP_OVERFLOW_BYTES + sizeof(TSCKSUM)) < 0) { @@ -1278,7 +1279,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDataCols * pBlock->keyFirst = dataColsKeyFirst(pDataCols); pBlock->keyLast = dataColsKeyLast(pDataCols); - tsdbDebug("vgId:%d uid:%"PRId64" a block of data is written to file %s, offset %" PRId64 + tsdbDebug("vgId:%d uid:%" PRId64 " a block of data is written to file %s, offset %" PRId64 " numOfRows %d len %d numOfCols %" PRId16 " keyFirst %" PRId64 " keyLast %" PRId64, REPO_ID(pRepo), TABLE_TID(pTable), TSDB_FILE_FULL_NAME(pDFile), offset, rowsToWrite, pBlock->len, pBlock->numOfCols, pBlock->keyFirst, pBlock->keyLast); @@ -1294,9 +1295,9 @@ static int tsdbWriteBlock(SCommitH *pCommith, SDFile *pDFile, SDataCols *pDataCo } static int tsdbWriteBlockInfo(SCommitH *pCommih) { - SDFile * pHeadf = TSDB_COMMIT_HEAD_FILE(pCommih); + SDFile *pHeadf = TSDB_COMMIT_HEAD_FILE(pCommih); SBlockIdx blkIdx; - STable * pTable = TSDB_COMMIT_TABLE(pCommih); + STable *pTable = TSDB_COMMIT_TABLE(pCommih); if (tsdbWriteBlockInfoImpl(pHeadf, pTable, pCommih->aSupBlk, pCommih->aSubBlk, (void **)(&(TSDB_COMMIT_BUF(pCommih))), &blkIdx) < 0) { @@ -1316,11 +1317,11 @@ static int tsdbWriteBlockInfo(SCommitH *pCommih) { } static int tsdbCommitMemData(SCommitH *pCommith, SCommitIter *pIter, TSKEY keyLimit, bool toData) { - STsdb * pRepo = TSDB_COMMIT_REPO(pCommith); - STsdbCfg * pCfg = REPO_CFG(pRepo); + STsdb *pRepo = TSDB_COMMIT_REPO(pCommith); + STsdbCfg *pCfg = REPO_CFG(pRepo); SMergeInfo mInfo; int32_t defaultRows = TSDB_COMMIT_DEFAULT_ROWS(pCommith); - SDFile * pDFile; + SDFile *pDFile; bool isLast; SBlock block; @@ -1349,16 +1350,16 @@ static int tsdbCommitMemData(SCommitH *pCommith, SCommitIter *pIter, TSKEY keyLi } static int tsdbMergeMemData(SCommitH *pCommith, SCommitIter *pIter, int bidx) { - STsdb * pRepo = TSDB_COMMIT_REPO(pCommith); - STsdbCfg * pCfg = REPO_CFG(pRepo); + STsdb *pRepo = TSDB_COMMIT_REPO(pCommith); + STsdbCfg *pCfg = REPO_CFG(pRepo); int nBlocks = pCommith->readh.pBlkIdx->numOfBlocks; - SBlock * pBlock = pCommith->readh.pBlkInfo->blocks + bidx; + SBlock *pBlock = pCommith->readh.pBlkInfo->blocks + bidx; TSKEY keyLimit; int16_t colId = PRIMARYKEY_TIMESTAMP_COL_ID; SMergeInfo mInfo; SBlock subBlocks[TSDB_MAX_SUBBLOCKS]; SBlock block, supBlock; - SDFile * pDFile; + SDFile *pDFile; if (bidx == nBlocks - 1) { keyLimit = pCommith->maxKey; @@ -1474,10 +1475,10 @@ static int tsdbCommitAddBlock(SCommitH *pCommith, const SBlock *pSupBlock, const static int tsdbMergeBlockData(SCommitH *pCommith, SCommitIter *pIter, SDataCols *pDataCols, TSKEY keyLimit, bool isLastOneBlock) { - STsdb * pRepo = TSDB_COMMIT_REPO(pCommith); + STsdb *pRepo = TSDB_COMMIT_REPO(pCommith); STsdbCfg *pCfg = REPO_CFG(pRepo); SBlock block; - SDFile * pDFile; + SDFile *pDFile; bool isLast; int32_t defaultRows = TSDB_COMMIT_DEFAULT_ROWS(pCommith); @@ -1598,7 +1599,7 @@ static void tsdbCloseCommitFile(SCommitH *pCommith, bool hasError) { } static bool tsdbCanAddSubBlock(SCommitH *pCommith, SBlock *pBlock, SMergeInfo *pInfo) { - STsdb * pRepo = TSDB_COMMIT_REPO(pCommith); + STsdb *pRepo = TSDB_COMMIT_REPO(pCommith); STsdbCfg *pCfg = REPO_CFG(pRepo); int mergeRows = pBlock->numOfRows + pInfo->rowsInserted - pInfo->rowsDeleteSucceed; diff --git a/source/dnode/vnode/src/tsdb/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c index b51539f697..01813af556 100644 --- a/source/dnode/vnode/src/tsdb/tsdbMemTable.c +++ b/source/dnode/vnode/src/tsdb/tsdbMemTable.c @@ -15,7 +15,7 @@ #include "tsdbDef.h" -static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitMsg *pMsg); +static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg); static int tsdbMemTableInsertTbData(STsdb *pRepo, SSubmitBlk *pBlock, int32_t *pAffectedRows); static STbData *tsdbNewTbData(tb_uid_t uid); static void tsdbFreeTbData(STbData *pTbData); @@ -73,7 +73,7 @@ void tsdbFreeMemTable(STsdb *pTsdb, STsdbMemTable *pMemTable) { } } -int tsdbMemTableInsert(STsdb *pTsdb, STsdbMemTable *pMemTable, SSubmitMsg *pMsg, SShellSubmitRsp *pRsp) { +int tsdbMemTableInsert(STsdb *pTsdb, STsdbMemTable *pMemTable, SSubmitReq *pMsg, SSubmitRsp *pRsp) { SSubmitBlk * pBlock = NULL; SSubmitMsgIter msgIter = {0}; int32_t affectedrows = 0, numOfRows = 0; @@ -227,7 +227,7 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey return 0; } -static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitMsg *pMsg) { +static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitReq *pMsg) { ASSERT(pMsg != NULL); // STsdbMeta * pMeta = pTsdb->tsdbMeta; SSubmitMsgIter msgIter = {0}; @@ -455,7 +455,7 @@ static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema * /* ------------------------ REFACTORING ------------------------ */ #if 0 -int tsdbInsertDataToMemTable(STsdbMemTable *pMemTable, SSubmitMsg *pMsg) { +int tsdbInsertDataToMemTable(STsdbMemTable *pMemTable, SSubmitReq *pMsg) { SMemAllocator *pMA = pMemTable->pMA; STbData * pTbData = (STbData *)TD_MA_MALLOC(pMA, sizeof(*pTbData)); if (pTbData == NULL) { @@ -496,9 +496,9 @@ static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables); static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, STSRow* row); static int tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter); static STSRow* tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter); -static int tsdbScanAndConvertSubmitMsg(STsdbRepo *pRepo, SSubmitMsg *pMsg); +static int tsdbScanAndConvertSubmitMsg(STsdbRepo *pRepo, SSubmitReq *pMsg); static int tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, int32_t *affectedrows); -static int tsdbInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter); +static int tsdbInitSubmitMsgIter(SSubmitReq *pMsg, SSubmitMsgIter *pIter); static int tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock); static int tsdbCheckTableSchema(STsdbRepo *pRepo, SSubmitBlk *pBlock, STable *pTable); static int tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable, STSRow* row); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 5bbc309661..bed2c0cd41 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -2174,8 +2174,8 @@ static int32_t createDataBlocksInfo(STsdbReadHandle* pTsdbReadHandle, int32_t nu assert(cnt <= numOfBlocks && numOfQualTables <= numOfTables); // the pTableQueryInfo[j]->numOfBlocks may be 0 sup.numOfTables = numOfQualTables; - SLoserTreeInfo* pTree = NULL; - uint8_t ret = tLoserTreeCreate(&pTree, sup.numOfTables, &sup, dataBlockOrderCompar); + SMultiwayMergeTreeInfo* pTree = NULL; + uint8_t ret = tMergeTreeCreate(&pTree, sup.numOfTables, &sup, dataBlockOrderCompar); if (ret != TSDB_CODE_SUCCESS) { cleanBlockOrderSupporter(&sup, numOfTables); return TSDB_CODE_TDB_OUT_OF_MEMORY; @@ -2184,7 +2184,7 @@ static int32_t createDataBlocksInfo(STsdbReadHandle* pTsdbReadHandle, int32_t nu int32_t numOfTotal = 0; while (numOfTotal < cnt) { - int32_t pos = pTree->pNode[0].index; + int32_t pos = tMergeTreeGetChosenIndex(pTree); int32_t index = sup.blockIndexArray[pos]++; STableBlockInfo* pBlocksInfo = sup.pDataBlockInfo[pos]; @@ -2195,7 +2195,7 @@ static int32_t createDataBlocksInfo(STsdbReadHandle* pTsdbReadHandle, int32_t nu sup.blockIndexArray[pos] = sup.numOfBlocksPerTable[pos] + 1; } - tLoserTreeAdjust(pTree, pos + sup.numOfTables); + tMergeTreeAdjust(pTree, tMergeTreeGetAdjustIndex(pTree)); } /* @@ -3643,13 +3643,13 @@ int32_t tsdbQuerySTableByTagCond(void* pMeta, uint64_t uid, TSKEY skey, const ch SColIndex* pColIndex, int32_t numOfCols, uint64_t reqId, uint64_t taskId) { STbCfg* pTbCfg = metaGetTbInfoByUid(pMeta, uid); if (pTbCfg == NULL) { -// tsdbError("%p failed to get stable, uid:%"PRIu64", TID:0x%"PRIx64" QID:0x%"PRIx64, tsdb, uid, taskId, reqId); + tsdbError("%p failed to get stable, uid:%"PRIu64", TID:0x%"PRIx64" QID:0x%"PRIx64, pMeta, uid, taskId, reqId); terrno = TSDB_CODE_TDB_INVALID_TABLE_ID; goto _error; } if (pTbCfg->type != META_SUPER_TABLE) { -// tsdbError("%p query normal tag not allowed, uid:%" PRIu64 ", TID:0x%"PRIx64" QID:0x%"PRIx64, tsdb, uid, taskId, reqId); + tsdbError("%p query normal tag not allowed, uid:%" PRIu64 ", TID:0x%"PRIx64" QID:0x%"PRIx64, pMeta, uid, taskId, reqId); terrno = TSDB_CODE_OPS_NOT_SUPPORT; //basically, this error is caused by invalid sql issued by client goto _error; } @@ -3668,8 +3668,8 @@ int32_t tsdbQuerySTableByTagCond(void* pMeta, uint64_t uid, TSKEY skey, const ch pGroupInfo->numOfTables = (uint32_t) taosArrayGetSize(res); pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols, skey); -// tsdbDebug("%p no table name/tag condition, all tables qualified, numOfTables:%u, group:%zu, TID:0x%"PRIx64" QID:0x%"PRIx64, tsdb, -// pGroupInfo->numOfTables, taosArrayGetSize(pGroupInfo->pGroupList), taskId, reqId); + tsdbDebug("%p no table name/tag condition, all tables qualified, numOfTables:%u, group:%zu, TID:0x%"PRIx64" QID:0x%"PRIx64, pMeta, + pGroupInfo->numOfTables, taosArrayGetSize(pGroupInfo->pGroupList), taskId, reqId); taosArrayDestroy(res); return ret; diff --git a/source/dnode/vnode/src/tsdb/tsdbWrite.c b/source/dnode/vnode/src/tsdb/tsdbWrite.c index 5f937f17e9..78067f8f83 100644 --- a/source/dnode/vnode/src/tsdb/tsdbWrite.c +++ b/source/dnode/vnode/src/tsdb/tsdbWrite.c @@ -15,7 +15,7 @@ #include "tsdbDef.h" -int tsdbInsertData(STsdb *pTsdb, SSubmitMsg *pMsg, SSubmitRsp *pRsp) { +int tsdbInsertData(STsdb *pTsdb, SSubmitReq *pMsg, SSubmitRsp *pRsp) { // Check if mem is there. If not, create one. if (pTsdb->mem == NULL) { pTsdb->mem = tsdbNewMemTable(pTsdb); diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index f541834bec..f779949f14 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -71,7 +71,6 @@ int vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg) { } static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) { - STableInfoReq * pReq = (STableInfoReq *)(pMsg->pCont); STbCfg * pTbCfg = NULL; STbCfg * pStbCfg = NULL; tb_uid_t uid; @@ -79,12 +78,19 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) { int32_t nTagCols; SSchemaWrapper *pSW = NULL; STableMetaRsp *pTbMetaMsg = NULL; - SSchema * pTagSchema; + STableMetaRsp metaRsp = {0}; + SSchema *pTagSchema; SRpcMsg rpcMsg; int msgLen = 0; int32_t code = TSDB_CODE_VND_APP_ERROR; - pTbCfg = metaGetTbInfoByName(pVnode->pMeta, pReq->tbName, &uid); + STableInfoReq infoReq = {0}; + if (tDeserializeSTableInfoReq(pMsg->pCont, pMsg->contLen, &infoReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + pTbCfg = metaGetTbInfoByName(pVnode->pMeta, infoReq.tbName, &uid); if (pTbCfg == NULL) { code = TSDB_CODE_VND_TB_NOT_EXIST; goto _exit; @@ -114,44 +120,51 @@ static int vnodeGetTableMeta(SVnode *pVnode, SRpcMsg *pMsg) { pTagSchema = NULL; } - msgLen = sizeof(STableMetaRsp) + sizeof(SSchema) * (nCols + nTagCols); - pTbMetaMsg = (STableMetaRsp *)rpcMallocCont(msgLen); - if (pTbMetaMsg == NULL) { + metaRsp.pSchemas = calloc(nCols + nTagCols, sizeof(SSchema)); + if (metaRsp.pSchemas == NULL) { code = TSDB_CODE_VND_OUT_OF_MEMORY; goto _exit; } - pTbMetaMsg->dbId = htobe64(pVnode->config.dbId); - memcpy(pTbMetaMsg->dbFName, pReq->dbFName, sizeof(pTbMetaMsg->dbFName)); - strcpy(pTbMetaMsg->tbName, pReq->tbName); + metaRsp.dbId = htobe64(pVnode->config.dbId); + memcpy(metaRsp.dbFName, infoReq.dbFName, sizeof(metaRsp.dbFName)); + strcpy(metaRsp.tbName, infoReq.tbName); if (pTbCfg->type == META_CHILD_TABLE) { - strcpy(pTbMetaMsg->stbName, pStbCfg->name); - pTbMetaMsg->suid = htobe64(pTbCfg->ctbCfg.suid); + strcpy(metaRsp.stbName, pStbCfg->name); + metaRsp.suid = pTbCfg->ctbCfg.suid; } else if (pTbCfg->type == META_SUPER_TABLE) { - strcpy(pTbMetaMsg->stbName, pTbCfg->name); - pTbMetaMsg->suid = htobe64(uid); + strcpy(metaRsp.stbName, pTbCfg->name); + metaRsp.suid = uid; } - pTbMetaMsg->numOfTags = htonl(nTagCols); - pTbMetaMsg->numOfColumns = htonl(nCols); - pTbMetaMsg->tableType = pTbCfg->type; - pTbMetaMsg->tuid = htobe64(uid); - pTbMetaMsg->vgId = htonl(pVnode->vgId); + metaRsp.numOfTags = nTagCols; + metaRsp.numOfColumns = nCols; + metaRsp.tableType = pTbCfg->type; + metaRsp.tuid = uid; + metaRsp.vgId = pVnode->vgId; - memcpy(pTbMetaMsg->pSchema, pSW->pSchema, sizeof(SSchema) * pSW->nCols); + memcpy(metaRsp.pSchemas, pSW->pSchema, sizeof(SSchema) * pSW->nCols); if (nTagCols) { - memcpy(POINTER_SHIFT(pTbMetaMsg->pSchema, sizeof(SSchema) * pSW->nCols), pTagSchema, sizeof(SSchema) * nTagCols); + memcpy(POINTER_SHIFT(metaRsp.pSchemas, sizeof(SSchema) * pSW->nCols), pTagSchema, sizeof(SSchema) * nTagCols); } - for (int i = 0; i < nCols + nTagCols; i++) { - SSchema *pSch = pTbMetaMsg->pSchema + i; - pSch->colId = htonl(pSch->colId); - pSch->bytes = htonl(pSch->bytes); + int32_t rspLen = tSerializeSTableMetaRsp(NULL, 0, &metaRsp); + if (rspLen < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; } + void *pRsp = rpcMallocCont(rspLen); + if (pRsp == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + tSerializeSTableMetaRsp(pRsp, rspLen, &metaRsp); + code = 0; _exit: + tFreeSTableMetaRsp(&metaRsp); if (pSW != NULL) { tfree(pSW->pSchema); tfree(pSW); @@ -170,13 +183,13 @@ _exit: rpcMsg.handle = pMsg->handle; rpcMsg.ahandle = pMsg->ahandle; - rpcMsg.pCont = pTbMetaMsg; - rpcMsg.contLen = msgLen; + rpcMsg.pCont = pRsp; + rpcMsg.contLen = rspLen; rpcMsg.code = code; rpcSendResponse(&rpcMsg); - return 0; + return code; } static void freeItemHelper(void *pItem) { diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index 4d048bc6e2..5c39c65f9f 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -16,19 +16,6 @@ #include "tq.h" #include "vnd.h" -#if 0 -int vnodeProcessNoWalWMsgs(SVnode *pVnode, SRpcMsg *pMsg) { - switch (pMsg->msgType) { - case TDMT_VND_MQ_SET_CUR: - if (tqSetCursor(pVnode->pTq, pMsg->pCont) < 0) { - // TODO: handle error - } - break; - } - return 0; -} -#endif - int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { SRpcMsg *pMsg; @@ -36,11 +23,12 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { pMsg = *(SRpcMsg **)taosArrayGet(pMsgs, i); // ser request version - void * pBuf = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); + void *pBuf = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int64_t ver = pVnode->state.processed++; - taosEncodeFixedU64(&pBuf, ver); + taosEncodeFixedI64(&pBuf, ver); if (walWrite(pVnode->pWal, ver, pMsg->msgType, pMsg->pCont, pMsg->contLen) < 0) { + /*ASSERT(false);*/ // TODO: handle error } } @@ -55,7 +43,7 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) { int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { SVCreateTbReq vCreateTbReq; SVCreateTbBatchReq vCreateTbBatchReq; - void * ptr = vnodeMalloc(pVnode, pMsg->contLen); + void *ptr = vnodeMalloc(pVnode, pMsg->contLen); if (ptr == NULL) { // TODO: handle error } @@ -64,8 +52,8 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { memcpy(ptr, pMsg->pCont, pMsg->contLen); // todo: change the interface here - uint64_t ver; - taosDecodeFixedU64(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &ver); + int64_t ver; + taosDecodeFixedI64(POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)), &ver); if (tqPushMsg(pVnode->pTq, ptr, ver) < 0) { // TODO: handle error } @@ -121,7 +109,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { // } break; case TDMT_VND_SUBMIT: - if (tsdbInsertData(pVnode->pTsdb, (SSubmitMsg *)ptr, NULL) < 0) { + if (tsdbInsertData(pVnode->pTsdb, (SSubmitReq *)ptr, NULL) < 0) { // TODO: handle error } break; @@ -132,7 +120,6 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { } break; case TDMT_VND_MQ_REB: { if (tqProcessRebReq(pVnode->pTq, POINTER_SHIFT(ptr, sizeof(SMsgHead))) < 0) { - } } break; default: diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index 1d23f333b2..049b69991f 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -13,3 +13,4 @@ add_subdirectory(function) add_subdirectory(qcom) add_subdirectory(qworker) add_subdirectory(tfs) +add_subdirectory(nodes) \ No newline at end of file diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index f1c2395de4..b5c7f44f98 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -1440,7 +1440,7 @@ int32_t ctgRefreshTblMeta(SCatalog* pCtg, void *pTransporter, const SEpSet* pMgm SCtgUpdateTblMsg *msg = NULL; STableMetaOutput moutput = {0}; - STableMetaOutput *output = malloc(sizeof(STableMetaOutput)); + STableMetaOutput *output = calloc(1, sizeof(STableMetaOutput)); if (NULL == output) { ctgError("malloc %d failed", (int32_t)sizeof(STableMetaOutput)); CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR); diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 769be97f7d..1c84f06a22 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -65,14 +65,14 @@ bool ctgTestDeadLoop = false; int32_t ctgTestPrintNum = 200000; int32_t ctgTestMTRunSec = 5; -int32_t ctgTestCurrentVgVersion = 0; -int32_t ctgTestVgVersion = 1; -int32_t ctgTestVgNum = 10; -int32_t ctgTestColNum = 2; -int32_t ctgTestTagNum = 1; -int32_t ctgTestSVersion = 1; -int32_t ctgTestTVersion = 1; -int32_t ctgTestSuid = 2; +int32_t ctgTestCurrentVgVersion = 0; +int32_t ctgTestVgVersion = 1; +int32_t ctgTestVgNum = 10; +int32_t ctgTestColNum = 2; +int32_t ctgTestTagNum = 1; +int32_t ctgTestSVersion = 1; +int32_t ctgTestTVersion = 1; +int32_t ctgTestSuid = 2; uint64_t ctgTestDbId = 33; uint64_t ctgTestClusterId = 0x1; @@ -85,35 +85,38 @@ int32_t ctgTestRspFunc[10] = {0}; int32_t ctgTestRspIdx = 0; void sendCreateDbMsg(void *shandle, SEpSet *pEpSet) { - SCreateDbReq *pReq = (SCreateDbReq *)rpcMallocCont(sizeof(SCreateDbReq)); - strcpy(pReq->db, "1.db1"); - pReq->numOfVgroups = htonl(2); - pReq->cacheBlockSize = htonl(16); - pReq->totalBlocks = htonl(10); - pReq->daysPerFile = htonl(10); - pReq->daysToKeep0 = htonl(3650); - pReq->daysToKeep1 = htonl(3650); - pReq->daysToKeep2 = htonl(3650); - pReq->minRows = htonl(100); - pReq->maxRows = htonl(4096); - pReq->commitTime = htonl(3600); - pReq->fsyncPeriod = htonl(3000); - pReq->walLevel = 1; - pReq->precision = 0; - pReq->compression = 2; - pReq->replications = 1; - pReq->quorum = 1; - pReq->update = 0; - pReq->cacheLastRow = 0; - pReq->ignoreExist = 1; + SCreateDbReq createReq = {0}; + strcpy(createReq.db, "1.db1"); + createReq.numOfVgroups = 2; + createReq.cacheBlockSize = 16; + createReq.totalBlocks = 10; + createReq.daysPerFile = 10; + createReq.daysToKeep0 = 3650; + createReq.daysToKeep1 = 3650; + createReq.daysToKeep2 = 3650; + createReq.minRows = 100; + createReq.maxRows = 4096; + createReq.commitTime = 3600; + createReq.fsyncPeriod = 3000; + createReq.walLevel = 1; + createReq.precision = 0; + createReq.compression = 2; + createReq.replications = 1; + createReq.quorum = 1; + createReq.update = 0; + createReq.cacheLastRow = 0; + createReq.ignoreExist = 1; + + int32_t contLen = tSerializeSCreateDbReq(NULL, 0, &createReq); + void *pReq = rpcMallocCont(contLen); + tSerializeSCreateDbReq(pReq, contLen, &createReq); SRpcMsg rpcMsg = {0}; rpcMsg.pCont = pReq; - rpcMsg.contLen = sizeof(SCreateDbReq); + rpcMsg.contLen = contLen; rpcMsg.msgType = TDMT_MND_CREATE_DB; SRpcMsg rpcRsp = {0}; - rpcSendRecv(shandle, pEpSet, &rpcMsg, &rpcRsp); ASSERT_EQ(rpcRsp.code, 0); @@ -219,7 +222,7 @@ void ctgTestBuildDBVgroup(SDBVgInfo **pdbVgroup) { for (int32_t n = 0; n < vgInfo.epset.numOfEps; ++n) { SEp *addr = &vgInfo.epset.eps[n]; strcpy(addr->fqdn, "a0"); - addr->port = htons(n + 22); + addr->port = n + 22; } taosHashPut(dbVgroup->vgHash, &vgInfo.vgId, sizeof(vgInfo.vgId), &vgInfo, sizeof(vgInfo)); @@ -228,7 +231,6 @@ void ctgTestBuildDBVgroup(SDBVgInfo **pdbVgroup) { *pdbVgroup = dbVgroup; } - void ctgTestBuildSTableMetaRsp(STableMetaRsp *rspMsg) { strcpy(rspMsg->dbFName, ctgTestDbname); sprintf(rspMsg->tbName, "%s", ctgTestSTablename); @@ -245,19 +247,19 @@ void ctgTestBuildSTableMetaRsp(STableMetaRsp *rspMsg) { rspMsg->vgId = 1; SSchema *s = NULL; - s = &rspMsg->pSchema[0]; + s = &rspMsg->pSchemas[0]; s->type = TSDB_DATA_TYPE_TIMESTAMP; s->colId = 1; s->bytes = 8; strcpy(s->name, "ts"); - s = &rspMsg->pSchema[1]; + s = &rspMsg->pSchemas[1]; s->type = TSDB_DATA_TYPE_INT; s->colId = 2; s->bytes = 4; strcpy(s->name, "col1s"); - s = &rspMsg->pSchema[2]; + s = &rspMsg->pSchemas[2]; s->type = TSDB_DATA_TYPE_BINARY; s->colId = 3; s->bytes = 12 + 1; @@ -266,211 +268,230 @@ void ctgTestBuildSTableMetaRsp(STableMetaRsp *rspMsg) { return; } - void ctgTestRspDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { - SUseDbRsp *rspMsg = NULL; // todo - - pRsp->code = 0; - pRsp->contLen = sizeof(SUseDbRsp) + ctgTestVgNum * sizeof(SVgroupInfo); - pRsp->pCont = calloc(1, pRsp->contLen); - rspMsg = (SUseDbRsp *)pRsp->pCont; - strcpy(rspMsg->db, ctgTestDbname); - rspMsg->vgVersion = htonl(ctgTestVgVersion); + SUseDbRsp usedbRsp = {0}; + strcpy(usedbRsp.db, ctgTestDbname); + usedbRsp.vgVersion = ctgTestVgVersion; ctgTestCurrentVgVersion = ctgTestVgVersion; - rspMsg->vgNum = htonl(ctgTestVgNum); - rspMsg->hashMethod = 0; - rspMsg->uid = htobe64(ctgTestDbId); + usedbRsp.vgNum = ctgTestVgNum; + usedbRsp.hashMethod = 0; + usedbRsp.uid = ctgTestDbId; + usedbRsp.pVgroupInfos = taosArrayInit(usedbRsp.vgNum, sizeof(SVgroupInfo)); - SVgroupInfo *vg = NULL; - uint32_t hashUnit = UINT32_MAX / ctgTestVgNum; + uint32_t hashUnit = UINT32_MAX / ctgTestVgNum; for (int32_t i = 0; i < ctgTestVgNum; ++i) { - vg = &rspMsg->vgroupInfo[i]; - - vg->vgId = htonl(i + 1); - vg->hashBegin = htonl(i * hashUnit); - vg->hashEnd = htonl(hashUnit * (i + 1) - 1); - vg->epset.numOfEps = i % TSDB_MAX_REPLICA + 1; - vg->epset.inUse = i % vg->epset.numOfEps; - for (int32_t n = 0; n < vg->epset.numOfEps; ++n) { - SEp *addr = &vg->epset.eps[n]; - strcpy(addr->fqdn, "a0"); - addr->port = htons(n + 22); + SVgroupInfo vg = {0}; + vg.vgId = i + 1; + vg.hashBegin = i * hashUnit; + vg.hashEnd = hashUnit * (i + 1) - 1; + if (i == ctgTestVgNum - 1) { + vg.hashEnd = htonl(UINT32_MAX); } + + vg.epset.numOfEps = i % TSDB_MAX_REPLICA + 1; + vg.epset.inUse = i % vg.epset.numOfEps; + for (int32_t n = 0; n < vg.epset.numOfEps; ++n) { + SEp *addr = &vg.epset.eps[n]; + strcpy(addr->fqdn, "a0"); + addr->port = n + 22; + } + + taosArrayPush(usedbRsp.pVgroupInfos, &vg); } - vg->hashEnd = htonl(UINT32_MAX); + int32_t contLen = tSerializeSUseDbRsp(NULL, 0, &usedbRsp); + void *pReq = rpcMallocCont(contLen); + tSerializeSUseDbRsp(pReq, contLen, &usedbRsp); - return; + pRsp->code = 0; + pRsp->contLen = contLen; + pRsp->pCont = pReq; } void ctgTestRspTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { - STableMetaRsp *rspMsg = NULL; // todo - - pRsp->code = 0; - pRsp->contLen = sizeof(STableMetaRsp) + (ctgTestColNum + ctgTestTagNum) * sizeof(SSchema); - pRsp->pCont = calloc(1, pRsp->contLen); - rspMsg = (STableMetaRsp *)pRsp->pCont; - strcpy(rspMsg->dbFName, ctgTestDbname); - strcpy(rspMsg->tbName, ctgTestTablename); - rspMsg->numOfTags = 0; - rspMsg->numOfColumns = htonl(ctgTestColNum); - rspMsg->precision = 1; - rspMsg->tableType = TSDB_NORMAL_TABLE; - rspMsg->update = 1; - rspMsg->sversion = htonl(ctgTestSVersion); - rspMsg->tversion = htonl(ctgTestTVersion); - rspMsg->suid = 0; - rspMsg->tuid = htobe64(0x0000000000000001); - rspMsg->vgId = htonl(8); + STableMetaRsp metaRsp = {0}; + strcpy(metaRsp.dbFName, ctgTestDbname); + strcpy(metaRsp.tbName, ctgTestTablename); + metaRsp.numOfTags = 0; + metaRsp.numOfColumns = ctgTestColNum; + metaRsp.precision = 1; + metaRsp.tableType = TSDB_NORMAL_TABLE; + metaRsp.update = 1; + metaRsp.sversion = ctgTestSVersion; + metaRsp.tversion = ctgTestTVersion; + metaRsp.suid = 0; + metaRsp.tuid = 0x0000000000000001; + metaRsp.vgId = 8; + metaRsp.pSchemas = (SSchema *)malloc((metaRsp.numOfTags + metaRsp.numOfColumns) * sizeof(SSchema)); SSchema *s = NULL; - s = &rspMsg->pSchema[0]; + s = &metaRsp.pSchemas[0]; s->type = TSDB_DATA_TYPE_TIMESTAMP; - s->colId = htonl(1); - s->bytes = htonl(8); + s->colId = 1; + s->bytes = 8; strcpy(s->name, "ts"); - s = &rspMsg->pSchema[1]; + s = &metaRsp.pSchemas[1]; s->type = TSDB_DATA_TYPE_INT; - s->colId = htonl(2); - s->bytes = htonl(4); + s->colId = 2; + s->bytes = 4; strcpy(s->name, "col1"); - return; + int32_t contLen = tSerializeSTableMetaRsp(NULL, 0, &metaRsp); + void *pReq = rpcMallocCont(contLen); + tSerializeSTableMetaRsp(pReq, contLen, &metaRsp); + + pRsp->code = 0; + pRsp->contLen = contLen; + pRsp->pCont = pReq; + + tFreeSTableMetaRsp(&metaRsp); } void ctgTestRspCTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { - STableMetaRsp *rspMsg = NULL; // todo - - pRsp->code = 0; - pRsp->contLen = sizeof(STableMetaRsp) + (ctgTestColNum + ctgTestTagNum) * sizeof(SSchema); - pRsp->pCont = calloc(1, pRsp->contLen); - rspMsg = (STableMetaRsp *)pRsp->pCont; - strcpy(rspMsg->dbFName, ctgTestDbname); - strcpy(rspMsg->tbName, ctgTestCTablename); - strcpy(rspMsg->stbName, ctgTestSTablename); - rspMsg->numOfTags = htonl(ctgTestTagNum); - rspMsg->numOfColumns = htonl(ctgTestColNum); - rspMsg->precision = 1; - rspMsg->tableType = TSDB_CHILD_TABLE; - rspMsg->update = 1; - rspMsg->sversion = htonl(ctgTestSVersion); - rspMsg->tversion = htonl(ctgTestTVersion); - rspMsg->suid = htobe64(0x0000000000000002); - rspMsg->tuid = htobe64(0x0000000000000003); - rspMsg->vgId = htonl(9); + STableMetaRsp metaRsp = {0}; + strcpy(metaRsp.dbFName, ctgTestDbname); + strcpy(metaRsp.tbName, ctgTestCTablename); + strcpy(metaRsp.stbName, ctgTestSTablename); + metaRsp.numOfTags = ctgTestTagNum; + metaRsp.numOfColumns = ctgTestColNum; + metaRsp.precision = 1; + metaRsp.tableType = TSDB_CHILD_TABLE; + metaRsp.update = 1; + metaRsp.sversion = ctgTestSVersion; + metaRsp.tversion = ctgTestTVersion; + metaRsp.suid = 0x0000000000000002; + metaRsp.tuid = 0x0000000000000003; + metaRsp.vgId = 9; + metaRsp.pSchemas = (SSchema *)malloc((metaRsp.numOfTags + metaRsp.numOfColumns) * sizeof(SSchema)); SSchema *s = NULL; - s = &rspMsg->pSchema[0]; + s = &metaRsp.pSchemas[0]; s->type = TSDB_DATA_TYPE_TIMESTAMP; - s->colId = htonl(1); - s->bytes = htonl(8); + s->colId = 1; + s->bytes = 8; strcpy(s->name, "ts"); - s = &rspMsg->pSchema[1]; + s = &metaRsp.pSchemas[1]; s->type = TSDB_DATA_TYPE_INT; - s->colId = htonl(2); - s->bytes = htonl(4); + s->colId = 2; + s->bytes = 4; strcpy(s->name, "col1s"); - s = &rspMsg->pSchema[2]; + s = &metaRsp.pSchemas[2]; s->type = TSDB_DATA_TYPE_BINARY; - s->colId = htonl(3); - s->bytes = htonl(12); + s->colId = 3; + s->bytes = 12; strcpy(s->name, "tag1s"); - return; + int32_t contLen = tSerializeSTableMetaRsp(NULL, 0, &metaRsp); + void *pReq = rpcMallocCont(contLen); + tSerializeSTableMetaRsp(pReq, contLen, &metaRsp); + + pRsp->code = 0; + pRsp->contLen = contLen; + pRsp->pCont = pReq; + + tFreeSTableMetaRsp(&metaRsp); } void ctgTestRspSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { - STableMetaRsp *rspMsg = NULL; // todo - - pRsp->code = 0; - pRsp->contLen = sizeof(STableMetaRsp) + (ctgTestColNum + ctgTestTagNum) * sizeof(SSchema); - pRsp->pCont = calloc(1, pRsp->contLen); - rspMsg = (STableMetaRsp *)pRsp->pCont; - strcpy(rspMsg->dbFName, ctgTestDbname); - strcpy(rspMsg->tbName, ctgTestSTablename); - strcpy(rspMsg->stbName, ctgTestSTablename); - rspMsg->numOfTags = htonl(ctgTestTagNum); - rspMsg->numOfColumns = htonl(ctgTestColNum); - rspMsg->precision = 1; - rspMsg->tableType = TSDB_SUPER_TABLE; - rspMsg->update = 1; - rspMsg->sversion = htonl(ctgTestSVersion); - rspMsg->tversion = htonl(ctgTestTVersion); - rspMsg->suid = htobe64(ctgTestSuid); - rspMsg->tuid = htobe64(ctgTestSuid); - rspMsg->vgId = 0; + STableMetaRsp metaRsp = {0}; + strcpy(metaRsp.dbFName, ctgTestDbname); + strcpy(metaRsp.tbName, ctgTestSTablename); + strcpy(metaRsp.stbName, ctgTestSTablename); + metaRsp.numOfTags = ctgTestTagNum; + metaRsp.numOfColumns = ctgTestColNum; + metaRsp.precision = 1; + metaRsp.tableType = TSDB_SUPER_TABLE; + metaRsp.update = 1; + metaRsp.sversion = ctgTestSVersion; + metaRsp.tversion = ctgTestTVersion; + metaRsp.suid = ctgTestSuid; + metaRsp.tuid = ctgTestSuid; + metaRsp.vgId = 0; + metaRsp.pSchemas = (SSchema *)malloc((metaRsp.numOfTags + metaRsp.numOfColumns) * sizeof(SSchema)); SSchema *s = NULL; - s = &rspMsg->pSchema[0]; + s = &metaRsp.pSchemas[0]; s->type = TSDB_DATA_TYPE_TIMESTAMP; - s->colId = htonl(1); - s->bytes = htonl(8); + s->colId = 1; + s->bytes = 8; strcpy(s->name, "ts"); - s = &rspMsg->pSchema[1]; + s = &metaRsp.pSchemas[1]; s->type = TSDB_DATA_TYPE_INT; - s->colId = htonl(2); - s->bytes = htonl(4); + s->colId = 2; + s->bytes = 4; strcpy(s->name, "col1s"); - s = &rspMsg->pSchema[2]; + s = &metaRsp.pSchemas[2]; s->type = TSDB_DATA_TYPE_BINARY; - s->colId = htonl(3); - s->bytes = htonl(12); + s->colId = 3; + s->bytes = 12; strcpy(s->name, "tag1s"); - return; + int32_t contLen = tSerializeSTableMetaRsp(NULL, 0, &metaRsp); + void *pReq = rpcMallocCont(contLen); + tSerializeSTableMetaRsp(pReq, contLen, &metaRsp); + + pRsp->code = 0; + pRsp->contLen = contLen; + pRsp->pCont = pReq; + + tFreeSTableMetaRsp(&metaRsp); } void ctgTestRspMultiSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { - STableMetaRsp *rspMsg = NULL; // todo static int32_t idx = 1; - pRsp->code = 0; - pRsp->contLen = sizeof(STableMetaRsp) + (ctgTestColNum + ctgTestTagNum) * sizeof(SSchema); - pRsp->pCont = calloc(1, pRsp->contLen); - rspMsg = (STableMetaRsp *)pRsp->pCont; - strcpy(rspMsg->dbFName, ctgTestDbname); - sprintf(rspMsg->tbName, "%s_%d", ctgTestSTablename, idx); - sprintf(rspMsg->stbName, "%s_%d", ctgTestSTablename, idx); - rspMsg->numOfTags = htonl(ctgTestTagNum); - rspMsg->numOfColumns = htonl(ctgTestColNum); - rspMsg->precision = 1; - rspMsg->tableType = TSDB_SUPER_TABLE; - rspMsg->update = 1; - rspMsg->sversion = htonl(ctgTestSVersion); - rspMsg->tversion = htonl(ctgTestTVersion); - rspMsg->suid = htobe64(ctgTestSuid + idx); - rspMsg->tuid = htobe64(ctgTestSuid + idx); - rspMsg->vgId = 0; + STableMetaRsp metaRsp = {0}; + strcpy(metaRsp.dbFName, ctgTestDbname); + sprintf(metaRsp.tbName, "%s_%d", ctgTestSTablename, idx); + sprintf(metaRsp.stbName, "%s_%d", ctgTestSTablename, idx); + metaRsp.numOfTags = ctgTestTagNum; + metaRsp.numOfColumns = ctgTestColNum; + metaRsp.precision = 1; + metaRsp.tableType = TSDB_SUPER_TABLE; + metaRsp.update = 1; + metaRsp.sversion = ctgTestSVersion; + metaRsp.tversion = ctgTestTVersion; + metaRsp.suid = ctgTestSuid + idx; + metaRsp.tuid = ctgTestSuid + idx; + metaRsp.vgId = 0; + metaRsp.pSchemas = (SSchema *)malloc((metaRsp.numOfTags + metaRsp.numOfColumns) * sizeof(SSchema)); SSchema *s = NULL; - s = &rspMsg->pSchema[0]; + s = &metaRsp.pSchemas[0]; s->type = TSDB_DATA_TYPE_TIMESTAMP; - s->colId = htonl(1); - s->bytes = htonl(8); + s->colId = 1; + s->bytes = 8; strcpy(s->name, "ts"); - s = &rspMsg->pSchema[1]; + s = &metaRsp.pSchemas[1]; s->type = TSDB_DATA_TYPE_INT; - s->colId = htonl(2); - s->bytes = htonl(4); + s->colId = 2; + s->bytes = 4; strcpy(s->name, "col1s"); - s = &rspMsg->pSchema[2]; + s = &metaRsp.pSchemas[2]; s->type = TSDB_DATA_TYPE_BINARY; - s->colId = htonl(3); - s->bytes = htonl(12); + s->colId = 3; + s->bytes = 12; strcpy(s->name, "tag1s"); ++idx; - return; -} + int32_t contLen = tSerializeSTableMetaRsp(NULL, 0, &metaRsp); + void *pReq = rpcMallocCont(contLen); + tSerializeSTableMetaRsp(pReq, contLen, &metaRsp); + pRsp->code = 0; + pRsp->contLen = contLen; + pRsp->pCont = pReq; + + tFreeSTableMetaRsp(&metaRsp); +} void ctgTestRspByIdx(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { switch (ctgTestRspFunc[ctgTestRspIdx]) { @@ -498,7 +519,6 @@ void ctgTestRspByIdx(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp return; } - void ctgTestRspDbVgroupsAndNormalMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { ctgTestRspDbVgroups(shandle, pEpSet, pMsg, pRsp); @@ -716,7 +736,6 @@ void *ctgTestSetSameDbVgroupThread(void *param) { return NULL; } - void *ctgTestSetDiffDbVgroupThread(void *param) { struct SCatalog *pCtg = (struct SCatalog *)param; int32_t code = 0; @@ -741,7 +760,6 @@ void *ctgTestSetDiffDbVgroupThread(void *param) { return NULL; } - void *ctgTestGetCtableMetaThread(void *param) { struct SCatalog *pCtg = (struct SCatalog *)param; int32_t code = 0; @@ -1008,7 +1026,7 @@ TEST(tableMeta, childTableCase) { } if (stbNum) { - printf("got expired stb,suid:%" PRId64 ",dbFName:%s, stbName:%s\n", stb->suid, stb->dbFName, stb->stbName); + printf("got expired stb,suid:%" PRId64 ",dbFName:%s, stbName:%s\n", stb->suid, stb->dbFName, stb->stbName); free(stb); stb = NULL; } else { @@ -1131,7 +1149,7 @@ TEST(tableMeta, superTableCase) { if (stbNum) { printf("got expired stb,suid:%" PRId64 ",dbFName:%s, stbName:%s\n", stb->suid, stb->dbFName, stb->stbName); - + free(stb); stb = NULL; } else { @@ -1215,7 +1233,7 @@ TEST(tableMeta, rmStbMeta) { ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_NUM), 0); ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_DB_RENT_NUM), 1); ASSERT_EQ(ctgDbgGetClusterCacheNum(pCtg, CTG_DBG_STB_RENT_NUM), 0); - + catalogDestroy(); memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); } @@ -1304,13 +1322,11 @@ TEST(tableMeta, updateStbMeta) { ASSERT_EQ(tableMeta->tableInfo.rowSize, 12); tfree(tableMeta); - + catalogDestroy(); memset(&gCtgMgmt.stat, 0, sizeof(gCtgMgmt.stat)); } - - TEST(tableDistVgroup, normalTable) { struct SCatalog *pCtg = NULL; void *mockPointer = (void *)0x1; @@ -1448,7 +1464,7 @@ TEST(dbVgroup, getSetDbVgroupCase) { void *mockPointer = (void *)0x1; SVgroupInfo vgInfo = {0}; SVgroupInfo *pvgInfo = NULL; - SDBVgInfo *dbVgroup = NULL; + SDBVgInfo *dbVgroup = NULL; SArray *vgList = NULL; ctgTestInitLogFile(); @@ -1634,8 +1650,6 @@ TEST(multiThread, getSetRmDiffDbVgroup) { memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); } - - TEST(multiThread, ctableMeta) { struct SCatalog *pCtg = NULL; void *mockPointer = (void *)0x1; @@ -1687,8 +1701,6 @@ TEST(multiThread, ctableMeta) { memset(&gCtgMgmt, 0, sizeof(gCtgMgmt)); } - - TEST(rentTest, allRent) { struct SCatalog *pCtg = NULL; void *mockPointer = (void *)0x1; @@ -1751,7 +1763,8 @@ TEST(rentTest, allRent) { printf("%d - expired stableNum:%d\n", i, num); if (stable) { for (int32_t n = 0; n < num; ++n) { - printf("suid:%" PRId64 ", dbFName:%s, stbName:%s, sversion:%d, tversion:%d\n", stable[n].suid, stable[n].dbFName, stable[n].stbName, stable[n].sversion, stable[n].tversion); + printf("suid:%" PRId64 ", dbFName:%s, stbName:%s, sversion:%d, tversion:%d\n", stable[n].suid, + stable[n].dbFName, stable[n].stbName, stable[n].sversion, stable[n].tversion); } free(stable); stable = NULL; diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 130e46fc4c..10d884cb3f 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -16,8 +16,8 @@ #define TDENGINE_QUERYUTIL_H #include "common.h" -#include "tpagedfile.h" #include "tbuffer.h" +#include "tpagedbuf.h" #define SET_RES_WINDOW_KEY(_k, _ori, _len, _uid) \ do { \ @@ -126,6 +126,13 @@ static FORCE_INLINE char* getPosInResultPage(struct STaskAttr* pQueryAttr, SFile // return ((char *)page->data) + rowOffset + offset * numOfRows; } +static FORCE_INLINE char* getPosInResultPage_rv(SFilePage* page, int32_t rowOffset, int32_t offset) { + assert(rowOffset >= 0); + + int32_t numOfRows = 1;//(int32_t)getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery); + return ((char *)page->data) + rowOffset + offset * numOfRows; +} + //bool isNullOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type); //bool notNullOperator(SColumnFilterElem *pFilter, const char* minval, const char* maxval, int16_t type); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 41ed1739b9..232b54554f 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -15,35 +15,40 @@ #ifndef TDENGINE_EXECUTORIMPL_H #define TDENGINE_EXECUTORIMPL_H +#ifdef __cplusplus +extern "C" { +#endif + #include "os.h" #include "common.h" +#include "tlosertree.h" #include "ttszip.h" #include "tvariant.h" #include "dataSinkMgt.h" #include "executil.h" +#include "executor.h" #include "planner.h" #include "taosdef.h" #include "tarray.h" #include "tfilter.h" #include "thash.h" #include "tlockfree.h" -#include "tpagedfile.h" -#include "executor.h" +#include "tpagedbuf.h" struct SColumnFilterElem; typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order); #define IS_QUERY_KILLED(_q) ((_q)->code == TSDB_CODE_TSC_QUERY_CANCELLED) -#define Q_STATUS_EQUAL(p, s) (((p) & (s)) != 0u) +#define Q_STATUS_EQUAL(p, s) (((p) & (s)) != 0u) #define QUERY_IS_ASC_QUERY(q) (GET_FORWARD_DIRECTION_FACTOR((q)->order.order) == QUERY_ASC_FORWARD_STEP) -#define GET_TABLEGROUP(q, _index) ((SArray*) taosArrayGetP((q)->tableqinfoGroupInfo.pGroupList, (_index))) +#define GET_TABLEGROUP(q, _index) ((SArray*)taosArrayGetP((q)->tableqinfoGroupInfo.pGroupList, (_index))) -#define GET_NUM_OF_RESULTS(_r) (((_r)->outputBuf) == NULL? 0:((_r)->outputBuf)->info.rows) +#define GET_NUM_OF_RESULTS(_r) (((_r)->outputBuf) == NULL ? 0 : ((_r)->outputBuf)->info.rows) -#define NEEDTO_COMPRESS_QUERY(size) ((size) > tsCompressColData? 1 : 0) +#define NEEDTO_COMPRESS_QUERY(size) ((size) > tsCompressColData ? 1 : 0) enum { // when this task starts to execute, this status will set @@ -62,8 +67,8 @@ enum { }; typedef struct SResultRowCell { - uint64_t groupId; - SResultRow *pRow; + uint64_t groupId; + SResultRow* pRow; } SResultRowCell; /** @@ -80,25 +85,23 @@ typedef struct SColumnFilterElem { int16_t bytes; // column length __filter_func_t fp; SColumnFilterInfo filterInfo; - void *q; + void* q; } SColumnFilterElem; typedef struct SSingleColumnFilterInfo { void* pData; - void* pData2; //used for nchar column + void* pData2; // used for nchar column int32_t numOfFilters; SColumnInfo info; SColumnFilterElem* pFilters; } SSingleColumnFilterInfo; typedef struct STableQueryInfo { - TSKEY lastKey; - int32_t groupIndex; // group id in table list - SVariant tag; - STimeWindow win; // todo remove it later - STSCursor cur; - void* pTable; // for retrieve the page id list - SResultRowInfo resInfo; + TSKEY lastKey; // last check ts + uint64_t uid; // table uid + int32_t groupIndex; // group id in table list +// SVariant tag; + SResultRowInfo resInfo; // result info } STableQueryInfo; typedef enum { @@ -109,11 +112,11 @@ typedef enum { typedef struct { EQueryProfEventType eventType; - int64_t eventTime; + int64_t eventTime; union { - uint8_t operatorType; //for operator event - int32_t abortCode; //for query abort event + uint8_t operatorType; // for operator event + int32_t abortCode; // for query abort event }; } SQueryProfEvent; @@ -124,33 +127,33 @@ typedef struct { } SOperatorProfResult; typedef struct STaskCostInfo { - int64_t created; - int64_t start; - int64_t end; + int64_t created; + int64_t start; + int64_t end; - uint64_t loadStatisTime; - uint64_t loadFileBlockTime; - uint64_t loadDataInCacheTime; - uint64_t loadStatisSize; - uint64_t loadFileBlockSize; - uint64_t loadDataInCacheSize; + uint64_t loadStatisTime; + uint64_t loadFileBlockTime; + uint64_t loadDataInCacheTime; + uint64_t loadStatisSize; + uint64_t loadFileBlockSize; + uint64_t loadDataInCacheSize; - uint64_t loadDataTime; - uint64_t totalRows; - uint64_t totalCheckedRows; - uint32_t totalBlocks; - uint32_t loadBlocks; - uint32_t loadBlockStatis; - uint32_t discardBlocks; - uint64_t elapsedTime; - uint64_t firstStageMergeTime; - uint64_t winInfoSize; - uint64_t tableInfoSize; - uint64_t hashSize; - uint64_t numOfTimeWindows; + uint64_t loadDataTime; + uint64_t totalRows; + uint64_t totalCheckedRows; + uint32_t totalBlocks; + uint32_t loadBlocks; + uint32_t loadBlockStatis; + uint32_t discardBlocks; + uint64_t elapsedTime; + uint64_t firstStageMergeTime; + uint64_t winInfoSize; + uint64_t tableInfoSize; + uint64_t hashSize; + uint64_t numOfTimeWindows; - SArray *queryProfEvents; //SArray - SHashObj *operatorProfResults; //map + SArray* queryProfEvents; // SArray + SHashObj* operatorProfResults; // map } STaskCostInfo; typedef struct { @@ -166,67 +169,67 @@ typedef struct { // The basic query information extracted from the SQueryInfo tree to support the // execution of query in a data node. typedef struct STaskAttr { - SLimit limit; - SLimit slimit; + SLimit limit; + SLimit slimit; // todo comment it - bool stableQuery; // super table query or not - bool topBotQuery; // TODO used bitwise flag - bool groupbyColumn; // denote if this is a groupby normal column query - bool hasTagResults; // if there are tag values in final result or not - bool timeWindowInterpo;// if the time window start/end required interpolation - bool queryBlockDist; // if query data block distribution - bool stabledev; // super table stddev query - bool tsCompQuery; // is tscomp query - bool diffQuery; // is diff query - bool simpleAgg; - bool pointInterpQuery; // point interpolation query - bool needReverseScan; // need reverse scan - bool distinct; // distinct query or not - bool stateWindow; // window State on sub/normal table - bool createFilterOperator; // if filter operator is needed - bool multigroupResult; // multigroup result can exist in one SSDataBlock - int32_t interBufSize; // intermediate buffer sizse + bool stableQuery; // super table query or not + bool topBotQuery; // TODO used bitwise flag + bool groupbyColumn; // denote if this is a groupby normal column query + bool hasTagResults; // if there are tag values in final result or not + bool timeWindowInterpo; // if the time window start/end required interpolation + bool queryBlockDist; // if query data block distribution + bool stabledev; // super table stddev query + bool tsCompQuery; // is tscomp query + bool diffQuery; // is diff query + bool simpleAgg; + bool pointInterpQuery; // point interpolation query + bool needReverseScan; // need reverse scan + bool distinct; // distinct query or not + bool stateWindow; // window State on sub/normal table + bool createFilterOperator; // if filter operator is needed + bool multigroupResult; // multigroup result can exist in one SSDataBlock + int32_t interBufSize; // intermediate buffer sizse - int32_t havingNum; // having expr number + int32_t havingNum; // having expr number - SOrder order; - int16_t numOfCols; - int16_t numOfTags; + SOrder order; + int16_t numOfCols; + int16_t numOfTags; - STimeWindow window; - SInterval interval; - SSessionWindow sw; - int16_t precision; - int16_t numOfOutput; - int16_t fillType; + STimeWindow window; + SInterval interval; + SSessionWindow sw; + int16_t precision; + int16_t numOfOutput; + int16_t fillType; - int32_t srcRowSize; // todo extract struct - int32_t resultRowSize; - int32_t intermediateResultRowSize; // intermediate result row size, in case of top-k query. - int32_t maxTableColumnWidth; - int32_t tagLen; // tag value length of current query - SGroupbyExpr *pGroupbyExpr; + int32_t srcRowSize; // todo extract struct + int32_t resultRowSize; + int32_t intermediateResultRowSize; // intermediate result row size, in case of top-k query. + int32_t maxTableColumnWidth; + int32_t tagLen; // tag value length of current query + SGroupbyExpr* pGroupbyExpr; - SExprInfo* pExpr1; - SExprInfo* pExpr2; - int32_t numOfExpr2; - SExprInfo* pExpr3; - int32_t numOfExpr3; + SExprInfo* pExpr1; + SExprInfo* pExpr2; + int32_t numOfExpr2; + SExprInfo* pExpr3; + int32_t numOfExpr3; - SColumnInfo* tableCols; - SColumnInfo* tagColList; - int32_t numOfFilterCols; - int64_t* fillVal; - SOrderedPrjQueryInfo prjInfo; // limit value for each vgroup, only available in global order projection query. + SColumnInfo* tableCols; + SColumnInfo* tagColList; + int32_t numOfFilterCols; + int64_t* fillVal; + SOrderedPrjQueryInfo prjInfo; // limit value for each vgroup, only available in global order projection query. SSingleColumnFilterInfo* pFilterInfo; -// SFilterInfo *pFilters; - - void* tsdb; - STableGroupInfo tableGroupInfo; // table list SArray - int32_t vgId; - SArray *pUdfInfo; // no need to free + // SFilterInfo *pFilters; + + void* tsdb; + STableGroupInfo tableGroupInfo; // table list SArray + int32_t vgId; + SArray* pUdfInfo; // no need to free } STaskAttr; typedef int32_t (*__optr_prepare_fn_t)(void* param); @@ -236,176 +239,185 @@ typedef void (*__optr_cleanup_fn_t)(void* param, int32_t num); struct SOperatorInfo; typedef struct STaskIdInfo { - uint64_t queryId; // this is also a request id - uint64_t subplanId; - uint64_t templateId; - char *str; + uint64_t queryId; // this is also a request id + uint64_t subplanId; + uint64_t templateId; + char* str; } STaskIdInfo; typedef struct SExecTaskInfo { STaskIdInfo id; - char *content; + char* content; uint32_t status; STimeWindow window; STaskCostInfo cost; - int64_t owner; // if it is in execution + int64_t owner; // if it is in execution int32_t code; - uint64_t totalRows; // total number of rows + uint64_t totalRows; // total number of rows STableGroupInfo tableqinfoGroupInfo; // this is a group array list, including SArray structure - char *sql; // query sql string - jmp_buf env; // - struct SOperatorInfo *pRoot; + char* sql; // query sql string + jmp_buf env; // + struct SOperatorInfo* pRoot; } SExecTaskInfo; typedef struct STaskRuntimeEnv { - jmp_buf env; - STaskAttr* pQueryAttr; - uint32_t status; // query status - void* qinfo; - uint8_t scanFlag; // denotes reversed scan of data or not - void* pTsdbReadHandle; + jmp_buf env; + STaskAttr* pQueryAttr; + uint32_t status; // query status + void* qinfo; + uint8_t scanFlag; // denotes reversed scan of data or not + void* pTsdbReadHandle; - int32_t prevGroupId; // previous executed group id - bool enableGroupData; - SDiskbasedResultBuf* pResultBuf; // query result buffer based on blocked-wised disk file - SHashObj* pResultRowHashTable; // quick locate the window object for each result - SHashObj* pResultRowListSet; // used to check if current ResultRowInfo has ResultRow object or not - SArray* pResultRowArrayList; // The array list that contains the Result rows - char* keyBuf; // window key buffer - SResultRowPool* pool; // The window result objects pool, all the resultRow Objects are allocated and managed by this object. - char** prevRow; + int32_t prevGroupId; // previous executed group id + bool enableGroupData; + SDiskbasedBuf* pResultBuf; // query result buffer based on blocked-wised disk file + SHashObj* pResultRowHashTable; // quick locate the window object for each result + SHashObj* pResultRowListSet; // used to check if current ResultRowInfo has ResultRow object or not + SArray* pResultRowArrayList; // The array list that contains the Result rows + char* keyBuf; // window key buffer + // The window result objects pool, all the resultRow Objects are allocated and managed by this object. + char** prevRow; + SResultRowPool* pool; - SArray* prevResult; // intermediate result, SArray - STSBuf* pTsBuf; // timestamp filter list - STSCursor cur; + SArray* prevResult; // intermediate result, SArray + STSBuf* pTsBuf; // timestamp filter list + STSCursor cur; - char* tagVal; // tag value of current data block - struct SScalarFunctionSupport * scalarSup; + char* tagVal; // tag value of current data block + struct SScalarFunctionSupport* scalarSup; - SSDataBlock *outputBuf; - STableGroupInfo tableqinfoGroupInfo; // this is a group array list, including SArray structure - struct SOperatorInfo *proot; + SSDataBlock* outputBuf; + STableGroupInfo tableqinfoGroupInfo; // this is a group array list, including SArray structure + struct SOperatorInfo* proot; SGroupResInfo groupResInfo; - int64_t currentOffset; // dynamic offset value + int64_t currentOffset; // dynamic offset value - STableQueryInfo *current; - SRspResultInfo resultInfo; - SHashObj *pTableRetrieveTsMap; - struct SUdfInfo *pUdfInfo; + STableQueryInfo* current; + SRspResultInfo resultInfo; + SHashObj* pTableRetrieveTsMap; + struct SUdfInfo* pUdfInfo; } STaskRuntimeEnv; enum { - OP_IN_EXECUTING = 1, - OP_RES_TO_RETURN = 2, - OP_EXEC_DONE = 3, + OP_IN_EXECUTING = 1, + OP_RES_TO_RETURN = 2, + OP_EXEC_DONE = 3, }; typedef struct SOperatorInfo { - uint8_t operatorType; - bool blockingOptr; // block operator or not - uint8_t status; // denote if current operator is completed - int32_t numOfOutput; // number of columns of the current operator results - char *name; // name, used to show the query execution plan - void *info; // extension attribution - SExprInfo *pExpr; - STaskRuntimeEnv *pRuntimeEnv; // todo remove it - SExecTaskInfo *pTaskInfo; + uint8_t operatorType; + bool blockingOptr; // block operator or not + uint8_t status; // denote if current operator is completed + int32_t numOfOutput; // number of columns of the current operator results + char* name; // name, used to show the query execution plan + void* info; // extension attribution + SExprInfo* pExpr; + STaskRuntimeEnv* pRuntimeEnv; // todo remove it + SExecTaskInfo* pTaskInfo; - struct SOperatorInfo **pDownstream; // downstram pointer list - int32_t numOfDownstream; // number of downstream. The value is always ONE expect for join operator - __optr_prepare_fn_t prepareFn; - __operator_fn_t exec; - __optr_cleanup_fn_t cleanupFn; + struct SOperatorInfo** pDownstream; // downstram pointer list + int32_t numOfDownstream; // number of downstream. The value is always ONE expect for join operator + __optr_prepare_fn_t prepareFn; + __operator_fn_t exec; + __optr_cleanup_fn_t cleanupFn; } SOperatorInfo; -enum { - QUERY_RESULT_NOT_READY = 1, - QUERY_RESULT_READY = 2, -}; - typedef struct { int32_t numOfTags; int32_t numOfCols; - SColumnInfo *colList; + SColumnInfo* colList; } SQueriedTableInfo; typedef struct SQInfo { - void* signature; - uint64_t qId; - int32_t code; // error code to returned to client - int64_t owner; // if it is in execution + void* signature; + uint64_t qId; + int32_t code; // error code to returned to client + int64_t owner; // if it is in execution STaskRuntimeEnv runtimeEnv; STaskAttr query; - void* pBuf; // allocated buffer for STableQueryInfo, sizeof(STableQueryInfo)*numOfTables; + void* pBuf; // allocated buffer for STableQueryInfo, sizeof(STableQueryInfo)*numOfTables; - pthread_mutex_t lock; // used to synchronize the rsp/query threads - tsem_t ready; - int32_t dataReady; // denote if query result is ready or not - void* rspContext; // response context - int64_t startExecTs; // start to exec timestamp - char* sql; // query sql string - STaskCostInfo summary; + pthread_mutex_t lock; // used to synchronize the rsp/query threads + tsem_t ready; + int32_t dataReady; // denote if query result is ready or not + void* rspContext; // response context + int64_t startExecTs; // start to exec timestamp + char* sql; // query sql string + STaskCostInfo summary; } SQInfo; typedef struct STaskParam { - char *sql; - char *tagCond; - char *colCond; - char *tbnameCond; - char *prevResult; - SArray *pTableIdList; - SSqlExpr **pExpr; - SSqlExpr **pSecExpr; - SExprInfo *pExprs; - SExprInfo *pSecExprs; + char* sql; + char* tagCond; + char* colCond; + char* tbnameCond; + char* prevResult; + SArray* pTableIdList; + SSqlExpr** pExpr; + SSqlExpr** pSecExpr; + SExprInfo* pExprs; + SExprInfo* pSecExprs; - SFilterInfo *pFilters; + SFilterInfo* pFilters; - SColIndex *pGroupColIndex; - SColumnInfo *pTagColumnInfo; - SGroupbyExpr *pGroupbyExpr; + SColIndex* pGroupColIndex; + SColumnInfo* pTagColumnInfo; + SGroupbyExpr* pGroupbyExpr; int32_t tableScanOperator; - SArray *pOperator; - struct SUdfInfo *pUdfInfo; + SArray* pOperator; + struct SUdfInfo* pUdfInfo; } STaskParam; -typedef struct SExchangeInfo { - SArray *pSources; - tsem_t ready; - void *pTransporter; - SRetrieveTableRsp *pRsp; - SSDataBlock *pResult; - int32_t current; - uint64_t rowsOfCurrentSource; +enum { + DATA_NOT_READY = 0x1, + DATA_READY = 0x2, + DATA_EXHAUSTED = 0x3, +}; - uint64_t totalSize; // total load bytes from remote - uint64_t totalRows; // total number of rows - uint64_t totalElapsed;// total elapsed time +typedef struct SSourceDataInfo { + struct SExchangeInfo *pEx; + int32_t index; + SRetrieveTableRsp *pRsp; + uint64_t totalRows; + int32_t status; +} SSourceDataInfo; + +typedef struct SExchangeInfo { + SArray* pSources; + SArray* pSourceDataInfo; + tsem_t ready; + void* pTransporter; + SSDataBlock* pResult; + bool seqLoadData; // sequential load data or not, false by default + int32_t current; + uint64_t totalSize; // total load bytes from remote + uint64_t totalRows; // total number of rows + uint64_t totalElapsed; // total elapsed time } SExchangeInfo; typedef struct STableScanInfo { - void *pTsdbReadHandle; - int32_t numOfBlocks; // extract basic running information. - int32_t numOfSkipped; - int32_t numOfBlockStatis; - int64_t numOfRows; - - int32_t order; // scan order - int32_t times; // repeat counts - int32_t current; - int32_t reverseTimes; // 0 by default + void* pTsdbReadHandle; + int32_t numOfBlocks; // extract basic running information. + int32_t numOfSkipped; + int32_t numOfBlockStatis; + int64_t numOfRows; - SqlFunctionCtx *pCtx; // next operator query context - SResultRowInfo *pResultRowInfo; - int32_t *rowCellInfoOffset; - SExprInfo *pExpr; + int32_t order; // scan order + int32_t times; // repeat counts + int32_t current; + int32_t reverseTimes; // 0 by default + + SqlFunctionCtx* pCtx; // next operator query context + SResultRowInfo* pResultRowInfo; + int32_t* rowCellInfoOffset; + SExprInfo* pExpr; SSDataBlock block; int32_t numOfOutput; int64_t elapsedTime; int32_t prevGroupId; // previous table group id - int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan + int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan } STableScanInfo; typedef struct STagScanInfo { @@ -416,32 +428,36 @@ typedef struct STagScanInfo { } STagScanInfo; typedef struct SStreamBlockScanInfo { - SSDataBlock *pRes; // result SSDataBlock - SColumnInfo *pCols; // the output column info - uint64_t numOfRows; // total scanned rows - uint64_t numOfExec; // execution times - void *readerHandle;// stream block reader handle + SSDataBlock* pRes; // result SSDataBlock + SColumnInfo* pCols; // the output column info + uint64_t numOfRows; // total scanned rows + uint64_t numOfExec; // execution times + void* readerHandle; // stream block reader handle } SStreamBlockScanInfo; typedef struct SOptrBasicInfo { - SResultRowInfo resultRowInfo; - int32_t *rowCellInfoOffset; // offset value for each row result cell info - SqlFunctionCtx *pCtx; - SSDataBlock *pRes; + SResultRowInfo resultRowInfo; + int32_t* rowCellInfoOffset; // offset value for each row result cell info + SqlFunctionCtx* pCtx; + SSDataBlock* pRes; + uint32_t resRowSize; + int32_t capacity; } SOptrBasicInfo; typedef struct SOptrBasicInfo STableIntervalOperatorInfo; typedef struct SAggOperatorInfo { - SOptrBasicInfo binfo; - uint32_t seed; - SDiskbasedResultBuf *pResultBuf; // query result buffer based on blocked-wised disk file - SHashObj* pResultRowHashTable; // quick locate the window object for each result - SHashObj* pResultRowListSet; // used to check if current ResultRowInfo has ResultRow object or not - SArray* pResultRowArrayList; // The array list that contains the Result rows - char* keyBuf; // window key buffer - SResultRowPool* pool; // The window result objects pool, all the resultRow Objects are allocated and managed by this object. - STableQueryInfo *current; + SOptrBasicInfo binfo; + SDiskbasedBuf *pResultBuf; // query result buffer based on blocked-wised disk file + SHashObj* pResultRowHashTable; // quick locate the window object for each result + SHashObj* pResultRowListSet; // used to check if current ResultRowInfo has ResultRow object or not + SArray* pResultRowArrayList; // The array list that contains the Result rows + char* keyBuf; // window key buffer + SResultRowPool *pool; // The window result objects pool, all the resultRow Objects are allocated and managed by this object. + STableQueryInfo *current; + uint32_t groupId; + SGroupResInfo groupResInfo; + STableQueryInfo *pTableQueryInfo; } SAggOperatorInfo; typedef struct SProjectOperatorInfo { @@ -449,52 +465,52 @@ typedef struct SProjectOperatorInfo { int32_t bufCapacity; uint32_t seed; - SSDataBlock *existDataBlock; + SSDataBlock* existDataBlock; } SProjectOperatorInfo; typedef struct SLimitOperatorInfo { - int64_t limit; - int64_t total; + int64_t limit; + int64_t total; } SLimitOperatorInfo; typedef struct SSLimitOperatorInfo { - int64_t groupTotal; - int64_t currentGroupOffset; + int64_t groupTotal; + int64_t currentGroupOffset; - int64_t rowsTotal; - int64_t currentOffset; - SLimit limit; - SLimit slimit; + int64_t rowsTotal; + int64_t currentOffset; + SLimit limit; + SLimit slimit; - char **prevRow; - SArray *orderColumnList; + char** prevRow; + SArray* orderColumnList; bool hasPrev; bool ignoreCurrentGroup; bool multigroupResult; - SSDataBlock *pRes; // result buffer - SSDataBlock *pPrevBlock; + SSDataBlock* pRes; // result buffer + SSDataBlock* pPrevBlock; int64_t capacity; int64_t threshold; } SSLimitOperatorInfo; typedef struct SFilterOperatorInfo { - SSingleColumnFilterInfo *pFilterInfo; - int32_t numOfFilterCols; + SSingleColumnFilterInfo* pFilterInfo; + int32_t numOfFilterCols; } SFilterOperatorInfo; typedef struct SFillOperatorInfo { - struct SFillInfo *pFillInfo; - SSDataBlock *pRes; - int64_t totalInputRows; - void **p; - SSDataBlock *existNewGroupBlock; - bool multigroupResult; + struct SFillInfo* pFillInfo; + SSDataBlock* pRes; + int64_t totalInputRows; + void** p; + SSDataBlock* existNewGroupBlock; + bool multigroupResult; } SFillOperatorInfo; typedef struct SGroupbyOperatorInfo { SOptrBasicInfo binfo; int32_t colIndex; - char *prevData; // previous group by value + char* prevData; // previous group by value } SGroupbyOperatorInfo; typedef struct SSWindowOperatorInfo { @@ -503,16 +519,16 @@ typedef struct SSWindowOperatorInfo { TSKEY prevTs; // previous timestamp int32_t numOfRows; // number of rows int32_t start; // start row index - bool reptScan; // next round scan + bool reptScan; // next round scan } SSWindowOperatorInfo; typedef struct SStateWindowOperatorInfo { SOptrBasicInfo binfo; STimeWindow curWindow; // current time window int32_t numOfRows; // number of rows - int32_t colIndex; // start row index + int32_t colIndex; // start row index int32_t start; - char* prevData; // previous data + char* prevData; // previous data bool reptScan; } SStateWindowOperatorInfo; @@ -520,147 +536,186 @@ typedef struct SDistinctDataInfo { int32_t index; int32_t type; int32_t bytes; -} SDistinctDataInfo; +} SDistinctDataInfo; typedef struct SDistinctOperatorInfo { - SHashObj *pSet; - SSDataBlock *pRes; - bool recordNullVal; //has already record the null value, no need to try again - int64_t threshold; - int64_t outputCapacity; - int32_t totalBytes; - char* buf; - SArray* pDistinctDataInfo; + SHashObj* pSet; + SSDataBlock* pRes; + bool recordNullVal; // has already record the null value, no need to try again + int64_t threshold; + int64_t outputCapacity; + int32_t totalBytes; + char* buf; + SArray* pDistinctDataInfo; } SDistinctOperatorInfo; struct SGlobalMerger; typedef struct SMultiwayMergeInfo { - struct SGlobalMerger *pMerge; - SOptrBasicInfo binfo; - int32_t bufCapacity; - int64_t seed; - char **prevRow; - SArray *orderColumnList; - int32_t resultRowFactor; + struct SGlobalMerger* pMerge; + SOptrBasicInfo binfo; + int32_t bufCapacity; + int64_t seed; + char** prevRow; + SArray* orderColumnList; + int32_t resultRowFactor; - bool hasGroupColData; - char **currentGroupColData; - SArray *groupColumnList; - bool hasDataBlockForNewGroup; - SSDataBlock *pExistBlock; + bool hasGroupColData; + char** currentGroupColData; + SArray* groupColumnList; + bool hasDataBlockForNewGroup; + SSDataBlock* pExistBlock; - SArray *udfInfo; - bool hasPrev; - bool multiGroupResults; + SArray* udfInfo; + bool hasPrev; + bool multiGroupResults; } SMultiwayMergeInfo; -// todo support the disk-based sort +typedef struct SMsortComparParam { + struct SExternalMemSource **pSources; + int32_t numOfSources; + SArray *orderInfo; // SArray + bool nullFirst; +} SMsortComparParam; + typedef struct SOrderOperatorInfo { - int32_t colIndex; - int32_t order; - SSDataBlock *pDataBlock; + int32_t sourceId; + uint32_t sortBufSize; // max buffer size for in-memory sort + SSDataBlock *pDataBlock; + bool hasVarCol; // has variable length column, such as binary/varchar/nchar + int32_t numOfCompleted; + SDiskbasedBuf *pSortInternalBuf; + SMultiwayMergeTreeInfo *pMergeTree; + SArray *pSources; // SArray + int32_t bufPageSize; + int32_t numOfRowsInRes; + + SMsortComparParam cmpParam; + + int64_t startTs; // sort start time + uint64_t sortElapsed; // sort elapsed time, time to flush to disk not included. + uint64_t totalSize; // total load bytes from remote + uint64_t totalRows; // total number of rows + uint64_t totalElapsed; // total elapsed time } SOrderOperatorInfo; SOperatorInfo* createExchangeOperatorInfo(const SArray* pSources, const SArray* pSchema, SExecTaskInfo* pTaskInfo); - -SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, int32_t reverseTime, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createTableSeqScanOperator(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv); -SOperatorInfo* createSubmitBlockScanOperatorInfo(void *pSubmitBlockReadHandle, int32_t numOfOutput, SExecTaskInfo* pTaskInfo); - -SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createProjectOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, + int32_t repeatTime, int32_t reverseTime, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv); +SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo); +SOperatorInfo* createMultiTableAggOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo); +SOperatorInfo* createProjectOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, + int32_t numOfOutput); SOperatorInfo* createLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream); -SOperatorInfo* createTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createSWindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createFillOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput, bool multigroupResult); -SOperatorInfo* createGroupbyOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createMultiTableAggOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, + int32_t numOfOutput); +SOperatorInfo* createAllTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, + SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createSWindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, + int32_t numOfOutput); +SOperatorInfo* createFillOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, + int32_t numOfOutput, bool multigroupResult); +SOperatorInfo* createGroupbyOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, + int32_t numOfOutput); + +SOperatorInfo* createMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, + SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createAllMultiTableTimeIntervalOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, + SExprInfo* pExpr, int32_t numOfOutput); SOperatorInfo* createTagScanOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createDistinctOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput); +SOperatorInfo* createDistinctOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, + int32_t numOfOutput); SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv); SOperatorInfo* createMultiwaySortOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SExprInfo* pExpr, int32_t numOfOutput, int32_t numOfRows, void* merger); -SOperatorInfo* createGlobalAggregateOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput, void* param, SArray* pUdfInfo, bool groupResultMixedUp); -SOperatorInfo* createStatewindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput); -SOperatorInfo* createSLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput, void* merger, bool multigroupResult); +SOperatorInfo* createGlobalAggregateOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, + SExprInfo* pExpr, int32_t numOfOutput, void* param, SArray* pUdfInfo, + bool groupResultMixedUp); +SOperatorInfo* createStatewindowOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, + int32_t numOfOutput); +SOperatorInfo* createSLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, + int32_t numOfOutput, void* merger, bool multigroupResult); SOperatorInfo* createFilterOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput, SColumnInfo* pCols, int32_t numOfFilter); -SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pdownstream, int32_t numOfDownstream, SSchema* pSchema, int32_t numOfOutput); -SOperatorInfo* createOrderOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput, SOrder* pOrderVal); +SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pdownstream, int32_t numOfDownstream, SSchema* pSchema, + int32_t numOfOutput); +SOperatorInfo* createOrderOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SArray* pOrderVal); +SOperatorInfo* createMergeSortOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput, + SOrder* pOrderVal); -//SSDataBlock* doGlobalAggregate(void* param, bool* newgroup); -//SSDataBlock* doMultiwayMergeSort(void* param, bool* newgroup); -//SSDataBlock* doSLimit(void* param, bool* newgroup); +// SSDataBlock* doGlobalAggregate(void* param, bool* newgroup); +// SSDataBlock* doMultiwayMergeSort(void* param, bool* newgroup); +// SSDataBlock* doSLimit(void* param, bool* newgroup); -//int32_t doCreateFilterInfo(SColumnInfo* pCols, int32_t numOfCols, int32_t numOfFilterCols, SSingleColumnFilterInfo** pFilterInfo, uint64_t qId); +// int32_t doCreateFilterInfo(SColumnInfo* pCols, int32_t numOfCols, int32_t numOfFilterCols, SSingleColumnFilterInfo** pFilterInfo, uint64_t qId); void doSetFilterColumnInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, SSDataBlock* pBlock); bool doFilterDataBlock(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, int32_t numOfRows, int8_t* p); void doCompactSDataBlock(SSDataBlock* pBlock, int32_t numOfRows, int8_t* p); SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numOfRows); -void* destroyOutputBuf(SSDataBlock* pBlock); void* doDestroyFilterInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols); void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order); -void finalizeQueryResult(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset); -void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity, int32_t numOfInputRows); -void clearOutputBuf(SOptrBasicInfo* pBInfo, int32_t *bufCapacity); +void finalizeQueryResult(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SResultRowInfo* pResultRowInfo, + int32_t* rowCellInfoOffset); +void updateOutputBuf(SOptrBasicInfo* pBInfo, int32_t* bufCapacity, int32_t numOfInputRows); +void clearOutputBuf(SOptrBasicInfo* pBInfo, int32_t* bufCapacity); void copyTsColoum(SSDataBlock* pRes, SqlFunctionCtx* pCtx, int32_t numOfOutput); -void freeParam(STaskParam *param); -int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExprInfo** pExprInfo, - SSqlExpr** pExprMsg, SColumnInfo* pTagCols, int32_t queryType, void* pMsg, struct SUdfInfo* pUdfInfo); +int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExprInfo** pExprInfo, SSqlExpr** pExprMsg, + SColumnInfo* pTagCols, int32_t queryType, void* pMsg, struct SUdfInfo* pUdfInfo); -int32_t createIndirectQueryFuncExprFromMsg(SQueryTableReq *pQueryMsg, int32_t numOfOutput, SExprInfo **pExprInfo, - SSqlExpr **pExpr, SExprInfo *prevExpr, struct SUdfInfo *pUdfInfo); +int32_t createIndirectQueryFuncExprFromMsg(SQueryTableReq* pQueryMsg, int32_t numOfOutput, SExprInfo** pExprInfo, + SSqlExpr** pExpr, SExprInfo* prevExpr, struct SUdfInfo* pUdfInfo); -int32_t createQueryFilter(char *data, uint16_t len, SFilterInfo** pFilters); +int32_t createQueryFilter(char* data, uint16_t len, SFilterInfo** pFilters); -SGroupbyExpr *createGroupbyExprFromMsg(SQueryTableReq *pQueryMsg, SColIndex *pColIndex, int32_t *code); +SGroupbyExpr* createGroupbyExprFromMsg(SQueryTableReq* pQueryMsg, SColIndex* pColIndex, int32_t* code); int32_t initQInfo(STsBufInfo* pTsBufInfo, void* tsdb, void* sourceOptr, SQInfo* pQInfo, STaskParam* param, char* start, int32_t prevResultLen, void* merger); int32_t createFilterInfo(STaskAttr* pQueryAttr, uint64_t qId); -void freeColumnFilterInfo(SColumnFilterInfo* pFilter, int32_t numOfFilters); +void freeColumnFilterInfo(SColumnFilterInfo* pFilter, int32_t numOfFilters); -STableQueryInfo *createTableQueryInfo(STaskAttr* pQueryAttr, void* pTable, bool groupbyColumn, STimeWindow win, void* buf); +STableQueryInfo* createTableQueryInfo(void* buf, bool groupbyColumn, STimeWindow win); STableQueryInfo* createTmpTableQueryInfo(STimeWindow win); -int32_t buildArithmeticExprFromMsg(SExprInfo *pArithExprInfo, void *pQueryMsg); +int32_t buildArithmeticExprFromMsg(SExprInfo* pArithExprInfo, void* pQueryMsg); -bool isTaskKilled(SExecTaskInfo *pTaskInfo); +bool isTaskKilled(SExecTaskInfo* pTaskInfo); int32_t checkForQueryBuf(size_t numOfTables); -bool checkNeedToCompressQueryCol(SQInfo *pQInfo); -void setQueryStatus(STaskRuntimeEnv *pRuntimeEnv, int8_t status); +bool checkNeedToCompressQueryCol(SQInfo* pQInfo); +void setQueryStatus(STaskRuntimeEnv* pRuntimeEnv, int8_t status); bool onlyQueryTags(STaskAttr* pQueryAttr); -//void destroyUdfInfo(struct SUdfInfo* pUdfInfo); +// void destroyUdfInfo(struct SUdfInfo* pUdfInfo); -int32_t doDumpQueryResult(SQInfo *pQInfo, char *data, int8_t compressed, int32_t *compLen); +int32_t doDumpQueryResult(SQInfo* pQInfo, char* data, int8_t compressed, int32_t* compLen); -size_t getResultSize(SQInfo *pQInfo, int64_t *numOfRows); -void setTaskKilled(SExecTaskInfo *pTaskInfo); +size_t getResultSize(SQInfo* pQInfo, int64_t* numOfRows); +void setTaskKilled(SExecTaskInfo* pTaskInfo); void publishOperatorProfEvent(SOperatorInfo* operatorInfo, EQueryProfEventType eventType); -void publishQueryAbortEvent(SExecTaskInfo * pTaskInfo, int32_t code); +void publishQueryAbortEvent(SExecTaskInfo* pTaskInfo, int32_t code); void calculateOperatorProfResults(SQInfo* pQInfo); -void queryCostStatis(SExecTaskInfo *pTaskInfo); +void queryCostStatis(SExecTaskInfo* pTaskInfo); -void doDestroyTask(SExecTaskInfo *pTaskInfo); -void freeQueryAttr(STaskAttr *pQuery); +void doDestroyTask(SExecTaskInfo* pTaskInfo); +void freeQueryAttr(STaskAttr* pQuery); int32_t getMaximumIdleDurationSec(); -void doInvokeUdf(struct SUdfInfo* pUdfInfo, SqlFunctionCtx *pCtx, int32_t idx, int32_t type); -void setTaskStatus(SExecTaskInfo *pTaskInfo, int8_t status); +void doInvokeUdf(struct SUdfInfo* pUdfInfo, SqlFunctionCtx* pCtx, int32_t idx, int32_t type); +void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status); int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId); +#ifdef __cplusplus +} +#endif + #endif // TDENGINE_EXECUTORIMPL_H diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 78093ce080..52ab8493f1 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -141,9 +141,9 @@ void clearResultRow(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResultRow, int16_ return; } - // the result does not put into the SDiskbasedResultBuf, ignore it. + // the result does not put into the SDiskbasedBuf, ignore it. if (pResultRow->pageId >= 0) { - SFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pResultRow->pageId); + SFilePage *page = getBufPage(pRuntimeEnv->pResultBuf, pResultRow->pageId); int16_t offset = 0; for (int32_t i = 0; i < pRuntimeEnv->pQueryAttr->numOfOutput; ++i) { @@ -358,7 +358,6 @@ void initGroupResInfo(SGroupResInfo* pGroupResInfo, SResultRowInfo* pResultInfo) pGroupResInfo->pRows = taosArrayFromList(pResultInfo->pResult, pResultInfo->size, POINTER_BYTES); pGroupResInfo->index = 0; - assert(pGroupResInfo->index <= getNumOfTotalRes(pGroupResInfo)); } @@ -533,7 +532,7 @@ static UNUSED_FUNC int32_t mergeIntoGroupResultImpl(STaskRuntimeEnv *pRuntimeEnv int32_t code = TSDB_CODE_SUCCESS; int32_t *posList = NULL; - SLoserTreeInfo *pTree = NULL; + SMultiwayMergeTreeInfo *pTree = NULL; STableQueryInfo **pTableQueryInfoList = NULL; size_t size = taosArrayGetSize(pTableList); @@ -566,7 +565,7 @@ static UNUSED_FUNC int32_t mergeIntoGroupResultImpl(STaskRuntimeEnv *pRuntimeEnv SCompSupporter cs = {pTableQueryInfoList, posList, pRuntimeEnv->pQueryAttr->order.order}; - int32_t ret = tLoserTreeCreate(&pTree, numOfTables, &cs, tableResultComparFn); + int32_t ret = tMergeTreeCreate(&pTree, numOfTables, &cs, tableResultComparFn); if (ret != TSDB_CODE_SUCCESS) { code = TSDB_CODE_QRY_OUT_OF_MEMORY; goto _end; @@ -576,7 +575,7 @@ static UNUSED_FUNC int32_t mergeIntoGroupResultImpl(STaskRuntimeEnv *pRuntimeEnv int64_t startt = taosGetTimestampMs(); while (1) { - int32_t tableIndex = pTree->pNode[0].index; + int32_t tableIndex = tMergeTreeGetChosenIndex(pTree); SResultRowInfo *pWindowResInfo = &pTableQueryInfoList[tableIndex]->resInfo; SResultRow *pWindowRes = getResultRow(pWindowResInfo, cs.rowIndex[tableIndex]); @@ -612,7 +611,7 @@ static UNUSED_FUNC int32_t mergeIntoGroupResultImpl(STaskRuntimeEnv *pRuntimeEnv } } - tLoserTreeAdjust(pTree, tableIndex + pTree->numOfEntries); + tMergeTreeAdjust(pTree, tMergeTreeGetAdjustIndex(pTree)); } int64_t endt = taosGetTimestampMs(); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 2d3085c82b..f5dc7a82b1 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -12,12 +12,13 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -#include "parser.h" -#include "tq.h" +#include #include "exception.h" #include "os.h" +#include "parser.h" #include "tglobal.h" #include "tmsg.h" +#include "tq.h" #include "ttime.h" #include "executorimpl.h" @@ -214,7 +215,7 @@ static void doSetOperatorCompleted(SOperatorInfo* pOperator) { } } -static int32_t doCopyToSDataBlock(STaskRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo, int32_t orderType, SSDataBlock* pBlock); +static int32_t doCopyToSDataBlock(SDiskbasedBuf *pBuf, SGroupResInfo* pGroupResInfo, int32_t orderType, SSDataBlock* pBlock, int32_t rowCapacity); static int32_t getGroupbyColumnIndex(SGroupbyExpr *pGroupbyExpr, SSDataBlock* pDataBlock); static int32_t setGroupResultOutputBuf(STaskRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *binf, int32_t numOfCols, char *pData, int16_t type, int16_t bytes, int32_t groupIndex); @@ -225,12 +226,10 @@ static void setResultBufSize(STaskAttr* pQueryAttr, SRspResultInfo* pResultInfo) static void setCtxTagForJoin(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, void* pTable); static void setParamForStableStddev(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr); static void setParamForStableStddevByColData(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, int32_t numOfOutput, SExprInfo* pExpr, char* val, int16_t bytes); -static void doSetTableGroupOutputBuf(STaskRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo, - SqlFunctionCtx* pCtx, int32_t* rowCellInfoOffset, int32_t numOfOutput, int32_t tableGroupId); +static void doSetTableGroupOutputBuf(SAggOperatorInfo* pAggInfo, int32_t numOfOutput, int32_t tableGroupId, SExecTaskInfo* pTaskInfo); SArray* getOrderCheckColumns(STaskAttr* pQuery); - typedef struct SRowCompSupporter { STaskRuntimeEnv *pRuntimeEnv; int16_t dataOffset; @@ -244,8 +243,8 @@ static int compareRowData(const void *a, const void *b, const void *userData) { SRowCompSupporter *supporter = (SRowCompSupporter *)userData; STaskRuntimeEnv* pRuntimeEnv = supporter->pRuntimeEnv; - SFilePage *page1 = getResBufPage(pRuntimeEnv->pResultBuf, pRow1->pageId); - SFilePage *page2 = getResBufPage(pRuntimeEnv->pResultBuf, pRow2->pageId); + SFilePage *page1 = getBufPage(pRuntimeEnv->pResultBuf, pRow1->pageId); + SFilePage *page2 = getBufPage(pRuntimeEnv->pResultBuf, pRow2->pageId); int16_t offset = supporter->dataOffset; char *in1 = getPosInResultPage(pRuntimeEnv->pQueryAttr, page1, pRow1->offset, offset); @@ -337,23 +336,6 @@ SSDataBlock* createOutputBuf_rv(SArray* pExprInfo, int32_t numOfRows) { return res; } -void* destroyOutputBuf(SSDataBlock* pBlock) { - if (pBlock == NULL) { - return NULL; - } - - int32_t numOfOutput = pBlock->info.numOfCols; - for(int32_t i = 0; i < numOfOutput; ++i) { - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); - tfree(pColInfoData->pData); - } - - taosArrayDestroy(pBlock->pDataBlock); - tfree(pBlock->pBlockAgg); - tfree(pBlock); - return NULL; -} - static bool isSelectivityWithTagsQuery(SqlFunctionCtx *pCtx, int32_t numOfOutput) { return true; // bool hasTags = false; @@ -709,7 +691,7 @@ static STimeWindow getCurrentActiveTimeWindow(SResultRowInfo * pResultRowInfo, i } // a new buffer page for each table. Needs to opt this design -static int32_t addNewWindowResultBuf(SResultRow *pWindowRes, SDiskbasedResultBuf *pResultBuf, int32_t tid, uint32_t size) { +static int32_t addNewWindowResultBuf(SResultRow *pWindowRes, SDiskbasedBuf *pResultBuf, int32_t tid, uint32_t size) { if (pWindowRes->pageId != -1) { return 0; } @@ -724,12 +706,12 @@ static int32_t addNewWindowResultBuf(SResultRow *pWindowRes, SDiskbasedResultBuf pData = getNewDataBuf(pResultBuf, tid, &pageId); } else { SPageInfo* pi = getLastPageInfo(list); - pData = getResBufPage(pResultBuf, pi->pageId); - pageId = pi->pageId; + pData = getBufPage(pResultBuf, getPageId(pi)); + pageId = getPageId(pi); - if (pData->num + size > pResultBuf->pageSize) { + if (pData->num + size > getBufPageSize(pResultBuf)) { // release current page first, and prepare the next one - releaseResBufPageInfo(pResultBuf, pi); + releaseBufPageInfo(pResultBuf, pi); pData = getNewDataBuf(pResultBuf, tid, &pageId); if (pData != NULL) { assert(pData->num == 0); // number of elements must be 0 for new allocated buffer @@ -764,7 +746,7 @@ static int32_t setResultOutputBufByKey(STaskRuntimeEnv *pRuntimeEnv, SResultRowI bool masterscan, SResultRow **pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset) { assert(win->skey <= win->ekey); - SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; + SDiskbasedBuf *pResultBuf = pRuntimeEnv->pResultBuf; SResultRow *pResultRow = doSetResultOutBufByKey(pRuntimeEnv, pResultRowInfo, tid, (char *)&win->skey, TSDB_KEYSIZE, masterscan, tableGroupId); if (pResultRow == NULL) { @@ -1771,7 +1753,7 @@ static void setResultRowKey(SResultRow* pResultRow, char* pData, int16_t type) { } static int32_t setGroupResultOutputBuf(STaskRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *binfo, int32_t numOfCols, char *pData, int16_t type, int16_t bytes, int32_t groupIndex) { - SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf; + SDiskbasedBuf *pResultBuf = pRuntimeEnv->pResultBuf; int32_t *rowCellInfoOffset = binfo->rowCellInfoOffset; SResultRowInfo *pResultRowInfo = &binfo->resultRowInfo; @@ -2024,7 +2006,7 @@ static SqlFunctionCtx* createSqlFunctionCtx(STaskRuntimeEnv* pRuntimeEnv, SExprI return pFuncCtx; } -static SqlFunctionCtx* createSqlFunctionCtx_rv(SArray* pExprInfo, int32_t** rowCellInfoOffset) { +static SqlFunctionCtx* createSqlFunctionCtx_rv(SArray* pExprInfo, int32_t** rowCellInfoOffset, uint32_t* pRowSize) { size_t numOfOutput = taosArrayGetSize(pExprInfo); SqlFunctionCtx * pFuncCtx = (SqlFunctionCtx *)calloc(numOfOutput, sizeof(SqlFunctionCtx)); @@ -2122,6 +2104,7 @@ static SqlFunctionCtx* createSqlFunctionCtx_rv(SArray* pExprInfo, int32_t** rowC for(int32_t i = 1; i < numOfOutput; ++i) { SExprInfo* pExpr = taosArrayGetP(pExprInfo, i - 1); (*rowCellInfoOffset)[i] = (int32_t)((*rowCellInfoOffset)[i - 1] + sizeof(SResultRowEntryInfo) + pExpr->base.interBytes); + *pRowSize += pExpr->base.resSchema.bytes; } setCtxTagColumnInfo(pFuncCtx, numOfOutput); @@ -2188,174 +2171,6 @@ static int32_t setupQueryRuntimeEnv(STaskRuntimeEnv *pRuntimeEnv, int32_t numOfT // group by normal column, sliding window query, interval query are handled by interval query processor // interval (down sampling operation) - int32_t numOfOperator = (int32_t) taosArrayGetSize(pOperator); - for(int32_t i = 0; i < numOfOperator; ++i) { - int32_t* op = taosArrayGet(pOperator, i); - - switch (*op) { -// case OP_TagScan: { -// pRuntimeEnv->proot = createTagScanOperatorInfo(pRuntimeEnv, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); -// break; -// } -// case OP_MultiTableTimeInterval: { -// pRuntimeEnv->proot = -// createMultiTableTimeIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); -// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->downstream[0]->info, pRuntimeEnv->proot); -// break; -// } -// case OP_AllMultiTableTimeInterval: { -// pRuntimeEnv->proot = -// createAllMultiTableTimeIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); -// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->downstream[0]->info, pRuntimeEnv->proot); -// break; -// } -// case OP_TimeWindow: { -// pRuntimeEnv->proot = -// createTimeIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); -// int32_t opType = pRuntimeEnv->proot->downstream[0]->operatorType; -// if (opType != OP_DummyInput && opType != OP_Join) { -// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->downstream[0]->info, pRuntimeEnv->proot); -// } -// break; -// } -// case OP_AllTimeWindow: { -// pRuntimeEnv->proot = -// createAllTimeIntervalOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); -// int32_t opType = pRuntimeEnv->proot->downstream[0]->operatorType; -// if (opType != OP_DummyInput && opType != OP_Join) { -// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->downstream[0]->info, pRuntimeEnv->proot); -// } -// break; -// } -// case OP_Groupby: { -// pRuntimeEnv->proot = -// createGroupbyOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); -// -// int32_t opType = pRuntimeEnv->proot->downstream[0]->operatorType; -// if (opType != OP_DummyInput) { -// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->downstream[0]->info, pRuntimeEnv->proot); -// } -// break; -// } -// case OP_SessionWindow: { -// pRuntimeEnv->proot = -// createSWindowOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); -// int32_t opType = pRuntimeEnv->proot->downstream[0]->operatorType; -// if (opType != OP_DummyInput) { -// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->downstream[0]->info, pRuntimeEnv->proot); -// } -// break; -// } -// case OP_MultiTableAggregate: { -// pRuntimeEnv->proot = -// createMultiTableAggOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); -// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->downstream[0]->info, pRuntimeEnv->proot); -// break; -// } -// case OP_Aggregate: { -// pRuntimeEnv->proot = -// createAggregateOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); -// -// int32_t opType = pRuntimeEnv->proot->downstream[0]->operatorType; -// if (opType != OP_DummyInput && opType != OP_Join) { -// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->downstream[0]->info, pRuntimeEnv->proot); -// } -// break; -// } -// -// case OP_Project: { // TODO refactor to remove arith operator. -// SOperatorInfo* prev = pRuntimeEnv->proot; -// if (i == 0) { -// pRuntimeEnv->proot = createProjectOperatorInfo(pRuntimeEnv, prev, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); -// if (pRuntimeEnv->proot != NULL && prev->operatorType != OP_DummyInput && prev->operatorType != OP_Join) { // TODO refactor -// setTableScanFilterOperatorInfo(prev->info, pRuntimeEnv->proot); -// } -// } else { -// prev = pRuntimeEnv->proot; -// assert(pQueryAttr->pExpr2 != NULL); -// pRuntimeEnv->proot = createProjectOperatorInfo(pRuntimeEnv, prev, pQueryAttr->pExpr2, pQueryAttr->numOfExpr2); -// } -// break; -// } -// -// case OP_StateWindow: { -// pRuntimeEnv->proot = createStatewindowOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); -// int32_t opType = pRuntimeEnv->proot->downstream[0]->operatorType; -// if (opType != OP_DummyInput) { -// setTableScanFilterOperatorInfo(pRuntimeEnv->proot->downstream[0]->info, pRuntimeEnv->proot); -// } -// break; -// } -// -// case OP_Limit: { -// pRuntimeEnv->proot = createLimitOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot); -// break; -// } -// -// case OP_Filter: { // todo refactor -// int32_t numOfFilterCols = 0; -// if (pQueryAttr->stableQuery) { -// SColumnInfo* pColInfo = -// extractColumnFilterInfo(pQueryAttr->pExpr3, pQueryAttr->numOfExpr3, &numOfFilterCols); -// pRuntimeEnv->proot = createFilterOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr3, -// pQueryAttr->numOfExpr3, pColInfo, numOfFilterCols); -// freeColumnInfo(pColInfo, pQueryAttr->numOfExpr3); -// } else { -// SColumnInfo* pColInfo = -// extractColumnFilterInfo(pQueryAttr->pExpr1, pQueryAttr->numOfOutput, &numOfFilterCols); -// pRuntimeEnv->proot = createFilterOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, -// pQueryAttr->numOfOutput, pColInfo, numOfFilterCols); -// freeColumnInfo(pColInfo, pQueryAttr->numOfOutput); -// } -// -// break; -// } -// -// case OP_Fill: { -// SOperatorInfo* pInfo = pRuntimeEnv->proot; -// pRuntimeEnv->proot = createFillOperatorInfo(pRuntimeEnv, pInfo, pInfo->pExpr, pInfo->numOfOutput, pQueryAttr->multigroupResult); -// break; -// } -// -// case OP_MultiwayMergeSort: { -// pRuntimeEnv->proot = createMultiwaySortOperatorInfo(pRuntimeEnv, pQueryAttr->pExpr1, pQueryAttr->numOfOutput, 4096, merger); -// break; -// } -// -// case OP_GlobalAggregate: { // If fill operator exists, the result rows of different group can not be in the same SSDataBlock. -// bool multigroupResult = pQueryAttr->multigroupResult; -// if (pQueryAttr->multigroupResult) { -// multigroupResult = (pQueryAttr->fillType == TSDB_FILL_NONE); -// } -// -// pRuntimeEnv->proot = createGlobalAggregateOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr3, -// pQueryAttr->numOfExpr3, merger, pQueryAttr->pUdfInfo, multigroupResult); -// break; -// } -// -// case OP_SLimit: { -// int32_t num = pRuntimeEnv->proot->numOfOutput; -// SExprInfo* pExpr = pRuntimeEnv->proot->pExpr; -// pRuntimeEnv->proot = createSLimitOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pExpr, num, merger, pQueryAttr->multigroupResult); -// break; -// } -// -// case OP_Distinct: { -// pRuntimeEnv->proot = createDistinctOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput); -// break; -// } -// -// case OP_Order: { -// pRuntimeEnv->proot = createOrderOperatorInfo(pRuntimeEnv, pRuntimeEnv->proot, pQueryAttr->pExpr1, pQueryAttr->numOfOutput, &pQueryAttr->order); -// break; -// } - - default: { - assert(0); - } - } - } - return TSDB_CODE_SUCCESS; _clean: @@ -2964,7 +2779,7 @@ void filterRowsInDataBlock(STaskRuntimeEnv* pRuntimeEnv, SSingleColumnFilterInfo int8_t *p = calloc(numOfRows, sizeof(int8_t)); bool all = true; - +#if 0 if (pRuntimeEnv->pTsBuf != NULL) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, 0); @@ -2992,6 +2807,7 @@ void filterRowsInDataBlock(STaskRuntimeEnv* pRuntimeEnv, SSingleColumnFilterInfo } else { all = doFilterDataBlock(pFilterInfo, numOfFilterCols, numOfRows, p); } +#endif if (!all) { doCompactSDataBlock(pBlock, numOfRows, p); @@ -3030,7 +2846,7 @@ void filterColRowsInDataBlock(STaskRuntimeEnv* pRuntimeEnv, SSDataBlock* pBlock, } // save the cursor status - pRuntimeEnv->current->cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); +// pRuntimeEnv->current->cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); } else { // all = filterExecute(pRuntimeEnv->pQueryAttr->pFilters, numOfRows, &p, pBlock->pBlockAgg, pRuntimeEnv->pQueryAttr->numOfCols); } @@ -3434,8 +3250,7 @@ void setTagValue(SOperatorInfo* pOperatorInfo, void *pTable, SqlFunctionCtx* pCt } } -void copyToSDataBlock(STaskRuntimeEnv* pRuntimeEnv, int32_t threshold, SSDataBlock* pBlock, int32_t* offset) { - SGroupResInfo* pGroupResInfo = &pRuntimeEnv->groupResInfo; +void copyToSDataBlock(SSDataBlock* pBlock, int32_t* offset, SGroupResInfo* pGroupResInfo, SDiskbasedBuf* pResBuf) { pBlock->info.rows = 0; int32_t code = TSDB_CODE_SUCCESS; @@ -3443,12 +3258,12 @@ void copyToSDataBlock(STaskRuntimeEnv* pRuntimeEnv, int32_t threshold, SSDataBlo // all results in current group have been returned to client, try next group if ((pGroupResInfo->pRows == NULL) || taosArrayGetSize(pGroupResInfo->pRows) == 0) { assert(pGroupResInfo->index == 0); - if ((code = mergeIntoGroupResult(&pRuntimeEnv->groupResInfo, pRuntimeEnv, offset)) != TSDB_CODE_SUCCESS) { +// if ((code = mergeIntoGroupResult(&pGroupResInfo, pRuntimeEnv, offset)) != TSDB_CODE_SUCCESS) { return; - } +// } } - doCopyToSDataBlock(pRuntimeEnv, pGroupResInfo, TSDB_ORDER_ASC, pBlock); +// doCopyToSDataBlock(pResBuf, pGroupResInfo, TSDB_ORDER_ASC, pBlock, ); // current data are all dumped to result buffer, clear it if (!hasRemainDataInCurrentGroup(pGroupResInfo)) { @@ -3459,9 +3274,9 @@ void copyToSDataBlock(STaskRuntimeEnv* pRuntimeEnv, int32_t threshold, SSDataBlo } // enough results in data buffer, return - if (pBlock->info.rows >= threshold) { - break; - } +// if (pBlock->info.rows >= threshold) { +// break; +// } } } @@ -3470,11 +3285,11 @@ static void updateTableQueryInfoForReverseScan(STableQueryInfo *pTableQueryInfo) return; } - TSWAP(pTableQueryInfo->win.skey, pTableQueryInfo->win.ekey, TSKEY); - pTableQueryInfo->lastKey = pTableQueryInfo->win.skey; +// TSWAP(pTableQueryInfo->win.skey, pTableQueryInfo->win.ekey, TSKEY); +// pTableQueryInfo->lastKey = pTableQueryInfo->win.skey; - SWITCH_ORDER(pTableQueryInfo->cur.order); - pTableQueryInfo->cur.vgroupIndex = -1; +// SWITCH_ORDER(pTableQueryInfo->cur.order); +// pTableQueryInfo->cur.vgroupIndex = -1; // set the index to be the end slot of result rows array SResultRowInfo* pResultRowInfo = &pTableQueryInfo->resInfo; @@ -3565,7 +3380,7 @@ void setDefaultOutputBuf(STaskRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *pInfo, in initCtxOutputBuffer(pCtx, pDataBlock->info.numOfCols); } -void setDefaultOutputBuf_rv(SAggOperatorInfo* pAggInfo, int64_t uid, int32_t stage, SExecTaskInfo* pTaskInfo) { +void setDefaultOutputBuf_rv(SAggOperatorInfo* pAggInfo, int32_t stage, SExecTaskInfo* pTaskInfo) { SOptrBasicInfo *pInfo = &pAggInfo->binfo; SqlFunctionCtx* pCtx = pInfo->pCtx; @@ -3574,8 +3389,10 @@ void setDefaultOutputBuf_rv(SAggOperatorInfo* pAggInfo, int64_t uid, int32_t sta SResultRowInfo* pResultRowInfo = &pInfo->resultRowInfo; int64_t tid = 0; + int64_t groupId = 0; + pAggInfo->keyBuf = realloc(pAggInfo->keyBuf, sizeof(tid) + sizeof(int64_t) + POINTER_BYTES); - SResultRow* pRow = doSetResultOutBufByKey_rv(pResultRowInfo, tid, (char *)&tid, sizeof(tid), true, uid, pTaskInfo, false, pAggInfo); + SResultRow* pRow = doSetResultOutBufByKey_rv(pResultRowInfo, tid, (char *)&tid, sizeof(tid), true, groupId, pTaskInfo, false, pAggInfo); for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { SColumnInfoData* pData = taosArrayGet(pDataBlock->pDataBlock, i); @@ -3781,24 +3598,19 @@ static bool hasMainOutput(STaskAttr *pQueryAttr) { return false; } -STableQueryInfo *createTableQueryInfo(STaskAttr* pQueryAttr, void* pTable, bool groupbyColumn, STimeWindow win, void* buf) { +STableQueryInfo *createTableQueryInfo(void* buf, bool groupbyColumn, STimeWindow win) { STableQueryInfo *pTableQueryInfo = buf; - - pTableQueryInfo->win = win; pTableQueryInfo->lastKey = win.skey; - pTableQueryInfo->pTable = pTable; - pTableQueryInfo->cur.vgroupIndex = -1; - // set more initial size of interval/groupby query - if (QUERY_IS_INTERVAL_QUERY(pQueryAttr) || groupbyColumn) { +// if (/*QUERY_IS_INTERVAL_QUERY(pQueryAttr) || */groupbyColumn) { int32_t initialSize = 128; int32_t code = initResultRowInfo(&pTableQueryInfo->resInfo, initialSize, TSDB_DATA_TYPE_INT); if (code != TSDB_CODE_SUCCESS) { return NULL; } - } else { // in other aggregate query, do not initialize the windowResInfo - } +// } else { // in other aggregate query, do not initialize the windowResInfo +// } return pTableQueryInfo; } @@ -3806,12 +3618,9 @@ STableQueryInfo *createTableQueryInfo(STaskAttr* pQueryAttr, void* pTable, bool STableQueryInfo* createTmpTableQueryInfo(STimeWindow win) { STableQueryInfo* pTableQueryInfo = calloc(1, sizeof(STableQueryInfo)); - pTableQueryInfo->win = win; +// pTableQueryInfo->win = win; pTableQueryInfo->lastKey = win.skey; - pTableQueryInfo->pTable = NULL; - pTableQueryInfo->cur.vgroupIndex = -1; - // set more initial size of interval/groupby query int32_t initialSize = 16; int32_t code = initResultRowInfo(&pTableQueryInfo->resInfo, initialSize, TSDB_DATA_TYPE_INT); @@ -3828,14 +3637,14 @@ void destroyTableQueryInfoImpl(STableQueryInfo *pTableQueryInfo) { return; } - taosVariantDestroy(&pTableQueryInfo->tag); +// taosVariantDestroy(&pTableQueryInfo->tag); cleanupResultRowInfo(&pTableQueryInfo->resInfo); } void setResultRowOutputBufInitCtx(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset) { // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group - SFilePage* bufPage = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pageId); + SFilePage* bufPage = getBufPage(pRuntimeEnv->pResultBuf, pResult->pageId); int32_t offset = 0; for (int32_t i = 0; i < numOfOutput; ++i) { @@ -3865,14 +3674,49 @@ void setResultRowOutputBufInitCtx(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pRes } } -void doSetTableGroupOutputBuf(STaskRuntimeEnv* pRuntimeEnv, SResultRowInfo* pResultRowInfo, SqlFunctionCtx* pCtx, - int32_t* rowCellInfoOffset, int32_t numOfOutput, int32_t tableGroupId) { +void setResultRowOutputBufInitCtx_rv(SDiskbasedBuf * pBuf, SResultRow *pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowCellInfoOffset) { + // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group + SFilePage* bufPage = getBufPage(pBuf, pResult->pageId); + + int32_t offset = 0; + for (int32_t i = 0; i < numOfOutput; ++i) { + pCtx[i].resultInfo = getResultCell(pResult, i, rowCellInfoOffset); + + struct SResultRowEntryInfo* pResInfo = pCtx[i].resultInfo; + if (isRowEntryCompleted(pResInfo) && isRowEntryInitialized(pResInfo)) { + offset += pCtx[i].resDataInfo.bytes; + continue; + } + + pCtx[i].pOutput = getPosInResultPage_rv(bufPage, pResult->offset, offset); + offset += pCtx[i].resDataInfo.bytes; + + int32_t functionId = pCtx[i].functionId; + if (functionId < 0) { + continue; + } + + if (functionId == FUNCTION_TOP || functionId == FUNCTION_BOTTOM || functionId == FUNCTION_DIFF) { + if (i > 0) pCtx[i].ptsOutputBuf = pCtx[i - 1].pOutput; + } + + // if (!pResInfo->initialized) { + // aAggs[functionId].init(&pCtx[i], pResInfo); + // } + } +} + +void doSetTableGroupOutputBuf(SAggOperatorInfo* pAggInfo, int32_t numOfOutput, int32_t tableGroupId, SExecTaskInfo* pTaskInfo) { // for simple group by query without interval, all the tables belong to one group result. int64_t uid = 0; int64_t tid = 0; + SResultRowInfo* pResultRowInfo = &pAggInfo->binfo.resultRowInfo; + SqlFunctionCtx* pCtx = pAggInfo->binfo.pCtx; + int32_t* rowCellInfoOffset = pAggInfo->binfo.rowCellInfoOffset; + SResultRow* pResultRow = - doSetResultOutBufByKey(pRuntimeEnv, pResultRowInfo, tid, (char*)&tableGroupId, sizeof(tableGroupId), true, uid); + doSetResultOutBufByKey_rv(pResultRowInfo, tid, (char*)&tableGroupId, sizeof(tableGroupId), true, uid, pTaskInfo, false, pAggInfo); assert (pResultRow != NULL); /* @@ -3880,35 +3724,32 @@ void doSetTableGroupOutputBuf(STaskRuntimeEnv* pRuntimeEnv, SResultRowInfo* pRes * all group belong to one result set, and each group result has different group id so set the id to be one */ if (pResultRow->pageId == -1) { - int32_t ret = addNewWindowResultBuf(pResultRow, pRuntimeEnv->pResultBuf, tableGroupId, pRuntimeEnv->pQueryAttr->resultRowSize); + int32_t ret = addNewWindowResultBuf(pResultRow, pAggInfo->pResultBuf, tableGroupId, pAggInfo->binfo.resRowSize); if (ret != TSDB_CODE_SUCCESS) { return; } } - setResultRowOutputBufInitCtx(pRuntimeEnv, pResultRow, pCtx, numOfOutput, rowCellInfoOffset); + setResultRowOutputBufInitCtx_rv(pAggInfo->pResultBuf, pResultRow, pCtx, numOfOutput, rowCellInfoOffset); } -void setExecutionContext(STaskRuntimeEnv* pRuntimeEnv, SOptrBasicInfo* pInfo, int32_t numOfOutput, int32_t tableGroupId, - TSKEY nextKey) { - STableQueryInfo *pTableQueryInfo = pRuntimeEnv->current; - +void setExecutionContext(int32_t numOfOutput, int32_t tableGroupId, TSKEY nextKey, SExecTaskInfo* pTaskInfo, STableQueryInfo *pTableQueryInfo, SAggOperatorInfo* pAggInfo) { // lastKey needs to be updated pTableQueryInfo->lastKey = nextKey; - if (pRuntimeEnv->prevGroupId != INT32_MIN && pRuntimeEnv->prevGroupId == tableGroupId) { + if (pAggInfo->groupId != INT32_MIN && pAggInfo->groupId == tableGroupId) { return; } - doSetTableGroupOutputBuf(pRuntimeEnv, &pInfo->resultRowInfo, pInfo->pCtx, pInfo->rowCellInfoOffset, numOfOutput, tableGroupId); + doSetTableGroupOutputBuf(pAggInfo, numOfOutput, tableGroupId, pTaskInfo); // record the current active group id - pRuntimeEnv->prevGroupId = tableGroupId; + pAggInfo->groupId = tableGroupId; } void setResultOutputBuf(STaskRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SqlFunctionCtx* pCtx, int32_t numOfCols, int32_t* rowCellInfoOffset) { // Note: pResult->pos[i]->num == 0, there is only fixed number of results for each group - SFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pResult->pageId); + SFilePage *page = getBufPage(pRuntimeEnv->pResultBuf, pResult->pageId); int16_t offset = 0; for (int32_t i = 0; i < numOfCols; ++i) { @@ -3957,7 +3798,7 @@ int32_t setTimestampListJoinInfo(STaskRuntimeEnv* pRuntimeEnv, SVariant* pTag, S STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; assert(pRuntimeEnv->pTsBuf != NULL); - +#if 0 // both the master and supplement scan needs to set the correct ts comp start position if (pTableQueryInfo->cur.vgroupIndex == -1) { taosVariantAssign(&pTableQueryInfo->tag, pTag); @@ -3991,7 +3832,7 @@ int32_t setTimestampListJoinInfo(STaskRuntimeEnv* pRuntimeEnv, SVariant* pTag, S //qDebug("QInfo:0x%"PRIx64" find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_TASKID(pRuntimeEnv), pTag->i, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); } } - +#endif return 0; } @@ -4081,7 +3922,7 @@ void setIntervalQueryRange(STaskRuntimeEnv *pRuntimeEnv, TSKEY key) { return; } - pTableQueryInfo->win.skey = key; +// pTableQueryInfo->win.skey = key; STimeWindow win = {.skey = key, .ekey = pQueryAttr->window.ekey}; /** @@ -4104,7 +3945,7 @@ void setIntervalQueryRange(STaskRuntimeEnv *pRuntimeEnv, TSKEY key) { // pResultRowInfo->prevSKey = w.skey; // } - pTableQueryInfo->lastKey = pTableQueryInfo->win.skey; +// pTableQueryInfo->lastKey = pTableQueryInfo->win.skey; } /** @@ -4117,9 +3958,7 @@ void setIntervalQueryRange(STaskRuntimeEnv *pRuntimeEnv, TSKEY key) { * @param result */ -static int32_t doCopyToSDataBlock(STaskRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo, int32_t orderType, SSDataBlock* pBlock) { - STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; - +static int32_t doCopyToSDataBlock(SDiskbasedBuf *pBuf, SGroupResInfo* pGroupResInfo, int32_t orderType, SSDataBlock* pBlock, int32_t rowCapacity) { int32_t numOfRows = getNumOfTotalRes(pGroupResInfo); int32_t numOfResult = pBlock->info.rows; // there are already exists result rows @@ -4145,13 +3984,13 @@ static int32_t doCopyToSDataBlock(STaskRuntimeEnv* pRuntimeEnv, SGroupResInfo* p } int32_t numOfRowsToCopy = pRow->numOfRows; - if (numOfResult + numOfRowsToCopy >= pRuntimeEnv->resultInfo.capacity) { + if (numOfResult + numOfRowsToCopy >= rowCapacity) { break; } pGroupResInfo->index += 1; - SFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pRow->pageId); + SFilePage *page = getBufPage(pBuf, pRow->pageId); int32_t offset = 0; for (int32_t j = 0; j < pBlock->info.numOfCols; ++j) { @@ -4159,14 +3998,14 @@ static int32_t doCopyToSDataBlock(STaskRuntimeEnv* pRuntimeEnv, SGroupResInfo* p int32_t bytes = pColInfoData->info.bytes; char *out = pColInfoData->pData + numOfResult * bytes; - char *in = getPosInResultPage(pQueryAttr, page, pRow->offset, offset); + char *in = getPosInResultPage_rv(page, pRow->offset, offset); memcpy(out, in, bytes * numOfRowsToCopy); offset += bytes; } numOfResult += numOfRowsToCopy; - if (numOfResult == pRuntimeEnv->resultInfo.capacity) { // output buffer is full + if (numOfResult == rowCapacity) { // output buffer is full break; } } @@ -4176,7 +4015,7 @@ static int32_t doCopyToSDataBlock(STaskRuntimeEnv* pRuntimeEnv, SGroupResInfo* p return 0; } -static void toSSDataBlock(SGroupResInfo *pGroupResInfo, STaskRuntimeEnv* pRuntimeEnv, SSDataBlock* pBlock) { +static void toSDatablock(SGroupResInfo *pGroupResInfo, SDiskbasedBuf* pBuf, SSDataBlock* pBlock, int32_t rowCapacity) { assert(pGroupResInfo->currentGroup <= pGroupResInfo->totalGroup); pBlock->info.rows = 0; @@ -4184,29 +4023,19 @@ static void toSSDataBlock(SGroupResInfo *pGroupResInfo, STaskRuntimeEnv* pRuntim return; } - STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; int32_t orderType = TSDB_ORDER_ASC;//(pQueryAttr->pGroupbyExpr != NULL) ? pQueryAttr->pGroupbyExpr->orderType : TSDB_ORDER_ASC; - doCopyToSDataBlock(pRuntimeEnv, pGroupResInfo, orderType, pBlock); + doCopyToSDataBlock(pBuf, pGroupResInfo, orderType, pBlock, rowCapacity); - // refactor : extract method - SColumnInfoData* pInfoData = taosArrayGet(pBlock->pDataBlock, 0); - - //add condition (pBlock->info.rows >= 1) just to runtime happy - if (pInfoData->info.type == TSDB_DATA_TYPE_TIMESTAMP && pBlock->info.rows >= 1) { - STimeWindow* w = &pBlock->info.window; - w->skey = *(int64_t*)pInfoData->pData; - w->ekey = *(int64_t*)(((char*)pInfoData->pData) + TSDB_KEYSIZE * (pBlock->info.rows - 1)); - } + // add condition (pBlock->info.rows >= 1) just to runtime happy + blockDataUpdateTsWindow(pBlock); } -static void updateNumOfRowsInResultRows(STaskRuntimeEnv* pRuntimeEnv, SqlFunctionCtx* pCtx, int32_t numOfOutput, +static void updateNumOfRowsInResultRows(SqlFunctionCtx* pCtx, int32_t numOfOutput, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset) { - STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - // update the number of result for each, only update the number of rows for the corresponding window result. - if (QUERY_IS_INTERVAL_QUERY(pQueryAttr)) { - return; - } +// if (QUERY_IS_INTERVAL_QUERY(pQueryAttr)) { +// return; +// } for (int32_t i = 0; i < pResultRowInfo->size; ++i) { SResultRow *pResult = pResultRowInfo->pResult[i]; @@ -4217,8 +4046,8 @@ static void updateNumOfRowsInResultRows(STaskRuntimeEnv* pRuntimeEnv, SqlFunctio continue; } -// SResultRowEntryInfo* pCell = getResultCell(pResult, j, rowCellInfoOffset); -// pResult->numOfRows = (uint16_t)(TMAX(pResult->numOfRows, pCell->numOfRes)); + SResultRowEntryInfo* pCell = getResultCell(pResult, j, rowCellInfoOffset); + pResult->numOfRows = (uint16_t)(TMAX(pResult->numOfRows, pCell->numOfRes)); } } } @@ -4799,7 +4628,7 @@ int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr getIntermediateBufInfo(pRuntimeEnv, &ps, &pQueryAttr->intermediateResultRowSize); int32_t TENMB = 1024*1024*10; - int32_t code = createDiskbasedResultBuffer(&pRuntimeEnv->pResultBuf, ps, TENMB, pQInfo->qId, tsTempDir); + int32_t code = createDiskbasedBuffer(&pRuntimeEnv->pResultBuf, ps, TENMB, pQInfo->qId, tsTempDir); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -4829,7 +4658,8 @@ int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr } static void doTableQueryInfoTimeWindowCheck(SExecTaskInfo* pTaskInfo, STableQueryInfo* pTableQueryInfo, int32_t order) { - if (order == TSDB_ORDER_ASC) { +#if 0 + if (order == TSDB_ORDER_ASC) { assert( (pTableQueryInfo->win.skey <= pTableQueryInfo->win.ekey) && (pTableQueryInfo->lastKey >= pTaskInfo->window.skey) && @@ -4840,6 +4670,8 @@ static void doTableQueryInfoTimeWindowCheck(SExecTaskInfo* pTaskInfo, STableQuer (pTableQueryInfo->lastKey <= pTaskInfo->window.skey) && (pTableQueryInfo->win.skey <= pTaskInfo->window.skey && pTableQueryInfo->win.ekey >= pTaskInfo->window.ekey)); } +#endif + } //STsdbQueryCond createTsdbQueryCond(STaskAttr* pQueryAttr, STimeWindow* win) { @@ -5102,14 +4934,16 @@ static SSDataBlock* doStreamBlockScan(void* param, bool* newgroup) { } int32_t loadRemoteDataCallback(void* param, const SDataBuf* pMsg, int32_t code) { - SExchangeInfo* pEx = (SExchangeInfo*) param; - pEx->pRsp = pMsg->pData; + SSourceDataInfo* pSourceDataInfo = (SSourceDataInfo*) param; + pSourceDataInfo->pRsp = pMsg->pData; - pEx->pRsp->numOfRows = htonl(pEx->pRsp->numOfRows); - pEx->pRsp->useconds = htobe64(pEx->pRsp->useconds); - pEx->pRsp->compLen = htonl(pEx->pRsp->compLen); + SRetrieveTableRsp* pRsp = pSourceDataInfo->pRsp; + pRsp->numOfRows = htonl(pRsp->numOfRows); + pRsp->useconds = htobe64(pRsp->useconds); + pRsp->compLen = htonl(pRsp->compLen); - tsem_post(&pEx->ready); + pSourceDataInfo->status = DATA_READY; + tsem_post(&pSourceDataInfo->pEx->ready); } static void destroySendMsgInfo(SMsgSendInfo* pMsgBody) { @@ -5139,115 +4973,234 @@ void qProcessFetchRsp(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { destroySendMsgInfo(pSendInfo); } -static SSDataBlock* doLoadRemoteData(void* param, bool* newgroup) { - SOperatorInfo *pOperator = (SOperatorInfo*) param; - - SExchangeInfo *pExchangeInfo = pOperator->info; - SExecTaskInfo *pTaskInfo = pOperator->pTaskInfo; - - *newgroup = false; - +static int32_t doSendFetchDataRequest(SExchangeInfo *pExchangeInfo, SExecTaskInfo *pTaskInfo, int32_t sourceIndex) { size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); - if (pExchangeInfo->current >= totalSources) { - qDebug("%s all %"PRIzu" source(s) are exhausted, total rows:%"PRIu64" bytes:%"PRIu64", elapsed:%.2f ms", GET_TASKID(pTaskInfo), totalSources, - pExchangeInfo->totalRows, pExchangeInfo->totalSize, pExchangeInfo->totalElapsed/1000.0); - return NULL; + + SResFetchReq* pMsg = calloc(1, sizeof(SResFetchReq)); + if (NULL == pMsg) { + pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + return pTaskInfo->code; } - SResFetchReq* pMsg = NULL; - SMsgSendInfo* pMsgSendInfo = NULL; + SDownstreamSource *pSource = taosArrayGet(pExchangeInfo->pSources, sourceIndex); + SSourceDataInfo *pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, sourceIndex); - while(1) { - pMsg = calloc(1, sizeof(SResFetchReq)); - if (NULL == pMsg) { // todo handle malloc error - pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto _error; + qDebug("%s build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", %d/%" PRIzu, + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->addr.epset.eps[0].fqdn, pSource->taskId, sourceIndex, totalSources); + + pMsg->header.vgId = htonl(pSource->addr.nodeId); + pMsg->sId = htobe64(pSource->schedId); + pMsg->taskId = htobe64(pSource->taskId); + pMsg->queryId = htobe64(pTaskInfo->id.queryId); + + // send the fetch remote task result reques + SMsgSendInfo* pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo)); + if (NULL == pMsgSendInfo) { + tfree(pMsg); + qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); + pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; + return pTaskInfo->code; + } + + pMsgSendInfo->param = pDataInfo; + pMsgSendInfo->msgInfo.pData = pMsg; + pMsgSendInfo->msgInfo.len = sizeof(SResFetchReq); + pMsgSendInfo->msgType = TDMT_VND_FETCH; + pMsgSendInfo->fp = loadRemoteDataCallback; + + int64_t transporterId = 0; + int32_t code = asyncSendMsgToServer(pExchangeInfo->pTransporter, &pSource->addr.epset, &transporterId, pMsgSendInfo); + return TSDB_CODE_SUCCESS; +} + +static int32_t setSDataBlockFromFetchRsp(SSDataBlock* pRes, SExchangeInfo *pExchangeInfo, SSourceDataInfo* pDataInfo, int32_t numOfOutput, int64_t startTs) { + char* pData = pDataInfo->pRsp->data; + SRetrieveTableRsp* pRsp = pDataInfo->pRsp; + + for (int32_t i = 0; i < numOfOutput; ++i) { + SColumnInfoData* pColInfoData = taosArrayGet(pRes->pDataBlock, i); + + char* tmp = realloc(pColInfoData->pData, pColInfoData->info.bytes * pRsp->numOfRows); + if (tmp == NULL) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; } - SDownstreamSource* pSource = taosArrayGet(pExchangeInfo->pSources, pExchangeInfo->current); + size_t len = pRsp->numOfRows * pColInfoData->info.bytes; + memcpy(tmp, pData, len); - int64_t startTs = taosGetTimestampUs(); - qDebug("%s build fetch msg and send to vgId:%d, ep:%s, taskId:0x%" PRIx64 ", %d/%" PRIzu, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->addr.epset.eps[0].fqdn, pSource->taskId, pExchangeInfo->current, totalSources); + pColInfoData->pData = tmp; + pData += len; + } - pMsg->header.vgId = htonl(pSource->addr.nodeId); - pMsg->sId = htobe64(pSource->schedId); - pMsg->taskId = htobe64(pSource->taskId); - pMsg->queryId = htobe64(pTaskInfo->id.queryId); + pRes->info.rows = pRsp->numOfRows; - // send the fetch remote task result reques - pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo)); - if (NULL == pMsgSendInfo) { - qError("%s prepare message %d failed", GET_TASKID(pTaskInfo), (int32_t)sizeof(SMsgSendInfo)); - pTaskInfo->code = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto _error; - } + int64_t el = taosGetTimestampUs() - startTs; - pMsgSendInfo->param = pExchangeInfo; - pMsgSendInfo->msgInfo.pData = pMsg; - pMsgSendInfo->msgInfo.len = sizeof(SResFetchReq); - pMsgSendInfo->msgType = TDMT_VND_FETCH; - pMsgSendInfo->fp = loadRemoteDataCallback; + pExchangeInfo->totalRows += pRsp->numOfRows; + pExchangeInfo->totalSize += pRsp->compLen; + pDataInfo->totalRows += pRsp->numOfRows; - int64_t transporterId = 0; - int32_t code = asyncSendMsgToServer(pExchangeInfo->pTransporter, &pSource->addr.epset, &transporterId, pMsgSendInfo); - tsem_wait(&pExchangeInfo->ready); + pExchangeInfo->totalElapsed += el; - SRetrieveTableRsp* pRsp = pExchangeInfo->pRsp; - if (pRsp->numOfRows == 0) { - qDebug("%s vgId:%d, taskID:0x%"PRIx64" %d of total completed, rowsOfSource:%"PRIu64", totalRows:%"PRIu64" try next", - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pExchangeInfo->current + 1, - pExchangeInfo->rowsOfCurrentSource, pExchangeInfo->totalRows); + return TSDB_CODE_SUCCESS; +} - pExchangeInfo->rowsOfCurrentSource = 0; - pExchangeInfo->current += 1; +static void* setAllSourcesCompleted(SOperatorInfo *pOperator, int64_t startTs) { + SExchangeInfo *pExchangeInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - if (pExchangeInfo->current >= totalSources) { - int64_t el = taosGetTimestampUs() - startTs; - pExchangeInfo->totalElapsed += el; + int64_t el = taosGetTimestampUs() - startTs; + pExchangeInfo->totalElapsed += el; - qDebug("%s all %"PRIzu" sources are exhausted, total rows: %"PRIu64" bytes:%"PRIu64", elapsed:%.2f ms", GET_TASKID(pTaskInfo), totalSources, - pExchangeInfo->totalRows, pExchangeInfo->totalSize, pExchangeInfo->totalElapsed/1000.0); - return NULL; - } else { + size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); + qDebug("%s all %"PRIzu" sources are exhausted, total rows: %"PRIu64" bytes:%"PRIu64", elapsed:%.2f ms", GET_TASKID(pTaskInfo), totalSources, + pExchangeInfo->totalRows, pExchangeInfo->totalSize, pExchangeInfo->totalElapsed/1000.0); + + doSetOperatorCompleted(pOperator); + return NULL; +} + +static SSDataBlock* concurrentlyLoadRemoteDataImpl(SOperatorInfo *pOperator, SExchangeInfo *pExchangeInfo, SExecTaskInfo *pTaskInfo) { + int32_t code = 0; + int64_t startTs = taosGetTimestampUs(); + size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); + + while (1) { + int32_t completed = 0; + for (int32_t i = 0; i < totalSources; ++i) { + SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, i); + + if (pDataInfo->status == DATA_EXHAUSTED) { + completed += 1; continue; } - } - SSDataBlock* pRes = pExchangeInfo->pResult; - char* pData = pRsp->data; + if (pDataInfo->status != DATA_READY) { + continue; + } - for (int32_t i = 0; i < pOperator->numOfOutput; ++i) { - SColumnInfoData* pColInfoData = taosArrayGet(pRes->pDataBlock, i); - char* tmp = realloc(pColInfoData->pData, pColInfoData->info.bytes * pRsp->numOfRows); - if (tmp == NULL) { + SRetrieveTableRsp* pRsp = pDataInfo->pRsp; + SDownstreamSource* pSource = taosArrayGet(pExchangeInfo->pSources, i); + + SSDataBlock* pRes = pExchangeInfo->pResult; + + if (pRsp->numOfRows == 0) { + qDebug("%s vgId:%d, taskID:0x%" PRIx64 " index:%d completed, rowsOfSource:%" PRIu64 ", totalRows:%" PRIu64 " try next", + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, i + 1, pDataInfo->totalRows, + pExchangeInfo->totalRows); + pDataInfo->status = DATA_EXHAUSTED; + completed += 1; + continue; + } + + code = setSDataBlockFromFetchRsp(pExchangeInfo->pResult, pExchangeInfo, pDataInfo, pOperator->numOfOutput, startTs); + if (code != 0) { goto _error; } - size_t len = pRsp->numOfRows * pColInfoData->info.bytes; - memcpy(tmp, pData, len); + if (pRsp->completed == 1) { + qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " numOfRows:%d, rowsOfSource:%" PRIu64 + ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 " try next %d/%" PRIzu, + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pRes->info.rows, + pDataInfo->totalRows, pExchangeInfo->totalRows, pExchangeInfo->totalSize, i + 1, + totalSources); + pDataInfo->status = DATA_EXHAUSTED; + } else { + qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " numOfRows:%d, totalRows:%" PRIu64 ", totalBytes:%" PRIu64, + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pRes->info.rows, pExchangeInfo->totalRows, + pExchangeInfo->totalSize); + } - pColInfoData->pData = tmp; - pData += len; + if (pDataInfo->status != DATA_EXHAUSTED) { + pDataInfo->status = DATA_NOT_READY; + code = doSendFetchDataRequest(pExchangeInfo, pTaskInfo, i); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + } + + return pExchangeInfo->pResult; } - pRes->info.numOfCols = pOperator->numOfOutput; - pRes->info.rows = pRsp->numOfRows; + if (completed == totalSources) { + return setAllSourcesCompleted(pOperator, startTs); + } + } - int64_t el = taosGetTimestampUs() - startTs; +_error: + pTaskInfo->code = code; + return NULL; +} - pExchangeInfo->totalRows += pRsp->numOfRows; - pExchangeInfo->totalSize += pRsp->compLen; - pExchangeInfo->rowsOfCurrentSource += pRsp->numOfRows; - pExchangeInfo->totalElapsed += el; +static SSDataBlock* concurrentlyLoadRemoteData(SOperatorInfo *pOperator) { + SExchangeInfo *pExchangeInfo = pOperator->info; + SExecTaskInfo *pTaskInfo = pOperator->pTaskInfo; + + if (pOperator->status == OP_RES_TO_RETURN) { + return concurrentlyLoadRemoteDataImpl(pOperator, pExchangeInfo, pTaskInfo); + } + + size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); + int64_t startTs = taosGetTimestampUs(); + + // Asynchronously send all fetch requests to all sources. + for(int32_t i = 0; i < totalSources; ++i) { + int32_t code = doSendFetchDataRequest(pExchangeInfo, pTaskInfo, i); + if (code != TSDB_CODE_SUCCESS) { + return NULL; + } + } + + int64_t endTs = taosGetTimestampUs(); + qDebug("%s send all fetch request to %"PRIzu" sources completed, elapsed:%"PRId64, GET_TASKID(pTaskInfo), totalSources, endTs - startTs); + + tsem_wait(&pExchangeInfo->ready); + + pOperator->status = OP_RES_TO_RETURN; + return concurrentlyLoadRemoteDataImpl(pOperator, pExchangeInfo, pTaskInfo); +} + +static SSDataBlock* seqLoadRemoteData(SOperatorInfo *pOperator) { + SExchangeInfo *pExchangeInfo = pOperator->info; + SExecTaskInfo *pTaskInfo = pOperator->pTaskInfo; + + size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); + int64_t startTs = taosGetTimestampUs(); + + while(1) { + if (pExchangeInfo->current >= totalSources) { + return setAllSourcesCompleted(pOperator, startTs); + } + + doSendFetchDataRequest(pExchangeInfo, pTaskInfo, pExchangeInfo->current); + + tsem_wait(&pExchangeInfo->ready); + + SSourceDataInfo* pDataInfo = taosArrayGet(pExchangeInfo->pSourceDataInfo, pExchangeInfo->current); + SDownstreamSource* pSource = taosArrayGet(pExchangeInfo->pSources, pExchangeInfo->current); + + SRetrieveTableRsp* pRsp = pDataInfo->pRsp; + if (pRsp->numOfRows == 0) { + qDebug("%s vgId:%d, taskID:0x%"PRIx64" %d of total completed, rowsOfSource:%"PRIu64", totalRows:%"PRIu64" try next", + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pExchangeInfo->current + 1, + pDataInfo->totalRows, pExchangeInfo->totalRows); + + pDataInfo->status = DATA_EXHAUSTED; + pExchangeInfo->current += 1; + continue; + } + + SSDataBlock* pRes = pExchangeInfo->pResult; + setSDataBlockFromFetchRsp(pExchangeInfo->pResult, pExchangeInfo, pDataInfo, pOperator->numOfOutput, startTs); if (pRsp->completed == 1) { qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " numOfRows:%d, rowsOfSource:%" PRIu64 - ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 " try next %d/%" PRIzu, - GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pRes->info.rows, pExchangeInfo->rowsOfCurrentSource, pExchangeInfo->totalRows, pExchangeInfo->totalSize, - pExchangeInfo->current + 1, totalSources); + ", totalRows:%" PRIu64 ", totalBytes:%" PRIu64 " try next %d/%" PRIzu, + GET_TASKID(pTaskInfo), pSource->addr.nodeId, pSource->taskId, pRes->info.rows, + pDataInfo->totalRows, pExchangeInfo->totalRows, pExchangeInfo->totalSize, pExchangeInfo->current + 1, + totalSources); - pExchangeInfo->rowsOfCurrentSource = 0; + pDataInfo->status = DATA_EXHAUSTED; pExchangeInfo->current += 1; } else { qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " numOfRows:%d, totalRows:%" PRIu64 ", totalBytes:%" PRIu64, @@ -5256,13 +5209,38 @@ static SSDataBlock* doLoadRemoteData(void* param, bool* newgroup) { return pExchangeInfo->pResult; } +} +static SSDataBlock* doLoadRemoteData(void* param, bool* newgroup) { + SOperatorInfo *pOperator = (SOperatorInfo*) param; + + SExchangeInfo *pExchangeInfo = pOperator->info; + SExecTaskInfo *pTaskInfo = pOperator->pTaskInfo; + + size_t totalSources = taosArrayGetSize(pExchangeInfo->pSources); + + if (pOperator->status == OP_EXEC_DONE) { + qDebug("%s all %"PRIzu" source(s) are exhausted, total rows:%"PRIu64" bytes:%"PRIu64", elapsed:%.2f ms", GET_TASKID(pTaskInfo), totalSources, + pExchangeInfo->totalRows, pExchangeInfo->totalSize, pExchangeInfo->totalElapsed/1000.0); + return NULL; + } + + *newgroup = false; + + if (pExchangeInfo->seqLoadData) { + return seqLoadRemoteData(pOperator); + } else { + return concurrentlyLoadRemoteData(pOperator); + } + +#if 0 _error: tfree(pMsg); tfree(pMsgSendInfo); terrno = pTaskInfo->code; return NULL; +#endif } static SSDataBlock* createResultDataBlock(const SArray* pExprInfo); @@ -5278,11 +5256,33 @@ SOperatorInfo* createExchangeOperatorInfo(const SArray* pSources, const SArray* return NULL; } + size_t numOfSources = taosArrayGetSize(pSources); + pInfo->pSources = taosArrayDup(pSources); - assert(taosArrayGetSize(pInfo->pSources) > 0); + pInfo->pSourceDataInfo = taosArrayInit(numOfSources, sizeof(SSourceDataInfo)); + if (pInfo->pSourceDataInfo == NULL || pInfo->pSources == NULL) { + tfree(pInfo); + tfree(pOperator); + taosArrayDestroy(pInfo->pSources); + taosArrayDestroy(pInfo->pSourceDataInfo); + terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + return NULL; + } + + for(int32_t i = 0; i < numOfSources; ++i) { + SSourceDataInfo dataInfo = {0}; + dataInfo.status = DATA_NOT_READY; + dataInfo.pEx = pInfo; + dataInfo.index = i; + + taosArrayPush(pInfo->pSourceDataInfo, &dataInfo); + } size_t size = taosArrayGetSize(pExprInfo); pInfo->pResult = createResultDataBlock(pExprInfo); + pInfo->seqLoadData = true; + + tsem_init(&pInfo->ready, 0, 0); pOperator->name = "ExchangeOperator"; pOperator->operatorType = OP_Exchange; @@ -5290,7 +5290,6 @@ SOperatorInfo* createExchangeOperatorInfo(const SArray* pSources, const SArray* pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; pOperator->numOfOutput = size; - pOperator->pRuntimeEnv = NULL; pOperator->exec = doLoadRemoteData; pOperator->pTaskInfo = pTaskInfo; @@ -5316,6 +5315,7 @@ SOperatorInfo* createExchangeOperatorInfo(const SArray* pSources, const SArray* } } #endif + return pOperator; } @@ -5344,7 +5344,7 @@ SSDataBlock* createResultDataBlock(const SArray* pExprInfo) { return pResBlock; } -SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, int32_t reverseTime, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, int32_t reverseTime, SExecTaskInfo* pTaskInfo) { assert(repeatTime > 0); STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); @@ -5364,8 +5364,8 @@ SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbReadHandle, int32_t order, pInfo->current = 0; pInfo->scanFlag = MAIN_SCAN; - pOperator->name = "DataBlocksOptimizedScanOperator"; - pOperator->operatorType = OP_DataBlocksOptScan; + pOperator->name = "TableScanOperator"; + pOperator->operatorType = OP_TableScan; pOperator->blockingOptr = false; pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; @@ -5376,7 +5376,7 @@ SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbReadHandle, int32_t order, return pOperator; } -SOperatorInfo* createTableSeqScanOperator(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv) { +SOperatorInfo* createTableSeqScanOperatorInfo(void* pTsdbReadHandle, STaskRuntimeEnv* pRuntimeEnv) { STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); pInfo->pTsdbReadHandle = pTsdbReadHandle; @@ -5604,10 +5604,11 @@ static void destroyGlobalAggOperatorInfo(void* param, int32_t numOfOutput) { tfree(pInfo->prevRow); tfree(pInfo->currentGroupColData); } + static void destroySlimitOperatorInfo(void* param, int32_t numOfOutput) { SSLimitOperatorInfo *pInfo = (SSLimitOperatorInfo*) param; taosArrayDestroy(pInfo->orderColumnList); - pInfo->pRes = destroyOutputBuf(pInfo->pRes); + pInfo->pRes = blockDataDestroy(pInfo->pRes); tfree(pInfo->prevRow); } @@ -5719,39 +5720,397 @@ SOperatorInfo *createMultiwaySortOperatorInfo(STaskRuntimeEnv *pRuntimeEnv, SExp return pOperator; } -static int32_t doMergeSDatablock(SSDataBlock* pDest, SSDataBlock* pSrc) { - assert(pSrc != NULL && pDest != NULL && pDest->info.numOfCols == pSrc->info.numOfCols); +typedef struct SExternalMemSource { + SArray* pageIdList; + int32_t pageIndex; + int32_t sourceId; + int32_t rowIndex; + SSDataBlock *pBlock; +} SExternalMemSource; - int32_t numOfCols = pSrc->info.numOfCols; - for(int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pCol2 = taosArrayGet(pDest->pDataBlock, i); - SColumnInfoData* pCol1 = taosArrayGet(pSrc->pDataBlock, i); +int32_t msortComparFn(const void *pLeft, const void *pRight, void *param) { + int32_t pLeftIdx = *(int32_t *)pLeft; + int32_t pRightIdx = *(int32_t *)pRight; - int32_t newSize = (pDest->info.rows + pSrc->info.rows) * pCol2->info.bytes; - char* tmp = realloc(pCol2->pData, newSize); - if (tmp != NULL) { - pCol2->pData = tmp; - int32_t offset = pCol2->info.bytes * pDest->info.rows; - memcpy(pCol2->pData + offset, pCol1->pData, pSrc->info.rows * pCol2->info.bytes); + SMsortComparParam *pParam = (SMsortComparParam *)param; + + SArray *pInfo = pParam->orderInfo; + + SExternalMemSource* pLeftSource = pParam->pSources[pLeftIdx]; + SExternalMemSource* pRightSource = pParam->pSources[pRightIdx]; + + // this input is exhausted, set the special value to denote this + if (pLeftSource->rowIndex == -1) { + return 1; + } + + if (pRightSource->rowIndex == -1) { + return -1; + } + + SSDataBlock* pLeftBlock = pLeftSource->pBlock; + SSDataBlock* pRightBlock = pRightSource->pBlock; + + for(int32_t i = 0; i < pInfo->size; ++i) { + SBlockOrderInfo* pOrder = TARRAY_GET_ELEM(pInfo, i); + + SColumnInfoData* pLeftColInfoData = TARRAY_GET_ELEM(pLeftBlock->pDataBlock, pOrder->colIndex); + + bool leftNull = false; + if (pLeftColInfoData->hasNull) { + leftNull = colDataIsNull(pLeftColInfoData, pLeftBlock->info.rows, pLeftSource->rowIndex, pLeftBlock->pBlockAgg); + } + + SColumnInfoData* pRightColInfoData = TARRAY_GET_ELEM(pRightBlock->pDataBlock, pOrder->colIndex); + bool rightNull = false; + if (pRightColInfoData->hasNull) { + rightNull = colDataIsNull(pRightColInfoData, pRightBlock->info.rows, pRightSource->rowIndex, pRightBlock->pBlockAgg); + } + + if (leftNull && rightNull) { + continue; // continue to next slot + } + + if (rightNull) { + return pParam->nullFirst? 1:-1; + } + + if (leftNull) { + return pParam->nullFirst? -1:1; + } + + void* left1 = colDataGet(pLeftColInfoData, pLeftSource->rowIndex); + void* right1 = colDataGet(pRightColInfoData, pRightSource->rowIndex); + + switch(pLeftColInfoData->info.type) { + case TSDB_DATA_TYPE_INT: { + int32_t leftv = *(int32_t*)left1; + int32_t rightv = *(int32_t*)right1; + + if (leftv == rightv) { + break; + } else { + if (pOrder->order == TSDB_ORDER_ASC) { + return leftv < rightv? -1 : 1; + } else { + return leftv < rightv? 1 : -1; + } + } + } + default: + assert(0); + } + } +} + +static int32_t adjustMergeTreeForNextTuple(SExternalMemSource *pSource, SMultiwayMergeTreeInfo *pTree, SOrderOperatorInfo* pInfo) { + /* + * load a new SDataBlock into memory of a given intermediate data-set source, + * since it's last record in buffer has been chosen to be processed, as the winner of loser-tree + */ + if (pSource->rowIndex >= pSource->pBlock->info.rows) { + pSource->rowIndex = 0; + pSource->pageIndex += 1; + + if (pSource->pageIndex >= taosArrayGetSize(pSource->pageIdList)) { + pInfo->numOfCompleted += 1; + pSource->rowIndex = -1; + pSource->pageIndex = -1; + pSource->pBlock = blockDataDestroy(pSource->pBlock); } else { - return TSDB_CODE_VND_OUT_OF_MEMORY; + SPageInfo* pPgInfo = *(SPageInfo**)taosArrayGet(pSource->pageIdList, pSource->pageIndex); + + SFilePage* pPage = getBufPage(pInfo->pSortInternalBuf, getPageId(pPgInfo)); + int32_t code = blockDataFromBuf(pSource->pBlock, pPage->data); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + releaseBufPage(pInfo->pSortInternalBuf, pPage); } } - pDest->info.rows += pSrc->info.rows; + /* + * Adjust loser tree otherwise, according to new candidate data + * if the loser tree is rebuild completed, we do not need to adjust + */ + int32_t leafNodeIndex = tMergeTreeGetAdjustIndex(pTree); + +#ifdef _DEBUG_VIEW + printf("before adjust:\t"); + tMergeTreePrint(pTree); +#endif + + tMergeTreeAdjust(pTree, leafNodeIndex); + +#ifdef _DEBUG_VIEW + printf("\nafter adjust:\t"); + tMergeTreePrint(pTree); +#endif +} + +static void appendOneRowToDataBlock(SSDataBlock *pBlock, const SSDataBlock* pSource, int32_t* rowIndex) { + for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); + + SColumnInfoData* pSrcColInfo = taosArrayGet(pSource->pDataBlock, i); + bool isNull = colDataIsNull(pSrcColInfo, pSource->info.rows, *rowIndex, NULL); + + if (isNull) { + colDataAppend(pColInfo, pBlock->info.rows, NULL, true); + } else { + char* pData = colDataGet(pSrcColInfo, *rowIndex); + colDataAppend(pColInfo, pBlock->info.rows, pData, false); + } + } + + pBlock->info.rows += 1; + *rowIndex += 1; +} + +static int32_t doAddNewSource(SOrderOperatorInfo* pInfo, SArray* pAllSources, int32_t numOfCols) { + SExternalMemSource* pSource = calloc(1, sizeof(SExternalMemSource)); + if (pSource == NULL) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + + pSource->pageIdList = getDataBufPagesIdList(pInfo->pSortInternalBuf, pInfo->sourceId); + pSource->sourceId = pInfo->sourceId; + + pSource->pBlock = calloc(1, sizeof(SSDataBlock)); + pSource->pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); + pSource->pBlock->info.numOfCols = numOfCols; + + for(int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData colInfo = {0}; + SColumnInfoData* p = taosArrayGet(pInfo->pDataBlock->pDataBlock, i); + colInfo.info = p->info; + taosArrayPush(pSource->pBlock->pDataBlock, &colInfo); + } + + taosArrayPush(pAllSources, &pSource); + + pInfo->sourceId += 1; + + int32_t rowSize = blockDataGetSerialRowSize(pSource->pBlock); + int32_t numOfRows = (getBufPageSize(pInfo->pSortInternalBuf) - blockDataGetSerialMetaSize(pInfo->pDataBlock))/rowSize; + + return blockDataEnsureCapacity(pSource->pBlock, numOfRows); +} + +void addToDiskbasedBuf(SOrderOperatorInfo* pInfo, SArray* pSources, jmp_buf env) { + int32_t start = 0; + + while(start < pInfo->pDataBlock->info.rows) { + int32_t stop = 0; + blockDataSplitRows(pInfo->pDataBlock, pInfo->hasVarCol, start, &stop, getBufPageSize(pInfo->pSortInternalBuf)); + SSDataBlock* p = blockDataExtractBlock(pInfo->pDataBlock, start, stop - start + 1); + if (p == NULL) { + longjmp(env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + int32_t pageId = -1; + SFilePage* pPage = getNewDataBuf(pInfo->pSortInternalBuf, pInfo->sourceId, &pageId); + if (pPage == NULL) { + assert(0); + longjmp(env, terrno); + } + + int32_t size = blockDataGetSize(p) + sizeof(int32_t) + p->info.numOfCols * sizeof(int32_t); + assert(size <= getBufPageSize(pInfo->pSortInternalBuf)); + + blockDataToBuf(pPage->data, p); + + setBufPageDirty(pPage, true); + releaseBufPage(pInfo->pSortInternalBuf, pPage); + + blockDataDestroy(p); + start = stop + 1; + } + + int32_t numOfCols = pInfo->pDataBlock->info.numOfCols; + blockDataClearup(pInfo->pDataBlock, pInfo->hasVarCol); + + int32_t code = doAddNewSource(pInfo, pSources, numOfCols); + if (code != TSDB_CODE_SUCCESS) { + longjmp(env, code); + } +} + +static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int32_t startIndex, int32_t endIndex, SDiskbasedBuf* pBuf) { + cmpParam->pSources = taosArrayGet(pSources, startIndex); + cmpParam->numOfSources = (endIndex - startIndex + 1); + + for(int32_t i = 0; i < cmpParam->numOfSources; ++i) { + SExternalMemSource* pSource = cmpParam->pSources[i]; + SPageInfo* pPgInfo = *(SPageInfo**) taosArrayGet(pSource->pageIdList, pSource->pageIndex); + + SFilePage* pPage = getBufPage(pBuf, getPageId(pPgInfo)); + int32_t code = blockDataFromBuf(cmpParam->pSources[i]->pBlock, pPage->data); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + releaseBufPage(pBuf, pPage); + } return TSDB_CODE_SUCCESS; } +static int32_t sortComparClearup(SMsortComparParam* cmpParam) { + for(int32_t i = 0; i < cmpParam->numOfSources; ++i) { + SExternalMemSource* pSource = cmpParam->pSources[i]; + blockDataDestroy(pSource->pBlock); + tfree(pSource); + } + + cmpParam->numOfSources = 0; +} + +static SSDataBlock* getSortedBlockData(SExecTaskInfo* pTaskInfo, SOrderOperatorInfo* pInfo, SMsortComparParam* cmpParam, int32_t capacity) { + blockDataClearup(pInfo->pDataBlock, pInfo->hasVarCol); + + while(1) { + if (cmpParam->numOfSources == pInfo->numOfCompleted) { + break; + } + + int32_t index = tMergeTreeGetChosenIndex(pInfo->pMergeTree); + + SExternalMemSource *pSource = (*cmpParam).pSources[index]; + appendOneRowToDataBlock(pInfo->pDataBlock, pSource->pBlock, &pSource->rowIndex); + + int32_t code = adjustMergeTreeForNextTuple(pSource, pInfo->pMergeTree, pInfo); + if (code != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, code); + } + + if (pInfo->pDataBlock->info.rows >= capacity) { + return pInfo->pDataBlock; + } + } + + return (pInfo->pDataBlock->info.rows > 0)? pInfo->pDataBlock:NULL; +} + +static int32_t doInternalSort(SExecTaskInfo* pTaskInfo, SOrderOperatorInfo* pInfo) { + size_t numOfSources = taosArrayGetSize(pInfo->pSources); + + // Calculate the I/O counts to complete the data sort. + double sortCount = floorl(log2(numOfSources) / log2(getNumOfInMemBufPages(pInfo->pSortInternalBuf))); + + pInfo->totalElapsed = taosGetTimestampUs() - pInfo->startTs; + qDebug("%s %d rounds mergesort required to complete the sort, first-round sorted data size:%"PRIzu", sort:%"PRId64", total elapsed:%"PRId64, + GET_TASKID(pTaskInfo), (int32_t) (sortCount + 1), getTotalBufSize(pInfo->pSortInternalBuf), pInfo->sortElapsed, + pInfo->totalElapsed); + + size_t pgSize = getBufPageSize(pInfo->pSortInternalBuf); + int32_t numOfRows = (pgSize - blockDataGetSerialMetaSize(pInfo->pDataBlock))/ blockDataGetSerialRowSize(pInfo->pDataBlock); + + blockDataEnsureCapacity(pInfo->pDataBlock, numOfRows); + + size_t numOfSorted = taosArrayGetSize(pInfo->pSources); + for(int32_t t = 0; t < sortCount; ++t) { + int64_t st = taosGetTimestampUs(); + + SArray* pResList = taosArrayInit(4, POINTER_BYTES); + SMsortComparParam resultParam = {.orderInfo = pInfo->cmpParam.orderInfo}; + + int32_t numOfInputSources = getNumOfInMemBufPages(pInfo->pSortInternalBuf); + int32_t sortGroup = (numOfSorted + numOfInputSources - 1) / numOfInputSources; + + // Only *numOfInputSources* can be loaded into buffer to perform the external sort. + for(int32_t i = 0; i < sortGroup; ++i) { + pInfo->sourceId += 1; + + int32_t end = (i + 1) * numOfInputSources - 1; + if (end > numOfSorted - 1) { + end = numOfSorted - 1; + } + + pInfo->cmpParam.numOfSources = end - i * numOfInputSources + 1; + + int32_t code = sortComparInit(&pInfo->cmpParam, pInfo->pSources, i * numOfInputSources, end, pInfo->pSortInternalBuf); + if (code != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, code); + } + + code = tMergeTreeCreate(&pInfo->pMergeTree, pInfo->cmpParam.numOfSources, &pInfo->cmpParam, msortComparFn); + if (code != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, code); + } + + while (1) { + SSDataBlock* pDataBlock = getSortedBlockData(pTaskInfo, pInfo, &pInfo->cmpParam, numOfRows); + if (pDataBlock == NULL) { + break; + } + + int32_t pageId = -1; + SFilePage* pPage = getNewDataBuf(pInfo->pSortInternalBuf, pInfo->sourceId, &pageId); + if (pPage == NULL) { + assert(0); + longjmp(pTaskInfo->env, terrno); + } + + int32_t size = blockDataGetSize(pDataBlock) + sizeof(int32_t) + pDataBlock->info.numOfCols * sizeof(int32_t); + assert(size <= getBufPageSize(pInfo->pSortInternalBuf)); + + blockDataToBuf(pPage->data, pDataBlock); + + setBufPageDirty(pPage, true); + releaseBufPage(pInfo->pSortInternalBuf, pPage); + + blockDataClearup(pDataBlock, pInfo->hasVarCol); + } + + tMergeTreeDestroy(pInfo->pMergeTree); + pInfo->numOfCompleted = 0; + + code = doAddNewSource(pInfo, pResList, pInfo->pDataBlock->info.numOfCols); + if (code != 0) { + longjmp(pTaskInfo->env, code); + } + } + + sortComparClearup(&pInfo->cmpParam); + + taosArrayClear(pInfo->pSources); + taosArrayAddAll(pInfo->pSources, pResList); + taosArrayDestroy(pResList); + + pInfo->cmpParam = resultParam; + numOfSorted = taosArrayGetSize(pInfo->pSources); + + int64_t el = taosGetTimestampUs() - st; + pInfo->totalElapsed += el; + + SDiskbasedBufStatis statis = getDBufStatis(pInfo->pSortInternalBuf); + + qDebug("%s %d round mergesort, elapsed:%"PRId64" readDisk:%.2f Kb, flushDisk:%.2f Kb", GET_TASKID(pTaskInfo), t + 1, el, statis.loadBytes/1024.0, + statis.flushBytes/1024.0); + } + + pInfo->cmpParam.numOfSources = taosArrayGetSize(pInfo->pSources); + return 0; +} + static SSDataBlock* doSort(void* param, bool* newgroup) { SOperatorInfo* pOperator = (SOperatorInfo*) param; if (pOperator->status == OP_EXEC_DONE) { return NULL; } + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SOrderOperatorInfo* pInfo = pOperator->info; - SSDataBlock* pBlock = NULL; + + if (pOperator->status == OP_RES_TO_RETURN) { + return getSortedBlockData(pTaskInfo, pInfo, &pInfo->cmpParam, pInfo->numOfRowsInRes); + } + + int64_t st = taosGetTimestampUs(); + while(1) { publishOperatorProfEvent(pOperator->pDownstream[0], QUERY_PROF_BEFORE_OPERATOR_EXEC); pBlock = pOperator->pDownstream[0]->exec(pOperator->pDownstream[0], newgroup); @@ -5759,68 +6118,130 @@ static SSDataBlock* doSort(void* param, bool* newgroup) { // start to flush data into disk and try do multiway merge sort if (pBlock == NULL) { - doSetOperatorCompleted(pOperator); break; } - int32_t code = doMergeSDatablock(pInfo->pDataBlock, pBlock); + int32_t code = blockDataMerge(pInfo->pDataBlock, pBlock); if (code != TSDB_CODE_SUCCESS) { - // todo handle error + longjmp(pOperator->pTaskInfo->env, code); + } + + size_t size = blockDataGetSize(pInfo->pDataBlock); + if (size > pInfo->sortBufSize) { + // Perform the in-memory sort and then flush data in the buffer into disk. + int64_t p = taosGetTimestampUs(); + blockDataSort(pInfo->pDataBlock, pInfo->cmpParam.orderInfo, pInfo->cmpParam.nullFirst); + + int64_t el = taosGetTimestampUs() - p; + pInfo->sortElapsed += el; + + addToDiskbasedBuf(pInfo, pInfo->pSources, pTaskInfo->env); } } - int32_t numOfCols = pInfo->pDataBlock->info.numOfCols; - void** pCols = calloc(numOfCols, POINTER_BYTES); - SSchema* pSchema = calloc(numOfCols, sizeof(SSchema)); + if (pInfo->pDataBlock->info.rows > 0) { + // Perform the in-memory sort and then flush data in the buffer into disk. + blockDataSort(pInfo->pDataBlock, pInfo->cmpParam.orderInfo, pInfo->cmpParam.nullFirst); - for(int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* p1 = taosArrayGet(pInfo->pDataBlock->pDataBlock, i); - pCols[i] = p1->pData; - pSchema[i].colId = p1->info.colId; - pSchema[i].bytes = p1->info.bytes; - pSchema[i].type = (uint8_t) p1->info.type; + // All sorted data are resident in memory, external memory sort is not needed. + // Return to the upstream operator directly + if (isAllDataInMemBuf(pInfo->pSortInternalBuf)) { + pOperator->status = OP_EXEC_DONE; + return (pInfo->pDataBlock->info.rows == 0)? NULL:pInfo->pDataBlock; + } + + addToDiskbasedBuf(pInfo, pInfo->pSources, pTaskInfo->env); } - __compar_fn_t comp = getKeyComparFunc(pSchema[pInfo->colIndex].type, pInfo->order); -// taosqsort(pCols, pSchema, numOfCols, pInfo->pDataBlock->info.rows, pInfo->colIndex, comp); + doInternalSort(pTaskInfo, pInfo); - tfree(pCols); - tfree(pSchema); - return (pInfo->pDataBlock->info.rows > 0)? pInfo->pDataBlock:NULL; + int32_t code = blockDataEnsureCapacity(pInfo->pDataBlock, pInfo->numOfRowsInRes); + if (code != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, code); + } + + int32_t numOfSources = taosArrayGetSize(pInfo->pSources); + ASSERT(numOfSources <= getNumOfInMemBufPages(pInfo->pSortInternalBuf)); + code = sortComparInit(&pInfo->cmpParam, pInfo->pSources, 0, numOfSources - 1, pInfo->pSortInternalBuf); + if (code != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, code); + } + + code = tMergeTreeCreate(&pInfo->pMergeTree, pInfo->cmpParam.numOfSources, &pInfo->cmpParam, msortComparFn); + if (code != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, code); + } + + pOperator->status = OP_RES_TO_RETURN; + return getSortedBlockData(pTaskInfo, pInfo, &pInfo->cmpParam, pInfo->numOfRowsInRes); } -SOperatorInfo *createOrderOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput, SOrder* pOrderVal) { - SOrderOperatorInfo* pInfo = calloc(1, sizeof(SOrderOperatorInfo)); +static SArray* createBlockOrder(SArray* pExprInfo, SArray* pOrderVal) { + SArray* pOrderInfo = taosArrayInit(1, sizeof(SBlockOrderInfo)); - { - SSDataBlock* pDataBlock = calloc(1, sizeof(SSDataBlock)); - pDataBlock->pDataBlock = taosArrayInit(numOfOutput, sizeof(SColumnInfoData)); - for(int32_t i = 0; i < numOfOutput; ++i) { - SColumnInfoData col = {{0}}; - col.info.colId = pExpr[i].base.pColumns->info.colId; -// col.info.bytes = pExpr[i].base.colBytes; -// col.info.type = pExpr[i].base.colType; - taosArrayPush(pDataBlock->pDataBlock, &col); + size_t numOfOrder = taosArrayGetSize(pOrderVal); + for (int32_t j = 0; j < numOfOrder; ++j) { + SBlockOrderInfo orderInfo = {0}; + SOrder* pOrder = taosArrayGet(pOrderVal, j); + orderInfo.order = pOrder->order; -// if (col.info.colId == pOrderVal->orderColId) { -// pInfo->colIndex = i; -// } + for (int32_t i = 0; i < taosArrayGetSize(pExprInfo); ++i) { + SExprInfo* pExpr = taosArrayGet(pExprInfo, i); + if (pExpr->base.resSchema.colId == pOrder->col.info.colId) { + orderInfo.colIndex = i; + break; } + } - pDataBlock->info.numOfCols = numOfOutput; -// pInfo->order = pOrderVal->order; - pInfo->pDataBlock = pDataBlock; + taosArrayPush(pOrderInfo, &orderInfo); } + return pOrderInfo; +} + +SOperatorInfo *createOrderOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SArray* pOrderVal) { + SOrderOperatorInfo* pInfo = calloc(1, sizeof(SOrderOperatorInfo)); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); - pOperator->name = "InMemoryOrder"; -// pOperator->operatorType = OP_Order; + if (pInfo == NULL || pOperator == NULL) { + tfree(pInfo); + + terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + return NULL; + } + + pInfo->sortBufSize = 1024 * 16; // 1MB + pInfo->bufPageSize = 1024; + pInfo->numOfRowsInRes = 1024; + + pInfo->pDataBlock = createOutputBuf_rv(pExprInfo, pInfo->numOfRowsInRes); + pInfo->pSources = taosArrayInit(4, POINTER_BYTES); + pInfo->cmpParam.orderInfo = createBlockOrder(pExprInfo, pOrderVal); + + for(int32_t i = 0; i < taosArrayGetSize(pExprInfo); ++i) { + SExprInfo* pExpr = taosArrayGetP(pExprInfo, i); + if (IS_VAR_DATA_TYPE(pExpr->base.resSchema.type)) { + pInfo->hasVarCol = true; + break; + } + } + + int32_t code = createDiskbasedBuffer(&pInfo->pSortInternalBuf, pInfo->bufPageSize, pInfo->sortBufSize, 1, "/tmp/"); + if (pInfo->pSources == NULL || code != 0 || pInfo->cmpParam.orderInfo == NULL || pInfo->pDataBlock == NULL) { + tfree(pOperator); + destroyOrderOperatorInfo(pInfo, taosArrayGetSize(pExprInfo)); + tfree(pInfo); + + terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; + return NULL; + } + + pOperator->name = "Order"; + pOperator->operatorType = OP_Order; pOperator->blockingOptr = true; pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; pOperator->exec = doSort; - pOperator->cleanupFn = destroyOrderOperatorInfo; - pOperator->pRuntimeEnv = pRuntimeEnv; + pOperator->cleanupFn = destroyOrderOperatorInfo; appendDownstream(pOperator, downstream); return pOperator; @@ -5869,7 +6290,7 @@ static SSDataBlock* doAggregate(void* param, bool* newgroup) { return (pInfo->pRes->info.rows != 0)? pInfo->pRes:NULL; } -static SSDataBlock* doSTableAggregate(void* param, bool* newgroup) { +static SSDataBlock* doMultiTableAggregate(void* param, bool* newgroup) { SOperatorInfo* pOperator = (SOperatorInfo*) param; if (pOperator->status == OP_EXEC_DONE) { return NULL; @@ -5877,22 +6298,20 @@ static SSDataBlock* doSTableAggregate(void* param, bool* newgroup) { SAggOperatorInfo* pAggInfo = pOperator->info; SOptrBasicInfo* pInfo = &pAggInfo->binfo; - - STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; if (pOperator->status == OP_RES_TO_RETURN) { - toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->pRes); + toSDatablock(&pAggInfo->groupResInfo, pAggInfo->pResultBuf, pInfo->pRes, pAggInfo->binfo.capacity); - if (pInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + if (pInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pAggInfo->groupResInfo)) { pOperator->status = OP_EXEC_DONE; } return pInfo->pRes; } - STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - int32_t order = pQueryAttr->order.order; - + // table scan order + int32_t order = TSDB_ORDER_ASC; SOperatorInfo* downstream = pOperator->pDownstream[0]; while(1) { @@ -5904,9 +6323,8 @@ static SSDataBlock* doSTableAggregate(void* param, bool* newgroup) { break; } - setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->pCtx, pOperator->numOfOutput); - -// if (downstream->operatorType == OP_DataBlocksOptScan) { +// setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->pCtx, pOperator->numOfOutput); +// if (downstream->operatorType == OP_TableScan) { // STableScanInfo* pScanInfo = downstream->info; // order = getTableScanOrder(pScanInfo); // } @@ -5915,7 +6333,7 @@ static SSDataBlock* doSTableAggregate(void* param, bool* newgroup) { setInputDataBlock(pOperator, pInfo->pCtx, pBlock, order); TSKEY key = 0; - if (QUERY_IS_ASC_QUERY(pQueryAttr)) { + if (order == TSDB_ORDER_ASC) { key = pBlock->info.window.ekey; TSKEY_MAX_ADD(key, 1); } else { @@ -5923,20 +6341,18 @@ static SSDataBlock* doSTableAggregate(void* param, bool* newgroup) { TSKEY_MIN_SUB(key, -1); } - setExecutionContext(pRuntimeEnv, pInfo, pOperator->numOfOutput, pRuntimeEnv->current->groupIndex, key); - doAggregateImpl(pOperator, pQueryAttr->window.skey, pInfo->pCtx, pBlock); + setExecutionContext(pOperator->numOfOutput, pAggInfo->current->groupIndex, key, pTaskInfo, pAggInfo->current, pAggInfo); + doAggregateImpl(pOperator, 0, pInfo->pCtx, pBlock); } pOperator->status = OP_RES_TO_RETURN; closeAllResultRows(&pInfo->resultRowInfo); + updateNumOfRowsInResultRows(pInfo->pCtx, pOperator->numOfOutput, &pInfo->resultRowInfo, pInfo->rowCellInfoOffset); - updateNumOfRowsInResultRows(pRuntimeEnv, pInfo->pCtx, pOperator->numOfOutput, &pInfo->resultRowInfo, - pInfo->rowCellInfoOffset); + initGroupResInfo(&pAggInfo->groupResInfo, &pInfo->resultRowInfo); + toSDatablock(&pAggInfo->groupResInfo, pAggInfo->pResultBuf, pInfo->pRes, pAggInfo->binfo.capacity); - initGroupResInfo(&pRuntimeEnv->groupResInfo, &pInfo->resultRowInfo); - - toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->pRes); - if (pInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { + if (pInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pAggInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); } @@ -5964,7 +6380,7 @@ static SSDataBlock* doProjectOperation(void* param, bool* newgroup) { // todo dynamic set tags if (pTableQueryInfo != NULL) { - setTagValue(pOperator, pTableQueryInfo->pTable, pInfo->pCtx, pOperator->numOfOutput); +// setTagValue(pOperator, pTableQueryInfo->pTable, pInfo->pCtx, pOperator->numOfOutput); } // the pDataBlock are always the same one, no need to call this again @@ -6014,7 +6430,7 @@ static SSDataBlock* doProjectOperation(void* param, bool* newgroup) { // todo dynamic set tags if (pTableQueryInfo != NULL) { - setTagValue(pOperator, pTableQueryInfo->pTable, pInfo->pCtx, pOperator->numOfOutput); +// setTagValue(pOperator, pTableQueryInfo->pTable, pInfo->pCtx, pOperator->numOfOutput); } // the pDataBlock are always the same one, no need to call this again @@ -6125,7 +6541,7 @@ static SSDataBlock* doIntervalAgg(void* param, bool* newgroup) { STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; if (pOperator->status == OP_RES_TO_RETURN) { - toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); +// toSDatablock(pAggInfo->pGroupResInfo, pAggInfo->pResultBuf, pInfo->pRes, pAggInfo->binfo.capacity); if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { doSetOperatorCompleted(pOperator); } @@ -6148,7 +6564,7 @@ static SSDataBlock* doIntervalAgg(void* param, bool* newgroup) { break; } - setTagValue(pOperator, pRuntimeEnv->current->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput); +// setTagValue(pOperator, pRuntimeEnv->current->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput); // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pIntervalInfo->pCtx, pBlock, pQueryAttr->order.order); @@ -6165,7 +6581,7 @@ static SSDataBlock* doIntervalAgg(void* param, bool* newgroup) { finalizeQueryResult(pOperator, pIntervalInfo->pCtx, &pIntervalInfo->resultRowInfo, pIntervalInfo->rowCellInfoOffset); initGroupResInfo(&pRuntimeEnv->groupResInfo, &pIntervalInfo->resultRowInfo); - toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); +// toSDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { doSetOperatorCompleted(pOperator); @@ -6184,7 +6600,7 @@ static SSDataBlock* doAllIntervalAgg(void* param, bool* newgroup) { STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; if (pOperator->status == OP_RES_TO_RETURN) { - toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); +// toSDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { doSetOperatorCompleted(pOperator); @@ -6208,7 +6624,7 @@ static SSDataBlock* doAllIntervalAgg(void* param, bool* newgroup) { break; } - setTagValue(pOperator, pRuntimeEnv->current->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput); +// setTagValue(pOperator, pRuntimeEnv->current->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput); // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pIntervalInfo->pCtx, pBlock, pQueryAttr->order.order); @@ -6225,7 +6641,7 @@ static SSDataBlock* doAllIntervalAgg(void* param, bool* newgroup) { finalizeQueryResult(pOperator, pIntervalInfo->pCtx, &pIntervalInfo->resultRowInfo, pIntervalInfo->rowCellInfoOffset); initGroupResInfo(&pRuntimeEnv->groupResInfo, &pIntervalInfo->resultRowInfo); - toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); +// toSDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { pOperator->status = OP_EXEC_DONE; @@ -6246,7 +6662,7 @@ static SSDataBlock* doSTableIntervalAgg(void* param, bool* newgroup) { if (pOperator->status == OP_RES_TO_RETURN) { int64_t st = taosGetTimestampUs(); - copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); +// copyToSDataBlock(NULL, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { doSetOperatorCompleted(pOperator); } @@ -6274,7 +6690,7 @@ static SSDataBlock* doSTableIntervalAgg(void* param, bool* newgroup) { // the pDataBlock are always the same one, no need to call this again STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; - setTagValue(pOperator, pTableQueryInfo->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput); +// setTagValue(pOperator, pTableQueryInfo->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput); setInputDataBlock(pOperator, pIntervalInfo->pCtx, pBlock, pQueryAttr->order.order); setIntervalQueryRange(pRuntimeEnv, pBlock->info.window.skey); @@ -6286,7 +6702,7 @@ static SSDataBlock* doSTableIntervalAgg(void* param, bool* newgroup) { doCloseAllTimeWindow(pRuntimeEnv); setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); - copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); +// copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { pOperator->status = OP_EXEC_DONE; } @@ -6304,7 +6720,7 @@ static SSDataBlock* doAllSTableIntervalAgg(void* param, bool* newgroup) { STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; if (pOperator->status == OP_RES_TO_RETURN) { - copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); +// copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { pOperator->status = OP_EXEC_DONE; } @@ -6329,7 +6745,7 @@ static SSDataBlock* doAllSTableIntervalAgg(void* param, bool* newgroup) { // the pDataBlock are always the same one, no need to call this again STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; - setTagValue(pOperator, pTableQueryInfo->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput); +// setTagValue(pOperator, pTableQueryInfo->pTable, pIntervalInfo->pCtx, pOperator->numOfOutput); setInputDataBlock(pOperator, pIntervalInfo->pCtx, pBlock, pQueryAttr->order.order); setIntervalQueryRange(pRuntimeEnv, pBlock->info.window.skey); @@ -6342,7 +6758,7 @@ static SSDataBlock* doAllSTableIntervalAgg(void* param, bool* newgroup) { setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); int64_t st = taosGetTimestampUs(); - copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); +// copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { pOperator->status = OP_EXEC_DONE; } @@ -6439,7 +6855,7 @@ static SSDataBlock* doStateWindowAgg(void *param, bool* newgroup) { STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; if (pOperator->status == OP_RES_TO_RETURN) { - toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pBInfo->pRes); +// toSDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pBInfo->pRes); if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { pOperator->status = OP_EXEC_DONE; @@ -6477,7 +6893,7 @@ static SSDataBlock* doStateWindowAgg(void *param, bool* newgroup) { finalizeQueryResult(pOperator, pBInfo->pCtx, &pBInfo->resultRowInfo, pBInfo->rowCellInfoOffset); initGroupResInfo(&pRuntimeEnv->groupResInfo, &pBInfo->resultRowInfo); - toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pBInfo->pRes); +// toSDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pBInfo->pRes); if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { pOperator->status = OP_EXEC_DONE; @@ -6498,7 +6914,7 @@ static SSDataBlock* doSessionWindowAgg(void* param, bool* newgroup) { STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; if (pOperator->status == OP_RES_TO_RETURN) { - toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pBInfo->pRes); +// toSDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pBInfo->pRes); if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { pOperator->status = OP_EXEC_DONE; @@ -6537,7 +6953,7 @@ static SSDataBlock* doSessionWindowAgg(void* param, bool* newgroup) { finalizeQueryResult(pOperator, pBInfo->pCtx, &pBInfo->resultRowInfo, pBInfo->rowCellInfoOffset); initGroupResInfo(&pRuntimeEnv->groupResInfo, &pBInfo->resultRowInfo); - toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pBInfo->pRes); +// toSDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pBInfo->pRes); if (pBInfo->pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { pOperator->status = OP_EXEC_DONE; @@ -6556,7 +6972,7 @@ static SSDataBlock* hashGroupbyAggregate(void* param, bool* newgroup) { STaskRuntimeEnv* pRuntimeEnv = pOperator->pRuntimeEnv; if (pOperator->status == OP_RES_TO_RETURN) { - toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->binfo.pRes); +// toSDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->binfo.pRes); if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { pOperator->status = OP_EXEC_DONE; @@ -6577,7 +6993,7 @@ static SSDataBlock* hashGroupbyAggregate(void* param, bool* newgroup) { // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pInfo->binfo.pCtx, pBlock, pRuntimeEnv->pQueryAttr->order.order); - setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->binfo.pCtx, pOperator->numOfOutput); +// setTagValue(pOperator, pRuntimeEnv->current->pTable, pInfo->binfo.pCtx, pOperator->numOfOutput); if (pInfo->colIndex == -1) { pInfo->colIndex = getGroupbyColumnIndex(pRuntimeEnv->pQueryAttr->pGroupbyExpr, pBlock); } @@ -6592,7 +7008,7 @@ static SSDataBlock* hashGroupbyAggregate(void* param, bool* newgroup) { if (!pRuntimeEnv->pQueryAttr->stableQuery) { // finalize include the update of result rows finalizeQueryResult(pOperator, pInfo->binfo.pCtx, &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset); } else { - updateNumOfRowsInResultRows(pRuntimeEnv, pInfo->binfo.pCtx, pOperator->numOfOutput, &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset); + updateNumOfRowsInResultRows(pInfo->binfo.pCtx, pOperator->numOfOutput, &pInfo->binfo.resultRowInfo, pInfo->binfo.rowCellInfoOffset); } initGroupResInfo(&pRuntimeEnv->groupResInfo, &pInfo->binfo.resultRowInfo); @@ -6600,8 +7016,7 @@ static SSDataBlock* hashGroupbyAggregate(void* param, bool* newgroup) { sortGroupResByOrderList(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->binfo.pRes); } - toSSDataBlock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->binfo.pRes); - +// toSDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pInfo->binfo.pRes); if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainDataInCurrentGroup(&pRuntimeEnv->groupResInfo)) { pOperator->status = OP_EXEC_DONE; } @@ -6745,41 +7160,67 @@ static void destroyOperatorInfo(SOperatorInfo* pOperator) { tfree(pOperator); } -SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo) { - SAggOperatorInfo* pInfo = calloc(1, sizeof(SAggOperatorInfo)); - - int32_t numOfRows = 1;//(int32_t)(getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery)); - - size_t numOfOutput = taosArrayGetSize(pExprInfo); +static int32_t initAggInfo(SAggOperatorInfo* pInfo, SArray* pExprInfo, int32_t numOfRows, const STableGroupInfo* pTableGroupInfo) { pInfo->binfo.pRes = createOutputBuf_rv(pExprInfo, numOfRows); - pInfo->binfo.pCtx = createSqlFunctionCtx_rv(pExprInfo, &pInfo->binfo.rowCellInfoOffset); + pInfo->binfo.pCtx = createSqlFunctionCtx_rv(pExprInfo, &pInfo->binfo.rowCellInfoOffset, &pInfo->binfo.resRowSize); + pInfo->binfo.capacity = 4096; - pInfo->pResultRowHashTable = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); - pInfo->pResultRowListSet = taosHashInit(100, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - pInfo->keyBuf = malloc(1024 + sizeof(int64_t) + POINTER_BYTES); // TODO: - pInfo->pool = initResultRowPool(getResultRowSize(pExprInfo)); + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + + pInfo->pResultRowHashTable = taosHashInit(10, hashFn, true, HASH_NO_LOCK); + pInfo->pResultRowListSet = taosHashInit(100, hashFn, false, HASH_NO_LOCK); + pInfo->pool = initResultRowPool(getResultRowSize(pExprInfo)); pInfo->pResultRowArrayList = taosArrayInit(10, sizeof(SResultRowCell)); - initResultRowInfo(&pInfo->binfo.resultRowInfo, 8, TSDB_DATA_TYPE_INT); + pInfo->pTableQueryInfo = calloc(pTableGroupInfo->numOfTables, sizeof(STableQueryInfo)); - pInfo->seed = rand(); - setDefaultOutputBuf_rv(pInfo, pInfo->seed, MAIN_SCAN, pTaskInfo); + int32_t index = 0; + for(int32_t i = 0; i < taosArrayGetSize(pTableGroupInfo->pGroupList); ++i) { + SArray* pa = taosArrayGetP(pTableGroupInfo->pGroupList, i); + for(int32_t j = 0; j < taosArrayGetSize(pa); ++j) { + STableKeyInfo* pk = taosArrayGet(pa, j); + STableQueryInfo* pTQueryInfo = &pInfo->pTableQueryInfo[index++]; + pTQueryInfo->uid = pk->uid; + pTQueryInfo->lastKey = pk->lastKey; + pTQueryInfo->groupIndex = i; + } + } + + STimeWindow win = {0, INT64_MAX}; + createTableQueryInfo(pInfo->pTableQueryInfo, false, win); + + return TSDB_CODE_SUCCESS; +} + +static SExprInfo* exprArrayDup(SArray* pExprInfo) { + size_t numOfOutput = taosArrayGetSize(pExprInfo); SExprInfo* p = calloc(numOfOutput, sizeof(SExprInfo)); - for(int32_t i = 0; i < taosArrayGetSize(pExprInfo); ++i) { + for (int32_t i = 0; i < taosArrayGetSize(pExprInfo); ++i) { SExprInfo* pExpr = taosArrayGetP(pExprInfo, i); assignExprInfo(&p[i], pExpr); } + return p; +} + +SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo) { + SAggOperatorInfo* pInfo = calloc(1, sizeof(SAggOperatorInfo)); + + int32_t numOfRows = 1; + //(int32_t)(getRowNumForMultioutput(pQueryAttr, pQueryAttr->topBotQuery, pQueryAttr->stableQuery)); + + initAggInfo(pInfo, pExprInfo, numOfRows, pTableGroupInfo); + setDefaultOutputBuf_rv(pInfo, MAIN_SCAN, pTaskInfo); + SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "TableAggregate"; pOperator->operatorType = OP_Aggregate; pOperator->blockingOptr = true; pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; - pOperator->pExpr = p; - pOperator->numOfOutput = numOfOutput; - pOperator->pRuntimeEnv = NULL; + pOperator->pExpr = exprArrayDup(pExprInfo); + pOperator->numOfOutput = taosArrayGetSize(pExprInfo); pOperator->pTaskInfo = pTaskInfo; pOperator->exec = doAggregate; @@ -6796,7 +7237,7 @@ static void doDestroyBasicInfo(SOptrBasicInfo* pInfo, int32_t numOfOutput) { tfree(pInfo->rowCellInfoOffset); cleanupResultRowInfo(&pInfo->resultRowInfo); - pInfo->pRes = destroyOutputBuf(pInfo->pRes); + pInfo->pRes = blockDataDestroy(pInfo->pRes); } static void destroyBasicOperatorInfo(void* param, int32_t numOfOutput) { @@ -6820,7 +7261,7 @@ static void destroySWindowOperatorInfo(void* param, int32_t numOfOutput) { static void destroySFillOperatorInfo(void* param, int32_t numOfOutput) { SFillOperatorInfo* pInfo = (SFillOperatorInfo*) param; pInfo->pFillInfo = taosDestroyFillInfo(pInfo->pFillInfo); - pInfo->pRes = destroyOutputBuf(pInfo->pRes); + pInfo->pRes = blockDataDestroy(pInfo->pRes); tfree(pInfo->p); } @@ -6837,12 +7278,17 @@ static void destroyProjectOperatorInfo(void* param, int32_t numOfOutput) { static void destroyTagScanOperatorInfo(void* param, int32_t numOfOutput) { STagScanInfo* pInfo = (STagScanInfo*) param; - pInfo->pRes = destroyOutputBuf(pInfo->pRes); + pInfo->pRes = blockDataDestroy(pInfo->pRes); } static void destroyOrderOperatorInfo(void* param, int32_t numOfOutput) { SOrderOperatorInfo* pInfo = (SOrderOperatorInfo*) param; - pInfo->pDataBlock = destroyOutputBuf(pInfo->pDataBlock); + pInfo->pDataBlock = blockDataDestroy(pInfo->pDataBlock); + + taosArrayDestroy(pInfo->cmpParam.orderInfo); + destroyResultBuf(pInfo->pSortInternalBuf); + + tMergeTreeDestroy(pInfo->pMergeTree); } static void destroyConditionOperatorInfo(void* param, int32_t numOfOutput) { @@ -6855,29 +7301,29 @@ static void destroyDistinctOperatorInfo(void* param, int32_t numOfOutput) { taosHashCleanup(pInfo->pSet); tfree(pInfo->buf); taosArrayDestroy(pInfo->pDistinctDataInfo); - pInfo->pRes = destroyOutputBuf(pInfo->pRes); + pInfo->pRes = blockDataDestroy(pInfo->pRes); } -SOperatorInfo* createMultiTableAggOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* downstream, SExprInfo* pExpr, int32_t numOfOutput) { +SOperatorInfo* createMultiTableAggOperatorInfo(SOperatorInfo* downstream, SArray* pExprInfo, SExecTaskInfo* pTaskInfo, const STableGroupInfo* pTableGroupInfo) { SAggOperatorInfo* pInfo = calloc(1, sizeof(SAggOperatorInfo)); - size_t tableGroup = GET_NUM_OF_TABLEGROUP(pRuntimeEnv); + int32_t numOfRows = 1; + size_t numOfOutput = taosArrayGetSize(pExprInfo); + initAggInfo(pInfo, pExprInfo, numOfRows, pTableGroupInfo); - pInfo->binfo.pRes = createOutputBuf(pExpr, numOfOutput, (int32_t) tableGroup); - pInfo->binfo.pCtx = createSqlFunctionCtx(pRuntimeEnv, pExpr, numOfOutput, &pInfo->binfo.rowCellInfoOffset); + size_t tableGroup = taosArrayGetSize(pTableGroupInfo->pGroupList); initResultRowInfo(&pInfo->binfo.resultRowInfo, (int32_t)tableGroup, TSDB_DATA_TYPE_INT); SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "MultiTableAggregate"; -// pOperator->operatorType = OP_MultiTableAggregate; + pOperator->operatorType = OP_MultiTableAggregate; pOperator->blockingOptr = true; pOperator->status = OP_IN_EXECUTING; pOperator->info = pInfo; - pOperator->pExpr = pExpr; + pOperator->pExpr = exprArrayDup(pExprInfo); pOperator->numOfOutput = numOfOutput; - pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->exec = doSTableAggregate; + pOperator->exec = doMultiTableAggregate; pOperator->cleanupFn = destroyAggOperatorInfo; appendDownstream(pOperator, downstream); @@ -7719,17 +8165,19 @@ static SExecTaskInfo* createExecTaskInfo(uint64_t queryId, uint64_t taskId) { } static tsdbReaderT doCreateDataReader(STableScanPhyNode* pTableScanNode, SReadHandle* pHandle, uint64_t queryId, uint64_t taskId); + static int32_t doCreateTableGroup(void* metaHandle, int32_t tableType, uint64_t tableUid, STableGroupInfo* pGroupInfo, uint64_t queryId, uint64_t taskId); -SOperatorInfo* doCreateOperatorTreeNode(SPhyNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, uint64_t queryId, uint64_t taskId) { +SOperatorInfo* doCreateOperatorTreeNode(SPhyNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, uint64_t queryId, uint64_t taskId, STableGroupInfo* pTableGroupInfo) { if (pPhyNode->pChildren == NULL || taosArrayGetSize(pPhyNode->pChildren) == 0) { - if (pPhyNode->info.type == OP_DataBlocksOptScan) { + if (pPhyNode->info.type == OP_TableScan) { SScanPhyNode* pScanPhyNode = (SScanPhyNode*)pPhyNode; - size_t numOfCols = taosArrayGetSize(pPhyNode->pTargets); + size_t numOfCols = taosArrayGetSize(pPhyNode->pTargets); tsdbReaderT pDataReader = doCreateDataReader((STableScanPhyNode*) pPhyNode, pHandle, (uint64_t) queryId, taskId); - return createDataBlocksOptScanInfo(pDataReader, pScanPhyNode->order, numOfCols, pScanPhyNode->count, pScanPhyNode->reverse, pTaskInfo); + int32_t code = doCreateTableGroup(pHandle->meta, pScanPhyNode->tableType, pScanPhyNode->uid, pTableGroupInfo, queryId, taskId); + return createTableScanOperatorInfo(pDataReader, pScanPhyNode->order, numOfCols, pScanPhyNode->count, pScanPhyNode->reverse, pTaskInfo); } else if (pPhyNode->info.type == OP_Exchange) { SExchangePhyNode* pEx = (SExchangePhyNode*) pPhyNode; return createExchangeOperatorInfo(pEx->pSrcEndPoints, pEx->node.pTargets, pTaskInfo); @@ -7762,10 +8210,20 @@ SOperatorInfo* doCreateOperatorTreeNode(SPhyNode* pPhyNode, SExecTaskInfo* pTask size_t size = taosArrayGetSize(pPhyNode->pChildren); assert(size == 1); + // TODO single table agg for (int32_t i = 0; i < size; ++i) { SPhyNode* pChildNode = taosArrayGetP(pPhyNode->pChildren, i); - SOperatorInfo* op = doCreateOperatorTreeNode(pChildNode, pTaskInfo, pHandle, queryId, taskId); - return createAggregateOperatorInfo(op, pPhyNode->pTargets, pTaskInfo); + SOperatorInfo* op = doCreateOperatorTreeNode(pChildNode, pTaskInfo, pHandle, queryId, taskId, pTableGroupInfo); + return createAggregateOperatorInfo(op, pPhyNode->pTargets, pTaskInfo, pTableGroupInfo); + } + } else if (pPhyNode->info.type == OP_MultiTableAggregate) { + size_t size = taosArrayGetSize(pPhyNode->pChildren); + assert(size == 1); + + for (int32_t i = 0; i < size; ++i) { + SPhyNode* pChildNode = taosArrayGetP(pPhyNode->pChildren, i); + SOperatorInfo* op = doCreateOperatorTreeNode(pChildNode, pTaskInfo, pHandle, queryId, taskId, pTableGroupInfo); + return createMultiTableAggOperatorInfo(op, pPhyNode->pTargets, pTaskInfo, pTableGroupInfo); } } } @@ -7840,7 +8298,8 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead goto _complete; } - (*pTaskInfo)->pRoot = doCreateOperatorTreeNode(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId); + STableGroupInfo group = {0}; + (*pTaskInfo)->pRoot = doCreateOperatorTreeNode(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId, &group); if ((*pTaskInfo)->pRoot == NULL) { code = TSDB_CODE_QRY_OUT_OF_MEMORY; goto _complete; @@ -7855,367 +8314,6 @@ _complete: return code; } -/** - * pQueryMsg->head has been converted before this function is called. - * - * @param pQueryMsg - * @param pTableIdList - * @param pExpr - * @return - */ -//int32_t convertQueryMsg(SQueryTableReq *pQueryMsg, STaskParam* param) { -// int32_t code = TSDB_CODE_SUCCESS; -// -//// if (taosCheckVersion(pQueryMsg->version, version, 3) != 0) { -//// return TSDB_CODE_QRY_INVALID_MSG; -//// } -// -// pQueryMsg->numOfTables = htonl(pQueryMsg->numOfTables); -// pQueryMsg->window.skey = htobe64(pQueryMsg->window.skey); -// pQueryMsg->window.ekey = htobe64(pQueryMsg->window.ekey); -// pQueryMsg->interval.interval = htobe64(pQueryMsg->interval.interval); -// pQueryMsg->interval.sliding = htobe64(pQueryMsg->interval.sliding); -// pQueryMsg->interval.offset = htobe64(pQueryMsg->interval.offset); -// pQueryMsg->limit = htobe64(pQueryMsg->limit); -// pQueryMsg->offset = htobe64(pQueryMsg->offset); -// pQueryMsg->vgroupLimit = htobe64(pQueryMsg->vgroupLimit); -// -// pQueryMsg->order = htons(pQueryMsg->order); -// pQueryMsg->orderColId = htons(pQueryMsg->orderColId); -// pQueryMsg->queryType = htonl(pQueryMsg->queryType); -//// pQueryMsg->tagNameRelType = htons(pQueryMsg->tagNameRelType); -// -// pQueryMsg->numOfCols = htons(pQueryMsg->numOfCols); -// pQueryMsg->numOfOutput = htons(pQueryMsg->numOfOutput); -// pQueryMsg->numOfGroupCols = htons(pQueryMsg->numOfGroupCols); -// -// pQueryMsg->tagCondLen = htons(pQueryMsg->tagCondLen); -// pQueryMsg->colCondLen = htons(pQueryMsg->colCondLen); -// -// pQueryMsg->tsBuf.tsOffset = htonl(pQueryMsg->tsBuf.tsOffset); -// pQueryMsg->tsBuf.tsLen = htonl(pQueryMsg->tsBuf.tsLen); -// pQueryMsg->tsBuf.tsNumOfBlocks = htonl(pQueryMsg->tsBuf.tsNumOfBlocks); -// pQueryMsg->tsBuf.tsOrder = htonl(pQueryMsg->tsBuf.tsOrder); -// -// pQueryMsg->numOfTags = htonl(pQueryMsg->numOfTags); -//// pQueryMsg->tbnameCondLen = htonl(pQueryMsg->tbnameCondLen); -// pQueryMsg->secondStageOutput = htonl(pQueryMsg->secondStageOutput); -// pQueryMsg->sqlstrLen = htonl(pQueryMsg->sqlstrLen); -// pQueryMsg->prevResultLen = htonl(pQueryMsg->prevResultLen); -//// pQueryMsg->sw.gap = htobe64(pQueryMsg->sw.gap); -//// pQueryMsg->sw.primaryColId = htonl(pQueryMsg->sw.primaryColId); -// pQueryMsg->tableScanOperator = htonl(pQueryMsg->tableScanOperator); -// pQueryMsg->numOfOperator = htonl(pQueryMsg->numOfOperator); -// pQueryMsg->udfContentOffset = htonl(pQueryMsg->udfContentOffset); -// pQueryMsg->udfContentLen = htonl(pQueryMsg->udfContentLen); -// pQueryMsg->udfNum = htonl(pQueryMsg->udfNum); -// -// // query msg safety check -// if (!validateQueryMsg(pQueryMsg)) { -// code = TSDB_CODE_QRY_INVALID_MSG; -// goto _cleanup; -// } -// -// char *pMsg = (char *)(pQueryMsg->tableCols) + sizeof(SColumnInfo) * pQueryMsg->numOfCols; -// for (int32_t col = 0; col < pQueryMsg->numOfCols; ++col) { -// SColumnInfo *pColInfo = &pQueryMsg->tableCols[col]; -// -// pColInfo->colId = htons(pColInfo->colId); -// pColInfo->type = htons(pColInfo->type); -// pColInfo->bytes = htons(pColInfo->bytes); -// pColInfo->flist.numOfFilters = 0; -// -// if (!isValidDataType(pColInfo->type)) { -// //qDebug("qmsg:%p, invalid data type in source column, index:%d, type:%d", pQueryMsg, col, pColInfo->type); -// code = TSDB_CODE_QRY_INVALID_MSG; -// goto _cleanup; -// } -// -///* -// int32_t numOfFilters = pColInfo->flist.numOfFilters; -// if (numOfFilters > 0) { -// pColInfo->flist.filterInfo = calloc(numOfFilters, sizeof(SColumnFilterInfo)); -// if (pColInfo->flist.filterInfo == NULL) { -// code = TSDB_CODE_QRY_OUT_OF_MEMORY; -// goto _cleanup; -// } -// } -// -// code = deserializeColFilterInfo(pColInfo->flist.filterInfo, numOfFilters, &pMsg); -// if (code != TSDB_CODE_SUCCESS) { -// goto _cleanup; -// } -//*/ -// } -// -// if (pQueryMsg->colCondLen > 0) { -// param->colCond = calloc(1, pQueryMsg->colCondLen); -// if (param->colCond == NULL) { -// code = TSDB_CODE_QRY_OUT_OF_MEMORY; -// goto _cleanup; -// } -// -// memcpy(param->colCond, pMsg, pQueryMsg->colCondLen); -// pMsg += pQueryMsg->colCondLen; -// } -// -// -// param->tableScanOperator = pQueryMsg->tableScanOperator; -// param->pExpr = calloc(pQueryMsg->numOfOutput, POINTER_BYTES); -// if (param->pExpr == NULL) { -// code = TSDB_CODE_QRY_OUT_OF_MEMORY; -// goto _cleanup; -// } -// -// SSqlExpr *pExprMsg = (SSqlExpr *)pMsg; -// -// for (int32_t i = 0; i < pQueryMsg->numOfOutput; ++i) { -// param->pExpr[i] = pExprMsg; -// -//// pExprMsg->colInfo.colIndex = htons(pExprMsg->colInfo.colIndex); -//// pExprMsg->colInfo.colId = htons(pExprMsg->colInfo.colId); -//// pExprMsg->colInfo.flag = htons(pExprMsg->colInfo.flag); -//// pExprMsg->colBytes = htons(pExprMsg->colBytes); -//// pExprMsg->colType = htons(pExprMsg->colType); -// -//// pExprMsg->resType = htons(pExprMsg->resType); -//// pExprMsg->resBytes = htons(pExprMsg->resBytes); -// pExprMsg->interBytes = htonl(pExprMsg->interBytes); -// -//// pExprMsg->functionId = htons(pExprMsg->functionId); -// pExprMsg->numOfParams = htons(pExprMsg->numOfParams); -//// pExprMsg->resColId = htons(pExprMsg->resColId); -//// pExprMsg->flist.numOfFilters = htons(pExprMsg->flist.numOfFilters); -// pMsg += sizeof(SSqlExpr); -// -// for (int32_t j = 0; j < pExprMsg->numOfParams; ++j) { -// pExprMsg->param[j].nType = htonl(pExprMsg->param[j].nType); -// pExprMsg->param[j].nLen = htonl(pExprMsg->param[j].nLen); -// -// if (pExprMsg->param[j].nType == TSDB_DATA_TYPE_BINARY) { -// pExprMsg->param[j].pz = pMsg; -// pMsg += pExprMsg->param[j].nLen; // one more for the string terminated char. -// } else { -// pExprMsg->param[j].i = htobe64(pExprMsg->param[j].i); -// } -// } -// -//// int16_t functionId = pExprMsg->functionId; -//// if (functionId == FUNCTION_TAG || functionId == FUNCTION_TAGPRJ || functionId == FUNCTION_TAG_DUMMY) { -//// if (!TSDB_COL_IS_TAG(pExprMsg->colInfo.flag)) { // ignore the column index check for arithmetic expression. -//// code = TSDB_CODE_QRY_INVALID_MSG; -//// goto _cleanup; -//// } -//// } -// -//// if (pExprMsg->flist.numOfFilters > 0) { -//// pExprMsg->flist.filterInfo = calloc(pExprMsg->flist.numOfFilters, sizeof(SColumnFilterInfo)); -//// } -//// -//// deserializeColFilterInfo(pExprMsg->flist.filterInfo, pExprMsg->flist.numOfFilters, &pMsg); -// pExprMsg = (SSqlExpr *)pMsg; -// } -// -// if (pQueryMsg->secondStageOutput) { -// pExprMsg = (SSqlExpr *)pMsg; -// param->pSecExpr = calloc(pQueryMsg->secondStageOutput, POINTER_BYTES); -// -// for (int32_t i = 0; i < pQueryMsg->secondStageOutput; ++i) { -// param->pSecExpr[i] = pExprMsg; -// -//// pExprMsg->colInfo.colIndex = htons(pExprMsg->colInfo.colIndex); -//// pExprMsg->colInfo.colId = htons(pExprMsg->colInfo.colId); -//// pExprMsg->colInfo.flag = htons(pExprMsg->colInfo.flag); -//// pExprMsg->resType = htons(pExprMsg->resType); -//// pExprMsg->resBytes = htons(pExprMsg->resBytes); -//// pExprMsg->colBytes = htons(pExprMsg->colBytes); -//// pExprMsg->colType = htons(pExprMsg->colType); -// -//// pExprMsg->functionId = htons(pExprMsg->functionId); -// pExprMsg->numOfParams = htons(pExprMsg->numOfParams); -// -// pMsg += sizeof(SSqlExpr); -// -// for (int32_t j = 0; j < pExprMsg->numOfParams; ++j) { -// pExprMsg->param[j].nType = htonl(pExprMsg->param[j].nType); -// pExprMsg->param[j].nLen = htonl(pExprMsg->param[j].nLen); -// -// if (pExprMsg->param[j].nType == TSDB_DATA_TYPE_BINARY) { -// pExprMsg->param[j].pz = pMsg; -// pMsg += pExprMsg->param[j].nLen; // one more for the string terminated char. -// } else { -// pExprMsg->param[j].i = htobe64(pExprMsg->param[j].i); -// } -// } -// -//// int16_t functionId = pExprMsg->functionId; -//// if (functionId == FUNCTION_TAG || functionId == FUNCTION_TAGPRJ || functionId == FUNCTION_TAG_DUMMY) { -//// if (!TSDB_COL_IS_TAG(pExprMsg->colInfo.flag)) { // ignore the column index check for arithmetic expression. -//// code = TSDB_CODE_QRY_INVALID_MSG; -//// goto _cleanup; -//// } -//// } -// -// pExprMsg = (SSqlExpr *)pMsg; -// } -// } -// -// pMsg = createTableIdList(pQueryMsg, pMsg, &(param->pTableIdList)); -// -// if (pQueryMsg->numOfGroupCols > 0) { // group by tag columns -// param->pGroupColIndex = malloc(pQueryMsg->numOfGroupCols * sizeof(SColIndex)); -// if (param->pGroupColIndex == NULL) { -// code = TSDB_CODE_QRY_OUT_OF_MEMORY; -// goto _cleanup; -// } -// -// for (int32_t i = 0; i < pQueryMsg->numOfGroupCols; ++i) { -// param->pGroupColIndex[i].colId = htons(*(int16_t *)pMsg); -// pMsg += sizeof(param->pGroupColIndex[i].colId); -// -// param->pGroupColIndex[i].colIndex = htons(*(int16_t *)pMsg); -// pMsg += sizeof(param->pGroupColIndex[i].colIndex); -// -// param->pGroupColIndex[i].flag = htons(*(int16_t *)pMsg); -// pMsg += sizeof(param->pGroupColIndex[i].flag); -// -// memcpy(param->pGroupColIndex[i].name, pMsg, tListLen(param->pGroupColIndex[i].name)); -// pMsg += tListLen(param->pGroupColIndex[i].name); -// } -// -// pQueryMsg->orderByIdx = htons(pQueryMsg->orderByIdx); -// pQueryMsg->orderType = htons(pQueryMsg->orderType); -// } -// -// pQueryMsg->fillType = htons(pQueryMsg->fillType); -// if (pQueryMsg->fillType != TSDB_FILL_NONE) { -// pQueryMsg->fillVal = (uint64_t)(pMsg); -// -// int64_t *v = (int64_t *)pMsg; -// for (int32_t i = 0; i < pQueryMsg->numOfOutput; ++i) { -// v[i] = htobe64(v[i]); -// } -// -// pMsg += sizeof(int64_t) * pQueryMsg->numOfOutput; -// } -// -// if (pQueryMsg->numOfTags > 0) { -// param->pTagColumnInfo = calloc(1, sizeof(SColumnInfo) * pQueryMsg->numOfTags); -// if (param->pTagColumnInfo == NULL) { -// code = TSDB_CODE_QRY_OUT_OF_MEMORY; -// goto _cleanup; -// } -// -// for (int32_t i = 0; i < pQueryMsg->numOfTags; ++i) { -// SColumnInfo* pTagCol = (SColumnInfo*) pMsg; -// -// pTagCol->colId = htons(pTagCol->colId); -// pTagCol->bytes = htons(pTagCol->bytes); -// pTagCol->type = htons(pTagCol->type); -//// pTagCol->flist.numOfFilters = 0; -// -// param->pTagColumnInfo[i] = *pTagCol; -// pMsg += sizeof(SColumnInfo); -// } -// } -// -// // the tag query condition expression string is located at the end of query msg -// if (pQueryMsg->tagCondLen > 0) { -// param->tagCond = calloc(1, pQueryMsg->tagCondLen); -// if (param->tagCond == NULL) { -// code = TSDB_CODE_QRY_OUT_OF_MEMORY; -// goto _cleanup; -// } -// -// memcpy(param->tagCond, pMsg, pQueryMsg->tagCondLen); -// pMsg += pQueryMsg->tagCondLen; -// } -// -// if (pQueryMsg->prevResultLen > 0) { -// param->prevResult = calloc(1, pQueryMsg->prevResultLen); -// if (param->prevResult == NULL) { -// code = TSDB_CODE_QRY_OUT_OF_MEMORY; -// goto _cleanup; -// } -// -// memcpy(param->prevResult, pMsg, pQueryMsg->prevResultLen); -// pMsg += pQueryMsg->prevResultLen; -// } -// -//// if (pQueryMsg->tbnameCondLen > 0) { -//// param->tbnameCond = calloc(1, pQueryMsg->tbnameCondLen + 1); -//// if (param->tbnameCond == NULL) { -//// code = TSDB_CODE_QRY_OUT_OF_MEMORY; -//// goto _cleanup; -//// } -//// -//// strncpy(param->tbnameCond, pMsg, pQueryMsg->tbnameCondLen); -//// pMsg += pQueryMsg->tbnameCondLen; -//// } -// -// //skip ts buf -// if ((pQueryMsg->tsBuf.tsOffset + pQueryMsg->tsBuf.tsLen) > 0) { -// pMsg = (char *)pQueryMsg + pQueryMsg->tsBuf.tsOffset + pQueryMsg->tsBuf.tsLen; -// } -// -// param->pOperator = taosArrayInit(pQueryMsg->numOfOperator, sizeof(int32_t)); -// for(int32_t i = 0; i < pQueryMsg->numOfOperator; ++i) { -// int32_t op = htonl(*(int32_t*)pMsg); -// taosArrayPush(param->pOperator, &op); -// -// pMsg += sizeof(int32_t); -// } -// -// if (pQueryMsg->udfContentLen > 0) { -// // todo extract udf function in tudf.c -//// param->pUdfInfo = calloc(1, sizeof(SUdfInfo)); -//// param->pUdfInfo->contLen = pQueryMsg->udfContentLen; -//// -//// pMsg = (char*)pQueryMsg + pQueryMsg->udfContentOffset; -//// param->pUdfInfo->resType = *(int8_t*) pMsg; -//// pMsg += sizeof(int8_t); -//// -//// param->pUdfInfo->resBytes = htons(*(int16_t*)pMsg); -//// pMsg += sizeof(int16_t); -//// -//// tstr* name = (tstr*)(pMsg); -//// param->pUdfInfo->name = strndup(name->data, name->len); -//// -//// pMsg += varDataTLen(name); -//// param->pUdfInfo->funcType = htonl(*(int32_t*)pMsg); -//// pMsg += sizeof(int32_t); -//// -//// param->pUdfInfo->bufSize = htonl(*(int32_t*)pMsg); -//// pMsg += sizeof(int32_t); -//// -//// param->pUdfInfo->content = malloc(pQueryMsg->udfContentLen); -//// memcpy(param->pUdfInfo->content, pMsg, pQueryMsg->udfContentLen); -// -// pMsg += pQueryMsg->udfContentLen; -// } -// -// param->sql = strndup(pMsg, pQueryMsg->sqlstrLen); -// -// SQueriedTableInfo info = { .numOfTags = pQueryMsg->numOfTags, .numOfCols = pQueryMsg->numOfCols, .colList = pQueryMsg->tableCols}; -// if (!validateQueryTableCols(&info, param->pExpr, pQueryMsg->numOfOutput, param->pTagColumnInfo, pQueryMsg)) { -// code = TSDB_CODE_QRY_INVALID_MSG; -// goto _cleanup; -// } -// -// //qDebug("qmsg:%p query %d tables, type:%d, qrange:%" PRId64 "-%" PRId64 ", numOfGroupbyTagCols:%d, order:%d, " -//// "outputCols:%d, numOfCols:%d, interval:%" PRId64 ", fillType:%d, comptsLen:%d, compNumOfBlocks:%d, limit:%" PRId64 ", offset:%" PRId64, -//// pQueryMsg, pQueryMsg->numOfTables, pQueryMsg->queryType, pQueryMsg->window.skey, pQueryMsg->window.ekey, pQueryMsg->numOfGroupCols, -//// pQueryMsg->order, pQueryMsg->numOfOutput, pQueryMsg->numOfCols, pQueryMsg->interval.interval, -//// pQueryMsg->fillType, pQueryMsg->tsBuf.tsLen, pQueryMsg->tsBuf.tsNumOfBlocks, pQueryMsg->limit, pQueryMsg->offset); -// -// //qDebug("qmsg:%p, sql:%s", pQueryMsg, param->sql); -// return TSDB_CODE_SUCCESS; -// -//_cleanup: -// freeParam(param); -// return code; -//} - int32_t cloneExprFilterInfo(SColumnFilterInfo **dst, SColumnFilterInfo* src, int32_t filterNum) { if (filterNum <= 0) { return TSDB_CODE_SUCCESS; diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index 8381a7c585..ebea6755d7 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -33,193 +33,312 @@ #include "stub.h" #include "executor.h" -/** -{ - "Id": { - "QueryId": 1.3108161807422521e+19, - "TemplateId": 0, - "SubplanId": 0 - }, - "Node": { - "Name": "TableScan", - "Targets": [{ - "Base": { - "Schema": { - "Type": 9, - "ColId": 5000, - "Bytes": 8 - }, - "Columns": [{ - "TableId": 1, - "Flag": 0, - "Info": { - "ColId": 1, - "Type": 9, - "Bytes": 8 - } - }], - "InterBytes": 0 - }, - "Expr": { - "Type": 4, - "Column": { - "Type": 9, - "ColId": 1, - "Bytes": 8 - } - } - }, { - "Base": { - "Schema": { - "Type": 4, - "ColId": 5001, - "Bytes": 4 - }, - "Columns": [{ - "TableId": 1, - "Flag": 0, - "Info": { - "ColId": 2, - "Type": 4, - "Bytes": 4 - } - }], - "InterBytes": 0 - }, - "Expr": { - "Type": 4, - "Column": { - "Type": 4, - "ColId": 2, - "Bytes": 4 - } - } - }], - "InputSchema": [{ - "Type": 9, - "ColId": 5000, - "Bytes": 8 - }, { - "Type": 4, - "ColId": 5001, - "Bytes": 4 - }], - "TableScan": { - "TableId": 1, - "TableType": 2, - "Flag": 0, - "Window": { - "StartKey": -9.2233720368547758e+18, - "EndKey": 9.2233720368547758e+18 - } - } - }, - "DataSink": { - "Name": "Dispatch", - "Dispatch": { - } - } -} - */ +namespace { +typedef struct SDummyInputInfo { + int32_t max; + int32_t current; + int32_t startVal; + SSDataBlock* pBlock; +} SDummyInputInfo; + +SSDataBlock* getDummyBlock(void* param, bool* newgroup) { + SOperatorInfo* pOperator = static_cast(param); + SDummyInputInfo* pInfo = static_cast(pOperator->info); + if (pInfo->current >= pInfo->max) { + return NULL; + } + + int32_t numOfRows = 1000; + + if (pInfo->pBlock == NULL) { + pInfo->pBlock = static_cast(calloc(1, sizeof(SSDataBlock))); + + pInfo->pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); + + SColumnInfoData colInfo = {0}; + colInfo.info.type = TSDB_DATA_TYPE_INT; + colInfo.info.bytes = sizeof(int32_t); + colInfo.info.colId = 1; + colInfo.pData = static_cast(calloc(numOfRows, sizeof(int32_t))); + colInfo.nullbitmap = static_cast(calloc(1, (numOfRows + 7) / 8)); + + taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo); + +// SColumnInfoData colInfo1 = {0}; +// colInfo1.info.type = TSDB_DATA_TYPE_BINARY; +// colInfo1.info.bytes = 40; +// colInfo1.info.colId = 2; +// +// colInfo1.varmeta.allocLen = 0;//numOfRows * sizeof(int32_t); +// colInfo1.varmeta.length = 0; +// colInfo1.varmeta.offset = static_cast(calloc(1, numOfRows * sizeof(int32_t))); +// +// taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo1); + } else { + blockDataClearup(pInfo->pBlock, true); + } + + SSDataBlock* pBlock = pInfo->pBlock; + + char buf[128] = {0}; + char b1[128] = {0}; + for(int32_t i = 0; i < numOfRows; ++i) { + SColumnInfoData* pColInfo = static_cast(TARRAY_GET_ELEM(pBlock->pDataBlock, 0)); + + int32_t v = (--pInfo->startVal); + colDataAppend(pColInfo, i, reinterpret_cast(&v), false); + +// sprintf(buf, "this is %d row", i); +// STR_TO_VARSTR(b1, buf); +// +// SColumnInfoData* pColInfo2 = static_cast(TARRAY_GET_ELEM(pBlock->pDataBlock, 1)); +// colDataAppend(pColInfo2, i, b1, false); + } + + pBlock->info.rows = numOfRows; + pBlock->info.numOfCols = 1; + + pInfo->current += 1; + return pBlock; +} + +SOperatorInfo* createDummyOperator(int32_t numOfBlocks) { + SOperatorInfo* pOperator = static_cast(calloc(1, sizeof(SOperatorInfo))); + pOperator->name = "dummyInputOpertor4Test"; + pOperator->exec = getDummyBlock; + + SDummyInputInfo *pInfo = (SDummyInputInfo*) calloc(1, sizeof(SDummyInputInfo)); + pInfo->max = numOfBlocks; + pInfo->startVal = 1500000; + + pOperator->info = pInfo; + return pOperator; +} +} int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } TEST(testCase, build_executor_tree_Test) { - - const char* msg = "{\n" - "\t\"Id\":\t{\n" - "\t\t\"QueryId\":\t1.3108161807422521e+19,\n" - "\t\t\"TemplateId\":\t0,\n" - "\t\t\"SubplanId\":\t0\n" - "\t},\n" - "\t\"Node\":\t{\n" - "\t\t\"Name\":\t\"TableScan\",\n" - "\t\t\"Targets\":\t[{\n" - "\t\t\t\t\"Base\":\t{\n" - "\t\t\t\t\t\"Schema\":\t{\n" - "\t\t\t\t\t\t\"Type\":\t9,\n" - "\t\t\t\t\t\t\"ColId\":\t5000,\n" - "\t\t\t\t\t\t\"Bytes\":\t8\n" - "\t\t\t\t\t},\n" - "\t\t\t\t\t\"Columns\":\t[{\n" - "\t\t\t\t\t\t\t\"TableId\":\t1,\n" - "\t\t\t\t\t\t\t\"Flag\":\t0,\n" - "\t\t\t\t\t\t\t\"Info\":\t{\n" - "\t\t\t\t\t\t\t\t\"ColId\":\t1,\n" - "\t\t\t\t\t\t\t\t\"Type\":\t9,\n" - "\t\t\t\t\t\t\t\t\"Bytes\":\t8\n" - "\t\t\t\t\t\t\t}\n" - "\t\t\t\t\t\t}],\n" - "\t\t\t\t\t\"InterBytes\":\t0\n" - "\t\t\t\t},\n" - "\t\t\t\t\"Expr\":\t{\n" - "\t\t\t\t\t\"Type\":\t4,\n" - "\t\t\t\t\t\"Column\":\t{\n" - "\t\t\t\t\t\t\"Type\":\t9,\n" - "\t\t\t\t\t\t\"ColId\":\t1,\n" - "\t\t\t\t\t\t\"Bytes\":\t8\n" - "\t\t\t\t\t}\n" - "\t\t\t\t}\n" - "\t\t\t}, {\n" - "\t\t\t\t\"Base\":\t{\n" - "\t\t\t\t\t\"Schema\":\t{\n" - "\t\t\t\t\t\t\"Type\":\t4,\n" - "\t\t\t\t\t\t\"ColId\":\t5001,\n" - "\t\t\t\t\t\t\"Bytes\":\t4\n" - "\t\t\t\t\t},\n" - "\t\t\t\t\t\"Columns\":\t[{\n" - "\t\t\t\t\t\t\t\"TableId\":\t1,\n" - "\t\t\t\t\t\t\t\"Flag\":\t0,\n" - "\t\t\t\t\t\t\t\"Info\":\t{\n" - "\t\t\t\t\t\t\t\t\"ColId\":\t2,\n" - "\t\t\t\t\t\t\t\t\"Type\":\t4,\n" - "\t\t\t\t\t\t\t\t\"Bytes\":\t4\n" - "\t\t\t\t\t\t\t}\n" - "\t\t\t\t\t\t}],\n" - "\t\t\t\t\t\"InterBytes\":\t0\n" - "\t\t\t\t},\n" - "\t\t\t\t\"Expr\":\t{\n" - "\t\t\t\t\t\"Type\":\t4,\n" - "\t\t\t\t\t\"Column\":\t{\n" - "\t\t\t\t\t\t\"Type\":\t4,\n" - "\t\t\t\t\t\t\"ColId\":\t2,\n" - "\t\t\t\t\t\t\"Bytes\":\t4\n" - "\t\t\t\t\t}\n" - "\t\t\t\t}\n" - "\t\t\t}],\n" - "\t\t\"InputSchema\":\t[{\n" - "\t\t\t\t\"Type\":\t9,\n" - "\t\t\t\t\"ColId\":\t5000,\n" - "\t\t\t\t\"Bytes\":\t8\n" - "\t\t\t}, {\n" - "\t\t\t\t\"Type\":\t4,\n" - "\t\t\t\t\"ColId\":\t5001,\n" - "\t\t\t\t\"Bytes\":\t4\n" - "\t\t\t}],\n" - "\t\t\"TableScan\":\t{\n" - "\t\t\t\"TableId\":\t1,\n" - "\t\t\t\"TableType\":\t2,\n" - "\t\t\t\"Flag\":\t0,\n" - "\t\t\t\"Window\":\t{\n" - "\t\t\t\t\"StartKey\":\t-9.2233720368547758e+18,\n" - "\t\t\t\t\"EndKey\":\t9.2233720368547758e+18\n" - "\t\t\t}\n" - "\t\t}\n" - "\t},\n" - "\t\"DataSink\":\t{\n" - "\t\t\"Name\":\t\"Dispatch\",\n" - "\t\t\"Dispatch\":\t{\n" - "\t\t}\n" - "\t}\n" - "}"; + "\t\"Id\":\t{\n" + "\t\t\"QueryId\":\t1.3108161807422521e+19,\n" + "\t\t\"TemplateId\":\t0,\n" + "\t\t\"SubplanId\":\t0\n" + "\t},\n" + "\t\"Node\":\t{\n" + "\t\t\"Name\":\t\"TableScan\",\n" + "\t\t\"Targets\":\t[{\n" + "\t\t\t\t\"Base\":\t{\n" + "\t\t\t\t\t\"Schema\":\t{\n" + "\t\t\t\t\t\t\"Type\":\t9,\n" + "\t\t\t\t\t\t\"ColId\":\t5000,\n" + "\t\t\t\t\t\t\"Bytes\":\t8\n" + "\t\t\t\t\t},\n" + "\t\t\t\t\t\"Columns\":\t[{\n" + "\t\t\t\t\t\t\t\"TableId\":\t1,\n" + "\t\t\t\t\t\t\t\"Flag\":\t0,\n" + "\t\t\t\t\t\t\t\"Info\":\t{\n" + "\t\t\t\t\t\t\t\t\"ColId\":\t1,\n" + "\t\t\t\t\t\t\t\t\"Type\":\t9,\n" + "\t\t\t\t\t\t\t\t\"Bytes\":\t8\n" + "\t\t\t\t\t\t\t}\n" + "\t\t\t\t\t\t}],\n" + "\t\t\t\t\t\"InterBytes\":\t0\n" + "\t\t\t\t},\n" + "\t\t\t\t\"Expr\":\t{\n" + "\t\t\t\t\t\"Type\":\t4,\n" + "\t\t\t\t\t\"Column\":\t{\n" + "\t\t\t\t\t\t\"Type\":\t9,\n" + "\t\t\t\t\t\t\"ColId\":\t1,\n" + "\t\t\t\t\t\t\"Bytes\":\t8\n" + "\t\t\t\t\t}\n" + "\t\t\t\t}\n" + "\t\t\t}, {\n" + "\t\t\t\t\"Base\":\t{\n" + "\t\t\t\t\t\"Schema\":\t{\n" + "\t\t\t\t\t\t\"Type\":\t4,\n" + "\t\t\t\t\t\t\"ColId\":\t5001,\n" + "\t\t\t\t\t\t\"Bytes\":\t4\n" + "\t\t\t\t\t},\n" + "\t\t\t\t\t\"Columns\":\t[{\n" + "\t\t\t\t\t\t\t\"TableId\":\t1,\n" + "\t\t\t\t\t\t\t\"Flag\":\t0,\n" + "\t\t\t\t\t\t\t\"Info\":\t{\n" + "\t\t\t\t\t\t\t\t\"ColId\":\t2,\n" + "\t\t\t\t\t\t\t\t\"Type\":\t4,\n" + "\t\t\t\t\t\t\t\t\"Bytes\":\t4\n" + "\t\t\t\t\t\t\t}\n" + "\t\t\t\t\t\t}],\n" + "\t\t\t\t\t\"InterBytes\":\t0\n" + "\t\t\t\t},\n" + "\t\t\t\t\"Expr\":\t{\n" + "\t\t\t\t\t\"Type\":\t4,\n" + "\t\t\t\t\t\"Column\":\t{\n" + "\t\t\t\t\t\t\"Type\":\t4,\n" + "\t\t\t\t\t\t\"ColId\":\t2,\n" + "\t\t\t\t\t\t\"Bytes\":\t4\n" + "\t\t\t\t\t}\n" + "\t\t\t\t}\n" + "\t\t\t}],\n" + "\t\t\"InputSchema\":\t[{\n" + "\t\t\t\t\"Type\":\t9,\n" + "\t\t\t\t\"ColId\":\t5000,\n" + "\t\t\t\t\"Bytes\":\t8\n" + "\t\t\t}, {\n" + "\t\t\t\t\"Type\":\t4,\n" + "\t\t\t\t\"ColId\":\t5001,\n" + "\t\t\t\t\"Bytes\":\t4\n" + "\t\t\t}],\n" + "\t\t\"TableScan\":\t{\n" + "\t\t\t\"TableId\":\t1,\n" + "\t\t\t\"TableType\":\t2,\n" + "\t\t\t\"Flag\":\t0,\n" + "\t\t\t\"Window\":\t{\n" + "\t\t\t\t\"StartKey\":\t-9.2233720368547758e+18,\n" + "\t\t\t\t\"EndKey\":\t9.2233720368547758e+18\n" + "\t\t\t}\n" + "\t\t}\n" + "\t},\n" + "\t\"DataSink\":\t{\n" + "\t\t\"Name\":\t\"Dispatch\",\n" + "\t\t\"Dispatch\":\t{\n" + "\t\t}\n" + "\t}\n" + "}"; SExecTaskInfo* pTaskInfo = nullptr; DataSinkHandle sinkHandle = nullptr; - int32_t code = qCreateExecTask((SReadHandle*) 1, 2, 1, NULL, (void**) &pTaskInfo, &sinkHandle); + SReadHandle handle = {.reader = reinterpret_cast(0x1), .meta = reinterpret_cast(0x1)}; + +// int32_t code = qCreateExecTask(&handle, 2, 1, NULL, (void**) &pTaskInfo, &sinkHandle); } +//TEST(testCase, inMem_sort_Test) { +// SArray* pOrderVal = taosArrayInit(4, sizeof(SOrder)); +// SOrder o = {.order = TSDB_ORDER_ASC}; +// o.col.info.colId = 1; +// o.col.info.type = TSDB_DATA_TYPE_INT; +// taosArrayPush(pOrderVal, &o); +// +// SArray* pExprInfo = taosArrayInit(4, sizeof(SExprInfo)); +// SExprInfo *exp = static_cast(calloc(1, sizeof(SExprInfo))); +// exp->base.resSchema = createSchema(TSDB_DATA_TYPE_INT, sizeof(int32_t), 1, "res"); +// taosArrayPush(pExprInfo, &exp); +// +// SExprInfo *exp1 = static_cast(calloc(1, sizeof(SExprInfo))); +// exp1->base.resSchema = createSchema(TSDB_DATA_TYPE_BINARY, 40, 2, "res1"); +// taosArrayPush(pExprInfo, &exp1); +// +// SOperatorInfo* pOperator = createOrderOperatorInfo(createDummyOperator(5), pExprInfo, pOrderVal); +// +// bool newgroup = false; +// SSDataBlock* pRes = pOperator->exec(pOperator, &newgroup); +// +// SColumnInfoData* pCol1 = static_cast(taosArrayGet(pRes->pDataBlock, 0)); +// SColumnInfoData* pCol2 = static_cast(taosArrayGet(pRes->pDataBlock, 1)); +// for(int32_t i = 0; i < pRes->info.rows; ++i) { +// char* p = colDataGet(pCol2, i); +// printf("%d: %d, %s\n", i, ((int32_t*)pCol1->pData)[i], (char*)varDataVal(p)); +// } +//} + +typedef struct su { + int32_t v; + char *c; +} su; + +int32_t cmp(const void* p1, const void* p2) { + su* v1 = (su*) p1; + su* v2 = (su*) p2; + + int32_t x1 = *(int32_t*) v1->c; + int32_t x2 = *(int32_t*) v2->c; + if (x1 == x2) { + return 0; + } else { + return x1 < x2? -1:1; + } +} + +TEST(testCase, external_sort_Test) { +#if 0 + su* v = static_cast(calloc(1000000, sizeof(su))); + for(int32_t i = 0; i < 1000000; ++i) { + v[i].v = rand(); + v[i].c = static_cast(malloc(4)); + *(int32_t*) v[i].c = i; + } + + qsort(v, 1000000, sizeof(su), cmp); +// for(int32_t i = 0; i < 1000; ++i) { +// printf("%d ", v[i]); +// } +// printf("\n"); + return; +#endif + + srand(time(NULL)); + + SArray* pOrderVal = taosArrayInit(4, sizeof(SOrder)); + SOrder o = {0}; + o.order = TSDB_ORDER_ASC; + o.col.info.colId = 1; + o.col.info.type = TSDB_DATA_TYPE_INT; + taosArrayPush(pOrderVal, &o); + + SArray* pExprInfo = taosArrayInit(4, sizeof(SExprInfo)); + SExprInfo *exp = static_cast(calloc(1, sizeof(SExprInfo))); + exp->base.resSchema = createSchema(TSDB_DATA_TYPE_INT, sizeof(int32_t), 1, "res"); + taosArrayPush(pExprInfo, &exp); + + SExprInfo *exp1 = static_cast(calloc(1, sizeof(SExprInfo))); + exp1->base.resSchema = createSchema(TSDB_DATA_TYPE_BINARY, 40, 2, "res1"); +// taosArrayPush(pExprInfo, &exp1); + + SOperatorInfo* pOperator = createOrderOperatorInfo(createDummyOperator(1500), pExprInfo, pOrderVal); + + bool newgroup = false; + SSDataBlock* pRes = NULL; + + int32_t total = 1; + + int64_t s1 = taosGetTimestampUs(); + int32_t t = 1; + + while(1) { + int64_t s = taosGetTimestampUs(); + pRes = pOperator->exec(pOperator, &newgroup); + + int64_t e = taosGetTimestampUs(); + if (t++ == 1) { + printf("---------------elapsed:%ld\n", e - s); + } + + if (pRes == NULL) { + break; + } + + SColumnInfoData* pCol1 = static_cast(taosArrayGet(pRes->pDataBlock, 0)); +// SColumnInfoData* pCol2 = static_cast(taosArrayGet(pRes->pDataBlock, 1)); + for (int32_t i = 0; i < pRes->info.rows; ++i) { +// char* p = colDataGet(pCol2, i); + printf("%d: %d\n", total++, ((int32_t*)pCol1->pData)[i]); +// printf("%d: %d, %s\n", total++, ((int32_t*)pCol1->pData)[i], (char*)varDataVal(p)); + } + } + + printStatisBeforeClose(((SOrderOperatorInfo*) pOperator->info)->pSortInternalBuf); + + int64_t s2 = taosGetTimestampUs(); + printf("total:%ld\n", s2 - s1); + + pOperator->cleanupFn(pOperator->info, 2); + tfree(exp); + tfree(exp1); + taosArrayDestroy(pExprInfo); + taosArrayDestroy(pOrderVal); +} #pragma GCC diagnostic pop diff --git a/source/libs/function/inc/thistogram.h b/source/libs/function/inc/thistogram.h index 3b5c2b4cfb..cb6560325b 100644 --- a/source/libs/function/inc/thistogram.h +++ b/source/libs/function/inc/thistogram.h @@ -49,7 +49,7 @@ typedef struct SHistogramInfo { SHistBin* elems; #else tSkipList* pList; - SLoserTreeInfo* pLoserTree; + SMultiwayMergeTreeInfo* pLoserTree; int32_t maxIndex; bool ordered; #endif diff --git a/source/libs/function/inc/tpercentile.h b/source/libs/function/inc/tpercentile.h index 563a63f6a5..dfb52f7694 100644 --- a/source/libs/function/inc/tpercentile.h +++ b/source/libs/function/inc/tpercentile.h @@ -20,7 +20,7 @@ extern "C" { #endif -#include "tpagedfile.h" +#include "tpagedbuf.h" #include "ttszip.h" typedef struct MinMaxEntry { @@ -63,7 +63,7 @@ typedef struct tMemBucket { __compar_fn_t comparFn; tMemBucketSlot * pSlots; - SDiskbasedResultBuf *pBuffer; + SDiskbasedBuf *pBuffer; __perc_hash_func_t hashFunc; } tMemBucket; diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 78d2feaa83..887f65a6ea 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -40,14 +40,8 @@ int32_t fmFuncMgtInit() { return TSDB_CODE_SUCCESS; } -int32_t fmGetHandle(FuncMgtHandle* pHandle) { - *pHandle = &gFunMgtService; - return TSDB_CODE_SUCCESS; -} - -int32_t fmGetFuncInfo(FuncMgtHandle handle, const char* pFuncName, int32_t* pFuncId, int32_t* pFuncType) { - SFuncMgtService* pService = (SFuncMgtService*)handle; - void* pVal = taosHashGet(pService->pFuncNameHashTable, pFuncName, strlen(pFuncName)); +int32_t fmGetFuncInfo(const char* pFuncName, int32_t* pFuncId, int32_t* pFuncType) { + void* pVal = taosHashGet(gFunMgtService.pFuncNameHashTable, pFuncName, strlen(pFuncName)); if (NULL == pVal) { return TSDB_CODE_FAILED; } @@ -66,6 +60,17 @@ int32_t fmGetFuncResultType(SFunctionNode* pFunc) { return funcMgtBuiltins[pFunc->funcId].checkFunc(pFunc); } +int32_t fmGetFuncExecFuncs(int32_t funcId, SFuncExecFuncs* pFpSet) { + if (funcId < 0 || funcId >= funcMgtBuiltinsNum) { + return TSDB_CODE_FAILED; + } + pFpSet->getEnv = funcMgtBuiltins[funcId].getEnvFunc; + pFpSet->init = funcMgtBuiltins[funcId].initFunc; + pFpSet->process = funcMgtBuiltins[funcId].processFunc; + pFpSet->finalize = funcMgtBuiltins[funcId].finalizeFunc; + return TSDB_CODE_SUCCESS; +} + bool fmIsAggFunc(int32_t funcId) { if (funcId < 0 || funcId >= funcMgtBuiltinsNum) { return false; diff --git a/source/libs/function/src/thistogram.c b/source/libs/function/src/thistogram.c index 2229ac8561..49799aef7a 100644 --- a/source/libs/function/src/thistogram.c +++ b/source/libs/function/src/thistogram.c @@ -117,14 +117,14 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) { if ((*pHisto)->ordered) { int32_t lastIndex = (*pHisto)->maxIndex; - SLoserTreeInfo* pTree = (*pHisto)->pLoserTree; + SMultiwayMergeTreeInfo* pTree = (*pHisto)->pLoserTree; (*pHisto)->pLoserTree->pNode[lastIndex + pTree->numOfEntries].pData = pResNode; pEntry1->index = (*pHisto)->pLoserTree->pNode[lastIndex + pTree->numOfEntries].index; // update the loser tree if ((*pHisto)->ordered) { - tLoserTreeAdjust(pTree, pEntry1->index + pTree->numOfEntries); + tMergeTreeAdjust(pTree, pEntry1->index + pTree->numOfEntries); } tSkipListKey kx = @@ -142,10 +142,10 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) { SHistBin* pPrevEntry = (SHistBin*)pResNode->pBackward[0]->pData; pPrevEntry->delta = val - pPrevEntry->val; - SLoserTreeInfo* pTree = (*pHisto)->pLoserTree; + SMultiwayMergeTreeInfo* pTree = (*pHisto)->pLoserTree; if ((*pHisto)->ordered) { - tLoserTreeAdjust(pTree, pPrevEntry->index + pTree->numOfEntries); - tLoserTreeDisplay(pTree); + tMergeTreeAdjust(pTree, pPrevEntry->index + pTree->numOfEntries); + tMergeTreePrint(pTree); } } @@ -155,7 +155,7 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) { if (!(*pHisto)->ordered) { SSkipListPrint((*pHisto)->pList, 1); - SLoserTreeInfo* pTree = (*pHisto)->pLoserTree; + SMultiwayMergeTreeInfo* pTree = (*pHisto)->pLoserTree; tSkipListNode* pHead = (*pHisto)->pList->pHead.pForward[0]; tSkipListNode* p1 = pHead; @@ -183,13 +183,13 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) { pTree->pNode[i].index = -1; } - tLoserTreeDisplay(pTree); + tMergeTreePrint(pTree); for (int32_t i = pTree->totalEntries - 1; i >= pTree->numOfEntries; i--) { - tLoserTreeAdjust(pTree, i); + tMergeTreeAdjust(pTree, i); } - tLoserTreeDisplay(pTree); + tMergeTreePrint(pTree); (*pHisto)->ordered = true; } @@ -219,7 +219,7 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) { pPrevEntry->delta = pEntry->val - pPrevEntry->val; } - SLoserTreeInfo* pTree = (*pHisto)->pLoserTree; + SMultiwayMergeTreeInfo* pTree = (*pHisto)->pLoserTree; if (pNextEntry->index != -1) { (*pHisto)->maxIndex = pNextEntry->index; @@ -230,12 +230,12 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) { printf("disappear index is:%d\n", f); } - tLoserTreeAdjust(pTree, pEntry->index + pTree->numOfEntries); + tMergeTreeAdjust(pTree, pEntry->index + pTree->numOfEntries); // remove the next node in skiplist tSkipListRemoveNode((*pHisto)->pList, pNext); SSkipListPrint((*pHisto)->pList, 1); - tLoserTreeDisplay((*pHisto)->pLoserTree); + tMergeTreePrint((*pHisto)->pLoserTree); } else { // add to heap if (pResNode->pForward[0] != NULL) { pEntry1->delta = ((SHistBin*)pResNode->pForward[0]->pData)->val - val; diff --git a/source/libs/function/src/tpercentile.c b/source/libs/function/src/tpercentile.c index 5d8876fee1..40731adc58 100644 --- a/source/libs/function/src/tpercentile.c +++ b/source/libs/function/src/tpercentile.c @@ -15,10 +15,10 @@ #include #include "os.h" -#include "tpercentile.h" -#include "tpagedfile.h" #include "taosdef.h" #include "tcompare.h" +#include "tpagedbuf.h" +#include "tpercentile.h" #include "ttypes.h" #define DEFAULT_NUM_OF_SLOT 1024 @@ -35,9 +35,9 @@ static SFilePage *loadDataFromFilePage(tMemBucket *pMemBucket, int32_t slotIdx) int32_t offset = 0; for(int32_t i = 0; i < list->size; ++i) { - SPageInfo* pgInfo = *(SPageInfo**) taosArrayGet(list, i); + struct SPageInfo* pgInfo = *(struct SPageInfo**) taosArrayGet(list, i); - SFilePage* pg = getResBufPage(pMemBucket->pBuffer, pgInfo->pageId); + SFilePage* pg = getBufPage(pMemBucket->pBuffer, getPageId(pgInfo)); memcpy(buffer->data + offset, pg->data, (size_t)(pg->num * pMemBucket->bytes)); offset += (int32_t)(pg->num * pMemBucket->bytes); @@ -98,8 +98,8 @@ double findOnlyResult(tMemBucket *pMemBucket) { SIDList list = getDataBufPagesIdList(pMemBucket->pBuffer, groupId); assert(list->size == 1); - SPageInfo* pgInfo = (SPageInfo*) taosArrayGetP(list, 0); - SFilePage* pPage = getResBufPage(pMemBucket->pBuffer, pgInfo->pageId); + struct SPageInfo* pgInfo = (struct SPageInfo*) taosArrayGetP(list, 0); + SFilePage* pPage = getBufPage(pMemBucket->pBuffer, getPageId(pgInfo)); assert(pPage->num == 1); double v = 0; @@ -254,7 +254,7 @@ tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, resetSlotInfo(pBucket); - int32_t ret = createDiskbasedResultBuffer(&pBucket->pBuffer, pBucket->bufPageSize, pBucket->bufPageSize * 512, 1, tsTempDir); + int32_t ret = createDiskbasedBuffer(&pBucket->pBuffer, pBucket->bufPageSize, pBucket->bufPageSize * 512, 1, tsTempDir); if (ret != 0) { tMemBucketDestroy(pBucket); return NULL; @@ -343,7 +343,7 @@ int32_t tMemBucketPut(tMemBucket *pBucket, const void *data, size_t size) { assert(pSlot->info.data->num >= pBucket->elemPerPage && pSlot->info.size > 0); // keep the pointer in memory - releaseResBufPage(pBucket->pBuffer, pSlot->info.data); + releaseBufPage(pBucket->pBuffer, pSlot->info.data); pSlot->info.data = NULL; } @@ -471,10 +471,10 @@ double getPercentileImpl(tMemBucket *pMemBucket, int32_t count, double fraction) for (int32_t f = 0; f < list->size; ++f) { SPageInfo *pgInfo = *(SPageInfo **)taosArrayGet(list, f); - SFilePage *pg = getResBufPage(pMemBucket->pBuffer, pgInfo->pageId); + SFilePage *pg = getBufPage(pMemBucket->pBuffer, getPageId(pgInfo)); tMemBucketPut(pMemBucket, pg->data, (int32_t)pg->num); - releaseResBufPageInfo(pMemBucket->pBuffer, pgInfo); + releaseBufPageInfo(pMemBucket->pBuffer, pgInfo); } return getPercentileImpl(pMemBucket, count - num, fraction); diff --git a/source/nodes/CMakeLists.txt b/source/libs/nodes/CMakeLists.txt similarity index 84% rename from source/nodes/CMakeLists.txt rename to source/libs/nodes/CMakeLists.txt index b30534f3f2..9a826e034c 100644 --- a/source/nodes/CMakeLists.txt +++ b/source/libs/nodes/CMakeLists.txt @@ -2,7 +2,7 @@ aux_source_directory(src NODES_SRC) add_library(nodes STATIC ${NODES_SRC}) target_include_directories( nodes - PUBLIC "${CMAKE_SOURCE_DIR}/include/nodes" + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/nodes" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) target_link_libraries( diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c new file mode 100644 index 0000000000..63bb11a821 --- /dev/null +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -0,0 +1,191 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "querynodes.h" +#include "taos.h" + +#define COPY_SCALAR_FIELD(fldname) \ + do { \ + (pDst)->fldname = (pSrc)->fldname; \ + } while (0) + +#define COPY_CHAR_ARRAY_FIELD(fldname) \ + do { \ + strcpy((pDst)->fldname, (pSrc)->fldname); \ + } while (0) + +#define COPY_CHAR_POINT_FIELD(fldname) \ + do { \ + (pDst)->fldname = strdup((pSrc)->fldname); \ + } while (0) + +#define COPY_NODE_FIELD(fldname) \ + do { \ + (pDst)->fldname = nodesCloneNode((pSrc)->fldname); \ + if (NULL == (pDst)->fldname) { \ + nodesDestroyNode((SNode*)(pDst)); \ + return NULL; \ + } \ + } while (0) + +#define COPY_NODE_LIST_FIELD(fldname) \ + do { \ + (pDst)->fldname = nodesCloneList((pSrc)->fldname); \ + if (NULL == (pDst)->fldname) { \ + nodesDestroyNode((SNode*)(pDst)); \ + return NULL; \ + } \ + } while (0) + +static void dataTypeCopy(const SDataType* pSrc, SDataType* pDst) { + COPY_SCALAR_FIELD(type); + COPY_SCALAR_FIELD(precision); + COPY_SCALAR_FIELD(scale); + COPY_SCALAR_FIELD(bytes); +} + +static void exprNodeCopy(const SExprNode* pSrc, SExprNode* pDst) { + COPY_SCALAR_FIELD(type); + dataTypeCopy(&pSrc->resType, &pDst->resType); + COPY_CHAR_ARRAY_FIELD(aliasName); + // COPY_NODE_LIST_FIELD(pAssociationList); +} + +static SNode* columnNodeCopy(const SColumnNode* pSrc, SColumnNode* pDst) { + exprNodeCopy((const SExprNode*)&pSrc, (SExprNode*)&pDst); + COPY_SCALAR_FIELD(colId); + COPY_SCALAR_FIELD(colType); + COPY_CHAR_ARRAY_FIELD(dbName); + COPY_CHAR_ARRAY_FIELD(tableName); + COPY_CHAR_ARRAY_FIELD(tableAlias); + COPY_CHAR_ARRAY_FIELD(colName); + // COPY_NODE_FIELD(pProjectRef); + return (SNode*)pDst; +} + +static SNode* valueNodeCopy(const SValueNode* pSrc, SValueNode* pDst) { + exprNodeCopy((const SExprNode*)&pSrc, (SExprNode*)&pDst); + COPY_CHAR_POINT_FIELD(literal); + COPY_SCALAR_FIELD(isDuration); + switch (pSrc->node.resType.type) { + case TSDB_DATA_TYPE_NULL: + break; + case TSDB_DATA_TYPE_BOOL: + COPY_SCALAR_FIELD(datum.b); + break; + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: + COPY_SCALAR_FIELD(datum.i); + break; + case TSDB_DATA_TYPE_UTINYINT: + case TSDB_DATA_TYPE_USMALLINT: + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_UBIGINT: + COPY_SCALAR_FIELD(datum.u); + break; + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: + COPY_SCALAR_FIELD(datum.d); + break; + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_VARCHAR: + case TSDB_DATA_TYPE_VARBINARY: + COPY_CHAR_POINT_FIELD(datum.p); + break; + case TSDB_DATA_TYPE_JSON: + case TSDB_DATA_TYPE_DECIMAL: + case TSDB_DATA_TYPE_BLOB: + // todo + default: + break; + } + return (SNode*)pDst; +} + +static SNode* operatorNodeCopy(const SOperatorNode* pSrc, SOperatorNode* pDst) { + exprNodeCopy((const SExprNode*)&pSrc, (SExprNode*)&pDst); + COPY_SCALAR_FIELD(opType); + COPY_NODE_FIELD(pLeft); + COPY_NODE_FIELD(pRight); + return (SNode*)pDst; +} + +static SNode* logicConditionNodeCopy(const SLogicConditionNode* pSrc, SLogicConditionNode* pDst) { + exprNodeCopy((const SExprNode*)&pSrc, (SExprNode*)&pDst); + COPY_SCALAR_FIELD(condType); + COPY_NODE_LIST_FIELD(pParameterList); + return (SNode*)pDst; +} + +static SNode* functionNodeCopy(const SFunctionNode* pSrc, SFunctionNode* pDst) { + exprNodeCopy((const SExprNode*)&pSrc, (SExprNode*)&pDst); + COPY_CHAR_ARRAY_FIELD(functionName); + COPY_SCALAR_FIELD(funcId); + COPY_SCALAR_FIELD(funcType); + COPY_NODE_LIST_FIELD(pParameterList); + return (SNode*)pDst; +} + +SNode* nodesCloneNode(const SNode* pNode) { + if (NULL == pNode) { + return NULL; + } + SNode* pDst = nodesMakeNode(nodeType(pNode)); + if (NULL == pDst) { + return NULL; + } + switch (nodeType(pNode)) { + case QUERY_NODE_COLUMN: + return columnNodeCopy((const SColumnNode*)pNode, (SColumnNode*)pDst); + case QUERY_NODE_VALUE: + return valueNodeCopy((const SValueNode*)pNode, (SValueNode*)pDst); + case QUERY_NODE_OPERATOR: + return operatorNodeCopy((const SOperatorNode*)pNode, (SOperatorNode*)pDst); + case QUERY_NODE_LOGIC_CONDITION: + return logicConditionNodeCopy((const SLogicConditionNode*)pNode, (SLogicConditionNode*)pDst); + case QUERY_NODE_FUNCTION: + return functionNodeCopy((const SFunctionNode*)pNode, (SFunctionNode*)pDst); + case QUERY_NODE_REAL_TABLE: + case QUERY_NODE_TEMP_TABLE: + case QUERY_NODE_JOIN_TABLE: + case QUERY_NODE_GROUPING_SET: + case QUERY_NODE_ORDER_BY_EXPR: + case QUERY_NODE_LIMIT: + default: + break; + } + return pDst; +} + +SNodeList* nodesCloneList(const SNodeList* pList) { + SNodeList* pDst = nodesMakeList(); + if (NULL == pDst) { + return NULL; + } + SNode* pNode; + FOREACH(pNode, pList) { + SNode* pNewNode = nodesCloneNode(pNode); + if (NULL == pNewNode) { + nodesDestroyList(pDst); + return NULL; + } + nodesListAppend(pDst, pNewNode); + } + return pDst; +} diff --git a/source/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c similarity index 97% rename from source/nodes/src/nodesCodeFuncs.c rename to source/libs/nodes/src/nodesCodeFuncs.c index 959e10fb82..ad83bdbd19 100644 --- a/source/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -21,7 +21,6 @@ int32_t nodesNodeToString(const SNode* pNode, char** pStr, int32_t* pLen) { case QUERY_NODE_VALUE: case QUERY_NODE_OPERATOR: case QUERY_NODE_LOGIC_CONDITION: - case QUERY_NODE_IS_NULL_CONDITION: case QUERY_NODE_FUNCTION: case QUERY_NODE_REAL_TABLE: case QUERY_NODE_TEMP_TABLE: diff --git a/source/nodes/src/nodesEqualFuncs.c b/source/libs/nodes/src/nodesEqualFuncs.c similarity index 92% rename from source/nodes/src/nodesEqualFuncs.c rename to source/libs/nodes/src/nodesEqualFuncs.c index 65eb6e9b32..fb2463d350 100644 --- a/source/nodes/src/nodesEqualFuncs.c +++ b/source/libs/nodes/src/nodesEqualFuncs.c @@ -89,12 +89,6 @@ static bool logicConditionNodeEqual(const SLogicConditionNode* a, const SLogicCo return true; } -static bool isNullConditionNodeEqual(const SIsNullCondNode* a, const SIsNullCondNode* b) { - COMPARE_NODE_FIELD(pExpr); - COMPARE_SCALAR_FIELD(isNull); - return true; -} - static bool functionNodeEqual(const SFunctionNode* a, const SFunctionNode* b) { COMPARE_SCALAR_FIELD(funcId); COMPARE_NODE_LIST_FIELD(pParameterList); @@ -123,8 +117,6 @@ bool nodesEqualNode(const SNode* a, const SNode* b) { return operatorNodeEqual((const SOperatorNode*)a, (const SOperatorNode*)b); case QUERY_NODE_LOGIC_CONDITION: return logicConditionNodeEqual((const SLogicConditionNode*)a, (const SLogicConditionNode*)b); - case QUERY_NODE_IS_NULL_CONDITION: - return isNullConditionNodeEqual((const SIsNullCondNode*)a, (const SIsNullCondNode*)b); case QUERY_NODE_FUNCTION: return functionNodeEqual((const SFunctionNode*)a, (const SFunctionNode*)b); case QUERY_NODE_REAL_TABLE: diff --git a/source/nodes/src/nodesTraverseFuncs.c b/source/libs/nodes/src/nodesTraverseFuncs.c similarity index 97% rename from source/nodes/src/nodesTraverseFuncs.c rename to source/libs/nodes/src/nodesTraverseFuncs.c index d1ded390db..b7e7ad6f0b 100644 --- a/source/nodes/src/nodesTraverseFuncs.c +++ b/source/libs/nodes/src/nodesTraverseFuncs.c @@ -53,9 +53,6 @@ static EDealRes walkNode(SNode* pNode, ETraversalOrder order, FNodeWalker walker case QUERY_NODE_LOGIC_CONDITION: res = walkList(((SLogicConditionNode*)pNode)->pParameterList, order, walker, pContext); break; - case QUERY_NODE_IS_NULL_CONDITION: - res = walkNode(((SIsNullCondNode*)pNode)->pExpr, order, walker, pContext); - break; case QUERY_NODE_FUNCTION: res = walkList(((SFunctionNode*)pNode)->pParameterList, order, walker, pContext); break; @@ -179,9 +176,6 @@ static EDealRes rewriteNode(SNode** pRawNode, ETraversalOrder order, FNodeRewrit case QUERY_NODE_LOGIC_CONDITION: res = rewriteList(((SLogicConditionNode*)pNode)->pParameterList, order, rewriter, pContext); break; - case QUERY_NODE_IS_NULL_CONDITION: - res = rewriteNode(&(((SIsNullCondNode*)pNode)->pExpr), order, rewriter, pContext); - break; case QUERY_NODE_FUNCTION: res = rewriteList(((SFunctionNode*)pNode)->pParameterList, order, rewriter, pContext); break; diff --git a/source/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c similarity index 98% rename from source/nodes/src/nodesUtilFuncs.c rename to source/libs/nodes/src/nodesUtilFuncs.c index fe2306dde3..8eef6aa364 100644 --- a/source/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -36,8 +36,6 @@ SNode* nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SOperatorNode)); case QUERY_NODE_LOGIC_CONDITION: return makeNode(type, sizeof(SLogicConditionNode)); - case QUERY_NODE_IS_NULL_CONDITION: - return makeNode(type, sizeof(SIsNullCondNode)); case QUERY_NODE_FUNCTION: return makeNode(type, sizeof(SFunctionNode)); case QUERY_NODE_REAL_TABLE: diff --git a/source/nodes/test/CMakeLists.txt b/source/libs/nodes/test/CMakeLists.txt similarity index 100% rename from source/nodes/test/CMakeLists.txt rename to source/libs/nodes/test/CMakeLists.txt diff --git a/source/nodes/test/nodesTest.cpp b/source/libs/nodes/test/nodesTest.cpp similarity index 100% rename from source/nodes/test/nodesTest.cpp rename to source/libs/nodes/test/nodesTest.cpp diff --git a/source/libs/parser/inc/astCreateFuncs.h b/source/libs/parser/inc/astCreateFuncs.h index 43bc5349e4..b44e621704 100644 --- a/source/libs/parser/inc/astCreateFuncs.h +++ b/source/libs/parser/inc/astCreateFuncs.h @@ -43,7 +43,6 @@ SNode* createLogicConditionNode(SAstCreateContext* pCxt, ELogicConditionType typ SNode* createOperatorNode(SAstCreateContext* pCxt, EOperatorType type, SNode* pLeft, SNode* pRight); SNode* createBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight); SNode* createNotBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight); -SNode* createIsNullCondNode(SAstCreateContext* pCxt, SNode* pExpr, bool isNull); SNode* createFunctionNode(SAstCreateContext* pCxt, const SToken* pFuncName, SNodeList* pParameterList); SNode* createNodeListNode(SAstCreateContext* pCxt, SNodeList* pList); SNode* createRealTableNode(SAstCreateContext* pCxt, const SToken* pDbName, const SToken* pTableName, const SToken* pTableAlias); diff --git a/source/libs/parser/inc/astToMsg.h b/source/libs/parser/inc/astToMsg.h index d7b6469abb..77d6900acf 100644 --- a/source/libs/parser/inc/astToMsg.h +++ b/source/libs/parser/inc/astToMsg.h @@ -1,18 +1,32 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + #ifndef TDENGINE_ASTTOMSG_H #define TDENGINE_ASTTOMSG_H #include "parserInt.h" #include "tmsg.h" - char* buildUserManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen); char* buildAcctManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen); char* buildDropUserMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen); -SShowReq* buildShowMsg(SShowInfo* pShowInfo, SParseContext* pParseCtx, SMsgBuf* pMsgBuf); -SCreateDbReq* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, SParseContext *pCtx, SMsgBuf* pMsgBuf); -char* buildCreateStbReq(SCreateTableSql* pCreateTableSql, int32_t* len, SParseContext* pParseCtx, SMsgBuf* pMsgBuf); -char* buildDropStableReq(SSqlInfo* pInfo, int32_t* len, SParseContext* pParseCtx, SMsgBuf* pMsgBuf); -char* buildCreateDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf); -char* buildDropDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf); +char* buildShowMsg(SShowInfo* pShowInfo, int32_t* outputLen, SParseContext* pParseCtx, SMsgBuf* pMsgBuf); +char* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, int32_t* outputLen, SParseContext* pCtx, SMsgBuf* pMsgBuf); +char* buildCreateStbReq(SCreateTableSql* pCreateTableSql, int32_t* outputLen, SParseContext* pParseCtx, SMsgBuf* pMsgBuf); +char* buildDropStableReq(SSqlInfo* pInfo, int32_t* outputLen, SParseContext* pParseCtx, SMsgBuf* pMsgBuf); +char* buildCreateDnodeMsg(SSqlInfo* pInfo, int32_t* outputLen, SMsgBuf* pMsgBuf); +char* buildDropDnodeMsg(SSqlInfo* pInfo, int32_t* outputLen, SMsgBuf* pMsgBuf); #endif // TDENGINE_ASTTOMSG_H diff --git a/source/libs/parser/inc/dataBlockMgt.h b/source/libs/parser/inc/dataBlockMgt.h index f93daa7f93..cbc35240a3 100644 --- a/source/libs/parser/inc/dataBlockMgt.h +++ b/source/libs/parser/inc/dataBlockMgt.h @@ -94,7 +94,7 @@ static FORCE_INLINE void getMemRowAppendInfo(SSchema *pSchema, uint8_t rowType, int32_t idx, int32_t *toffset, int32_t *colIdx) { int32_t schemaIdx = 0; if (IS_DATA_COL_ORDERED(spd)) { - schemaIdx = spd->boundedColumns[idx] - 1; + schemaIdx = spd->boundedColumns[idx] - PRIMARYKEY_TIMESTAMP_COL_ID; if (TD_IS_TP_ROW_T(rowType)) { *toffset = (spd->cols + schemaIdx)->toffset; // the offset of firstPart *colIdx = schemaIdx; @@ -104,7 +104,7 @@ static FORCE_INLINE void getMemRowAppendInfo(SSchema *pSchema, uint8_t rowType, } } else { ASSERT(idx == (spd->colIdxInfo + idx)->boundIdx); - schemaIdx = (spd->colIdxInfo + idx)->schemaColIdx; + schemaIdx = (spd->colIdxInfo + idx)->schemaColIdx - PRIMARYKEY_TIMESTAMP_COL_ID; if (TD_IS_TP_ROW_T(rowType)) { *toffset = (spd->cols + schemaIdx)->toffset; *colIdx = schemaIdx; diff --git a/source/libs/parser/inc/new_sql.y b/source/libs/parser/inc/new_sql.y index eef04b17c3..065cb95650 100644 --- a/source/libs/parser/inc/new_sql.y +++ b/source/libs/parser/inc/new_sql.y @@ -55,6 +55,7 @@ %left OR. %left AND. +//%right NOT. %left UNION ALL MINUS EXCEPT INTERSECT. //%left BITAND BITOR LSHIFT RSHIFT. %left NK_PLUS NK_MINUS. @@ -169,13 +170,41 @@ column_reference(A) ::= table_name(B) NK_DOT column_name(C). //pseudo_column(A) ::= NK_NOW. { PARSER_TRACE; A = createFunctionNode(pCxt, NULL, NULL); } /************************************************ predicate ***********************************************************/ -predicate(A) ::= expression(B) compare_op(C) expression(D). { PARSER_TRACE; A = createOperatorNode(pCxt, C, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, D)); } +predicate(A) ::= expression(B) compare_op(C) expression(D). { + PARSER_TRACE; + SToken s = getTokenFromRawExprNode(pCxt, B); + SToken e = getTokenFromRawExprNode(pCxt, D); + A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, C, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, D))); + } //predicate(A) ::= expression(B) compare_op sub_type expression(B). -predicate(A) ::= expression(B) BETWEEN expression(C) AND expression(D). { PARSER_TRACE; A = createBetweenAnd(pCxt, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C), releaseRawExprNode(pCxt, D)); } -predicate(A) ::= expression(B) NOT BETWEEN expression(C) AND expression(D). { PARSER_TRACE; A = createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, C), releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, D)); } -predicate(A) ::= expression(B) IS NULL. { PARSER_TRACE; A = createIsNullCondNode(pCxt, releaseRawExprNode(pCxt, B), true); } -predicate(A) ::= expression(B) IS NOT NULL. { PARSER_TRACE; A = createIsNullCondNode(pCxt, releaseRawExprNode(pCxt, B), false); } -predicate(A) ::= expression(B) in_op(C) in_predicate_value(D). { PARSER_TRACE; A = createOperatorNode(pCxt, C, releaseRawExprNode(pCxt, B), D); } +predicate(A) ::= expression(B) BETWEEN expression(C) AND expression(D). { + PARSER_TRACE; + SToken s = getTokenFromRawExprNode(pCxt, B); + SToken e = getTokenFromRawExprNode(pCxt, D); + A = createRawExprNodeExt(pCxt, &s, &e, createBetweenAnd(pCxt, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C), releaseRawExprNode(pCxt, D))); + } +predicate(A) ::= expression(B) NOT BETWEEN expression(C) AND expression(D). { + PARSER_TRACE; + SToken s = getTokenFromRawExprNode(pCxt, B); + SToken e = getTokenFromRawExprNode(pCxt, D); + A = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, C), releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, D))); + } +predicate(A) ::= expression(B) IS NULL(C). { + PARSER_TRACE; + SToken s = getTokenFromRawExprNode(pCxt, B); + A = createRawExprNodeExt(pCxt, &s, &C, createOperatorNode(pCxt, OP_TYPE_IS_NULL, releaseRawExprNode(pCxt, B), NULL)); + } +predicate(A) ::= expression(B) IS NOT NULL(C). { + PARSER_TRACE; + SToken s = getTokenFromRawExprNode(pCxt, B); + A = createRawExprNodeExt(pCxt, &s, &C, createOperatorNode(pCxt, OP_TYPE_IS_NOT_NULL, releaseRawExprNode(pCxt, B), NULL)); + } +predicate(A) ::= expression(B) in_op(C) in_predicate_value(D). { + PARSER_TRACE; + SToken s = getTokenFromRawExprNode(pCxt, B); + SToken e = getTokenFromRawExprNode(pCxt, D); + A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, C, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, D))); + } %type compare_op { EOperatorType } %destructor compare_op { PARSER_DESTRUCTOR_TRACE; } @@ -195,18 +224,36 @@ compare_op(A) ::= NMATCH. in_op(A) ::= IN. { PARSER_TRACE; A = OP_TYPE_IN; } in_op(A) ::= NOT IN. { PARSER_TRACE; A = OP_TYPE_NOT_IN; } -in_predicate_value(A) ::= NK_LP expression_list(B) NK_RP. { PARSER_TRACE; A = createNodeListNode(pCxt, B); } +in_predicate_value(A) ::= NK_LP(C) expression_list(B) NK_RP(D). { PARSER_TRACE; A = createRawExprNodeExt(pCxt, &C, &D, createNodeListNode(pCxt, B)); } /************************************************ boolean_value_expression ********************************************/ boolean_value_expression(A) ::= boolean_primary(B). { PARSER_TRACE; A = B; } -boolean_value_expression(A) ::= NOT boolean_primary(B). { PARSER_TRACE; A = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, B, NULL); } +boolean_value_expression(A) ::= NOT(C) boolean_primary(B). { + PARSER_TRACE; + SToken e = getTokenFromRawExprNode(pCxt, B); + A = createRawExprNodeExt(pCxt, &C, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, B), NULL)); + } boolean_value_expression(A) ::= - boolean_value_expression(B) OR boolean_value_expression(C). { PARSER_TRACE; A = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, B, C); } + boolean_value_expression(B) OR boolean_value_expression(C). { + PARSER_TRACE; + SToken s = getTokenFromRawExprNode(pCxt, B); + SToken e = getTokenFromRawExprNode(pCxt, C); + A = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C))); + } boolean_value_expression(A) ::= - boolean_value_expression(B) AND boolean_value_expression(C). { PARSER_TRACE; A = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, B, C); } + boolean_value_expression(B) AND boolean_value_expression(C). { + PARSER_TRACE; + SToken s = getTokenFromRawExprNode(pCxt, B); + SToken e = getTokenFromRawExprNode(pCxt, C); + A = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C))); + } boolean_primary(A) ::= predicate(B). { PARSER_TRACE; A = B; } -boolean_primary(A) ::= NK_LP boolean_value_expression(B) NK_RP. { PARSER_TRACE; A = B; } +boolean_primary(A) ::= NK_LP(C) boolean_value_expression(B) NK_RP(D). { PARSER_TRACE; A = createRawExprNodeExt(pCxt, &C, &D, releaseRawExprNode(pCxt, B)); } + +/************************************************ common_expression ********************************************/ +common_expression(A) ::= expression(B). { A = B; } +common_expression(A) ::= boolean_value_expression(B). { A = B; } /************************************************ from_clause *********************************************************/ from_clause(A) ::= FROM table_reference_list(B). { PARSER_TRACE; A = B; } @@ -271,13 +318,13 @@ select_list(A) ::= select_sublist(B). select_sublist(A) ::= select_item(B). { PARSER_TRACE; A = createNodeList(pCxt, B); } select_sublist(A) ::= select_sublist(B) NK_COMMA select_item(C). { PARSER_TRACE; A = addNodeToList(pCxt, B, C); } -select_item(A) ::= expression(B). { +select_item(A) ::= common_expression(B). { PARSER_TRACE; SToken t = getTokenFromRawExprNode(pCxt, B); A = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, B), &t); } -select_item(A) ::= expression(B) column_alias(C). { PARSER_TRACE; A = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, B), &C); } -select_item(A) ::= expression(B) AS column_alias(C). { PARSER_TRACE; A = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, B), &C); } +select_item(A) ::= common_expression(B) column_alias(C). { PARSER_TRACE; A = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, B), &C); } +select_item(A) ::= common_expression(B) AS column_alias(C). { PARSER_TRACE; A = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, B), &C); } select_item(A) ::= table_name(B) NK_DOT NK_STAR(C). { PARSER_TRACE; A = createColumnNode(pCxt, &B, &C); } where_clause_opt(A) ::= . { PARSER_TRACE; A = NULL; } @@ -364,7 +411,7 @@ limit_clause_opt(A) ::= LIMIT NK_INTEGER(C) NK_COMMA NK_INTEGER(B). subquery(A) ::= NK_LP(B) query_expression(C) NK_RP(D). { PARSER_TRACE; A = createRawExprNodeExt(pCxt, &B, &D, C); } /************************************************ search_condition ****************************************************/ -search_condition(A) ::= boolean_value_expression(B). { PARSER_TRACE; A = B; } +search_condition(A) ::= common_expression(B). { PARSER_TRACE; A = releaseRawExprNode(pCxt, B); } /************************************************ sort_specification_list *********************************************/ %type sort_specification_list { SNodeList* } diff --git a/source/libs/parser/inc/parserImpl.h b/source/libs/parser/inc/parserImpl.h index b55060def7..4f6f0cf387 100644 --- a/source/libs/parser/inc/parserImpl.h +++ b/source/libs/parser/inc/parserImpl.h @@ -13,9 +13,6 @@ * along with this program. If not, see . */ -#include "querynodes.h" -#include "parser.h" - #ifndef _TD_AST_CREATE_FUNCS_H_ #define _TD_AST_CREATE_FUNCS_H_ @@ -23,9 +20,19 @@ extern "C" { #endif +#include "querynodes.h" +#include "parser.h" + +typedef enum EStmtType { + STMT_TYPE_CMD = 1, + STMT_TYPE_QUERY +} EStmtType; + typedef struct SQuery { + EStmtType stmtType; SNode* pRoot; - // todo reslut meta + int32_t numOfResCols; + SSchema* pResSchema; } SQuery; int32_t doParse(SParseContext* pParseCxt, SQuery* pQuery); diff --git a/source/libs/parser/src/astCreateFuncs.c b/source/libs/parser/src/astCreateFuncs.c index ce2903f6aa..2615aca7f3 100644 --- a/source/libs/parser/src/astCreateFuncs.c +++ b/source/libs/parser/src/astCreateFuncs.c @@ -84,6 +84,10 @@ SNode* releaseRawExprNode(SAstCreateContext* pCxt, SNode* pNode) { } SToken getTokenFromRawExprNode(SAstCreateContext* pCxt, SNode* pNode) { + if (NULL == pNode || QUERY_NODE_RAW_EXPR != nodeType(pNode)) { + pCxt->valid = false; + return nil_token; + } SRawExprNode* target = (SRawExprNode*)pNode; SToken t = { .type = 0, .z = target->p, .n = target->n}; return t; @@ -166,14 +170,6 @@ SNode* createNotBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, createOperatorNode(pCxt, OP_TYPE_LOWER_THAN, pExpr, pLeft), createOperatorNode(pCxt, OP_TYPE_GREATER_THAN, pExpr, pRight)); } -SNode* createIsNullCondNode(SAstCreateContext* pCxt, SNode* pExpr, bool isNull) { - SIsNullCondNode* cond = (SIsNullCondNode*)nodesMakeNode(QUERY_NODE_IS_NULL_CONDITION); - CHECK_OUT_OF_MEM(cond); - cond->pExpr = pExpr; - cond->isNull = isNull; - return (SNode*)cond; -} - SNode* createFunctionNode(SAstCreateContext* pCxt, const SToken* pFuncName, SNodeList* pParameterList) { SFunctionNode* func = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); CHECK_OUT_OF_MEM(func); @@ -292,7 +288,11 @@ SNode* createGroupingSetNode(SAstCreateContext* pCxt, SNode* pNode) { } SNode* setProjectionAlias(SAstCreateContext* pCxt, SNode* pNode, const SToken* pAlias) { - strncpy(((SExprNode*)pNode)->aliasName, pAlias->z, pAlias->n); + if (NULL == pNode || !pCxt->valid) { + return pNode; + } + uint32_t maxLen = sizeof(((SExprNode*)pNode)->aliasName); + strncpy(((SExprNode*)pNode)->aliasName, pAlias->z, pAlias->n > maxLen ? maxLen : pAlias->n); return pNode; } diff --git a/source/libs/parser/src/astToMsg.c b/source/libs/parser/src/astToMsg.c index 1fae979ee9..0b4867de03 100644 --- a/source/libs/parser/src/astToMsg.c +++ b/source/libs/parser/src/astToMsg.c @@ -1,3 +1,18 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + #include "astGenerator.h" #include "parserInt.h" #include "parserUtil.h" @@ -11,7 +26,7 @@ char* buildUserManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, createReq.superUser = (int8_t)pUser->type; if (pUser->type == TSDB_ALTER_USER_PRIVILEGES) { - // pMsg->privilege = (char)pCmd->count; + // pMsg->privilege = (char)pCmd->count; } else { strncpy(createReq.pass, pUser->passwd.z, pUser->passwd.n); } @@ -39,14 +54,11 @@ char* buildAcctManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, SCreateAcctInfo* pAcctOpt = &pInfo->pMiscInfo->acctOpt; - createReq.maxUsers = htonl(pAcctOpt->maxUsers); - createReq.maxDbs = htonl(pAcctOpt->maxDbs); - createReq.maxTimeSeries = htonl(pAcctOpt->maxTimeSeries); - createReq.maxStreams = htonl(pAcctOpt->maxStreams); - // createReq.maxPointsPerSecond = htonl(pAcctOpt->maxPointsPerSecond); - createReq.maxStorage = htobe64(pAcctOpt->maxStorage); - // createReq.maxQueryTime = htobe64(pAcctOpt->maxQueryTime); - // createReq.maxConnections = htonl(pAcctOpt->maxConnections); + createReq.maxUsers = pAcctOpt->maxUsers; + createReq.maxDbs = pAcctOpt->maxDbs; + createReq.maxTimeSeries = pAcctOpt->maxTimeSeries; + createReq.maxStreams = pAcctOpt->maxStreams; + createReq.maxStorage = pAcctOpt->maxStorage; if (pAcctOpt->stat.n == 0) { createReq.accessState = -1; @@ -74,7 +86,7 @@ char* buildAcctManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, return pReq; } -char* buildDropUserMsg(SSqlInfo* pInfo, int32_t* msgLen, int64_t id, char* msgBuf, int32_t msgBufLen) { +char* buildDropUserMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgBufLen) { SDropUserReq dropReq = {0}; SToken* pName = taosArrayGet(pInfo->pMiscInfo->a, 0); @@ -92,30 +104,26 @@ char* buildDropUserMsg(SSqlInfo* pInfo, int32_t* msgLen, int64_t id, char* msgBu } tSerializeSDropUserReq(pReq, tlen, &dropReq); - *msgLen = tlen; + *outputLen = tlen; return pReq; } -SShowReq* buildShowMsg(SShowInfo* pShowInfo, SParseContext* pCtx, SMsgBuf* pMsgBuf) { - SShowReq* pShowMsg = calloc(1, sizeof(SShowReq)); - if (pShowMsg == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return pShowMsg; - } +char* buildShowMsg(SShowInfo* pShowInfo, int32_t* outputLen, SParseContext* pCtx, SMsgBuf* pMsgBuf) { + SShowReq showReq = {.type = pShowInfo->showType}; - pShowMsg->type = pShowInfo->showType; if (pShowInfo->showType != TSDB_MGMT_TABLE_VNODES) { SToken* pPattern = &pShowInfo->pattern; if (pPattern->type > 0) { // only show tables support wildcard query - strncpy(pShowMsg->payload, pPattern->z, pPattern->n); - pShowMsg->payloadLen = htons(pPattern->n); + showReq.payloadLen = pPattern->n; + showReq.payload = malloc(showReq.payloadLen); + strncpy(showReq.payload, pPattern->z, pPattern->n); } } else { SToken* pEpAddr = &pShowInfo->prefix; assert(pEpAddr->n > 0 && pEpAddr->type > 0); - - strncpy(pShowMsg->payload, pEpAddr->z, pEpAddr->n); - pShowMsg->payloadLen = htons(pEpAddr->n); + showReq.payloadLen = pEpAddr->n; + showReq.payload = malloc(showReq.payloadLen); + strncpy(showReq.payload, pEpAddr->z, pEpAddr->n); } if (pShowInfo->showType == TSDB_MGMT_TABLE_STB || pShowInfo->showType == TSDB_MGMT_TABLE_VGROUP) { @@ -124,22 +132,32 @@ SShowReq* buildShowMsg(SShowInfo* pShowInfo, SParseContext* pCtx, SMsgBuf* pMsgB if (pShowInfo->prefix.n > 0) { if (pShowInfo->prefix.n >= TSDB_DB_FNAME_LEN) { terrno = buildInvalidOperationMsg(pMsgBuf, "prefix name is too long"); - tfree(pShowMsg); + tFreeSShowReq(&showReq); return NULL; } tNameSetDbName(&n, pCtx->acctId, pShowInfo->prefix.z, pShowInfo->prefix.n); } else if (pCtx->db == NULL || strlen(pCtx->db) == 0) { terrno = buildInvalidOperationMsg(pMsgBuf, "database is not specified"); - tfree(pShowMsg); + tFreeSShowReq(&showReq); return NULL; } else { tNameSetDbName(&n, pCtx->acctId, pCtx->db, strlen(pCtx->db)); } - tNameGetFullDbName(&n, pShowMsg->db); + tNameGetFullDbName(&n, showReq.db); } - return pShowMsg; + int32_t tlen = tSerializeSShowReq(NULL, 0, &showReq); + void* pReq = malloc(tlen); + if (pReq == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + tSerializeSShowReq(pReq, tlen, &showReq); + tFreeSShowReq(&showReq); + *outputLen = tlen; + return pReq; } static int32_t setKeepOption(SCreateDbReq* pMsg, const SCreateDbInfo* pCreateDb, SMsgBuf* pMsgBuf) { @@ -147,9 +165,9 @@ static int32_t setKeepOption(SCreateDbReq* pMsg, const SCreateDbInfo* pCreateDb, const char* msg2 = "invalid keep value"; const char* msg3 = "invalid keep value, should be keep0 <= keep1 <= keep2"; - pMsg->daysToKeep0 = htonl(-1); - pMsg->daysToKeep1 = htonl(-1); - pMsg->daysToKeep2 = htonl(-1); + pMsg->daysToKeep0 = -1; + pMsg->daysToKeep1 = -1; + pMsg->daysToKeep2 = -1; SArray* pKeep = pCreateDb->keep; if (pKeep != NULL) { @@ -209,13 +227,13 @@ static int32_t setTimePrecision(SCreateDbReq* pMsg, const SCreateDbInfo* pCreate } static void doSetDbOptions(SCreateDbReq* pMsg, const SCreateDbInfo* pCreateDb) { - pMsg->cacheBlockSize = htonl(pCreateDb->cacheBlockSize); - pMsg->totalBlocks = htonl(pCreateDb->numOfBlocks); - pMsg->daysPerFile = htonl(pCreateDb->daysPerFile); - pMsg->commitTime = htonl((int32_t)pCreateDb->commitTime); - pMsg->minRows = htonl(pCreateDb->minRowsPerBlock); - pMsg->maxRows = htonl(pCreateDb->maxRowsPerBlock); - pMsg->fsyncPeriod = htonl(pCreateDb->fsyncPeriod); + pMsg->cacheBlockSize = pCreateDb->cacheBlockSize; + pMsg->totalBlocks = pCreateDb->numOfBlocks; + pMsg->daysPerFile = pCreateDb->daysPerFile; + pMsg->commitTime = (int32_t)pCreateDb->commitTime; + pMsg->minRows = pCreateDb->minRowsPerBlock; + pMsg->maxRows = pCreateDb->maxRowsPerBlock; + pMsg->fsyncPeriod = pCreateDb->fsyncPeriod; pMsg->compression = (int8_t)pCreateDb->compressionLevel; pMsg->walLevel = (char)pCreateDb->walLevel; pMsg->replications = pCreateDb->replica; @@ -223,7 +241,7 @@ static void doSetDbOptions(SCreateDbReq* pMsg, const SCreateDbInfo* pCreateDb) { pMsg->ignoreExist = pCreateDb->ignoreExists; pMsg->update = pCreateDb->update; pMsg->cacheLastRow = pCreateDb->cachelast; - pMsg->numOfVgroups = htonl(pCreateDb->numOfVgroups); + pMsg->numOfVgroups = pCreateDb->numOfVgroups; } int32_t setDbOptions(SCreateDbReq* pCreateDbMsg, const SCreateDbInfo* pCreateDbSql, SMsgBuf* pMsgBuf) { @@ -240,12 +258,104 @@ int32_t setDbOptions(SCreateDbReq* pCreateDbMsg, const SCreateDbInfo* pCreateDbS return TSDB_CODE_SUCCESS; } -SCreateDbReq* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, SParseContext* pCtx, SMsgBuf* pMsgBuf) { - SCreateDbReq* pCreateMsg = calloc(1, sizeof(SCreateDbReq)); - if (setDbOptions(pCreateMsg, pCreateDbInfo, pMsgBuf) != TSDB_CODE_SUCCESS) { - tfree(pCreateMsg); - terrno = TSDB_CODE_TSC_INVALID_OPERATION; +// can only perform the parameters based on the macro definitation +static int32_t doCheckDbOptions(SCreateDbReq* pCreate, SMsgBuf* pMsgBuf) { + char msg[512] = {0}; + if (pCreate->walLevel != -1 && (pCreate->walLevel < TSDB_MIN_WAL_LEVEL || pCreate->walLevel > TSDB_MAX_WAL_LEVEL)) { + snprintf(msg, tListLen(msg), "invalid db option walLevel: %d, only 1-2 allowed", pCreate->walLevel); + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + if (pCreate->replications != -1 && + (pCreate->replications < TSDB_MIN_DB_REPLICA_OPTION || pCreate->replications > TSDB_MAX_DB_REPLICA_OPTION)) { + snprintf(msg, tListLen(msg), "invalid db option replications: %d valid range: [%d, %d]", pCreate->replications, + TSDB_MIN_DB_REPLICA_OPTION, TSDB_MAX_DB_REPLICA_OPTION); + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + int32_t blocks = pCreate->totalBlocks; + if (blocks != -1 && (blocks < TSDB_MIN_TOTAL_BLOCKS || blocks > TSDB_MAX_TOTAL_BLOCKS)) { + snprintf(msg, tListLen(msg), "invalid db option totalBlocks: %d valid range: [%d, %d]", blocks, + TSDB_MIN_TOTAL_BLOCKS, TSDB_MAX_TOTAL_BLOCKS); + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + if (pCreate->quorum != -1 && + (pCreate->quorum < TSDB_MIN_DB_QUORUM_OPTION || pCreate->quorum > TSDB_MAX_DB_QUORUM_OPTION)) { + snprintf(msg, tListLen(msg), "invalid db option quorum: %d valid range: [%d, %d]", pCreate->quorum, + TSDB_MIN_DB_QUORUM_OPTION, TSDB_MAX_DB_QUORUM_OPTION); + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + int32_t val = pCreate->daysPerFile; + if (val != -1 && (val < TSDB_MIN_DAYS_PER_FILE || val > TSDB_MAX_DAYS_PER_FILE)) { + snprintf(msg, tListLen(msg), "invalid db option daysPerFile: %d valid range: [%d, %d]", val, TSDB_MIN_DAYS_PER_FILE, + TSDB_MAX_DAYS_PER_FILE); + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + val = pCreate->cacheBlockSize; + if (val != -1 && (val < TSDB_MIN_CACHE_BLOCK_SIZE || val > TSDB_MAX_CACHE_BLOCK_SIZE)) { + snprintf(msg, tListLen(msg), "invalid db option cacheBlockSize: %d valid range: [%d, %d]", val, + TSDB_MIN_CACHE_BLOCK_SIZE, TSDB_MAX_CACHE_BLOCK_SIZE); + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + if (pCreate->precision != TSDB_TIME_PRECISION_MILLI && pCreate->precision != TSDB_TIME_PRECISION_MICRO && + pCreate->precision != TSDB_TIME_PRECISION_NANO) { + snprintf(msg, tListLen(msg), "invalid db option timePrecision: %d valid value: [%d, %d, %d]", pCreate->precision, + TSDB_TIME_PRECISION_MILLI, TSDB_TIME_PRECISION_MICRO, TSDB_TIME_PRECISION_NANO); + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + val = pCreate->commitTime; + if (val != -1 && (val < TSDB_MIN_COMMIT_TIME || val > TSDB_MAX_COMMIT_TIME)) { + snprintf(msg, tListLen(msg), "invalid db option commitTime: %d valid range: [%d, %d]", val, TSDB_MIN_COMMIT_TIME, + TSDB_MAX_COMMIT_TIME); + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + val = pCreate->fsyncPeriod; + if (val != -1 && (val < TSDB_MIN_FSYNC_PERIOD || val > TSDB_MAX_FSYNC_PERIOD)) { + snprintf(msg, tListLen(msg), "invalid db option fsyncPeriod: %d valid range: [%d, %d]", val, TSDB_MIN_FSYNC_PERIOD, + TSDB_MAX_FSYNC_PERIOD); + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + if (pCreate->compression != -1 && + (pCreate->compression < TSDB_MIN_COMP_LEVEL || pCreate->compression > TSDB_MAX_COMP_LEVEL)) { + snprintf(msg, tListLen(msg), "invalid db option compression: %d valid range: [%d, %d]", pCreate->compression, + TSDB_MIN_COMP_LEVEL, TSDB_MAX_COMP_LEVEL); + return buildInvalidOperationMsg(pMsgBuf, msg); + } + + val = pCreate->numOfVgroups; + if (val < TSDB_MIN_VNODES_PER_DB || val > TSDB_MAX_VNODES_PER_DB) { + snprintf(msg, tListLen(msg), "invalid number of vgroups for DB:%d valid range: [%d, %d]", val, + TSDB_MIN_VNODES_PER_DB, TSDB_MAX_VNODES_PER_DB); + } + + val = pCreate->maxRows; + if (val < TSDB_MIN_MAX_ROW_FBLOCK || val > TSDB_MAX_MAX_ROW_FBLOCK) { + snprintf(msg, tListLen(msg), "invalid number of max rows in file block for DB:%d valid range: [%d, %d]", val, + TSDB_MIN_MAX_ROW_FBLOCK, TSDB_MAX_MAX_ROW_FBLOCK); + } + + val = pCreate->minRows; + if (val < TSDB_MIN_MIN_ROW_FBLOCK || val > TSDB_MAX_MIN_ROW_FBLOCK) { + snprintf(msg, tListLen(msg), "invalid number of min rows in file block for DB:%d valid range: [%d, %d]", val, + TSDB_MIN_MIN_ROW_FBLOCK, TSDB_MAX_MIN_ROW_FBLOCK); + } + + return TSDB_CODE_SUCCESS; +} + +char* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, int32_t* outputLen, SParseContext* pCtx, SMsgBuf* pMsgBuf) { + SCreateDbReq createReq = {0}; + + if (setDbOptions(&createReq, pCreateDbInfo, pMsgBuf) != TSDB_CODE_SUCCESS) { + terrno = TSDB_CODE_TSC_INVALID_OPERATION; return NULL; } @@ -256,11 +366,27 @@ SCreateDbReq* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, SParseContext* pCtx return NULL; } - tNameGetFullDbName(&name, pCreateMsg->db); - return pCreateMsg; + tNameGetFullDbName(&name, createReq.db); + + if (doCheckDbOptions(&createReq, pMsgBuf) != TSDB_CODE_SUCCESS) { + terrno = TSDB_CODE_TSC_INVALID_OPERATION; + return NULL; + } + + int32_t tlen = tSerializeSCreateDbReq(NULL, 0, &createReq); + void* pReq = malloc(tlen); + if (pReq == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + tSerializeSCreateDbReq(pReq, tlen, &createReq); + *outputLen = tlen; + return pReq; } -char* buildCreateStbReq(SCreateTableSql* pCreateTableSql, int32_t* len, SParseContext* pParseCtx, SMsgBuf* pMsgBuf) { +char* buildCreateStbReq(SCreateTableSql* pCreateTableSql, int32_t* outputLen, SParseContext* pParseCtx, + SMsgBuf* pMsgBuf) { SMCreateStbReq createReq = {0}; createReq.igExists = pCreateTableSql->existCheck ? 1 : 0; createReq.pColumns = pCreateTableSql->colInfo.pColumns; @@ -278,20 +404,19 @@ char* buildCreateStbReq(SCreateTableSql* pCreateTableSql, int32_t* len, SParseCo return NULL; } - int32_t tlen = tSerializeSMCreateStbReq(NULL, &createReq); + int32_t tlen = tSerializeSMCreateStbReq(NULL, 0, &createReq); void* pReq = malloc(tlen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - void* pBuf = pReq; - tSerializeSMCreateStbReq(&pBuf, &createReq); - *len = tlen; + tSerializeSMCreateStbReq(pReq, tlen, &createReq); + *outputLen = tlen; return pReq; } -char* buildDropStableReq(SSqlInfo* pInfo, int32_t* len, SParseContext* pParseCtx, SMsgBuf* pMsgBuf) { +char* buildDropStableReq(SSqlInfo* pInfo, int32_t* outputLen, SParseContext* pParseCtx, SMsgBuf* pMsgBuf) { SToken* tableName = taosArrayGet(pInfo->pMiscInfo->a, 0); SName name = {0}; @@ -307,20 +432,19 @@ char* buildDropStableReq(SSqlInfo* pInfo, int32_t* len, SParseContext* pParseCtx assert(code == TSDB_CODE_SUCCESS && name.type == TSDB_TABLE_NAME_T); dropReq.igNotExists = pInfo->pMiscInfo->existsCheck ? 1 : 0; - int32_t tlen = tSerializeSMDropStbReq(NULL, &dropReq); + int32_t tlen = tSerializeSMDropStbReq(NULL, 0, &dropReq); void* pReq = malloc(tlen); if (pReq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } - void* pBuf = pReq; - tSerializeSMDropStbReq(&pBuf, &dropReq); - *len = tlen; + tSerializeSMDropStbReq(pReq, tlen, &dropReq); + *outputLen = tlen; return pReq; } -char* buildCreateDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf) { +char* buildCreateDnodeMsg(SSqlInfo* pInfo, int32_t* outputLen, SMsgBuf* pMsgBuf) { const char* msg1 = "invalid host name (name too long, maximum length 128)"; const char* msg2 = "dnode name can not be string"; const char* msg3 = "port should be an integer that is less than 65535 and greater than 0"; @@ -365,11 +489,11 @@ char* buildCreateDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf) { } tSerializeSCreateDnodeReq(pReq, tlen, &createReq); - *len = tlen; + *outputLen = tlen; return pReq; } -char* buildDropDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf) { +char* buildDropDnodeMsg(SSqlInfo* pInfo, int32_t* outputLen, SMsgBuf* pMsgBuf) { SDropDnodeReq dropReq = {0}; SToken* pzName = taosArrayGet(pInfo->pMiscInfo->a, 0); @@ -390,6 +514,6 @@ char* buildDropDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf) { } tSerializeSDropDnodeReq(pReq, tlen, &dropReq); - *len = tlen; + *outputLen = tlen; return pReq; } \ No newline at end of file diff --git a/source/libs/parser/src/dCDAstProcess.c b/source/libs/parser/src/dCDAstProcess.c index 6f22d51c4f..b4125b0a24 100644 --- a/source/libs/parser/src/dCDAstProcess.c +++ b/source/libs/parser/src/dCDAstProcess.c @@ -114,105 +114,10 @@ static int32_t setShowInfo(SShowInfo* pShowInfo, SParseContext* pCtx, void** out } *pEpSet = pCtx->mgmtEpSet; - *output = buildShowMsg(pShowInfo, pCtx, pMsgBuf); + *output = buildShowMsg(pShowInfo, outputLen, pCtx, pMsgBuf); if (*output == NULL) { return terrno; } - - *outputLen = sizeof(SShowReq) /* + htons(pShowMsg->payloadLen)*/; - } - - return TSDB_CODE_SUCCESS; -} - -// can only perform the parameters based on the macro definitation -static int32_t doCheckDbOptions(SCreateDbReq* pCreate, SMsgBuf* pMsgBuf) { - char msg[512] = {0}; - - if (pCreate->walLevel != -1 && (pCreate->walLevel < TSDB_MIN_WAL_LEVEL || pCreate->walLevel > TSDB_MAX_WAL_LEVEL)) { - snprintf(msg, tListLen(msg), "invalid db option walLevel: %d, only 1-2 allowed", pCreate->walLevel); - return buildInvalidOperationMsg(pMsgBuf, msg); - } - - if (pCreate->replications != -1 && - (pCreate->replications < TSDB_MIN_DB_REPLICA_OPTION || pCreate->replications > TSDB_MAX_DB_REPLICA_OPTION)) { - snprintf(msg, tListLen(msg), "invalid db option replications: %d valid range: [%d, %d]", pCreate->replications, - TSDB_MIN_DB_REPLICA_OPTION, TSDB_MAX_DB_REPLICA_OPTION); - return buildInvalidOperationMsg(pMsgBuf, msg); - } - - int32_t blocks = ntohl(pCreate->totalBlocks); - if (blocks != -1 && (blocks < TSDB_MIN_TOTAL_BLOCKS || blocks > TSDB_MAX_TOTAL_BLOCKS)) { - snprintf(msg, tListLen(msg), "invalid db option totalBlocks: %d valid range: [%d, %d]", blocks, - TSDB_MIN_TOTAL_BLOCKS, TSDB_MAX_TOTAL_BLOCKS); - return buildInvalidOperationMsg(pMsgBuf, msg); - } - - if (pCreate->quorum != -1 && - (pCreate->quorum < TSDB_MIN_DB_QUORUM_OPTION || pCreate->quorum > TSDB_MAX_DB_QUORUM_OPTION)) { - snprintf(msg, tListLen(msg), "invalid db option quorum: %d valid range: [%d, %d]", pCreate->quorum, - TSDB_MIN_DB_QUORUM_OPTION, TSDB_MAX_DB_QUORUM_OPTION); - return buildInvalidOperationMsg(pMsgBuf, msg); - } - - int32_t val = htonl(pCreate->daysPerFile); - if (val != -1 && (val < TSDB_MIN_DAYS_PER_FILE || val > TSDB_MAX_DAYS_PER_FILE)) { - snprintf(msg, tListLen(msg), "invalid db option daysPerFile: %d valid range: [%d, %d]", val, TSDB_MIN_DAYS_PER_FILE, - TSDB_MAX_DAYS_PER_FILE); - return buildInvalidOperationMsg(pMsgBuf, msg); - } - - val = htonl(pCreate->cacheBlockSize); - if (val != -1 && (val < TSDB_MIN_CACHE_BLOCK_SIZE || val > TSDB_MAX_CACHE_BLOCK_SIZE)) { - snprintf(msg, tListLen(msg), "invalid db option cacheBlockSize: %d valid range: [%d, %d]", val, - TSDB_MIN_CACHE_BLOCK_SIZE, TSDB_MAX_CACHE_BLOCK_SIZE); - return buildInvalidOperationMsg(pMsgBuf, msg); - } - - if (pCreate->precision != TSDB_TIME_PRECISION_MILLI && pCreate->precision != TSDB_TIME_PRECISION_MICRO && - pCreate->precision != TSDB_TIME_PRECISION_NANO) { - snprintf(msg, tListLen(msg), "invalid db option timePrecision: %d valid value: [%d, %d, %d]", pCreate->precision, - TSDB_TIME_PRECISION_MILLI, TSDB_TIME_PRECISION_MICRO, TSDB_TIME_PRECISION_NANO); - return buildInvalidOperationMsg(pMsgBuf, msg); - } - - val = htonl(pCreate->commitTime); - if (val != -1 && (val < TSDB_MIN_COMMIT_TIME || val > TSDB_MAX_COMMIT_TIME)) { - snprintf(msg, tListLen(msg), "invalid db option commitTime: %d valid range: [%d, %d]", val, TSDB_MIN_COMMIT_TIME, - TSDB_MAX_COMMIT_TIME); - return buildInvalidOperationMsg(pMsgBuf, msg); - } - - val = htonl(pCreate->fsyncPeriod); - if (val != -1 && (val < TSDB_MIN_FSYNC_PERIOD || val > TSDB_MAX_FSYNC_PERIOD)) { - snprintf(msg, tListLen(msg), "invalid db option fsyncPeriod: %d valid range: [%d, %d]", val, TSDB_MIN_FSYNC_PERIOD, - TSDB_MAX_FSYNC_PERIOD); - return buildInvalidOperationMsg(pMsgBuf, msg); - } - - if (pCreate->compression != -1 && - (pCreate->compression < TSDB_MIN_COMP_LEVEL || pCreate->compression > TSDB_MAX_COMP_LEVEL)) { - snprintf(msg, tListLen(msg), "invalid db option compression: %d valid range: [%d, %d]", pCreate->compression, - TSDB_MIN_COMP_LEVEL, TSDB_MAX_COMP_LEVEL); - return buildInvalidOperationMsg(pMsgBuf, msg); - } - - val = htonl(pCreate->numOfVgroups); - if (val < TSDB_MIN_VNODES_PER_DB || val > TSDB_MAX_VNODES_PER_DB) { - snprintf(msg, tListLen(msg), "invalid number of vgroups for DB:%d valid range: [%d, %d]", val, - TSDB_MIN_VNODES_PER_DB, TSDB_MAX_VNODES_PER_DB); - } - - val = htonl(pCreate->maxRows); - if (val < TSDB_MIN_MAX_ROW_FBLOCK || val > TSDB_MAX_MAX_ROW_FBLOCK) { - snprintf(msg, tListLen(msg), "invalid number of max rows in file block for DB:%d valid range: [%d, %d]", val, - TSDB_MIN_MAX_ROW_FBLOCK, TSDB_MAX_MAX_ROW_FBLOCK); - } - - val = htonl(pCreate->minRows); - if (val < TSDB_MIN_MIN_ROW_FBLOCK || val > TSDB_MAX_MIN_ROW_FBLOCK) { - snprintf(msg, tListLen(msg), "invalid number of min rows in file block for DB:%d valid range: [%d, %d]", val, - TSDB_MIN_MIN_ROW_FBLOCK, TSDB_MAX_MIN_ROW_FBLOCK); } return TSDB_CODE_SUCCESS; @@ -852,11 +757,15 @@ SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseContext* pCtx, ch goto _error; } - SUseDbReq* pUseDbMsg = (SUseDbReq*)calloc(1, sizeof(SUseDbReq)); - tNameExtractFullName(&n, pUseDbMsg->db); + SUseDbReq usedbReq = {0}; + tNameExtractFullName(&n, usedbReq.db); - pDcl->pMsg = (char*)pUseDbMsg; - pDcl->msgLen = sizeof(SUseDbReq); + int32_t bufLen = tSerializeSUseDbReq(NULL, 0, &usedbReq); + void* pBuf = malloc(bufLen); + tSerializeSUseDbReq(pBuf, bufLen, &usedbReq); + + pDcl->pMsg = pBuf; + pDcl->msgLen = bufLen; pDcl->msgType = TDMT_MND_USE_DB; break; } @@ -880,14 +789,11 @@ SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseContext* pCtx, ch goto _error; } - SCreateDbReq* pCreateMsg = buildCreateDbMsg(pCreateDB, pCtx, pMsgBuf); - if (doCheckDbOptions(pCreateMsg, pMsgBuf) != TSDB_CODE_SUCCESS) { - code = TSDB_CODE_TSC_INVALID_OPERATION; - goto _error; - } + int32_t bufLen = 0; + char* pBuf = buildCreateDbMsg(pCreateDB, &bufLen, pCtx, pMsgBuf); - pDcl->pMsg = (char*)pCreateMsg; - pDcl->msgLen = sizeof(SCreateDbReq); + pDcl->pMsg = pBuf; + pDcl->msgLen = bufLen; pDcl->msgType = (pInfo->type == TSDB_SQL_CREATE_DB) ? TDMT_MND_CREATE_DB : TDMT_MND_ALTER_DB; break; } @@ -905,15 +811,18 @@ SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseContext* pCtx, ch goto _error; } - SDropDbReq* pDropDbMsg = (SDropDbReq*)calloc(1, sizeof(SDropDbReq)); - - code = tNameExtractFullName(&name, pDropDbMsg->db); - pDropDbMsg->ignoreNotExists = pInfo->pMiscInfo->existsCheck ? 1 : 0; + SDropDbReq dropdbReq = {0}; + code = tNameExtractFullName(&name, dropdbReq.db); + dropdbReq.ignoreNotExists = pInfo->pMiscInfo->existsCheck ? 1 : 0; assert(code == TSDB_CODE_SUCCESS && name.type == TSDB_DB_NAME_T); + int32_t bufLen = tSerializeSDropDbReq(NULL, 0, &dropdbReq); + void* pBuf = malloc(bufLen); + tSerializeSDropDbReq(pBuf, bufLen, &dropdbReq); + pDcl->msgType = TDMT_MND_DROP_DB; - pDcl->msgLen = sizeof(SDropDbReq); - pDcl->pMsg = (char*)pDropDbMsg; + pDcl->msgLen = bufLen; + pDcl->pMsg = pBuf; break; } diff --git a/source/libs/parser/src/dataBlockMgt.c b/source/libs/parser/src/dataBlockMgt.c index eefaa2c147..381dec4f15 100644 --- a/source/libs/parser/src/dataBlockMgt.c +++ b/source/libs/parser/src/dataBlockMgt.c @@ -417,7 +417,7 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SB } int32_t mergeTableDataBlocks(SHashObj* pHashObj, int8_t schemaAttached, uint8_t payloadType, SArray** pVgDataBlocks) { - const int INSERT_HEAD_SIZE = sizeof(SSubmitMsg); + const int INSERT_HEAD_SIZE = sizeof(SSubmitReq); int code = 0; bool isRawPayload = IS_RAW_PAYLOAD(payloadType); SHashObj* pVnodeDataBlockHashList = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false); diff --git a/source/libs/parser/src/insertParser.c b/source/libs/parser/src/insertParser.c index 08fe0852b1..745982e869 100644 --- a/source/libs/parser/src/insertParser.c +++ b/source/libs/parser/src/insertParser.c @@ -121,7 +121,7 @@ static int32_t findCol(SToken* pColname, int32_t start, int32_t end, SSchema* pS } static void buildMsgHeader(SVgDataBlocks* blocks) { - SSubmitMsg* submit = (SSubmitMsg*)blocks->pData; + SSubmitReq* submit = (SSubmitReq*)blocks->pData; submit->header.vgId = htonl(blocks->vg.vgId); submit->header.contLen = htonl(blocks->size); submit->length = submit->header.contLen; @@ -322,7 +322,7 @@ static int32_t parseBoundColumns(SInsertParseContext* pCxt, SParsedDataColInfo* } lastColIdx = index; pColList->cols[index].valStat = VAL_STAT_HAS; - pColList->boundedColumns[pColList->numOfBound] = index; + pColList->boundedColumns[pColList->numOfBound] = index + PRIMARYKEY_TIMESTAMP_COL_ID; ++pColList->numOfBound; switch (pSchema[t].type) { case TSDB_DATA_TYPE_BINARY: diff --git a/source/libs/parser/src/new_sql.c b/source/libs/parser/src/new_sql.c index b7a82e5d97..2d0c4b7eba 100644 --- a/source/libs/parser/src/new_sql.c +++ b/source/libs/parser/src/new_sql.c @@ -109,21 +109,21 @@ #endif /************* Begin control #defines *****************************************/ #define YYCODETYPE unsigned char -#define YYNOCODE 125 +#define YYNOCODE 126 #define YYACTIONTYPE unsigned short int #define NewParseTOKENTYPE SToken typedef union { int yyinit; NewParseTOKENTYPE yy0; - EOperatorType yy40; - EFillMode yy44; - SToken yy79; - ENullOrder yy107; - EJoinType yy162; - SNodeList* yy174; - EOrder yy188; - SNode* yy212; - bool yy237; + EFillMode yy18; + SToken yy29; + EJoinType yy36; + SNode* yy56; + ENullOrder yy109; + EOperatorType yy128; + bool yy173; + SNodeList* yy208; + EOrder yy218; } YYMINORTYPE; #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 @@ -138,17 +138,17 @@ typedef union { #define NewParseCTX_PARAM #define NewParseCTX_FETCH #define NewParseCTX_STORE -#define YYNSTATE 147 -#define YYNRULE 138 +#define YYNSTATE 148 +#define YYNRULE 140 #define YYNTOKEN 72 -#define YY_MAX_SHIFT 146 -#define YY_MIN_SHIFTREDUCE 243 -#define YY_MAX_SHIFTREDUCE 380 -#define YY_ERROR_ACTION 381 -#define YY_ACCEPT_ACTION 382 -#define YY_NO_ACTION 383 -#define YY_MIN_REDUCE 384 -#define YY_MAX_REDUCE 521 +#define YY_MAX_SHIFT 147 +#define YY_MIN_SHIFTREDUCE 245 +#define YY_MAX_SHIFTREDUCE 384 +#define YY_ERROR_ACTION 385 +#define YY_ACCEPT_ACTION 386 +#define YY_NO_ACTION 387 +#define YY_MIN_REDUCE 388 +#define YY_MAX_REDUCE 527 /************* End control #defines *******************************************/ #define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) @@ -215,208 +215,215 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (737) +#define YY_ACTTAB_COUNT (750) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 393, 391, 92, 23, 66, 394, 391, 469, 30, 28, - /* 10 */ 26, 25, 24, 401, 391, 141, 416, 141, 416, 142, - /* 20 */ 72, 112, 46, 402, 265, 405, 22, 87, 96, 117, - /* 30 */ 283, 284, 285, 286, 287, 288, 289, 291, 292, 293, - /* 40 */ 30, 28, 26, 25, 24, 401, 391, 141, 416, 141, - /* 50 */ 416, 142, 146, 116, 46, 402, 54, 405, 22, 87, - /* 60 */ 96, 514, 283, 284, 285, 286, 287, 288, 289, 291, - /* 70 */ 292, 293, 131, 401, 391, 128, 9, 141, 416, 142, - /* 80 */ 4, 318, 39, 402, 54, 405, 441, 109, 29, 27, - /* 90 */ 86, 437, 132, 513, 63, 245, 246, 247, 248, 143, - /* 100 */ 251, 500, 101, 1, 118, 113, 111, 10, 245, 246, - /* 110 */ 247, 248, 143, 251, 500, 53, 453, 401, 391, 498, - /* 120 */ 258, 141, 416, 142, 126, 54, 40, 402, 499, 405, - /* 130 */ 441, 453, 498, 450, 94, 437, 453, 401, 391, 69, - /* 140 */ 330, 141, 416, 142, 473, 417, 40, 402, 449, 405, - /* 150 */ 441, 134, 256, 448, 94, 437, 41, 401, 391, 344, - /* 160 */ 8, 141, 416, 142, 496, 74, 40, 402, 347, 405, - /* 170 */ 441, 458, 110, 318, 94, 437, 131, 401, 391, 107, - /* 180 */ 322, 141, 416, 142, 457, 57, 78, 402, 60, 405, - /* 190 */ 128, 9, 29, 27, 108, 345, 346, 348, 349, 245, - /* 200 */ 246, 247, 248, 143, 251, 500, 101, 1, 19, 31, - /* 210 */ 133, 10, 295, 54, 30, 28, 26, 25, 24, 53, - /* 220 */ 29, 27, 321, 498, 26, 25, 24, 245, 246, 247, - /* 230 */ 248, 143, 251, 480, 101, 5, 401, 391, 7, 6, - /* 240 */ 141, 416, 142, 7, 6, 40, 402, 251, 405, 441, - /* 250 */ 122, 138, 280, 440, 437, 55, 398, 54, 396, 401, - /* 260 */ 391, 301, 105, 141, 416, 142, 379, 380, 40, 402, - /* 270 */ 106, 405, 441, 29, 27, 323, 121, 437, 479, 59, - /* 280 */ 245, 246, 247, 248, 143, 251, 93, 101, 5, 30, - /* 290 */ 28, 26, 25, 24, 401, 391, 139, 131, 129, 416, - /* 300 */ 142, 125, 44, 45, 402, 265, 405, 3, 460, 135, - /* 310 */ 42, 120, 20, 30, 28, 26, 25, 24, 62, 67, - /* 320 */ 446, 124, 290, 123, 70, 294, 500, 259, 104, 21, - /* 330 */ 102, 466, 48, 64, 2, 30, 28, 26, 25, 24, - /* 340 */ 53, 16, 31, 423, 498, 262, 71, 318, 255, 29, - /* 350 */ 27, 130, 136, 43, 258, 454, 245, 246, 247, 248, - /* 360 */ 143, 251, 32, 101, 5, 29, 27, 30, 28, 26, - /* 370 */ 25, 24, 245, 246, 247, 248, 143, 251, 65, 101, - /* 380 */ 1, 401, 391, 259, 470, 141, 416, 142, 97, 140, - /* 390 */ 40, 402, 515, 405, 441, 29, 27, 497, 137, 438, - /* 400 */ 73, 256, 245, 246, 247, 248, 143, 251, 56, 101, - /* 410 */ 5, 13, 30, 28, 26, 25, 24, 31, 14, 401, - /* 420 */ 391, 341, 58, 141, 416, 142, 376, 377, 84, 402, - /* 430 */ 100, 405, 401, 391, 35, 343, 129, 416, 142, 47, - /* 440 */ 61, 45, 402, 337, 405, 36, 114, 401, 391, 382, - /* 450 */ 144, 141, 416, 142, 336, 37, 84, 402, 103, 405, - /* 460 */ 115, 396, 401, 391, 18, 315, 141, 416, 142, 467, - /* 470 */ 6, 80, 402, 15, 405, 281, 314, 401, 391, 500, - /* 480 */ 68, 141, 416, 142, 33, 34, 84, 402, 95, 405, - /* 490 */ 395, 52, 17, 53, 263, 401, 391, 498, 370, 141, - /* 500 */ 416, 142, 11, 119, 79, 402, 38, 405, 401, 391, - /* 510 */ 365, 75, 141, 416, 142, 364, 98, 81, 402, 369, - /* 520 */ 405, 368, 401, 391, 385, 99, 141, 416, 142, 249, - /* 530 */ 12, 76, 402, 384, 405, 145, 383, 401, 391, 383, - /* 540 */ 383, 141, 416, 142, 383, 383, 82, 402, 383, 405, - /* 550 */ 383, 383, 401, 391, 383, 383, 141, 416, 142, 383, - /* 560 */ 383, 77, 402, 383, 405, 383, 383, 383, 383, 383, - /* 570 */ 401, 391, 383, 383, 141, 416, 142, 383, 383, 83, - /* 580 */ 402, 383, 405, 401, 391, 383, 383, 141, 416, 142, - /* 590 */ 383, 383, 413, 402, 383, 405, 383, 401, 391, 383, - /* 600 */ 383, 141, 416, 142, 383, 383, 412, 402, 383, 405, - /* 610 */ 383, 383, 401, 391, 383, 383, 141, 416, 142, 383, - /* 620 */ 383, 411, 402, 383, 405, 383, 383, 401, 391, 383, - /* 630 */ 383, 141, 416, 142, 383, 383, 90, 402, 383, 405, - /* 640 */ 383, 383, 383, 383, 383, 401, 391, 383, 383, 141, - /* 650 */ 416, 142, 383, 383, 89, 402, 383, 405, 401, 391, - /* 660 */ 383, 383, 141, 416, 142, 383, 383, 91, 402, 383, - /* 670 */ 405, 383, 401, 391, 383, 383, 141, 416, 142, 383, - /* 680 */ 383, 88, 402, 383, 405, 125, 44, 401, 391, 383, - /* 690 */ 383, 141, 416, 142, 42, 383, 85, 402, 383, 405, - /* 700 */ 125, 44, 127, 49, 446, 447, 383, 451, 383, 42, - /* 710 */ 383, 383, 383, 383, 125, 44, 383, 383, 50, 446, - /* 720 */ 447, 383, 451, 42, 383, 383, 383, 383, 383, 383, - /* 730 */ 383, 383, 51, 446, 447, 383, 451, + /* 0 */ 397, 395, 93, 23, 72, 398, 395, 73, 30, 28, + /* 10 */ 26, 25, 24, 405, 395, 142, 420, 142, 420, 143, + /* 20 */ 110, 113, 80, 406, 267, 409, 22, 88, 97, 147, + /* 30 */ 285, 286, 287, 288, 289, 290, 291, 293, 294, 295, + /* 40 */ 30, 28, 26, 25, 24, 405, 395, 142, 420, 142, + /* 50 */ 420, 143, 118, 117, 46, 406, 260, 409, 22, 88, + /* 60 */ 97, 55, 285, 286, 287, 288, 289, 290, 291, 293, + /* 70 */ 294, 295, 132, 405, 395, 70, 65, 142, 420, 143, + /* 80 */ 127, 10, 39, 406, 55, 409, 445, 119, 114, 112, + /* 90 */ 87, 441, 334, 133, 519, 247, 248, 249, 250, 144, + /* 100 */ 253, 459, 506, 55, 258, 405, 395, 475, 261, 128, + /* 110 */ 420, 143, 127, 10, 40, 406, 54, 409, 445, 456, + /* 120 */ 504, 459, 95, 441, 49, 405, 395, 56, 135, 142, + /* 130 */ 420, 143, 383, 384, 81, 406, 71, 409, 20, 455, + /* 140 */ 405, 395, 103, 472, 128, 420, 143, 125, 292, 40, + /* 150 */ 406, 296, 409, 445, 29, 27, 421, 95, 441, 49, + /* 160 */ 75, 247, 248, 249, 250, 144, 253, 120, 102, 1, + /* 170 */ 459, 9, 8, 11, 26, 25, 24, 464, 473, 322, + /* 180 */ 405, 395, 386, 145, 142, 420, 143, 134, 454, 40, + /* 190 */ 406, 55, 409, 445, 303, 29, 27, 95, 441, 518, + /* 200 */ 2, 326, 247, 248, 249, 250, 144, 253, 479, 102, + /* 210 */ 1, 405, 395, 506, 11, 142, 420, 143, 6, 322, + /* 220 */ 40, 406, 261, 409, 445, 9, 8, 54, 95, 441, + /* 230 */ 518, 504, 405, 395, 506, 139, 142, 420, 143, 502, + /* 240 */ 111, 40, 406, 325, 409, 445, 486, 136, 505, 95, + /* 250 */ 441, 518, 504, 476, 29, 27, 327, 108, 124, 45, + /* 260 */ 463, 247, 248, 249, 250, 144, 253, 43, 102, 1, + /* 270 */ 57, 42, 351, 11, 348, 253, 126, 50, 452, 453, + /* 280 */ 140, 457, 132, 405, 395, 106, 107, 142, 420, 143, + /* 290 */ 137, 59, 79, 406, 62, 409, 29, 27, 109, 349, + /* 300 */ 350, 352, 353, 247, 248, 249, 250, 144, 253, 485, + /* 310 */ 102, 7, 506, 30, 28, 26, 25, 24, 31, 405, + /* 320 */ 395, 297, 94, 142, 420, 143, 54, 5, 41, 406, + /* 330 */ 504, 409, 445, 55, 61, 466, 444, 441, 405, 395, + /* 340 */ 121, 64, 142, 420, 143, 48, 105, 41, 406, 4, + /* 350 */ 409, 445, 130, 322, 282, 129, 441, 132, 66, 257, + /* 360 */ 31, 124, 45, 264, 29, 27, 131, 402, 44, 400, + /* 370 */ 43, 247, 248, 249, 250, 144, 253, 260, 102, 7, + /* 380 */ 68, 452, 123, 32, 122, 29, 27, 506, 460, 67, + /* 390 */ 16, 427, 247, 248, 249, 250, 144, 253, 521, 102, + /* 400 */ 1, 54, 98, 405, 395, 504, 141, 142, 420, 143, + /* 410 */ 138, 503, 41, 406, 258, 409, 445, 29, 27, 74, + /* 420 */ 3, 442, 14, 31, 247, 248, 249, 250, 144, 253, + /* 430 */ 58, 102, 7, 115, 345, 60, 405, 395, 35, 347, + /* 440 */ 142, 420, 143, 47, 63, 85, 406, 101, 409, 405, + /* 450 */ 395, 116, 341, 142, 420, 143, 36, 400, 85, 406, + /* 460 */ 104, 409, 405, 395, 340, 18, 142, 420, 143, 37, + /* 470 */ 69, 85, 406, 96, 409, 405, 395, 319, 15, 142, + /* 480 */ 420, 143, 318, 33, 46, 406, 34, 409, 405, 395, + /* 490 */ 8, 399, 142, 420, 143, 53, 283, 82, 406, 265, + /* 500 */ 409, 405, 395, 374, 17, 142, 420, 143, 12, 38, + /* 510 */ 77, 406, 369, 409, 368, 99, 405, 395, 373, 372, + /* 520 */ 142, 420, 143, 100, 520, 83, 406, 76, 409, 405, + /* 530 */ 395, 251, 13, 142, 420, 143, 389, 388, 78, 406, + /* 540 */ 146, 409, 405, 395, 387, 387, 142, 420, 143, 387, + /* 550 */ 387, 84, 406, 387, 409, 405, 395, 387, 387, 142, + /* 560 */ 420, 143, 387, 387, 417, 406, 387, 409, 405, 395, + /* 570 */ 387, 387, 142, 420, 143, 387, 387, 416, 406, 387, + /* 580 */ 409, 405, 395, 387, 387, 142, 420, 143, 387, 387, + /* 590 */ 415, 406, 387, 409, 387, 405, 395, 387, 387, 142, + /* 600 */ 420, 143, 387, 387, 91, 406, 387, 409, 405, 395, + /* 610 */ 387, 387, 142, 420, 143, 387, 387, 90, 406, 387, + /* 620 */ 409, 405, 395, 387, 387, 142, 420, 143, 387, 387, + /* 630 */ 92, 406, 387, 409, 405, 395, 387, 387, 142, 420, + /* 640 */ 143, 387, 387, 89, 406, 387, 409, 405, 395, 387, + /* 650 */ 387, 142, 420, 143, 387, 387, 86, 406, 387, 409, + /* 660 */ 124, 45, 387, 387, 387, 124, 45, 387, 387, 43, + /* 670 */ 387, 387, 387, 387, 43, 387, 387, 387, 387, 51, + /* 680 */ 452, 453, 387, 457, 52, 452, 453, 387, 457, 30, + /* 690 */ 28, 26, 25, 24, 19, 387, 387, 387, 387, 387, + /* 700 */ 30, 28, 26, 25, 24, 21, 387, 387, 387, 387, + /* 710 */ 387, 30, 28, 26, 25, 24, 387, 387, 387, 387, + /* 720 */ 30, 28, 26, 25, 24, 387, 387, 387, 387, 387, + /* 730 */ 387, 387, 387, 387, 387, 387, 267, 387, 387, 387, + /* 740 */ 387, 387, 387, 387, 387, 387, 387, 387, 380, 381, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 74, 75, 76, 88, 89, 74, 75, 82, 8, 9, + /* 0 */ 74, 75, 76, 88, 89, 74, 75, 124, 8, 9, /* 10 */ 10, 11, 12, 74, 75, 78, 79, 78, 79, 80, - /* 20 */ 123, 84, 83, 84, 24, 86, 26, 27, 28, 22, + /* 20 */ 115, 84, 83, 84, 24, 86, 26, 27, 28, 13, /* 30 */ 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, /* 40 */ 8, 9, 10, 11, 12, 74, 75, 78, 79, 78, - /* 50 */ 79, 80, 13, 84, 83, 84, 45, 86, 26, 27, - /* 60 */ 28, 122, 30, 31, 32, 33, 34, 35, 36, 37, - /* 70 */ 38, 39, 73, 74, 75, 22, 23, 78, 79, 80, - /* 80 */ 43, 44, 83, 84, 45, 86, 87, 114, 8, 9, - /* 90 */ 91, 92, 121, 122, 107, 15, 16, 17, 18, 19, - /* 100 */ 20, 102, 22, 23, 50, 51, 52, 27, 15, 16, - /* 110 */ 17, 18, 19, 20, 102, 116, 81, 74, 75, 120, - /* 120 */ 22, 78, 79, 80, 100, 45, 83, 84, 116, 86, - /* 130 */ 87, 81, 120, 98, 91, 92, 81, 74, 75, 41, - /* 140 */ 10, 78, 79, 80, 101, 79, 83, 84, 98, 86, - /* 150 */ 87, 3, 22, 98, 91, 92, 21, 74, 75, 24, - /* 160 */ 103, 78, 79, 80, 101, 117, 83, 84, 29, 86, - /* 170 */ 87, 42, 54, 44, 91, 92, 73, 74, 75, 53, - /* 180 */ 4, 78, 79, 80, 101, 21, 83, 84, 24, 86, - /* 190 */ 22, 23, 8, 9, 55, 56, 57, 58, 59, 15, - /* 200 */ 16, 17, 18, 19, 20, 102, 22, 23, 2, 21, - /* 210 */ 62, 27, 24, 45, 8, 9, 10, 11, 12, 116, - /* 220 */ 8, 9, 46, 120, 10, 11, 12, 15, 16, 17, - /* 230 */ 18, 19, 20, 113, 22, 23, 74, 75, 1, 2, - /* 240 */ 78, 79, 80, 1, 2, 83, 84, 20, 86, 87, - /* 250 */ 27, 21, 29, 91, 92, 112, 23, 45, 25, 74, - /* 260 */ 75, 24, 75, 78, 79, 80, 70, 71, 83, 84, - /* 270 */ 75, 86, 87, 8, 9, 10, 91, 92, 113, 112, - /* 280 */ 15, 16, 17, 18, 19, 20, 75, 22, 23, 8, - /* 290 */ 9, 10, 11, 12, 74, 75, 66, 73, 78, 79, - /* 300 */ 80, 77, 78, 83, 84, 24, 86, 61, 109, 21, - /* 310 */ 86, 60, 26, 8, 9, 10, 11, 12, 108, 95, - /* 320 */ 96, 97, 36, 99, 104, 39, 102, 22, 48, 2, - /* 330 */ 110, 111, 106, 105, 47, 8, 9, 10, 11, 12, - /* 340 */ 116, 23, 21, 90, 120, 24, 41, 44, 22, 8, - /* 350 */ 9, 10, 64, 78, 22, 81, 15, 16, 17, 18, - /* 360 */ 19, 20, 40, 22, 23, 8, 9, 8, 9, 10, - /* 370 */ 11, 12, 15, 16, 17, 18, 19, 20, 93, 22, - /* 380 */ 23, 74, 75, 22, 82, 78, 79, 80, 69, 65, - /* 390 */ 83, 84, 124, 86, 87, 8, 9, 119, 63, 92, - /* 400 */ 118, 22, 15, 16, 17, 18, 19, 20, 24, 22, - /* 410 */ 23, 21, 8, 9, 10, 11, 12, 21, 49, 74, - /* 420 */ 75, 24, 23, 78, 79, 80, 67, 68, 83, 84, - /* 430 */ 85, 86, 74, 75, 21, 24, 78, 79, 80, 23, - /* 440 */ 23, 83, 84, 24, 86, 23, 15, 74, 75, 72, - /* 450 */ 73, 78, 79, 80, 24, 23, 83, 84, 85, 86, - /* 460 */ 21, 25, 74, 75, 21, 24, 78, 79, 80, 111, - /* 470 */ 2, 83, 84, 49, 86, 29, 24, 74, 75, 102, - /* 480 */ 25, 78, 79, 80, 42, 21, 83, 84, 85, 86, - /* 490 */ 25, 25, 21, 116, 24, 74, 75, 120, 24, 78, - /* 500 */ 79, 80, 49, 115, 83, 84, 4, 86, 74, 75, - /* 510 */ 15, 25, 78, 79, 80, 15, 15, 83, 84, 15, - /* 520 */ 86, 15, 74, 75, 0, 15, 78, 79, 80, 17, - /* 530 */ 23, 83, 84, 0, 86, 14, 125, 74, 75, 125, - /* 540 */ 125, 78, 79, 80, 125, 125, 83, 84, 125, 86, - /* 550 */ 125, 125, 74, 75, 125, 125, 78, 79, 80, 125, - /* 560 */ 125, 83, 84, 125, 86, 125, 125, 125, 125, 125, - /* 570 */ 74, 75, 125, 125, 78, 79, 80, 125, 125, 83, - /* 580 */ 84, 125, 86, 74, 75, 125, 125, 78, 79, 80, - /* 590 */ 125, 125, 83, 84, 125, 86, 125, 74, 75, 125, - /* 600 */ 125, 78, 79, 80, 125, 125, 83, 84, 125, 86, - /* 610 */ 125, 125, 74, 75, 125, 125, 78, 79, 80, 125, - /* 620 */ 125, 83, 84, 125, 86, 125, 125, 74, 75, 125, - /* 630 */ 125, 78, 79, 80, 125, 125, 83, 84, 125, 86, - /* 640 */ 125, 125, 125, 125, 125, 74, 75, 125, 125, 78, - /* 650 */ 79, 80, 125, 125, 83, 84, 125, 86, 74, 75, - /* 660 */ 125, 125, 78, 79, 80, 125, 125, 83, 84, 125, - /* 670 */ 86, 125, 74, 75, 125, 125, 78, 79, 80, 125, - /* 680 */ 125, 83, 84, 125, 86, 77, 78, 74, 75, 125, - /* 690 */ 125, 78, 79, 80, 86, 125, 83, 84, 125, 86, - /* 700 */ 77, 78, 94, 95, 96, 97, 125, 99, 125, 86, - /* 710 */ 125, 125, 125, 125, 77, 78, 125, 125, 95, 96, - /* 720 */ 97, 125, 99, 86, 125, 125, 125, 125, 125, 125, - /* 730 */ 125, 125, 95, 96, 97, 125, 99, + /* 50 */ 79, 80, 22, 84, 83, 84, 22, 86, 26, 27, + /* 60 */ 28, 45, 30, 31, 32, 33, 34, 35, 36, 37, + /* 70 */ 38, 39, 73, 74, 75, 41, 108, 78, 79, 80, + /* 80 */ 22, 23, 83, 84, 45, 86, 87, 50, 51, 52, + /* 90 */ 91, 92, 10, 122, 123, 15, 16, 17, 18, 19, + /* 100 */ 20, 81, 103, 45, 22, 74, 75, 82, 22, 78, + /* 110 */ 79, 80, 22, 23, 83, 84, 117, 86, 87, 99, + /* 120 */ 121, 81, 91, 92, 93, 74, 75, 41, 3, 78, + /* 130 */ 79, 80, 70, 71, 83, 84, 105, 86, 26, 99, + /* 140 */ 74, 75, 111, 112, 78, 79, 80, 101, 36, 83, + /* 150 */ 84, 39, 86, 87, 8, 9, 79, 91, 92, 93, + /* 160 */ 118, 15, 16, 17, 18, 19, 20, 116, 22, 23, + /* 170 */ 81, 1, 2, 27, 10, 11, 12, 42, 112, 44, + /* 180 */ 74, 75, 72, 73, 78, 79, 80, 62, 99, 83, + /* 190 */ 84, 45, 86, 87, 24, 8, 9, 91, 92, 93, + /* 200 */ 104, 4, 15, 16, 17, 18, 19, 20, 102, 22, + /* 210 */ 23, 74, 75, 103, 27, 78, 79, 80, 43, 44, + /* 220 */ 83, 84, 22, 86, 87, 1, 2, 117, 91, 92, + /* 230 */ 93, 121, 74, 75, 103, 21, 78, 79, 80, 102, + /* 240 */ 54, 83, 84, 46, 86, 87, 114, 21, 117, 91, + /* 250 */ 92, 93, 121, 82, 8, 9, 10, 53, 77, 78, + /* 260 */ 102, 15, 16, 17, 18, 19, 20, 86, 22, 23, + /* 270 */ 113, 21, 29, 27, 24, 20, 95, 96, 97, 98, + /* 280 */ 66, 100, 73, 74, 75, 75, 75, 78, 79, 80, + /* 290 */ 64, 21, 83, 84, 24, 86, 8, 9, 55, 56, + /* 300 */ 57, 58, 59, 15, 16, 17, 18, 19, 20, 114, + /* 310 */ 22, 23, 103, 8, 9, 10, 11, 12, 21, 74, + /* 320 */ 75, 24, 75, 78, 79, 80, 117, 61, 83, 84, + /* 330 */ 121, 86, 87, 45, 113, 110, 91, 92, 74, 75, + /* 340 */ 60, 109, 78, 79, 80, 107, 48, 83, 84, 47, + /* 350 */ 86, 87, 27, 44, 29, 91, 92, 73, 106, 22, + /* 360 */ 21, 77, 78, 24, 8, 9, 10, 23, 78, 25, + /* 370 */ 86, 15, 16, 17, 18, 19, 20, 22, 22, 23, + /* 380 */ 96, 97, 98, 40, 100, 8, 9, 103, 81, 94, + /* 390 */ 23, 90, 15, 16, 17, 18, 19, 20, 125, 22, + /* 400 */ 23, 117, 69, 74, 75, 121, 65, 78, 79, 80, + /* 410 */ 63, 120, 83, 84, 22, 86, 87, 8, 9, 119, + /* 420 */ 21, 92, 49, 21, 15, 16, 17, 18, 19, 20, + /* 430 */ 24, 22, 23, 15, 24, 23, 74, 75, 21, 24, + /* 440 */ 78, 79, 80, 23, 23, 83, 84, 85, 86, 74, + /* 450 */ 75, 21, 24, 78, 79, 80, 23, 25, 83, 84, + /* 460 */ 85, 86, 74, 75, 24, 21, 78, 79, 80, 23, + /* 470 */ 25, 83, 84, 85, 86, 74, 75, 24, 49, 78, + /* 480 */ 79, 80, 24, 42, 83, 84, 21, 86, 74, 75, + /* 490 */ 2, 25, 78, 79, 80, 25, 29, 83, 84, 24, + /* 500 */ 86, 74, 75, 24, 21, 78, 79, 80, 49, 4, + /* 510 */ 83, 84, 15, 86, 15, 15, 74, 75, 15, 15, + /* 520 */ 78, 79, 80, 15, 123, 83, 84, 25, 86, 74, + /* 530 */ 75, 17, 23, 78, 79, 80, 0, 0, 83, 84, + /* 540 */ 14, 86, 74, 75, 126, 126, 78, 79, 80, 126, + /* 550 */ 126, 83, 84, 126, 86, 74, 75, 126, 126, 78, + /* 560 */ 79, 80, 126, 126, 83, 84, 126, 86, 74, 75, + /* 570 */ 126, 126, 78, 79, 80, 126, 126, 83, 84, 126, + /* 580 */ 86, 74, 75, 126, 126, 78, 79, 80, 126, 126, + /* 590 */ 83, 84, 126, 86, 126, 74, 75, 126, 126, 78, + /* 600 */ 79, 80, 126, 126, 83, 84, 126, 86, 74, 75, + /* 610 */ 126, 126, 78, 79, 80, 126, 126, 83, 84, 126, + /* 620 */ 86, 74, 75, 126, 126, 78, 79, 80, 126, 126, + /* 630 */ 83, 84, 126, 86, 74, 75, 126, 126, 78, 79, + /* 640 */ 80, 126, 126, 83, 84, 126, 86, 74, 75, 126, + /* 650 */ 126, 78, 79, 80, 126, 126, 83, 84, 126, 86, + /* 660 */ 77, 78, 126, 126, 126, 77, 78, 126, 126, 86, + /* 670 */ 126, 126, 126, 126, 86, 126, 126, 126, 126, 96, + /* 680 */ 97, 98, 126, 100, 96, 97, 98, 126, 100, 8, + /* 690 */ 9, 10, 11, 12, 2, 126, 126, 126, 126, 126, + /* 700 */ 8, 9, 10, 11, 12, 2, 126, 126, 126, 126, + /* 710 */ 126, 8, 9, 10, 11, 12, 126, 126, 126, 126, + /* 720 */ 8, 9, 10, 11, 12, 126, 126, 126, 126, 126, + /* 730 */ 126, 126, 126, 126, 126, 126, 24, 126, 126, 126, + /* 740 */ 126, 126, 126, 126, 126, 126, 126, 126, 67, 68, + /* 750 */ 126, 126, 126, 126, 126, 126, 126, 126, 126, 126, + /* 760 */ 126, 126, 126, 126, 126, 126, 126, 126, 126, 126, + /* 770 */ 126, 126, 126, 126, 126, 126, 126, 126, 126, 126, + /* 780 */ 126, 126, 126, 126, 126, 126, 126, 126, 126, 126, + /* 790 */ 126, 126, 126, }; -#define YY_SHIFT_COUNT (146) +#define YY_SHIFT_COUNT (147) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (533) +#define YY_SHIFT_MAX (712) static const unsigned short int yy_shift_ofst[] = { - /* 0 */ 39, 80, 184, 184, 184, 212, 184, 184, 265, 168, - /* 10 */ 357, 387, 341, 387, 387, 387, 387, 387, 387, 387, - /* 20 */ 387, 387, 387, 387, 387, 387, 387, 387, 387, 387, - /* 30 */ 387, 387, 53, 53, 53, 93, 7, 7, 11, 0, - /* 40 */ 32, 93, 98, 98, 98, 305, 359, 139, 54, 129, - /* 50 */ 37, 129, 130, 148, 176, 118, 126, 227, 227, 118, - /* 60 */ 126, 227, 246, 251, 280, 287, 318, 303, 326, 332, - /* 70 */ 322, 361, 319, 324, 335, 379, 206, 327, 281, 404, - /* 80 */ 404, 404, 404, 404, 404, 404, 237, 286, 214, 214, - /* 90 */ 214, 214, 135, 164, 242, 188, 223, 196, 288, 230, - /* 100 */ 321, 233, 390, 396, 369, 384, 397, 399, 413, 411, - /* 110 */ 416, 417, 419, 422, 430, 431, 439, 436, 432, 443, - /* 120 */ 424, 468, 446, 441, 452, 455, 442, 464, 465, 466, - /* 130 */ 470, 474, 471, 453, 502, 495, 500, 501, 504, 506, - /* 140 */ 510, 486, 507, 512, 524, 533, 521, + /* 0 */ 16, 146, 246, 187, 187, 187, 187, 288, 187, 187, + /* 10 */ 58, 377, 409, 356, 409, 409, 409, 409, 409, 409, + /* 20 */ 409, 409, 409, 409, 409, 409, 409, 409, 409, 409, + /* 30 */ 409, 409, 90, 90, 90, 80, 30, 30, 39, 0, + /* 40 */ 32, 32, 80, 34, 34, 34, 681, 243, 37, 86, + /* 50 */ 135, 175, 135, 82, 125, 197, 200, 186, 204, 255, + /* 60 */ 255, 186, 204, 255, 266, 280, 298, 302, 309, 337, + /* 70 */ 355, 343, 367, 333, 341, 347, 392, 692, 703, 712, + /* 80 */ 305, 305, 305, 305, 305, 305, 305, 170, 112, 164, + /* 90 */ 164, 164, 164, 250, 270, 224, 297, 325, 62, 226, + /* 100 */ 214, 339, 344, 399, 402, 373, 406, 410, 412, 417, + /* 110 */ 415, 420, 421, 428, 433, 440, 418, 430, 432, 446, + /* 120 */ 444, 429, 453, 458, 445, 441, 465, 466, 470, 488, + /* 130 */ 467, 475, 479, 483, 459, 505, 497, 499, 500, 503, + /* 140 */ 504, 508, 502, 509, 514, 536, 537, 526, }; -#define YY_REDUCE_COUNT (75) -#define YY_REDUCE_MIN (-103) -#define YY_REDUCE_MAX (637) +#define YY_REDUCE_COUNT (76) +#define YY_REDUCE_MIN (-117) +#define YY_REDUCE_MAX (588) static const short yy_reduce_ofst[] = { - /* 0 */ 377, -1, 43, 63, 83, 103, 162, 185, 220, 224, - /* 10 */ 307, -29, 345, 358, 373, 388, 403, -61, 421, 434, - /* 20 */ 448, 463, 478, 496, 509, 523, 538, 553, 571, 584, - /* 30 */ 598, 613, 608, 623, 637, -74, -63, -31, 12, -85, - /* 40 */ -85, -69, 35, 50, 55, -75, -103, -27, -13, 24, - /* 50 */ 24, 24, 66, 48, 57, 120, 143, 187, 195, 165, - /* 60 */ 167, 211, 199, 210, 226, 228, 253, 24, 275, 274, - /* 70 */ 285, 302, 268, 278, 282, 66, + /* 0 */ 110, -1, 31, 66, 106, 137, 158, 209, 245, 264, + /* 10 */ 284, 329, -29, 362, 375, 51, 388, 401, -61, 414, + /* 20 */ 427, 442, 455, 468, 481, 494, 507, 521, 534, 547, + /* 30 */ 560, 573, 181, 583, 588, -74, -63, -31, 131, -85, + /* 40 */ -85, -85, -69, 20, 40, 89, -117, -95, -32, 25, + /* 50 */ 46, 46, 46, 77, 42, 96, 171, 132, 157, 210, + /* 60 */ 211, 195, 221, 247, 225, 232, 238, 252, 46, 290, + /* 70 */ 307, 295, 301, 273, 291, 300, 77, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 381, 381, 381, 381, 381, 381, 381, 381, 381, 381, - /* 10 */ 381, 381, 381, 381, 381, 381, 381, 381, 381, 381, - /* 20 */ 381, 381, 381, 381, 381, 381, 381, 381, 381, 381, - /* 30 */ 381, 381, 381, 381, 381, 381, 381, 381, 381, 381, - /* 40 */ 381, 381, 452, 452, 452, 468, 516, 381, 476, 444, - /* 50 */ 458, 445, 381, 501, 461, 483, 481, 381, 381, 483, - /* 60 */ 481, 381, 495, 491, 474, 472, 381, 458, 381, 381, - /* 70 */ 381, 381, 519, 507, 503, 381, 381, 381, 381, 494, - /* 80 */ 493, 420, 419, 418, 414, 415, 381, 381, 409, 410, - /* 90 */ 408, 407, 381, 381, 512, 381, 381, 381, 504, 508, - /* 100 */ 381, 397, 465, 475, 381, 381, 381, 381, 381, 381, - /* 110 */ 381, 381, 381, 381, 381, 381, 381, 397, 381, 492, - /* 120 */ 381, 439, 381, 451, 447, 381, 381, 443, 396, 381, - /* 130 */ 381, 381, 502, 381, 381, 381, 381, 381, 381, 381, - /* 140 */ 381, 381, 381, 381, 381, 381, 381, + /* 0 */ 385, 385, 385, 385, 385, 385, 385, 385, 385, 385, + /* 10 */ 385, 385, 385, 385, 385, 385, 385, 385, 385, 385, + /* 20 */ 385, 385, 385, 385, 385, 385, 385, 385, 385, 385, + /* 30 */ 385, 385, 385, 385, 385, 385, 385, 385, 385, 385, + /* 40 */ 447, 385, 385, 458, 458, 458, 522, 385, 482, 474, + /* 50 */ 450, 464, 451, 385, 507, 467, 385, 489, 487, 385, + /* 60 */ 385, 489, 487, 385, 501, 497, 480, 478, 464, 385, + /* 70 */ 385, 385, 385, 525, 513, 509, 385, 385, 385, 385, + /* 80 */ 500, 499, 424, 423, 422, 418, 419, 385, 385, 413, + /* 90 */ 414, 412, 411, 385, 385, 448, 385, 385, 385, 510, + /* 100 */ 514, 385, 401, 471, 481, 385, 385, 385, 385, 385, + /* 110 */ 385, 385, 385, 385, 385, 385, 385, 385, 401, 385, + /* 120 */ 498, 385, 457, 453, 385, 385, 449, 400, 385, 443, + /* 130 */ 385, 385, 385, 508, 385, 385, 385, 385, 385, 385, + /* 140 */ 385, 385, 385, 385, 385, 385, 385, 385, }; /********** End of lemon-generated parsing tables *****************************/ @@ -616,38 +623,39 @@ static const char *const yyTokenName[] = { /* 90 */ "in_predicate_value", /* 91 */ "boolean_value_expression", /* 92 */ "boolean_primary", - /* 93 */ "from_clause", - /* 94 */ "table_reference_list", - /* 95 */ "table_reference", - /* 96 */ "table_primary", - /* 97 */ "joined_table", - /* 98 */ "alias_opt", - /* 99 */ "parenthesized_joined_table", - /* 100 */ "join_type", - /* 101 */ "search_condition", - /* 102 */ "query_specification", - /* 103 */ "set_quantifier_opt", - /* 104 */ "select_list", - /* 105 */ "where_clause_opt", - /* 106 */ "partition_by_clause_opt", - /* 107 */ "twindow_clause_opt", - /* 108 */ "group_by_clause_opt", - /* 109 */ "having_clause_opt", - /* 110 */ "select_sublist", - /* 111 */ "select_item", - /* 112 */ "sliding_opt", - /* 113 */ "fill_opt", - /* 114 */ "fill_mode", - /* 115 */ "group_by_list", - /* 116 */ "query_expression_body", - /* 117 */ "order_by_clause_opt", - /* 118 */ "slimit_clause_opt", - /* 119 */ "limit_clause_opt", - /* 120 */ "query_primary", - /* 121 */ "sort_specification_list", - /* 122 */ "sort_specification", - /* 123 */ "ordering_specification_opt", - /* 124 */ "null_ordering_opt", + /* 93 */ "common_expression", + /* 94 */ "from_clause", + /* 95 */ "table_reference_list", + /* 96 */ "table_reference", + /* 97 */ "table_primary", + /* 98 */ "joined_table", + /* 99 */ "alias_opt", + /* 100 */ "parenthesized_joined_table", + /* 101 */ "join_type", + /* 102 */ "search_condition", + /* 103 */ "query_specification", + /* 104 */ "set_quantifier_opt", + /* 105 */ "select_list", + /* 106 */ "where_clause_opt", + /* 107 */ "partition_by_clause_opt", + /* 108 */ "twindow_clause_opt", + /* 109 */ "group_by_clause_opt", + /* 110 */ "having_clause_opt", + /* 111 */ "select_sublist", + /* 112 */ "select_item", + /* 113 */ "sliding_opt", + /* 114 */ "fill_opt", + /* 115 */ "fill_mode", + /* 116 */ "group_by_list", + /* 117 */ "query_expression_body", + /* 118 */ "order_by_clause_opt", + /* 119 */ "slimit_clause_opt", + /* 120 */ "limit_clause_opt", + /* 121 */ "query_primary", + /* 122 */ "sort_specification_list", + /* 123 */ "sort_specification", + /* 124 */ "ordering_specification_opt", + /* 125 */ "null_ordering_opt", }; #endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ @@ -714,85 +722,87 @@ static const char *const yyRuleName[] = { /* 56 */ "boolean_value_expression ::= boolean_value_expression AND boolean_value_expression", /* 57 */ "boolean_primary ::= predicate", /* 58 */ "boolean_primary ::= NK_LP boolean_value_expression NK_RP", - /* 59 */ "from_clause ::= FROM table_reference_list", - /* 60 */ "table_reference_list ::= table_reference", - /* 61 */ "table_reference_list ::= table_reference_list NK_COMMA table_reference", - /* 62 */ "table_reference ::= table_primary", - /* 63 */ "table_reference ::= joined_table", - /* 64 */ "table_primary ::= table_name alias_opt", - /* 65 */ "table_primary ::= db_name NK_DOT table_name alias_opt", - /* 66 */ "table_primary ::= subquery alias_opt", - /* 67 */ "table_primary ::= parenthesized_joined_table", - /* 68 */ "alias_opt ::=", - /* 69 */ "alias_opt ::= table_alias", - /* 70 */ "alias_opt ::= AS table_alias", - /* 71 */ "parenthesized_joined_table ::= NK_LP joined_table NK_RP", - /* 72 */ "parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP", - /* 73 */ "joined_table ::= table_reference join_type JOIN table_reference ON search_condition", - /* 74 */ "join_type ::=", - /* 75 */ "join_type ::= INNER", - /* 76 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt", - /* 77 */ "set_quantifier_opt ::=", - /* 78 */ "set_quantifier_opt ::= DISTINCT", - /* 79 */ "set_quantifier_opt ::= ALL", - /* 80 */ "select_list ::= NK_STAR", - /* 81 */ "select_list ::= select_sublist", - /* 82 */ "select_sublist ::= select_item", - /* 83 */ "select_sublist ::= select_sublist NK_COMMA select_item", - /* 84 */ "select_item ::= expression", - /* 85 */ "select_item ::= expression column_alias", - /* 86 */ "select_item ::= expression AS column_alias", - /* 87 */ "select_item ::= table_name NK_DOT NK_STAR", - /* 88 */ "where_clause_opt ::=", - /* 89 */ "where_clause_opt ::= WHERE search_condition", - /* 90 */ "partition_by_clause_opt ::=", - /* 91 */ "partition_by_clause_opt ::= PARTITION BY expression_list", - /* 92 */ "twindow_clause_opt ::=", - /* 93 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA NK_INTEGER NK_RP", - /* 94 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP", - /* 95 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", - /* 96 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", - /* 97 */ "sliding_opt ::=", - /* 98 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", - /* 99 */ "fill_opt ::=", - /* 100 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", - /* 101 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP", - /* 102 */ "fill_mode ::= NONE", - /* 103 */ "fill_mode ::= PREV", - /* 104 */ "fill_mode ::= NULL", - /* 105 */ "fill_mode ::= LINEAR", - /* 106 */ "fill_mode ::= NEXT", - /* 107 */ "group_by_clause_opt ::=", - /* 108 */ "group_by_clause_opt ::= GROUP BY group_by_list", - /* 109 */ "group_by_list ::= expression", - /* 110 */ "group_by_list ::= group_by_list NK_COMMA expression", - /* 111 */ "having_clause_opt ::=", - /* 112 */ "having_clause_opt ::= HAVING search_condition", - /* 113 */ "query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt", - /* 114 */ "query_expression_body ::= query_primary", - /* 115 */ "query_expression_body ::= query_expression_body UNION ALL query_expression_body", - /* 116 */ "query_primary ::= query_specification", - /* 117 */ "order_by_clause_opt ::=", - /* 118 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", - /* 119 */ "slimit_clause_opt ::=", - /* 120 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", - /* 121 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", - /* 122 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 123 */ "limit_clause_opt ::=", - /* 124 */ "limit_clause_opt ::= LIMIT NK_INTEGER", - /* 125 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", - /* 126 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 127 */ "subquery ::= NK_LP query_expression NK_RP", - /* 128 */ "search_condition ::= boolean_value_expression", - /* 129 */ "sort_specification_list ::= sort_specification", - /* 130 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", - /* 131 */ "sort_specification ::= expression ordering_specification_opt null_ordering_opt", - /* 132 */ "ordering_specification_opt ::=", - /* 133 */ "ordering_specification_opt ::= ASC", - /* 134 */ "ordering_specification_opt ::= DESC", - /* 135 */ "null_ordering_opt ::=", - /* 136 */ "null_ordering_opt ::= NULLS FIRST", - /* 137 */ "null_ordering_opt ::= NULLS LAST", + /* 59 */ "common_expression ::= expression", + /* 60 */ "common_expression ::= boolean_value_expression", + /* 61 */ "from_clause ::= FROM table_reference_list", + /* 62 */ "table_reference_list ::= table_reference", + /* 63 */ "table_reference_list ::= table_reference_list NK_COMMA table_reference", + /* 64 */ "table_reference ::= table_primary", + /* 65 */ "table_reference ::= joined_table", + /* 66 */ "table_primary ::= table_name alias_opt", + /* 67 */ "table_primary ::= db_name NK_DOT table_name alias_opt", + /* 68 */ "table_primary ::= subquery alias_opt", + /* 69 */ "table_primary ::= parenthesized_joined_table", + /* 70 */ "alias_opt ::=", + /* 71 */ "alias_opt ::= table_alias", + /* 72 */ "alias_opt ::= AS table_alias", + /* 73 */ "parenthesized_joined_table ::= NK_LP joined_table NK_RP", + /* 74 */ "parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP", + /* 75 */ "joined_table ::= table_reference join_type JOIN table_reference ON search_condition", + /* 76 */ "join_type ::=", + /* 77 */ "join_type ::= INNER", + /* 78 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt", + /* 79 */ "set_quantifier_opt ::=", + /* 80 */ "set_quantifier_opt ::= DISTINCT", + /* 81 */ "set_quantifier_opt ::= ALL", + /* 82 */ "select_list ::= NK_STAR", + /* 83 */ "select_list ::= select_sublist", + /* 84 */ "select_sublist ::= select_item", + /* 85 */ "select_sublist ::= select_sublist NK_COMMA select_item", + /* 86 */ "select_item ::= common_expression", + /* 87 */ "select_item ::= common_expression column_alias", + /* 88 */ "select_item ::= common_expression AS column_alias", + /* 89 */ "select_item ::= table_name NK_DOT NK_STAR", + /* 90 */ "where_clause_opt ::=", + /* 91 */ "where_clause_opt ::= WHERE search_condition", + /* 92 */ "partition_by_clause_opt ::=", + /* 93 */ "partition_by_clause_opt ::= PARTITION BY expression_list", + /* 94 */ "twindow_clause_opt ::=", + /* 95 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA NK_INTEGER NK_RP", + /* 96 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP", + /* 97 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", + /* 98 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", + /* 99 */ "sliding_opt ::=", + /* 100 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", + /* 101 */ "fill_opt ::=", + /* 102 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", + /* 103 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP", + /* 104 */ "fill_mode ::= NONE", + /* 105 */ "fill_mode ::= PREV", + /* 106 */ "fill_mode ::= NULL", + /* 107 */ "fill_mode ::= LINEAR", + /* 108 */ "fill_mode ::= NEXT", + /* 109 */ "group_by_clause_opt ::=", + /* 110 */ "group_by_clause_opt ::= GROUP BY group_by_list", + /* 111 */ "group_by_list ::= expression", + /* 112 */ "group_by_list ::= group_by_list NK_COMMA expression", + /* 113 */ "having_clause_opt ::=", + /* 114 */ "having_clause_opt ::= HAVING search_condition", + /* 115 */ "query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt", + /* 116 */ "query_expression_body ::= query_primary", + /* 117 */ "query_expression_body ::= query_expression_body UNION ALL query_expression_body", + /* 118 */ "query_primary ::= query_specification", + /* 119 */ "order_by_clause_opt ::=", + /* 120 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", + /* 121 */ "slimit_clause_opt ::=", + /* 122 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", + /* 123 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", + /* 124 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 125 */ "limit_clause_opt ::=", + /* 126 */ "limit_clause_opt ::= LIMIT NK_INTEGER", + /* 127 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", + /* 128 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 129 */ "subquery ::= NK_LP query_expression NK_RP", + /* 130 */ "search_condition ::= common_expression", + /* 131 */ "sort_specification_list ::= sort_specification", + /* 132 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", + /* 133 */ "sort_specification ::= expression ordering_specification_opt null_ordering_opt", + /* 134 */ "ordering_specification_opt ::=", + /* 135 */ "ordering_specification_opt ::= ASC", + /* 136 */ "ordering_specification_opt ::= DESC", + /* 137 */ "null_ordering_opt ::=", + /* 138 */ "null_ordering_opt ::= NULLS FIRST", + /* 139 */ "null_ordering_opt ::= NULLS LAST", }; #endif /* NDEBUG */ @@ -930,40 +940,41 @@ static void yy_destructor( case 90: /* in_predicate_value */ case 91: /* boolean_value_expression */ case 92: /* boolean_primary */ - case 93: /* from_clause */ - case 94: /* table_reference_list */ - case 95: /* table_reference */ - case 96: /* table_primary */ - case 97: /* joined_table */ - case 99: /* parenthesized_joined_table */ - case 101: /* search_condition */ - case 102: /* query_specification */ - case 105: /* where_clause_opt */ - case 107: /* twindow_clause_opt */ - case 109: /* having_clause_opt */ - case 111: /* select_item */ - case 112: /* sliding_opt */ - case 113: /* fill_opt */ - case 116: /* query_expression_body */ - case 118: /* slimit_clause_opt */ - case 119: /* limit_clause_opt */ - case 120: /* query_primary */ - case 122: /* sort_specification */ + case 93: /* common_expression */ + case 94: /* from_clause */ + case 95: /* table_reference_list */ + case 96: /* table_reference */ + case 97: /* table_primary */ + case 98: /* joined_table */ + case 100: /* parenthesized_joined_table */ + case 102: /* search_condition */ + case 103: /* query_specification */ + case 106: /* where_clause_opt */ + case 108: /* twindow_clause_opt */ + case 110: /* having_clause_opt */ + case 112: /* select_item */ + case 113: /* sliding_opt */ + case 114: /* fill_opt */ + case 117: /* query_expression_body */ + case 119: /* slimit_clause_opt */ + case 120: /* limit_clause_opt */ + case 121: /* query_primary */ + case 123: /* sort_specification */ { - PARSER_DESTRUCTOR_TRACE; nodesDestroyNode((yypminor->yy212)); + PARSER_DESTRUCTOR_TRACE; nodesDestroyNode((yypminor->yy56)); } break; case 76: /* literal_list */ case 85: /* expression_list */ - case 104: /* select_list */ - case 106: /* partition_by_clause_opt */ - case 108: /* group_by_clause_opt */ - case 110: /* select_sublist */ - case 115: /* group_by_list */ - case 117: /* order_by_clause_opt */ - case 121: /* sort_specification_list */ + case 105: /* select_list */ + case 107: /* partition_by_clause_opt */ + case 109: /* group_by_clause_opt */ + case 111: /* select_sublist */ + case 116: /* group_by_list */ + case 118: /* order_by_clause_opt */ + case 122: /* sort_specification_list */ { - PARSER_DESTRUCTOR_TRACE; nodesDestroyList((yypminor->yy174)); + PARSER_DESTRUCTOR_TRACE; nodesDestroyList((yypminor->yy208)); } break; case 77: /* db_name */ @@ -972,7 +983,7 @@ static void yy_destructor( case 80: /* function_name */ case 81: /* table_alias */ case 82: /* column_alias */ - case 98: /* alias_opt */ + case 99: /* alias_opt */ { PARSER_DESTRUCTOR_TRACE; } @@ -983,27 +994,27 @@ static void yy_destructor( PARSER_DESTRUCTOR_TRACE; } break; - case 100: /* join_type */ + case 101: /* join_type */ { PARSER_DESTRUCTOR_TRACE; } break; - case 103: /* set_quantifier_opt */ + case 104: /* set_quantifier_opt */ { PARSER_DESTRUCTOR_TRACE; } break; - case 114: /* fill_mode */ + case 115: /* fill_mode */ { PARSER_DESTRUCTOR_TRACE; } break; - case 123: /* ordering_specification_opt */ + case 124: /* ordering_specification_opt */ { PARSER_DESTRUCTOR_TRACE; } break; - case 124: /* null_ordering_opt */ + case 125: /* null_ordering_opt */ { PARSER_DESTRUCTOR_TRACE; } @@ -1361,85 +1372,87 @@ static const struct { { 91, -3 }, /* (56) boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ { 92, -1 }, /* (57) boolean_primary ::= predicate */ { 92, -3 }, /* (58) boolean_primary ::= NK_LP boolean_value_expression NK_RP */ - { 93, -2 }, /* (59) from_clause ::= FROM table_reference_list */ - { 94, -1 }, /* (60) table_reference_list ::= table_reference */ - { 94, -3 }, /* (61) table_reference_list ::= table_reference_list NK_COMMA table_reference */ - { 95, -1 }, /* (62) table_reference ::= table_primary */ - { 95, -1 }, /* (63) table_reference ::= joined_table */ - { 96, -2 }, /* (64) table_primary ::= table_name alias_opt */ - { 96, -4 }, /* (65) table_primary ::= db_name NK_DOT table_name alias_opt */ - { 96, -2 }, /* (66) table_primary ::= subquery alias_opt */ - { 96, -1 }, /* (67) table_primary ::= parenthesized_joined_table */ - { 98, 0 }, /* (68) alias_opt ::= */ - { 98, -1 }, /* (69) alias_opt ::= table_alias */ - { 98, -2 }, /* (70) alias_opt ::= AS table_alias */ - { 99, -3 }, /* (71) parenthesized_joined_table ::= NK_LP joined_table NK_RP */ - { 99, -3 }, /* (72) parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ - { 97, -6 }, /* (73) joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ - { 100, 0 }, /* (74) join_type ::= */ - { 100, -1 }, /* (75) join_type ::= INNER */ - { 102, -9 }, /* (76) query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ - { 103, 0 }, /* (77) set_quantifier_opt ::= */ - { 103, -1 }, /* (78) set_quantifier_opt ::= DISTINCT */ - { 103, -1 }, /* (79) set_quantifier_opt ::= ALL */ - { 104, -1 }, /* (80) select_list ::= NK_STAR */ - { 104, -1 }, /* (81) select_list ::= select_sublist */ - { 110, -1 }, /* (82) select_sublist ::= select_item */ - { 110, -3 }, /* (83) select_sublist ::= select_sublist NK_COMMA select_item */ - { 111, -1 }, /* (84) select_item ::= expression */ - { 111, -2 }, /* (85) select_item ::= expression column_alias */ - { 111, -3 }, /* (86) select_item ::= expression AS column_alias */ - { 111, -3 }, /* (87) select_item ::= table_name NK_DOT NK_STAR */ - { 105, 0 }, /* (88) where_clause_opt ::= */ - { 105, -2 }, /* (89) where_clause_opt ::= WHERE search_condition */ - { 106, 0 }, /* (90) partition_by_clause_opt ::= */ - { 106, -3 }, /* (91) partition_by_clause_opt ::= PARTITION BY expression_list */ - { 107, 0 }, /* (92) twindow_clause_opt ::= */ - { 107, -6 }, /* (93) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA NK_INTEGER NK_RP */ - { 107, -4 }, /* (94) twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP */ - { 107, -6 }, /* (95) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ - { 107, -8 }, /* (96) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ - { 112, 0 }, /* (97) sliding_opt ::= */ - { 112, -4 }, /* (98) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ - { 113, 0 }, /* (99) fill_opt ::= */ - { 113, -4 }, /* (100) fill_opt ::= FILL NK_LP fill_mode NK_RP */ - { 113, -6 }, /* (101) fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ - { 114, -1 }, /* (102) fill_mode ::= NONE */ - { 114, -1 }, /* (103) fill_mode ::= PREV */ - { 114, -1 }, /* (104) fill_mode ::= NULL */ - { 114, -1 }, /* (105) fill_mode ::= LINEAR */ - { 114, -1 }, /* (106) fill_mode ::= NEXT */ - { 108, 0 }, /* (107) group_by_clause_opt ::= */ - { 108, -3 }, /* (108) group_by_clause_opt ::= GROUP BY group_by_list */ - { 115, -1 }, /* (109) group_by_list ::= expression */ - { 115, -3 }, /* (110) group_by_list ::= group_by_list NK_COMMA expression */ - { 109, 0 }, /* (111) having_clause_opt ::= */ - { 109, -2 }, /* (112) having_clause_opt ::= HAVING search_condition */ - { 73, -4 }, /* (113) query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ - { 116, -1 }, /* (114) query_expression_body ::= query_primary */ - { 116, -4 }, /* (115) query_expression_body ::= query_expression_body UNION ALL query_expression_body */ - { 120, -1 }, /* (116) query_primary ::= query_specification */ - { 117, 0 }, /* (117) order_by_clause_opt ::= */ - { 117, -3 }, /* (118) order_by_clause_opt ::= ORDER BY sort_specification_list */ - { 118, 0 }, /* (119) slimit_clause_opt ::= */ - { 118, -2 }, /* (120) slimit_clause_opt ::= SLIMIT NK_INTEGER */ - { 118, -4 }, /* (121) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - { 118, -4 }, /* (122) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - { 119, 0 }, /* (123) limit_clause_opt ::= */ - { 119, -2 }, /* (124) limit_clause_opt ::= LIMIT NK_INTEGER */ - { 119, -4 }, /* (125) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ - { 119, -4 }, /* (126) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - { 86, -3 }, /* (127) subquery ::= NK_LP query_expression NK_RP */ - { 101, -1 }, /* (128) search_condition ::= boolean_value_expression */ - { 121, -1 }, /* (129) sort_specification_list ::= sort_specification */ - { 121, -3 }, /* (130) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ - { 122, -3 }, /* (131) sort_specification ::= expression ordering_specification_opt null_ordering_opt */ - { 123, 0 }, /* (132) ordering_specification_opt ::= */ - { 123, -1 }, /* (133) ordering_specification_opt ::= ASC */ - { 123, -1 }, /* (134) ordering_specification_opt ::= DESC */ - { 124, 0 }, /* (135) null_ordering_opt ::= */ - { 124, -2 }, /* (136) null_ordering_opt ::= NULLS FIRST */ - { 124, -2 }, /* (137) null_ordering_opt ::= NULLS LAST */ + { 93, -1 }, /* (59) common_expression ::= expression */ + { 93, -1 }, /* (60) common_expression ::= boolean_value_expression */ + { 94, -2 }, /* (61) from_clause ::= FROM table_reference_list */ + { 95, -1 }, /* (62) table_reference_list ::= table_reference */ + { 95, -3 }, /* (63) table_reference_list ::= table_reference_list NK_COMMA table_reference */ + { 96, -1 }, /* (64) table_reference ::= table_primary */ + { 96, -1 }, /* (65) table_reference ::= joined_table */ + { 97, -2 }, /* (66) table_primary ::= table_name alias_opt */ + { 97, -4 }, /* (67) table_primary ::= db_name NK_DOT table_name alias_opt */ + { 97, -2 }, /* (68) table_primary ::= subquery alias_opt */ + { 97, -1 }, /* (69) table_primary ::= parenthesized_joined_table */ + { 99, 0 }, /* (70) alias_opt ::= */ + { 99, -1 }, /* (71) alias_opt ::= table_alias */ + { 99, -2 }, /* (72) alias_opt ::= AS table_alias */ + { 100, -3 }, /* (73) parenthesized_joined_table ::= NK_LP joined_table NK_RP */ + { 100, -3 }, /* (74) parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ + { 98, -6 }, /* (75) joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ + { 101, 0 }, /* (76) join_type ::= */ + { 101, -1 }, /* (77) join_type ::= INNER */ + { 103, -9 }, /* (78) query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ + { 104, 0 }, /* (79) set_quantifier_opt ::= */ + { 104, -1 }, /* (80) set_quantifier_opt ::= DISTINCT */ + { 104, -1 }, /* (81) set_quantifier_opt ::= ALL */ + { 105, -1 }, /* (82) select_list ::= NK_STAR */ + { 105, -1 }, /* (83) select_list ::= select_sublist */ + { 111, -1 }, /* (84) select_sublist ::= select_item */ + { 111, -3 }, /* (85) select_sublist ::= select_sublist NK_COMMA select_item */ + { 112, -1 }, /* (86) select_item ::= common_expression */ + { 112, -2 }, /* (87) select_item ::= common_expression column_alias */ + { 112, -3 }, /* (88) select_item ::= common_expression AS column_alias */ + { 112, -3 }, /* (89) select_item ::= table_name NK_DOT NK_STAR */ + { 106, 0 }, /* (90) where_clause_opt ::= */ + { 106, -2 }, /* (91) where_clause_opt ::= WHERE search_condition */ + { 107, 0 }, /* (92) partition_by_clause_opt ::= */ + { 107, -3 }, /* (93) partition_by_clause_opt ::= PARTITION BY expression_list */ + { 108, 0 }, /* (94) twindow_clause_opt ::= */ + { 108, -6 }, /* (95) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA NK_INTEGER NK_RP */ + { 108, -4 }, /* (96) twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP */ + { 108, -6 }, /* (97) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ + { 108, -8 }, /* (98) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ + { 113, 0 }, /* (99) sliding_opt ::= */ + { 113, -4 }, /* (100) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ + { 114, 0 }, /* (101) fill_opt ::= */ + { 114, -4 }, /* (102) fill_opt ::= FILL NK_LP fill_mode NK_RP */ + { 114, -6 }, /* (103) fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ + { 115, -1 }, /* (104) fill_mode ::= NONE */ + { 115, -1 }, /* (105) fill_mode ::= PREV */ + { 115, -1 }, /* (106) fill_mode ::= NULL */ + { 115, -1 }, /* (107) fill_mode ::= LINEAR */ + { 115, -1 }, /* (108) fill_mode ::= NEXT */ + { 109, 0 }, /* (109) group_by_clause_opt ::= */ + { 109, -3 }, /* (110) group_by_clause_opt ::= GROUP BY group_by_list */ + { 116, -1 }, /* (111) group_by_list ::= expression */ + { 116, -3 }, /* (112) group_by_list ::= group_by_list NK_COMMA expression */ + { 110, 0 }, /* (113) having_clause_opt ::= */ + { 110, -2 }, /* (114) having_clause_opt ::= HAVING search_condition */ + { 73, -4 }, /* (115) query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ + { 117, -1 }, /* (116) query_expression_body ::= query_primary */ + { 117, -4 }, /* (117) query_expression_body ::= query_expression_body UNION ALL query_expression_body */ + { 121, -1 }, /* (118) query_primary ::= query_specification */ + { 118, 0 }, /* (119) order_by_clause_opt ::= */ + { 118, -3 }, /* (120) order_by_clause_opt ::= ORDER BY sort_specification_list */ + { 119, 0 }, /* (121) slimit_clause_opt ::= */ + { 119, -2 }, /* (122) slimit_clause_opt ::= SLIMIT NK_INTEGER */ + { 119, -4 }, /* (123) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + { 119, -4 }, /* (124) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + { 120, 0 }, /* (125) limit_clause_opt ::= */ + { 120, -2 }, /* (126) limit_clause_opt ::= LIMIT NK_INTEGER */ + { 120, -4 }, /* (127) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ + { 120, -4 }, /* (128) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + { 86, -3 }, /* (129) subquery ::= NK_LP query_expression NK_RP */ + { 102, -1 }, /* (130) search_condition ::= common_expression */ + { 122, -1 }, /* (131) sort_specification_list ::= sort_specification */ + { 122, -3 }, /* (132) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ + { 123, -3 }, /* (133) sort_specification ::= expression ordering_specification_opt null_ordering_opt */ + { 124, 0 }, /* (134) ordering_specification_opt ::= */ + { 124, -1 }, /* (135) ordering_specification_opt ::= ASC */ + { 124, -1 }, /* (136) ordering_specification_opt ::= DESC */ + { 125, 0 }, /* (137) null_ordering_opt ::= */ + { 125, -2 }, /* (138) null_ordering_opt ::= NULLS FIRST */ + { 125, -2 }, /* (139) null_ordering_opt ::= NULLS LAST */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -1530,27 +1543,27 @@ static YYACTIONTYPE yy_reduce( { PARSER_TRACE; createShowStmt(pCxt, SHOW_TYPE_DATABASE); } break; case 1: /* cmd ::= query_expression */ -{ PARSER_TRACE; pCxt->pRootNode = yymsp[0].minor.yy212; } +{ PARSER_TRACE; pCxt->pRootNode = yymsp[0].minor.yy56; } break; case 2: /* literal ::= NK_INTEGER */ -{ PARSER_TRACE; yylhsminor.yy212 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy212 = yylhsminor.yy212; +{ PARSER_TRACE; yylhsminor.yy56 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy56 = yylhsminor.yy56; break; case 3: /* literal ::= NK_FLOAT */ -{ PARSER_TRACE; yylhsminor.yy212 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy212 = yylhsminor.yy212; +{ PARSER_TRACE; yylhsminor.yy56 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy56 = yylhsminor.yy56; break; case 4: /* literal ::= NK_STRING */ -{ PARSER_TRACE; yylhsminor.yy212 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy212 = yylhsminor.yy212; +{ PARSER_TRACE; yylhsminor.yy56 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy56 = yylhsminor.yy56; break; case 5: /* literal ::= NK_BOOL */ -{ PARSER_TRACE; yylhsminor.yy212 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy212 = yylhsminor.yy212; +{ PARSER_TRACE; yylhsminor.yy56 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy56 = yylhsminor.yy56; break; case 6: /* literal ::= TIMESTAMP NK_STRING */ -{ PARSER_TRACE; yylhsminor.yy212 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0)); } - yymsp[-1].minor.yy212 = yylhsminor.yy212; +{ PARSER_TRACE; yylhsminor.yy56 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0)); } + yymsp[-1].minor.yy56 = yylhsminor.yy56; break; case 7: /* literal ::= duration_literal */ case 17: /* expression ::= literal */ yytestcase(yyruleno==17); @@ -1558,29 +1571,28 @@ static YYACTIONTYPE yy_reduce( case 21: /* expression ::= subquery */ yytestcase(yyruleno==21); case 53: /* boolean_value_expression ::= boolean_primary */ yytestcase(yyruleno==53); case 57: /* boolean_primary ::= predicate */ yytestcase(yyruleno==57); - case 60: /* table_reference_list ::= table_reference */ yytestcase(yyruleno==60); - case 62: /* table_reference ::= table_primary */ yytestcase(yyruleno==62); - case 63: /* table_reference ::= joined_table */ yytestcase(yyruleno==63); - case 67: /* table_primary ::= parenthesized_joined_table */ yytestcase(yyruleno==67); - case 114: /* query_expression_body ::= query_primary */ yytestcase(yyruleno==114); - case 116: /* query_primary ::= query_specification */ yytestcase(yyruleno==116); - case 128: /* search_condition ::= boolean_value_expression */ yytestcase(yyruleno==128); -{ PARSER_TRACE; yylhsminor.yy212 = yymsp[0].minor.yy212; } - yymsp[0].minor.yy212 = yylhsminor.yy212; + case 62: /* table_reference_list ::= table_reference */ yytestcase(yyruleno==62); + case 64: /* table_reference ::= table_primary */ yytestcase(yyruleno==64); + case 65: /* table_reference ::= joined_table */ yytestcase(yyruleno==65); + case 69: /* table_primary ::= parenthesized_joined_table */ yytestcase(yyruleno==69); + case 116: /* query_expression_body ::= query_primary */ yytestcase(yyruleno==116); + case 118: /* query_primary ::= query_specification */ yytestcase(yyruleno==118); +{ PARSER_TRACE; yylhsminor.yy56 = yymsp[0].minor.yy56; } + yymsp[0].minor.yy56 = yylhsminor.yy56; break; case 8: /* duration_literal ::= NK_VARIABLE */ -{ PARSER_TRACE; yylhsminor.yy212 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy212 = yylhsminor.yy212; +{ PARSER_TRACE; yylhsminor.yy56 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy56 = yylhsminor.yy56; break; case 9: /* literal_list ::= literal */ case 30: /* expression_list ::= expression */ yytestcase(yyruleno==30); -{ PARSER_TRACE; yylhsminor.yy174 = createNodeList(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy212)); } - yymsp[0].minor.yy174 = yylhsminor.yy174; +{ PARSER_TRACE; yylhsminor.yy208 = createNodeList(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy56)); } + yymsp[0].minor.yy208 = yylhsminor.yy208; break; case 10: /* literal_list ::= literal_list NK_COMMA literal */ case 31: /* expression_list ::= expression_list NK_COMMA expression */ yytestcase(yyruleno==31); -{ PARSER_TRACE; yylhsminor.yy174 = addNodeToList(pCxt, yymsp[-2].minor.yy174, releaseRawExprNode(pCxt, yymsp[0].minor.yy212)); } - yymsp[-2].minor.yy174 = yylhsminor.yy174; +{ PARSER_TRACE; yylhsminor.yy208 = addNodeToList(pCxt, yymsp[-2].minor.yy208, releaseRawExprNode(pCxt, yymsp[0].minor.yy56)); } + yymsp[-2].minor.yy208 = yylhsminor.yy208; break; case 11: /* db_name ::= NK_ID */ case 12: /* table_name ::= NK_ID */ yytestcase(yyruleno==12); @@ -1588,380 +1600,425 @@ static YYACTIONTYPE yy_reduce( case 14: /* function_name ::= NK_ID */ yytestcase(yyruleno==14); case 15: /* table_alias ::= NK_ID */ yytestcase(yyruleno==15); case 16: /* column_alias ::= NK_ID */ yytestcase(yyruleno==16); -{ PARSER_TRACE; yylhsminor.yy79 = yymsp[0].minor.yy0; } - yymsp[0].minor.yy79 = yylhsminor.yy79; +{ PARSER_TRACE; yylhsminor.yy29 = yymsp[0].minor.yy0; } + yymsp[0].minor.yy29 = yylhsminor.yy29; break; case 19: /* expression ::= function_name NK_LP expression_list NK_RP */ -{ PARSER_TRACE; yylhsminor.yy212 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy79, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy79, yymsp[-1].minor.yy174)); } - yymsp[-3].minor.yy212 = yylhsminor.yy212; +{ PARSER_TRACE; yylhsminor.yy56 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy29, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy29, yymsp[-1].minor.yy208)); } + yymsp[-3].minor.yy56 = yylhsminor.yy56; break; case 20: /* expression ::= function_name NK_LP NK_STAR NK_RP */ -{ PARSER_TRACE; yylhsminor.yy212 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy79, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy79, createNodeList(pCxt, createColumnNode(pCxt, NULL, &yymsp[-1].minor.yy0)))); } - yymsp[-3].minor.yy212 = yylhsminor.yy212; +{ PARSER_TRACE; yylhsminor.yy56 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy29, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy29, createNodeList(pCxt, createColumnNode(pCxt, NULL, &yymsp[-1].minor.yy0)))); } + yymsp[-3].minor.yy56 = yylhsminor.yy56; break; case 22: /* expression ::= NK_LP expression NK_RP */ -{ PARSER_TRACE; yylhsminor.yy212 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy212)); } - yymsp[-2].minor.yy212 = yylhsminor.yy212; + case 58: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ yytestcase(yyruleno==58); +{ PARSER_TRACE; yylhsminor.yy56 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy56)); } + yymsp[-2].minor.yy56 = yylhsminor.yy56; break; case 23: /* expression ::= NK_PLUS expression */ { PARSER_TRACE; - SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy212); - yylhsminor.yy212 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, releaseRawExprNode(pCxt, yymsp[0].minor.yy212)); + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy56); + yylhsminor.yy56 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, releaseRawExprNode(pCxt, yymsp[0].minor.yy56)); } - yymsp[-1].minor.yy212 = yylhsminor.yy212; + yymsp[-1].minor.yy56 = yylhsminor.yy56; break; case 24: /* expression ::= NK_MINUS expression */ { PARSER_TRACE; - SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy212); - yylhsminor.yy212 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[0].minor.yy212), NULL)); + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy56); + yylhsminor.yy56 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[0].minor.yy56), NULL)); } - yymsp[-1].minor.yy212 = yylhsminor.yy212; + yymsp[-1].minor.yy56 = yylhsminor.yy56; break; case 25: /* expression ::= expression NK_PLUS expression */ { PARSER_TRACE; - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy212); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy212); - yylhsminor.yy212 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy212), releaseRawExprNode(pCxt, yymsp[0].minor.yy212))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy56); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy56); + yylhsminor.yy56 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy56), releaseRawExprNode(pCxt, yymsp[0].minor.yy56))); } - yymsp[-2].minor.yy212 = yylhsminor.yy212; + yymsp[-2].minor.yy56 = yylhsminor.yy56; break; case 26: /* expression ::= expression NK_MINUS expression */ { PARSER_TRACE; - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy212); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy212); - yylhsminor.yy212 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[-2].minor.yy212), releaseRawExprNode(pCxt, yymsp[0].minor.yy212))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy56); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy56); + yylhsminor.yy56 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[-2].minor.yy56), releaseRawExprNode(pCxt, yymsp[0].minor.yy56))); } - yymsp[-2].minor.yy212 = yylhsminor.yy212; + yymsp[-2].minor.yy56 = yylhsminor.yy56; break; case 27: /* expression ::= expression NK_STAR expression */ { PARSER_TRACE; - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy212); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy212); - yylhsminor.yy212 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, yymsp[-2].minor.yy212), releaseRawExprNode(pCxt, yymsp[0].minor.yy212))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy56); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy56); + yylhsminor.yy56 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, yymsp[-2].minor.yy56), releaseRawExprNode(pCxt, yymsp[0].minor.yy56))); } - yymsp[-2].minor.yy212 = yylhsminor.yy212; + yymsp[-2].minor.yy56 = yylhsminor.yy56; break; case 28: /* expression ::= expression NK_SLASH expression */ { PARSER_TRACE; - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy212); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy212); - yylhsminor.yy212 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, yymsp[-2].minor.yy212), releaseRawExprNode(pCxt, yymsp[0].minor.yy212))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy56); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy56); + yylhsminor.yy56 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, yymsp[-2].minor.yy56), releaseRawExprNode(pCxt, yymsp[0].minor.yy56))); } - yymsp[-2].minor.yy212 = yylhsminor.yy212; + yymsp[-2].minor.yy56 = yylhsminor.yy56; break; case 29: /* expression ::= expression NK_REM expression */ { PARSER_TRACE; - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy212); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy212); - yylhsminor.yy212 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MOD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy212), releaseRawExprNode(pCxt, yymsp[0].minor.yy212))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy56); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy56); + yylhsminor.yy56 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MOD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy56), releaseRawExprNode(pCxt, yymsp[0].minor.yy56))); } - yymsp[-2].minor.yy212 = yylhsminor.yy212; + yymsp[-2].minor.yy56 = yylhsminor.yy56; break; case 32: /* column_reference ::= column_name */ -{ PARSER_TRACE; yylhsminor.yy212 = createRawExprNode(pCxt, &yymsp[0].minor.yy79, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy79)); } - yymsp[0].minor.yy212 = yylhsminor.yy212; +{ PARSER_TRACE; yylhsminor.yy56 = createRawExprNode(pCxt, &yymsp[0].minor.yy29, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy29)); } + yymsp[0].minor.yy56 = yylhsminor.yy56; break; case 33: /* column_reference ::= table_name NK_DOT column_name */ -{ PARSER_TRACE; yylhsminor.yy212 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy79, &yymsp[0].minor.yy79, createColumnNode(pCxt, &yymsp[-2].minor.yy79, &yymsp[0].minor.yy79)); } - yymsp[-2].minor.yy212 = yylhsminor.yy212; +{ PARSER_TRACE; yylhsminor.yy56 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy29, &yymsp[0].minor.yy29, createColumnNode(pCxt, &yymsp[-2].minor.yy29, &yymsp[0].minor.yy29)); } + yymsp[-2].minor.yy56 = yylhsminor.yy56; break; case 34: /* predicate ::= expression compare_op expression */ -{ PARSER_TRACE; yylhsminor.yy212 = createOperatorNode(pCxt, yymsp[-1].minor.yy40, releaseRawExprNode(pCxt, yymsp[-2].minor.yy212), releaseRawExprNode(pCxt, yymsp[0].minor.yy212)); } - yymsp[-2].minor.yy212 = yylhsminor.yy212; - break; - case 35: /* predicate ::= expression BETWEEN expression AND expression */ -{ PARSER_TRACE; yylhsminor.yy212 = createBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-4].minor.yy212), releaseRawExprNode(pCxt, yymsp[-2].minor.yy212), releaseRawExprNode(pCxt, yymsp[0].minor.yy212)); } - yymsp[-4].minor.yy212 = yylhsminor.yy212; - break; - case 36: /* predicate ::= expression NOT BETWEEN expression AND expression */ -{ PARSER_TRACE; yylhsminor.yy212 = createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy212), releaseRawExprNode(pCxt, yymsp[-5].minor.yy212), releaseRawExprNode(pCxt, yymsp[0].minor.yy212)); } - yymsp[-5].minor.yy212 = yylhsminor.yy212; - break; - case 37: /* predicate ::= expression IS NULL */ -{ PARSER_TRACE; yylhsminor.yy212 = createIsNullCondNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy212), true); } - yymsp[-2].minor.yy212 = yylhsminor.yy212; - break; - case 38: /* predicate ::= expression IS NOT NULL */ -{ PARSER_TRACE; yylhsminor.yy212 = createIsNullCondNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy212), false); } - yymsp[-3].minor.yy212 = yylhsminor.yy212; - break; - case 39: /* predicate ::= expression in_op in_predicate_value */ -{ PARSER_TRACE; yylhsminor.yy212 = createOperatorNode(pCxt, yymsp[-1].minor.yy40, releaseRawExprNode(pCxt, yymsp[-2].minor.yy212), yymsp[0].minor.yy212); } - yymsp[-2].minor.yy212 = yylhsminor.yy212; - break; - case 40: /* compare_op ::= NK_LT */ -{ PARSER_TRACE; yymsp[0].minor.yy40 = OP_TYPE_LOWER_THAN; } - break; - case 41: /* compare_op ::= NK_GT */ -{ PARSER_TRACE; yymsp[0].minor.yy40 = OP_TYPE_GREATER_THAN; } - break; - case 42: /* compare_op ::= NK_LE */ -{ PARSER_TRACE; yymsp[0].minor.yy40 = OP_TYPE_LOWER_EQUAL; } - break; - case 43: /* compare_op ::= NK_GE */ -{ PARSER_TRACE; yymsp[0].minor.yy40 = OP_TYPE_GREATER_EQUAL; } - break; - case 44: /* compare_op ::= NK_NE */ -{ PARSER_TRACE; yymsp[0].minor.yy40 = OP_TYPE_NOT_EQUAL; } - break; - case 45: /* compare_op ::= NK_EQ */ -{ PARSER_TRACE; yymsp[0].minor.yy40 = OP_TYPE_EQUAL; } - break; - case 46: /* compare_op ::= LIKE */ -{ PARSER_TRACE; yymsp[0].minor.yy40 = OP_TYPE_LIKE; } - break; - case 47: /* compare_op ::= NOT LIKE */ -{ PARSER_TRACE; yymsp[-1].minor.yy40 = OP_TYPE_NOT_LIKE; } - break; - case 48: /* compare_op ::= MATCH */ -{ PARSER_TRACE; yymsp[0].minor.yy40 = OP_TYPE_MATCH; } - break; - case 49: /* compare_op ::= NMATCH */ -{ PARSER_TRACE; yymsp[0].minor.yy40 = OP_TYPE_NMATCH; } - break; - case 50: /* in_op ::= IN */ -{ PARSER_TRACE; yymsp[0].minor.yy40 = OP_TYPE_IN; } - break; - case 51: /* in_op ::= NOT IN */ -{ PARSER_TRACE; yymsp[-1].minor.yy40 = OP_TYPE_NOT_IN; } - break; - case 52: /* in_predicate_value ::= NK_LP expression_list NK_RP */ -{ PARSER_TRACE; yymsp[-2].minor.yy212 = createNodeListNode(pCxt, yymsp[-1].minor.yy174); } - break; - case 54: /* boolean_value_expression ::= NOT boolean_primary */ -{ PARSER_TRACE; yymsp[-1].minor.yy212 = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, yymsp[0].minor.yy212, NULL); } - break; - case 55: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ -{ PARSER_TRACE; yylhsminor.yy212 = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, yymsp[-2].minor.yy212, yymsp[0].minor.yy212); } - yymsp[-2].minor.yy212 = yylhsminor.yy212; - break; - case 56: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ -{ PARSER_TRACE; yylhsminor.yy212 = createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, yymsp[-2].minor.yy212, yymsp[0].minor.yy212); } - yymsp[-2].minor.yy212 = yylhsminor.yy212; - break; - case 58: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ - case 71: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ yytestcase(yyruleno==71); - case 72: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==72); -{ PARSER_TRACE; yymsp[-2].minor.yy212 = yymsp[-1].minor.yy212; } - break; - case 59: /* from_clause ::= FROM table_reference_list */ - case 89: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==89); - case 112: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==112); -{ PARSER_TRACE; yymsp[-1].minor.yy212 = yymsp[0].minor.yy212; } - break; - case 61: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ -{ PARSER_TRACE; yylhsminor.yy212 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy212, yymsp[0].minor.yy212, NULL); } - yymsp[-2].minor.yy212 = yylhsminor.yy212; - break; - case 64: /* table_primary ::= table_name alias_opt */ -{ PARSER_TRACE; yylhsminor.yy212 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy79, &yymsp[0].minor.yy79); } - yymsp[-1].minor.yy212 = yylhsminor.yy212; - break; - case 65: /* table_primary ::= db_name NK_DOT table_name alias_opt */ -{ PARSER_TRACE; yylhsminor.yy212 = createRealTableNode(pCxt, &yymsp[-3].minor.yy79, &yymsp[-1].minor.yy79, &yymsp[0].minor.yy79); } - yymsp[-3].minor.yy212 = yylhsminor.yy212; - break; - case 66: /* table_primary ::= subquery alias_opt */ -{ PARSER_TRACE; yylhsminor.yy212 = createTempTableNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy212), &yymsp[0].minor.yy79); } - yymsp[-1].minor.yy212 = yylhsminor.yy212; - break; - case 68: /* alias_opt ::= */ -{ PARSER_TRACE; yymsp[1].minor.yy79 = nil_token; } - break; - case 69: /* alias_opt ::= table_alias */ -{ PARSER_TRACE; yylhsminor.yy79 = yymsp[0].minor.yy79; } - yymsp[0].minor.yy79 = yylhsminor.yy79; - break; - case 70: /* alias_opt ::= AS table_alias */ -{ PARSER_TRACE; yymsp[-1].minor.yy79 = yymsp[0].minor.yy79; } - break; - case 73: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ -{ PARSER_TRACE; yylhsminor.yy212 = createJoinTableNode(pCxt, yymsp[-4].minor.yy162, yymsp[-5].minor.yy212, yymsp[-2].minor.yy212, yymsp[0].minor.yy212); } - yymsp[-5].minor.yy212 = yylhsminor.yy212; - break; - case 74: /* join_type ::= */ -{ PARSER_TRACE; yymsp[1].minor.yy162 = JOIN_TYPE_INNER; } - break; - case 75: /* join_type ::= INNER */ -{ PARSER_TRACE; yymsp[0].minor.yy162 = JOIN_TYPE_INNER; } - break; - case 76: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ -{ - PARSER_TRACE; - yymsp[-8].minor.yy212 = createSelectStmt(pCxt, yymsp[-7].minor.yy237, yymsp[-6].minor.yy174, yymsp[-5].minor.yy212); - yymsp[-8].minor.yy212 = addWhereClause(pCxt, yymsp[-8].minor.yy212, yymsp[-4].minor.yy212); - yymsp[-8].minor.yy212 = addPartitionByClause(pCxt, yymsp[-8].minor.yy212, yymsp[-3].minor.yy174); - yymsp[-8].minor.yy212 = addWindowClauseClause(pCxt, yymsp[-8].minor.yy212, yymsp[-2].minor.yy212); - yymsp[-8].minor.yy212 = addGroupByClause(pCxt, yymsp[-8].minor.yy212, yymsp[-1].minor.yy174); - yymsp[-8].minor.yy212 = addHavingClause(pCxt, yymsp[-8].minor.yy212, yymsp[0].minor.yy212); - } - break; - case 77: /* set_quantifier_opt ::= */ -{ PARSER_TRACE; yymsp[1].minor.yy237 = false; } - break; - case 78: /* set_quantifier_opt ::= DISTINCT */ -{ PARSER_TRACE; yymsp[0].minor.yy237 = true; } - break; - case 79: /* set_quantifier_opt ::= ALL */ -{ PARSER_TRACE; yymsp[0].minor.yy237 = false; } - break; - case 80: /* select_list ::= NK_STAR */ -{ PARSER_TRACE; yymsp[0].minor.yy174 = NULL; } - break; - case 81: /* select_list ::= select_sublist */ -{ PARSER_TRACE; yylhsminor.yy174 = yymsp[0].minor.yy174; } - yymsp[0].minor.yy174 = yylhsminor.yy174; - break; - case 82: /* select_sublist ::= select_item */ - case 129: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==129); -{ PARSER_TRACE; yylhsminor.yy174 = createNodeList(pCxt, yymsp[0].minor.yy212); } - yymsp[0].minor.yy174 = yylhsminor.yy174; - break; - case 83: /* select_sublist ::= select_sublist NK_COMMA select_item */ - case 130: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==130); -{ PARSER_TRACE; yylhsminor.yy174 = addNodeToList(pCxt, yymsp[-2].minor.yy174, yymsp[0].minor.yy212); } - yymsp[-2].minor.yy174 = yylhsminor.yy174; - break; - case 84: /* select_item ::= expression */ + case 39: /* predicate ::= expression in_op in_predicate_value */ yytestcase(yyruleno==39); { PARSER_TRACE; - SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy212); - yylhsminor.yy212 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy212), &t); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy56); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy56); + yylhsminor.yy56 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, yymsp[-1].minor.yy128, releaseRawExprNode(pCxt, yymsp[-2].minor.yy56), releaseRawExprNode(pCxt, yymsp[0].minor.yy56))); } - yymsp[0].minor.yy212 = yylhsminor.yy212; + yymsp[-2].minor.yy56 = yylhsminor.yy56; break; - case 85: /* select_item ::= expression column_alias */ -{ PARSER_TRACE; yylhsminor.yy212 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy212), &yymsp[0].minor.yy79); } - yymsp[-1].minor.yy212 = yylhsminor.yy212; + case 35: /* predicate ::= expression BETWEEN expression AND expression */ +{ + PARSER_TRACE; + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-4].minor.yy56); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy56); + yylhsminor.yy56 = createRawExprNodeExt(pCxt, &s, &e, createBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-4].minor.yy56), releaseRawExprNode(pCxt, yymsp[-2].minor.yy56), releaseRawExprNode(pCxt, yymsp[0].minor.yy56))); + } + yymsp[-4].minor.yy56 = yylhsminor.yy56; break; - case 86: /* select_item ::= expression AS column_alias */ -{ PARSER_TRACE; yylhsminor.yy212 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy212), &yymsp[0].minor.yy79); } - yymsp[-2].minor.yy212 = yylhsminor.yy212; + case 36: /* predicate ::= expression NOT BETWEEN expression AND expression */ +{ + PARSER_TRACE; + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-5].minor.yy56); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy56); + yylhsminor.yy56 = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy56), releaseRawExprNode(pCxt, yymsp[-5].minor.yy56), releaseRawExprNode(pCxt, yymsp[0].minor.yy56))); + } + yymsp[-5].minor.yy56 = yylhsminor.yy56; break; - case 87: /* select_item ::= table_name NK_DOT NK_STAR */ -{ PARSER_TRACE; yylhsminor.yy212 = createColumnNode(pCxt, &yymsp[-2].minor.yy79, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy212 = yylhsminor.yy212; + case 37: /* predicate ::= expression IS NULL */ +{ + PARSER_TRACE; + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy56); + yylhsminor.yy56 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NULL, releaseRawExprNode(pCxt, yymsp[-2].minor.yy56), NULL)); + } + yymsp[-2].minor.yy56 = yylhsminor.yy56; break; - case 88: /* where_clause_opt ::= */ - case 92: /* twindow_clause_opt ::= */ yytestcase(yyruleno==92); - case 97: /* sliding_opt ::= */ yytestcase(yyruleno==97); - case 99: /* fill_opt ::= */ yytestcase(yyruleno==99); - case 111: /* having_clause_opt ::= */ yytestcase(yyruleno==111); - case 119: /* slimit_clause_opt ::= */ yytestcase(yyruleno==119); - case 123: /* limit_clause_opt ::= */ yytestcase(yyruleno==123); -{ PARSER_TRACE; yymsp[1].minor.yy212 = NULL; } + case 38: /* predicate ::= expression IS NOT NULL */ +{ + PARSER_TRACE; + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-3].minor.yy56); + yylhsminor.yy56 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NOT_NULL, releaseRawExprNode(pCxt, yymsp[-3].minor.yy56), NULL)); + } + yymsp[-3].minor.yy56 = yylhsminor.yy56; break; - case 90: /* partition_by_clause_opt ::= */ - case 107: /* group_by_clause_opt ::= */ yytestcase(yyruleno==107); - case 117: /* order_by_clause_opt ::= */ yytestcase(yyruleno==117); -{ PARSER_TRACE; yymsp[1].minor.yy174 = NULL; } + case 40: /* compare_op ::= NK_LT */ +{ PARSER_TRACE; yymsp[0].minor.yy128 = OP_TYPE_LOWER_THAN; } break; - case 91: /* partition_by_clause_opt ::= PARTITION BY expression_list */ - case 108: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==108); - case 118: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==118); -{ PARSER_TRACE; yymsp[-2].minor.yy174 = yymsp[0].minor.yy174; } + case 41: /* compare_op ::= NK_GT */ +{ PARSER_TRACE; yymsp[0].minor.yy128 = OP_TYPE_GREATER_THAN; } break; - case 93: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA NK_INTEGER NK_RP */ -{ PARSER_TRACE; yymsp[-5].minor.yy212 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy212), &yymsp[-1].minor.yy0); } + case 42: /* compare_op ::= NK_LE */ +{ PARSER_TRACE; yymsp[0].minor.yy128 = OP_TYPE_LOWER_EQUAL; } break; - case 94: /* twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP */ -{ PARSER_TRACE; yymsp[-3].minor.yy212 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy212)); } + case 43: /* compare_op ::= NK_GE */ +{ PARSER_TRACE; yymsp[0].minor.yy128 = OP_TYPE_GREATER_EQUAL; } break; - case 95: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ -{ PARSER_TRACE; yymsp[-5].minor.yy212 = createIntervalWindowNode(pCxt, yymsp[-3].minor.yy212, NULL, yymsp[-1].minor.yy212, yymsp[0].minor.yy212); } + case 44: /* compare_op ::= NK_NE */ +{ PARSER_TRACE; yymsp[0].minor.yy128 = OP_TYPE_NOT_EQUAL; } break; - case 96: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ -{ PARSER_TRACE; yymsp[-7].minor.yy212 = createIntervalWindowNode(pCxt, yymsp[-5].minor.yy212, yymsp[-3].minor.yy212, yymsp[-1].minor.yy212, yymsp[0].minor.yy212); } + case 45: /* compare_op ::= NK_EQ */ +{ PARSER_TRACE; yymsp[0].minor.yy128 = OP_TYPE_EQUAL; } break; - case 98: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ -{ PARSER_TRACE; yymsp[-3].minor.yy212 = yymsp[-1].minor.yy212; } + case 46: /* compare_op ::= LIKE */ +{ PARSER_TRACE; yymsp[0].minor.yy128 = OP_TYPE_LIKE; } break; - case 100: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ -{ PARSER_TRACE; yymsp[-3].minor.yy212 = createFillNode(pCxt, yymsp[-1].minor.yy44, NULL); } + case 47: /* compare_op ::= NOT LIKE */ +{ PARSER_TRACE; yymsp[-1].minor.yy128 = OP_TYPE_NOT_LIKE; } break; - case 101: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ -{ PARSER_TRACE; yymsp[-5].minor.yy212 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy174)); } + case 48: /* compare_op ::= MATCH */ +{ PARSER_TRACE; yymsp[0].minor.yy128 = OP_TYPE_MATCH; } break; - case 102: /* fill_mode ::= NONE */ -{ PARSER_TRACE; yymsp[0].minor.yy44 = FILL_MODE_NONE; } + case 49: /* compare_op ::= NMATCH */ +{ PARSER_TRACE; yymsp[0].minor.yy128 = OP_TYPE_NMATCH; } break; - case 103: /* fill_mode ::= PREV */ -{ PARSER_TRACE; yymsp[0].minor.yy44 = FILL_MODE_PREV; } + case 50: /* in_op ::= IN */ +{ PARSER_TRACE; yymsp[0].minor.yy128 = OP_TYPE_IN; } break; - case 104: /* fill_mode ::= NULL */ -{ PARSER_TRACE; yymsp[0].minor.yy44 = FILL_MODE_NULL; } + case 51: /* in_op ::= NOT IN */ +{ PARSER_TRACE; yymsp[-1].minor.yy128 = OP_TYPE_NOT_IN; } break; - case 105: /* fill_mode ::= LINEAR */ -{ PARSER_TRACE; yymsp[0].minor.yy44 = FILL_MODE_LINEAR; } + case 52: /* in_predicate_value ::= NK_LP expression_list NK_RP */ +{ PARSER_TRACE; yylhsminor.yy56 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, createNodeListNode(pCxt, yymsp[-1].minor.yy208)); } + yymsp[-2].minor.yy56 = yylhsminor.yy56; break; - case 106: /* fill_mode ::= NEXT */ -{ PARSER_TRACE; yymsp[0].minor.yy44 = FILL_MODE_NEXT; } + case 54: /* boolean_value_expression ::= NOT boolean_primary */ +{ + PARSER_TRACE; + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy56); + yylhsminor.yy56 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, yymsp[0].minor.yy56), NULL)); + } + yymsp[-1].minor.yy56 = yylhsminor.yy56; break; - case 109: /* group_by_list ::= expression */ -{ PARSER_TRACE; yylhsminor.yy174 = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy212))); } - yymsp[0].minor.yy174 = yylhsminor.yy174; + case 55: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ +{ + PARSER_TRACE; + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy56); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy56); + yylhsminor.yy56 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy56), releaseRawExprNode(pCxt, yymsp[0].minor.yy56))); + } + yymsp[-2].minor.yy56 = yylhsminor.yy56; break; - case 110: /* group_by_list ::= group_by_list NK_COMMA expression */ -{ PARSER_TRACE; yylhsminor.yy174 = addNodeToList(pCxt, yymsp[-2].minor.yy174, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy212))); } - yymsp[-2].minor.yy174 = yylhsminor.yy174; + case 56: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ +{ + PARSER_TRACE; + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy56); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy56); + yylhsminor.yy56 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy56), releaseRawExprNode(pCxt, yymsp[0].minor.yy56))); + } + yymsp[-2].minor.yy56 = yylhsminor.yy56; break; - case 113: /* query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ + case 59: /* common_expression ::= expression */ + case 60: /* common_expression ::= boolean_value_expression */ yytestcase(yyruleno==60); +{ yylhsminor.yy56 = yymsp[0].minor.yy56; } + yymsp[0].minor.yy56 = yylhsminor.yy56; + break; + case 61: /* from_clause ::= FROM table_reference_list */ + case 91: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==91); + case 114: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==114); +{ PARSER_TRACE; yymsp[-1].minor.yy56 = yymsp[0].minor.yy56; } + break; + case 63: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ +{ PARSER_TRACE; yylhsminor.yy56 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy56, yymsp[0].minor.yy56, NULL); } + yymsp[-2].minor.yy56 = yylhsminor.yy56; + break; + case 66: /* table_primary ::= table_name alias_opt */ +{ PARSER_TRACE; yylhsminor.yy56 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy29, &yymsp[0].minor.yy29); } + yymsp[-1].minor.yy56 = yylhsminor.yy56; + break; + case 67: /* table_primary ::= db_name NK_DOT table_name alias_opt */ +{ PARSER_TRACE; yylhsminor.yy56 = createRealTableNode(pCxt, &yymsp[-3].minor.yy29, &yymsp[-1].minor.yy29, &yymsp[0].minor.yy29); } + yymsp[-3].minor.yy56 = yylhsminor.yy56; + break; + case 68: /* table_primary ::= subquery alias_opt */ +{ PARSER_TRACE; yylhsminor.yy56 = createTempTableNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy56), &yymsp[0].minor.yy29); } + yymsp[-1].minor.yy56 = yylhsminor.yy56; + break; + case 70: /* alias_opt ::= */ +{ PARSER_TRACE; yymsp[1].minor.yy29 = nil_token; } + break; + case 71: /* alias_opt ::= table_alias */ +{ PARSER_TRACE; yylhsminor.yy29 = yymsp[0].minor.yy29; } + yymsp[0].minor.yy29 = yylhsminor.yy29; + break; + case 72: /* alias_opt ::= AS table_alias */ +{ PARSER_TRACE; yymsp[-1].minor.yy29 = yymsp[0].minor.yy29; } + break; + case 73: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ + case 74: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==74); +{ PARSER_TRACE; yymsp[-2].minor.yy56 = yymsp[-1].minor.yy56; } + break; + case 75: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ +{ PARSER_TRACE; yylhsminor.yy56 = createJoinTableNode(pCxt, yymsp[-4].minor.yy36, yymsp[-5].minor.yy56, yymsp[-2].minor.yy56, yymsp[0].minor.yy56); } + yymsp[-5].minor.yy56 = yylhsminor.yy56; + break; + case 76: /* join_type ::= */ +{ PARSER_TRACE; yymsp[1].minor.yy36 = JOIN_TYPE_INNER; } + break; + case 77: /* join_type ::= INNER */ +{ PARSER_TRACE; yymsp[0].minor.yy36 = JOIN_TYPE_INNER; } + break; + case 78: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ { PARSER_TRACE; - yylhsminor.yy212 = addOrderByClause(pCxt, yymsp[-3].minor.yy212, yymsp[-2].minor.yy174); - yylhsminor.yy212 = addSlimitClause(pCxt, yylhsminor.yy212, yymsp[-1].minor.yy212); - yylhsminor.yy212 = addLimitClause(pCxt, yylhsminor.yy212, yymsp[0].minor.yy212); + yymsp[-8].minor.yy56 = createSelectStmt(pCxt, yymsp[-7].minor.yy173, yymsp[-6].minor.yy208, yymsp[-5].minor.yy56); + yymsp[-8].minor.yy56 = addWhereClause(pCxt, yymsp[-8].minor.yy56, yymsp[-4].minor.yy56); + yymsp[-8].minor.yy56 = addPartitionByClause(pCxt, yymsp[-8].minor.yy56, yymsp[-3].minor.yy208); + yymsp[-8].minor.yy56 = addWindowClauseClause(pCxt, yymsp[-8].minor.yy56, yymsp[-2].minor.yy56); + yymsp[-8].minor.yy56 = addGroupByClause(pCxt, yymsp[-8].minor.yy56, yymsp[-1].minor.yy208); + yymsp[-8].minor.yy56 = addHavingClause(pCxt, yymsp[-8].minor.yy56, yymsp[0].minor.yy56); } - yymsp[-3].minor.yy212 = yylhsminor.yy212; break; - case 115: /* query_expression_body ::= query_expression_body UNION ALL query_expression_body */ -{ PARSER_TRACE; yylhsminor.yy212 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy212, yymsp[0].minor.yy212); } - yymsp[-3].minor.yy212 = yylhsminor.yy212; + case 79: /* set_quantifier_opt ::= */ +{ PARSER_TRACE; yymsp[1].minor.yy173 = false; } break; - case 120: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ - case 124: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==124); -{ PARSER_TRACE; yymsp[-1].minor.yy212 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } + case 80: /* set_quantifier_opt ::= DISTINCT */ +{ PARSER_TRACE; yymsp[0].minor.yy173 = true; } break; - case 121: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - case 125: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==125); -{ PARSER_TRACE; yymsp[-3].minor.yy212 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } + case 81: /* set_quantifier_opt ::= ALL */ +{ PARSER_TRACE; yymsp[0].minor.yy173 = false; } break; - case 122: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - case 126: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==126); -{ PARSER_TRACE; yymsp[-3].minor.yy212 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } + case 82: /* select_list ::= NK_STAR */ +{ PARSER_TRACE; yymsp[0].minor.yy208 = NULL; } break; - case 127: /* subquery ::= NK_LP query_expression NK_RP */ -{ PARSER_TRACE; yylhsminor.yy212 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy212); } - yymsp[-2].minor.yy212 = yylhsminor.yy212; + case 83: /* select_list ::= select_sublist */ +{ PARSER_TRACE; yylhsminor.yy208 = yymsp[0].minor.yy208; } + yymsp[0].minor.yy208 = yylhsminor.yy208; break; - case 131: /* sort_specification ::= expression ordering_specification_opt null_ordering_opt */ -{ PARSER_TRACE; yylhsminor.yy212 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy212), yymsp[-1].minor.yy188, yymsp[0].minor.yy107); } - yymsp[-2].minor.yy212 = yylhsminor.yy212; + case 84: /* select_sublist ::= select_item */ + case 131: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==131); +{ PARSER_TRACE; yylhsminor.yy208 = createNodeList(pCxt, yymsp[0].minor.yy56); } + yymsp[0].minor.yy208 = yylhsminor.yy208; break; - case 132: /* ordering_specification_opt ::= */ -{ PARSER_TRACE; yymsp[1].minor.yy188 = ORDER_ASC; } + case 85: /* select_sublist ::= select_sublist NK_COMMA select_item */ + case 132: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==132); +{ PARSER_TRACE; yylhsminor.yy208 = addNodeToList(pCxt, yymsp[-2].minor.yy208, yymsp[0].minor.yy56); } + yymsp[-2].minor.yy208 = yylhsminor.yy208; break; - case 133: /* ordering_specification_opt ::= ASC */ -{ PARSER_TRACE; yymsp[0].minor.yy188 = ORDER_ASC; } + case 86: /* select_item ::= common_expression */ +{ + PARSER_TRACE; + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy56); + yylhsminor.yy56 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy56), &t); + } + yymsp[0].minor.yy56 = yylhsminor.yy56; break; - case 134: /* ordering_specification_opt ::= DESC */ -{ PARSER_TRACE; yymsp[0].minor.yy188 = ORDER_DESC; } + case 87: /* select_item ::= common_expression column_alias */ +{ PARSER_TRACE; yylhsminor.yy56 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy56), &yymsp[0].minor.yy29); } + yymsp[-1].minor.yy56 = yylhsminor.yy56; break; - case 135: /* null_ordering_opt ::= */ -{ PARSER_TRACE; yymsp[1].minor.yy107 = NULL_ORDER_DEFAULT; } + case 88: /* select_item ::= common_expression AS column_alias */ +{ PARSER_TRACE; yylhsminor.yy56 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy56), &yymsp[0].minor.yy29); } + yymsp[-2].minor.yy56 = yylhsminor.yy56; break; - case 136: /* null_ordering_opt ::= NULLS FIRST */ -{ PARSER_TRACE; yymsp[-1].minor.yy107 = NULL_ORDER_FIRST; } + case 89: /* select_item ::= table_name NK_DOT NK_STAR */ +{ PARSER_TRACE; yylhsminor.yy56 = createColumnNode(pCxt, &yymsp[-2].minor.yy29, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy56 = yylhsminor.yy56; break; - case 137: /* null_ordering_opt ::= NULLS LAST */ -{ PARSER_TRACE; yymsp[-1].minor.yy107 = NULL_ORDER_LAST; } + case 90: /* where_clause_opt ::= */ + case 94: /* twindow_clause_opt ::= */ yytestcase(yyruleno==94); + case 99: /* sliding_opt ::= */ yytestcase(yyruleno==99); + case 101: /* fill_opt ::= */ yytestcase(yyruleno==101); + case 113: /* having_clause_opt ::= */ yytestcase(yyruleno==113); + case 121: /* slimit_clause_opt ::= */ yytestcase(yyruleno==121); + case 125: /* limit_clause_opt ::= */ yytestcase(yyruleno==125); +{ PARSER_TRACE; yymsp[1].minor.yy56 = NULL; } + break; + case 92: /* partition_by_clause_opt ::= */ + case 109: /* group_by_clause_opt ::= */ yytestcase(yyruleno==109); + case 119: /* order_by_clause_opt ::= */ yytestcase(yyruleno==119); +{ PARSER_TRACE; yymsp[1].minor.yy208 = NULL; } + break; + case 93: /* partition_by_clause_opt ::= PARTITION BY expression_list */ + case 110: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==110); + case 120: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==120); +{ PARSER_TRACE; yymsp[-2].minor.yy208 = yymsp[0].minor.yy208; } + break; + case 95: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA NK_INTEGER NK_RP */ +{ PARSER_TRACE; yymsp[-5].minor.yy56 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy56), &yymsp[-1].minor.yy0); } + break; + case 96: /* twindow_clause_opt ::= STATE_WINDOW NK_LP column_reference NK_RP */ +{ PARSER_TRACE; yymsp[-3].minor.yy56 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy56)); } + break; + case 97: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ +{ PARSER_TRACE; yymsp[-5].minor.yy56 = createIntervalWindowNode(pCxt, yymsp[-3].minor.yy56, NULL, yymsp[-1].minor.yy56, yymsp[0].minor.yy56); } + break; + case 98: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ +{ PARSER_TRACE; yymsp[-7].minor.yy56 = createIntervalWindowNode(pCxt, yymsp[-5].minor.yy56, yymsp[-3].minor.yy56, yymsp[-1].minor.yy56, yymsp[0].minor.yy56); } + break; + case 100: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ +{ PARSER_TRACE; yymsp[-3].minor.yy56 = yymsp[-1].minor.yy56; } + break; + case 102: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ +{ PARSER_TRACE; yymsp[-3].minor.yy56 = createFillNode(pCxt, yymsp[-1].minor.yy18, NULL); } + break; + case 103: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ +{ PARSER_TRACE; yymsp[-5].minor.yy56 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy208)); } + break; + case 104: /* fill_mode ::= NONE */ +{ PARSER_TRACE; yymsp[0].minor.yy18 = FILL_MODE_NONE; } + break; + case 105: /* fill_mode ::= PREV */ +{ PARSER_TRACE; yymsp[0].minor.yy18 = FILL_MODE_PREV; } + break; + case 106: /* fill_mode ::= NULL */ +{ PARSER_TRACE; yymsp[0].minor.yy18 = FILL_MODE_NULL; } + break; + case 107: /* fill_mode ::= LINEAR */ +{ PARSER_TRACE; yymsp[0].minor.yy18 = FILL_MODE_LINEAR; } + break; + case 108: /* fill_mode ::= NEXT */ +{ PARSER_TRACE; yymsp[0].minor.yy18 = FILL_MODE_NEXT; } + break; + case 111: /* group_by_list ::= expression */ +{ PARSER_TRACE; yylhsminor.yy208 = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy56))); } + yymsp[0].minor.yy208 = yylhsminor.yy208; + break; + case 112: /* group_by_list ::= group_by_list NK_COMMA expression */ +{ PARSER_TRACE; yylhsminor.yy208 = addNodeToList(pCxt, yymsp[-2].minor.yy208, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy56))); } + yymsp[-2].minor.yy208 = yylhsminor.yy208; + break; + case 115: /* query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ +{ + PARSER_TRACE; + yylhsminor.yy56 = addOrderByClause(pCxt, yymsp[-3].minor.yy56, yymsp[-2].minor.yy208); + yylhsminor.yy56 = addSlimitClause(pCxt, yylhsminor.yy56, yymsp[-1].minor.yy56); + yylhsminor.yy56 = addLimitClause(pCxt, yylhsminor.yy56, yymsp[0].minor.yy56); + } + yymsp[-3].minor.yy56 = yylhsminor.yy56; + break; + case 117: /* query_expression_body ::= query_expression_body UNION ALL query_expression_body */ +{ PARSER_TRACE; yylhsminor.yy56 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy56, yymsp[0].minor.yy56); } + yymsp[-3].minor.yy56 = yylhsminor.yy56; + break; + case 122: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ + case 126: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==126); +{ PARSER_TRACE; yymsp[-1].minor.yy56 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } + break; + case 123: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + case 127: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==127); +{ PARSER_TRACE; yymsp[-3].minor.yy56 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } + break; + case 124: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + case 128: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==128); +{ PARSER_TRACE; yymsp[-3].minor.yy56 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } + break; + case 129: /* subquery ::= NK_LP query_expression NK_RP */ +{ PARSER_TRACE; yylhsminor.yy56 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy56); } + yymsp[-2].minor.yy56 = yylhsminor.yy56; + break; + case 130: /* search_condition ::= common_expression */ +{ PARSER_TRACE; yylhsminor.yy56 = releaseRawExprNode(pCxt, yymsp[0].minor.yy56); } + yymsp[0].minor.yy56 = yylhsminor.yy56; + break; + case 133: /* sort_specification ::= expression ordering_specification_opt null_ordering_opt */ +{ PARSER_TRACE; yylhsminor.yy56 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy56), yymsp[-1].minor.yy218, yymsp[0].minor.yy109); } + yymsp[-2].minor.yy56 = yylhsminor.yy56; + break; + case 134: /* ordering_specification_opt ::= */ +{ PARSER_TRACE; yymsp[1].minor.yy218 = ORDER_ASC; } + break; + case 135: /* ordering_specification_opt ::= ASC */ +{ PARSER_TRACE; yymsp[0].minor.yy218 = ORDER_ASC; } + break; + case 136: /* ordering_specification_opt ::= DESC */ +{ PARSER_TRACE; yymsp[0].minor.yy218 = ORDER_DESC; } + break; + case 137: /* null_ordering_opt ::= */ +{ PARSER_TRACE; yymsp[1].minor.yy109 = NULL_ORDER_DEFAULT; } + break; + case 138: /* null_ordering_opt ::= NULLS FIRST */ +{ PARSER_TRACE; yymsp[-1].minor.yy109 = NULL_ORDER_FIRST; } + break; + case 139: /* null_ordering_opt ::= NULLS LAST */ +{ PARSER_TRACE; yymsp[-1].minor.yy109 = NULL_ORDER_LAST; } break; default: break; diff --git a/source/libs/parser/src/parserImpl.c b/source/libs/parser/src/parserImpl.c index 84486079c3..a2602c42ee 100644 --- a/source/libs/parser/src/parserImpl.c +++ b/source/libs/parser/src/parserImpl.c @@ -174,6 +174,19 @@ static uint32_t getToken(const char* z, uint32_t* tokenId) { return n; } +static EStmtType getStmtType(const SNode* pRootNode) { + if (NULL == pRootNode) { + return STMT_TYPE_CMD; + } + switch (nodeType(pRootNode)) { + case QUERY_NODE_SELECT_STMT: + return STMT_TYPE_QUERY; + default: + break; + } + return STMT_TYPE_CMD; +} + int32_t doParse(SParseContext* pParseCxt, SQuery* pQuery) { SAstCreateContext cxt; createAstCreateContext(pParseCxt, &cxt); @@ -181,15 +194,12 @@ int32_t doParse(SParseContext* pParseCxt, SQuery* pQuery) { int32_t i = 0; while (1) { SToken t0 = {0}; - // printf("===========================\n"); if (cxt.pQueryCxt->pSql[i] == 0) { NewParse(pParser, 0, t0, &cxt); goto abort_parse; } - // printf("input: [%s]\n", cxt.pQueryCxt->pSql + i); t0.n = getToken((char *)&cxt.pQueryCxt->pSql[i], &t0.type); t0.z = (char *)(cxt.pQueryCxt->pSql + i); - // printf("token : %d %d [%s]\n", t0.type, t0.n, t0.z); i += t0.n; switch (t0.type) { @@ -201,14 +211,12 @@ int32_t doParse(SParseContext* pParseCxt, SQuery* pQuery) { NewParse(pParser, 0, t0, &cxt); goto abort_parse; } - case TK_QUESTION: case TK_ILLEGAL: { snprintf(cxt.pQueryCxt->pMsg, cxt.pQueryCxt->msgLen, "unrecognized token: \"%s\"", t0.z); cxt.valid = false; goto abort_parse; } - case TK_HEX: case TK_OCT: case TK_BIN: { @@ -216,7 +224,6 @@ int32_t doParse(SParseContext* pParseCxt, SQuery* pQuery) { cxt.valid = false; goto abort_parse; } - default: NewParse(pParser, t0.type, t0, &cxt); // NewParseTrace(stdout, ""); @@ -227,9 +234,9 @@ int32_t doParse(SParseContext* pParseCxt, SQuery* pQuery) { } abort_parse: - // printf("doParse completed.\n"); NewParseFree(pParser, free); destroyAstCreateContext(&cxt); + pQuery->stmtType = getStmtType(cxt.pRootNode); pQuery->pRoot = cxt.pRootNode; return cxt.valid ? TSDB_CODE_SUCCESS : TSDB_CODE_FAILED; } @@ -255,7 +262,6 @@ static bool beforeHaving(ESqlClause clause) { typedef struct STranslateContext { SParseContext* pParseCxt; - FuncMgtHandle fmgt; int32_t errCode; SMsgBuf msgBuf; SArray* pNsLevel; // element is SArray*, the element of this subarray is STableNode* @@ -292,6 +298,8 @@ static char* getSyntaxErrFormat(int32_t errCode) { return "Not SELECTed expression"; case TSDB_CODE_PAR_NOT_SINGLE_GROUP: return "Not a single-group group function"; + case TSDB_CODE_PAR_OUT_OF_MEMORY: + return "Out of memory"; default: return "Unknown error"; } @@ -376,12 +384,15 @@ static void setColumnInfoByExpr(const STableNode* pTable, SExprNode* pExpr, SCol pCol->node.resType = pExpr->resType; } -static int32_t createColumnNodeByTable(const STableNode* pTable, SNodeList* pList) { +static int32_t createColumnNodeByTable(STranslateContext* pCxt, const STableNode* pTable, SNodeList* pList) { if (QUERY_NODE_REAL_TABLE == nodeType(pTable)) { const STableMeta* pMeta = ((SRealTableNode*)pTable)->pMeta; int32_t nums = pMeta->tableInfo.numOfTags + pMeta->tableInfo.numOfColumns; for (int32_t i = 0; i < nums; ++i) { SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); + if (NULL == pCol) { + return generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_OUT_OF_MEMORY); + } setColumnInfoBySchema(pTable, pMeta->schema + i, pCol); nodesListAppend(pList, (SNode*)pCol); } @@ -390,10 +401,14 @@ static int32_t createColumnNodeByTable(const STableNode* pTable, SNodeList* pLis SNode* pNode; FOREACH(pNode, pProjectList) { SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); + if (NULL == pCol) { + return generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_OUT_OF_MEMORY); + } setColumnInfoByExpr(pTable, (SExprNode*)pNode, pCol); nodesListAppend(pList, (SNode*)pCol); } } + return TSDB_CODE_SUCCESS; } static bool findAndSetColumn(SColumnNode* pCol, const STableNode* pTable) { @@ -539,7 +554,7 @@ static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) { case TSDB_DATA_TYPE_UTINYINT: case TSDB_DATA_TYPE_USMALLINT: case TSDB_DATA_TYPE_UINT: - case TSDB_DATA_TYPE_UBIGINT:{ + case TSDB_DATA_TYPE_UBIGINT: { char* endPtr = NULL; pVal->datum.u = strtoull(pVal->literal, &endPtr, 10); break; @@ -556,12 +571,20 @@ static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) { case TSDB_DATA_TYPE_VARBINARY: { int32_t n = strlen(pVal->literal); pVal->datum.p = calloc(1, n); + if (NULL == pVal->datum.p) { + generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_OUT_OF_MEMORY); + return DEAL_RES_ERROR; + } trimStringCopy(pVal->literal, n, pVal->datum.p); break; } case TSDB_DATA_TYPE_TIMESTAMP: { int32_t n = strlen(pVal->literal); char* tmp = calloc(1, n); + if (NULL == tmp) { + generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_OUT_OF_MEMORY); + return DEAL_RES_ERROR; + } int32_t len = trimStringCopy(pVal->literal, n, tmp); if (taosParseTime(tmp, &pVal->datum.i, len, pVal->node.resType.precision, tsDaylight) != TSDB_CODE_SUCCESS) { tfree(tmp); @@ -608,7 +631,7 @@ static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) { } static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode* pFunc) { - if (TSDB_CODE_SUCCESS != fmGetFuncInfo(pCxt->fmgt, pFunc->functionName, &pFunc->funcId, &pFunc->funcType)) { + if (TSDB_CODE_SUCCESS != fmGetFuncInfo(pFunc->functionName, &pFunc->funcId, &pFunc->funcType)) { generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_INVALID_FUNTION, pFunc->functionName); return DEAL_RES_ERROR; } @@ -628,6 +651,12 @@ static EDealRes translateExprSubquery(STranslateContext* pCxt, SNode* pNode) { return (TSDB_CODE_SUCCESS == translateSubquery(pCxt, pNode) ? DEAL_RES_CONTINUE : DEAL_RES_ERROR); } +static EDealRes translateLogicCond(STranslateContext* pCxt, SLogicConditionNode* pCond) { + pCond->node.resType.type = TSDB_DATA_TYPE_BOOL; + pCond->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; + return DEAL_RES_CONTINUE; +} + static EDealRes doTranslateExpr(SNode* pNode, void* pContext) { STranslateContext* pCxt = (STranslateContext*)pContext; switch (nodeType(pNode)) { @@ -639,6 +668,8 @@ static EDealRes doTranslateExpr(SNode* pNode, void* pContext) { return translateOperator(pCxt, (SOperatorNode*)pNode); case QUERY_NODE_FUNCTION: return translateFunction(pCxt, (SFunctionNode*)pNode); + case QUERY_NODE_LOGIC_CONDITION: + return translateLogicCond(pCxt, (SLogicConditionNode*)pNode); case QUERY_NODE_TEMP_TABLE: return translateExprSubquery(pCxt, ((STempTableNode*)pNode)->pSubquery); default: @@ -798,9 +829,15 @@ static int32_t translateStar(STranslateContext* pCxt, SSelectStmt* pSelect, bool SArray* pTables = taosArrayGetP(pCxt->pNsLevel, pCxt->currLevel); size_t nums = taosArrayGetSize(pTables); pSelect->pProjectionList = nodesMakeList(); + if (NULL == pSelect->pProjectionList) { + return generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_OUT_OF_MEMORY); + } for (size_t i = 0; i < nums; ++i) { STableNode* pTable = taosArrayGetP(pTables, i); - createColumnNodeByTable(pTable, pSelect->pProjectionList); + int32_t code = createColumnNodeByTable(pCxt, pTable, pSelect->pProjectionList); + if (TSDB_CODE_SUCCESS != code) { + return code; + } } *pIsSelectStar = true; } else { @@ -840,7 +877,7 @@ static int32_t getPositionValue(const SValueNode* pVal) { return -1; } -static bool translateOrderByPosition(STranslateContext* pCxt, SNodeList* pProjectionList, SNodeList* pOrderByList, bool* pOther) { +static int32_t translateOrderByPosition(STranslateContext* pCxt, SNodeList* pProjectionList, SNodeList* pOrderByList, bool* pOther) { *pOther = false; SNode* pNode; FOREACH(pNode, pOrderByList) { @@ -848,18 +885,20 @@ static bool translateOrderByPosition(STranslateContext* pCxt, SNodeList* pProjec if (QUERY_NODE_VALUE == nodeType(pExpr)) { SValueNode* pVal = (SValueNode*)pExpr; if (!translateValue(pCxt, pVal)) { - return false; + return pCxt->errCode; } - int32_t pos = getPositionValue((SValueNode*)pExpr); + int32_t pos = getPositionValue(pVal); if (pos < 0) { ERASE_NODE(pOrderByList); nodesDestroyNode(pNode); continue; } else if (0 == pos || pos > LIST_LENGTH(pProjectionList)) { - generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT); - return false; + return generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_WRONG_NUMBER_OF_SELECT); } else { SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); + if (NULL == pCol) { + return generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_OUT_OF_MEMORY); + } setColumnInfoByExpr(NULL, (SExprNode*)nodesListGetNode(pProjectionList, pos - 1), pCol); ((SOrderByExprNode*)pNode)->pExpr = (SNode*)pCol; nodesDestroyNode(pExpr); @@ -868,19 +907,20 @@ static bool translateOrderByPosition(STranslateContext* pCxt, SNodeList* pProjec *pOther = true; } } - return true; + return TSDB_CODE_SUCCESS; } static int32_t translateOrderBy(STranslateContext* pCxt, SSelectStmt* pSelect) { bool other; - if (!translateOrderByPosition(pCxt, pSelect->pProjectionList, pSelect->pOrderByList, &other)) { - return pCxt->errCode; + int32_t code = translateOrderByPosition(pCxt, pSelect->pProjectionList, pSelect->pOrderByList, &other); + if (TSDB_CODE_SUCCESS != code) { + return code; } if (!other) { return TSDB_CODE_SUCCESS; } pCxt->currClause = SQL_CLAUSE_ORDER_BY; - int32_t code = translateExprList(pCxt, pSelect->pOrderByList); + code = translateExprList(pCxt, pSelect->pOrderByList); if (TSDB_CODE_SUCCESS == code) { code = checkExprListForGroupBy(pCxt, pSelect->pOrderByList); } @@ -937,12 +977,6 @@ static int32_t translateFrom(STranslateContext* pCxt, SNode* pTable) { return translateTable(pCxt, pTable); } -// typedef struct SSelectStmt { -// bool isDistinct; -// SNode* pLimit; -// SNode* pSlimit; -// } SSelectStmt; - static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) { pCxt->pCurrStmt = pSelect; int32_t code = translateFrom(pCxt, pSelect->pFromTable); @@ -996,6 +1030,26 @@ static int32_t translateSubquery(STranslateContext* pCxt, SNode* pNode) { return code; } +int32_t setReslutSchema(STranslateContext* pCxt, SQuery* pQuery) { + if (QUERY_NODE_SELECT_STMT == nodeType(pQuery->pRoot)) { + SSelectStmt* pSelect = (SSelectStmt*)pQuery->pRoot; + pQuery->numOfResCols = LIST_LENGTH(pSelect->pProjectionList); + pQuery->pResSchema = calloc(pQuery->numOfResCols, sizeof(SSchema)); + if (NULL == pQuery->pResSchema) { + return generateSyntaxErrMsg(pCxt, TSDB_CODE_PAR_OUT_OF_MEMORY); + } + SNode* pNode; + int32_t index = 0; + FOREACH(pNode, pSelect->pProjectionList) { + SExprNode* pExpr = (SExprNode*)pNode; + pQuery->pResSchema[index].type = pExpr->resType.type; + pQuery->pResSchema[index].bytes = pExpr->resType.bytes; + strcpy(pQuery->pResSchema[index].name, pExpr->aliasName); + } + } + return TSDB_CODE_SUCCESS; +} + int32_t doTranslate(SParseContext* pParseCxt, SQuery* pQuery) { STranslateContext cxt = { .pParseCxt = pParseCxt, @@ -1006,12 +1060,11 @@ int32_t doTranslate(SParseContext* pParseCxt, SQuery* pQuery) { .currClause = 0 }; int32_t code = fmFuncMgtInit(); - if (TSDB_CODE_SUCCESS != code) { - return code; + if (TSDB_CODE_SUCCESS == code) { + code = translateQuery(&cxt, pQuery->pRoot); } - code = fmGetHandle(&cxt.fmgt); - if (TSDB_CODE_SUCCESS != code) { - return code; + if (TSDB_CODE_SUCCESS == code && STMT_TYPE_QUERY == pQuery->stmtType) { + code = setReslutSchema(&cxt, pQuery); } - return translateQuery(&cxt, pQuery->pRoot); + return code; } diff --git a/source/libs/parser/test/insertParserTest.cpp b/source/libs/parser/test/insertParserTest.cpp index 86e8b1d7aa..b6992e5157 100644 --- a/source/libs/parser/test/insertParserTest.cpp +++ b/source/libs/parser/test/insertParserTest.cpp @@ -70,7 +70,7 @@ protected: for (size_t i = 0; i < num; ++i) { SVgDataBlocks* vg = (SVgDataBlocks*)taosArrayGetP(res_->pDataBlocks, i); cout << "vgId:" << vg->vg.vgId << ", numOfTables:" << vg->numOfTables << ", dataSize:" << vg->size << endl; - SSubmitMsg* submit = (SSubmitMsg*)vg->pData; + SSubmitReq* submit = (SSubmitReq*)vg->pData; cout << "length:" << ntohl(submit->length) << ", numOfBlocks:" << ntohl(submit->numOfBlocks) << endl; int32_t numOfBlocks = ntohl(submit->numOfBlocks); SSubmitBlk* blk = (SSubmitBlk*)(submit + 1); @@ -93,7 +93,7 @@ protected: SVgDataBlocks* vg = (SVgDataBlocks*)taosArrayGetP(res_->pDataBlocks, i); ASSERT_EQ(vg->numOfTables, numOfTables); ASSERT_GE(vg->size, 0); - SSubmitMsg* submit = (SSubmitMsg*)vg->pData; + SSubmitReq* submit = (SSubmitReq*)vg->pData; ASSERT_GE(ntohl(submit->length), 0); ASSERT_GE(ntohl(submit->numOfBlocks), 0); int32_t numOfBlocks = ntohl(submit->numOfBlocks); diff --git a/source/libs/parser/test/newParserTest.cpp b/source/libs/parser/test/newParserTest.cpp index 2616c6c251..996e941848 100644 --- a/source/libs/parser/test/newParserTest.cpp +++ b/source/libs/parser/test/newParserTest.cpp @@ -76,7 +76,8 @@ private: case QUERY_NODE_COLUMN: case QUERY_NODE_VALUE: case QUERY_NODE_OPERATOR: - case QUERY_NODE_FUNCTION: { + case QUERY_NODE_FUNCTION: + case QUERY_NODE_LOGIC_CONDITION: { SExprNode* pExpr = (SExprNode*)node; str.append(" [" + dataTypeToStr(pExpr->resType) + "]"); if (isProject) { @@ -215,6 +216,34 @@ private: str.append((NULL_ORDER_FIRST == pOrderBy->nullOrder ? " NULLS FIRST" : " NULLS LAST")); } + string logicConditionTypeToStr(ELogicConditionType type) { + switch (type) { + case LOGIC_COND_TYPE_AND: + return "AND"; + case LOGIC_COND_TYPE_OR: + return "OR"; + case LOGIC_COND_TYPE_NOT: + return "NOT"; + default: + break; + } + return "Unknown logic cond type"; + } + + void logicCondToStr(SLogicConditionNode* pCond, string& str) { + SNode* node = nullptr; + bool first = true; + FOREACH(node, pCond->pParameterList) { + if (first && LOGIC_COND_TYPE_NOT == pCond->condType) { + str.append(logicConditionTypeToStr(pCond->condType) + " "); + } else if (!first && LOGIC_COND_TYPE_NOT != pCond->condType) { + str.append(" " + logicConditionTypeToStr(pCond->condType) + " "); + } + first = false; + nodeToStr(node, str, false); + } + } + void nodeToStr(const SNode* node, string& str, bool isProject) { if (nullptr == node) { return; @@ -237,6 +266,10 @@ private: functionToStr((SFunctionNode*)node, str); break; } + case QUERY_NODE_LOGIC_CONDITION: { + logicCondToStr((SLogicConditionNode*)node, str); + break; + } case QUERY_NODE_GROUPING_SET: { groupingSetToStr((SGroupingSetNode*)node, str); break; @@ -511,6 +544,12 @@ TEST_F(NewParserTest, selectExpression) { bind("SELECT ts + 10s, c1 + 10, concat(c2, 'abc') FROM t1"); ASSERT_TRUE(run()); + + bind("SELECT ts > 0, c1 < 20 AND c2 = 'qaz' FROM t1"); + ASSERT_TRUE(run()); + + bind("SELECT ts > 0, c1 BETWEEN 10 AND 20 AND c2 = 'qaz' FROM t1"); + ASSERT_TRUE(run()); } TEST_F(NewParserTest, selectClause) { diff --git a/source/libs/planner/CMakeLists.txt b/source/libs/planner/CMakeLists.txt index 8d8c148fde..14b4488e6a 100644 --- a/source/libs/planner/CMakeLists.txt +++ b/source/libs/planner/CMakeLists.txt @@ -8,7 +8,7 @@ target_include_directories( target_link_libraries( planner - PRIVATE os util catalog cjson parser function qcom + PRIVATE os util nodes catalog cjson parser function qcom PUBLIC transport ) diff --git a/source/libs/planner/inc/plannerImpl.h b/source/libs/planner/inc/plannerImpl.h index 050329693c..1f8df990b1 100644 --- a/source/libs/planner/inc/plannerImpl.h +++ b/source/libs/planner/inc/plannerImpl.h @@ -20,6 +20,33 @@ extern "C" { #endif +#include "querynodes.h" +#include "planner.h" + +typedef struct SLogicNode { + ENodeType type; + SNodeList* pTargets; + SNode* pConditions; + SNodeList* pChildren; + struct SLogicNode* pParent; +} SLogicNode; + +typedef struct SScanLogicNode { + SLogicNode node; + SNodeList* pScanCols; + struct STableMeta* pMeta; +} SScanLogicNode; + +typedef struct SFilterLogicNode { + SLogicNode node; +} SFilterLogicNode; + +typedef struct SAggLogicNode { + SLogicNode node; + SNodeList* pGroupKeys; + SNodeList* pAggFuncs; +} SAggLogicNode; + #ifdef __cplusplus } #endif diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index 20e8378c26..eadc95b98d 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -216,7 +216,7 @@ static SPhyNode* createMultiTableScanNode(SQueryPlanNode* pPlanNode, SQueryTable } else if (needSeqScan(pPlanNode)) { return createUserTableScanNode(pPlanNode, pTable, OP_TableSeqScan); } - int32_t type = (pPlanNode->info.type == QNODE_TABLESCAN)? OP_DataBlocksOptScan:OP_StreamScan; + int32_t type = (pPlanNode->info.type == QNODE_TABLESCAN)? OP_TableScan:OP_StreamScan; return createUserTableScanNode(pPlanNode, pTable, type); } @@ -288,7 +288,7 @@ static bool needMultiNodeScan(SQueryTableInfo* pTable) { static SPhyNode* createSingleTableScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTableInfo, SSubplan* subplan) { SVgroupsInfo* pVgroupsInfo = pTableInfo->pMeta->vgroupList; vgroupInfoToNodeAddr(&(pVgroupsInfo->vgroups[0]), &subplan->execNode); - int32_t type = (pPlanNode->info.type == QNODE_TABLESCAN)? OP_DataBlocksOptScan:OP_StreamScan; + int32_t type = (pPlanNode->info.type == QNODE_TABLESCAN)? OP_TableScan:OP_StreamScan; return createUserTableScanNode(pPlanNode, pTableInfo, type); } diff --git a/source/libs/planner/src/physicalPlanJson.c b/source/libs/planner/src/physicalPlanJson.c index e367f2e74b..b2109c0a4f 100644 --- a/source/libs/planner/src/physicalPlanJson.c +++ b/source/libs/planner/src/physicalPlanJson.c @@ -88,7 +88,7 @@ static const char* jkPnodeType = "Type"; static int32_t getPnodeTypeSize(cJSON* json) { switch (getNumber(json, jkPnodeType)) { case OP_StreamScan: - case OP_DataBlocksOptScan: + case OP_TableScan: case OP_TableSeqScan: return sizeof(STableScanPhyNode); case OP_TagScan: @@ -830,7 +830,7 @@ static bool specificPhyNodeToJson(const void* obj, cJSON* json) { const SPhyNode* phyNode = (const SPhyNode*)obj; switch (phyNode->info.type) { case OP_StreamScan: - case OP_DataBlocksOptScan: + case OP_TableScan: case OP_TableSeqScan: return tableScanNodeToJson(obj, json); case OP_TagScan: @@ -868,7 +868,7 @@ static bool specificPhyNodeFromJson(const cJSON* json, void* obj) { SPhyNode* phyNode = (SPhyNode*)obj; switch (phyNode->info.type) { case OP_StreamScan: - case OP_DataBlocksOptScan: + case OP_TableScan: case OP_TableSeqScan: return tableScanNodeFromJson(json, obj); case OP_TagScan: diff --git a/source/libs/planner/src/plannerImpl.c b/source/libs/planner/src/plannerImpl.c index 6acb927db9..ddb3023bb3 100644 --- a/source/libs/planner/src/plannerImpl.c +++ b/source/libs/planner/src/plannerImpl.c @@ -13,7 +13,192 @@ * along with this program. If not, see . */ +#include "plannerImpl.h" +#include "functionMgt.h" -// int32_t getPlan(SNode* pRoot, SQueryPlanNode** pQueryPlan) { +static SLogicNode* createQueryLogicNode(SNode* pStmt); -// } \ No newline at end of file +typedef struct SCollectColumnsCxt { + SNodeList* pCols; + SHashObj* pColIdHash; +} SCollectColumnsCxt; + +static EDealRes doCollectColumns(SNode* pNode, void* pContext) { + if (QUERY_NODE_COLUMN == nodeType(pNode)) { + SCollectColumnsCxt* pCxt = (SCollectColumnsCxt*)pContext; + int16_t colId = ((SColumnNode*)pNode)->colId; + if (colId > 0) { + if (NULL == taosHashGet(pCxt->pColIdHash, &colId, sizeof(colId))) { + taosHashPut(pCxt->pColIdHash, &colId, sizeof(colId), NULL, 0); + nodesListAppend(pCxt->pCols, pNode); + } + } + } + return DEAL_RES_CONTINUE; +} + +static SNodeList* collectColumns(SSelectStmt* pSelect) { + SCollectColumnsCxt cxt = { .pCols = nodesMakeList(), .pColIdHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK) }; + if (NULL == cxt.pCols || NULL == cxt.pColIdHash) { + return NULL; + } + nodesWalkNode(pSelect->pFromTable, doCollectColumns, &cxt); + nodesWalkNode(pSelect->pWhere, doCollectColumns, &cxt); + nodesWalkList(pSelect->pPartitionByList, doCollectColumns, &cxt); + nodesWalkNode(pSelect->pWindow, doCollectColumns, &cxt); + nodesWalkList(pSelect->pGroupByList, doCollectColumns, &cxt); + nodesWalkNode(pSelect->pHaving, doCollectColumns, &cxt); + nodesWalkList(pSelect->pProjectionList, doCollectColumns, &cxt); + nodesWalkList(pSelect->pOrderByList, doCollectColumns, &cxt); + taosHashCleanup(cxt.pColIdHash); + return cxt.pCols; +} + +typedef struct SCollectAggFuncsCxt { + SNodeList* pAggFuncs; +} SCollectAggFuncsCxt; + +static EDealRes doCollectAggFuncs(SNode* pNode, void* pContext) { + if (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsAggFunc(((SFunctionNode*)pNode)->funcId)) { + SCollectAggFuncsCxt* pCxt = (SCollectAggFuncsCxt*)pContext; + nodesListAppend(pCxt->pAggFuncs, pNode); + return DEAL_RES_IGNORE_CHILD; + } + return DEAL_RES_CONTINUE; +} + +static SNodeList* collectAggFuncs(SSelectStmt* pSelect) { + SCollectAggFuncsCxt cxt = { .pAggFuncs = nodesMakeList() }; + if (NULL == cxt.pAggFuncs) { + return NULL; + } + nodesWalkNode(pSelect->pHaving, doCollectAggFuncs, &cxt); + nodesWalkList(pSelect->pProjectionList, doCollectAggFuncs, &cxt); + if (!pSelect->isDistinct) { + nodesWalkList(pSelect->pOrderByList, doCollectAggFuncs, &cxt); + } + return cxt.pAggFuncs; +} + +typedef struct SRewriteExprCxt { + SNodeList* pTargets; +} SRewriteExprCxt; + +static EDealRes doRewriteExpr(SNode** pNode, void* pContext) { + SRewriteExprCxt* pCxt = (SRewriteExprCxt*)pContext; + SNode* pTarget; + int32_t index = 0; + FOREACH(pTarget, pCxt->pTargets) { + if (nodesEqualNode(pTarget, *pNode)) { + nodesDestroyNode(*pNode); + *pNode = nodesMakeNode(QUERY_NODE_COLUMN_REF); + ((SColumnRef*)*pNode)->slotId = index; + return DEAL_RES_IGNORE_CHILD; + } + ++index; + } + return DEAL_RES_CONTINUE; +} + +static int32_t rewriteExpr(SNodeList* pTargets, SSelectStmt* pSelect) { + SRewriteExprCxt cxt = { .pTargets = pTargets }; + nodesRewriteNode(&(pSelect->pFromTable), doRewriteExpr, &cxt); + nodesRewriteNode(&(pSelect->pWhere), doRewriteExpr, &cxt); + nodesRewriteList(pSelect->pPartitionByList, doRewriteExpr, &cxt); + nodesRewriteNode(&(pSelect->pWindow), doRewriteExpr, &cxt); + nodesRewriteList(pSelect->pGroupByList, doRewriteExpr, &cxt); + nodesRewriteNode(&(pSelect->pHaving), doRewriteExpr, &cxt); + nodesRewriteList(pSelect->pProjectionList, doRewriteExpr, &cxt); + nodesRewriteList(pSelect->pOrderByList, doRewriteExpr, &cxt); + return TSDB_CODE_SUCCESS; +} + +static SLogicNode* pushLogicNode(SLogicNode* pRoot, SLogicNode* pNode) { + if (NULL == pRoot) { + return pNode; + } + if (NULL == pNode) { + return pRoot; + } + pRoot->pParent = pNode; + if (NULL == pNode->pChildren) { + pNode->pChildren = nodesMakeList(); + } + nodesListAppend(pNode->pChildren, (SNode*)pRoot); + return pNode; +} + +static SNodeList* createScanTargets(SNodeList* pCols) { + +} + +static SLogicNode* createScanLogicNode(SSelectStmt* pSelect, SRealTableNode* pRealTable) { + SScanLogicNode* pScan = (SScanLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_SCAN); + SNodeList* pCols = collectColumns(pSelect); + pScan->pScanCols = nodesCloneList(pCols); + // + rewriteExpr(pScan->pScanCols, pSelect); + pScan->node.pTargets = createScanTargets(pCols); + pScan->pMeta = pRealTable->pMeta; + return (SLogicNode*)pScan; +} + +static SLogicNode* createSubqueryLogicNode(SSelectStmt* pSelect, STempTableNode* pTable) { + return createQueryLogicNode(pTable->pSubquery); +} + +static SLogicNode* createLogicNodeByTable(SSelectStmt* pSelect, SNode* pTable) { + switch (nodeType(pTable)) { + case QUERY_NODE_REAL_TABLE: + return createScanLogicNode(pSelect, (SRealTableNode*)pTable); + case QUERY_NODE_TEMP_TABLE: + return createSubqueryLogicNode(pSelect, (STempTableNode*)pTable); + case QUERY_NODE_JOIN_TABLE: + default: + break; + } + return NULL; +} + +static SLogicNode* createFilterLogicNode(SNode* pWhere) { + if (NULL == pWhere) { + return NULL; + } + SFilterLogicNode* pFilter = (SFilterLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_FILTER); + pFilter->node.pConditions = nodesCloneNode(pWhere); + return (SLogicNode*)pFilter; +} + +static SLogicNode* createAggLogicNode(SSelectStmt* pSelect, SNodeList* pGroupByList, SNode* pHaving) { + SNodeList* pAggFuncs = collectAggFuncs(pSelect); + if (NULL == pAggFuncs && NULL == pGroupByList) { + return NULL; + } + SAggLogicNode* pAgg = (SAggLogicNode*)nodesMakeNode(QUERY_NODE_LOGIC_PLAN_AGG); + pAgg->pGroupKeys = nodesCloneList(pGroupByList); + pAgg->pAggFuncs = nodesCloneList(pAggFuncs); + pAgg->node.pConditions = nodesCloneNode(pHaving); + return (SLogicNode*)pAgg; +} + +static SLogicNode* createSelectLogicNode(SSelectStmt* pSelect) { + SLogicNode* pRoot = createLogicNodeByTable(pSelect, pSelect->pFromTable); + pRoot = pushLogicNode(pRoot, createFilterLogicNode(pSelect->pWhere)); + pRoot = pushLogicNode(pRoot, createAggLogicNode(pSelect, pSelect->pGroupByList, pSelect->pHaving)); + // pRoot = pushLogicNode(pRoot, createProjectLogicNode(pSelect, pSelect->pProjectionList)); + return pRoot; +} + +static SLogicNode* createQueryLogicNode(SNode* pStmt) { + switch (nodeType(pStmt)) { + case QUERY_NODE_SELECT_STMT: + return createSelectLogicNode((SSelectStmt*)pStmt); + default: + break; + } +} + +int32_t createLogicPlan(SNode* pNode, SLogicNode** pLogicNode) { + *pLogicNode = createQueryLogicNode(pNode); + return TSDB_CODE_SUCCESS; +} diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index 9f5e1b93e3..753ac75860 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -21,153 +21,117 @@ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wformat-truncation" -int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen) = {0}; +int32_t (*queryBuildMsg[TDMT_MAX])(void *input, char **msg, int32_t msgSize, int32_t *msgLen) = {0}; +int32_t (*queryProcessMsgRsp[TDMT_MAX])(void *output, char *msg, int32_t msgSize) = {0}; -int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char *msg, int32_t msgSize) = {0}; - -int32_t queryBuildTableMetaReqMsg(void* input, char **msg, int32_t msgSize, int32_t *msgLen) { +int32_t queryBuildTableMetaReqMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen) { + SBuildTableMetaInput *pInput = input; if (NULL == input || NULL == msg || NULL == msgLen) { return TSDB_CODE_TSC_INVALID_INPUT; } - SBuildTableMetaInput* bInput = (SBuildTableMetaInput *)input; - - int32_t estimateSize = sizeof(STableInfoReq); - if (NULL == *msg || msgSize < estimateSize) { - tfree(*msg); - *msg = rpcMallocCont(estimateSize); - if (NULL == *msg) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } + STableInfoReq infoReq = {0}; + infoReq.header.vgId = pInput->vgId; + if (pInput->dbFName) { + tstrncpy(infoReq.dbFName, pInput->dbFName, TSDB_DB_FNAME_LEN); } + tstrncpy(infoReq.tbName, pInput->tbName, TSDB_TABLE_NAME_LEN); - STableInfoReq *bMsg = (STableInfoReq *)*msg; + int32_t bufLen = tSerializeSTableInfoReq(NULL, 0, &infoReq); + void *pBuf = rpcMallocCont(bufLen); + tSerializeSTableInfoReq(pBuf, bufLen, &infoReq); - bMsg->header.vgId = htonl(bInput->vgId); + *msg = pBuf; + *msgLen = bufLen; - if (bInput->dbFName) { - tstrncpy(bMsg->dbFName, bInput->dbFName, tListLen(bMsg->dbFName)); - } - - tstrncpy(bMsg->tbName, bInput->tbName, tListLen(bMsg->tbName)); - - *msgLen = (int32_t)sizeof(*bMsg); return TSDB_CODE_SUCCESS; } -int32_t queryBuildUseDbMsg(void* input, char **msg, int32_t msgSize, int32_t *msgLen) { - if (NULL == input || NULL == msg || NULL == msgLen) { +int32_t queryBuildUseDbMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen) { + SBuildUseDBInput *pInput = input; + if (NULL == pInput || NULL == msg || NULL == msgLen) { return TSDB_CODE_TSC_INVALID_INPUT; } - SBuildUseDBInput* bInput = (SBuildUseDBInput *)input; + SUseDbReq usedbReq = {0}; + strncpy(usedbReq.db, pInput->db, sizeof(usedbReq.db)); + usedbReq.db[sizeof(usedbReq.db) - 1] = 0; + usedbReq.vgVersion = pInput->vgVersion; - int32_t estimateSize = sizeof(SUseDbReq); - if (NULL == *msg || msgSize < estimateSize) { - tfree(*msg); - *msg = rpcMallocCont(estimateSize); - if (NULL == *msg) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - } + int32_t bufLen = tSerializeSUseDbReq(NULL, 0, &usedbReq); + void *pBuf = rpcMallocCont(bufLen); + tSerializeSUseDbReq(pBuf, bufLen, &usedbReq); - SUseDbReq *bMsg = (SUseDbReq *)*msg; + *msg = pBuf; + *msgLen = bufLen; - strncpy(bMsg->db, bInput->db, sizeof(bMsg->db)); - bMsg->db[sizeof(bMsg->db) - 1] = 0; - - bMsg->vgVersion = bInput->vgVersion; - - *msgLen = (int32_t)sizeof(*bMsg); - - return TSDB_CODE_SUCCESS; + return TSDB_CODE_SUCCESS; } +int32_t queryProcessUseDBRsp(void *output, char *msg, int32_t msgSize) { + SUseDbOutput *pOut = output; + SUseDbRsp usedbRsp = {0}; + int32_t code = -1; -int32_t queryProcessUseDBRsp(void* output, char *msg, int32_t msgSize) { if (NULL == output || NULL == msg || msgSize <= 0) { - return TSDB_CODE_TSC_INVALID_INPUT; + code = TSDB_CODE_TSC_INVALID_INPUT; + goto PROCESS_USEDB_OVER; } - SUseDbRsp *pRsp = (SUseDbRsp *)msg; - SUseDbOutput *pOut = (SUseDbOutput *)output; - int32_t code = 0; - - if (msgSize <= sizeof(*pRsp)) { - qError("invalid use db rsp msg size, msgSize:%d", msgSize); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; - } - - pRsp->vgVersion = ntohl(pRsp->vgVersion); - pRsp->vgNum = ntohl(pRsp->vgNum); - pRsp->uid = be64toh(pRsp->uid); - - if (pRsp->vgNum < 0) { - qError("invalid db[%s] vgroup number[%d]", pRsp->db, pRsp->vgNum); - return TSDB_CODE_TSC_INVALID_VALUE; + if (tDeserializeSUseDbRsp(msg, msgSize, &usedbRsp) != 0) { + qError("invalid use db rsp msg, msgSize:%d", msgSize); + code = TSDB_CODE_INVALID_MSG; + goto PROCESS_USEDB_OVER; } - int32_t expectSize = pRsp->vgNum * sizeof(pRsp->vgroupInfo[0]) + sizeof(*pRsp); - if (msgSize != expectSize) { - qError("use db rsp size mis-match, msgSize:%d, expected:%d, vgnumber:%d", msgSize, expectSize, pRsp->vgNum); - return TSDB_CODE_TSC_VALUE_OUT_OF_RANGE; + if (usedbRsp.vgNum < 0) { + qError("invalid db[%s] vgroup number[%d]", usedbRsp.db, usedbRsp.vgNum); + code = TSDB_CODE_TSC_INVALID_VALUE; + goto PROCESS_USEDB_OVER; } + memcpy(pOut->db, usedbRsp.db, TSDB_DB_FNAME_LEN); + pOut->dbId = usedbRsp.uid; pOut->dbVgroup = calloc(1, sizeof(SDBVgInfo)); if (NULL == pOut->dbVgroup) { - qError("calloc %d failed", (int32_t)sizeof(SDBVgInfo)); - return TSDB_CODE_TSC_OUT_OF_MEMORY; + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto PROCESS_USEDB_OVER; } - pOut->dbId = pRsp->uid; - pOut->dbVgroup->vgVersion = pRsp->vgVersion; - pOut->dbVgroup->hashMethod = pRsp->hashMethod; - pOut->dbVgroup->vgHash = taosHashInit(pRsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); + pOut->dbVgroup->vgVersion = usedbRsp.vgVersion; + pOut->dbVgroup->hashMethod = usedbRsp.hashMethod; + pOut->dbVgroup->vgHash = + taosHashInit(usedbRsp.vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); if (NULL == pOut->dbVgroup->vgHash) { - qError("taosHashInit %d failed", pRsp->vgNum); tfree(pOut->dbVgroup); - return TSDB_CODE_TSC_OUT_OF_MEMORY; + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto PROCESS_USEDB_OVER; } - for (int32_t i = 0; i < pRsp->vgNum; ++i) { - pRsp->vgroupInfo[i].vgId = ntohl(pRsp->vgroupInfo[i].vgId); - pRsp->vgroupInfo[i].hashBegin = ntohl(pRsp->vgroupInfo[i].hashBegin); - pRsp->vgroupInfo[i].hashEnd = ntohl(pRsp->vgroupInfo[i].hashEnd); - - for (int32_t n = 0; n < pRsp->vgroupInfo[i].epset.numOfEps; ++n) { - pRsp->vgroupInfo[i].epset.eps[n].port = ntohs(pRsp->vgroupInfo[i].epset.eps[n].port); - } - - if (0 != taosHashPut(pOut->dbVgroup->vgHash, &pRsp->vgroupInfo[i].vgId, sizeof(pRsp->vgroupInfo[i].vgId), &pRsp->vgroupInfo[i], sizeof(pRsp->vgroupInfo[i]))) { - qError("taosHashPut failed"); - goto _return; + for (int32_t i = 0; i < usedbRsp.vgNum; ++i) { + SVgroupInfo *pVgInfo = taosArrayGet(usedbRsp.pVgroupInfos, i); + if (0 != taosHashPut(pOut->dbVgroup->vgHash, &pVgInfo->vgId, sizeof(int32_t), pVgInfo, sizeof(SVgroupInfo))) { + code = TSDB_CODE_TSC_OUT_OF_MEMORY; + goto PROCESS_USEDB_OVER; } } - memcpy(pOut->db, pRsp->db, sizeof(pOut->db)); + code = 0; - return code; - -_return: - - if (pOut) { - taosHashCleanup(pOut->dbVgroup->vgHash); - tfree(pOut->dbVgroup); +PROCESS_USEDB_OVER: + if (code != 0) { + if (pOut) { + if (pOut->dbVgroup) taosHashCleanup(pOut->dbVgroup->vgHash); + tfree(pOut->dbVgroup); + } + qError("failed to process usedb rsp since %s", terrstr()); } - + + tFreeSUsedbRsp(&usedbRsp); return code; } -static int32_t queryConvertTableMetaMsg(STableMetaRsp* pMetaMsg) { - pMetaMsg->dbId = be64toh(pMetaMsg->dbId); - pMetaMsg->numOfTags = ntohl(pMetaMsg->numOfTags); - pMetaMsg->numOfColumns = ntohl(pMetaMsg->numOfColumns); - pMetaMsg->sversion = ntohl(pMetaMsg->sversion); - pMetaMsg->tversion = ntohl(pMetaMsg->tversion); - pMetaMsg->tuid = be64toh(pMetaMsg->tuid); - pMetaMsg->suid = be64toh(pMetaMsg->suid); - pMetaMsg->vgId = ntohl(pMetaMsg->vgId); - +static int32_t queryConvertTableMetaMsg(STableMetaRsp *pMetaMsg) { if (pMetaMsg->numOfTags < 0 || pMetaMsg->numOfTags > TSDB_MAX_TAGS) { qError("invalid numOfTags[%d] in table meta rsp msg", pMetaMsg->numOfTags); return TSDB_CODE_TSC_INVALID_VALUE; @@ -178,7 +142,8 @@ static int32_t queryConvertTableMetaMsg(STableMetaRsp* pMetaMsg) { return TSDB_CODE_TSC_INVALID_VALUE; } - if (pMetaMsg->tableType != TSDB_SUPER_TABLE && pMetaMsg->tableType != TSDB_CHILD_TABLE && pMetaMsg->tableType != TSDB_NORMAL_TABLE) { + if (pMetaMsg->tableType != TSDB_SUPER_TABLE && pMetaMsg->tableType != TSDB_CHILD_TABLE && + pMetaMsg->tableType != TSDB_NORMAL_TABLE) { qError("invalid tableType[%d] in table meta rsp msg", pMetaMsg->tableType); return TSDB_CODE_TSC_INVALID_VALUE; } @@ -192,30 +157,20 @@ static int32_t queryConvertTableMetaMsg(STableMetaRsp* pMetaMsg) { qError("invalid tversion[%d] in table meta rsp msg", pMetaMsg->tversion); return TSDB_CODE_TSC_INVALID_VALUE; } - - SSchema* pSchema = pMetaMsg->pSchema; - int32_t numOfTotalCols = pMetaMsg->numOfColumns + pMetaMsg->numOfTags; - for (int i = 0; i < numOfTotalCols; ++i) { - pSchema->bytes = ntohl(pSchema->bytes); - pSchema->colId = ntohl(pSchema->colId); - - pSchema++; - } - - if (pMetaMsg->pSchema[0].colId != PRIMARYKEY_TIMESTAMP_COL_ID) { - qError("invalid colId[%d] for the first column in table meta rsp msg", pMetaMsg->pSchema[0].colId); + if (pMetaMsg->pSchemas[0].colId != PRIMARYKEY_TIMESTAMP_COL_ID) { + qError("invalid colId[%d] for the first column in table meta rsp msg", pMetaMsg->pSchemas[0].colId); return TSDB_CODE_TSC_INVALID_VALUE; } return TSDB_CODE_SUCCESS; } -int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STableMeta **pMeta) { +int32_t queryCreateTableMetaFromMsg(STableMetaRsp *msg, bool isSuperTable, STableMeta **pMeta) { int32_t total = msg->numOfColumns + msg->numOfTags; int32_t metaSize = sizeof(STableMeta) + sizeof(SSchema) * total; - - STableMeta* pTableMeta = calloc(1, metaSize); + + STableMeta *pTableMeta = calloc(1, metaSize); if (NULL == pTableMeta) { qError("calloc size[%d] failed", metaSize); return TSDB_CODE_TSC_OUT_OF_MEMORY; @@ -223,7 +178,7 @@ int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STabl pTableMeta->vgId = isSuperTable ? 0 : msg->vgId; pTableMeta->tableType = isSuperTable ? TSDB_SUPER_TABLE : msg->tableType; - pTableMeta->uid = isSuperTable ? msg->suid : msg->tuid; + pTableMeta->uid = isSuperTable ? msg->suid : msg->tuid; pTableMeta->suid = msg->suid; pTableMeta->sversion = msg->sversion; pTableMeta->tversion = msg->tversion; @@ -232,60 +187,71 @@ int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STabl pTableMeta->tableInfo.precision = msg->precision; pTableMeta->tableInfo.numOfColumns = msg->numOfColumns; - memcpy(pTableMeta->schema, msg->pSchema, sizeof(SSchema) * total); + memcpy(pTableMeta->schema, msg->pSchemas, sizeof(SSchema) * total); - for(int32_t i = 0; i < msg->numOfColumns; ++i) { + for (int32_t i = 0; i < msg->numOfColumns; ++i) { pTableMeta->tableInfo.rowSize += pTableMeta->schema[i].bytes; } *pMeta = pTableMeta; - return TSDB_CODE_SUCCESS; } +int32_t queryProcessTableMetaRsp(void *output, char *msg, int32_t msgSize) { + int32_t code = -1; + STableMetaRsp metaRsp = {0}; -int32_t queryProcessTableMetaRsp(void* output, char *msg, int32_t msgSize) { - STableMetaRsp *pMetaMsg = (STableMetaRsp *)msg; - int32_t code = queryConvertTableMetaMsg(pMetaMsg); + if (NULL == output || NULL == msg || msgSize <= 0) { + code = TSDB_CODE_TSC_INVALID_INPUT; + goto PROCESS_META_OVER; + } + + if (tDeserializeSTableMetaRsp(msg, msgSize, &metaRsp) != 0) { + code = TSDB_CODE_INVALID_MSG; + goto PROCESS_META_OVER; + } + + code = queryConvertTableMetaMsg(&metaRsp); if (code != TSDB_CODE_SUCCESS) { - return code; + goto PROCESS_META_OVER; } - STableMetaOutput *pOut = (STableMetaOutput *)output; - - if (!tIsValidSchema(pMetaMsg->pSchema, pMetaMsg->numOfColumns, pMetaMsg->numOfTags)) { - qError("validate table meta schema in rsp msg failed"); - return TSDB_CODE_TSC_INVALID_VALUE; + if (!tIsValidSchema(metaRsp.pSchemas, metaRsp.numOfColumns, metaRsp.numOfTags)) { + code = TSDB_CODE_TSC_INVALID_VALUE; + goto PROCESS_META_OVER; } - strcpy(pOut->dbFName, pMetaMsg->dbFName); - - pOut->dbId = pMetaMsg->dbId; + STableMetaOutput *pOut = output; + strcpy(pOut->dbFName, metaRsp.dbFName); + pOut->dbId = metaRsp.dbId; - if (pMetaMsg->tableType == TSDB_CHILD_TABLE) { + if (metaRsp.tableType == TSDB_CHILD_TABLE) { SET_META_TYPE_BOTH_TABLE(pOut->metaType); - strcpy(pOut->ctbName, pMetaMsg->tbName); - strcpy(pOut->tbName, pMetaMsg->stbName); - - pOut->ctbMeta.vgId = pMetaMsg->vgId; - pOut->ctbMeta.tableType = pMetaMsg->tableType; - pOut->ctbMeta.uid = pMetaMsg->tuid; - pOut->ctbMeta.suid = pMetaMsg->suid; + strcpy(pOut->ctbName, metaRsp.tbName); + strcpy(pOut->tbName, metaRsp.stbName); - code = queryCreateTableMetaFromMsg(pMetaMsg, true, &pOut->tbMeta); + pOut->ctbMeta.vgId = metaRsp.vgId; + pOut->ctbMeta.tableType = metaRsp.tableType; + pOut->ctbMeta.uid = metaRsp.tuid; + pOut->ctbMeta.suid = metaRsp.suid; + + code = queryCreateTableMetaFromMsg(&metaRsp, true, &pOut->tbMeta); } else { SET_META_TYPE_TABLE(pOut->metaType); - - strcpy(pOut->tbName, pMetaMsg->tbName); - - code = queryCreateTableMetaFromMsg(pMetaMsg, (pMetaMsg->tableType == TSDB_SUPER_TABLE), &pOut->tbMeta); + strcpy(pOut->tbName, metaRsp.tbName); + code = queryCreateTableMetaFromMsg(&metaRsp, (metaRsp.tableType == TSDB_SUPER_TABLE), &pOut->tbMeta); } - + +PROCESS_META_OVER: + if (code != 0) { + qError("failed to process table meta rsp since %s", terrstr()); + } + + tFreeSTableMetaRsp(&metaRsp); return code; } - void initQueryModuleMsgHandle() { queryBuildMsg[TMSG_INDEX(TDMT_VND_TABLE_META)] = queryBuildTableMetaReqMsg; queryBuildMsg[TMSG_INDEX(TDMT_MND_STB_META)] = queryBuildTableMetaReqMsg; diff --git a/source/libs/qworker/src/qworkerMsg.c b/source/libs/qworker/src/qworkerMsg.c index 368a590918..a1b431fd7d 100644 --- a/source/libs/qworker/src/qworkerMsg.c +++ b/source/libs/qworker/src/qworkerMsg.c @@ -172,45 +172,54 @@ int32_t qwBuildAndSendDropRsp(void *connection, int32_t code) { } int32_t qwBuildAndSendShowRsp(SRpcMsg *pMsg, int32_t code) { - int32_t numOfCols = 6; - int32_t msgSize = sizeof(SVShowTablesRsp) + sizeof(SSchema) * numOfCols; + int32_t numOfCols = 6; + SVShowTablesRsp showRsp = {0}; - SVShowTablesRsp *pRsp = (SVShowTablesRsp *)rpcMallocCont(msgSize); + // showRsp.showId = 1; + showRsp.tableMeta.pSchemas = calloc(numOfCols, sizeof(SSchema)); + if (showRsp.tableMeta.pSchemas == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } int32_t cols = 0; - SSchema *pSchema = pRsp->metaInfo.pSchema; + SSchema *pSchema = showRsp.tableMeta.pSchemas; const SSchema *s = tGetTbnameColumnSchema(); - *pSchema = createSchema(s->type, htonl(s->bytes), htonl(++cols), "name"); + *pSchema = createSchema(s->type, s->bytes, ++cols, "name"); pSchema++; int32_t type = TSDB_DATA_TYPE_TIMESTAMP; - *pSchema = createSchema(type, htonl(tDataTypes[type].bytes), htonl(++cols), "created"); + *pSchema = createSchema(type, tDataTypes[type].bytes, ++cols, "created"); pSchema++; type = TSDB_DATA_TYPE_SMALLINT; - *pSchema = createSchema(type, htonl(tDataTypes[type].bytes), htonl(++cols), "columns"); + *pSchema = createSchema(type, tDataTypes[type].bytes, ++cols, "columns"); pSchema++; - *pSchema = createSchema(s->type, htonl(s->bytes), htonl(++cols), "stable"); + *pSchema = createSchema(s->type, s->bytes, ++cols, "stable"); pSchema++; type = TSDB_DATA_TYPE_BIGINT; - *pSchema = createSchema(type, htonl(tDataTypes[type].bytes), htonl(++cols), "uid"); + *pSchema = createSchema(type, tDataTypes[type].bytes, ++cols, "uid"); pSchema++; type = TSDB_DATA_TYPE_INT; - *pSchema = createSchema(type, htonl(tDataTypes[type].bytes), htonl(++cols), "vgId"); + *pSchema = createSchema(type, tDataTypes[type].bytes, ++cols, "vgId"); assert(cols == numOfCols); - pRsp->metaInfo.numOfColumns = htonl(cols); + showRsp.tableMeta.numOfColumns = cols; + + int32_t bufLen = tSerializeSShowRsp(NULL, 0, &showRsp); + void *pBuf = rpcMallocCont(bufLen); + tSerializeSShowRsp(pBuf, bufLen, &showRsp); SRpcMsg rpcMsg = { - .handle = pMsg->handle, + .handle = pMsg->handle, .ahandle = pMsg->ahandle, - .pCont = pRsp, - .contLen = msgSize, - .code = code, + .pCont = pBuf, + .contLen = bufLen, + .code = code, }; rpcSendResponse(&rpcMsg); diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 1b6d871274..c14ae40b59 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -823,7 +823,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch SCH_ERR_RET(schProcessOnTaskFailure(pJob, pTask, rspCode)); } - SShellSubmitRsp *rsp = (SShellSubmitRsp *)msg; + SSubmitRsp *rsp = (SSubmitRsp *)msg; if (rsp) { pJob->resNumOfRows += rsp->affectedRows; } diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 2c75683ecb..6bfff0197c 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -273,7 +273,7 @@ void *schtSendRsp(void *param) { while (pIter) { SSchTask *task = *(SSchTask **)pIter; - SShellSubmitRsp rsp = {0}; + SSubmitRsp rsp = {0}; rsp.affectedRows = 10; schHandleResponseMsg(job, task, TDMT_VND_SUBMIT_RSP, (char *)&rsp, sizeof(rsp), 0); diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 6f8da57ee7..d5a8cf5f84 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -140,6 +140,7 @@ typedef struct { SRpcMsg* pRsp; // for synchronous API tsem_t* pSem; // for synchronous API + int hThrdIdx; char* ip; uint32_t port; // SEpSet* pSet; // for synchronous API diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 00d9174e76..9cab863ed7 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -17,6 +17,7 @@ #include "transComm.h" +#define CONN_HOST_THREAD_INDEX(conn) (conn ? ((SCliConn*)conn)->hThrdIdx : -1) #define CONN_PERSIST_TIME(para) (para * 1000 * 10) typedef struct SCliConn { @@ -28,7 +29,8 @@ typedef struct SCliConn { void* data; queue conn; uint64_t expireTime; - int8_t notifyCount; // timers already notify to client + int8_t ctnRdCnt; // continue read count + int hThrdIdx; SRpcPush* push; int persist; // @@ -131,16 +133,21 @@ static void clientHandleResp(SCliConn* conn) { rpcMsg.msgType = pHead->msgType; rpcMsg.ahandle = pCtx->ahandle; + if (rpcMsg.msgType == TDMT_VND_QUERY_RSP || rpcMsg.msgType == TDMT_VND_FETCH_RSP) { + rpcMsg.handle = conn; + conn->persist = 1; + } + tDebug("client conn %p %s received from %s:%d, local info: %s:%d", conn, TMSG_INFO(pHead->msgType), inet_ntoa(conn->addr.sin_addr), ntohs(conn->addr.sin_port), inet_ntoa(conn->locaddr.sin_addr), ntohs(conn->locaddr.sin_port)); - if (conn->push != NULL && conn->notifyCount != 0) { + if (conn->push != NULL && conn->ctnRdCnt != 0) { (*conn->push->callback)(conn->push->arg, &rpcMsg); conn->push = NULL; } else { if (pCtx->pSem == NULL) { - tTrace("client conn%p handle resp", conn); + tTrace("client conn %p handle resp", conn); (pRpc->cfp)(pRpc->parent, &rpcMsg, NULL); } else { tTrace("client conn(sync) %p handle resp", conn); @@ -148,7 +155,7 @@ static void clientHandleResp(SCliConn* conn) { tsem_post(pCtx->pSem); } } - conn->notifyCount += 1; + conn->ctnRdCnt += 1; conn->secured = pHead->secured; // buf's mem alread translated to rpcMsg.pCont @@ -157,13 +164,15 @@ static void clientHandleResp(SCliConn* conn) { uv_read_start((uv_stream_t*)conn->stream, clientAllocBufferCb, clientReadCb); SCliThrdObj* pThrd = conn->hostThrd; + // user owns conn->persist = 1 - if (conn->push == NULL) { + if (conn->push == NULL || conn->persist == 0) { addConnToPool(pThrd->pool, pCtx->ip, pCtx->port, conn); destroyCmsg(conn->data); conn->data = NULL; } + // start thread's timer of conn pool if not active if (!uv_is_active((uv_handle_t*)pThrd->timer) && pRpc->idleTime > 0) { uv_timer_start((uv_timer_t*)pThrd->timer, clientTimeoutCb, CONN_PERSIST_TIME(pRpc->idleTime) / 2, 0); @@ -178,13 +187,18 @@ static void clientHandleExcept(SCliConn* pConn) { tTrace("client conn %p start to destroy", pConn); SCliMsg* pMsg = pConn->data; + tmsg_t msgType = TDMT_MND_CONNECT; + if (pMsg != NULL) { + msgType = pMsg->msg.msgType; + } STransConnCtx* pCtx = pMsg->ctx; SRpcMsg rpcMsg = {0}; rpcMsg.ahandle = pCtx->ahandle; rpcMsg.code = TSDB_CODE_RPC_NETWORK_UNAVAIL; + rpcMsg.msgType = msgType + 1; - if (pConn->push != NULL && pConn->notifyCount != 0) { + if (pConn->push != NULL && pConn->ctnRdCnt != 0) { (*pConn->push->callback)(pConn->push->arg, &rpcMsg); pConn->push = NULL; } else { @@ -205,7 +219,7 @@ static void clientHandleExcept(SCliConn* pConn) { } // transDestroyConnCtx(pCtx); clientConnDestroy(pConn, true); - pConn->notifyCount += 1; + pConn->ctnRdCnt += 1; } static void clientTimeoutCb(uv_timer_t* handle) { @@ -287,7 +301,7 @@ static void addConnToPool(void* pool, char* ip, uint32_t port, SCliConn* conn) { conn->expireTime = taosGetTimestampMs() + CONN_PERSIST_TIME(pRpc->idleTime); SConnList* plist = taosHashGet((SHashObj*)pool, key, strlen(key)); - conn->notifyCount = 0; + conn->ctnRdCnt = 0; // list already create before assert(plist != NULL); QUEUE_PUSH(&plist->conn, &conn->conn); @@ -418,6 +432,8 @@ static void clientWrite(SCliConn* pConn) { pHead->msgType = pMsg->msgType; pHead->msgLen = (int32_t)htonl((uint32_t)msgLen); + // if (pHead->msgType == TDMT_VND_QUERY || pHead->msgType == TDMT_VND_) + uv_buf_t wb = uv_buf_init((char*)pHead, msgLen); tDebug("client conn %p %s is send to %s:%d, local info %s:%d", pConn, TMSG_INFO(pHead->msgType), inet_ntoa(pConn->addr.sin_addr), ntohs(pConn->addr.sin_port), inet_ntoa(pConn->locaddr.sin_addr), @@ -457,14 +473,25 @@ static void clientHandleQuit(SCliMsg* pMsg, SCliThrdObj* pThrd) { static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { uint64_t et = taosGetTimestampUs(); uint64_t el = et - pMsg->st; - tTrace("client msg tran time cost: %" PRIu64 "", el); + tTrace("client msg tran time cost: %" PRIu64 "us", el); et = taosGetTimestampUs(); STransConnCtx* pCtx = pMsg->ctx; - SCliConn* conn = getConnFromPool(pThrd->pool, pCtx->ip, pCtx->port); + + SCliConn* conn = NULL; + if (pMsg->msg.handle == NULL) { + conn = getConnFromPool(pThrd->pool, pCtx->ip, pCtx->port); + if (conn != NULL) { + tTrace("client conn %p get from conn pool", conn); + } + } else { + conn = (SCliConn*)(pMsg->msg.handle); + if (conn != NULL) { + tTrace("client conn %p reused", conn); + } + } + if (conn != NULL) { - // impl later - tTrace("client get conn %p from pool", conn); conn->data = pMsg; conn->writeReq->data = conn; transDestroyBuffer(&conn->readBuf); @@ -475,10 +502,8 @@ static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { } clientWrite(conn); - conn->push = pMsg->msg.push; - } else { - SCliConn* conn = calloc(1, sizeof(SCliConn)); + conn = calloc(1, sizeof(SCliConn)); conn->ref++; // read/write stream handle conn->stream = (uv_stream_t*)malloc(sizeof(uv_tcp_t)); @@ -495,13 +520,18 @@ static void clientHandleReq(SCliMsg* pMsg, SCliThrdObj* pThrd) { conn->data = pMsg; conn->hostThrd = pThrd; - conn->push = pMsg->msg.push; + // conn->push = pMsg->msg.push; + // conn->ctnRdCnt = 0; struct sockaddr_in addr; uv_ip4_addr(pMsg->ctx->ip, pMsg->ctx->port, &addr); // handle error in callback if fail to connect uv_tcp_connect(&conn->connReq, (uv_tcp_t*)(conn->stream), (const struct sockaddr*)&addr, clientConnCb); } + + conn->push = pMsg->msg.push; + conn->ctnRdCnt = 0; + conn->hThrdIdx = pCtx->hThrdIdx; } static void clientAsyncCb(uv_async_t* handle) { SAsyncItem* item = handle->data; @@ -622,15 +652,9 @@ static void clientSendQuit(SCliThrdObj* thrd) { SCliMsg* msg = calloc(1, sizeof(SCliMsg)); msg->ctx = NULL; // - // pthread_mutex_lock(&thrd->msgMtx); - // QUEUE_PUSH(&thrd->msg, &msg->q); - // pthread_mutex_unlock(&thrd->msgMtx); - transSendAsync(thrd->asyncPool, &msg->q); - // uv_async_send(thrd->cliAsync); } void taosCloseClient(void* arg) { - // impl later SClientObj* cli = arg; for (int i = 0; i < cli->numOfThreads; i++) { clientSendQuit(cli->pThreadObj[i]); @@ -639,6 +663,13 @@ void taosCloseClient(void* arg) { free(cli->pThreadObj); free(cli); } +static int clientRBChoseIdx(SRpcInfo* pRpc) { + int64_t index = pRpc->index; + if (pRpc->index++ >= pRpc->numOfThreads) { + pRpc->index = 0; + } + return index % pRpc->numOfThreads; +} void rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid) { // impl later char* ip = (char*)(pEpSet->eps[pEpSet->inUse].fqdn); @@ -646,76 +677,63 @@ void rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* SRpcInfo* pRpc = (SRpcInfo*)shandle; + int index = CONN_HOST_THREAD_INDEX(pMsg->handle); + if (index == -1) { + index = clientRBChoseIdx(pRpc); + } int32_t flen = 0; if (transCompressMsg(pMsg->pCont, pMsg->contLen, &flen)) { // imp later } - STransConnCtx* pCtx = calloc(1, sizeof(STransConnCtx)); - pCtx->pTransInst = (SRpcInfo*)shandle; pCtx->ahandle = pMsg->ahandle; pCtx->msgType = pMsg->msgType; pCtx->ip = strdup(ip); pCtx->port = port; + pCtx->hThrdIdx = index; assert(pRpc->connType == TAOS_CONN_CLIENT); // atomic or not - int64_t index = pRpc->index; - if (pRpc->index++ >= pRpc->numOfThreads) { - pRpc->index = 0; - } + SCliMsg* cliMsg = malloc(sizeof(SCliMsg)); cliMsg->ctx = pCtx; cliMsg->msg = *pMsg; cliMsg->st = taosGetTimestampUs(); - SCliThrdObj* thrd = ((SClientObj*)pRpc->tcphandle)->pThreadObj[index % pRpc->numOfThreads]; - - // pthread_mutex_lock(&thrd->msgMtx); - // QUEUE_PUSH(&thrd->msg, &cliMsg->q); - // pthread_mutex_unlock(&thrd->msgMtx); - - // int start = taosGetTimestampUs(); + SCliThrdObj* thrd = ((SClientObj*)pRpc->tcphandle)->pThreadObj[index]; transSendAsync(thrd->asyncPool, &(cliMsg->q)); - // uv_async_send(thrd->cliAsync); - // int end = taosGetTimestampUs() - start; - // tError("client sent to rpc, time cost: %d", (int)end); } + void rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pReq, SRpcMsg* pRsp) { char* ip = (char*)(pEpSet->eps[pEpSet->inUse].fqdn); uint32_t port = pEpSet->eps[pEpSet->inUse].port; SRpcInfo* pRpc = (SRpcInfo*)shandle; + int index = CONN_HOST_THREAD_INDEX(pReq->handle); + if (index == -1) { + index = clientRBChoseIdx(pRpc); + } + STransConnCtx* pCtx = calloc(1, sizeof(STransConnCtx)); pCtx->pTransInst = (SRpcInfo*)shandle; pCtx->ahandle = pReq->ahandle; pCtx->msgType = pReq->msgType; pCtx->ip = strdup(ip); pCtx->port = port; + pCtx->hThrdIdx = index; pCtx->pSem = calloc(1, sizeof(tsem_t)); pCtx->pRsp = pRsp; tsem_init(pCtx->pSem, 0, 0); - int64_t index = pRpc->index; - if (pRpc->index++ >= pRpc->numOfThreads) { - pRpc->index = 0; - } SCliMsg* cliMsg = malloc(sizeof(SCliMsg)); cliMsg->ctx = pCtx; cliMsg->msg = *pReq; cliMsg->st = taosGetTimestampUs(); - SCliThrdObj* thrd = ((SClientObj*)pRpc->tcphandle)->pThreadObj[index % pRpc->numOfThreads]; - - // pthread_mutex_lock(&thrd->msgMtx); - // QUEUE_PUSH(&thrd->msg, &cliMsg->q); - // pthread_mutex_unlock(&thrd->msgMtx); - - // int start = taosGetTimestampUs(); + SCliThrdObj* thrd = ((SClientObj*)pRpc->tcphandle)->pThreadObj[index]; transSendAsync(thrd->asyncPool, &(cliMsg->q)); - tsem_t* pSem = pCtx->pSem; tsem_wait(pSem); tsem_destroy(pSem); diff --git a/source/libs/transport/test/rclient.c b/source/libs/transport/test/rclient.c index 4e29c02508..cc6a63d3cd 100644 --- a/source/libs/transport/test/rclient.c +++ b/source/libs/transport/test/rclient.c @@ -96,7 +96,7 @@ static void *sendRequest(void *param) { int main(int argc, char *argv[]) { SRpcInit rpcInit; - SEpSet epSet; + SEpSet epSet = {0}; int msgSize = 128; int numOfReqs = 0; int appThreads = 1; diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 4cb1e392b9..70608e117e 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -77,7 +77,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_PTR, "Invalid pointer") TAOS_DEFINE_ERROR(TSDB_CODE_MEMORY_CORRUPTED, "Memory corrupted") TAOS_DEFINE_ERROR(TSDB_CODE_FILE_CORRUPTED, "Data file corrupted") TAOS_DEFINE_ERROR(TSDB_CODE_CHECKSUM_ERROR, "Checksum error") -TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_MSG, "Invalid config message") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_MSG, "Invalid message") TAOS_DEFINE_ERROR(TSDB_CODE_MSG_NOT_PROCESSED, "Message not processed") TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_PARA, "Invalid parameters") TAOS_DEFINE_ERROR(TSDB_CODE_REPEAT_INIT, "Repeat initialization") diff --git a/source/util/src/tlosertree.c b/source/util/src/tlosertree.c index 6155ba4c1a..80bbac2c78 100644 --- a/source/util/src/tlosertree.c +++ b/source/util/src/tlosertree.c @@ -14,82 +14,85 @@ */ #include "os.h" -#include "tlosertree.h" #include "ulog.h" +#include "tlosertree.h" +#include "taoserror.h" -// set initial value for loser tree -void tLoserTreeInit(SLoserTreeInfo* pTree) { - assert((pTree->totalEntries & 0x01) == 0 && (pTree->numOfEntries << 1 == pTree->totalEntries)); - for (int32_t i = 0; i < pTree->totalEntries; ++i) { - if (i < pTree->numOfEntries) { + +// Set the initial value of the multiway merge tree. +static void tMergeTreeInit(SMultiwayMergeTreeInfo* pTree) { + assert((pTree->totalSources & 0x01) == 0 && (pTree->numOfSources << 1 == pTree->totalSources)); + + for (int32_t i = 0; i < pTree->totalSources; ++i) { + if (i < pTree->numOfSources) { pTree->pNode[i].index = -1; } else { - pTree->pNode[i].index = i - pTree->numOfEntries; + pTree->pNode[i].index = i - pTree->numOfSources; } } } -/* - * display whole loser tree on screen for debug purpose only. - */ -void tLoserTreeDisplay(SLoserTreeInfo* pTree) { - printf("the value of loser tree:\t"); - for (int32_t i = 0; i < pTree->totalEntries; ++i) printf("%d\t", pTree->pNode[i].index); - printf("\n"); -} +int32_t tMergeTreeCreate(SMultiwayMergeTreeInfo** pTree, uint32_t numOfSources, void* param, __merge_compare_fn_t compareFn) { + int32_t totalEntries = numOfSources << 1u; -uint32_t tLoserTreeCreate(SLoserTreeInfo** pTree, int32_t numOfEntries, void* param, __merge_compare_fn_t compareFn) { - int32_t totalEntries = numOfEntries << 1; - - *pTree = (SLoserTreeInfo*)calloc(1, sizeof(SLoserTreeInfo) + sizeof(SLoserTreeNode) * totalEntries); - if ((*pTree) == NULL) { + SMultiwayMergeTreeInfo* pTreeInfo = (SMultiwayMergeTreeInfo*)calloc(1, sizeof(SMultiwayMergeTreeInfo) + sizeof(STreeNode) * totalEntries); + if (pTreeInfo == NULL) { uError("allocate memory for loser-tree failed. reason:%s", strerror(errno)); - return -1; + return TAOS_SYSTEM_ERROR(errno); } - (*pTree)->pNode = (SLoserTreeNode*)(((char*)(*pTree)) + sizeof(SLoserTreeInfo)); + pTreeInfo->pNode = (STreeNode*)(((char*)pTreeInfo) + sizeof(SMultiwayMergeTreeInfo)); - (*pTree)->numOfEntries = numOfEntries; - (*pTree)->totalEntries = totalEntries; - (*pTree)->param = param; - (*pTree)->comparFn = compareFn; + pTreeInfo->numOfSources = numOfSources; + pTreeInfo->totalSources = totalEntries; + pTreeInfo->param = param; + pTreeInfo->comparFn = compareFn; // set initial value for loser tree - tLoserTreeInit(*pTree); + tMergeTreeInit(pTreeInfo); #ifdef _DEBUG_VIEW printf("the initial value of loser tree:\n"); - tLoserTreeDisplay(*pTree); + tLoserTreeDisplaypTreeInfo; #endif - for (int32_t i = totalEntries - 1; i >= numOfEntries; i--) { - tLoserTreeAdjust(*pTree, i); + for (int32_t i = totalEntries - 1; i >= numOfSources; i--) { + tMergeTreeAdjust(pTreeInfo, i); } #if defined(_DEBUG_VIEW) printf("after adjust:\n"); - tLoserTreeDisplay(*pTree); + tLoserTreeDisplaypTreeInfo; printf("initialize local reducer completed!\n"); #endif + *pTree = pTreeInfo; return 0; } -void tLoserTreeAdjust(SLoserTreeInfo* pTree, int32_t idx) { - assert(idx <= pTree->totalEntries - 1 && idx >= pTree->numOfEntries && pTree->totalEntries >= 2); +void tMergeTreeDestroy(SMultiwayMergeTreeInfo* pTree) { + if (pTree == NULL) { + return; + } - if (pTree->totalEntries == 2) { + tfree(pTree); +} + +void tMergeTreeAdjust(SMultiwayMergeTreeInfo* pTree, int32_t idx) { + assert(idx <= pTree->totalSources - 1 && idx >= pTree->numOfSources && pTree->totalSources >= 2); + + if (pTree->totalSources == 2) { pTree->pNode[0].index = 0; pTree->pNode[1].index = 0; return; } int32_t parentId = idx >> 1; - SLoserTreeNode kLeaf = pTree->pNode[idx]; + STreeNode kLeaf = pTree->pNode[idx]; while (parentId > 0) { - SLoserTreeNode* pCur = &pTree->pNode[parentId]; + STreeNode* pCur = &pTree->pNode[parentId]; if (pCur->index == -1) { pTree->pNode[parentId] = kLeaf; return; @@ -97,7 +100,7 @@ void tLoserTreeAdjust(SLoserTreeInfo* pTree, int32_t idx) { int32_t ret = pTree->comparFn(pCur, &kLeaf, pTree->param); if (ret < 0) { - SLoserTreeNode t = pTree->pNode[parentId]; + STreeNode t = pTree->pNode[parentId]; pTree->pNode[parentId] = kLeaf; kLeaf = t; } @@ -111,11 +114,23 @@ void tLoserTreeAdjust(SLoserTreeInfo* pTree, int32_t idx) { } } -void tLoserTreeRebuild(SLoserTreeInfo* pTree) { - assert((pTree->totalEntries & 0x1) == 0); +void tMergeTreeRebuild(SMultiwayMergeTreeInfo* pTree) { + assert((pTree->totalSources & 0x1) == 0); - tLoserTreeInit(pTree); - for (int32_t i = pTree->totalEntries - 1; i >= pTree->numOfEntries; i--) { - tLoserTreeAdjust(pTree, i); + tMergeTreeInit(pTree); + for (int32_t i = pTree->totalSources - 1; i >= pTree->numOfSources; i--) { + tMergeTreeAdjust(pTree, i); } } + +/* + * display whole loser tree on screen for debug purpose only. + */ +void tMergeTreePrint(const SMultiwayMergeTreeInfo* pTree) { + printf("the value of loser tree:\t"); + for (int32_t i = 0; i < pTree->totalSources; ++i) { + printf("%d\t", pTree->pNode[i].index); + } + + printf("\n"); +} diff --git a/source/util/src/tpagedbuf.c b/source/util/src/tpagedbuf.c new file mode 100644 index 0000000000..0e8d85492c --- /dev/null +++ b/source/util/src/tpagedbuf.c @@ -0,0 +1,597 @@ +#include "os.h" +#include "ulog.h" +#include "tpagedbuf.h" +#include "taoserror.h" +#include "tcompression.h" +#include "thash.h" + +#define GET_DATA_PAYLOAD(_p) ((char *)(_p)->pData + POINTER_BYTES) +#define NO_IN_MEM_AVAILABLE_PAGES(_b) (listNEles((_b)->lruList) >= (_b)->inMemPages) + +typedef struct SFreeListItem { + int32_t offset; + int32_t len; +} SFreeListItem; + +typedef struct SPageDiskInfo { + int64_t offset; + int32_t length; +} SPageDiskInfo; + +typedef struct SPageInfo { + SListNode* pn; // point to list node + void* pData; + int64_t offset; + int32_t pageId; + int32_t length:30; + bool used:1; // set current page is in used + bool dirty:1; // set current buffer page is dirty or not +} SPageInfo; + +typedef struct SDiskbasedBuf { + int32_t numOfPages; + int64_t totalBufSize; + uint64_t fileSize; // disk file size + FILE* file; + int32_t allocateId; // allocated page id + char* path; // file path + int32_t pageSize; // current used page size + int32_t inMemPages; // numOfPages that are allocated in memory + SHashObj* groupSet; // id hash table + SHashObj* all; + SList* lruList; + void* emptyDummyIdList; // dummy id list + void* assistBuf; // assistant buffer for compress/decompress data + SArray* pFree; // free area in file + bool comp; // compressed before flushed to disk + uint64_t nextPos; // next page flush position + + uint64_t qId; // for debug purpose + bool printStatis; // Print statistics info when closing this buffer. + SDiskbasedBufStatis statis; +} SDiskbasedBuf; + +static void printStatisData(const SDiskbasedBuf* pBuf); + + int32_t createDiskbasedBuffer(SDiskbasedBuf** pBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId, const char* dir) { + *pBuf = calloc(1, sizeof(SDiskbasedBuf)); + + SDiskbasedBuf* pResBuf = *pBuf; + if (pResBuf == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pResBuf->pageSize = pagesize; + pResBuf->numOfPages = 0; // all pages are in buffer in the first place + pResBuf->totalBufSize = 0; + pResBuf->inMemPages = inMemBufSize/pagesize; // maximum allowed pages, it is a soft limit. + pResBuf->allocateId = -1; + pResBuf->comp = true; + pResBuf->file = NULL; + pResBuf->qId = qId; + pResBuf->fileSize = 0; + + // at least more than 2 pages must be in memory + assert(inMemBufSize >= pagesize * 2); + + pResBuf->lruList = tdListNew(POINTER_BYTES); + + // init id hash table + pResBuf->groupSet = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false); + pResBuf->assistBuf = malloc(pResBuf->pageSize + 2); // EXTRA BYTES + pResBuf->all = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false); + + char path[PATH_MAX] = {0}; + taosGetTmpfilePath(dir, "qbuf", path); + pResBuf->path = strdup(path); + + pResBuf->emptyDummyIdList = taosArrayInit(1, sizeof(int32_t)); + +// qDebug("QInfo:0x%"PRIx64" create resBuf for output, page size:%d, inmem buf pages:%d, file:%s", qId, pResBuf->pageSize, +// pResBuf->inMemPages, pResBuf->path); + + return TSDB_CODE_SUCCESS; +} + +static int32_t createDiskFile(SDiskbasedBuf* pBuf) { + pBuf->file = fopen(pBuf->path, "wb+"); + if (pBuf->file == NULL) { +// qError("failed to create tmp file: %s on disk. %s", pBuf->path, strerror(errno)); + return TAOS_SYSTEM_ERROR(errno); + } + + return TSDB_CODE_SUCCESS; +} + +static char* doCompressData(void* data, int32_t srcSize, int32_t *dst, SDiskbasedBuf* pBuf) { // do nothing + if (!pBuf->comp) { + *dst = srcSize; + return data; + } + + *dst = tsCompressString(data, srcSize, 1, pBuf->assistBuf, srcSize, ONE_STAGE_COMP, NULL, 0); + + memcpy(data, pBuf->assistBuf, *dst); + return data; +} + +static char* doDecompressData(void* data, int32_t srcSize, int32_t *dst, SDiskbasedBuf* pBuf) { // do nothing + if (!pBuf->comp) { + *dst = srcSize; + return data; + } + + *dst = tsDecompressString(data, srcSize, 1, pBuf->assistBuf, pBuf->pageSize+sizeof(SFilePage), ONE_STAGE_COMP, NULL, 0); + if (*dst > 0) { + memcpy(data, pBuf->assistBuf, *dst); + } + return data; +} + +static uint64_t allocatePositionInFile(SDiskbasedBuf* pBuf, size_t size) { + if (pBuf->pFree == NULL) { + return pBuf->nextPos; + } else { + int32_t offset = -1; + + size_t num = taosArrayGetSize(pBuf->pFree); + for(int32_t i = 0; i < num; ++i) { + SFreeListItem* pi = taosArrayGet(pBuf->pFree, i); + if (pi->len >= size) { + offset = pi->offset; + pi->offset += (int32_t)size; + pi->len -= (int32_t)size; + + return offset; + } + } + + // no available recycle space, allocate new area in file + return pBuf->nextPos; + } +} + +static char* doFlushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) { + assert(!pg->used && pg->pData != NULL); + + int32_t size = -1; + char* t = NULL; + if (pg->offset == -1 || pg->dirty) { + SFilePage* pPage = (SFilePage*) GET_DATA_PAYLOAD(pg); + t = doCompressData(pPage->data, pBuf->pageSize, &size, pBuf); + } + + // this page is flushed to disk for the first time + if (pg->offset == -1) { + assert(pg->dirty == true); + + pg->offset = allocatePositionInFile(pBuf, size); + pBuf->nextPos += size; + + int32_t ret = fseek(pBuf->file, pg->offset, SEEK_SET); + if (ret != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return NULL; + } + + ret = (int32_t) fwrite(t, 1, size, pBuf->file); + if (ret != size) { + terrno = TAOS_SYSTEM_ERROR(errno); + return NULL; + } + + if (pBuf->fileSize < pg->offset + size) { + pBuf->fileSize = pg->offset + size; + } + + pBuf->statis.flushBytes += size; + pBuf->statis.flushPages += 1; + } else if (pg->dirty) { + // length becomes greater, current space is not enough, allocate new place, otherwise, do nothing + if (pg->length < size) { + // 1. add current space to free list + SPageDiskInfo dinfo = {.length = pg->length, .offset = pg->offset}; + taosArrayPush(pBuf->pFree, &dinfo); + + // 2. allocate new position, and update the info + pg->offset = allocatePositionInFile(pBuf, size); + pBuf->nextPos += size; + } + + // 3. write to disk. + int32_t ret = fseek(pBuf->file, pg->offset, SEEK_SET); + if (ret != 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return NULL; + } + + ret = (int32_t)fwrite(t, 1, size, pBuf->file); + if (ret != size) { + terrno = TAOS_SYSTEM_ERROR(errno); + return NULL; + } + + if (pBuf->fileSize < pg->offset + size) { + pBuf->fileSize = pg->offset + size; + } + + pBuf->statis.flushBytes += size; + pBuf->statis.flushPages += 1; + } + + char* pDataBuf = pg->pData; + memset(pDataBuf, 0, pBuf->pageSize + sizeof(SFilePage)); + + pg->pData = NULL; // this means the data is not in buffer + pg->length = size; + pg->dirty = false; + + return pDataBuf; +} + +static char* flushPageToDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) { + int32_t ret = TSDB_CODE_SUCCESS; + assert(((int64_t) pBuf->numOfPages * pBuf->pageSize) == pBuf->totalBufSize && pBuf->numOfPages >= pBuf->inMemPages); + + if (pBuf->file == NULL) { + if ((ret = createDiskFile(pBuf)) != TSDB_CODE_SUCCESS) { + terrno = ret; + return NULL; + } + } + + return doFlushPageToDisk(pBuf, pg); +} + +// load file block data in disk +static int32_t loadPageFromDisk(SDiskbasedBuf* pBuf, SPageInfo* pg) { + int32_t ret = fseek(pBuf->file, pg->offset, SEEK_SET); + if (ret != 0) { + ret = TAOS_SYSTEM_ERROR(errno); + return ret; + } + + SFilePage* pPage = (SFilePage*) GET_DATA_PAYLOAD(pg); + ret = (int32_t)fread(pPage->data, 1, pg->length, pBuf->file); + if (ret != pg->length) { + ret = TAOS_SYSTEM_ERROR(errno); + return ret; + } + + pBuf->statis.loadBytes += pg->length; + pBuf->statis.loadPages += 1; + + int32_t fullSize = 0; + doDecompressData(pPage->data, pg->length, &fullSize, pBuf); + return 0; +} + +static SIDList addNewGroup(SDiskbasedBuf* pBuf, int32_t groupId) { + assert(taosHashGet(pBuf->groupSet, (const char*) &groupId, sizeof(int32_t)) == NULL); + + SArray* pa = taosArrayInit(1, POINTER_BYTES); + int32_t ret = taosHashPut(pBuf->groupSet, (const char*)&groupId, sizeof(int32_t), &pa, POINTER_BYTES); + assert(ret == 0); + + return pa; +} + +static SPageInfo* registerPage(SDiskbasedBuf* pBuf, int32_t groupId, int32_t pageId) { + SIDList list = NULL; + + char** p = taosHashGet(pBuf->groupSet, (const char*)&groupId, sizeof(int32_t)); + if (p == NULL) { // it is a new group id + list = addNewGroup(pBuf, groupId); + } else { + list = (SIDList) (*p); + } + + pBuf->numOfPages += 1; + + SPageInfo* ppi = malloc(sizeof(SPageInfo));//{ .info = PAGE_INFO_INITIALIZER, .pageId = pageId, .pn = NULL}; + + ppi->pageId = pageId; + ppi->pData = NULL; + ppi->offset = -1; + ppi->length = -1; + ppi->used = true; + ppi->pn = NULL; + + return *(SPageInfo**) taosArrayPush(list, &ppi); +} + +static SListNode* getEldestUnrefedPage(SDiskbasedBuf* pBuf) { + SListIter iter = {0}; + tdListInitIter(pBuf->lruList, &iter, TD_LIST_BACKWARD); + + SListNode* pn = NULL; + while((pn = tdListNext(&iter)) != NULL) { + assert(pn != NULL); + + SPageInfo* pageInfo = *(SPageInfo**) pn->data; + assert(pageInfo->pageId >= 0 && pageInfo->pn == pn); + + if (!pageInfo->used) { + break; + } + } + + return pn; +} + +static char* evacOneDataPage(SDiskbasedBuf* pBuf) { + char* bufPage = NULL; + SListNode* pn = getEldestUnrefedPage(pBuf); + + // all pages are referenced by user, try to allocate new space + if (pn == NULL) { + assert(0); + int32_t prev = pBuf->inMemPages; + + // increase by 50% of previous mem pages + pBuf->inMemPages = (int32_t)(pBuf->inMemPages * 1.5f); + +// qWarn("%p in memory buf page not sufficient, expand from %d to %d, page size:%d", pBuf, prev, +// pBuf->inMemPages, pBuf->pageSize); + } else { + tdListPopNode(pBuf->lruList, pn); + + SPageInfo* d = *(SPageInfo**) pn->data; + assert(d->pn == pn); + + d->pn = NULL; + tfree(pn); + + bufPage = flushPageToDisk(pBuf, d); + } + + return bufPage; +} + +static void lruListPushFront(SList *pList, SPageInfo* pi) { + tdListPrepend(pList, &pi); + SListNode* front = tdListGetHead(pList); + pi->pn = front; +} + +static void lruListMoveToFront(SList *pList, SPageInfo* pi) { + tdListPopNode(pList, pi->pn); + tdListPrependNode(pList, pi->pn); +} + +static FORCE_INLINE size_t getAllocPageSize(int32_t pageSize) { + return pageSize + POINTER_BYTES + 2 + sizeof(SFilePage); +} + +SFilePage* getNewDataBuf(SDiskbasedBuf* pBuf, int32_t groupId, int32_t* pageId) { + pBuf->statis.getPages += 1; + + char* availablePage = NULL; + if (NO_IN_MEM_AVAILABLE_PAGES(pBuf)) { + availablePage = evacOneDataPage(pBuf); + + // Failed to allocate a new buffer page, and there is an error occurs. + if (availablePage == NULL) { + return NULL; + } + } + + // register new id in this group + *pageId = (++pBuf->allocateId); + + // register page id info + SPageInfo* pi = registerPage(pBuf, groupId, *pageId); + + // add to LRU list + assert(listNEles(pBuf->lruList) < pBuf->inMemPages && pBuf->inMemPages > 0); + lruListPushFront(pBuf->lruList, pi); + + // add to hash map + taosHashPut(pBuf->all, pageId, sizeof(int32_t), &pi, POINTER_BYTES); + + // allocate buf + if (availablePage == NULL) { + pi->pData = calloc(1, getAllocPageSize(pBuf->pageSize)); // add extract bytes in case of zipped buffer increased. + } else { + pi->pData = availablePage; + } + + pBuf->totalBufSize += pBuf->pageSize; + + ((void**)pi->pData)[0] = pi; + pi->used = true; + + return (void *)(GET_DATA_PAYLOAD(pi)); +} + +SFilePage* getBufPage(SDiskbasedBuf* pBuf, int32_t id) { + assert(pBuf != NULL && id >= 0); + pBuf->statis.getPages += 1; + + SPageInfo** pi = taosHashGet(pBuf->all, &id, sizeof(int32_t)); + assert(pi != NULL && *pi != NULL); + + if ((*pi)->pData != NULL) { // it is in memory + // no need to update the LRU list if only one page exists + if (pBuf->numOfPages == 1) { + (*pi)->used = true; + return (void *)(GET_DATA_PAYLOAD(*pi)); + } + + SPageInfo** pInfo = (SPageInfo**) ((*pi)->pn->data); + assert(*pInfo == *pi); + + lruListMoveToFront(pBuf->lruList, (*pi)); + (*pi)->used = true; + + return (void *)(GET_DATA_PAYLOAD(*pi)); + + } else { // not in memory + assert((*pi)->pData == NULL && (*pi)->pn == NULL && (*pi)->length >= 0 && (*pi)->offset >= 0); + + char* availablePage = NULL; + if (NO_IN_MEM_AVAILABLE_PAGES(pBuf)) { + availablePage = evacOneDataPage(pBuf); + if (availablePage == NULL) { + return NULL; + } + } + + if (availablePage == NULL) { + (*pi)->pData = calloc(1, getAllocPageSize(pBuf->pageSize)); + } else { + (*pi)->pData = availablePage; + } + + ((void**)((*pi)->pData))[0] = (*pi); + + lruListPushFront(pBuf->lruList, *pi); + (*pi)->used = true; + + int32_t code = loadPageFromDisk(pBuf, *pi); + if (code != 0) { + return NULL; + } + + return (void *)(GET_DATA_PAYLOAD(*pi)); + } +} + +void releaseBufPage(SDiskbasedBuf* pBuf, void* page) { + assert(pBuf != NULL && page != NULL); + int32_t offset = offsetof(SPageInfo, pData); + char* p = page - offset; + + SPageInfo* ppi = ((SPageInfo**) p)[0]; + releaseBufPageInfo(pBuf, ppi); +} + +void releaseBufPageInfo(SDiskbasedBuf* pBuf, SPageInfo* pi) { + assert(pi->pData != NULL && pi->used); + + pi->used = false; + pBuf->statis.releasePages += 1; +} + +size_t getNumOfResultBufGroupId(const SDiskbasedBuf* pBuf) { return taosHashGetSize(pBuf->groupSet); } + +size_t getTotalBufSize(const SDiskbasedBuf* pBuf) { return (size_t)pBuf->totalBufSize; } + +SIDList getDataBufPagesIdList(SDiskbasedBuf* pBuf, int32_t groupId) { + assert(pBuf != NULL); + + char** p = taosHashGet(pBuf->groupSet, (const char*)&groupId, sizeof(int32_t)); + if (p == NULL) { // it is a new group id + return pBuf->emptyDummyIdList; + } else { + return (SArray*) (*p); + } +} + +void destroyResultBuf(SDiskbasedBuf* pBuf) { + if (pBuf == NULL) { + return; + } + + printStatisData(pBuf); + + if (pBuf->file != NULL) { + uDebug("Paged buffer closed, total:%.2f Kb (%d Pages), inmem size:%.2f Kb (%d Pages), file size:%.2f Kb, page size:%.2f Kb, %"PRIx64"\n", + pBuf->totalBufSize/1024.0, pBuf->numOfPages, listNEles(pBuf->lruList) * pBuf->pageSize / 1024.0, + listNEles(pBuf->lruList), pBuf->fileSize/1024.0, pBuf->pageSize/1024.0f, pBuf->qId); + + fclose(pBuf->file); + } else { + uDebug("Paged buffer closed, total:%.2f Kb, no file created, %"PRIx64, pBuf->totalBufSize/1024.0, pBuf->qId); + } + + // print the statistics information + { + SDiskbasedBufStatis *ps = &pBuf->statis; + uDebug("Get/Release pages:%d/%d, flushToDisk:%.2f Kb (%d Pages), loadFromDisk:%.2f Kb (%d Pages), avgPageSize:%.2f Kb\n" + , ps->getPages, ps->releasePages, ps->flushBytes/1024.0f, ps->flushPages, ps->loadBytes/1024.0f, ps->loadPages + , ps->loadBytes/(1024.0 * ps->loadPages)); + } + + remove(pBuf->path); + tfree(pBuf->path); + + SArray** p = taosHashIterate(pBuf->groupSet, NULL); + while(p) { + size_t n = taosArrayGetSize(*p); + for(int32_t i = 0; i < n; ++i) { + SPageInfo* pi = taosArrayGetP(*p, i); + tfree(pi->pData); + tfree(pi); + } + + taosArrayDestroy(*p); + p = taosHashIterate(pBuf->groupSet, p); + } + + tdListFree(pBuf->lruList); + taosArrayDestroy(pBuf->emptyDummyIdList); + taosHashCleanup(pBuf->groupSet); + taosHashCleanup(pBuf->all); + + tfree(pBuf->assistBuf); + tfree(pBuf); +} + +SPageInfo* getLastPageInfo(SIDList pList) { + size_t size = taosArrayGetSize(pList); + SPageInfo* pPgInfo = taosArrayGetP(pList, size - 1); + return pPgInfo; +} + +int32_t getPageId(const SPageInfo* pPgInfo) { + ASSERT(pPgInfo != NULL); + return pPgInfo->pageId; +} + +int32_t getBufPageSize(const SDiskbasedBuf* pBuf) { + return pBuf->pageSize; +} + +int32_t getNumOfInMemBufPages(const SDiskbasedBuf* pBuf) { + return pBuf->inMemPages; +} + +bool isAllDataInMemBuf(const SDiskbasedBuf* pBuf) { + return pBuf->fileSize == 0; +} + +void setBufPageDirty(SFilePage* pPage, bool dirty) { + int32_t offset = offsetof(SPageInfo, pData); // todo extract method + char* p = (char*)pPage - offset; + + SPageInfo* ppi = ((SPageInfo**) p)[0]; + ppi->dirty = dirty; +} + +void printStatisBeforeClose(SDiskbasedBuf* pBuf) { + pBuf->printStatis = true; +} + +SDiskbasedBufStatis getDBufStatis(const SDiskbasedBuf* pBuf) { + return pBuf->statis; +} + +void printStatisData(const SDiskbasedBuf* pBuf) { + if (!pBuf->printStatis) { + return; + } + + const SDiskbasedBufStatis* ps = &pBuf->statis; + + printf( + "Paged buffer closed, total:%.2f Kb (%d Pages), inmem size:%.2f Kb (%d Pages), file size:%.2f Kb, page size:%.2f " + "Kb, %" PRIx64 "\n", + pBuf->totalBufSize / 1024.0, pBuf->numOfPages, listNEles(pBuf->lruList) * pBuf->pageSize / 1024.0, + listNEles(pBuf->lruList), pBuf->fileSize / 1024.0, pBuf->pageSize / 1024.0f, pBuf->qId); + + printf( + "Get/Release pages:%d/%d, flushToDisk:%.2f Kb (%d Pages), loadFromDisk:%.2f Kb (%d Pages), avgPageSize:%.2f Kb\n", + ps->getPages, ps->releasePages, ps->flushBytes / 1024.0f, ps->flushPages, ps->loadBytes / 1024.0f, ps->loadPages, + ps->loadBytes / (1024.0 * ps->loadPages)); +} diff --git a/source/util/src/tpagedfile.c b/source/util/src/tpagedfile.c deleted file mode 100644 index 3cdba580d4..0000000000 --- a/source/util/src/tpagedfile.c +++ /dev/null @@ -1,468 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE - -#include "tpagedfile.h" -#include "thash.h" -#include "stddef.h" -#include "taoserror.h" -#include "tcompression.h" - -#define GET_DATA_PAYLOAD(_p) ((char *)(_p)->pData + POINTER_BYTES) -#define NO_IN_MEM_AVAILABLE_PAGES(_b) (listNEles((_b)->lruList) >= (_b)->inMemPages) - -int32_t createDiskbasedResultBuffer(SDiskbasedResultBuf** pResultBuf, int32_t pagesize, int32_t inMemBufSize, uint64_t qId, const char* dir) { - *pResultBuf = calloc(1, sizeof(SDiskbasedResultBuf)); - - SDiskbasedResultBuf* pResBuf = *pResultBuf; - if (pResBuf == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - pResBuf->pageSize = pagesize; - pResBuf->numOfPages = 0; // all pages are in buffer in the first place - pResBuf->totalBufSize = 0; - pResBuf->inMemPages = inMemBufSize/pagesize; // maximum allowed pages, it is a soft limit. - pResBuf->allocateId = -1; - pResBuf->comp = true; - pResBuf->file = NULL; - pResBuf->qId = qId; - pResBuf->fileSize = 0; - - // at least more than 2 pages must be in memory - assert(inMemBufSize >= pagesize * 2); - - pResBuf->lruList = tdListNew(POINTER_BYTES); - - // init id hash table - pResBuf->groupSet = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false); - pResBuf->assistBuf = malloc(pResBuf->pageSize + 2); // EXTRA BYTES - pResBuf->all = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, false); - - char path[PATH_MAX] = {0}; - taosGetTmpfilePath(dir, "qbuf", path); - pResBuf->path = strdup(path); - - pResBuf->emptyDummyIdList = taosArrayInit(1, sizeof(int32_t)); - -// qDebug("QInfo:0x%"PRIx64" create resBuf for output, page size:%d, inmem buf pages:%d, file:%s", qId, pResBuf->pageSize, -// pResBuf->inMemPages, pResBuf->path); - - return TSDB_CODE_SUCCESS; -} - -static int32_t createDiskFile(SDiskbasedResultBuf* pResultBuf) { - pResultBuf->file = fopen(pResultBuf->path, "wb+"); - if (pResultBuf->file == NULL) { -// qError("failed to create tmp file: %s on disk. %s", pResultBuf->path, strerror(errno)); - return TAOS_SYSTEM_ERROR(errno); - } - - return TSDB_CODE_SUCCESS; -} - -static char* doCompressData(void* data, int32_t srcSize, int32_t *dst, SDiskbasedResultBuf* pResultBuf) { // do nothing - if (!pResultBuf->comp) { - *dst = srcSize; - return data; - } - - *dst = tsCompressString(data, srcSize, 1, pResultBuf->assistBuf, srcSize, ONE_STAGE_COMP, NULL, 0); - - memcpy(data, pResultBuf->assistBuf, *dst); - return data; -} - -static char* doDecompressData(void* data, int32_t srcSize, int32_t *dst, SDiskbasedResultBuf* pResultBuf) { // do nothing - if (!pResultBuf->comp) { - *dst = srcSize; - return data; - } - - *dst = tsDecompressString(data, srcSize, 1, pResultBuf->assistBuf, pResultBuf->pageSize, ONE_STAGE_COMP, NULL, 0); - if (*dst > 0) { - memcpy(data, pResultBuf->assistBuf, *dst); - } - return data; -} - -static int32_t allocatePositionInFile(SDiskbasedResultBuf* pResultBuf, size_t size) { - if (pResultBuf->pFree == NULL) { - return pResultBuf->nextPos; - } else { - int32_t offset = -1; - - size_t num = taosArrayGetSize(pResultBuf->pFree); - for(int32_t i = 0; i < num; ++i) { - SFreeListItem* pi = taosArrayGet(pResultBuf->pFree, i); - if (pi->len >= size) { - offset = pi->offset; - pi->offset += (int32_t)size; - pi->len -= (int32_t)size; - - return offset; - } - } - - // no available recycle space, allocate new area in file - return pResultBuf->nextPos; - } -} - -static char* doFlushPageToDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) { - assert(!pg->used && pg->pData != NULL); - - int32_t size = -1; - char* t = doCompressData(GET_DATA_PAYLOAD(pg), pResultBuf->pageSize, &size, pResultBuf); - - // this page is flushed to disk for the first time - if (pg->info.offset == -1) { - pg->info.offset = allocatePositionInFile(pResultBuf, size); - pResultBuf->nextPos += size; - - int32_t ret = fseek(pResultBuf->file, pg->info.offset, SEEK_SET); - assert(ret == 0); - - ret = (int32_t) fwrite(t, 1, size, pResultBuf->file); - assert(ret == size); - - if (pResultBuf->fileSize < pg->info.offset + pg->info.length) { - pResultBuf->fileSize = pg->info.offset + pg->info.length; - } - } else { - // length becomes greater, current space is not enough, allocate new place, otherwise, do nothing - if (pg->info.length < size) { - // 1. add current space to free list - taosArrayPush(pResultBuf->pFree, &pg->info); - - // 2. allocate new position, and update the info - pg->info.offset = allocatePositionInFile(pResultBuf, size); - pResultBuf->nextPos += size; - } - - //3. write to disk. - int32_t ret = fseek(pResultBuf->file, pg->info.offset, SEEK_SET); - if (ret != 0) { // todo handle the error case - - } - - ret = (int32_t)fwrite(t, size, 1, pResultBuf->file); - if (ret != size) { // todo handle the error case - - } - - if (pResultBuf->fileSize < pg->info.offset + pg->info.length) { - pResultBuf->fileSize = pg->info.offset + pg->info.length; - } - } - - char* ret = pg->pData; - memset(ret, 0, pResultBuf->pageSize); - - pg->pData = NULL; - pg->info.length = size; - - pResultBuf->statis.flushBytes += pg->info.length; - - return ret; -} - -static char* flushPageToDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) { - int32_t ret = TSDB_CODE_SUCCESS; - assert(((int64_t) pResultBuf->numOfPages * pResultBuf->pageSize) == pResultBuf->totalBufSize && pResultBuf->numOfPages >= pResultBuf->inMemPages); - - if (pResultBuf->file == NULL) { - if ((ret = createDiskFile(pResultBuf)) != TSDB_CODE_SUCCESS) { - terrno = ret; - return NULL; - } - } - - return doFlushPageToDisk(pResultBuf, pg); -} - -// load file block data in disk -static char* loadPageFromDisk(SDiskbasedResultBuf* pResultBuf, SPageInfo* pg) { - int32_t ret = fseek(pResultBuf->file, pg->info.offset, SEEK_SET); - ret = (int32_t)fread(GET_DATA_PAYLOAD(pg), 1, pg->info.length, pResultBuf->file); - if (ret != pg->info.length) { - terrno = errno; - return NULL; - } - - pResultBuf->statis.loadBytes += pg->info.length; - - int32_t fullSize = 0; - doDecompressData(GET_DATA_PAYLOAD(pg), pg->info.length, &fullSize, pResultBuf); - - return (char*)GET_DATA_PAYLOAD(pg); -} - -static SIDList addNewGroup(SDiskbasedResultBuf* pResultBuf, int32_t groupId) { - assert(taosHashGet(pResultBuf->groupSet, (const char*) &groupId, sizeof(int32_t)) == NULL); - - SArray* pa = taosArrayInit(1, POINTER_BYTES); - int32_t ret = taosHashPut(pResultBuf->groupSet, (const char*)&groupId, sizeof(int32_t), &pa, POINTER_BYTES); - assert(ret == 0); - - return pa; -} - -static SPageInfo* registerPage(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t pageId) { - SIDList list = NULL; - - char** p = taosHashGet(pResultBuf->groupSet, (const char*)&groupId, sizeof(int32_t)); - if (p == NULL) { // it is a new group id - list = addNewGroup(pResultBuf, groupId); - } else { - list = (SIDList) (*p); - } - - pResultBuf->numOfPages += 1; - - SPageInfo* ppi = malloc(sizeof(SPageInfo));//{ .info = PAGE_INFO_INITIALIZER, .pageId = pageId, .pn = NULL}; - - ppi->pageId = pageId; - ppi->pData = NULL; - ppi->info = PAGE_INFO_INITIALIZER; - ppi->used = true; - ppi->pn = NULL; - - return *(SPageInfo**) taosArrayPush(list, &ppi); -} - -static SListNode* getEldestUnrefedPage(SDiskbasedResultBuf* pResultBuf) { - SListIter iter = {0}; - tdListInitIter(pResultBuf->lruList, &iter, TD_LIST_BACKWARD); - - SListNode* pn = NULL; - while((pn = tdListNext(&iter)) != NULL) { - assert(pn != NULL); - - SPageInfo* pageInfo = *(SPageInfo**) pn->data; - assert(pageInfo->pageId >= 0 && pageInfo->pn == pn); - - if (!pageInfo->used) { - break; - } - } - - return pn; -} - -static char* evicOneDataPage(SDiskbasedResultBuf* pResultBuf) { - char* bufPage = NULL; - SListNode* pn = getEldestUnrefedPage(pResultBuf); - - // all pages are referenced by user, try to allocate new space - if (pn == NULL) { - int32_t prev = pResultBuf->inMemPages; - - // increase by 50% of previous mem pages - pResultBuf->inMemPages = (int32_t)(pResultBuf->inMemPages * 1.5f); - -// qWarn("%p in memory buf page not sufficient, expand from %d to %d, page size:%d", pResultBuf, prev, -// pResultBuf->inMemPages, pResultBuf->pageSize); - } else { - pResultBuf->statis.flushPages += 1; - tdListPopNode(pResultBuf->lruList, pn); - - SPageInfo* d = *(SPageInfo**) pn->data; - assert(d->pn == pn); - - d->pn = NULL; - tfree(pn); - - bufPage = flushPageToDisk(pResultBuf, d); - } - - return bufPage; -} - -static void lruListPushFront(SList *pList, SPageInfo* pi) { - tdListPrepend(pList, &pi); - SListNode* front = tdListGetHead(pList); - pi->pn = front; -} - -static void lruListMoveToFront(SList *pList, SPageInfo* pi) { - tdListPopNode(pList, pi->pn); - tdListPrependNode(pList, pi->pn); -} - -static FORCE_INLINE size_t getAllocPageSize(int32_t pageSize) { - return pageSize + POINTER_BYTES + 2 + sizeof(SFilePage); -} - -SFilePage* getNewDataBuf(SDiskbasedResultBuf* pResultBuf, int32_t groupId, int32_t* pageId) { - pResultBuf->statis.getPages += 1; - - char* availablePage = NULL; - if (NO_IN_MEM_AVAILABLE_PAGES(pResultBuf)) { - availablePage = evicOneDataPage(pResultBuf); - } - - // register new id in this group - *pageId = (++pResultBuf->allocateId); - - // register page id info - SPageInfo* pi = registerPage(pResultBuf, groupId, *pageId); - - // add to LRU list - assert(listNEles(pResultBuf->lruList) < pResultBuf->inMemPages && pResultBuf->inMemPages > 0); - - lruListPushFront(pResultBuf->lruList, pi); - - // add to hash map - taosHashPut(pResultBuf->all, pageId, sizeof(int32_t), &pi, POINTER_BYTES); - - // allocate buf - if (availablePage == NULL) { - pi->pData = calloc(1, getAllocPageSize(pResultBuf->pageSize)); // add extract bytes in case of zipped buffer increased. - } else { - pi->pData = availablePage; - } - - pResultBuf->totalBufSize += pResultBuf->pageSize; - - ((void**)pi->pData)[0] = pi; - pi->used = true; - - return (void *)(GET_DATA_PAYLOAD(pi)); -} - -SFilePage* getResBufPage(SDiskbasedResultBuf* pResultBuf, int32_t id) { - assert(pResultBuf != NULL && id >= 0); - pResultBuf->statis.getPages += 1; - - SPageInfo** pi = taosHashGet(pResultBuf->all, &id, sizeof(int32_t)); - assert(pi != NULL && *pi != NULL); - - if ((*pi)->pData != NULL) { // it is in memory - // no need to update the LRU list if only one page exists - if (pResultBuf->numOfPages == 1) { - (*pi)->used = true; - return (void *)(GET_DATA_PAYLOAD(*pi)); - } - - SPageInfo** pInfo = (SPageInfo**) ((*pi)->pn->data); - assert(*pInfo == *pi); - - lruListMoveToFront(pResultBuf->lruList, (*pi)); - (*pi)->used = true; - - return (void *)(GET_DATA_PAYLOAD(*pi)); - - } else { // not in memory - assert((*pi)->pData == NULL && (*pi)->pn == NULL && (*pi)->info.length >= 0 && (*pi)->info.offset >= 0); - - char* availablePage = NULL; - if (NO_IN_MEM_AVAILABLE_PAGES(pResultBuf)) { - availablePage = evicOneDataPage(pResultBuf); - } - - if (availablePage == NULL) { - (*pi)->pData = calloc(1, getAllocPageSize(pResultBuf->pageSize)); - } else { - (*pi)->pData = availablePage; - } - - ((void**)((*pi)->pData))[0] = (*pi); - - lruListPushFront(pResultBuf->lruList, *pi); - (*pi)->used = true; - - loadPageFromDisk(pResultBuf, *pi); - return (void *)(GET_DATA_PAYLOAD(*pi)); - } -} - -void releaseResBufPage(SDiskbasedResultBuf* pResultBuf, void* page) { - assert(pResultBuf != NULL && page != NULL); - char* p = (char*) page - POINTER_BYTES; - - SPageInfo* ppi = ((SPageInfo**) p)[0]; - releaseResBufPageInfo(pResultBuf, ppi); -} - -void releaseResBufPageInfo(SDiskbasedResultBuf* pResultBuf, SPageInfo* pi) { - assert(pi->pData != NULL && pi->used); - - pi->used = false; - pResultBuf->statis.releasePages += 1; -} - -size_t getNumOfResultBufGroupId(const SDiskbasedResultBuf* pResultBuf) { return taosHashGetSize(pResultBuf->groupSet); } - -size_t getResBufSize(const SDiskbasedResultBuf* pResultBuf) { return (size_t)pResultBuf->totalBufSize; } - -SIDList getDataBufPagesIdList(SDiskbasedResultBuf* pResultBuf, int32_t groupId) { - assert(pResultBuf != NULL); - - char** p = taosHashGet(pResultBuf->groupSet, (const char*)&groupId, sizeof(int32_t)); - if (p == NULL) { // it is a new group id - return pResultBuf->emptyDummyIdList; - } else { - return (SArray*) (*p); - } -} - -void destroyResultBuf(SDiskbasedResultBuf* pResultBuf) { - if (pResultBuf == NULL) { - return; - } - - if (pResultBuf->file != NULL) { -// qDebug("QInfo:0x%"PRIx64" res output buffer closed, total:%.2f Kb, inmem size:%.2f Kb, file size:%.2f Kb", -// pResultBuf->qId, pResultBuf->totalBufSize/1024.0, listNEles(pResultBuf->lruList) * pResultBuf->pageSize / 1024.0, -// pResultBuf->fileSize/1024.0); - - fclose(pResultBuf->file); - } else { -// qDebug("QInfo:0x%"PRIx64" res output buffer closed, total:%.2f Kb, no file created", pResultBuf->qId, -// pResultBuf->totalBufSize/1024.0); - } - - remove(pResultBuf->path); - tfree(pResultBuf->path); - - SArray** p = taosHashIterate(pResultBuf->groupSet, NULL); - while(p) { - size_t n = taosArrayGetSize(*p); - for(int32_t i = 0; i < n; ++i) { - SPageInfo* pi = taosArrayGetP(*p, i); - tfree(pi->pData); - tfree(pi); - } - - taosArrayDestroy(*p); - p = taosHashIterate(pResultBuf->groupSet, p); - } - - tdListFree(pResultBuf->lruList); - taosArrayDestroy(pResultBuf->emptyDummyIdList); - taosHashCleanup(pResultBuf->groupSet); - taosHashCleanup(pResultBuf->all); - - tfree(pResultBuf->assistBuf); - tfree(pResultBuf); -} - -SPageInfo* getLastPageInfo(SIDList pList) { - size_t size = taosArrayGetSize(pList); - return (SPageInfo*) taosArrayGetP(pList, size - 1); -} - diff --git a/source/util/test/pageBufferTest.cpp b/source/util/test/pageBufferTest.cpp new file mode 100644 index 0000000000..8fa8216223 --- /dev/null +++ b/source/util/test/pageBufferTest.cpp @@ -0,0 +1,165 @@ +#include +#include +#include + +#include "taos.h" +#include "tpagedbuf.h" + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" + +namespace { +// simple test +void simpleTest() { + SDiskbasedBuf* pResultBuf = NULL; + int32_t ret = createDiskbasedBuffer(&pResultBuf, 1024, 4096, 1, "/tmp/"); + + int32_t pageId = 0; + int32_t groupId = 0; + + SFilePage* pBufPage = getNewDataBuf(pResultBuf, groupId, &pageId); + ASSERT_TRUE(pBufPage != NULL); + + ASSERT_EQ(getTotalBufSize(pResultBuf), 1024); + + SIDList list = getDataBufPagesIdList(pResultBuf, groupId); + ASSERT_EQ(taosArrayGetSize(list), 1); + ASSERT_EQ(getNumOfResultBufGroupId(pResultBuf), 1); + + releaseBufPage(pResultBuf, pBufPage); + + SFilePage* pBufPage1 = getNewDataBuf(pResultBuf, groupId, &pageId); + + SFilePage* t = getBufPage(pResultBuf, pageId); + ASSERT_TRUE(t == pBufPage1); + + SFilePage* pBufPage2 = getNewDataBuf(pResultBuf, groupId, &pageId); + SFilePage* t1 = getBufPage(pResultBuf, pageId); + ASSERT_TRUE(t1 == pBufPage2); + + SFilePage* pBufPage3 = getNewDataBuf(pResultBuf, groupId, &pageId); + SFilePage* t2 = getBufPage(pResultBuf, pageId); + ASSERT_TRUE(t2 == pBufPage3); + + SFilePage* pBufPage4 = getNewDataBuf(pResultBuf, groupId, &pageId); + SFilePage* t3 = getBufPage(pResultBuf, pageId); + ASSERT_TRUE(t3 == pBufPage4); + + SFilePage* pBufPage5 = getNewDataBuf(pResultBuf, groupId, &pageId); + SFilePage* t4 = getBufPage(pResultBuf, pageId); + ASSERT_TRUE(t4 == pBufPage5); + + destroyResultBuf(pResultBuf); +} + +void writeDownTest() { + SDiskbasedBuf* pResultBuf = NULL; + int32_t ret = createDiskbasedBuffer(&pResultBuf, 1024, 4*1024, 1, "/tmp/"); + + int32_t pageId = 0; + int32_t writePageId = 0; + int32_t groupId = 0; + int32_t nx = 12345; + + SFilePage* pBufPage = getNewDataBuf(pResultBuf, groupId, &pageId); + ASSERT_TRUE(pBufPage != NULL); + + *(int32_t*)(pBufPage->data) = nx; + writePageId = pageId; + releaseBufPage(pResultBuf, pBufPage); + + SFilePage* pBufPage1 = getNewDataBuf(pResultBuf, groupId, &pageId); + SFilePage* t1 = getBufPage(pResultBuf, pageId); + ASSERT_TRUE(t1 == pBufPage1); + ASSERT_TRUE(pageId == 1); + + SFilePage* pBufPage2 = getNewDataBuf(pResultBuf, groupId, &pageId); + SFilePage* t2 = getBufPage(pResultBuf, pageId); + ASSERT_TRUE(t2 == pBufPage2); + ASSERT_TRUE(pageId == 2); + + SFilePage* pBufPage3 = getNewDataBuf(pResultBuf, groupId, &pageId); + SFilePage* t3 = getBufPage(pResultBuf, pageId); + ASSERT_TRUE(t3 == pBufPage3); + ASSERT_TRUE(pageId == 3); + + SFilePage* pBufPage4 = getNewDataBuf(pResultBuf, groupId, &pageId); + SFilePage* t4 = getBufPage(pResultBuf, pageId); + ASSERT_TRUE(t4 == pBufPage4); + ASSERT_TRUE(pageId == 4); + releaseBufPage(pResultBuf, t4); + + // flush the written page to disk, and read it out again + SFilePage* pBufPagex = getBufPage(pResultBuf, writePageId); + ASSERT_EQ(*(int32_t*)pBufPagex->data, nx); + + SArray* pa = getDataBufPagesIdList(pResultBuf, groupId); + ASSERT_EQ(taosArrayGetSize(pa), 5); + + destroyResultBuf(pResultBuf); +} + +void recyclePageTest() { + SDiskbasedBuf* pResultBuf = NULL; + int32_t ret = createDiskbasedBuffer(&pResultBuf, 1024, 4*1024, 1, "/tmp/"); + + int32_t pageId = 0; + int32_t writePageId = 0; + int32_t groupId = 0; + int32_t nx = 12345; + + SFilePage* pBufPage = getNewDataBuf(pResultBuf, groupId, &pageId); + ASSERT_TRUE(pBufPage != NULL); + releaseBufPage(pResultBuf, pBufPage); + + SFilePage* pBufPage1 = getNewDataBuf(pResultBuf, groupId, &pageId); + SFilePage* t1 = getBufPage(pResultBuf, pageId); + ASSERT_TRUE(t1 == pBufPage1); + ASSERT_TRUE(pageId == 1); + + SFilePage* pBufPage2 = getNewDataBuf(pResultBuf, groupId, &pageId); + SFilePage* t2 = getBufPage(pResultBuf, pageId); + ASSERT_TRUE(t2 == pBufPage2); + ASSERT_TRUE(pageId == 2); + + SFilePage* pBufPage3 = getNewDataBuf(pResultBuf, groupId, &pageId); + SFilePage* t3 = getBufPage(pResultBuf, pageId); + ASSERT_TRUE(t3 == pBufPage3); + ASSERT_TRUE(pageId == 3); + + SFilePage* pBufPage4 = getNewDataBuf(pResultBuf, groupId, &pageId); + SFilePage* t4 = getBufPage(pResultBuf, pageId); + ASSERT_TRUE(t4 == pBufPage4); + ASSERT_TRUE(pageId == 4); + releaseBufPage(pResultBuf, t4); + + SFilePage* pBufPage5 = getNewDataBuf(pResultBuf, groupId, &pageId); + SFilePage* t5 = getBufPage(pResultBuf, pageId); + ASSERT_TRUE(t5 == pBufPage5); + ASSERT_TRUE(pageId == 5); + + // flush the written page to disk, and read it out again + SFilePage* pBufPagex = getBufPage(pResultBuf, writePageId); + *(int32_t*)(pBufPagex->data) = nx; + writePageId = pageId; // update the data + releaseBufPage(pResultBuf, pBufPagex); + + SFilePage* pBufPagex1 = getBufPage(pResultBuf, 1); + + SArray* pa = getDataBufPagesIdList(pResultBuf, groupId); + ASSERT_EQ(taosArrayGetSize(pa), 6); + + destroyResultBuf(pResultBuf); +} +} // namespace + + +TEST(testCase, resultBufferTest) { + srand(time(NULL)); + simpleTest(); + writeDownTest(); + recyclePageTest(); +} + +#pragma GCC diagnostic pop \ No newline at end of file