From 49be2ab41d2c2d5f194783f56f394e8ac5f74c1e Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 10 Jan 2022 11:43:58 +0000 Subject: [PATCH 01/57] more tkv --- source/libs/tkv/src/tDiskMgr.c | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/source/libs/tkv/src/tDiskMgr.c b/source/libs/tkv/src/tDiskMgr.c index d759171f85..c8ce3c6c2a 100644 --- a/source/libs/tkv/src/tDiskMgr.c +++ b/source/libs/tkv/src/tDiskMgr.c @@ -25,12 +25,35 @@ struct SDiskMgr { #define PAGE_OFFSET(PGID, PGSIZE) ((PGID) * (PGSIZE)) int tdmOpen(SDiskMgr **ppDiskMgr, const char *fname, uint16_t pgsize) { - // TODO + SDiskMgr *pDiskMgr; + + pDiskMgr = malloc(sizeof(*pDiskMgr)); + if (pDiskMgr == NULL) { + return -1; + } + + pDiskMgr->fname = strdup(fname); + if (pDiskMgr->fname == NULL) { + free(pDiskMgr); + return -1; + } + pDiskMgr->pgsize = pgsize; + pDiskMgr->fd = open(fname, O_CREAT | O_RDWR, 0755); + if (pDiskMgr->fd < 0) { + free(pDiskMgr->fname); + free(pDiskMgr); + return -1; + } + + *ppDiskMgr = pDiskMgr; + return 0; } int tdmClose(SDiskMgr *pDiskMgr) { - // TODO + close(pDiskMgr->fd); + free(pDiskMgr->fname); + free(pDiskMgr); return 0; } From 2d53617d5c885eccad52b928d9ace840f8a8cc21 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Mon, 10 Jan 2022 19:48:21 +0800 Subject: [PATCH 02/57] [td-11818] support select * on child table. --- include/common/common.h | 6 + include/dnode/vnode/tsdb/tsdb.h | 134 ++ include/libs/executor/executor.h | 26 +- include/libs/planner/planner.h | 2 +- source/client/src/clientImpl.c | 20 +- source/client/test/clientTests.cpp | 900 ++++---- source/dnode/vnode/impl/src/vnodeQuery.c | 2 +- source/dnode/vnode/tsdb/CMakeLists.txt | 1 + source/dnode/vnode/tsdb/src/tsdbCommit.c | 2 +- source/dnode/vnode/tsdb/src/tsdbRead.c | 2138 +++++++++---------- source/libs/executor/CMakeLists.txt | 31 +- source/libs/executor/inc/dataSinkMgt.h | 2 +- source/libs/executor/inc/executil.h | 4 +- source/libs/executor/inc/executorimpl.h | 50 +- source/libs/executor/src/executil.c | 8 +- source/libs/executor/src/executorMain.c | 241 +-- source/libs/executor/src/executorimpl.c | 323 ++- source/libs/executor/test/CMakeLists.txt | 18 + source/libs/executor/test/executorTests.cpp | 221 ++ source/libs/parser/src/parser.c | 3 + source/libs/planner/inc/plannerInt.h | 2 +- source/libs/planner/src/physicalPlan.c | 11 +- source/libs/planner/src/physicalPlanJson.c | 12 +- source/libs/planner/src/planner.c | 36 +- source/libs/planner/test/phyPlanTests.cpp | 5 +- source/libs/qworker/CMakeLists.txt | 26 +- source/libs/qworker/src/qworker.c | 11 +- source/libs/scheduler/src/scheduler.c | 87 +- 28 files changed, 2298 insertions(+), 2024 deletions(-) create mode 100644 source/libs/executor/test/CMakeLists.txt diff --git a/include/common/common.h b/include/common/common.h index 0ae06e5a7f..9b8a465442 100644 --- a/include/common/common.h +++ b/include/common/common.h @@ -62,6 +62,12 @@ typedef struct SConstantItem { SVariant value; } SConstantItem; +typedef struct { + uint32_t numOfTables; + SArray *pGroupList; + SHashObj *map; // speedup acquire the tableQueryInfo by table uid +} STableGroupInfo; + typedef struct SSDataBlock { SColumnDataAgg *pBlockAgg; SArray *pDataBlock; // SArray diff --git a/include/dnode/vnode/tsdb/tsdb.h b/include/dnode/vnode/tsdb/tsdb.h index 3aeef4d013..f3d9f71346 100644 --- a/include/dnode/vnode/tsdb/tsdb.h +++ b/include/dnode/vnode/tsdb/tsdb.h @@ -18,6 +18,7 @@ #include "mallocator.h" #include "meta.h" +#include "common.h" #ifdef __cplusplus extern "C" { @@ -39,6 +40,10 @@ typedef struct STable { STSchema *pSchema; } STable; +#define BLOCK_LOAD_OFFSET_SEQ_ORDER 1 +#define BLOCK_LOAD_TABLE_SEQ_ORDER 2 +#define BLOCK_LOAD_TABLE_RR_ORDER 3 + #define TABLE_TID(t) (t)->tid #define TABLE_UID(t) (t)->uid @@ -58,6 +63,22 @@ typedef struct STsdbCfg { int8_t compression; } STsdbCfg; +// query condition to build multi-table data block iterator +typedef struct STsdbQueryCond { + STimeWindow twindow; + int32_t order; // desc|asc order to iterate the data block + int32_t numOfCols; + SColumnInfo *colList; + bool loadExternalRows; // load external rows or not + int32_t type; // data block load type: +} STsdbQueryCond; + +typedef struct { + void *pTable; + TSKEY lastKey; + uint64_t uid; +} STableKeyInfo; + // STsdb STsdb *tsdbOpen(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF, SMeta *pMeta); void tsdbClose(STsdb *); @@ -70,6 +91,119 @@ int tsdbCommit(STsdb *pTsdb); int tsdbOptionsInit(STsdbCfg *); void tsdbOptionsClear(STsdbCfg *); +typedef void* tsdbReadHandleT; +/** + * Get the data block iterator, starting from position according to the query condition + * + * @param tsdb tsdb handle + * @param pCond query condition, including time window, result set order, and basic required columns for each block + * @param tableInfoGroup table object list in the form of set, grouped into different sets according to the + * group by condition + * @param qinfo query info handle from query processor + * @return + */ +tsdbReadHandleT *tsdbQueryTables(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, uint64_t qId, + void *pRef); + +/** + * Get the last row of the given query time window for all the tables in STableGroupInfo object. + * Note that only one data block with only row will be returned while invoking retrieve data block function for + * all tables in this group. + * + * @param tsdb tsdb handle + * @param pCond query condition, including time window, result set order, and basic required columns for each block + * @param tableInfo table list. + * @return + */ +//tsdbReadHandleT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfo, uint64_t qId, +// SMemRef *pRef); + + +tsdbReadHandleT tsdbQueryCacheLast(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, void* pMemRef); + +bool isTsdbCacheLastRow(tsdbReadHandleT* pTsdbReadHandle); + +/** + * get num of rows in mem table + * + * @param pHandle + * @return row size + */ + +int64_t tsdbGetNumOfRowsInMemTable(tsdbReadHandleT* pHandle); + +/** + * move to next block if exists + * + * @param pTsdbReadHandle + * @return + */ +bool tsdbNextDataBlock(tsdbReadHandleT pTsdbReadHandle); + +/** + * Get current data block information + * + * @param pTsdbReadHandle + * @param pBlockInfo + * @return + */ +void tsdbRetrieveDataBlockInfo(tsdbReadHandleT *pTsdbReadHandle, SDataBlockInfo *pBlockInfo); + +/** + * + * Get the pre-calculated information w.r.t. current data block. + * + * In case of data block in cache, the pBlockStatis will always be NULL. + * If a block is not completed loaded from disk, the pBlockStatis will be NULL. + + * @pBlockStatis the pre-calculated value for current data blocks. if the block is a cache block, always return 0 + * @return + */ +int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReadHandleT *pTsdbReadHandle, SDataStatis **pBlockStatis); + +/** + * + * The query condition with primary timestamp is passed to iterator during its constructor function, + * the returned data block must be satisfied with the time window condition in any cases, + * which means the SData data block is not actually the completed disk data blocks. + * + * @param pTsdbReadHandle query handle + * @param pColumnIdList required data columns id list + * @return + */ +SArray *tsdbRetrieveDataBlock(tsdbReadHandleT *pTsdbReadHandle, SArray *pColumnIdList); + +/** + * destroy the created table group list, which is generated by tag query + * @param pGroupList + */ +void tsdbDestroyTableGroup(STableGroupInfo *pGroupList); + +/** + * create the table group result including only one table, used to handle the normal table query + * + * @param tsdb tsdbHandle + * @param uid table uid + * @param pGroupInfo the generated result + * @return + */ +int32_t tsdbGetOneTableGroup(STsdb *tsdb, uint64_t uid, TSKEY startKey, STableGroupInfo *pGroupInfo); + +/** + * + * @param tsdb + * @param pTableIdList + * @param pGroupInfo + * @return + */ +int32_t tsdbGetTableGroupFromIdList(STsdb *tsdb, SArray *pTableIdList, STableGroupInfo *pGroupInfo); + +/** + * clean up the query handle + * @param queryHandle + */ +void tsdbCleanupQueryHandle(tsdbReadHandleT queryHandle); + #ifdef __cplusplus } #endif diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 17c11b5d09..cf3ff101aa 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -22,23 +22,25 @@ extern "C" { typedef void* qTaskInfo_t; -/** - * create the qinfo object according to QueryTableMsg - * @param tsdb - * @param pQueryTableMsg - * @param pTaskInfo - * @return - */ -int32_t qCreateTask(void* tsdb, int32_t vgId, void* pQueryTableMsg, qTaskInfo_t* pTaskInfo, uint64_t qId); + /** + * Create the exec task object according to task json + * @param tsdb + * @param vgId + * @param pTaskInfoMsg + * @param pTaskInfo + * @param qId + * @return + */ +int32_t qCreateExecTask(void* tsdb, int32_t vgId, struct SSubplan* pPlan, qTaskInfo_t* pTaskInfo); /** - * the main query execution function, including query on both table and multiple tables, + * the main task execution function, including query on both table and multiple tables, * which are decided according to the tag or table name query conditions * * @param qinfo * @return */ -bool qExecTask(qTaskInfo_t qinfo, uint64_t *qId); +bool qExecTask(qTaskInfo_t qTask); /** * Retrieve the produced results information, if current query is not paused or completed, @@ -81,7 +83,7 @@ int32_t qKillTask(qTaskInfo_t qinfo); * @param qinfo * @return */ -int32_t qIsQueryCompleted(qTaskInfo_t qinfo); +int32_t qIsTaskCompleted(qTaskInfo_t qinfo); /** * destroy query info structure @@ -113,7 +115,7 @@ int32_t qGetQualifiedTableIdList(void* pTableList, const char* tagCond, int32_t * @param numOfIndex * @return */ -int32_t qCreateTableGroupByGroupExpr(SArray* pTableIdList, TSKEY skey, STableGroupInfo groupInfo, SColIndex* groupByIndex, int32_t numOfIndex); +//int32_t qCreateTableGroupByGroupExpr(SArray* pTableIdList, TSKEY skey, STableGroupInfo groupInfo, SColIndex* groupByIndex, int32_t numOfIndex); /** * Update the table id list of a given query. diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index c5da68f0a6..e58fa351d0 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -150,7 +150,7 @@ struct SQueryNode; * @param requestId * @return */ -int32_t qCreateQueryDag(const struct SQueryNode* pQueryInfo, struct SQueryDag** pDag, uint64_t requestId); +int32_t qCreateQueryDag(const struct SQueryNode* pQueryInfo, struct SQueryDag** pDag, SSchema** pSchema, uint32_t* numOfResCols, uint64_t requestId); // Set datasource of this subplan, multiple calls may be made to a subplan. // @subplan subplan to be schedule diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index d18142cebf..26f97cc6ef 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -197,7 +197,25 @@ int32_t execDdlQuery(SRequestObj* pRequest, SQueryNode* pQuery) { int32_t getPlan(SRequestObj* pRequest, SQueryNode* pQueryNode, SQueryDag** pDag) { pRequest->type = pQueryNode->type; - return qCreateQueryDag(pQueryNode, pDag, pRequest->requestId); + + SSchema *pSchema = NULL; + SReqResultInfo* pResInfo = &pRequest->body.resInfo; + + int32_t code = qCreateQueryDag(pQueryNode, pDag, &pSchema, &pResInfo->numOfCols, pRequest->requestId); + if (code != 0) { + return code; + } + + if (pQueryNode->type == TSDB_SQL_SELECT) { + pResInfo->fields = calloc(1, sizeof(TAOS_FIELD)); + for (int32_t i = 0; i < pResInfo->numOfCols; ++i) { + pResInfo->fields[i].bytes = pSchema[i].bytes; + pResInfo->fields[i].type = pSchema[i].type; + tstrncpy(pResInfo->fields[i].name, pSchema[i].name, tListLen(pResInfo->fields[i].name)); + } + } + + return code; } int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, void** pJob) { diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 73c9fc5e9f..af5ad874f8 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -24,7 +24,6 @@ #include "../inc/clientInt.h" #include "taos.h" -#include "tglobal.h" namespace { void showDB(TAOS* pConn) { @@ -57,449 +56,449 @@ TEST(testCase, connect_Test) { taos_close(pConn); } -TEST(testCase, create_user_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "create user abc pass 'abc'"); - if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { - printf("failed to create user, reason:%s\n", taos_errstr(pRes)); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, create_account_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "create account aabc pass 'abc'"); - if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { - printf("failed to create user, reason:%s\n", taos_errstr(pRes)); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, drop_account_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "drop account aabc"); - if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { - printf("failed to create user, reason:%s\n", taos_errstr(pRes)); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, show_user_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "show users"); - TAOS_ROW pRow = NULL; - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); - - char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, drop_user_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "drop user abc"); - if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { - printf("failed to create user, reason:%s\n", taos_errstr(pRes)); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, show_db_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "show databases"); - TAOS_ROW pRow = NULL; - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); - - char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } - - taos_close(pConn); -} - -TEST(testCase, create_db_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); - if (taos_errno(pRes) != 0) { - printf("error in create db, reason:%s\n", taos_errstr(pRes)); - } - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - ASSERT_TRUE(pFields == NULL); - - int32_t numOfFields = taos_num_fields(pRes); - ASSERT_EQ(numOfFields, 0); - - taos_free_result(pRes); - - pRes = taos_query(pConn, "create database abc1 vgroups 4"); - if (taos_errno(pRes) != 0) { - printf("error in create db, reason:%s\n", taos_errstr(pRes)); - } - taos_close(pConn); -} - -TEST(testCase, create_dnode_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "create dnode abc1 port 7000"); - if (taos_errno(pRes) != 0) { - printf("error in create dnode, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create dnode 1.1.1.1 port 9000"); - if (taos_errno(pRes) != 0) { - printf("failed to create dnode, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - taos_close(pConn); -} - -TEST(testCase, drop_dnode_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "drop dnode 2"); - if (taos_errno(pRes) != 0) { - printf("error in drop dnode, reason:%s\n", taos_errstr(pRes)); - } - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - ASSERT_TRUE(pFields == NULL); - - int32_t numOfFields = taos_num_fields(pRes); - ASSERT_EQ(numOfFields, 0); - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, use_db_test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("error in use db, reason:%s\n", taos_errstr(pRes)); - } - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - ASSERT_TRUE(pFields == NULL); - - int32_t numOfFields = taos_num_fields(pRes); - ASSERT_EQ(numOfFields, 0); - - taos_close(pConn); -} - -// TEST(testCase, drop_db_test) { +//TEST(testCase, create_user_Test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // assert(pConn != NULL); // -// showDB(pConn); -// -// TAOS_RES* pRes = taos_query(pConn, "drop database abc1"); -// if (taos_errno(pRes) != 0) { -// printf("failed to drop db, reason:%s\n", taos_errstr(pRes)); +// TAOS_RES* pRes = taos_query(pConn, "create user abc pass 'abc'"); +// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { +// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); // } -// taos_free_result(pRes); // -// showDB(pConn); -// -// pRes = taos_query(pConn, "create database abc1"); -// if (taos_errno(pRes) != 0) { -// printf("create to drop db, reason:%s\n", taos_errstr(pRes)); -// } // taos_free_result(pRes); // taos_close(pConn); //} - -TEST(testCase, create_stable_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); - if (taos_errno(pRes) != 0) { - printf("error in create db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("error in use db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create stable st1(ts timestamp, k int) tags(a int)"); - if (taos_errno(pRes) != 0) { - printf("error in create stable, reason:%s\n", taos_errstr(pRes)); - } - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - ASSERT_TRUE(pFields == NULL); - - int32_t numOfFields = taos_num_fields(pRes); - ASSERT_EQ(numOfFields, 0); - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, create_table_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table tm0(ts timestamp, k int)"); - taos_free_result(pRes); - - taos_close(pConn); -} - -TEST(testCase, create_ctable_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("failed to use db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table tm0 using st1 tags(1)"); - if (taos_errno(pRes) != 0) { - printf("failed to create child table tm0, reason:%s\n", taos_errstr(pRes)); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, show_stable_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("failed to use db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "show stables"); - if (taos_errno(pRes) != 0) { - printf("failed to show stables, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); - } - - TAOS_ROW pRow = NULL; - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); - - char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, show_vgroup_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("failed to use db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "show vgroups"); - if (taos_errno(pRes) != 0) { - printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); - } - - TAOS_ROW pRow = NULL; - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); - - char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, create_multiple_tables) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - ASSERT_NE(pConn, nullptr); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("failed to use db, reason:%s", taos_errstr(pRes)); - taos_free_result(pRes); - taos_close(pConn); - return; - } - - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table t_2 using st1 tags(1)"); - if (taos_errno(pRes) != 0) { - printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); - } - - taos_free_result(pRes); - pRes = taos_query(pConn, "create table t_3 using st1 tags(2)"); - if (taos_errno(pRes) != 0) { - printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); - } - - TAOS_ROW pRow = NULL; - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); - - char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } - - taos_free_result(pRes); - - for (int32_t i = 0; i < 20; ++i) { - char sql[512] = {0}; - snprintf(sql, tListLen(sql), - "create table t_x_%d using st1 tags(2) t_x_%d using st1 tags(5) t_x_%d using st1 tags(911)", i, - (i + 1) * 30, (i + 2) * 40); - TAOS_RES* pres = taos_query(pConn, sql); - if (taos_errno(pres) != 0) { - printf("failed to create table %d\n, reason:%s", i, taos_errstr(pres)); - } - taos_free_result(pres); - } - - taos_close(pConn); -} - -TEST(testCase, show_table_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - taos_free_result(pRes); - - pRes = taos_query(pConn, "show tables"); - if (taos_errno(pRes) != 0) { - printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); - } - - TAOS_ROW pRow = NULL; - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); - - char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, drop_stable_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "create database abc1"); - if (taos_errno(pRes) != 0) { - printf("error in creating db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("error in using db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "drop stable st1"); - if (taos_errno(pRes) != 0) { - printf("failed to drop stable, reason:%s\n", taos_errstr(pRes)); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, generated_request_id_test) { - SHashObj* phash = taosHashInit(10000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); - - for (int32_t i = 0; i < 50000; ++i) { - uint64_t v = generateRequestId(); - void* result = taosHashGet(phash, &v, sizeof(v)); - if (result != nullptr) { - printf("0x%lx, index:%d\n", v, i); - } - assert(result == nullptr); - taosHashPut(phash, &v, sizeof(v), NULL, 0); - } - - taosHashCleanup(phash); -} +// +//TEST(testCase, create_account_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "create account aabc pass 'abc'"); +// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { +// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, drop_account_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "drop account aabc"); +// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { +// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, show_user_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "show users"); +// TAOS_ROW pRow = NULL; +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, drop_user_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "drop user abc"); +// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { +// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, show_db_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "show databases"); +// TAOS_ROW pRow = NULL; +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// +// taos_close(pConn); +//} +// +//TEST(testCase, create_db_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); +// if (taos_errno(pRes) != 0) { +// printf("error in create db, reason:%s\n", taos_errstr(pRes)); +// } +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// ASSERT_TRUE(pFields == NULL); +// +// int32_t numOfFields = taos_num_fields(pRes); +// ASSERT_EQ(numOfFields, 0); +// +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create database abc1 vgroups 4"); +// if (taos_errno(pRes) != 0) { +// printf("error in create db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_close(pConn); +//} +// +//TEST(testCase, create_dnode_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "create dnode abc1 port 7000"); +// if (taos_errno(pRes) != 0) { +// printf("error in create dnode, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create dnode 1.1.1.1 port 9000"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create dnode, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// taos_close(pConn); +//} +// +//TEST(testCase, drop_dnode_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "drop dnode 2"); +// if (taos_errno(pRes) != 0) { +// printf("error in drop dnode, reason:%s\n", taos_errstr(pRes)); +// } +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// ASSERT_TRUE(pFields == NULL); +// +// int32_t numOfFields = taos_num_fields(pRes); +// ASSERT_EQ(numOfFields, 0); +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, use_db_test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("error in use db, reason:%s\n", taos_errstr(pRes)); +// } +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// ASSERT_TRUE(pFields == NULL); +// +// int32_t numOfFields = taos_num_fields(pRes); +// ASSERT_EQ(numOfFields, 0); +// +// taos_close(pConn); +//} +// +//// TEST(testCase, drop_db_test) { +//// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +//// assert(pConn != NULL); +//// +//// showDB(pConn); +//// +//// TAOS_RES* pRes = taos_query(pConn, "drop database abc1"); +//// if (taos_errno(pRes) != 0) { +//// printf("failed to drop db, reason:%s\n", taos_errstr(pRes)); +//// } +//// taos_free_result(pRes); +//// +//// showDB(pConn); +//// +//// pRes = taos_query(pConn, "create database abc1"); +//// if (taos_errno(pRes) != 0) { +//// printf("create to drop db, reason:%s\n", taos_errstr(pRes)); +//// } +//// taos_free_result(pRes); +//// taos_close(pConn); +////} +// +//TEST(testCase, create_stable_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); +// if (taos_errno(pRes) != 0) { +// printf("error in create db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("error in use db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create stable st1(ts timestamp, k int) tags(a int)"); +// if (taos_errno(pRes) != 0) { +// printf("error in create stable, reason:%s\n", taos_errstr(pRes)); +// } +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// ASSERT_TRUE(pFields == NULL); +// +// int32_t numOfFields = taos_num_fields(pRes); +// ASSERT_EQ(numOfFields, 0); +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, create_table_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create table tm0(ts timestamp, k int)"); +// taos_free_result(pRes); +// +// taos_close(pConn); +//} +// +//TEST(testCase, create_ctable_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create table tm0 using st1 tags(1)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create child table tm0, reason:%s\n", taos_errstr(pRes)); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, show_stable_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "show stables"); +// if (taos_errno(pRes) != 0) { +// printf("failed to show stables, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// ASSERT_TRUE(false); +// } +// +// TAOS_ROW pRow = NULL; +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, show_vgroup_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "show vgroups"); +// if (taos_errno(pRes) != 0) { +// printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// ASSERT_TRUE(false); +// } +// +// TAOS_ROW pRow = NULL; +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, create_multiple_tables) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// ASSERT_NE(pConn, nullptr); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// taos_close(pConn); +// return; +// } +// +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create table t_2 using st1 tags(1)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// ASSERT_TRUE(false); +// } +// +// taos_free_result(pRes); +// pRes = taos_query(pConn, "create table t_3 using st1 tags(2)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// ASSERT_TRUE(false); +// } +// +// TAOS_ROW pRow = NULL; +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// +// taos_free_result(pRes); +// +// for (int32_t i = 0; i < 20; ++i) { +// char sql[512] = {0}; +// snprintf(sql, tListLen(sql), +// "create table t_x_%d using st1 tags(2) t_x_%d using st1 tags(5) t_x_%d using st1 tags(911)", i, +// (i + 1) * 30, (i + 2) * 40); +// TAOS_RES* pres = taos_query(pConn, sql); +// if (taos_errno(pres) != 0) { +// printf("failed to create table %d\n, reason:%s", i, taos_errstr(pres)); +// } +// taos_free_result(pres); +// } +// +// taos_close(pConn); +//} +// +//TEST(testCase, show_table_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "show tables"); +// if (taos_errno(pRes) != 0) { +// printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// ASSERT_TRUE(false); +// } +// +// TAOS_ROW pRow = NULL; +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, drop_stable_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "create database abc1"); +// if (taos_errno(pRes) != 0) { +// printf("error in creating db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("error in using db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "drop stable st1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to drop stable, reason:%s\n", taos_errstr(pRes)); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, generated_request_id_test) { +// SHashObj* phash = taosHashInit(10000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); +// +// for (int32_t i = 0; i < 50000; ++i) { +// uint64_t v = generateRequestId(); +// void* result = taosHashGet(phash, &v, sizeof(v)); +// if (result != nullptr) { +// printf("0x%lx, index:%d\n", v, i); +// } +// assert(result == nullptr); +// taosHashPut(phash, &v, sizeof(v), NULL, 0); +// } +// +// taosHashCleanup(phash); +//} // TEST(testCase, create_topic_Test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -557,18 +556,29 @@ TEST(testCase, projection_query_tables) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(pConn, nullptr); - TAOS_RES* pRes = taos_query(pConn, "use test1"); - if (taos_errno(pRes) != 0) { - printf("failed to use db, reason:%s", taos_errstr(pRes)); - taos_free_result(pRes); - return; - } +// TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); +// if (taos_errno(pRes) != 0) { +// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// return; +// } - taos_free_result(pRes); +// taos_free_result(pRes); - pRes = taos_query(pConn, "select * from tm0"); + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + +// pRes = taos_query(pConn, "create table m1 (ts timestamp, k int) tags(a int)"); +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create table tu using m1 tags(1)"); +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "insert into tu values(now, 1)"); +// taos_free_result(pRes); + + pRes = taos_query(pConn, "select * from tu"); if (taos_errno(pRes) != 0) { - printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); + printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); taos_free_result(pRes); ASSERT_TRUE(false); } diff --git a/source/dnode/vnode/impl/src/vnodeQuery.c b/source/dnode/vnode/impl/src/vnodeQuery.c index a32ee50df5..0f3ef9a015 100644 --- a/source/dnode/vnode/impl/src/vnodeQuery.c +++ b/source/dnode/vnode/impl/src/vnodeQuery.c @@ -23,7 +23,7 @@ int vnodeQueryOpen(SVnode *pVnode) { return qWorkerInit(NULL, &pVnode->pQuery); int vnodeProcessQueryReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { vTrace("query message is processed"); - return qWorkerProcessQueryMsg(pVnode, pVnode->pQuery, pMsg); + return qWorkerProcessQueryMsg(pVnode->pTsdb, pVnode->pQuery, pMsg); } int vnodeProcessFetchReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { diff --git a/source/dnode/vnode/tsdb/CMakeLists.txt b/source/dnode/vnode/tsdb/CMakeLists.txt index 65dc95a9d8..e38ba1c466 100644 --- a/source/dnode/vnode/tsdb/CMakeLists.txt +++ b/source/dnode/vnode/tsdb/CMakeLists.txt @@ -13,6 +13,7 @@ else(0) "src/tsdbReadImpl.c" "src/tsdbFile.c" "src/tsdbFS.c" + "src/tsdbRead.c" ) endif(0) diff --git a/source/dnode/vnode/tsdb/src/tsdbCommit.c b/source/dnode/vnode/tsdb/src/tsdbCommit.c index 2be032a335..5aab343c4b 100644 --- a/source/dnode/vnode/tsdb/src/tsdbCommit.c +++ b/source/dnode/vnode/tsdb/src/tsdbCommit.c @@ -1253,7 +1253,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDataCols * pBlock->keyFirst = dataColsKeyFirst(pDataCols); pBlock->keyLast = dataColsKeyLast(pDataCols); - tsdbDebug("vgId:%d tid:%d a block of data is written to file %s, offset %" PRId64 + tsdbDebug("vgId:%d uid:%"PRId64" a block of data is written to file %s, offset %" PRId64 " numOfRows %d len %d numOfCols %" PRId16 " keyFirst %" PRId64 " keyLast %" PRId64, REPO_ID(pRepo), TABLE_TID(pTable), TSDB_FILE_FULL_NAME(pDFile), offset, rowsToWrite, pBlock->len, pBlock->numOfCols, pBlock->keyFirst, pBlock->keyLast); diff --git a/source/dnode/vnode/tsdb/src/tsdbRead.c b/source/dnode/vnode/tsdb/src/tsdbRead.c index da5481dae1..906046ed9a 100644 --- a/source/dnode/vnode/tsdb/src/tsdbRead.c +++ b/source/dnode/vnode/tsdb/src/tsdbRead.c @@ -13,18 +13,23 @@ * along with this program. If not, see . */ +#include "tsdb.h" +#include "tsdbDef.h" +#include "tsdbFS.h" +#include "tsdbLog.h" +#include "tsdbReadImpl.h" +#include "ttime.h" +#include "exception.h" #include "os.h" -#include "tdataformat.h" -#include "tskiplist.h" -#include "tulog.h" #include "talgo.h" #include "tcompare.h" -#include "exception.h" +#include "tdataformat.h" +#include "tskiplist.h" #include "taosdef.h" #include "tlosertree.h" #include "tsdbint.h" -#include "texpr.h" +#include "tmsg.h" #define EXTRA_BYTES 2 #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) @@ -34,8 +39,7 @@ ((SDataBlockInfo){.window = {.skey = (_block)->keyFirst, .ekey = (_block)->keyLast}, \ .numOfCols = (_block)->numOfCols, \ .rows = (_block)->numOfRows, \ - .tid = (_checkInfo)->tableId.tid, \ - .uid = (_checkInfo)->tableId.uid}) + .uid = (_checkInfo)->tableId}) enum { TSDB_QUERY_TYPE_ALL = 1, @@ -62,7 +66,7 @@ typedef struct SQueryFilePos { typedef struct SDataBlockLoadInfo { SDFileSet* fileGroup; int32_t slot; - int32_t tid; + uint64_t uid; SArray* pLoadedCols; } SDataBlockLoadInfo; @@ -79,13 +83,13 @@ enum { typedef struct STableCheckInfo { - STableId tableId; + uint64_t tableId; TSKEY lastKey; STable* pTableObj; SBlockInfo* pCompInfo; int32_t compSize; int32_t numOfBlocks:29; // number of qualified data blocks not the original blocks - uint8_t chosen:2; // indicate which iterator should move forward + uint8_t chosen:2; // indicate which iterator should move forward bool initBuf; // whether to initialize the in-memory skip list iterator or not SSkipListIterator* iter; // mem buffer skip list iterator SSkipListIterator* iiter; // imem buffer skip list iterator @@ -111,8 +115,8 @@ typedef struct SIOCostSummary { int64_t headFileLoadTime; } SIOCostSummary; -typedef struct STsdbQueryHandle { - STsdbRepo* pTsdb; +typedef struct STsdbReadHandle { + STsdb* pTsdb; SQueryFilePos cur; // current position int16_t order; STimeWindow window; // the primary query time window that applies to all queries @@ -137,7 +141,8 @@ typedef struct STsdbQueryHandle { STableBlockInfo* pDataBlockInfo; SDataCols *pDataCols; // in order to hold current file data block int32_t allocSize; // allocated data block size - SMemRef *pMemRef; +// STsdb +// STsdbMemTable * pMemTable; SArray *defaultLoadColumn;// default load column SDataBlockLoadInfo dataBlockLoadInfo; /* record current block load information */ SLoadCompBlockInfo compBlockLoadInfo; /* record current compblock information in SQueryAttr */ @@ -145,7 +150,7 @@ typedef struct STsdbQueryHandle { SArray *prev; // previous row which is before than time window SArray *next; // next row which is after the query time window SIOCostSummary cost; -} STsdbQueryHandle; +} STsdbReadHandle; typedef struct STableGroupSupporter { int32_t numOfCols; @@ -154,23 +159,23 @@ typedef struct STableGroupSupporter { } STableGroupSupporter; static STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList); -static int32_t checkForCachedLastRow(STsdbQueryHandle* pQueryHandle, STableGroupInfo *groupList); -static int32_t checkForCachedLast(STsdbQueryHandle* pQueryHandle); -static int32_t tsdbGetCachedLastRow(STable* pTable, SMemRow* pRes, TSKEY* lastKey); +static int32_t checkForCachedLastRow(STsdbReadHandle* pTsdbReadHandle, STableGroupInfo *groupList); +static int32_t checkForCachedLast(STsdbReadHandle* pTsdbReadHandle); +//static int32_t tsdbGetCachedLastRow(STable* pTable, SMemRow* pRes, TSKEY* lastKey); -static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle); -static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SBlock* pBlock); +static void changeQueryHandleForInterpQuery(tsdbReadHandleT pHandle); +static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInfo* pCheckInfo, SBlock* pBlock); static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); -static int32_t tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win, STsdbQueryHandle* pQueryHandle); +static int32_t tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win, STsdbReadHandle* pTsdbReadHandle); static int32_t tsdbCheckInfoCompar(const void* key1, const void* key2); -static int32_t doGetExternalRow(STsdbQueryHandle* pQueryHandle, int16_t type, SMemRef* pMemRef); -static void* doFreeColumnInfoData(SArray* pColumnInfoData); -static void* destroyTableCheckInfo(SArray* pTableCheckInfo); -static bool tsdbGetExternalRow(TsdbQueryHandleT pHandle); +//static int32_t doGetExternalRow(STsdbReadHandle* pTsdbReadHandle, int16_t type, void* pMemRef); +//static void* doFreeColumnInfoData(SArray* pColumnInfoData); +//static void* destroyTableCheckInfo(SArray* pTableCheckInfo); +static bool tsdbGetExternalRow(tsdbReadHandleT pHandle); static void tsdbInitDataBlockLoadInfo(SDataBlockLoadInfo* pBlockLoadInfo) { pBlockLoadInfo->slot = -1; - pBlockLoadInfo->tid = -1; + pBlockLoadInfo->uid = 0; pBlockLoadInfo->fileGroup = NULL; } @@ -179,21 +184,21 @@ static void tsdbInitCompBlockLoadInfo(SLoadCompBlockInfo* pCompBlockLoadInfo) { pCompBlockLoadInfo->fileId = -1; } -static SArray* getColumnIdList(STsdbQueryHandle* pQueryHandle) { - size_t numOfCols = QH_GET_NUM_OF_COLS(pQueryHandle); +static SArray* getColumnIdList(STsdbReadHandle* pTsdbReadHandle) { + size_t numOfCols = QH_GET_NUM_OF_COLS(pTsdbReadHandle); assert(numOfCols <= TSDB_MAX_COLUMNS); SArray* pIdList = taosArrayInit(numOfCols, sizeof(int16_t)); for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i); + SColumnInfoData* pCol = taosArrayGet(pTsdbReadHandle->pColumns, i); taosArrayPush(pIdList, &pCol->info.colId); } return pIdList; } -static SArray* getDefaultLoadColumns(STsdbQueryHandle* pQueryHandle, bool loadTS) { - SArray* pLocalIdList = getColumnIdList(pQueryHandle); +static SArray* getDefaultLoadColumns(STsdbReadHandle* pTsdbReadHandle, bool loadTS) { + SArray* pLocalIdList = getColumnIdList(pTsdbReadHandle); // check if the primary time stamp column needs to load int16_t colId = *(int16_t*)taosArrayGet(pLocalIdList, 0); @@ -207,63 +212,63 @@ static SArray* getDefaultLoadColumns(STsdbQueryHandle* pQueryHandle, bool loadTS return pLocalIdList; } -static void tsdbMayTakeMemSnapshot(STsdbQueryHandle* pQueryHandle, SArray* psTable) { - assert(pQueryHandle != NULL && pQueryHandle->pMemRef != NULL); - - SMemRef* pMemRef = pQueryHandle->pMemRef; - if (pQueryHandle->pMemRef->ref++ == 0) { - tsdbTakeMemSnapshot(pQueryHandle->pTsdb, &(pMemRef->snapshot), psTable); - } - - taosArrayDestroy(psTable); +static void tsdbMayTakeMemSnapshot(STsdbReadHandle* pTsdbReadHandle, SArray* psTable) { +// assert(pTsdbReadHandle != NULL && pTsdbReadHandle->pMemRef != NULL); +// +// STsdbMemTable* pMemRef = pTsdbReadHandle->pMemRef; +// if (pTsdbReadHandle->pMemRef->ref++ == 0) { +// tsdbTakeMemSnapshot(pTsdbReadHandle->pTsdb, &(pMemRef->snapshot), psTable); +// } +// +// taosArrayDestroy(psTable); } -static void tsdbMayUnTakeMemSnapshot(STsdbQueryHandle* pQueryHandle) { - assert(pQueryHandle != NULL); - SMemRef* pMemRef = pQueryHandle->pMemRef; - if (pMemRef == NULL) { // it has been freed - return; - } - - if (--pMemRef->ref == 0) { - tsdbUnTakeMemSnapShot(pQueryHandle->pTsdb, &(pMemRef->snapshot)); - } - - pQueryHandle->pMemRef = NULL; +static void tsdbMayUnTakeMemSnapshot(STsdbReadHandle* pTsdbReadHandle) { +// assert(pTsdbReadHandle != NULL); +// STsdbMemTable* pMemRef = pTsdbReadHandle->pMemRef; +// if (pMemRef == NULL) { // it has been freed +// return; +// } +// +// if (--pMemRef->ref == 0) { +// tsdbUnTakeMemSnapShot(pTsdbReadHandle->pTsdb, &(pMemRef->snapshot)); +// } +// +// pTsdbReadHandle->pMemRef = NULL; } -int64_t tsdbGetNumOfRowsInMemTable(TsdbQueryHandleT* pHandle) { - STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; +//int64_t tsdbGetNumOfRowsInMemTable(tsdbReadHandleT* pHandle) { +// STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) pHandle; +// +// int64_t rows = 0; +// STsdbMemTable* pMemTable = pTsdbReadHandle->pMemTable; +// if (pMemTable == NULL) { return rows; } +// +//// STableData* pMem = NULL; +//// STableData* pIMem = NULL; +// +//// SMemTable* pMemT = pMemRef->snapshot.mem; +//// SMemTable* pIMemT = pMemRef->snapshot.imem; +// +// size_t size = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); +// for (int32_t i = 0; i < size; ++i) { +// STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, i); +// +//// if (pMemT && pCheckInfo->tableId < pMemT->maxTables) { +//// pMem = pMemT->tData[pCheckInfo->tableId]; +//// rows += (pMem && pMem->uid == pCheckInfo->tableId) ? pMem->numOfRows : 0; +//// } +//// if (pIMemT && pCheckInfo->tableId < pIMemT->maxTables) { +//// pIMem = pIMemT->tData[pCheckInfo->tableId]; +//// rows += (pIMem && pIMem->uid == pCheckInfo->tableId) ? pIMem->numOfRows : 0; +//// } +// } +// return rows; +//} - int64_t rows = 0; - SMemRef* pMemRef = pQueryHandle->pMemRef; - if (pMemRef == NULL) { return rows; } - - STableData* pMem = NULL; - STableData* pIMem = NULL; - - SMemTable* pMemT = pMemRef->snapshot.mem; - SMemTable* pIMemT = pMemRef->snapshot.imem; - - size_t size = taosArrayGetSize(pQueryHandle->pTableCheckInfo); - for (int32_t i = 0; i < size; ++i) { - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); - - if (pMemT && pCheckInfo->tableId.tid < pMemT->maxTables) { - pMem = pMemT->tData[pCheckInfo->tableId.tid]; - rows += (pMem && pMem->uid == pCheckInfo->tableId.uid) ? pMem->numOfRows : 0; - } - if (pIMemT && pCheckInfo->tableId.tid < pIMemT->maxTables) { - pIMem = pIMemT->tData[pCheckInfo->tableId.tid]; - rows += (pIMem && pIMem->uid == pCheckInfo->tableId.uid) ? pIMem->numOfRows : 0; - } - } - return rows; -} - -static SArray* createCheckInfoFromTableGroup(STsdbQueryHandle* pQueryHandle, STableGroupInfo* pGroupList, STsdbMeta* pMeta, SArray** psTable) { +static SArray* createCheckInfoFromTableGroup(STsdbReadHandle* pTsdbReadHandle, STableGroupInfo* pGroupList, SArray** psTable) { size_t sizeOfGroup = taosArrayGetSize(pGroupList->pGroupList); - assert(sizeOfGroup >= 1 && pMeta != NULL); + assert(sizeOfGroup >= 1); // allocate buffer in order to load data blocks from file SArray* pTableCheckInfo = taosArrayInit(pGroupList->numOfTables, sizeof(STableCheckInfo)); @@ -288,32 +293,28 @@ static SArray* createCheckInfoFromTableGroup(STsdbQueryHandle* pQueryHandle, STa STableKeyInfo* pKeyInfo = (STableKeyInfo*) taosArrayGet(group, j); STableCheckInfo info = { .lastKey = pKeyInfo->lastKey, .pTableObj = pKeyInfo->pTable }; - assert(info.pTableObj != NULL && (info.pTableObj->type == TSDB_NORMAL_TABLE || - info.pTableObj->type == TSDB_CHILD_TABLE || info.pTableObj->type == TSDB_STREAM_TABLE)); +// assert(info.pTableObj != NULL && (info.pTableObj->type == TSDB_NORMAL_TABLE || +// info.pTableObj->type == TSDB_CHILD_TABLE || info.pTableObj->type == TSDB_STREAM_TABLE)); - info.tableId.tid = info.pTableObj->tableId.tid; - info.tableId.uid = info.pTableObj->tableId.uid; + info.tableId = pKeyInfo->uid; - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { - if (info.lastKey == INT64_MIN || info.lastKey < pQueryHandle->window.skey) { - info.lastKey = pQueryHandle->window.skey; + if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { + if (info.lastKey == INT64_MIN || info.lastKey < pTsdbReadHandle->window.skey) { + info.lastKey = pTsdbReadHandle->window.skey; } - assert(info.lastKey >= pQueryHandle->window.skey && info.lastKey <= pQueryHandle->window.ekey); + assert(info.lastKey >= pTsdbReadHandle->window.skey && info.lastKey <= pTsdbReadHandle->window.ekey); } else { - assert(info.lastKey >= pQueryHandle->window.ekey && info.lastKey <= pQueryHandle->window.skey); + assert(info.lastKey >= pTsdbReadHandle->window.ekey && info.lastKey <= pTsdbReadHandle->window.skey); } taosArrayPush(pTableCheckInfo, &info); - tsdbDebug("%p check table uid:%"PRId64", tid:%d from lastKey:%"PRId64" 0x%"PRIx64, pQueryHandle, info.tableId.uid, - info.tableId.tid, info.lastKey, pQueryHandle->qId); + tsdbDebug("%p check table uid:%"PRId64" from lastKey:%"PRId64" 0x%"PRIx64, pTsdbReadHandle, info.tableId, info.lastKey, pTsdbReadHandle->qId); } } - taosArraySort(pTableCheckInfo, tsdbCheckInfoCompar); - +// taosArraySort(pTableCheckInfo, tsdbCheckInfoCompar); size_t gsize = taosArrayGetSize(pTableCheckInfo); - for (int32_t i = 0; i < gsize; ++i) { STableCheckInfo* pInfo = (STableCheckInfo*) taosArrayGet(pTableCheckInfo, i); taosArrayPush(pTable, &pInfo->pTableObj); @@ -323,22 +324,22 @@ static SArray* createCheckInfoFromTableGroup(STsdbQueryHandle* pQueryHandle, STa return pTableCheckInfo; } -static void resetCheckInfo(STsdbQueryHandle* pQueryHandle) { - size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); +static void resetCheckInfo(STsdbReadHandle* pTsdbReadHandle) { + size_t numOfTables = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); assert(numOfTables >= 1); // todo apply the lastkey of table check to avoid to load header file for (int32_t i = 0; i < numOfTables; ++i) { - STableCheckInfo* pCheckInfo = (STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, i); - pCheckInfo->lastKey = pQueryHandle->window.skey; + STableCheckInfo* pCheckInfo = (STableCheckInfo*) taosArrayGet(pTsdbReadHandle->pTableCheckInfo, i); + pCheckInfo->lastKey = pTsdbReadHandle->window.skey; pCheckInfo->iter = tSkipListDestroyIter(pCheckInfo->iter); pCheckInfo->iiter = tSkipListDestroyIter(pCheckInfo->iiter); pCheckInfo->initBuf = false; - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { - assert(pCheckInfo->lastKey >= pQueryHandle->window.skey); + if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { + assert(pCheckInfo->lastKey >= pTsdbReadHandle->window.skey); } else { - assert(pCheckInfo->lastKey <= pQueryHandle->window.skey); + assert(pCheckInfo->lastKey <= pTsdbReadHandle->window.skey); } } } @@ -358,38 +359,38 @@ static SArray* createCheckInfoFromCheckInfo(STableCheckInfo* pCheckInfo, TSKEY s return pNew; } -static bool emptyQueryTimewindow(STsdbQueryHandle* pQueryHandle) { - assert(pQueryHandle != NULL); +static bool emptyQueryTimewindow(STsdbReadHandle* pTsdbReadHandle) { + assert(pTsdbReadHandle != NULL); - STimeWindow* w = &pQueryHandle->window; - bool asc = ASCENDING_TRAVERSE(pQueryHandle->order); + STimeWindow* w = &pTsdbReadHandle->window; + bool asc = ASCENDING_TRAVERSE(pTsdbReadHandle->order); return ((asc && w->skey > w->ekey) || (!asc && w->ekey > w->skey)); } // Update the query time window according to the data time to live(TTL) information, in order to avoid to return // the expired data to client, even it is queried already. -static int64_t getEarliestValidTimestamp(STsdbRepo* pTsdb) { +static int64_t getEarliestValidTimestamp(STsdb* pTsdb) { STsdbCfg* pCfg = &pTsdb->config; int64_t now = taosGetTimestamp(pCfg->precision); return now - (tsTickPerDay[pCfg->precision] * pCfg->keep) + 1; // needs to add one tick } -static void setQueryTimewindow(STsdbQueryHandle* pQueryHandle, STsdbQueryCond* pCond) { - pQueryHandle->window = pCond->twindow; +static void setQueryTimewindow(STsdbReadHandle* pTsdbReadHandle, STsdbQueryCond* pCond) { + pTsdbReadHandle->window = pCond->twindow; bool updateTs = false; - int64_t startTs = getEarliestValidTimestamp(pQueryHandle->pTsdb); - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { - if (startTs > pQueryHandle->window.skey) { - pQueryHandle->window.skey = startTs; + int64_t startTs = getEarliestValidTimestamp(pTsdbReadHandle->pTsdb); + if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { + if (startTs > pTsdbReadHandle->window.skey) { + pTsdbReadHandle->window.skey = startTs; pCond->twindow.skey = startTs; updateTs = true; } } else { - if (startTs > pQueryHandle->window.ekey) { - pQueryHandle->window.ekey = startTs; + if (startTs > pTsdbReadHandle->window.ekey) { + pTsdbReadHandle->window.ekey = startTs; pCond->twindow.ekey = startTs; updateTs = true; } @@ -397,51 +398,50 @@ static void setQueryTimewindow(STsdbQueryHandle* pQueryHandle, STsdbQueryCond* p if (updateTs) { tsdbDebug("%p update the query time window, old:%" PRId64 " - %" PRId64 ", new:%" PRId64 " - %" PRId64 - ", 0x%" PRIx64, pQueryHandle, pCond->twindow.skey, pCond->twindow.ekey, pQueryHandle->window.skey, - pQueryHandle->window.ekey, pQueryHandle->qId); + ", 0x%" PRIx64, pTsdbReadHandle, pCond->twindow.skey, pCond->twindow.ekey, pTsdbReadHandle->window.skey, + pTsdbReadHandle->window.ekey, pTsdbReadHandle->qId); } } -static STsdbQueryHandle* tsdbQueryTablesImpl(STsdbRepo* tsdb, STsdbQueryCond* pCond, uint64_t qId, SMemRef* pMemRef) { - STsdbQueryHandle* pQueryHandle = calloc(1, sizeof(STsdbQueryHandle)); - if (pQueryHandle == NULL) { +static STsdbReadHandle* tsdbQueryTablesImpl(STsdb* tsdb, STsdbQueryCond* pCond, uint64_t qId, STsdbMemTable* pMemRef) { + STsdbReadHandle* pReadHandle = calloc(1, sizeof(STsdbReadHandle)); + if (pReadHandle == NULL) { goto _end; } - pQueryHandle->order = pCond->order; - pQueryHandle->pTsdb = tsdb; - pQueryHandle->type = TSDB_QUERY_TYPE_ALL; - pQueryHandle->cur.fid = INT32_MIN; - pQueryHandle->cur.win = TSWINDOW_INITIALIZER; - pQueryHandle->checkFiles = true; - pQueryHandle->activeIndex = 0; // current active table index - pQueryHandle->qId = qId; - pQueryHandle->allocSize = 0; - pQueryHandle->locateStart = false; - pQueryHandle->pMemRef = pMemRef; - pQueryHandle->loadType = pCond->type; + pReadHandle->order = pCond->order; + pReadHandle->pTsdb = tsdb; + pReadHandle->type = TSDB_QUERY_TYPE_ALL; + pReadHandle->cur.fid = INT32_MIN; + pReadHandle->cur.win = TSWINDOW_INITIALIZER; + pReadHandle->checkFiles = true; + pReadHandle->activeIndex = 0; // current active table index + pReadHandle->qId = qId; + pReadHandle->allocSize = 0; + pReadHandle->locateStart = false; + pReadHandle->loadType = pCond->type; - pQueryHandle->outputCapacity = ((STsdbRepo*)tsdb)->config.maxRowsPerFileBlock; - pQueryHandle->loadExternalRow = pCond->loadExternalRows; - pQueryHandle->currentLoadExternalRows = pCond->loadExternalRows; + pReadHandle->outputCapacity = 4096;//((STsdb*)tsdb)->config.maxRowsPerFileBlock; + pReadHandle->loadExternalRow = pCond->loadExternalRows; + pReadHandle->currentLoadExternalRows = pCond->loadExternalRows; - if (tsdbInitReadH(&pQueryHandle->rhelper, (STsdbRepo*)tsdb) != 0) { + if (tsdbInitReadH(&pReadHandle->rhelper, (STsdb*)tsdb) != 0) { goto _end; } - assert(pCond != NULL && pMemRef != NULL); - setQueryTimewindow(pQueryHandle, pCond); + assert(pCond != NULL); + setQueryTimewindow(pReadHandle, pCond); if (pCond->numOfCols > 0) { // allocate buffer in order to load data blocks from file - pQueryHandle->statis = calloc(pCond->numOfCols, sizeof(SDataStatis)); - if (pQueryHandle->statis == NULL) { + pReadHandle->statis = calloc(pCond->numOfCols, sizeof(SDataStatis)); + if (pReadHandle->statis == NULL) { goto _end; } // todo: use list instead of array? - pQueryHandle->pColumns = taosArrayInit(pCond->numOfCols, sizeof(SColumnInfoData)); - if (pQueryHandle->pColumns == NULL) { + pReadHandle->pColumns = taosArrayInit(pCond->numOfCols, sizeof(SColumnInfoData)); + if (pReadHandle->pColumns == NULL) { goto _end; } @@ -449,147 +449,144 @@ static STsdbQueryHandle* tsdbQueryTablesImpl(STsdbRepo* tsdb, STsdbQueryCond* pC SColumnInfoData colInfo = {{0}, 0}; colInfo.info = pCond->colList[i]; - colInfo.pData = calloc(1, EXTRA_BYTES + pQueryHandle->outputCapacity * pCond->colList[i].bytes); + colInfo.pData = calloc(1, EXTRA_BYTES + pReadHandle->outputCapacity * pCond->colList[i].bytes); if (colInfo.pData == NULL) { goto _end; } - taosArrayPush(pQueryHandle->pColumns, &colInfo); - pQueryHandle->statis[i].colId = colInfo.info.colId; + taosArrayPush(pReadHandle->pColumns, &colInfo); + pReadHandle->statis[i].colId = colInfo.info.colId; } - pQueryHandle->defaultLoadColumn = getDefaultLoadColumns(pQueryHandle, true); + pReadHandle->defaultLoadColumn = getDefaultLoadColumns(pReadHandle, true); } - STsdbMeta* pMeta = tsdbGetMeta(tsdb); - assert(pMeta != NULL); +// STsdbMeta* pMeta = NULL;//tsdbGetMeta(tsdb); +// assert(pMeta != NULL); - pQueryHandle->pDataCols = tdNewDataCols(pMeta->maxCols, pQueryHandle->pTsdb->config.maxRowsPerFileBlock); - if (pQueryHandle->pDataCols == NULL) { - tsdbError("%p failed to malloc buf for pDataCols, %"PRIu64, pQueryHandle, pQueryHandle->qId); + pReadHandle->pDataCols = tdNewDataCols(1000, pReadHandle->pTsdb->config.maxRowsPerFileBlock); + if (pReadHandle->pDataCols == NULL) { + tsdbError("%p failed to malloc buf for pDataCols, %"PRIu64, pReadHandle, pReadHandle->qId); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _end; } - tsdbInitDataBlockLoadInfo(&pQueryHandle->dataBlockLoadInfo); - tsdbInitCompBlockLoadInfo(&pQueryHandle->compBlockLoadInfo); + tsdbInitDataBlockLoadInfo(&pReadHandle->dataBlockLoadInfo); + tsdbInitCompBlockLoadInfo(&pReadHandle->compBlockLoadInfo); - return (TsdbQueryHandleT) pQueryHandle; + return (tsdbReadHandleT)pReadHandle; _end: - tsdbCleanupQueryHandle(pQueryHandle); +// tsdbCleanupQueryHandle(pTsdbReadHandle); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return NULL; } -TsdbQueryHandleT* tsdbQueryTables(STsdbRepo* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, uint64_t qId, SMemRef* pRef) { - STsdbQueryHandle* pQueryHandle = tsdbQueryTablesImpl(tsdb, pCond, qId, pRef); - if (pQueryHandle == NULL) { +tsdbReadHandleT* tsdbQueryTables(STsdb* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, uint64_t qId, void* pRef) { + STsdbReadHandle* pTsdbReadHandle = tsdbQueryTablesImpl(tsdb, pCond, qId, pRef); + if (pTsdbReadHandle == NULL) { return NULL; } - if (emptyQueryTimewindow(pQueryHandle)) { - return (TsdbQueryHandleT*) pQueryHandle; + if (emptyQueryTimewindow(pTsdbReadHandle)) { + return (tsdbReadHandleT*) pTsdbReadHandle; } - STsdbMeta* pMeta = tsdbGetMeta(tsdb); - assert(pMeta != NULL); - SArray* psTable = NULL; // todo apply the lastkey of table check to avoid to load header file - pQueryHandle->pTableCheckInfo = createCheckInfoFromTableGroup(pQueryHandle, groupList, pMeta, &psTable); - if (pQueryHandle->pTableCheckInfo == NULL) { - tsdbCleanupQueryHandle(pQueryHandle); + pTsdbReadHandle->pTableCheckInfo = createCheckInfoFromTableGroup(pTsdbReadHandle, groupList, &psTable); + if (pTsdbReadHandle->pTableCheckInfo == NULL) { +// tsdbCleanupQueryHandle(pTsdbReadHandle); taosArrayDestroy(psTable); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return NULL; } - tsdbMayTakeMemSnapshot(pQueryHandle, psTable); +// tsdbMayTakeMemSnapshot(pTsdbReadHandle, psTable); - tsdbDebug("%p total numOfTable:%" PRIzu " in query, 0x%"PRIx64, pQueryHandle, taosArrayGetSize(pQueryHandle->pTableCheckInfo), pQueryHandle->qId); - return (TsdbQueryHandleT) pQueryHandle; + tsdbDebug("%p total numOfTable:%" PRIzu " in query, 0x%"PRIx64, pTsdbReadHandle, taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo), pTsdbReadHandle->qId); + return (tsdbReadHandleT) pTsdbReadHandle; } -void tsdbResetQueryHandle(TsdbQueryHandleT queryHandle, STsdbQueryCond *pCond) { - STsdbQueryHandle* pQueryHandle = queryHandle; +void tsdbResetQueryHandle(tsdbReadHandleT queryHandle, STsdbQueryCond *pCond) { + STsdbReadHandle* pTsdbReadHandle = queryHandle; - if (emptyQueryTimewindow(pQueryHandle)) { - if (pCond->order != pQueryHandle->order) { - pQueryHandle->order = pCond->order; - SWAP(pQueryHandle->window.skey, pQueryHandle->window.ekey, int64_t); + if (emptyQueryTimewindow(pTsdbReadHandle)) { + if (pCond->order != pTsdbReadHandle->order) { + pTsdbReadHandle->order = pCond->order; + SWAP(pTsdbReadHandle->window.skey, pTsdbReadHandle->window.ekey, int64_t); } return; } - pQueryHandle->order = pCond->order; - pQueryHandle->window = pCond->twindow; - pQueryHandle->type = TSDB_QUERY_TYPE_ALL; - pQueryHandle->cur.fid = -1; - pQueryHandle->cur.win = TSWINDOW_INITIALIZER; - pQueryHandle->checkFiles = true; - pQueryHandle->activeIndex = 0; // current active table index - pQueryHandle->locateStart = false; - pQueryHandle->loadExternalRow = pCond->loadExternalRows; + pTsdbReadHandle->order = pCond->order; + pTsdbReadHandle->window = pCond->twindow; + pTsdbReadHandle->type = TSDB_QUERY_TYPE_ALL; + pTsdbReadHandle->cur.fid = -1; + pTsdbReadHandle->cur.win = TSWINDOW_INITIALIZER; + pTsdbReadHandle->checkFiles = true; + pTsdbReadHandle->activeIndex = 0; // current active table index + pTsdbReadHandle->locateStart = false; + pTsdbReadHandle->loadExternalRow = pCond->loadExternalRows; if (ASCENDING_TRAVERSE(pCond->order)) { - assert(pQueryHandle->window.skey <= pQueryHandle->window.ekey); + assert(pTsdbReadHandle->window.skey <= pTsdbReadHandle->window.ekey); } else { - assert(pQueryHandle->window.skey >= pQueryHandle->window.ekey); + assert(pTsdbReadHandle->window.skey >= pTsdbReadHandle->window.ekey); } // allocate buffer in order to load data blocks from file - memset(pQueryHandle->statis, 0, sizeof(SDataStatis)); + memset(pTsdbReadHandle->statis, 0, sizeof(SDataStatis)); - tsdbInitDataBlockLoadInfo(&pQueryHandle->dataBlockLoadInfo); - tsdbInitCompBlockLoadInfo(&pQueryHandle->compBlockLoadInfo); + tsdbInitDataBlockLoadInfo(&pTsdbReadHandle->dataBlockLoadInfo); + tsdbInitCompBlockLoadInfo(&pTsdbReadHandle->compBlockLoadInfo); - resetCheckInfo(pQueryHandle); + resetCheckInfo(pTsdbReadHandle); } -void tsdbResetQueryHandleForNewTable(TsdbQueryHandleT queryHandle, STsdbQueryCond *pCond, STableGroupInfo* groupList) { - STsdbQueryHandle* pQueryHandle = queryHandle; +void tsdbResetQueryHandleForNewTable(tsdbReadHandleT queryHandle, STsdbQueryCond *pCond, STableGroupInfo* groupList) { + STsdbReadHandle* pTsdbReadHandle = queryHandle; - pQueryHandle->order = pCond->order; - pQueryHandle->window = pCond->twindow; - pQueryHandle->type = TSDB_QUERY_TYPE_ALL; - pQueryHandle->cur.fid = -1; - pQueryHandle->cur.win = TSWINDOW_INITIALIZER; - pQueryHandle->checkFiles = true; - pQueryHandle->activeIndex = 0; // current active table index - pQueryHandle->locateStart = false; - pQueryHandle->loadExternalRow = pCond->loadExternalRows; + pTsdbReadHandle->order = pCond->order; + pTsdbReadHandle->window = pCond->twindow; + pTsdbReadHandle->type = TSDB_QUERY_TYPE_ALL; + pTsdbReadHandle->cur.fid = -1; + pTsdbReadHandle->cur.win = TSWINDOW_INITIALIZER; + pTsdbReadHandle->checkFiles = true; + pTsdbReadHandle->activeIndex = 0; // current active table index + pTsdbReadHandle->locateStart = false; + pTsdbReadHandle->loadExternalRow = pCond->loadExternalRows; if (ASCENDING_TRAVERSE(pCond->order)) { - assert(pQueryHandle->window.skey <= pQueryHandle->window.ekey); + assert(pTsdbReadHandle->window.skey <= pTsdbReadHandle->window.ekey); } else { - assert(pQueryHandle->window.skey >= pQueryHandle->window.ekey); + assert(pTsdbReadHandle->window.skey >= pTsdbReadHandle->window.ekey); } // allocate buffer in order to load data blocks from file - memset(pQueryHandle->statis, 0, sizeof(SDataStatis)); + memset(pTsdbReadHandle->statis, 0, sizeof(SDataStatis)); - tsdbInitDataBlockLoadInfo(&pQueryHandle->dataBlockLoadInfo); - tsdbInitCompBlockLoadInfo(&pQueryHandle->compBlockLoadInfo); + tsdbInitDataBlockLoadInfo(&pTsdbReadHandle->dataBlockLoadInfo); + tsdbInitCompBlockLoadInfo(&pTsdbReadHandle->compBlockLoadInfo); SArray* pTable = NULL; - STsdbMeta* pMeta = tsdbGetMeta(pQueryHandle->pTsdb); +// STsdbMeta* pMeta = tsdbGetMeta(pTsdbReadHandle->pTsdb); - pQueryHandle->pTableCheckInfo = destroyTableCheckInfo(pQueryHandle->pTableCheckInfo); +// pTsdbReadHandle->pTableCheckInfo = destroyTableCheckInfo(pTsdbReadHandle->pTableCheckInfo); - pQueryHandle->pTableCheckInfo = createCheckInfoFromTableGroup(pQueryHandle, groupList, pMeta, &pTable); - if (pQueryHandle->pTableCheckInfo == NULL) { - tsdbCleanupQueryHandle(pQueryHandle); + pTsdbReadHandle->pTableCheckInfo = NULL;//createCheckInfoFromTableGroup(pTsdbReadHandle, groupList, pMeta, &pTable); + if (pTsdbReadHandle->pTableCheckInfo == NULL) { +// tsdbCleanupQueryHandle(pTsdbReadHandle); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; } - pQueryHandle->prev = doFreeColumnInfoData(pQueryHandle->prev); - pQueryHandle->next = doFreeColumnInfoData(pQueryHandle->next); +// pTsdbReadHandle->prev = doFreeColumnInfoData(pTsdbReadHandle->prev); +// pTsdbReadHandle->next = doFreeColumnInfoData(pTsdbReadHandle->next); } -TsdbQueryHandleT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, SMemRef* pMemRef) { +tsdbReadHandleT tsdbQueryLastRow(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, STsdbMemTable* pMemRef) { pCond->twindow = updateLastrowForEachGroup(groupList); // no qualified table @@ -597,56 +594,56 @@ TsdbQueryHandleT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STable return NULL; } - STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qId, pMemRef); - if (pQueryHandle == NULL) { + STsdbReadHandle *pTsdbReadHandle = (STsdbReadHandle*) tsdbQueryTables(tsdb, pCond, groupList, qId, pMemRef); + if (pTsdbReadHandle == NULL) { return NULL; } - int32_t code = checkForCachedLastRow(pQueryHandle, groupList); + int32_t code = checkForCachedLastRow(pTsdbReadHandle, groupList); if (code != TSDB_CODE_SUCCESS) { // set the numOfTables to be 0 terrno = code; return NULL; } assert(pCond->order == TSDB_ORDER_ASC && pCond->twindow.skey <= pCond->twindow.ekey); - if (pQueryHandle->cachelastrow) { - pQueryHandle->type = TSDB_QUERY_TYPE_LAST; + if (pTsdbReadHandle->cachelastrow) { + pTsdbReadHandle->type = TSDB_QUERY_TYPE_LAST; } - return pQueryHandle; + return pTsdbReadHandle; } - -TsdbQueryHandleT tsdbQueryCacheLast(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, SMemRef* pMemRef) { - STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qId, pMemRef); - if (pQueryHandle == NULL) { +#if 0 +tsdbReadHandleT tsdbQueryCacheLast(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, STsdbMemTable* pMemRef) { + STsdbReadHandle *pTsdbReadHandle = (STsdbReadHandle*) tsdbQueryTables(tsdb, pCond, groupList, qId, pMemRef); + if (pTsdbReadHandle == NULL) { return NULL; } - int32_t code = checkForCachedLast(pQueryHandle); + int32_t code = checkForCachedLast(pTsdbReadHandle); if (code != TSDB_CODE_SUCCESS) { // set the numOfTables to be 0 terrno = code; return NULL; } - if (pQueryHandle->cachelastrow) { - pQueryHandle->type = TSDB_QUERY_TYPE_LAST; + if (pTsdbReadHandle->cachelastrow) { + pTsdbReadHandle->type = TSDB_QUERY_TYPE_LAST; } - return pQueryHandle; + return pTsdbReadHandle; } - -SArray* tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle) { +#endif +SArray* tsdbGetQueriedTableList(tsdbReadHandleT *pHandle) { assert(pHandle != NULL); - STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) pHandle; + STsdbReadHandle *pTsdbReadHandle = (STsdbReadHandle*) pHandle; - size_t size = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + size_t size = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); SArray* res = taosArrayInit(size, POINTER_BYTES); for(int32_t i = 0; i < size; ++i) { - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); + STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, i); taosArrayPush(res, &pCheckInfo->pTableObj); } @@ -668,11 +665,11 @@ static STableGroupInfo* trimTableGroup(STimeWindow* window, STableGroupInfo* pGr SArray* px = taosArrayInit(4, sizeof(STableKeyInfo)); for (int32_t j = 0; j < numOfTables; ++j) { STableKeyInfo* pInfo = (STableKeyInfo*)taosArrayGet(oneGroup, j); - if (window->skey <= pInfo->lastKey && ((STable*)pInfo->pTable)->lastKey != TSKEY_INITIAL_VAL) { - taosArrayPush(px, pInfo); - pNew->numOfTables += 1; - break; - } +// if (window->skey <= pInfo->lastKey && ((STable*)pInfo->pTable)->lastKey != TSKEY_INITIAL_VAL) { +// taosArrayPush(px, pInfo); +// pNew->numOfTables += 1; +// break; +// } } // there are no data in this group @@ -686,7 +683,7 @@ static STableGroupInfo* trimTableGroup(STimeWindow* window, STableGroupInfo* pGr return pNew; } -TsdbQueryHandleT tsdbQueryRowsInExternalWindow(STsdbRepo *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, uint64_t qId, SMemRef* pRef) { +tsdbReadHandleT tsdbQueryRowsInExternalWindow(STsdb *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, uint64_t qId, STsdbMemTable* pRef) { STableGroupInfo* pNew = trimTableGroup(&pCond->twindow, groupList); if (pNew->numOfTables == 0) { @@ -701,17 +698,14 @@ TsdbQueryHandleT tsdbQueryRowsInExternalWindow(STsdbRepo *tsdb, STsdbQueryCond* } } - STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, pNew, qId, pRef); - pQueryHandle->loadExternalRow = true; - pQueryHandle->currentLoadExternalRows = true; + STsdbReadHandle *pTsdbReadHandle = (STsdbReadHandle*) tsdbQueryTables(tsdb, pCond, pNew, qId, pRef); + pTsdbReadHandle->loadExternalRow = true; + pTsdbReadHandle->currentLoadExternalRows = true; - return pQueryHandle; + return pTsdbReadHandle; } -static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCheckInfo) { - STable* pTable = pCheckInfo->pTableObj; - assert(pTable != NULL); - +static bool initTableMemIterator(STsdbReadHandle* pHandle, STableCheckInfo* pCheckInfo) { if (pCheckInfo->initBuf) { return true; } @@ -720,33 +714,29 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh int32_t order = pHandle->order; // no data in buffer, abort - if (pHandle->pMemRef->snapshot.mem == NULL && pHandle->pMemRef->snapshot.imem == NULL) { - return false; - } +// if (pHandle->pMemTable->snapshot.mem == NULL && pHandle->pMemTable->snapshot.imem == NULL) { +// return false; +// } +// +// assert(pCheckInfo->iter == NULL && pCheckInfo->iiter == NULL); +// + STbData** pMem = NULL; + STbData** pIMem = NULL; - assert(pCheckInfo->iter == NULL && pCheckInfo->iiter == NULL); - - STableData* pMem = NULL; - STableData* pIMem = NULL; - - SMemTable* pMemT = pHandle->pMemRef->snapshot.mem; - SMemTable* pIMemT = pHandle->pMemRef->snapshot.imem; - - if (pMemT && pCheckInfo->tableId.tid < pMemT->maxTables) { - pMem = pMemT->tData[pCheckInfo->tableId.tid]; - if (pMem != NULL && pMem->uid == pCheckInfo->tableId.uid) { // check uid - TKEY tLastKey = keyToTkey(pCheckInfo->lastKey); + TKEY tLastKey = 0; /// keyToTkey(pCheckInfo->lastKey); + if (pHandle->pTsdb->mem != NULL) { + pMem = taosHashGet(pHandle->pTsdb->mem->pHashIdx, &pCheckInfo->tableId, sizeof(pCheckInfo->tableId)); + if (pMem != NULL) { pCheckInfo->iter = - tSkipListCreateIterFromVal(pMem->pData, (const char*)&tLastKey, TSDB_DATA_TYPE_TIMESTAMP, order); + tSkipListCreateIterFromVal((*pMem)->pData, (const char*)&tLastKey, TSDB_DATA_TYPE_TIMESTAMP, order); } } - if (pIMemT && pCheckInfo->tableId.tid < pIMemT->maxTables) { - pIMem = pIMemT->tData[pCheckInfo->tableId.tid]; - if (pIMem != NULL && pIMem->uid == pCheckInfo->tableId.uid) { // check uid - TKEY tLastKey = keyToTkey(pCheckInfo->lastKey); + if (pHandle->pTsdb->imem != NULL) { + pIMem = taosHashGet(pHandle->pTsdb->imem->pHashIdx, &pCheckInfo->tableId, sizeof(pCheckInfo->tableId)); + if (pIMem != NULL) { pCheckInfo->iiter = - tSkipListCreateIterFromVal(pIMem->pData, (const char*)&tLastKey, TSDB_DATA_TYPE_TIMESTAMP, order); + tSkipListCreateIterFromVal((*pIMem)->pData, (const char*)&tLastKey, TSDB_DATA_TYPE_TIMESTAMP, order); } } @@ -767,10 +757,9 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh SMemRow row = (SMemRow)SL_GET_NODE_DATA(node); TSKEY key = memRowKey(row); // first timestamp in buffer - tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in mem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 + tsdbDebug("%p uid:%" PRId64 ", check data in mem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 "-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", 0x%"PRIx64, - pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, key, order, pMem->keyFirst, pMem->keyLast, - pCheckInfo->lastKey, pMem->numOfRows, pHandle->qId); + pHandle, pCheckInfo->tableId, key, order, (*pMem)->keyMin, (*pMem)->keyMax, pCheckInfo->lastKey, (*pMem)->nrows, pHandle->qId); if (ASCENDING_TRAVERSE(order)) { assert(pCheckInfo->lastKey <= key); @@ -779,8 +768,7 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh } } else { - tsdbDebug("%p uid:%"PRId64", tid:%d no data in mem, 0x%"PRIx64, pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, - pHandle->qId); + tsdbDebug("%p uid:%"PRId64", no data in mem, 0x%"PRIx64, pHandle, pCheckInfo->tableId, pHandle->qId); } if (!imemEmpty) { @@ -789,10 +777,9 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh SMemRow row = (SMemRow)SL_GET_NODE_DATA(node); TSKEY key = memRowKey(row); // first timestamp in buffer - tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in imem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 + tsdbDebug("%p uid:%" PRId64 ", check data in imem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 "-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", 0x%"PRIx64, - pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, key, order, pIMem->keyFirst, pIMem->keyLast, - pCheckInfo->lastKey, pIMem->numOfRows, pHandle->qId); + pHandle, pCheckInfo->tableId, key, order, (*pIMem)->keyMin, (*pIMem)->keyMax, pCheckInfo->lastKey, (*pIMem)->nrows, pHandle->qId); if (ASCENDING_TRAVERSE(order)) { assert(pCheckInfo->lastKey <= key); @@ -800,7 +787,7 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh assert(pCheckInfo->lastKey >= key); } } else { - tsdbDebug("%p uid:%"PRId64", tid:%d no data in imem, 0x%"PRIx64, pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, + tsdbDebug("%p uid:%"PRId64", no data in imem, 0x%"PRIx64, pHandle, pCheckInfo->tableId, pHandle->qId); } @@ -973,17 +960,13 @@ static bool moveToNextRowInMem(STableCheckInfo* pCheckInfo) { return hasNext; } -static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) { +static bool hasMoreDataInCache(STsdbReadHandle* pHandle) { STsdbCfg *pCfg = &pHandle->pTsdb->config; size_t size = taosArrayGetSize(pHandle->pTableCheckInfo); assert(pHandle->activeIndex < size && pHandle->activeIndex >= 0 && size >= 1); pHandle->cur.fid = INT32_MIN; STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); - - STable* pTable = pCheckInfo->pTableObj; - assert(pTable != NULL); - if (!pCheckInfo->initBuf) { initTableMemIterator(pHandle, pCheckInfo); } @@ -994,8 +977,8 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) { } pCheckInfo->lastKey = memRowKey(row); // first timestamp in buffer - tsdbDebug("%p uid:%" PRId64", tid:%d check data in buffer from skey:%" PRId64 ", order:%d, 0x%"PRIx64, pHandle, - pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, pCheckInfo->lastKey, pHandle->order, pHandle->qId); + tsdbDebug("%p uid:%" PRId64", check data in buffer from skey:%" PRId64 ", order:%d, 0x%"PRIx64, pHandle, + pCheckInfo->tableId, pCheckInfo->lastKey, pHandle->order, pHandle->qId); // all data in mem are checked already. if ((pCheckInfo->lastKey > pHandle->window.ekey && ASCENDING_TRAVERSE(pHandle->order)) || @@ -1068,21 +1051,21 @@ static int32_t binarySearchForBlock(SBlock* pBlock, int32_t numOfBlocks, TSKEY s return midSlot; } -static int32_t loadBlockInfo(STsdbQueryHandle * pQueryHandle, int32_t index, int32_t* numOfBlocks) { +static int32_t loadBlockInfo(STsdbReadHandle * pTsdbReadHandle, int32_t index, int32_t* numOfBlocks) { int32_t code = 0; - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, index); + STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, index); pCheckInfo->numOfBlocks = 0; - if (tsdbSetReadTable(&pQueryHandle->rhelper, pCheckInfo->pTableObj) != TSDB_CODE_SUCCESS) { + if (tsdbSetReadTable(&pTsdbReadHandle->rhelper, pCheckInfo->pTableObj) != TSDB_CODE_SUCCESS) { code = terrno; return code; } - SBlockIdx* compIndex = pQueryHandle->rhelper.pBlkIdx; + SBlockIdx* compIndex = pTsdbReadHandle->rhelper.pBlkIdx; // no data block in this file, try next file - if (compIndex == NULL || compIndex->uid != pCheckInfo->tableId.uid) { + if (compIndex == NULL || compIndex->uid != pCheckInfo->tableId) { return 0; // no data blocks in the file belongs to pCheckInfo->pTable } @@ -1100,21 +1083,21 @@ static int32_t loadBlockInfo(STsdbQueryHandle * pQueryHandle, int32_t index, int pCheckInfo->compSize = compIndex->len; } - if (tsdbLoadBlockInfo(&(pQueryHandle->rhelper), (void*)(pCheckInfo->pCompInfo)) < 0) { + if (tsdbLoadBlockInfo(&(pTsdbReadHandle->rhelper), (void*)(pCheckInfo->pCompInfo)) < 0) { return terrno; } SBlockInfo* pCompInfo = pCheckInfo->pCompInfo; TSKEY s = TSKEY_INITIAL_VAL, e = TSKEY_INITIAL_VAL; - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { - assert(pCheckInfo->lastKey <= pQueryHandle->window.ekey && pQueryHandle->window.skey <= pQueryHandle->window.ekey); + if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { + assert(pCheckInfo->lastKey <= pTsdbReadHandle->window.ekey && pTsdbReadHandle->window.skey <= pTsdbReadHandle->window.ekey); } else { - assert(pCheckInfo->lastKey >= pQueryHandle->window.ekey && pQueryHandle->window.skey >= pQueryHandle->window.ekey); + assert(pCheckInfo->lastKey >= pTsdbReadHandle->window.ekey && pTsdbReadHandle->window.skey >= pTsdbReadHandle->window.ekey); } - s = MIN(pCheckInfo->lastKey, pQueryHandle->window.ekey); - e = MAX(pCheckInfo->lastKey, pQueryHandle->window.ekey); + s = MIN(pCheckInfo->lastKey, pTsdbReadHandle->window.ekey); + e = MAX(pCheckInfo->lastKey, pTsdbReadHandle->window.ekey); // discard the unqualified data block based on the query time window int32_t start = binarySearchForBlock(pCompInfo->blocks, compIndex->numOfBlocks, s, TSDB_ORDER_ASC); @@ -1139,26 +1122,26 @@ static int32_t loadBlockInfo(STsdbQueryHandle * pQueryHandle, int32_t index, int return 0; } -static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlocks) { +static int32_t getFileCompInfo(STsdbReadHandle* pTsdbReadHandle, int32_t* numOfBlocks) { // load all the comp offset value for all tables in this file int32_t code = TSDB_CODE_SUCCESS; *numOfBlocks = 0; - pQueryHandle->cost.headFileLoad += 1; + pTsdbReadHandle->cost.headFileLoad += 1; int64_t s = taosGetTimestampUs(); size_t numOfTables = 0; - if (pQueryHandle->loadType == BLOCK_LOAD_TABLE_SEQ_ORDER) { - code = loadBlockInfo(pQueryHandle, pQueryHandle->activeIndex, numOfBlocks); - } else if (pQueryHandle->loadType == BLOCK_LOAD_OFFSET_SEQ_ORDER) { - numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + if (pTsdbReadHandle->loadType == BLOCK_LOAD_TABLE_SEQ_ORDER) { + code = loadBlockInfo(pTsdbReadHandle, pTsdbReadHandle->activeIndex, numOfBlocks); + } else if (pTsdbReadHandle->loadType == BLOCK_LOAD_OFFSET_SEQ_ORDER) { + numOfTables = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); for (int32_t i = 0; i < numOfTables; ++i) { - code = loadBlockInfo(pQueryHandle, i, numOfBlocks); + code = loadBlockInfo(pTsdbReadHandle, i, numOfBlocks); if (code != TSDB_CODE_SUCCESS) { int64_t e = taosGetTimestampUs(); - pQueryHandle->cost.headFileLoadTime += (e - s); + pTsdbReadHandle->cost.headFileLoadTime += (e - s); return code; } } @@ -1167,57 +1150,57 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo } int64_t e = taosGetTimestampUs(); - pQueryHandle->cost.headFileLoadTime += (e - s); + pTsdbReadHandle->cost.headFileLoadTime += (e - s); return code; } -static int32_t doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SBlock* pBlock, STableCheckInfo* pCheckInfo, int32_t slotIndex) { +static int32_t doLoadFileDataBlock(STsdbReadHandle* pTsdbReadHandle, SBlock* pBlock, STableCheckInfo* pCheckInfo, int32_t slotIndex) { int64_t st = taosGetTimestampUs(); - STSchema *pSchema = tsdbGetTableSchema(pCheckInfo->pTableObj); - int32_t code = tdInitDataCols(pQueryHandle->pDataCols, pSchema); + STSchema *pSchema = NULL;//tsdbGetTableSchema(pCheckInfo->pTableObj); + int32_t code = tdInitDataCols(pTsdbReadHandle->pDataCols, pSchema); if (code != TSDB_CODE_SUCCESS) { - tsdbError("%p failed to malloc buf for pDataCols, 0x%"PRIx64, pQueryHandle, pQueryHandle->qId); + tsdbError("%p failed to malloc buf for pDataCols, 0x%"PRIx64, pTsdbReadHandle, pTsdbReadHandle->qId); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _error; } - code = tdInitDataCols(pQueryHandle->rhelper.pDCols[0], pSchema); + code = tdInitDataCols(pTsdbReadHandle->rhelper.pDCols[0], pSchema); if (code != TSDB_CODE_SUCCESS) { - tsdbError("%p failed to malloc buf for rhelper.pDataCols[0], 0x%"PRIx64, pQueryHandle, pQueryHandle->qId); + tsdbError("%p failed to malloc buf for rhelper.pDataCols[0], 0x%"PRIx64, pTsdbReadHandle, pTsdbReadHandle->qId); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _error; } - code = tdInitDataCols(pQueryHandle->rhelper.pDCols[1], pSchema); + code = tdInitDataCols(pTsdbReadHandle->rhelper.pDCols[1], pSchema); if (code != TSDB_CODE_SUCCESS) { - tsdbError("%p failed to malloc buf for rhelper.pDataCols[1], 0x%"PRIx64, pQueryHandle, pQueryHandle->qId); + tsdbError("%p failed to malloc buf for rhelper.pDataCols[1], 0x%"PRIx64, pTsdbReadHandle, pTsdbReadHandle->qId); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _error; } - int16_t* colIds = pQueryHandle->defaultLoadColumn->pData; + int16_t* colIds = pTsdbReadHandle->defaultLoadColumn->pData; - int32_t ret = tsdbLoadBlockDataCols(&(pQueryHandle->rhelper), pBlock, pCheckInfo->pCompInfo, colIds, (int)(QH_GET_NUM_OF_COLS(pQueryHandle))); + int32_t ret = tsdbLoadBlockDataCols(&(pTsdbReadHandle->rhelper), pBlock, pCheckInfo->pCompInfo, colIds, (int)(QH_GET_NUM_OF_COLS(pTsdbReadHandle))); if (ret != TSDB_CODE_SUCCESS) { int32_t c = terrno; assert(c != TSDB_CODE_SUCCESS); goto _error; } - SDataBlockLoadInfo* pBlockLoadInfo = &pQueryHandle->dataBlockLoadInfo; + SDataBlockLoadInfo* pBlockLoadInfo = &pTsdbReadHandle->dataBlockLoadInfo; - pBlockLoadInfo->fileGroup = pQueryHandle->pFileGroup; - pBlockLoadInfo->slot = pQueryHandle->cur.slot; - pBlockLoadInfo->tid = pCheckInfo->pTableObj->tableId.tid; + pBlockLoadInfo->fileGroup = pTsdbReadHandle->pFileGroup; + pBlockLoadInfo->slot = pTsdbReadHandle->cur.slot; + pBlockLoadInfo->uid = pCheckInfo->pTableObj->uid; - SDataCols* pCols = pQueryHandle->rhelper.pDCols[0]; + SDataCols* pCols = pTsdbReadHandle->rhelper.pDCols[0]; assert(pCols->numOfRows != 0 && pCols->numOfRows <= pBlock->numOfRows); pBlock->numOfRows = pCols->numOfRows; // Convert from TKEY to TSKEY for primary timestamp column if current block has timestamp before 1970-01-01T00:00:00Z - if(pBlock->keyFirst < 0 && colIds[0] == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + if(pBlock->keyFirst < 0 && colIds[0] == PRIMARYKEY_TIMESTAMP_COL_ID) { int64_t* src = pCols->cols[0].pData; for(int32_t i = 0; i < pBlock->numOfRows; ++i) { src[i] = tdGetKey(src[i]); @@ -1225,60 +1208,60 @@ static int32_t doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SBlock* pBloc } int64_t elapsedTime = (taosGetTimestampUs() - st); - pQueryHandle->cost.blockLoadTime += elapsedTime; + pTsdbReadHandle->cost.blockLoadTime += elapsedTime; tsdbDebug("%p load file block into buffer, index:%d, brange:%"PRId64"-%"PRId64", rows:%d, elapsed time:%"PRId64 " us, 0x%"PRIx64, - pQueryHandle, slotIndex, pBlock->keyFirst, pBlock->keyLast, pBlock->numOfRows, elapsedTime, pQueryHandle->qId); + pTsdbReadHandle, slotIndex, pBlock->keyFirst, pBlock->keyLast, pBlock->numOfRows, elapsedTime, pTsdbReadHandle->qId); return TSDB_CODE_SUCCESS; _error: pBlock->numOfRows = 0; tsdbError("%p error occurs in loading file block, index:%d, brange:%"PRId64"-%"PRId64", rows:%d, 0x%"PRIx64, - pQueryHandle, slotIndex, pBlock->keyFirst, pBlock->keyLast, pBlock->numOfRows, pQueryHandle->qId); + pTsdbReadHandle, slotIndex, pBlock->keyFirst, pBlock->keyLast, pBlock->numOfRows, pTsdbReadHandle->qId); return terrno; } -static int32_t getEndPosInDataBlock(STsdbQueryHandle* pQueryHandle, SDataBlockInfo* pBlockInfo); -static int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, int32_t start, int32_t end); -static void moveDataToFront(STsdbQueryHandle* pQueryHandle, int32_t numOfRows, int32_t numOfCols); -static void doCheckGeneratedBlockRange(STsdbQueryHandle* pQueryHandle); -static void copyAllRemainRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SDataBlockInfo* pBlockInfo, int32_t endPos); +static int32_t getEndPosInDataBlock(STsdbReadHandle* pTsdbReadHandle, SDataBlockInfo* pBlockInfo); +static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t capacity, int32_t numOfRows, int32_t start, int32_t end); +static void moveDataToFront(STsdbReadHandle* pTsdbReadHandle, int32_t numOfRows, int32_t numOfCols); +static void doCheckGeneratedBlockRange(STsdbReadHandle* pTsdbReadHandle); +static void copyAllRemainRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, STableCheckInfo* pCheckInfo, SDataBlockInfo* pBlockInfo, int32_t endPos); -static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SBlock* pBlock, STableCheckInfo* pCheckInfo){ - SQueryFilePos* cur = &pQueryHandle->cur; - STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; +static int32_t handleDataMergeIfNeeded(STsdbReadHandle* pTsdbReadHandle, SBlock* pBlock, STableCheckInfo* pCheckInfo){ + SQueryFilePos* cur = &pTsdbReadHandle->cur; + STsdbCfg* pCfg = &pTsdbReadHandle->pTsdb->config; SDataBlockInfo binfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock); TSKEY key; int32_t code = TSDB_CODE_SUCCESS; - /*bool hasData = */ initTableMemIterator(pQueryHandle, pCheckInfo); + /*bool hasData = */ initTableMemIterator(pTsdbReadHandle, pCheckInfo); assert(cur->pos >= 0 && cur->pos <= binfo.rows); - key = extractFirstTraverseKey(pCheckInfo, pQueryHandle->order, pCfg->update); + key = extractFirstTraverseKey(pCheckInfo, pTsdbReadHandle->order, pCfg->update); if (key != TSKEY_INITIAL_VAL) { - tsdbDebug("%p key in mem:%"PRId64", 0x%"PRIx64, pQueryHandle, key, pQueryHandle->qId); + tsdbDebug("%p key in mem:%"PRId64", 0x%"PRIx64, pTsdbReadHandle, key, pTsdbReadHandle->qId); } else { - tsdbDebug("%p no data in mem, 0x%"PRIx64, pQueryHandle, pQueryHandle->qId); + tsdbDebug("%p no data in mem, 0x%"PRIx64, pTsdbReadHandle, pTsdbReadHandle->qId); } - if ((ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key <= binfo.window.ekey)) || - (!ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key >= binfo.window.skey))) { + if ((ASCENDING_TRAVERSE(pTsdbReadHandle->order) && (key != TSKEY_INITIAL_VAL && key <= binfo.window.ekey)) || + (!ASCENDING_TRAVERSE(pTsdbReadHandle->order) && (key != TSKEY_INITIAL_VAL && key >= binfo.window.skey))) { - if ((ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key < binfo.window.skey)) || - (!ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key > binfo.window.ekey))) { + if ((ASCENDING_TRAVERSE(pTsdbReadHandle->order) && (key != TSKEY_INITIAL_VAL && key < binfo.window.skey)) || + (!ASCENDING_TRAVERSE(pTsdbReadHandle->order) && (key != TSKEY_INITIAL_VAL && key > binfo.window.ekey))) { // do not load file block into buffer - int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order) ? 1 : -1; + int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order) ? 1 : -1; - TSKEY maxKey = ASCENDING_TRAVERSE(pQueryHandle->order)? (binfo.window.skey - step):(binfo.window.ekey - step); - cur->rows = tsdbReadRowsFromCache(pCheckInfo, maxKey, pQueryHandle->outputCapacity, &cur->win, pQueryHandle); - pQueryHandle->realNumOfRows = cur->rows; + TSKEY maxKey = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? (binfo.window.skey - step):(binfo.window.ekey - step); + cur->rows = tsdbReadRowsFromCache(pCheckInfo, maxKey, pTsdbReadHandle->outputCapacity, &cur->win, pTsdbReadHandle); + pTsdbReadHandle->realNumOfRows = cur->rows; // update the last key value pCheckInfo->lastKey = cur->win.ekey + step; - if (!ASCENDING_TRAVERSE(pQueryHandle->order)) { + if (!ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { SWAP(cur->win.skey, cur->win.ekey, TSKEY); } @@ -1289,11 +1272,11 @@ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SBlock* p // return error, add test cases - if ((code = doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo, cur->slot)) != TSDB_CODE_SUCCESS) { + if ((code = doLoadFileDataBlock(pTsdbReadHandle, pBlock, pCheckInfo, cur->slot)) != TSDB_CODE_SUCCESS) { return code; } - doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock); + doMergeTwoLevelData(pTsdbReadHandle, pCheckInfo, pBlock); } else { /* * no data in cache, only load data from file @@ -1301,19 +1284,19 @@ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SBlock* p * * Here the buffer is not enough, so only part of file block can be loaded into memory buffer */ - assert(pQueryHandle->outputCapacity >= binfo.rows); - int32_t endPos = getEndPosInDataBlock(pQueryHandle, &binfo); + assert(pTsdbReadHandle->outputCapacity >= binfo.rows); + int32_t endPos = getEndPosInDataBlock(pTsdbReadHandle, &binfo); - if ((cur->pos == 0 && endPos == binfo.rows -1 && ASCENDING_TRAVERSE(pQueryHandle->order)) || - (cur->pos == (binfo.rows - 1) && endPos == 0 && (!ASCENDING_TRAVERSE(pQueryHandle->order)))) { - pQueryHandle->realNumOfRows = binfo.rows; + if ((cur->pos == 0 && endPos == binfo.rows -1 && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || + (cur->pos == (binfo.rows - 1) && endPos == 0 && (!ASCENDING_TRAVERSE(pTsdbReadHandle->order)))) { + pTsdbReadHandle->realNumOfRows = binfo.rows; cur->rows = binfo.rows; cur->win = binfo.window; cur->mixBlock = false; cur->blockCompleted = true; - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { + if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { cur->lastKey = binfo.window.ekey + 1; cur->pos = binfo.rows; } else { @@ -1321,17 +1304,17 @@ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SBlock* p cur->pos = -1; } } else { // partially copy to dest buffer - copyAllRemainRowsFromFileBlock(pQueryHandle, pCheckInfo, &binfo, endPos); + copyAllRemainRowsFromFileBlock(pTsdbReadHandle, pCheckInfo, &binfo, endPos); cur->mixBlock = true; } assert(cur->blockCompleted); if (cur->rows == binfo.rows) { tsdbDebug("%p whole file block qualified, brange:%"PRId64"-%"PRId64", rows:%d, lastKey:%"PRId64", %"PRIx64, - pQueryHandle, cur->win.skey, cur->win.ekey, cur->rows, cur->lastKey, pQueryHandle->qId); + pTsdbReadHandle, cur->win.skey, cur->win.ekey, cur->rows, cur->lastKey, pTsdbReadHandle->qId); } else { tsdbDebug("%p create data block from remain file block, brange:%"PRId64"-%"PRId64", rows:%d, total:%d, lastKey:%"PRId64", %"PRIx64, - pQueryHandle, cur->win.skey, cur->win.ekey, cur->rows, binfo.rows, cur->lastKey, pQueryHandle->qId); + pTsdbReadHandle, cur->win.skey, cur->win.ekey, cur->rows, binfo.rows, cur->lastKey, pTsdbReadHandle->qId); } } @@ -1339,58 +1322,58 @@ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SBlock* p return code; } -static int32_t loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SBlock* pBlock, STableCheckInfo* pCheckInfo, bool* exists) { - SQueryFilePos* cur = &pQueryHandle->cur; +static int32_t loadFileDataBlock(STsdbReadHandle* pTsdbReadHandle, SBlock* pBlock, STableCheckInfo* pCheckInfo, bool* exists) { + SQueryFilePos* cur = &pTsdbReadHandle->cur; int32_t code = TSDB_CODE_SUCCESS; - bool asc = ASCENDING_TRAVERSE(pQueryHandle->order); + bool asc = ASCENDING_TRAVERSE(pTsdbReadHandle->order); if (asc) { // query ended in/started from current block - if (pQueryHandle->window.ekey < pBlock->keyLast || pCheckInfo->lastKey > pBlock->keyFirst) { - if ((code = doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo, cur->slot)) != TSDB_CODE_SUCCESS) { + if (pTsdbReadHandle->window.ekey < pBlock->keyLast || pCheckInfo->lastKey > pBlock->keyFirst) { + if ((code = doLoadFileDataBlock(pTsdbReadHandle, pBlock, pCheckInfo, cur->slot)) != TSDB_CODE_SUCCESS) { *exists = false; return code; } - SDataCols* pTSCol = pQueryHandle->rhelper.pDCols[0]; + SDataCols* pTSCol = pTsdbReadHandle->rhelper.pDCols[0]; assert(pTSCol->cols->type == TSDB_DATA_TYPE_TIMESTAMP && pTSCol->numOfRows == pBlock->numOfRows); if (pCheckInfo->lastKey > pBlock->keyFirst) { cur->pos = - binarySearchForKey(pTSCol->cols[0].pData, pBlock->numOfRows, pCheckInfo->lastKey, pQueryHandle->order); + binarySearchForKey(pTSCol->cols[0].pData, pBlock->numOfRows, pCheckInfo->lastKey, pTsdbReadHandle->order); } else { cur->pos = 0; } assert(pCheckInfo->lastKey <= pBlock->keyLast); - doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock); + doMergeTwoLevelData(pTsdbReadHandle, pCheckInfo, pBlock); } else { // the whole block is loaded in to buffer cur->pos = asc? 0:(pBlock->numOfRows - 1); - code = handleDataMergeIfNeeded(pQueryHandle, pBlock, pCheckInfo); + code = handleDataMergeIfNeeded(pTsdbReadHandle, pBlock, pCheckInfo); } } else { //desc order, query ended in current block - if (pQueryHandle->window.ekey > pBlock->keyFirst || pCheckInfo->lastKey < pBlock->keyLast) { - if ((code = doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo, cur->slot)) != TSDB_CODE_SUCCESS) { + if (pTsdbReadHandle->window.ekey > pBlock->keyFirst || pCheckInfo->lastKey < pBlock->keyLast) { + if ((code = doLoadFileDataBlock(pTsdbReadHandle, pBlock, pCheckInfo, cur->slot)) != TSDB_CODE_SUCCESS) { *exists = false; return code; } - SDataCols* pTsCol = pQueryHandle->rhelper.pDCols[0]; + SDataCols* pTsCol = pTsdbReadHandle->rhelper.pDCols[0]; if (pCheckInfo->lastKey < pBlock->keyLast) { - cur->pos = binarySearchForKey(pTsCol->cols[0].pData, pBlock->numOfRows, pCheckInfo->lastKey, pQueryHandle->order); + cur->pos = binarySearchForKey(pTsCol->cols[0].pData, pBlock->numOfRows, pCheckInfo->lastKey, pTsdbReadHandle->order); } else { cur->pos = pBlock->numOfRows - 1; } assert(pCheckInfo->lastKey >= pBlock->keyFirst); - doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock); + doMergeTwoLevelData(pTsdbReadHandle, pCheckInfo, pBlock); } else { cur->pos = asc? 0:(pBlock->numOfRows-1); - code = handleDataMergeIfNeeded(pQueryHandle, pBlock, pCheckInfo); + code = handleDataMergeIfNeeded(pTsdbReadHandle, pBlock, pCheckInfo); } } - *exists = pQueryHandle->realNumOfRows > 0; + *exists = pTsdbReadHandle->realNumOfRows > 0; return code; } @@ -1456,11 +1439,11 @@ static int doBinarySearchKey(char* pValue, int num, TSKEY key, int order) { return midPos; } -static int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, int32_t start, int32_t end) { +static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t capacity, int32_t numOfRows, int32_t start, int32_t end) { char* pData = NULL; - int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1 : -1; + int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? 1 : -1; - SDataCols* pCols = pQueryHandle->rhelper.pDCols[0]; + SDataCols* pCols = pTsdbReadHandle->rhelper.pDCols[0]; TSKEY* tsArray = pCols->cols[0].pData; int32_t num = end - start + 1; @@ -1470,12 +1453,12 @@ static int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t c return numOfRows; } - int32_t requiredNumOfCols = (int32_t)taosArrayGetSize(pQueryHandle->pColumns); + int32_t requiredNumOfCols = (int32_t)taosArrayGetSize(pTsdbReadHandle->pColumns); //data in buffer has greater timestamp, copy data in file block int32_t i = 0, j = 0; while(i < requiredNumOfCols && j < pCols->numOfCols) { - SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); + SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); SDataCol* src = &pCols->cols[j]; if (src->colId < pColInfo->info.colId) { @@ -1485,7 +1468,7 @@ static int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t c int32_t bytes = pColInfo->info.bytes; - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { + if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; } else { pData = (char*)pColInfo->pData + (capacity - numOfRows - num) * pColInfo->info.bytes; @@ -1523,8 +1506,8 @@ static int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t c } while (i < requiredNumOfCols) { // the remain columns are all null data - SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { + SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); + if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; } else { pData = (char*)pColInfo->pData + (capacity - numOfRows - num) * pColInfo->info.bytes; @@ -1544,15 +1527,15 @@ static int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t c i++; } - pQueryHandle->cur.win.ekey = tsArray[end]; - pQueryHandle->cur.lastKey = tsArray[end] + step; + pTsdbReadHandle->cur.win.ekey = tsArray[end]; + pTsdbReadHandle->cur.lastKey = tsArray[end] + step; return numOfRows + num; } // Note: row1 always has high priority -static void mergeTwoRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, - SMemRow row1, SMemRow row2, int32_t numOfCols, STable* pTable, +static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacity, int32_t numOfRows, + SMemRow row1, SMemRow row2, int32_t numOfCols, uint64_t uid, STSchema* pSchema1, STSchema* pSchema2, bool forceSetNull) { char* pData = NULL; STSchema* pSchema; @@ -1570,8 +1553,9 @@ static void mergeTwoRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfColsOfRow1 = 0; if (pSchema1 == NULL) { - pSchema1 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row1)); + pSchema1 = metaGetTbTSchema(pTsdbReadHandle->pTsdb->pMeta, uid, 0); } + if(isRow1DataRow) { numOfColsOfRow1 = schemaNCols(pSchema1); } else { @@ -1582,7 +1566,7 @@ static void mergeTwoRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, if(row2) { isRow2DataRow = isDataRow(row2); if (pSchema2 == NULL) { - pSchema2 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row2)); + pSchema2 = metaGetTbTSchema(pTsdbReadHandle->pTsdb->pMeta, uid, 0); } if(isRow2DataRow) { numOfColsOfRow2 = schemaNCols(pSchema2); @@ -1594,9 +1578,9 @@ static void mergeTwoRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t i = 0, j = 0, k = 0; while(i < numOfCols && (j < numOfColsOfRow1 || k < numOfColsOfRow2)) { - SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); + SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { + if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; } else { pData = (char*)pColInfo->pData + (capacity - numOfRows - 1) * pColInfo->info.bytes; @@ -1699,7 +1683,7 @@ static void mergeTwoRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, SET_DOUBLE_PTR(pData, value); break; case TSDB_DATA_TYPE_TIMESTAMP: - if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) { *(TSKEY *)pData = tdGetKey(*(TKEY *)value); } else { *(TSKEY *)pData = *(TSKEY *)value; @@ -1730,8 +1714,8 @@ static void mergeTwoRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, if(forceSetNull) { while (i < numOfCols) { // the remain columns are all null data - SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { + SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); + if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; } else { pData = (char*)pColInfo->pData + (capacity - numOfRows - 1) * pColInfo->info.bytes; @@ -1748,29 +1732,29 @@ static void mergeTwoRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, } } -static void moveDataToFront(STsdbQueryHandle* pQueryHandle, int32_t numOfRows, int32_t numOfCols) { - if (numOfRows == 0 || ASCENDING_TRAVERSE(pQueryHandle->order)) { +static void moveDataToFront(STsdbReadHandle* pTsdbReadHandle, int32_t numOfRows, int32_t numOfCols) { + if (numOfRows == 0 || ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { return; } // if the buffer is not full in case of descending order query, move the data in the front of the buffer - if (numOfRows < pQueryHandle->outputCapacity) { - int32_t emptySize = pQueryHandle->outputCapacity - numOfRows; + if (numOfRows < pTsdbReadHandle->outputCapacity) { + int32_t emptySize = pTsdbReadHandle->outputCapacity - numOfRows; for(int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); + SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); memmove((char*)pColInfo->pData, (char*)pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes); } } } -static void getQualifiedRowsPos(STsdbQueryHandle* pQueryHandle, int32_t startPos, int32_t endPos, int32_t numOfExisted, +static void getQualifiedRowsPos(STsdbReadHandle* pTsdbReadHandle, int32_t startPos, int32_t endPos, int32_t numOfExisted, int32_t* start, int32_t* end) { *start = -1; - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { + if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { int32_t remain = endPos - startPos + 1; - if (remain + numOfExisted > pQueryHandle->outputCapacity) { - *end = (pQueryHandle->outputCapacity - numOfExisted) + startPos - 1; + if (remain + numOfExisted > pTsdbReadHandle->outputCapacity) { + *end = (pTsdbReadHandle->outputCapacity - numOfExisted) + startPos - 1; } else { *end = endPos; } @@ -1778,8 +1762,8 @@ static void getQualifiedRowsPos(STsdbQueryHandle* pQueryHandle, int32_t startPos *start = startPos; } else { int32_t remain = (startPos - endPos) + 1; - if (remain + numOfExisted > pQueryHandle->outputCapacity) { - *end = startPos + 1 - (pQueryHandle->outputCapacity - numOfExisted); + if (remain + numOfExisted > pTsdbReadHandle->outputCapacity) { + *end = startPos + 1 - (pTsdbReadHandle->outputCapacity - numOfExisted); } else { *end = endPos; } @@ -1789,55 +1773,55 @@ static void getQualifiedRowsPos(STsdbQueryHandle* pQueryHandle, int32_t startPos } } -static void updateInfoAfterMerge(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, int32_t numOfRows, int32_t endPos) { - SQueryFilePos* cur = &pQueryHandle->cur; +static void updateInfoAfterMerge(STsdbReadHandle* pTsdbReadHandle, STableCheckInfo* pCheckInfo, int32_t numOfRows, int32_t endPos) { + SQueryFilePos* cur = &pTsdbReadHandle->cur; pCheckInfo->lastKey = cur->lastKey; - pQueryHandle->realNumOfRows = numOfRows; + pTsdbReadHandle->realNumOfRows = numOfRows; cur->rows = numOfRows; cur->pos = endPos; } -static void doCheckGeneratedBlockRange(STsdbQueryHandle* pQueryHandle) { - SQueryFilePos* cur = &pQueryHandle->cur; +static void doCheckGeneratedBlockRange(STsdbReadHandle* pTsdbReadHandle) { + SQueryFilePos* cur = &pTsdbReadHandle->cur; if (cur->rows > 0) { - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { - assert(cur->win.skey >= pQueryHandle->window.skey && cur->win.ekey <= pQueryHandle->window.ekey); + if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { + assert(cur->win.skey >= pTsdbReadHandle->window.skey && cur->win.ekey <= pTsdbReadHandle->window.ekey); } else { - assert(cur->win.skey >= pQueryHandle->window.ekey && cur->win.ekey <= pQueryHandle->window.skey); + assert(cur->win.skey >= pTsdbReadHandle->window.ekey && cur->win.ekey <= pTsdbReadHandle->window.skey); } - SColumnInfoData* pColInfoData = taosArrayGet(pQueryHandle->pColumns, 0); + SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pColumns, 0); assert(cur->win.skey == ((TSKEY*)pColInfoData->pData)[0] && cur->win.ekey == ((TSKEY*)pColInfoData->pData)[cur->rows-1]); } else { - cur->win = pQueryHandle->window; + cur->win = pTsdbReadHandle->window; - int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1; - cur->lastKey = pQueryHandle->window.ekey + step; + int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? 1:-1; + cur->lastKey = pTsdbReadHandle->window.ekey + step; } } -static void copyAllRemainRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SDataBlockInfo* pBlockInfo, int32_t endPos) { - SQueryFilePos* cur = &pQueryHandle->cur; +static void copyAllRemainRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, STableCheckInfo* pCheckInfo, SDataBlockInfo* pBlockInfo, int32_t endPos) { + SQueryFilePos* cur = &pTsdbReadHandle->cur; - SDataCols* pCols = pQueryHandle->rhelper.pDCols[0]; + SDataCols* pCols = pTsdbReadHandle->rhelper.pDCols[0]; TSKEY* tsArray = pCols->cols[0].pData; - int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1; - int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pQueryHandle)); + int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? 1:-1; + int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pTsdbReadHandle)); int32_t pos = cur->pos; int32_t start = cur->pos; int32_t end = endPos; - if (!ASCENDING_TRAVERSE(pQueryHandle->order)) { + if (!ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { SWAP(start, end, int32_t); } - assert(pQueryHandle->outputCapacity >= (end - start + 1)); - int32_t numOfRows = doCopyRowsFromFileBlock(pQueryHandle, pQueryHandle->outputCapacity, 0, start, end); + assert(pTsdbReadHandle->outputCapacity >= (end - start + 1)); + int32_t numOfRows = doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, 0, start, end); // the time window should always be ascending order: skey <= ekey cur->win = (STimeWindow) {.skey = tsArray[start], .ekey = tsArray[end]}; @@ -1846,35 +1830,34 @@ static void copyAllRemainRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, STabl cur->blockCompleted = true; // if the buffer is not full in case of descending order query, move the data in the front of the buffer - moveDataToFront(pQueryHandle, numOfRows, numOfCols); + moveDataToFront(pTsdbReadHandle, numOfRows, numOfCols); // The value of pos may be -1 or pBlockInfo->rows, and it is invalid in both cases. pos = endPos + step; - updateInfoAfterMerge(pQueryHandle, pCheckInfo, numOfRows, pos); - doCheckGeneratedBlockRange(pQueryHandle); + updateInfoAfterMerge(pTsdbReadHandle, pCheckInfo, numOfRows, pos); + doCheckGeneratedBlockRange(pTsdbReadHandle); - tsdbDebug("%p uid:%" PRIu64",tid:%d data block created, mixblock:%d, brange:%"PRIu64"-%"PRIu64" rows:%d, 0x%"PRIx64, - pQueryHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, cur->mixBlock, cur->win.skey, - cur->win.ekey, cur->rows, pQueryHandle->qId); + tsdbDebug("%p uid:%" PRIu64", data block created, mixblock:%d, brange:%"PRIu64"-%"PRIu64" rows:%d, 0x%"PRIx64, + pTsdbReadHandle, pCheckInfo->tableId, cur->mixBlock, cur->win.skey, cur->win.ekey, cur->rows, pTsdbReadHandle->qId); } -int32_t getEndPosInDataBlock(STsdbQueryHandle* pQueryHandle, SDataBlockInfo* pBlockInfo) { +int32_t getEndPosInDataBlock(STsdbReadHandle* pTsdbReadHandle, SDataBlockInfo* pBlockInfo) { // NOTE: reverse the order to find the end position in data block int32_t endPos = -1; - int32_t order = ASCENDING_TRAVERSE(pQueryHandle->order)? TSDB_ORDER_DESC : TSDB_ORDER_ASC; + int32_t order = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? TSDB_ORDER_DESC : TSDB_ORDER_ASC; - SQueryFilePos* cur = &pQueryHandle->cur; - SDataCols* pCols = pQueryHandle->rhelper.pDCols[0]; + SQueryFilePos* cur = &pTsdbReadHandle->cur; + SDataCols* pCols = pTsdbReadHandle->rhelper.pDCols[0]; - if (ASCENDING_TRAVERSE(pQueryHandle->order) && pQueryHandle->window.ekey >= pBlockInfo->window.ekey) { + if (ASCENDING_TRAVERSE(pTsdbReadHandle->order) && pTsdbReadHandle->window.ekey >= pBlockInfo->window.ekey) { endPos = pBlockInfo->rows - 1; cur->mixBlock = (cur->pos != 0); - } else if (!ASCENDING_TRAVERSE(pQueryHandle->order) && pQueryHandle->window.ekey <= pBlockInfo->window.skey) { + } else if (!ASCENDING_TRAVERSE(pTsdbReadHandle->order) && pTsdbReadHandle->window.ekey <= pBlockInfo->window.skey) { endPos = 0; cur->mixBlock = (cur->pos != pBlockInfo->rows - 1); } else { assert(pCols->numOfRows > 0); - endPos = doBinarySearchKey(pCols->cols[0].pData, pCols->numOfRows, pQueryHandle->window.ekey, order); + endPos = doBinarySearchKey(pCols->cols[0].pData, pCols->numOfRows, pTsdbReadHandle->window.ekey, order); cur->mixBlock = true; } @@ -1883,33 +1866,33 @@ int32_t getEndPosInDataBlock(STsdbQueryHandle* pQueryHandle, SDataBlockInfo* pBl // only return the qualified data to client in terms of query time window, data rows in the same block but do not // be included in the query time window will be discarded -static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SBlock* pBlock) { - SQueryFilePos* cur = &pQueryHandle->cur; - SDataBlockInfo blockInfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock); - STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; +static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInfo* pCheckInfo, SBlock* pBlock) { + SQueryFilePos* cur = &pTsdbReadHandle->cur; + SDataBlockInfo blockInfo = {0};//GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock); + STsdbCfg* pCfg = &pTsdbReadHandle->pTsdb->config; - initTableMemIterator(pQueryHandle, pCheckInfo); + initTableMemIterator(pTsdbReadHandle, pCheckInfo); - SDataCols* pCols = pQueryHandle->rhelper.pDCols[0]; - assert(pCols->cols[0].type == TSDB_DATA_TYPE_TIMESTAMP && pCols->cols[0].colId == PRIMARYKEY_TIMESTAMP_COL_INDEX && + SDataCols* pCols = pTsdbReadHandle->rhelper.pDCols[0]; + assert(pCols->cols[0].type == TSDB_DATA_TYPE_TIMESTAMP && pCols->cols[0].colId == PRIMARYKEY_TIMESTAMP_COL_ID && cur->pos >= 0 && cur->pos < pBlock->numOfRows); TSKEY* tsArray = pCols->cols[0].pData; assert(pCols->numOfRows == pBlock->numOfRows && tsArray[0] == pBlock->keyFirst && tsArray[pBlock->numOfRows-1] == pBlock->keyLast); // for search the endPos, so the order needs to reverse - int32_t order = (pQueryHandle->order == TSDB_ORDER_ASC)? TSDB_ORDER_DESC:TSDB_ORDER_ASC; + int32_t order = (pTsdbReadHandle->order == TSDB_ORDER_ASC)? TSDB_ORDER_DESC:TSDB_ORDER_ASC; - int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1; - int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pQueryHandle)); + int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? 1:-1; + int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pTsdbReadHandle)); STable* pTable = pCheckInfo->pTableObj; - int32_t endPos = getEndPosInDataBlock(pQueryHandle, &blockInfo); + int32_t endPos = getEndPosInDataBlock(pTsdbReadHandle, &blockInfo); - tsdbDebug("%p uid:%" PRIu64",tid:%d start merge data block, file block range:%"PRIu64"-%"PRIu64" rows:%d, start:%d," + tsdbDebug("%p uid:%" PRIu64" start merge data block, file block range:%"PRIu64"-%"PRIu64" rows:%d, start:%d," "end:%d, 0x%"PRIx64, - pQueryHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, blockInfo.window.skey, blockInfo.window.ekey, - blockInfo.rows, cur->pos, endPos, pQueryHandle->qId); + pTsdbReadHandle, pCheckInfo->tableId, blockInfo.window.skey, blockInfo.window.ekey, + blockInfo.rows, cur->pos, endPos, pTsdbReadHandle->qId); // compared with the data from in-memory buffer, to generate the correct timestamp array list int32_t numOfRows = 0; @@ -1924,40 +1907,40 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* // no data in buffer, load data from file directly if (pCheckInfo->iiter == NULL && pCheckInfo->iter == NULL) { - copyAllRemainRowsFromFileBlock(pQueryHandle, pCheckInfo, &blockInfo, endPos); + copyAllRemainRowsFromFileBlock(pTsdbReadHandle, pCheckInfo, &blockInfo, endPos); return; } else if (pCheckInfo->iter != NULL || pCheckInfo->iiter != NULL) { SSkipListNode* node = NULL; do { SMemRow row2 = NULL; - SMemRow row1 = getSMemRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update, &row2); + SMemRow row1 = getSMemRowInTableMem(pCheckInfo, pTsdbReadHandle->order, pCfg->update, &row2); if (row1 == NULL) { break; } TSKEY key = memRowKey(row1); - if ((key > pQueryHandle->window.ekey && ASCENDING_TRAVERSE(pQueryHandle->order)) || - (key < pQueryHandle->window.ekey && !ASCENDING_TRAVERSE(pQueryHandle->order))) { + if ((key > pTsdbReadHandle->window.ekey && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || + (key < pTsdbReadHandle->window.ekey && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) { break; } - if (((pos > endPos || tsArray[pos] > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) || - ((pos < endPos || tsArray[pos] < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order))) { + if (((pos > endPos || tsArray[pos] > pTsdbReadHandle->window.ekey) && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || + ((pos < endPos || tsArray[pos] < pTsdbReadHandle->window.ekey) && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) { break; } - if ((key < tsArray[pos] && ASCENDING_TRAVERSE(pQueryHandle->order)) || - (key > tsArray[pos] && !ASCENDING_TRAVERSE(pQueryHandle->order))) { + if ((key < tsArray[pos] && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || + (key > tsArray[pos] && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) { if (rv1 != memRowVersion(row1)) { - pSchema1 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row1)); +// pSchema1 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row1)); rv1 = memRowVersion(row1); } if(row2 && rv2 != memRowVersion(row2)) { - pSchema2 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row2)); +// pSchema2 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row2)); rv2 = memRowVersion(row2); } - mergeTwoRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row1, row2, numOfCols, pTable, pSchema1, pSchema2, true); + mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, numOfRows, row1, row2, numOfCols, pTable, pSchema1, pSchema2, true); numOfRows += 1; if (cur->win.skey == TSKEY_INITIAL_VAL) { cur->win.skey = key; @@ -1971,19 +1954,19 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* } else if (key == tsArray[pos]) { // data in buffer has the same timestamp of data in file block, ignore it if (pCfg->update) { if(pCfg->update == TD_ROW_PARTIAL_UPDATE) { - doCopyRowsFromFileBlock(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, pos, pos); + doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, numOfRows, pos, pos); } if (rv1 != memRowVersion(row1)) { - pSchema1 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row1)); +// pSchema1 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row1)); rv1 = memRowVersion(row1); } if(row2 && rv2 != memRowVersion(row2)) { - pSchema2 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row2)); +// pSchema2 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row2)); rv2 = memRowVersion(row2); } bool forceSetNull = pCfg->update != TD_ROW_PARTIAL_UPDATE; - mergeTwoRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row1, row2, numOfCols, pTable, pSchema1, pSchema2, forceSetNull); + mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, numOfRows, row1, row2, numOfCols, pTable, pSchema1, pSchema2, forceSetNull); numOfRows += 1; if (cur->win.skey == TSKEY_INITIAL_VAL) { cur->win.skey = key; @@ -1998,8 +1981,8 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* } else { moveToNextRowInMem(pCheckInfo); } - } else if ((key > tsArray[pos] && ASCENDING_TRAVERSE(pQueryHandle->order)) || - (key < tsArray[pos] && !ASCENDING_TRAVERSE(pQueryHandle->order))) { + } else if ((key > tsArray[pos] && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || + (key < tsArray[pos] && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) { if (cur->win.skey == TSKEY_INITIAL_VAL) { cur->win.skey = tsArray[pos]; } @@ -2016,38 +1999,38 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* } int32_t qstart = 0, qend = 0; - getQualifiedRowsPos(pQueryHandle, pos, end, numOfRows, &qstart, &qend); + getQualifiedRowsPos(pTsdbReadHandle, pos, end, numOfRows, &qstart, &qend); - numOfRows = doCopyRowsFromFileBlock(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, qstart, qend); + numOfRows = doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, numOfRows, qstart, qend); pos += (qend - qstart + 1) * step; - cur->win.ekey = ASCENDING_TRAVERSE(pQueryHandle->order)? tsArray[qend]:tsArray[qstart]; + cur->win.ekey = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? tsArray[qend]:tsArray[qstart]; cur->lastKey = cur->win.ekey + step; } - } while (numOfRows < pQueryHandle->outputCapacity); + } while (numOfRows < pTsdbReadHandle->outputCapacity); - if (numOfRows < pQueryHandle->outputCapacity) { + if (numOfRows < pTsdbReadHandle->outputCapacity) { /** * if cache is empty, load remain file block data. In contrast, if there are remain data in cache, do NOT * copy them all to result buffer, since it may be overlapped with file data block. */ if (node == NULL || - ((memRowKey((SMemRow)SL_GET_NODE_DATA(node)) > pQueryHandle->window.ekey) && - ASCENDING_TRAVERSE(pQueryHandle->order)) || - ((memRowKey((SMemRow)SL_GET_NODE_DATA(node)) < pQueryHandle->window.ekey) && - !ASCENDING_TRAVERSE(pQueryHandle->order))) { + ((memRowKey((SMemRow)SL_GET_NODE_DATA(node)) > pTsdbReadHandle->window.ekey) && + ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || + ((memRowKey((SMemRow)SL_GET_NODE_DATA(node)) < pTsdbReadHandle->window.ekey) && + !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) { // no data in cache or data in cache is greater than the ekey of time window, load data from file block if (cur->win.skey == TSKEY_INITIAL_VAL) { cur->win.skey = tsArray[pos]; } int32_t start = -1, end = -1; - getQualifiedRowsPos(pQueryHandle, pos, endPos, numOfRows, &start, &end); + getQualifiedRowsPos(pTsdbReadHandle, pos, endPos, numOfRows, &start, &end); - numOfRows = doCopyRowsFromFileBlock(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, start, end); + numOfRows = doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, numOfRows, start, end); pos += (end - start + 1) * step; - cur->win.ekey = ASCENDING_TRAVERSE(pQueryHandle->order)? tsArray[end]:tsArray[start]; + cur->win.ekey = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? tsArray[end]:tsArray[start]; cur->lastKey = cur->win.ekey + step; cur->mixBlock = true; } @@ -2055,20 +2038,19 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* } cur->blockCompleted = - (((pos > endPos || cur->lastKey > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) || - ((pos < endPos || cur->lastKey < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order))); + (((pos > endPos || cur->lastKey > pTsdbReadHandle->window.ekey) && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || + ((pos < endPos || cur->lastKey < pTsdbReadHandle->window.ekey) && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))); - if (!ASCENDING_TRAVERSE(pQueryHandle->order)) { + if (!ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { SWAP(cur->win.skey, cur->win.ekey, TSKEY); } - moveDataToFront(pQueryHandle, numOfRows, numOfCols); - updateInfoAfterMerge(pQueryHandle, pCheckInfo, numOfRows, pos); - doCheckGeneratedBlockRange(pQueryHandle); + moveDataToFront(pTsdbReadHandle, numOfRows, numOfCols); + updateInfoAfterMerge(pTsdbReadHandle, pCheckInfo, numOfRows, pos); + doCheckGeneratedBlockRange(pTsdbReadHandle); - tsdbDebug("%p uid:%" PRIu64",tid:%d data block created, mixblock:%d, brange:%"PRIu64"-%"PRIu64" rows:%d, 0x%"PRIx64, - pQueryHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, cur->mixBlock, cur->win.skey, - cur->win.ekey, cur->rows, pQueryHandle->qId); + tsdbDebug("%p uid:%" PRIu64", data block created, mixblock:%d, brange:%"PRIu64"-%"PRIu64" rows:%d, 0x%"PRIx64, + pTsdbReadHandle, pCheckInfo->tableId, cur->mixBlock, cur->win.skey, cur->win.ekey, cur->rows, pTsdbReadHandle->qId); } int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) { @@ -2174,24 +2156,24 @@ static int32_t dataBlockOrderCompar(const void* pLeft, const void* pRight, void* return pLeftBlockInfoEx->compBlock->offset > pRightBlockInfoEx->compBlock->offset ? 1 : -1; } -static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numOfBlocks, int32_t* numOfAllocBlocks) { +static int32_t createDataBlocksInfo(STsdbReadHandle* pTsdbReadHandle, int32_t numOfBlocks, int32_t* numOfAllocBlocks) { size_t size = sizeof(STableBlockInfo) * numOfBlocks; - if (pQueryHandle->allocSize < size) { - pQueryHandle->allocSize = (int32_t)size; - char* tmp = realloc(pQueryHandle->pDataBlockInfo, pQueryHandle->allocSize); + if (pTsdbReadHandle->allocSize < size) { + pTsdbReadHandle->allocSize = (int32_t)size; + char* tmp = realloc(pTsdbReadHandle->pDataBlockInfo, pTsdbReadHandle->allocSize); if (tmp == NULL) { return TSDB_CODE_TDB_OUT_OF_MEMORY; } - pQueryHandle->pDataBlockInfo = (STableBlockInfo*) tmp; + pTsdbReadHandle->pDataBlockInfo = (STableBlockInfo*) tmp; } - memset(pQueryHandle->pDataBlockInfo, 0, size); + memset(pTsdbReadHandle->pDataBlockInfo, 0, size); *numOfAllocBlocks = numOfBlocks; // access data blocks according to the offset of each block in asc/desc order. - int32_t numOfTables = (int32_t)taosArrayGetSize(pQueryHandle->pTableCheckInfo); + int32_t numOfTables = (int32_t)taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); SBlockOrderSupporter sup = {0}; sup.numOfTables = numOfTables; @@ -2208,7 +2190,7 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO int32_t numOfQualTables = 0; for (int32_t j = 0; j < numOfTables; ++j) { - STableCheckInfo* pTableCheck = (STableCheckInfo*)taosArrayGet(pQueryHandle->pTableCheckInfo, j); + STableCheckInfo* pTableCheck = (STableCheckInfo*)taosArrayGet(pTsdbReadHandle->pTableCheckInfo, j); if (pTableCheck->numOfBlocks <= 0) { continue; } @@ -2239,16 +2221,16 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO // since there is only one table qualified, blocks are not sorted if (numOfQualTables == 1) { - memcpy(pQueryHandle->pDataBlockInfo, sup.pDataBlockInfo[0], sizeof(STableBlockInfo) * numOfBlocks); + memcpy(pTsdbReadHandle->pDataBlockInfo, sup.pDataBlockInfo[0], sizeof(STableBlockInfo) * numOfBlocks); cleanBlockOrderSupporter(&sup, numOfQualTables); - tsdbDebug("%p create data blocks info struct completed for 1 table, %d blocks not sorted 0x%"PRIx64, pQueryHandle, cnt, - pQueryHandle->qId); + tsdbDebug("%p create data blocks info struct completed for 1 table, %d blocks not sorted 0x%"PRIx64, pTsdbReadHandle, cnt, + pTsdbReadHandle->qId); return TSDB_CODE_SUCCESS; } - tsdbDebug("%p create data blocks info struct completed, %d blocks in %d tables 0x%"PRIx64, pQueryHandle, cnt, - numOfQualTables, pQueryHandle->qId); + tsdbDebug("%p create data blocks info struct completed, %d blocks in %d tables 0x%"PRIx64, pTsdbReadHandle, cnt, + numOfQualTables, pTsdbReadHandle->qId); assert(cnt <= numOfBlocks && numOfQualTables <= numOfTables); // the pTableQueryInfo[j]->numOfBlocks may be 0 sup.numOfTables = numOfQualTables; @@ -2267,7 +2249,7 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO int32_t index = sup.blockIndexArray[pos]++; STableBlockInfo* pBlocksInfo = sup.pDataBlockInfo[pos]; - pQueryHandle->pDataBlockInfo[numOfTotal++] = pBlocksInfo[index]; + pTsdbReadHandle->pDataBlockInfo[numOfTotal++] = pBlocksInfo[index]; // set data block index overflow, in order to disable the offset comparator if (sup.blockIndexArray[pos] >= sup.numOfBlocksPerTable[pos]) { @@ -2284,90 +2266,90 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO * } */ - tsdbDebug("%p %d data blocks sort completed, 0x%"PRIx64, pQueryHandle, cnt, pQueryHandle->qId); + tsdbDebug("%p %d data blocks sort completed, 0x%"PRIx64, pTsdbReadHandle, cnt, pTsdbReadHandle->qId); cleanBlockOrderSupporter(&sup, numOfTables); free(pTree); return TSDB_CODE_SUCCESS; } -static int32_t getFirstFileDataBlock(STsdbQueryHandle* pQueryHandle, bool* exists); +static int32_t getFirstFileDataBlock(STsdbReadHandle* pTsdbReadHandle, bool* exists); -static int32_t getDataBlockRv(STsdbQueryHandle* pQueryHandle, STableBlockInfo* pNext, bool *exists) { - int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1 : -1; - SQueryFilePos* cur = &pQueryHandle->cur; +static int32_t getDataBlockRv(STsdbReadHandle* pTsdbReadHandle, STableBlockInfo* pNext, bool *exists) { + int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? 1 : -1; + SQueryFilePos* cur = &pTsdbReadHandle->cur; while(1) { - int32_t code = loadFileDataBlock(pQueryHandle, pNext->compBlock, pNext->pTableCheckInfo, exists); + int32_t code = loadFileDataBlock(pTsdbReadHandle, pNext->compBlock, pNext->pTableCheckInfo, exists); if (code != TSDB_CODE_SUCCESS || *exists) { return code; } - if ((cur->slot == pQueryHandle->numOfBlocks - 1 && ASCENDING_TRAVERSE(pQueryHandle->order)) || - (cur->slot == 0 && !ASCENDING_TRAVERSE(pQueryHandle->order))) { + if ((cur->slot == pTsdbReadHandle->numOfBlocks - 1 && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || + (cur->slot == 0 && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) { // all data blocks in current file has been checked already, try next file if exists - return getFirstFileDataBlock(pQueryHandle, exists); + return getFirstFileDataBlock(pTsdbReadHandle, exists); } else { // next block of the same file cur->slot += step; cur->mixBlock = false; cur->blockCompleted = false; - pNext = &pQueryHandle->pDataBlockInfo[cur->slot]; + pNext = &pTsdbReadHandle->pDataBlockInfo[cur->slot]; } } } -static int32_t getFirstFileDataBlock(STsdbQueryHandle* pQueryHandle, bool* exists) { - pQueryHandle->numOfBlocks = 0; - SQueryFilePos* cur = &pQueryHandle->cur; +static int32_t getFirstFileDataBlock(STsdbReadHandle* pTsdbReadHandle, bool* exists) { + pTsdbReadHandle->numOfBlocks = 0; + SQueryFilePos* cur = &pTsdbReadHandle->cur; int32_t code = TSDB_CODE_SUCCESS; int32_t numOfBlocks = 0; - int32_t numOfTables = (int32_t)taosArrayGetSize(pQueryHandle->pTableCheckInfo); + int32_t numOfTables = (int32_t)taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); - STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; + STsdbCfg* pCfg = &pTsdbReadHandle->pTsdb->config; STimeWindow win = TSWINDOW_INITIALIZER; while (true) { - tsdbRLockFS(REPO_FS(pQueryHandle->pTsdb)); + tsdbRLockFS(REPO_FS(pTsdbReadHandle->pTsdb)); - if ((pQueryHandle->pFileGroup = tsdbFSIterNext(&pQueryHandle->fileIter)) == NULL) { - tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); + if ((pTsdbReadHandle->pFileGroup = tsdbFSIterNext(&pTsdbReadHandle->fileIter)) == NULL) { + tsdbUnLockFS(REPO_FS(pTsdbReadHandle->pTsdb)); break; } - tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, pQueryHandle->pFileGroup->fid, &win.skey, &win.ekey); + tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, pTsdbReadHandle->pFileGroup->fid, &win.skey, &win.ekey); // current file are not overlapped with query time window, ignore remain files - if ((ASCENDING_TRAVERSE(pQueryHandle->order) && win.skey > pQueryHandle->window.ekey) || - (!ASCENDING_TRAVERSE(pQueryHandle->order) && win.ekey < pQueryHandle->window.ekey)) { - tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); - tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, 0x%"PRIx64, pQueryHandle, - pQueryHandle->window.skey, pQueryHandle->window.ekey, pQueryHandle->qId); - pQueryHandle->pFileGroup = NULL; - assert(pQueryHandle->numOfBlocks == 0); + if ((ASCENDING_TRAVERSE(pTsdbReadHandle->order) && win.skey > pTsdbReadHandle->window.ekey) || + (!ASCENDING_TRAVERSE(pTsdbReadHandle->order) && win.ekey < pTsdbReadHandle->window.ekey)) { + tsdbUnLockFS(REPO_FS(pTsdbReadHandle->pTsdb)); + tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, 0x%"PRIx64, pTsdbReadHandle, + pTsdbReadHandle->window.skey, pTsdbReadHandle->window.ekey, pTsdbReadHandle->qId); + pTsdbReadHandle->pFileGroup = NULL; + assert(pTsdbReadHandle->numOfBlocks == 0); break; } - if (tsdbSetAndOpenReadFSet(&pQueryHandle->rhelper, pQueryHandle->pFileGroup) < 0) { - tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); + if (tsdbSetAndOpenReadFSet(&pTsdbReadHandle->rhelper, pTsdbReadHandle->pFileGroup) < 0) { + tsdbUnLockFS(REPO_FS(pTsdbReadHandle->pTsdb)); code = terrno; break; } - tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); + tsdbUnLockFS(REPO_FS(pTsdbReadHandle->pTsdb)); - if (tsdbLoadBlockIdx(&pQueryHandle->rhelper) < 0) { + if (tsdbLoadBlockIdx(&pTsdbReadHandle->rhelper) < 0) { code = terrno; break; } - if ((code = getFileCompInfo(pQueryHandle, &numOfBlocks)) != TSDB_CODE_SUCCESS) { + if ((code = getFileCompInfo(pTsdbReadHandle, &numOfBlocks)) != TSDB_CODE_SUCCESS) { break; } - tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, 0x%"PRIx64, pQueryHandle, numOfBlocks, numOfTables, - pQueryHandle->pFileGroup->fid, pQueryHandle->qId); + tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, 0x%"PRIx64, pTsdbReadHandle, numOfBlocks, numOfTables, + pTsdbReadHandle->pFileGroup->fid, pTsdbReadHandle->qId); assert(numOfBlocks >= 0); if (numOfBlocks == 0) { @@ -2375,20 +2357,20 @@ static int32_t getFirstFileDataBlock(STsdbQueryHandle* pQueryHandle, bool* exist } // todo return error code to query engine - if ((code = createDataBlocksInfo(pQueryHandle, numOfBlocks, &pQueryHandle->numOfBlocks)) != TSDB_CODE_SUCCESS) { + if ((code = createDataBlocksInfo(pTsdbReadHandle, numOfBlocks, &pTsdbReadHandle->numOfBlocks)) != TSDB_CODE_SUCCESS) { break; } - assert(numOfBlocks >= pQueryHandle->numOfBlocks); - if (pQueryHandle->numOfBlocks > 0) { + assert(numOfBlocks >= pTsdbReadHandle->numOfBlocks); + if (pTsdbReadHandle->numOfBlocks > 0) { break; } } // no data in file anymore - if (pQueryHandle->numOfBlocks <= 0 || code != TSDB_CODE_SUCCESS) { + if (pTsdbReadHandle->numOfBlocks <= 0 || code != TSDB_CODE_SUCCESS) { if (code == TSDB_CODE_SUCCESS) { - assert(pQueryHandle->pFileGroup == NULL); + assert(pTsdbReadHandle->pFileGroup == NULL); } cur->fid = INT32_MIN; // denote that there are no data in file anymore @@ -2396,12 +2378,12 @@ static int32_t getFirstFileDataBlock(STsdbQueryHandle* pQueryHandle, bool* exist return code; } - assert(pQueryHandle->pFileGroup != NULL && pQueryHandle->numOfBlocks > 0); - cur->slot = ASCENDING_TRAVERSE(pQueryHandle->order)? 0:pQueryHandle->numOfBlocks-1; - cur->fid = pQueryHandle->pFileGroup->fid; + assert(pTsdbReadHandle->pFileGroup != NULL && pTsdbReadHandle->numOfBlocks > 0); + cur->slot = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? 0:pTsdbReadHandle->numOfBlocks-1; + cur->fid = pTsdbReadHandle->pFileGroup->fid; - STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot]; - return getDataBlockRv(pQueryHandle, pBlockInfo, exists); + STableBlockInfo* pBlockInfo = &pTsdbReadHandle->pDataBlockInfo[cur->slot]; + return getDataBlockRv(pTsdbReadHandle, pBlockInfo, exists); } static bool isEndFileDataBlock(SQueryFilePos* cur, int32_t numOfBlocks, bool ascTrav) { @@ -2409,90 +2391,90 @@ static bool isEndFileDataBlock(SQueryFilePos* cur, int32_t numOfBlocks, bool asc return (cur->slot == numOfBlocks - 1 && ascTrav) || (cur->slot == 0 && !ascTrav); } -static void moveToNextDataBlockInCurrentFile(STsdbQueryHandle* pQueryHandle) { - int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1 : -1; +static void moveToNextDataBlockInCurrentFile(STsdbReadHandle* pTsdbReadHandle) { + int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? 1 : -1; - SQueryFilePos* cur = &pQueryHandle->cur; - assert(cur->slot < pQueryHandle->numOfBlocks && cur->slot >= 0); + SQueryFilePos* cur = &pTsdbReadHandle->cur; + assert(cur->slot < pTsdbReadHandle->numOfBlocks && cur->slot >= 0); cur->slot += step; cur->mixBlock = false; cur->blockCompleted = false; } - -int32_t tsdbGetFileBlocksDistInfo(TsdbQueryHandleT* queryHandle, STableBlockDist* pTableBlockInfo) { - STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) queryHandle; +#if 0 +int32_t tsdbGetFileBlocksDistInfo(tsdbReadHandleT* queryHandle, STableBlockDist* pTableBlockInfo) { + STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) queryHandle; pTableBlockInfo->totalSize = 0; pTableBlockInfo->totalRows = 0; - STsdbFS* pFileHandle = REPO_FS(pQueryHandle->pTsdb); + STsdbFS* pFileHandle = REPO_FS(pTsdbReadHandle->pTsdb); // find the start data block in file - pQueryHandle->locateStart = true; - STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; - int32_t fid = getFileIdFromKey(pQueryHandle->window.skey, pCfg->daysPerFile, pCfg->precision); + pTsdbReadHandle->locateStart = true; + STsdbCfg* pCfg = &pTsdbReadHandle->pTsdb->config; + int32_t fid = getFileIdFromKey(pTsdbReadHandle->window.skey, pCfg->daysPerFile, pCfg->precision); tsdbRLockFS(pFileHandle); - tsdbFSIterInit(&pQueryHandle->fileIter, pFileHandle, pQueryHandle->order); - tsdbFSIterSeek(&pQueryHandle->fileIter, fid); + tsdbFSIterInit(&pTsdbReadHandle->fileIter, pFileHandle, pTsdbReadHandle->order); + tsdbFSIterSeek(&pTsdbReadHandle->fileIter, fid); tsdbUnLockFS(pFileHandle); pTableBlockInfo->numOfFiles += 1; int32_t code = TSDB_CODE_SUCCESS; int32_t numOfBlocks = 0; - int32_t numOfTables = (int32_t)taosArrayGetSize(pQueryHandle->pTableCheckInfo); + int32_t numOfTables = (int32_t)taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); int defaultRows = TSDB_DEFAULT_BLOCK_ROWS(pCfg->maxRowsPerFileBlock); STimeWindow win = TSWINDOW_INITIALIZER; while (true) { numOfBlocks = 0; - tsdbRLockFS(REPO_FS(pQueryHandle->pTsdb)); + tsdbRLockFS(REPO_FS(pTsdbReadHandle->pTsdb)); - if ((pQueryHandle->pFileGroup = tsdbFSIterNext(&pQueryHandle->fileIter)) == NULL) { - tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); + if ((pTsdbReadHandle->pFileGroup = tsdbFSIterNext(&pTsdbReadHandle->fileIter)) == NULL) { + tsdbUnLockFS(REPO_FS(pTsdbReadHandle->pTsdb)); break; } - tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, pQueryHandle->pFileGroup->fid, &win.skey, &win.ekey); + tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, pTsdbReadHandle->pFileGroup->fid, &win.skey, &win.ekey); // current file are not overlapped with query time window, ignore remain files - if ((ASCENDING_TRAVERSE(pQueryHandle->order) && win.skey > pQueryHandle->window.ekey) || - (!ASCENDING_TRAVERSE(pQueryHandle->order) && win.ekey < pQueryHandle->window.ekey)) { - tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); - tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, 0x%"PRIx64, pQueryHandle, - pQueryHandle->window.skey, pQueryHandle->window.ekey, pQueryHandle->qId); - pQueryHandle->pFileGroup = NULL; + if ((ASCENDING_TRAVERSE(pTsdbReadHandle->order) && win.skey > pTsdbReadHandle->window.ekey) || + (!ASCENDING_TRAVERSE(pTsdbReadHandle->order) && win.ekey < pTsdbReadHandle->window.ekey)) { + tsdbUnLockFS(REPO_FS(pTsdbReadHandle->pTsdb)); + tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, 0x%"PRIx64, pTsdbReadHandle, + pTsdbReadHandle->window.skey, pTsdbReadHandle->window.ekey, pTsdbReadHandle->qId); + pTsdbReadHandle->pFileGroup = NULL; break; } pTableBlockInfo->numOfFiles += 1; - if (tsdbSetAndOpenReadFSet(&pQueryHandle->rhelper, pQueryHandle->pFileGroup) < 0) { - tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); + if (tsdbSetAndOpenReadFSet(&pTsdbReadHandle->rhelper, pTsdbReadHandle->pFileGroup) < 0) { + tsdbUnLockFS(REPO_FS(pTsdbReadHandle->pTsdb)); code = terrno; break; } - tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); + tsdbUnLockFS(REPO_FS(pTsdbReadHandle->pTsdb)); - if (tsdbLoadBlockIdx(&pQueryHandle->rhelper) < 0) { + if (tsdbLoadBlockIdx(&pTsdbReadHandle->rhelper) < 0) { code = terrno; break; } - if ((code = getFileCompInfo(pQueryHandle, &numOfBlocks)) != TSDB_CODE_SUCCESS) { + if ((code = getFileCompInfo(pTsdbReadHandle, &numOfBlocks)) != TSDB_CODE_SUCCESS) { break; } - tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, 0x%"PRIx64, pQueryHandle, numOfBlocks, numOfTables, - pQueryHandle->pFileGroup->fid, pQueryHandle->qId); + tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, 0x%"PRIx64, pTsdbReadHandle, numOfBlocks, numOfTables, + pTsdbReadHandle->pFileGroup->fid, pTsdbReadHandle->qId); if (numOfBlocks == 0) { continue; } for (int32_t i = 0; i < numOfTables; ++i) { - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); + STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, i); SBlock* pBlock = pCheckInfo->pCompInfo->blocks; for (int32_t j = 0; j < pCheckInfo->numOfBlocks; ++j) { @@ -2512,36 +2494,37 @@ int32_t tsdbGetFileBlocksDistInfo(TsdbQueryHandleT* queryHandle, STableBlockDist return code; } +#endif -static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists) { - STsdbFS* pFileHandle = REPO_FS(pQueryHandle->pTsdb); - SQueryFilePos* cur = &pQueryHandle->cur; +static int32_t getDataBlocksInFiles(STsdbReadHandle* pTsdbReadHandle, bool* exists) { + STsdbFS* pFileHandle = REPO_FS(pTsdbReadHandle->pTsdb); + SQueryFilePos* cur = &pTsdbReadHandle->cur; // find the start data block in file - if (!pQueryHandle->locateStart) { - pQueryHandle->locateStart = true; - STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; - int32_t fid = getFileIdFromKey(pQueryHandle->window.skey, pCfg->daysPerFile, pCfg->precision); + if (!pTsdbReadHandle->locateStart) { + pTsdbReadHandle->locateStart = true; + STsdbCfg* pCfg = &pTsdbReadHandle->pTsdb->config; + int32_t fid = getFileIdFromKey(pTsdbReadHandle->window.skey, pCfg->daysPerFile, pCfg->precision); tsdbRLockFS(pFileHandle); - tsdbFSIterInit(&pQueryHandle->fileIter, pFileHandle, pQueryHandle->order); - tsdbFSIterSeek(&pQueryHandle->fileIter, fid); + tsdbFSIterInit(&pTsdbReadHandle->fileIter, pFileHandle, pTsdbReadHandle->order); + tsdbFSIterSeek(&pTsdbReadHandle->fileIter, fid); tsdbUnLockFS(pFileHandle); - return getFirstFileDataBlock(pQueryHandle, exists); + return getFirstFileDataBlock(pTsdbReadHandle, exists); } else { // check if current file block is all consumed - STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot]; + STableBlockInfo* pBlockInfo = &pTsdbReadHandle->pDataBlockInfo[cur->slot]; STableCheckInfo* pCheckInfo = pBlockInfo->pTableCheckInfo; // current block is done, try next if ((!cur->mixBlock) || cur->blockCompleted) { // all data blocks in current file has been checked already, try next file if exists } else { - tsdbDebug("%p continue in current data block, index:%d, pos:%d, 0x%"PRIx64, pQueryHandle, cur->slot, cur->pos, - pQueryHandle->qId); - int32_t code = handleDataMergeIfNeeded(pQueryHandle, pBlockInfo->compBlock, pCheckInfo); - *exists = (pQueryHandle->realNumOfRows > 0); + tsdbDebug("%p continue in current data block, index:%d, pos:%d, 0x%"PRIx64, pTsdbReadHandle, cur->slot, cur->pos, + pTsdbReadHandle->qId); + int32_t code = handleDataMergeIfNeeded(pTsdbReadHandle, pBlockInfo->compBlock, pCheckInfo); + *exists = (pTsdbReadHandle->realNumOfRows > 0); if (code != TSDB_CODE_SUCCESS || *exists) { return code; @@ -2550,50 +2533,50 @@ static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists // current block is empty, try next block in file // all data blocks in current file has been checked already, try next file if exists - if (isEndFileDataBlock(cur, pQueryHandle->numOfBlocks, ASCENDING_TRAVERSE(pQueryHandle->order))) { - return getFirstFileDataBlock(pQueryHandle, exists); + if (isEndFileDataBlock(cur, pTsdbReadHandle->numOfBlocks, ASCENDING_TRAVERSE(pTsdbReadHandle->order))) { + return getFirstFileDataBlock(pTsdbReadHandle, exists); } else { - moveToNextDataBlockInCurrentFile(pQueryHandle); - STableBlockInfo* pNext = &pQueryHandle->pDataBlockInfo[cur->slot]; - return getDataBlockRv(pQueryHandle, pNext, exists); + moveToNextDataBlockInCurrentFile(pTsdbReadHandle); + STableBlockInfo* pNext = &pTsdbReadHandle->pDataBlockInfo[cur->slot]; + return getDataBlockRv(pTsdbReadHandle, pNext, exists); } } } -static bool doHasDataInBuffer(STsdbQueryHandle* pQueryHandle) { - size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); +static bool doHasDataInBuffer(STsdbReadHandle* pTsdbReadHandle) { + size_t numOfTables = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); - while (pQueryHandle->activeIndex < numOfTables) { - if (hasMoreDataInCache(pQueryHandle)) { + while (pTsdbReadHandle->activeIndex < numOfTables) { + if (hasMoreDataInCache(pTsdbReadHandle)) { return true; } - pQueryHandle->activeIndex += 1; + pTsdbReadHandle->activeIndex += 1; } // no data in memtable or imemtable, decrease the memory reference. // TODO !! -// tsdbMayUnTakeMemSnapshot(pQueryHandle); +// tsdbMayUnTakeMemSnapshot(pTsdbReadHandle); return false; } //todo not unref yet, since it is not support multi-group interpolation query -static UNUSED_FUNC void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle) { +static UNUSED_FUNC void changeQueryHandleForInterpQuery(tsdbReadHandleT pHandle) { // filter the queried time stamp in the first place - STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; + STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) pHandle; // starts from the buffer in case of descending timestamp order check data blocks - size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + size_t numOfTables = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); int32_t i = 0; while(i < numOfTables) { - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); + STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, i); // the first qualified table for interpolation query - if ((pQueryHandle->window.skey <= pCheckInfo->pTableObj->lastKey) && - (pCheckInfo->pTableObj->lastKey != TSKEY_INITIAL_VAL)) { - break; - } +// if ((pTsdbReadHandle->window.skey <= pCheckInfo->pTableObj->lastKey) && +// (pCheckInfo->pTableObj->lastKey != TSKEY_INITIAL_VAL)) { +// break; +// } i++; } @@ -2603,35 +2586,34 @@ static UNUSED_FUNC void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle return; } - STableCheckInfo info = *(STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, i); - taosArrayClear(pQueryHandle->pTableCheckInfo); + STableCheckInfo info = *(STableCheckInfo*) taosArrayGet(pTsdbReadHandle->pTableCheckInfo, i); + taosArrayClear(pTsdbReadHandle->pTableCheckInfo); - info.lastKey = pQueryHandle->window.skey; - taosArrayPush(pQueryHandle->pTableCheckInfo, &info); + info.lastKey = pTsdbReadHandle->window.skey; + taosArrayPush(pTsdbReadHandle->pTableCheckInfo, &info); } static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win, - STsdbQueryHandle* pQueryHandle) { + STsdbReadHandle* pTsdbReadHandle) { int numOfRows = 0; - int32_t numOfCols = (int32_t)taosArrayGetSize(pQueryHandle->pColumns); - STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; + int32_t numOfCols = (int32_t)taosArrayGetSize(pTsdbReadHandle->pColumns); + STsdbCfg* pCfg = &pTsdbReadHandle->pTsdb->config; win->skey = TSKEY_INITIAL_VAL; int64_t st = taosGetTimestampUs(); - STable* pTable = pCheckInfo->pTableObj; int16_t rv = -1; STSchema* pSchema = NULL; do { - SMemRow row = getSMemRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update, NULL); + SMemRow row = getSMemRowInTableMem(pCheckInfo, pTsdbReadHandle->order, pCfg->update, NULL); if (row == NULL) { break; } TSKEY key = memRowKey(row); - if ((key > maxKey && ASCENDING_TRAVERSE(pQueryHandle->order)) || (key < maxKey && !ASCENDING_TRAVERSE(pQueryHandle->order))) { - tsdbDebug("%p key:%"PRIu64" beyond qrange:%"PRId64" - %"PRId64", no more data in buffer", pQueryHandle, key, pQueryHandle->window.skey, - pQueryHandle->window.ekey); + if ((key > maxKey && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || (key < maxKey && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) { + tsdbDebug("%p key:%"PRIu64" beyond qrange:%"PRId64" - %"PRId64", no more data in buffer", pTsdbReadHandle, key, pTsdbReadHandle->window.skey, + pTsdbReadHandle->window.ekey); break; } @@ -2642,10 +2624,10 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int win->ekey = key; if (rv != memRowVersion(row)) { - pSchema = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row)); + pSchema = metaGetTbTSchema(pTsdbReadHandle->pTsdb->pMeta, pCheckInfo->tableId, 0); rv = memRowVersion(row); } - mergeTwoRowFromMem(pQueryHandle, maxRowsToRead, numOfRows, row, NULL, numOfCols, pTable, pSchema, NULL, true); + mergeTwoRowFromMem(pTsdbReadHandle, maxRowsToRead, numOfRows, row, NULL, numOfCols, pCheckInfo->tableId, pSchema, NULL, true); if (++numOfRows >= maxRowsToRead) { moveToNextRowInMem(pCheckInfo); @@ -2657,24 +2639,24 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int assert(numOfRows <= maxRowsToRead); // if the buffer is not full in case of descending order query, move the data in the front of the buffer - if (!ASCENDING_TRAVERSE(pQueryHandle->order) && numOfRows < maxRowsToRead) { + if (!ASCENDING_TRAVERSE(pTsdbReadHandle->order) && numOfRows < maxRowsToRead) { int32_t emptySize = maxRowsToRead - numOfRows; for(int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); + SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); memmove((char*)pColInfo->pData, (char*)pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes); } } int64_t elapsedTime = taosGetTimestampUs() - st; - tsdbDebug("%p build data block from cache completed, elapsed time:%"PRId64" us, numOfRows:%d, numOfCols:%d, 0x%"PRIx64, pQueryHandle, - elapsedTime, numOfRows, numOfCols, pQueryHandle->qId); + tsdbDebug("%p build data block from cache completed, elapsed time:%"PRId64" us, numOfRows:%d, numOfCols:%d, 0x%"PRIx64, pTsdbReadHandle, + elapsedTime, numOfRows, numOfCols, pTsdbReadHandle->qId); return numOfRows; } static int32_t getAllTableList(STable* pSuperTable, SArray* list) { - SSkipListIterator* iter = tSkipListCreateIter(pSuperTable->pIndex); + SSkipListIterator* iter = NULL;//tSkipListCreateIter(pSuperTable->pIndex); while (tSkipListIterNext(iter)) { SSkipListNode* pNode = tSkipListIterGet(iter); @@ -2693,58 +2675,61 @@ static void destroyHelper(void* param) { return; } - tQueryInfo* pInfo = (tQueryInfo*)param; - if (pInfo->optr != TSDB_RELATION_IN) { - tfree(pInfo->q); - } else { - taosHashCleanup((SHashObj *)(pInfo->q)); - } +// tQueryInfo* pInfo = (tQueryInfo*)param; +// if (pInfo->optr != TSDB_RELATION_IN) { +// tfree(pInfo->q); +// } else { +// taosHashCleanup((SHashObj *)(pInfo->q)); +// } free(param); } -static bool loadBlockOfActiveTable(STsdbQueryHandle* pQueryHandle) { - if (pQueryHandle->checkFiles) { +#define TSDB_PREV_ROW 0x1 +#define TSDB_NEXT_ROW 0x2 + +static bool loadBlockOfActiveTable(STsdbReadHandle* pTsdbReadHandle) { + if (pTsdbReadHandle->checkFiles) { // check if the query range overlaps with the file data block bool exists = true; - int32_t code = getDataBlocksInFiles(pQueryHandle, &exists); + int32_t code = getDataBlocksInFiles(pTsdbReadHandle, &exists); if (code != TSDB_CODE_SUCCESS) { - pQueryHandle->checkFiles = false; + pTsdbReadHandle->checkFiles = false; return false; } if (exists) { - tsdbRetrieveDataBlock((TsdbQueryHandleT*) pQueryHandle, NULL); - if (pQueryHandle->currentLoadExternalRows && pQueryHandle->window.skey == pQueryHandle->window.ekey) { - SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, 0); - assert(*(int64_t*)pColInfo->pData == pQueryHandle->window.skey); + tsdbRetrieveDataBlock((tsdbReadHandleT*) pTsdbReadHandle, NULL); + if (pTsdbReadHandle->currentLoadExternalRows && pTsdbReadHandle->window.skey == pTsdbReadHandle->window.ekey) { + SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, 0); + assert(*(int64_t*)pColInfo->pData == pTsdbReadHandle->window.skey); } - pQueryHandle->currentLoadExternalRows = false; // clear the flag, since the exact matched row is found. + pTsdbReadHandle->currentLoadExternalRows = false; // clear the flag, since the exact matched row is found. return exists; } - pQueryHandle->checkFiles = false; + pTsdbReadHandle->checkFiles = false; } - if (hasMoreDataInCache(pQueryHandle)) { - pQueryHandle->currentLoadExternalRows = false; + if (hasMoreDataInCache(pTsdbReadHandle)) { + pTsdbReadHandle->currentLoadExternalRows = false; return true; } // current result is empty - if (pQueryHandle->currentLoadExternalRows && pQueryHandle->window.skey == pQueryHandle->window.ekey && pQueryHandle->cur.rows == 0) { - SMemRef* pMemRef = pQueryHandle->pMemRef; + if (pTsdbReadHandle->currentLoadExternalRows && pTsdbReadHandle->window.skey == pTsdbReadHandle->window.ekey && pTsdbReadHandle->cur.rows == 0) { +// STsdbMemTable* pMemRef = pTsdbReadHandle->pMemTable; - doGetExternalRow(pQueryHandle, TSDB_PREV_ROW, pMemRef); - doGetExternalRow(pQueryHandle, TSDB_NEXT_ROW, pMemRef); +// doGetExternalRow(pTsdbReadHandle, TSDB_PREV_ROW, pMemRef); +// doGetExternalRow(pTsdbReadHandle, TSDB_NEXT_ROW, pMemRef); - bool result = tsdbGetExternalRow(pQueryHandle); + bool result = tsdbGetExternalRow(pTsdbReadHandle); - pQueryHandle->prev = doFreeColumnInfoData(pQueryHandle->prev); - pQueryHandle->next = doFreeColumnInfoData(pQueryHandle->next); - pQueryHandle->currentLoadExternalRows = false; +// pTsdbReadHandle->prev = doFreeColumnInfoData(pTsdbReadHandle->prev); +// pTsdbReadHandle->next = doFreeColumnInfoData(pTsdbReadHandle->next); + pTsdbReadHandle->currentLoadExternalRows = false; return result; } @@ -2752,26 +2737,26 @@ static bool loadBlockOfActiveTable(STsdbQueryHandle* pQueryHandle) { return false; } -static bool loadCachedLastRow(STsdbQueryHandle* pQueryHandle) { +static bool loadCachedLastRow(STsdbReadHandle* pTsdbReadHandle) { // the last row is cached in buffer, return it directly. - // here note that the pQueryHandle->window must be the TS_INITIALIZER - int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pQueryHandle)); - size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + // here note that the pTsdbReadHandle->window must be the TS_INITIALIZER + int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pTsdbReadHandle)); + size_t numOfTables = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); assert(numOfTables > 0 && numOfCols > 0); - SQueryFilePos* cur = &pQueryHandle->cur; + SQueryFilePos* cur = &pTsdbReadHandle->cur; SMemRow pRow = NULL; TSKEY key = TSKEY_INITIAL_VAL; - int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1; + int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? 1:-1; - if (++pQueryHandle->activeIndex < numOfTables) { - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); - int32_t ret = tsdbGetCachedLastRow(pCheckInfo->pTableObj, &pRow, &key); - if (ret != TSDB_CODE_SUCCESS) { - return false; - } - mergeTwoRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, 0, pRow, NULL, numOfCols, pCheckInfo->pTableObj, NULL, NULL, true); + if (++pTsdbReadHandle->activeIndex < numOfTables) { + STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, pTsdbReadHandle->activeIndex); +// int32_t ret = tsdbGetCachedLastRow(pCheckInfo->pTableObj, &pRow, &key); +// if (ret != TSDB_CODE_SUCCESS) { +// return false; +// } + mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, 0, pRow, NULL, numOfCols, pCheckInfo->pTableObj, NULL, NULL, true); tfree(pRow); // update the last key value @@ -2791,192 +2776,191 @@ static bool loadCachedLastRow(STsdbQueryHandle* pQueryHandle) { -static bool loadCachedLast(STsdbQueryHandle* pQueryHandle) { - // the last row is cached in buffer, return it directly. - // here note that the pQueryHandle->window must be the TS_INITIALIZER - int32_t tgNumOfCols = (int32_t)QH_GET_NUM_OF_COLS(pQueryHandle); - size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); - int32_t numOfRows = 0; - assert(numOfTables > 0 && tgNumOfCols > 0); - SQueryFilePos* cur = &pQueryHandle->cur; - TSKEY priKey = TSKEY_INITIAL_VAL; - int32_t priIdx = -1; - SColumnInfoData* pColInfo = NULL; +//static bool loadCachedLast(STsdbReadHandle* pTsdbReadHandle) { +// // the last row is cached in buffer, return it directly. +// // here note that the pTsdbReadHandle->window must be the TS_INITIALIZER +// int32_t tgNumOfCols = (int32_t)QH_GET_NUM_OF_COLS(pTsdbReadHandle); +// size_t numOfTables = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); +// int32_t numOfRows = 0; +// assert(numOfTables > 0 && tgNumOfCols > 0); +// SQueryFilePos* cur = &pTsdbReadHandle->cur; +// TSKEY priKey = TSKEY_INITIAL_VAL; +// int32_t priIdx = -1; +// SColumnInfoData* pColInfo = NULL; +// +// while (++pTsdbReadHandle->activeIndex < numOfTables) { +// STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, pTsdbReadHandle->activeIndex); +// STable* pTable = pCheckInfo->pTableObj; +// char* pData = NULL; +// +// int32_t numOfCols = pTable->maxColNum; +// +// if (pTable->lastCols == NULL || pTable->maxColNum <= 0) { +// tsdbWarn("no last cached for table %s, uid:%" PRIu64 ",tid:%d", pTable->name->data, pTable->uid, pTable->tableId); +// continue; +// } +// +// int32_t i = 0, j = 0; +// while(i < tgNumOfCols && j < numOfCols) { +// pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); +// if (pTable->lastCols[j].colId < pColInfo->info.colId) { +// j++; +// continue; +// } else if (pTable->lastCols[j].colId > pColInfo->info.colId) { +// i++; +// continue; +// } +// +// pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; +// +// if (pTable->lastCols[j].bytes > 0) { +// void* value = pTable->lastCols[j].pData; +// switch (pColInfo->info.type) { +// case TSDB_DATA_TYPE_BINARY: +// case TSDB_DATA_TYPE_NCHAR: +// memcpy(pData, value, varDataTLen(value)); +// break; +// case TSDB_DATA_TYPE_NULL: +// case TSDB_DATA_TYPE_BOOL: +// case TSDB_DATA_TYPE_TINYINT: +// case TSDB_DATA_TYPE_UTINYINT: +// *(uint8_t *)pData = *(uint8_t *)value; +// break; +// case TSDB_DATA_TYPE_SMALLINT: +// case TSDB_DATA_TYPE_USMALLINT: +// *(uint16_t *)pData = *(uint16_t *)value; +// break; +// case TSDB_DATA_TYPE_INT: +// case TSDB_DATA_TYPE_UINT: +// *(uint32_t *)pData = *(uint32_t *)value; +// break; +// case TSDB_DATA_TYPE_BIGINT: +// case TSDB_DATA_TYPE_UBIGINT: +// *(uint64_t *)pData = *(uint64_t *)value; +// break; +// case TSDB_DATA_TYPE_FLOAT: +// SET_FLOAT_PTR(pData, value); +// break; +// case TSDB_DATA_TYPE_DOUBLE: +// SET_DOUBLE_PTR(pData, value); +// break; +// case TSDB_DATA_TYPE_TIMESTAMP: +// if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) { +// priKey = tdGetKey(*(TKEY *)value); +// priIdx = i; +// +// i++; +// j++; +// continue; +// } else { +// *(TSKEY *)pData = *(TSKEY *)value; +// } +// break; +// default: +// memcpy(pData, value, pColInfo->info.bytes); +// } +// +// for (int32_t n = 0; n < tgNumOfCols; ++n) { +// if (n == i) { +// continue; +// } +// +// pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, n); +// pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;; +// +// if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) { +//// *(TSKEY *)pData = pTable->lastCols[j].ts; +// continue; +// } +// +// if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { +// setVardataNull(pData, pColInfo->info.type); +// } else { +// setNull(pData, pColInfo->info.type, pColInfo->info.bytes); +// } +// } +// +// numOfRows++; +// assert(numOfRows < pTsdbReadHandle->outputCapacity); +// } +// +// i++; +// j++; +// } +// +// // leave the real ts column as the last row, because last function only (not stable) use the last row as res +// if (priKey != TSKEY_INITIAL_VAL) { +// pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, priIdx); +// pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; +// +// *(TSKEY *)pData = priKey; +// +// for (int32_t n = 0; n < tgNumOfCols; ++n) { +// if (n == priIdx) { +// continue; +// } +// +// pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, n); +// pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;; +// +// assert (pColInfo->info.colId != PRIMARYKEY_TIMESTAMP_COL_ID); +// +// if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { +// setVardataNull(pData, pColInfo->info.type); +// } else { +// setNull(pData, pColInfo->info.type, pColInfo->info.bytes); +// } +// } +// +// numOfRows++; +// } +// +// if (numOfRows > 0) { +// cur->rows = numOfRows; +// cur->mixBlock = true; +// +// return true; +// } +// } +// +// return false; +//} - while (++pQueryHandle->activeIndex < numOfTables) { - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); - STable* pTable = pCheckInfo->pTableObj; - char* pData = NULL; - - int32_t numOfCols = pTable->maxColNum; - - if (pTable->lastCols == NULL || pTable->maxColNum <= 0) { - tsdbWarn("no last cached for table %s, uid:%" PRIu64 ",tid:%d", pTable->name->data, pTable->tableId.uid, pTable->tableId.tid); - continue; - } - - int32_t i = 0, j = 0; - while(i < tgNumOfCols && j < numOfCols) { - pColInfo = taosArrayGet(pQueryHandle->pColumns, i); - if (pTable->lastCols[j].colId < pColInfo->info.colId) { - j++; - continue; - } else if (pTable->lastCols[j].colId > pColInfo->info.colId) { - i++; - continue; - } - - pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; - - if (pTable->lastCols[j].bytes > 0) { - void* value = pTable->lastCols[j].pData; - switch (pColInfo->info.type) { - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: - memcpy(pData, value, varDataTLen(value)); - break; - case TSDB_DATA_TYPE_NULL: - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_UTINYINT: - *(uint8_t *)pData = *(uint8_t *)value; - break; - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_USMALLINT: - *(uint16_t *)pData = *(uint16_t *)value; - break; - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_UINT: - *(uint32_t *)pData = *(uint32_t *)value; - break; - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_UBIGINT: - *(uint64_t *)pData = *(uint64_t *)value; - break; - case TSDB_DATA_TYPE_FLOAT: - SET_FLOAT_PTR(pData, value); - break; - case TSDB_DATA_TYPE_DOUBLE: - SET_DOUBLE_PTR(pData, value); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - priKey = tdGetKey(*(TKEY *)value); - priIdx = i; - - i++; - j++; - continue; - } else { - *(TSKEY *)pData = *(TSKEY *)value; - } - break; - default: - memcpy(pData, value, pColInfo->info.bytes); - } - - for (int32_t n = 0; n < tgNumOfCols; ++n) { - if (n == i) { - continue; - } - - pColInfo = taosArrayGet(pQueryHandle->pColumns, n); - pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;; - - if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - *(TSKEY *)pData = pTable->lastCols[j].ts; - continue; - } - - if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { - setVardataNull(pData, pColInfo->info.type); - } else { - setNull(pData, pColInfo->info.type, pColInfo->info.bytes); - } - } - - numOfRows++; - assert(numOfRows < pQueryHandle->outputCapacity); - } - - i++; - j++; - } - - // leave the real ts column as the last row, because last function only (not stable) use the last row as res - if (priKey != TSKEY_INITIAL_VAL) { - pColInfo = taosArrayGet(pQueryHandle->pColumns, priIdx); - pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; - - *(TSKEY *)pData = priKey; - - for (int32_t n = 0; n < tgNumOfCols; ++n) { - if (n == priIdx) { - continue; - } - - pColInfo = taosArrayGet(pQueryHandle->pColumns, n); - pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;; - - assert (pColInfo->info.colId != PRIMARYKEY_TIMESTAMP_COL_INDEX); - - if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { - setVardataNull(pData, pColInfo->info.type); - } else { - setNull(pData, pColInfo->info.type, pColInfo->info.bytes); - } - } - - numOfRows++; - } - - if (numOfRows > 0) { - cur->rows = numOfRows; - cur->mixBlock = true; - - return true; - } - } - - return false; -} - - -static bool loadDataBlockFromTableSeq(STsdbQueryHandle* pQueryHandle) { - size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); +static bool loadDataBlockFromTableSeq(STsdbReadHandle* pTsdbReadHandle) { + size_t numOfTables = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); assert(numOfTables > 0); int64_t stime = taosGetTimestampUs(); - while(pQueryHandle->activeIndex < numOfTables) { - if (loadBlockOfActiveTable(pQueryHandle)) { + while(pTsdbReadHandle->activeIndex < numOfTables) { + if (loadBlockOfActiveTable(pTsdbReadHandle)) { return true; } - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); + STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, pTsdbReadHandle->activeIndex); pCheckInfo->numOfBlocks = 0; - pQueryHandle->activeIndex += 1; - pQueryHandle->locateStart = false; - pQueryHandle->checkFiles = true; - pQueryHandle->cur.rows = 0; - pQueryHandle->currentLoadExternalRows = pQueryHandle->loadExternalRow; + pTsdbReadHandle->activeIndex += 1; + pTsdbReadHandle->locateStart = false; + pTsdbReadHandle->checkFiles = true; + pTsdbReadHandle->cur.rows = 0; + pTsdbReadHandle->currentLoadExternalRows = pTsdbReadHandle->loadExternalRow; terrno = TSDB_CODE_SUCCESS; int64_t elapsedTime = taosGetTimestampUs() - stime; - pQueryHandle->cost.checkForNextTime += elapsedTime; + pTsdbReadHandle->cost.checkForNextTime += elapsedTime; } return false; } // handle data in cache situation -bool tsdbNextDataBlock(TsdbQueryHandleT pHandle) { - STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; +bool tsdbNextDataBlock(tsdbReadHandleT pHandle) { + STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) pHandle; - if (emptyQueryTimewindow(pQueryHandle)) { - tsdbDebug("%p query window not overlaps with the data set, no result returned, 0x%"PRIx64, pQueryHandle, pQueryHandle->qId); + if (emptyQueryTimewindow(pTsdbReadHandle)) { + tsdbDebug("%p query window not overlaps with the data set, no result returned, 0x%"PRIx64, pTsdbReadHandle, pTsdbReadHandle->qId); return false; } @@ -2984,177 +2968,177 @@ bool tsdbNextDataBlock(TsdbQueryHandleT pHandle) { int64_t elapsedTime = stime; // TODO refactor: remove "type" - if (pQueryHandle->type == TSDB_QUERY_TYPE_LAST) { - if (pQueryHandle->cachelastrow == TSDB_CACHED_TYPE_LASTROW) { - return loadCachedLastRow(pQueryHandle); - } else if (pQueryHandle->cachelastrow == TSDB_CACHED_TYPE_LAST) { - return loadCachedLast(pQueryHandle); + if (pTsdbReadHandle->type == TSDB_QUERY_TYPE_LAST) { + if (pTsdbReadHandle->cachelastrow == TSDB_CACHED_TYPE_LASTROW) { +// return loadCachedLastRow(pTsdbReadHandle); + } else if (pTsdbReadHandle->cachelastrow == TSDB_CACHED_TYPE_LAST) { +// return loadCachedLast(pTsdbReadHandle); } } - if (pQueryHandle->loadType == BLOCK_LOAD_TABLE_SEQ_ORDER) { - return loadDataBlockFromTableSeq(pQueryHandle); + if (pTsdbReadHandle->loadType == BLOCK_LOAD_TABLE_SEQ_ORDER) { + return loadDataBlockFromTableSeq(pTsdbReadHandle); } else { // loadType == RR and Offset Order - if (pQueryHandle->checkFiles) { + if (pTsdbReadHandle->checkFiles) { // check if the query range overlaps with the file data block bool exists = true; - int32_t code = getDataBlocksInFiles(pQueryHandle, &exists); + int32_t code = getDataBlocksInFiles(pTsdbReadHandle, &exists); if (code != TSDB_CODE_SUCCESS) { - pQueryHandle->activeIndex = 0; - pQueryHandle->checkFiles = false; + pTsdbReadHandle->activeIndex = 0; + pTsdbReadHandle->checkFiles = false; return false; } if (exists) { - pQueryHandle->cost.checkForNextTime += (taosGetTimestampUs() - stime); + pTsdbReadHandle->cost.checkForNextTime += (taosGetTimestampUs() - stime); return exists; } - pQueryHandle->activeIndex = 0; - pQueryHandle->checkFiles = false; + pTsdbReadHandle->activeIndex = 0; + pTsdbReadHandle->checkFiles = false; } // TODO: opt by consider the scan order - bool ret = doHasDataInBuffer(pQueryHandle); + bool ret = doHasDataInBuffer(pTsdbReadHandle); terrno = TSDB_CODE_SUCCESS; elapsedTime = taosGetTimestampUs() - stime; - pQueryHandle->cost.checkForNextTime += elapsedTime; + pTsdbReadHandle->cost.checkForNextTime += elapsedTime; return ret; } } -static int32_t doGetExternalRow(STsdbQueryHandle* pQueryHandle, int16_t type, SMemRef* pMemRef) { - STsdbQueryHandle* pSecQueryHandle = NULL; +//static int32_t doGetExternalRow(STsdbReadHandle* pTsdbReadHandle, int16_t type, STsdbMemTable* pMemRef) { +// STsdbReadHandle* pSecQueryHandle = NULL; +// +// if (type == TSDB_PREV_ROW && pTsdbReadHandle->prev) { +// return TSDB_CODE_SUCCESS; +// } +// +// if (type == TSDB_NEXT_ROW && pTsdbReadHandle->next) { +// return TSDB_CODE_SUCCESS; +// } +// +// // prepare the structure +// int32_t numOfCols = (int32_t) QH_GET_NUM_OF_COLS(pTsdbReadHandle); +// +// if (type == TSDB_PREV_ROW) { +// pTsdbReadHandle->prev = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); +// if (pTsdbReadHandle->prev == NULL) { +// terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; +// goto out_of_memory; +// } +// } else { +// pTsdbReadHandle->next = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); +// if (pTsdbReadHandle->next == NULL) { +// terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; +// goto out_of_memory; +// } +// } +// +// SArray* row = (type == TSDB_PREV_ROW)? pTsdbReadHandle->prev : pTsdbReadHandle->next; +// +// for (int32_t i = 0; i < numOfCols; ++i) { +// SColumnInfoData* pCol = taosArrayGet(pTsdbReadHandle->pColumns, i); +// +// SColumnInfoData colInfo = {{0}, 0}; +// colInfo.info = pCol->info; +// colInfo.pData = calloc(1, pCol->info.bytes); +// if (colInfo.pData == NULL) { +// terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; +// goto out_of_memory; +// } +// +// taosArrayPush(row, &colInfo); +// } +// +// // load the previous row +// STsdbQueryCond cond = {.numOfCols = numOfCols, .loadExternalRows = false, .type = BLOCK_LOAD_OFFSET_SEQ_ORDER}; +// if (type == TSDB_PREV_ROW) { +// cond.order = TSDB_ORDER_DESC; +// cond.twindow = (STimeWindow){pTsdbReadHandle->window.skey, INT64_MIN}; +// } else { +// cond.order = TSDB_ORDER_ASC; +// cond.twindow = (STimeWindow){pTsdbReadHandle->window.skey, INT64_MAX}; +// } +// +// cond.colList = calloc(cond.numOfCols, sizeof(SColumnInfo)); +// if (cond.colList == NULL) { +// terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; +// goto out_of_memory; +// } +// +// for (int32_t i = 0; i < cond.numOfCols; ++i) { +// SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pColumns, i); +// memcpy(&cond.colList[i], &pColInfoData->info, sizeof(SColumnInfo)); +// } +// +// pSecQueryHandle = tsdbQueryTablesImpl(pTsdbReadHandle->pTsdb, &cond, pTsdbReadHandle->qId, pMemRef); +// tfree(cond.colList); +// +// // current table, only one table +// STableCheckInfo* pCurrent = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, pTsdbReadHandle->activeIndex); +// +// SArray* psTable = NULL; +// pSecQueryHandle->pTableCheckInfo = createCheckInfoFromCheckInfo(pCurrent, pSecQueryHandle->window.skey, &psTable); +// if (pSecQueryHandle->pTableCheckInfo == NULL) { +// taosArrayDestroy(psTable); +// terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; +// goto out_of_memory; +// } +// +// +// tsdbMayTakeMemSnapshot(pSecQueryHandle, psTable); +// if (!tsdbNextDataBlock((void*)pSecQueryHandle)) { +// // no result in current query, free the corresponding result rows structure +// if (type == TSDB_PREV_ROW) { +// pTsdbReadHandle->prev = doFreeColumnInfoData(pTsdbReadHandle->prev); +// } else { +// pTsdbReadHandle->next = doFreeColumnInfoData(pTsdbReadHandle->next); +// } +// +// goto out_of_memory; +// } +// +// SDataBlockInfo blockInfo = {{0}, 0}; +// tsdbRetrieveDataBlockInfo((void*)pSecQueryHandle, &blockInfo); +// tsdbRetrieveDataBlock((void*)pSecQueryHandle, pSecQueryHandle->defaultLoadColumn); +// +// row = (type == TSDB_PREV_ROW)? pTsdbReadHandle->prev:pTsdbReadHandle->next; +// int32_t pos = (type == TSDB_PREV_ROW)?pSecQueryHandle->cur.rows - 1:0; +// +// for (int32_t i = 0; i < numOfCols; ++i) { +// SColumnInfoData* pCol = taosArrayGet(row, i); +// SColumnInfoData* s = taosArrayGet(pSecQueryHandle->pColumns, i); +// memcpy((char*)pCol->pData, (char*)s->pData + s->info.bytes * pos, pCol->info.bytes); +// } +// +//out_of_memory: +// tsdbCleanupQueryHandle(pSecQueryHandle); +// return terrno; +//} - if (type == TSDB_PREV_ROW && pQueryHandle->prev) { - return TSDB_CODE_SUCCESS; - } - - if (type == TSDB_NEXT_ROW && pQueryHandle->next) { - return TSDB_CODE_SUCCESS; - } - - // prepare the structure - int32_t numOfCols = (int32_t) QH_GET_NUM_OF_COLS(pQueryHandle); - - if (type == TSDB_PREV_ROW) { - pQueryHandle->prev = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); - if (pQueryHandle->prev == NULL) { - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto out_of_memory; - } - } else { - pQueryHandle->next = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); - if (pQueryHandle->next == NULL) { - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto out_of_memory; - } - } - - SArray* row = (type == TSDB_PREV_ROW)? pQueryHandle->prev : pQueryHandle->next; - - for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i); - - SColumnInfoData colInfo = {{0}, 0}; - colInfo.info = pCol->info; - colInfo.pData = calloc(1, pCol->info.bytes); - if (colInfo.pData == NULL) { - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto out_of_memory; - } - - taosArrayPush(row, &colInfo); - } - - // load the previous row - STsdbQueryCond cond = {.numOfCols = numOfCols, .loadExternalRows = false, .type = BLOCK_LOAD_OFFSET_SEQ_ORDER}; - if (type == TSDB_PREV_ROW) { - cond.order = TSDB_ORDER_DESC; - cond.twindow = (STimeWindow){pQueryHandle->window.skey, INT64_MIN}; - } else { - cond.order = TSDB_ORDER_ASC; - cond.twindow = (STimeWindow){pQueryHandle->window.skey, INT64_MAX}; - } - - cond.colList = calloc(cond.numOfCols, sizeof(SColumnInfo)); - if (cond.colList == NULL) { - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto out_of_memory; - } - - for (int32_t i = 0; i < cond.numOfCols; ++i) { - SColumnInfoData* pColInfoData = taosArrayGet(pQueryHandle->pColumns, i); - memcpy(&cond.colList[i], &pColInfoData->info, sizeof(SColumnInfo)); - } - - pSecQueryHandle = tsdbQueryTablesImpl(pQueryHandle->pTsdb, &cond, pQueryHandle->qId, pMemRef); - tfree(cond.colList); - - // current table, only one table - STableCheckInfo* pCurrent = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); - - SArray* psTable = NULL; - pSecQueryHandle->pTableCheckInfo = createCheckInfoFromCheckInfo(pCurrent, pSecQueryHandle->window.skey, &psTable); - if (pSecQueryHandle->pTableCheckInfo == NULL) { - taosArrayDestroy(psTable); - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto out_of_memory; - } - - - tsdbMayTakeMemSnapshot(pSecQueryHandle, psTable); - if (!tsdbNextDataBlock((void*)pSecQueryHandle)) { - // no result in current query, free the corresponding result rows structure - if (type == TSDB_PREV_ROW) { - pQueryHandle->prev = doFreeColumnInfoData(pQueryHandle->prev); - } else { - pQueryHandle->next = doFreeColumnInfoData(pQueryHandle->next); - } - - goto out_of_memory; - } - - SDataBlockInfo blockInfo = {{0}, 0}; - tsdbRetrieveDataBlockInfo((void*)pSecQueryHandle, &blockInfo); - tsdbRetrieveDataBlock((void*)pSecQueryHandle, pSecQueryHandle->defaultLoadColumn); - - row = (type == TSDB_PREV_ROW)? pQueryHandle->prev:pQueryHandle->next; - int32_t pos = (type == TSDB_PREV_ROW)?pSecQueryHandle->cur.rows - 1:0; - - for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pCol = taosArrayGet(row, i); - SColumnInfoData* s = taosArrayGet(pSecQueryHandle->pColumns, i); - memcpy((char*)pCol->pData, (char*)s->pData + s->info.bytes * pos, pCol->info.bytes); - } - -out_of_memory: - tsdbCleanupQueryHandle(pSecQueryHandle); - return terrno; -} - -bool tsdbGetExternalRow(TsdbQueryHandleT pHandle) { - STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; - SQueryFilePos* cur = &pQueryHandle->cur; +bool tsdbGetExternalRow(tsdbReadHandleT pHandle) { + STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) pHandle; + SQueryFilePos* cur = &pTsdbReadHandle->cur; cur->fid = INT32_MIN; cur->mixBlock = true; - if (pQueryHandle->prev == NULL || pQueryHandle->next == NULL) { + if (pTsdbReadHandle->prev == NULL || pTsdbReadHandle->next == NULL) { cur->rows = 0; return false; } - int32_t numOfCols = (int32_t) QH_GET_NUM_OF_COLS(pQueryHandle); + int32_t numOfCols = (int32_t) QH_GET_NUM_OF_COLS(pTsdbReadHandle); for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pColInfoData = taosArrayGet(pQueryHandle->pColumns, i); - SColumnInfoData* first = taosArrayGet(pQueryHandle->prev, i); + SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pColumns, i); + SColumnInfoData* first = taosArrayGet(pTsdbReadHandle->prev, i); memcpy(pColInfoData->pData, first->pData, pColInfoData->info.bytes); - SColumnInfoData* sec = taosArrayGet(pQueryHandle->next, i); + SColumnInfoData* sec = taosArrayGet(pTsdbReadHandle->next, i); memcpy(((char*)pColInfoData->pData) + pColInfoData->info.bytes, sec->pData, pColInfoData->info.bytes); if (i == 0 && pColInfoData->info.type == TSDB_DATA_TYPE_TIMESTAMP) { @@ -3171,76 +3155,75 @@ bool tsdbGetExternalRow(TsdbQueryHandleT pHandle) { * if lastRow == NULL, return TSDB_CODE_TDB_NO_CACHE_LAST_ROW * else set pRes and return TSDB_CODE_SUCCESS and save lastKey */ -int32_t tsdbGetCachedLastRow(STable* pTable, SMemRow* pRes, TSKEY* lastKey) { - int32_t code = TSDB_CODE_SUCCESS; +//int32_t tsdbGetCachedLastRow(STable* pTable, SMemRow* pRes, TSKEY* lastKey) { +// int32_t code = TSDB_CODE_SUCCESS; +// +// TSDB_RLOCK_TABLE(pTable); +// +// if (!pTable->lastRow) { +// code = TSDB_CODE_TDB_NO_CACHE_LAST_ROW; +// goto out; +// } +// +// if (pRes) { +// *pRes = tdMemRowDup(pTable->lastRow); +// if (*pRes == NULL) { +// code = TSDB_CODE_TDB_OUT_OF_MEMORY; +// } +// } +// +//out: +// TSDB_RUNLOCK_TABLE(pTable); +// return code; +//} - TSDB_RLOCK_TABLE(pTable); - - if (!pTable->lastRow) { - code = TSDB_CODE_TDB_NO_CACHE_LAST_ROW; - goto out; - } - - if (pRes) { - *pRes = tdMemRowDup(pTable->lastRow); - if (*pRes == NULL) { - code = TSDB_CODE_TDB_OUT_OF_MEMORY; - } - } - -out: - TSDB_RUNLOCK_TABLE(pTable); - return code; +bool isTsdbCacheLastRow(tsdbReadHandleT* pTsdbReadHandle) { + return ((STsdbReadHandle *)pTsdbReadHandle)->cachelastrow > TSDB_CACHED_TYPE_NONE; } -bool isTsdbCacheLastRow(TsdbQueryHandleT* pQueryHandle) { - return ((STsdbQueryHandle *)pQueryHandle)->cachelastrow > TSDB_CACHED_TYPE_NONE; +int32_t checkForCachedLastRow(STsdbReadHandle* pTsdbReadHandle, STableGroupInfo *groupList) { + assert(pTsdbReadHandle != NULL && groupList != NULL); + +// TSKEY key = TSKEY_INITIAL_VAL; +// +// SArray* group = taosArrayGetP(groupList->pGroupList, 0); +// assert(group != NULL); +// +// STableKeyInfo* pInfo = (STableKeyInfo*)taosArrayGet(group, 0); +// +// int32_t code = 0; +// +// if (((STable*)pInfo->pTable)->lastRow) { +// code = tsdbGetCachedLastRow(pInfo->pTable, NULL, &key); +// if (code != TSDB_CODE_SUCCESS) { +// pTsdbReadHandle->cachelastrow = TSDB_CACHED_TYPE_NONE; +// } else { +// pTsdbReadHandle->cachelastrow = TSDB_CACHED_TYPE_LASTROW; +// } +// } +// +// // update the tsdb query time range +// if (pTsdbReadHandle->cachelastrow != TSDB_CACHED_TYPE_NONE) { +// pTsdbReadHandle->window = TSWINDOW_INITIALIZER; +// pTsdbReadHandle->checkFiles = false; +// pTsdbReadHandle->activeIndex = -1; // start from -1 +// } + + return TSDB_CODE_SUCCESS; } -int32_t checkForCachedLastRow(STsdbQueryHandle* pQueryHandle, STableGroupInfo *groupList) { - assert(pQueryHandle != NULL && groupList != NULL); - - TSKEY key = TSKEY_INITIAL_VAL; - - SArray* group = taosArrayGetP(groupList->pGroupList, 0); - assert(group != NULL); - - STableKeyInfo* pInfo = (STableKeyInfo*)taosArrayGet(group, 0); +int32_t checkForCachedLast(STsdbReadHandle* pTsdbReadHandle) { + assert(pTsdbReadHandle != NULL); int32_t code = 0; - - if (((STable*)pInfo->pTable)->lastRow) { - code = tsdbGetCachedLastRow(pInfo->pTable, NULL, &key); - if (code != TSDB_CODE_SUCCESS) { - pQueryHandle->cachelastrow = TSDB_CACHED_TYPE_NONE; - } else { - pQueryHandle->cachelastrow = TSDB_CACHED_TYPE_LASTROW; - } - } +// if (pTsdbReadHandle->pTsdb && atomic_load_8(&pTsdbReadHandle->pTsdb->hasCachedLastColumn)){ +// pTsdbReadHandle->cachelastrow = TSDB_CACHED_TYPE_LAST; +// } // update the tsdb query time range - if (pQueryHandle->cachelastrow != TSDB_CACHED_TYPE_NONE) { - pQueryHandle->window = TSWINDOW_INITIALIZER; - pQueryHandle->checkFiles = false; - pQueryHandle->activeIndex = -1; // start from -1 - } - - return code; -} - -int32_t checkForCachedLast(STsdbQueryHandle* pQueryHandle) { - assert(pQueryHandle != NULL); - - int32_t code = 0; - - if (pQueryHandle->pTsdb && atomic_load_8(&pQueryHandle->pTsdb->hasCachedLastColumn)){ - pQueryHandle->cachelastrow = TSDB_CACHED_TYPE_LAST; - } - - // update the tsdb query time range - if (pQueryHandle->cachelastrow) { - pQueryHandle->checkFiles = false; - pQueryHandle->activeIndex = -1; // start from -1 + if (pTsdbReadHandle->cachelastrow) { + pTsdbReadHandle->checkFiles = false; + pTsdbReadHandle->activeIndex = -1; // start from -1 } return code; @@ -3266,7 +3249,7 @@ STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList) { STableKeyInfo* pInfo = (STableKeyInfo*) taosArrayGet(pGroup, i); // if the lastKey equals to INT64_MIN, there is no data in this table - TSKEY lastKey = ((STable*)(pInfo->pTable))->lastKey; + TSKEY lastKey = 0;//((STable*)(pInfo->pTable))->lastKey; if (key < lastKey) { key = lastKey; @@ -3290,7 +3273,7 @@ STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList) { // keyInfo.pTable may be NULL here. if (pInfo->pTable != keyInfo.pTable) { - tsdbUnRefTable(pInfo->pTable); +// tsdbUnRefTable(pInfo->pTable); } } @@ -3322,23 +3305,23 @@ STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList) { return window; } -void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT* pQueryHandle, SDataBlockInfo* pDataBlockInfo) { - STsdbQueryHandle* pHandle = (STsdbQueryHandle*)pQueryHandle; +void tsdbRetrieveDataBlockInfo(tsdbReadHandleT* pTsdbReadHandle, SDataBlockInfo* pDataBlockInfo) { + STsdbReadHandle* pHandle = (STsdbReadHandle*)pTsdbReadHandle; SQueryFilePos* cur = &pHandle->cur; - STable* pTable = NULL; + + uint64_t uid = 0; // there are data in file if (pHandle->cur.fid != INT32_MIN) { STableBlockInfo* pBlockInfo = &pHandle->pDataBlockInfo[cur->slot]; - pTable = pBlockInfo->pTableCheckInfo->pTableObj; + uid = pBlockInfo->pTableCheckInfo->tableId; } else { STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); - pTable = pCheckInfo->pTableObj; + uid = pCheckInfo->tableId; } - pDataBlockInfo->uid = pTable->tableId.uid; - pDataBlockInfo->tid = pTable->tableId.tid; - pDataBlockInfo->rows = cur->rows; + pDataBlockInfo->uid = uid; + pDataBlockInfo->rows = cur->rows; pDataBlockInfo->window = cur->win; pDataBlockInfo->numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pHandle)); } @@ -3346,8 +3329,8 @@ void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT* pQueryHandle, SDataBlockInfo* p /* * return null for mixed data block, if not a complete file data block, the statistics value will always return NULL */ -int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataStatis** pBlockStatis) { - STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle; +int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReadHandleT* pTsdbReadHandle, SDataStatis** pBlockStatis) { + STsdbReadHandle* pHandle = (STsdbReadHandle*) pTsdbReadHandle; SQueryFilePos* c = &pHandle->cur; if (c->mixBlock) { @@ -3381,7 +3364,7 @@ int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataSta // always load the first primary timestamp column data SDataStatis* pPrimaryColStatis = &pHandle->statis[0]; - assert(pPrimaryColStatis->colId == PRIMARYKEY_TIMESTAMP_COL_INDEX); + assert(pPrimaryColStatis->colId == PRIMARYKEY_TIMESTAMP_COL_ID); pPrimaryColStatis->numOfNull = 0; pPrimaryColStatis->min = pBlockInfo->compBlock->keyFirst; @@ -3401,12 +3384,12 @@ int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataSta return TSDB_CODE_SUCCESS; } -SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) { +SArray* tsdbRetrieveDataBlock(tsdbReadHandleT* pTsdbReadHandle, SArray* pIdList) { /** * In the following two cases, the data has been loaded to SColumnInfoData. * 1. data is from cache, 2. data block is not completed qualified to query time range */ - STsdbQueryHandle* pHandle = (STsdbQueryHandle*)pQueryHandle; + STsdbReadHandle* pHandle = (STsdbReadHandle*)pTsdbReadHandle; if (pHandle->cur.fid == INT32_MIN) { return pHandle->pColumns; @@ -3417,14 +3400,14 @@ SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) { if (pHandle->cur.mixBlock) { return pHandle->pColumns; } else { - SDataBlockInfo binfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlockInfo->compBlock); + SDataBlockInfo binfo = {0};/*GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlockInfo->compBlock);*/ assert(pHandle->realNumOfRows <= binfo.rows); // data block has been loaded, todo extract method SDataBlockLoadInfo* pBlockLoadInfo = &pHandle->dataBlockLoadInfo; if (pBlockLoadInfo->slot == pHandle->cur.slot && pBlockLoadInfo->fileGroup->fid == pHandle->cur.fid && - pBlockLoadInfo->tid == pCheckInfo->pTableObj->tableId.tid) { + pBlockLoadInfo->uid == pCheckInfo->pTableObj->tid) { return pHandle->pColumns; } else { // only load the file block SBlock* pBlock = pBlockInfo->compBlock; @@ -3451,7 +3434,7 @@ SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) { } } } - +#if 0 void filterPrepare(void* expr, void* param) { tExprNode* pExpr = (tExprNode*)expr; if (pExpr->_node.info != NULL) { @@ -3550,11 +3533,12 @@ static int32_t tableGroupComparFn(const void *p1, const void *p2, const void *pa return 0; } +#endif static int tsdbCheckInfoCompar(const void* key1, const void* key2) { - if (((STableCheckInfo*)key1)->tableId.tid < ((STableCheckInfo*)key2)->tableId.tid) { + if (((STableCheckInfo*)key1)->tableId < ((STableCheckInfo*)key2)->tableId) { return -1; - } else if (((STableCheckInfo*)key1)->tableId.tid > ((STableCheckInfo*)key2)->tableId.tid) { + } else if (((STableCheckInfo*)key1)->tableId > ((STableCheckInfo*)key2)->tableId) { return 1; } else { ASSERT(false); @@ -3570,7 +3554,6 @@ void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTable STableKeyInfo info = {.pTable = pTable, .lastKey = skey}; taosArrayPush(g, &info); - tsdbRefTable(pTable); for (int32_t i = 1; i < numOfTables; ++i) { STable** prev = taosArrayGet(pTableList, i - 1); @@ -3579,8 +3562,7 @@ void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTable int32_t ret = compareFn(prev, p, pSupp); assert(ret == 0 || ret == -1); - tsdbRefTable(*p); - assert((*p)->type == TSDB_CHILD_TABLE); +// assert((*p)->type == TSDB_CHILD_TABLE); if (ret == 0) { STableKeyInfo info1 = {.pTable = *p, .lastKey = skey}; @@ -3597,6 +3579,7 @@ void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTable taosArrayPush(pGroups, &g); } +#if 0 SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pCols, int32_t numOfOrderCols, TSKEY skey) { assert(pTableList != NULL); SArray* pTableGroup = taosArrayInit(1, POINTER_BYTES); @@ -3616,7 +3599,6 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC for(int32_t i = 0; i < size; ++i) { STableKeyInfo *pKeyInfo = taosArrayGet(pTableList, i); - tsdbRefTable(pKeyInfo->pTable); STableKeyInfo info = {.pTable = pKeyInfo->pTable, .lastKey = skey}; taosArrayPush(sa, &info); @@ -3737,7 +3719,7 @@ static int32_t doQueryTableList(STable* pSTable, SArray* pRes, tExprNode* pExpr) return TSDB_CODE_SUCCESS; } -int32_t tsdbQuerySTableByTagCond(STsdbRepo* tsdb, uint64_t uid, TSKEY skey, const char* pTagCond, size_t len, +int32_t tsdbQuerySTableByTagCond(STsdb* tsdb, uint64_t uid, TSKEY skey, const char* pTagCond, size_t len, int16_t tagNameRelType, const char* tbnameCond, STableGroupInfo* pGroupInfo, SColIndex* pColIndex, int32_t numOfCols) { if (tsdbRLockRepoMeta(tsdb) < 0) goto _error; @@ -3752,7 +3734,7 @@ int32_t tsdbQuerySTableByTagCond(STsdbRepo* tsdb, uint64_t uid, TSKEY skey, cons } if (pTable->type != TSDB_SUPER_TABLE) { - tsdbError("%p query normal tag not allowed, uid:%" PRIu64 ", tid:%d, name:%s", tsdb, uid, pTable->tableId.tid, + tsdbError("%p query normal tag not allowed, uid:%" PRIu64 ", tid:%d, name:%s", tsdb, uid, pTable->tableId, pTable->name->data); terrno = TSDB_CODE_COM_OPS_NOT_SUPPORT; //basically, this error is caused by invalid sql issued by client @@ -3821,8 +3803,8 @@ int32_t tsdbQuerySTableByTagCond(STsdbRepo* tsdb, uint64_t uid, TSKEY skey, cons pGroupInfo->numOfTables = (uint32_t)taosArrayGetSize(res); pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols, skey); - tsdbDebug("%p stable tid:%d, uid:%"PRIu64" query, numOfTables:%u, belong to %" PRIzu " groups", tsdb, pTable->tableId.tid, - pTable->tableId.uid, pGroupInfo->numOfTables, taosArrayGetSize(pGroupInfo->pGroupList)); + tsdbDebug("%p stable tid:%d, uid:%"PRIu64" query, numOfTables:%u, belong to %" PRIzu " groups", tsdb, pTable->tableId, + pTable->uid, pGroupInfo->numOfTables, taosArrayGetSize(pGroupInfo->pGroupList)); taosArrayDestroy(res); @@ -3833,7 +3815,7 @@ int32_t tsdbQuerySTableByTagCond(STsdbRepo* tsdb, uint64_t uid, TSKEY skey, cons return terrno; } -int32_t tsdbGetOneTableGroup(STsdbRepo* tsdb, uint64_t uid, TSKEY startKey, STableGroupInfo* pGroupInfo) { +int32_t tsdbGetOneTableGroup(STsdb* tsdb, uint64_t uid, TSKEY startKey, STableGroupInfo* pGroupInfo) { if (tsdbRLockRepoMeta(tsdb) < 0) goto _error; STable* pTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid); @@ -3844,7 +3826,6 @@ int32_t tsdbGetOneTableGroup(STsdbRepo* tsdb, uint64_t uid, TSKEY startKey, STab } assert(pTable->type == TSDB_CHILD_TABLE || pTable->type == TSDB_NORMAL_TABLE || pTable->type == TSDB_STREAM_TABLE); - tsdbRefTable(pTable); if (tsdbUnlockRepoMeta(tsdb) < 0) goto _error; pGroupInfo->numOfTables = 1; @@ -3862,7 +3843,7 @@ int32_t tsdbGetOneTableGroup(STsdbRepo* tsdb, uint64_t uid, TSKEY startKey, STab return terrno; } -int32_t tsdbGetTableGroupFromIdList(STsdbRepo* tsdb, SArray* pTableIdList, STableGroupInfo* pGroupInfo) { +int32_t tsdbGetTableGroupFromIdList(STsdb* tsdb, SArray* pTableIdList, STableGroupInfo* pGroupInfo) { if (tsdbRLockRepoMeta(tsdb) < 0) { return terrno; } @@ -3889,8 +3870,6 @@ int32_t tsdbGetTableGroupFromIdList(STsdbRepo* tsdb, SArray* pTableIdList, STabl return terrno; } - tsdbRefTable(pTable); - STableKeyInfo info = {.pTable = pTable, .lastKey = id->key}; taosArrayPush(group, &info); } @@ -3938,42 +3917,42 @@ static void* destroyTableCheckInfo(SArray* pTableCheckInfo) { return NULL; } -void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) { - STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*)queryHandle; - if (pQueryHandle == NULL) { +void tsdbCleanupQueryHandle(tsdbReadHandleT queryHandle) { + STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*)queryHandle; + if (pTsdbReadHandle == NULL) { return; } - pQueryHandle->pColumns = doFreeColumnInfoData(pQueryHandle->pColumns); + pTsdbReadHandle->pColumns = doFreeColumnInfoData(pTsdbReadHandle->pColumns); - taosArrayDestroy(pQueryHandle->defaultLoadColumn); - tfree(pQueryHandle->pDataBlockInfo); - tfree(pQueryHandle->statis); + taosArrayDestroy(pTsdbReadHandle->defaultLoadColumn); + tfree(pTsdbReadHandle->pDataBlockInfo); + tfree(pTsdbReadHandle->statis); - if (!emptyQueryTimewindow(pQueryHandle)) { - tsdbMayUnTakeMemSnapshot(pQueryHandle); + if (!emptyQueryTimewindow(pTsdbReadHandle)) { + tsdbMayUnTakeMemSnapshot(pTsdbReadHandle); } else { - assert(pQueryHandle->pTableCheckInfo == NULL); + assert(pTsdbReadHandle->pTableCheckInfo == NULL); } - if (pQueryHandle->pTableCheckInfo != NULL) { - pQueryHandle->pTableCheckInfo = destroyTableCheckInfo(pQueryHandle->pTableCheckInfo); + if (pTsdbReadHandle->pTableCheckInfo != NULL) { + pTsdbReadHandle->pTableCheckInfo = destroyTableCheckInfo(pTsdbReadHandle->pTableCheckInfo); } - tsdbDestroyReadH(&pQueryHandle->rhelper); + tsdbDestroyReadH(&pTsdbReadHandle->rhelper); - tdFreeDataCols(pQueryHandle->pDataCols); - pQueryHandle->pDataCols = NULL; + tdFreeDataCols(pTsdbReadHandle->pDataCols); + pTsdbReadHandle->pDataCols = NULL; - pQueryHandle->prev = doFreeColumnInfoData(pQueryHandle->prev); - pQueryHandle->next = doFreeColumnInfoData(pQueryHandle->next); + pTsdbReadHandle->prev = doFreeColumnInfoData(pTsdbReadHandle->prev); + pTsdbReadHandle->next = doFreeColumnInfoData(pTsdbReadHandle->next); - SIOCostSummary* pCost = &pQueryHandle->cost; + SIOCostSummary* pCost = &pTsdbReadHandle->cost; tsdbDebug("%p :io-cost summary: head-file read cnt:%"PRIu64", head-file time:%"PRIu64" us, statis-info:%"PRId64" us, datablock:%" PRId64" us, check data:%"PRId64" us, 0x%"PRIx64, - pQueryHandle, pCost->headFileLoad, pCost->headFileLoadTime, pCost->statisInfoLoadTime, pCost->blockLoadTime, pCost->checkForNextTime, pQueryHandle->qId); + pTsdbReadHandle, pCost->headFileLoad, pCost->headFileLoadTime, pCost->statisInfoLoadTime, pCost->blockLoadTime, pCost->checkForNextTime, pTsdbReadHandle->qId); - tfree(pQueryHandle); + tfree(pTsdbReadHandle); } void tsdbDestroyTableGroup(STableGroupInfo *pGroupList) { @@ -4257,3 +4236,4 @@ void getTableListfromSkipList(tExprNode *pExpr, SSkipList *pSkipList, SArray *re //apply the hierarchical filter expression to every node in skiplist to find the qualified nodes applyFilterToSkipListNode(pSkipList, pExpr, result, param); } +#endif \ No newline at end of file diff --git a/source/libs/executor/CMakeLists.txt b/source/libs/executor/CMakeLists.txt index ba941ab22d..04b5fab4bf 100644 --- a/source/libs/executor/CMakeLists.txt +++ b/source/libs/executor/CMakeLists.txt @@ -1,12 +1,27 @@ aux_source_directory(src EXECUTOR_SRC) -add_library(executor ${EXECUTOR_SRC}) +#add_library(executor ${EXECUTOR_SRC}) + + +#target_link_libraries( +# executor +# PRIVATE os util common function parser planner qcom tsdb +#) + +add_library(executor STATIC ${EXECUTOR_SRC}) +#set_target_properties(executor PROPERTIES +# IMPORTED_LOCATION "${CMAKE_CURRENT_SOURCE_DIR}/libexecutor.a" +# INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_SOURCE_DIR}/include/libs/executor" +# ) +target_link_libraries(executor + PRIVATE os util common function parser planner qcom tsdb + ) + target_include_directories( - executor - PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/executor" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" + executor + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/executor" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) -target_link_libraries( - executor - PRIVATE os util common function parser planner qcom -) \ No newline at end of file +#if(${BUILD_TEST}) + ADD_SUBDIRECTORY(test) +#endif(${BUILD_TEST}) \ No newline at end of file diff --git a/source/libs/executor/inc/dataSinkMgt.h b/source/libs/executor/inc/dataSinkMgt.h index d13423b25d..d0057a213a 100644 --- a/source/libs/executor/inc/dataSinkMgt.h +++ b/source/libs/executor/inc/dataSinkMgt.h @@ -33,7 +33,7 @@ struct SDataSink; struct SSDataBlock; typedef struct SDataSinkMgtCfg { - uint32_t maxDataBlockNum; + uint32_t maxDataBlockNum; // todo: this should be numOfRows? uint32_t maxDataBlockNumPerQuery; } SDataSinkMgtCfg; diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 2c1bf71638..79766b22ac 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -38,7 +38,7 @@ #define GET_RES_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t)) #define GET_RES_EXT_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t) + POINTER_BYTES) -#define GET_QID(_r) (((SQInfo*)((_r)->qinfo))->qId) +#define GET_TASKID(_t) (((SExecTaskInfo*)(_t))->id.queryId) #define curTimeWindowIndex(_winres) ((_winres)->curIndex) @@ -157,6 +157,6 @@ int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo); int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, struct STaskRuntimeEnv *pRuntimeEnv, int32_t* offset); -int32_t initUdfInfo(struct SUdfInfo* pUdfInfo); +//int32_t initUdfInfo(struct SUdfInfo* pUdfInfo); #endif // TDENGINE_QUERYUTIL_H diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index a268215d3d..86d708856f 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -29,15 +29,8 @@ #include "tpagedfile.h" #include "planner.h" - struct SColumnFilterElem; -typedef struct { - uint32_t numOfTables; - SArray *pGroupList; - SHashObj *map; // speedup acquire the tableQueryInfo by table uid -} STableGroupInfo; - typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order); #define IS_QUERY_KILLED(_q) ((_q)->code == TSDB_CODE_TSC_QUERY_CANCELLED) @@ -51,19 +44,19 @@ typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int #define NEEDTO_COMPRESS_QUERY(size) ((size) > tsCompressColData? 1 : 0) enum { - // when query starts to execute, this status will set - QUERY_NOT_COMPLETED = 0x1u, + // when this task starts to execute, this status will set + TASK_NOT_COMPLETED = 0x1u, - /* query is over + /* Task is over * 1. this status is used in one row result query process, e.g., count/sum/first/last/ avg...etc. * 2. when all data within queried time window, it is also denoted as query_completed */ - QUERY_COMPLETED = 0x2u, + TASK_COMPLETED = 0x2u, /* when the result is not completed return to client, this status will be * usually used in case of interval query with interpolation option */ - QUERY_OVER = 0x4u, + TASK_OVER = 0x4u, }; typedef struct SResultRowCell { @@ -129,6 +122,7 @@ typedef struct { } SOperatorProfResult; typedef struct STaskCostInfo { + int64_t created; int64_t start; int64_t end; @@ -246,13 +240,14 @@ typedef struct STaskIdInfo { uint64_t taskId; // this is a subplan id } STaskIdInfo; -typedef struct STaskInfo { +typedef struct SExecTaskInfo { STaskIdInfo id; char *content; uint32_t status; STimeWindow window; STaskCostInfo cost; int64_t owner; // if it is in execution + int32_t code; STableGroupInfo tableqinfoGroupInfo; // this is a group array list, including SArray structure pthread_mutex_t lock; // used to synchronize the rsp/query threads @@ -260,8 +255,9 @@ typedef struct STaskInfo { // int32_t dataReady; // denote if query result is ready or not // void* rspContext; // response context char *sql; // query sql string - jmp_buf env; -} STaskInfo; + jmp_buf env; // + struct SOperatorInfo *pRoot; +} SExecTaskInfo; typedef struct STaskRuntimeEnv { jmp_buf env; @@ -269,7 +265,7 @@ typedef struct STaskRuntimeEnv { uint32_t status; // query status void* qinfo; uint8_t scanFlag; // denotes reversed scan of data or not - void* pQueryHandle; + void* pTsdbReadHandle; int32_t prevGroupId; // previous executed group id bool enableGroupData; @@ -314,8 +310,8 @@ typedef struct SOperatorInfo { char *name; // name, used to show the query execution plan void *info; // extension attribution SExprInfo *pExpr; - STaskRuntimeEnv *pRuntimeEnv; - STaskInfo *pTaskInfo; + STaskRuntimeEnv *pRuntimeEnv; // todo remove it + SExecTaskInfo *pTaskInfo; struct SOperatorInfo **pDownstream; // downstram pointer list int32_t numOfDownstream; // number of downstream. The value is always ONE expect for join operator @@ -376,7 +372,7 @@ typedef struct STaskParam { } STaskParam; typedef struct STableScanInfo { - void *pQueryHandle; + void *pTsdbReadHandle; int32_t numOfBlocks; int32_t numOfSkipped; int32_t numOfBlockStatis; @@ -544,7 +540,7 @@ typedef struct SOrderOperatorInfo { void appendUpstream(SOperatorInfo* p, SOperatorInfo* pUpstream); SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, STaskRuntimeEnv* pRuntimeEnv, int32_t repeatTime, int32_t reverseTime); -SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime); +SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, SExecTaskInfo* pTaskInfo); SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, STaskRuntimeEnv* pRuntimeEnv); SOperatorInfo* createAggregateOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); @@ -572,11 +568,11 @@ SOperatorInfo* createFilterOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pUpstream, int32_t numOfUpstream, SSchema* pSchema, int32_t numOfOutput); SOperatorInfo* createOrderOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, SOrder* pOrderVal); -SSDataBlock* doGlobalAggregate(void* param, bool* newgroup); -SSDataBlock* doMultiwayMergeSort(void* param, bool* newgroup); -SSDataBlock* doSLimit(void* param, bool* newgroup); +//SSDataBlock* doGlobalAggregate(void* param, bool* newgroup); +//SSDataBlock* doMultiwayMergeSort(void* param, bool* newgroup); +//SSDataBlock* doSLimit(void* param, bool* newgroup); -int32_t doCreateFilterInfo(SColumnInfo* pCols, int32_t numOfCols, int32_t numOfFilterCols, SSingleColumnFilterInfo** pFilterInfo, uint64_t qId); +//int32_t doCreateFilterInfo(SColumnInfo* pCols, int32_t numOfCols, int32_t numOfFilterCols, SSingleColumnFilterInfo** pFilterInfo, uint64_t qId); void doSetFilterColumnInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, SSDataBlock* pBlock); bool doFilterDataBlock(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, int32_t numOfRows, int8_t* p); void doCompactSDataBlock(SSDataBlock* pBlock, int32_t numOfRows, int8_t* p); @@ -617,14 +613,14 @@ STableQueryInfo* createTmpTableQueryInfo(STimeWindow win); int32_t buildArithmeticExprFromMsg(SExprInfo *pArithExprInfo, void *pQueryMsg); -bool isQueryKilled(SQInfo *pQInfo); +bool isTaskKilled(SExecTaskInfo *pTaskInfo); int32_t checkForQueryBuf(size_t numOfTables); bool checkNeedToCompressQueryCol(SQInfo *pQInfo); bool doBuildResCheck(SQInfo* pQInfo); void setQueryStatus(STaskRuntimeEnv *pRuntimeEnv, int8_t status); bool onlyQueryTags(STaskAttr* pQueryAttr); -void destroyUdfInfo(struct SUdfInfo* pUdfInfo); +//void destroyUdfInfo(struct SUdfInfo* pUdfInfo); bool isValidQInfo(void *param); @@ -644,5 +640,7 @@ void freeQueryAttr(STaskAttr *pQuery); int32_t getMaximumIdleDurationSec(); void doInvokeUdf(struct SUdfInfo* pUdfInfo, SQLFunctionCtx *pCtx, int32_t idx, int32_t type); +void setTaskStatus(SExecTaskInfo *pTaskInfo, int8_t status); +int32_t doCreateExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, void* readerHandle); #endif // TDENGINE_EXECUTORIMPL_H diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index e8ecffb72c..5253add5af 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -547,7 +547,7 @@ static UNUSED_FUNC int32_t mergeIntoGroupResultImpl(STaskRuntimeEnv *pRuntimeEnv pTableQueryInfoList = malloc(POINTER_BYTES * size); if (pTableQueryInfoList == NULL || posList == NULL || pGroupResInfo->pRows == NULL || pGroupResInfo->pRows == NULL) { -// qError("QInfo:%"PRIu64" failed alloc memory", GET_QID(pRuntimeEnv)); +// qError("QInfo:%"PRIu64" failed alloc memory", GET_TASKID(pRuntimeEnv)); code = TSDB_CODE_QRY_OUT_OF_MEMORY; goto _end; } @@ -619,7 +619,7 @@ static UNUSED_FUNC int32_t mergeIntoGroupResultImpl(STaskRuntimeEnv *pRuntimeEnv int64_t endt = taosGetTimestampMs(); -// qDebug("QInfo:%"PRIx64" result merge completed for group:%d, elapsed time:%" PRId64 " ms", GET_QID(pRuntimeEnv), +// qDebug("QInfo:%"PRIx64" result merge completed for group:%d, elapsed time:%" PRId64 " ms", GET_TASKID(pRuntimeEnv), // pGroupResInfo->currentGroup, endt - startt); _end: @@ -641,13 +641,13 @@ int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, STaskRuntimeEnv* pRun break; } -// qDebug("QInfo:%"PRIu64" no result in group %d, continue", GET_QID(pRuntimeEnv), pGroupResInfo->currentGroup); +// qDebug("QInfo:%"PRIu64" no result in group %d, continue", GET_TASKID(pRuntimeEnv), pGroupResInfo->currentGroup); cleanupGroupResInfo(pGroupResInfo); incNextGroup(pGroupResInfo); } // int64_t elapsedTime = taosGetTimestampUs() - st; -// qDebug("QInfo:%"PRIu64" merge res data into group, index:%d, total group:%d, elapsed time:%" PRId64 "us", GET_QID(pRuntimeEnv), +// qDebug("QInfo:%"PRIu64" merge res data into group, index:%d, total group:%d, elapsed time:%" PRId64 "us", GET_TASKID(pRuntimeEnv), // pGroupResInfo->currentGroup, pGroupResInfo->totalGroup, elapsedTime); return TSDB_CODE_SUCCESS; diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index b98b7fef5c..778521ae2d 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -13,11 +13,12 @@ * along with this program. If not, see . */ +#include +#include "exception.h" #include "os.h" #include "tcache.h" #include "tglobal.h" #include "tmsg.h" -#include "exception.h" #include "thash.h" #include "executorimpl.h" @@ -66,152 +67,24 @@ void freeParam(STaskParam *param) { tfree(param->prevResult); } -// todo parse json to get the operator tree. +int32_t qCreateExecTask(void* tsdb, int32_t vgId, SSubplan* pSubplan, qTaskInfo_t* pTaskInfo) { + assert(tsdb != NULL && pSubplan != NULL); -int32_t qCreateTask(void* tsdb, int32_t vgId, void* pQueryMsg, qTaskInfo_t* pTaskInfo, uint64_t taskId) { - assert(pQueryMsg != NULL && tsdb != NULL); - - int32_t code = TSDB_CODE_SUCCESS; -#if 0 - STaskParam param = {0}; - code = convertQueryMsg(pQueryMsg, ¶m); + int32_t code = doCreateExecTaskInfo(pSubplan, (SExecTaskInfo**) pTaskInfo, tsdb); if (code != TSDB_CODE_SUCCESS) { - goto _over; + goto _error; } - if (pQueryMsg->numOfTables <= 0) { - qError("Invalid number of tables to query, numOfTables:%d", pQueryMsg->numOfTables); - code = TSDB_CODE_QRY_INVALID_MSG; - goto _over; - } - - if (param.pTableIdList == NULL || taosArrayGetSize(param.pTableIdList) == 0) { - qError("qmsg:%p, SQueryTableMsg wrong format", pQueryMsg); - code = TSDB_CODE_QRY_INVALID_MSG; - goto _over; - } - - SQueriedTableInfo info = { .numOfTags = pQueryMsg->numOfTags, .numOfCols = pQueryMsg->numOfCols, .colList = pQueryMsg->tableCols}; - if ((code = createQueryFunc(&info, pQueryMsg->numOfOutput, ¶m.pExprs, param.pExpr, param.pTagColumnInfo, - pQueryMsg->queryType, pQueryMsg, param.pUdfInfo)) != TSDB_CODE_SUCCESS) { - goto _over; - } - - if (param.pSecExpr != NULL) { - if ((code = createIndirectQueryFuncExprFromMsg(pQueryMsg, pQueryMsg->secondStageOutput, ¶m.pSecExprs, param.pSecExpr, param.pExprs, param.pUdfInfo)) != TSDB_CODE_SUCCESS) { - goto _over; - } - } - - if (param.colCond != NULL) { - if ((code = createQueryFilter(param.colCond, pQueryMsg->colCondLen, ¶m.pFilters)) != TSDB_CODE_SUCCESS) { - goto _over; - } - } - - param.pGroupbyExpr = createGroupbyExprFromMsg(pQueryMsg, param.pGroupColIndex, &code); - if ((param.pGroupbyExpr == NULL && pQueryMsg->numOfGroupCols != 0) || code != TSDB_CODE_SUCCESS) { - goto _over; - } - - bool isSTableQuery = false; - STableGroupInfo tableGroupInfo = {0}; - int64_t st = taosGetTimestampUs(); - - if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_TABLE_QUERY)) { - STableIdInfo *id = taosArrayGet(param.pTableIdList, 0); - - qDebug("qmsg:%p query normal table, uid:%"PRId64", tid:%d", pQueryMsg, id->uid, id->tid); - if ((code = tsdbGetOneTableGroup(tsdb, id->uid, pQueryMsg->window.skey, &tableGroupInfo)) != TSDB_CODE_SUCCESS) { - goto _over; - } - } else if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY|TSDB_QUERY_TYPE_STABLE_QUERY)) { - isSTableQuery = true; - - // also note there's possibility that only one table in the super table - if (!TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY)) { - STableIdInfo *id = taosArrayGet(param.pTableIdList, 0); - - // group by normal column, do not pass the group by condition to tsdb to group table into different group - int32_t numOfGroupByCols = pQueryMsg->numOfGroupCols; - if (pQueryMsg->numOfGroupCols == 1 && !TSDB_COL_IS_TAG(param.pGroupColIndex->flag)) { - numOfGroupByCols = 0; - } - - qDebug("qmsg:%p query stable, uid:%"PRIu64", tid:%d", pQueryMsg, id->uid, id->tid); - code = tsdbQuerySTableByTagCond(tsdb, id->uid, pQueryMsg->window.skey, param.tagCond, pQueryMsg->tagCondLen, - pQueryMsg->tagNameRelType, param.tbnameCond, &tableGroupInfo, param.pGroupColIndex, numOfGroupByCols); - - if (code != TSDB_CODE_SUCCESS) { - qError("qmsg:%p failed to query stable, reason: %s", pQueryMsg, tstrerror(code)); - goto _over; - } - } else { - code = tsdbGetTableGroupFromIdList(tsdb, param.pTableIdList, &tableGroupInfo); - if (code != TSDB_CODE_SUCCESS) { - goto _over; - } - - qDebug("qmsg:%p query on %u tables in one group from client", pQueryMsg, tableGroupInfo.numOfTables); - } - - int64_t el = taosGetTimestampUs() - st; - qDebug("qmsg:%p tag filter completed, numOfTables:%u, elapsed time:%"PRId64"us", pQueryMsg, tableGroupInfo.numOfTables, el); - } else { - assert(0); - } - - code = checkForQueryBuf(tableGroupInfo.numOfTables); - if (code != TSDB_CODE_SUCCESS) { // not enough query buffer, abort - goto _over; - } - - assert(pQueryMsg->stableQuery == isSTableQuery); - (*pTaskInfo) = createQInfoImpl(pQueryMsg, param.pGroupbyExpr, param.pExprs, param.pSecExprs, &tableGroupInfo, - param.pTagColumnInfo, param.pFilters, vgId, param.sql, qId, param.pUdfInfo); - - param.sql = NULL; - param.pExprs = NULL; - param.pSecExprs = NULL; - param.pGroupbyExpr = NULL; - param.pTagColumnInfo = NULL; - param.pFilters = NULL; - - if ((*pTaskInfo) == NULL) { - code = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto _over; - } - param.pUdfInfo = NULL; - - code = initQInfo(&pQueryMsg->tsBuf, tsdb, NULL, *pTaskInfo, ¶m, (char*)pQueryMsg, pQueryMsg->prevResultLen, NULL); - - _over: - if (param.pGroupbyExpr != NULL) { - taosArrayDestroy(param.pGroupbyExpr->columnInfo); - } - - tfree(param.colCond); - - destroyUdfInfo(param.pUdfInfo); - - taosArrayDestroy(param.pTableIdList); - param.pTableIdList = NULL; - - freeParam(¶m); - - for (int32_t i = 0; i < pQueryMsg->numOfCols; i++) { - SColumnInfo* column = pQueryMsg->tableCols + i; - freeColumnFilterInfo(column->flist.filterInfo, column->flist.numOfFilters); - } - - filterFreeInfo(param.pFilters); - - //pTaskInfo already freed in initQInfo, but *pTaskInfo may not pointer to null; + SDataSinkMgtCfg cfg = {.maxDataBlockNum = 1000}; + code = dsDataSinkMgtInit(&cfg); if (code != TSDB_CODE_SUCCESS) { - *pTaskInfo = NULL; + goto _error; } -#endif + DataSinkHandle pHandle = NULL; + code = dsCreateDataSinker(pSubplan->pDataSink, &pHandle); + + _error: // if failed to add ref for all tables in this query, abort current query return code; } @@ -250,7 +123,7 @@ int waitMoment(SQInfo* pQInfo){ while(used_ms < ms) { taosMsleep(1000); used_ms += 1000; - if(isQueryKilled(pQInfo)){ + if(isTaskKilled(pQInfo)){ printf("test check query is canceled, sleep break.%s\n", pQInfo->sql); break; } @@ -261,68 +134,66 @@ int waitMoment(SQInfo* pQInfo){ } #endif -bool qExecTask(qTaskInfo_t qinfo, uint64_t *qId) { - SQInfo *pQInfo = (SQInfo *)qinfo; - assert(pQInfo && pQInfo->signature == pQInfo); +bool qExecTask(qTaskInfo_t qinfo) { + SExecTaskInfo * pTaskInfo = (SExecTaskInfo *)qinfo; int64_t threadId = taosGetSelfPthreadId(); int64_t curOwner = 0; - if ((curOwner = atomic_val_compare_exchange_64(&pQInfo->owner, 0, threadId)) != 0) { - qError("QInfo:0x%"PRIx64"-%p qhandle is now executed by thread:%p", pQInfo->qId, pQInfo, (void*) curOwner); - pQInfo->code = TSDB_CODE_QRY_IN_EXEC; + if ((curOwner = atomic_val_compare_exchange_64(&pTaskInfo->owner, 0, threadId)) != 0) { + qError("QInfo:0x%"PRIx64"-%p qhandle is now executed by thread:%p", GET_TASKID(pTaskInfo), pTaskInfo, (void*) curOwner); + pTaskInfo->code = TSDB_CODE_QRY_IN_EXEC; return false; } - *qId = pQInfo->qId; - if(pQInfo->startExecTs == 0) - pQInfo->startExecTs = taosGetTimestampMs(); - - if (isQueryKilled(pQInfo)) { - qDebug("QInfo:0x%"PRIx64" it is already killed, abort", pQInfo->qId); - return doBuildResCheck(pQInfo); + if(pTaskInfo->cost.start == 0) { + pTaskInfo->cost.start = taosGetTimestampMs(); } - STaskRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - if (pRuntimeEnv->tableqinfoGroupInfo.numOfTables == 0) { - qDebug("QInfo:0x%"PRIx64" no table exists for query, abort", pQInfo->qId); -// setTaskStatus(pRuntimeEnv, QUERY_COMPLETED); - return doBuildResCheck(pQInfo); + if (isTaskKilled(pTaskInfo)) { + qDebug("QInfo:0x%"PRIx64" it is already killed, abort", GET_TASKID(pTaskInfo)); +// return doBuildResCheck(pTaskInfo); } +// STaskRuntimeEnv* pRuntimeEnv = &pTaskInfo->runtimeEnv; +// if (pTaskInfo->tableqinfoGroupInfo.numOfTables == 0) { +// qDebug("QInfo:0x%"PRIx64" no table exists for query, abort", GET_TASKID(pTaskInfo)); +// setTaskStatus(pTaskInfo, TASK_COMPLETED); +// return doBuildResCheck(pTaskInfo); +// } + // error occurs, record the error code and return to client - int32_t ret = setjmp(pQInfo->runtimeEnv.env); + int32_t ret = setjmp(pTaskInfo->env); if (ret != TSDB_CODE_SUCCESS) { - publishQueryAbortEvent(pQInfo, ret); - pQInfo->code = ret; - qDebug("QInfo:0x%"PRIx64" query abort due to error/cancel occurs, code:%s", pQInfo->qId, tstrerror(pQInfo->code)); - return doBuildResCheck(pQInfo); + publishQueryAbortEvent(pTaskInfo, ret); + pTaskInfo->code = ret; + qDebug("QInfo:0x%"PRIx64" query abort due to error/cancel occurs, code:%s", GET_TASKID(pTaskInfo), tstrerror(pTaskInfo->code)); +// return doBuildResCheck(pTaskInfo); } - qDebug("QInfo:0x%"PRIx64" query task is launched", pQInfo->qId); + qDebug("QInfo:0x%"PRIx64" query task is launched", GET_TASKID(pTaskInfo)); bool newgroup = false; - publishOperatorProfEvent(pRuntimeEnv->proot, QUERY_PROF_BEFORE_OPERATOR_EXEC); + publishOperatorProfEvent(pTaskInfo->pRoot, QUERY_PROF_BEFORE_OPERATOR_EXEC); int64_t st = taosGetTimestampUs(); - pRuntimeEnv->outputBuf = pRuntimeEnv->proot->exec(pRuntimeEnv->proot, &newgroup); - pQInfo->summary.elapsedTime += (taosGetTimestampUs() - st); -#ifdef TEST_IMPL - waitMoment(pQInfo); -#endif - publishOperatorProfEvent(pRuntimeEnv->proot, QUERY_PROF_AFTER_OPERATOR_EXEC); - pRuntimeEnv->resultInfo.total += GET_NUM_OF_RESULTS(pRuntimeEnv); + void* pOutput = pTaskInfo->pRoot->exec(pTaskInfo->pRoot, &newgroup); + // todo put the result into sink node. - if (isQueryKilled(pQInfo)) { - qDebug("QInfo:0x%"PRIx64" query is killed", pQInfo->qId); - } else if (GET_NUM_OF_RESULTS(pRuntimeEnv) == 0) { - qDebug("QInfo:0x%"PRIx64" over, %u tables queried, total %"PRId64" rows returned", pQInfo->qId, pRuntimeEnv->tableqinfoGroupInfo.numOfTables, - pRuntimeEnv->resultInfo.total); + + pTaskInfo->cost.elapsedTime += (taosGetTimestampUs() - st); + + publishOperatorProfEvent(pTaskInfo->pRoot, QUERY_PROF_AFTER_OPERATOR_EXEC); + + if (isTaskKilled(pTaskInfo)) { + qDebug("QInfo:0x%"PRIx64" query is killed", GET_TASKID(pTaskInfo)); +// } else if (GET_NUM_OF_RESULTS(pRuntimeEnv) == 0) { +// qDebug("QInfo:0x%"PRIx64" over, %u tables queried, total %"PRId64" rows returned", pTaskInfo->qId, pRuntimeEnv->tableqinfoGroupInfo.numOfTables, +// pRuntimeEnv->resultInfo.total); } else { - qDebug("QInfo:0x%"PRIx64" query paused, %d rows returned, total:%" PRId64 " rows", pQInfo->qId, - GET_NUM_OF_RESULTS(pRuntimeEnv), pRuntimeEnv->resultInfo.total); +// qDebug("QInfo:0x%"PRIx64" query paused, %d rows returned, total:%" PRId64 " rows", pTaskInfo->qId, +// GET_NUM_OF_RESULTS(pRuntimeEnv), pRuntimeEnv->resultInfo.total); } - - return doBuildResCheck(pQInfo); +// return doBuildResCheck(pTaskInfo); } int32_t qRetrieveQueryResultInfo(qTaskInfo_t qinfo, bool* buildRes, void* pRspContext) { @@ -398,13 +269,13 @@ int32_t qKillTask(qTaskInfo_t qinfo) { } int32_t qIsTaskCompleted(qTaskInfo_t qinfo) { - SQInfo *pQInfo = (SQInfo *)qinfo; + SExecTaskInfo *pTaskInfo = (SExecTaskInfo *)qinfo; - if (pQInfo == NULL || !isValidQInfo(pQInfo)) { + if (pTaskInfo == NULL /*|| !isValidQInfo(pTaskInfo)*/) { return TSDB_CODE_QRY_INVALID_QHANDLE; } - return isQueryKilled(pQInfo) || Q_STATUS_EQUAL(pQInfo->runtimeEnv.status, QUERY_OVER); + return isTaskKilled(pTaskInfo) || Q_STATUS_EQUAL(pTaskInfo->status, TASK_OVER); } void qDestroyTask(qTaskInfo_t qHandle) { diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 1d2740f0e0..b5cd9f9287 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -26,6 +26,7 @@ #include "thash.h" #include "ttypes.h" #include "query.h" +#include "tsdb.h" #define IS_MAIN_SCAN(runtime) ((runtime)->scanFlag == MAIN_SCAN) #define IS_REVERSE_SCAN(runtime) ((runtime)->scanFlag == REVERSE_SCAN) @@ -204,12 +205,10 @@ static void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput); static void destroyAggOperatorInfo(void* param, int32_t numOfOutput); static void destroyOperatorInfo(SOperatorInfo* pOperator); -void setTaskStatus(STaskInfo *pTaskInfo, int8_t status); - static void doSetOperatorCompleted(SOperatorInfo* pOperator) { pOperator->status = OP_EXEC_DONE; if (pOperator->pTaskInfo != NULL) { - setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); + setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); } } @@ -1509,7 +1508,7 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SGroupbyOperatorInfo *pIn int16_t type = pColInfoData->info.type; if (type == TSDB_DATA_TYPE_FLOAT || type == TSDB_DATA_TYPE_DOUBLE) { - //qError("QInfo:0x%"PRIx64" group by not supported on double/float columns, abort", GET_QID(pRuntimeEnv)); + //qError("QInfo:0x%"PRIx64" group by not supported on double/float columns, abort", GET_TASKID(pRuntimeEnv)); return; } @@ -1908,7 +1907,7 @@ static SQLFunctionCtx* createSQLFunctionCtx(STaskRuntimeEnv* pRuntimeEnv, SExprI pCtx->param[2].i = pQueryAttr->window.ekey; pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; } else if (functionId == FUNCTION_ARITHM) { - pCtx->param[1].pz = (char*) getScalarFuncSupport(pRuntimeEnv->scalarSup, i); +// pCtx->param[1].pz = (char*) getScalarFuncSupport(pRuntimeEnv->scalarSup, i); } } @@ -1940,7 +1939,7 @@ static void* destroySQLFunctionCtx(SQLFunctionCtx* pCtx, int32_t numOfOutput) { } static int32_t setupQueryRuntimeEnv(STaskRuntimeEnv *pRuntimeEnv, int32_t numOfTables, SArray* pOperator, void* merger) { - //qDebug("QInfo:0x%"PRIx64" setup runtime env", GET_QID(pRuntimeEnv)); + //qDebug("QInfo:0x%"PRIx64" setup runtime env", GET_TASKID(pRuntimeEnv)); STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; pRuntimeEnv->prevGroupId = INT32_MIN; @@ -1977,7 +1976,7 @@ static int32_t setupQueryRuntimeEnv(STaskRuntimeEnv *pRuntimeEnv, int32_t numOfT } } - //qDebug("QInfo:0x%"PRIx64" init runtime environment completed", GET_QID(pRuntimeEnv)); + //qDebug("QInfo:0x%"PRIx64" init runtime environment completed", GET_TASKID(pRuntimeEnv)); // group by normal column, sliding window query, interval query are handled by interval query processor // interval (down sampling operation) @@ -2164,8 +2163,8 @@ _clean: static void doFreeQueryHandle(STaskRuntimeEnv* pRuntimeEnv) { STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; -// tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle); - pRuntimeEnv->pQueryHandle = NULL; +// tsdbCleanupQueryHandle(pRuntimeEnv->pTsdbReadHandle); + pRuntimeEnv->pTsdbReadHandle = NULL; // SMemRef* pMemRef = &pQueryAttr->memRef; // assert(pMemRef->ref == 0 && pMemRef->snapshot.imem == NULL && pMemRef->snapshot.mem == NULL); @@ -2191,7 +2190,7 @@ static void teardownQueryRuntimeEnv(STaskRuntimeEnv *pRuntimeEnv) { //qDebug("QInfo:0x%"PRIx64" teardown runtime env", pQInfo->qId); destroyScalarFuncSupport(pRuntimeEnv->scalarSup, pQueryAttr->numOfOutput); - destroyUdfInfo(pRuntimeEnv->pUdfInfo); +// destroyUdfInfo(pRuntimeEnv->pUdfInfo); destroyResultBuf(pRuntimeEnv->pResultBuf); doFreeQueryHandle(pRuntimeEnv); @@ -2224,18 +2223,18 @@ static bool needBuildResAfterQueryComplete(SQInfo* pQInfo) { return pQInfo->rspContext != NULL; } -bool isQueryKilled(SQInfo *pQInfo) { - if (IS_QUERY_KILLED(pQInfo)) { +bool isTaskKilled(SExecTaskInfo *pTaskInfo) { + if (IS_QUERY_KILLED(pTaskInfo)) { return true; } // query has been executed more than tsShellActivityTimer, and the retrieve has not arrived // abort current query execution. - if (pQInfo->owner != 0 && ((taosGetTimestampSec() - pQInfo->startExecTs/1000) > getMaximumIdleDurationSec()) && - (!needBuildResAfterQueryComplete(pQInfo))) { + if (pTaskInfo->owner != 0 && ((taosGetTimestampSec() - pTaskInfo->cost.start/1000) > 10*getMaximumIdleDurationSec()) + /*(!needBuildResAfterQueryComplete(pTaskInfo))*/) { - assert(pQInfo->startExecTs != 0); - //qDebug("QInfo:%" PRIu64 " retrieve not arrive beyond %d ms, abort current query execution, start:%" PRId64 + assert(pTaskInfo->cost.start != 0); +// qDebug("QInfo:%" PRIu64 " retrieve not arrive beyond %d ms, abort current query execution, start:%" PRId64 // ", current:%d", pQInfo->qId, 1, pQInfo->startExecTs, taosGetTimestampSec()); return true; } @@ -2887,7 +2886,7 @@ void doSetFilterColumnInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFi } } -int32_t loadDataBlock(STaskInfo *pTaskInfo, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { +int32_t loadDataBlock(SExecTaskInfo *pTaskInfo, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { STaskCostInfo* pCost = &pTaskInfo->cost; pCost->totalBlocks += 1; @@ -2896,13 +2895,15 @@ int32_t loadDataBlock(STaskInfo *pTaskInfo, STableScanInfo* pTableScanInfo, SSDa pCost->totalCheckedRows += pBlock->info.rows; pCost->loadBlocks += 1; -// pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pQueryHandle, NULL); + pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pTsdbReadHandle, NULL); if (pBlock->pDataBlock == NULL) { return terrno; + } else { + return TSDB_CODE_SUCCESS; } } -int32_t loadDataBlockOnDemand(STaskInfo *pTaskInfo, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { +int32_t loadDataBlockOnDemand(SExecTaskInfo *pTaskInfo, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { *status = BLK_DATA_NO_NEEDED; pBlock->pDataBlock = NULL; @@ -2991,10 +2992,10 @@ int32_t loadDataBlockOnDemand(STaskInfo *pTaskInfo, STableScanInfo* pTableScanIn } else if ((*status) == BLK_DATA_STATIS_NEEDED) { // this function never returns error? pCost->loadBlockStatis += 1; -// tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pQueryHandle, &pBlock->pBlockAgg); +// tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pTsdbReadHandle, &pBlock->pBlockAgg); if (pBlock->pBlockAgg == NULL) { // data block statistics does not exist, load data block -// pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pQueryHandle, NULL); +// pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pTsdbReadHandle, NULL); pCost->totalCheckedRows += pBlock->info.rows; } } else { @@ -3002,7 +3003,7 @@ int32_t loadDataBlockOnDemand(STaskInfo *pTaskInfo, STableScanInfo* pTableScanIn // load the data block statistics to perform further filter pCost->loadBlockStatis += 1; -// tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pQueryHandle, &pBlock->pBlockAgg); +// tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pTsdbReadHandle, &pBlock->pBlockAgg); if (pQueryAttr->topBotQuery && pBlock->pBlockAgg != NULL) { { // set previous window @@ -3048,7 +3049,7 @@ int32_t loadDataBlockOnDemand(STaskInfo *pTaskInfo, STableScanInfo* pTableScanIn pCost->totalCheckedRows += pBlockInfo->rows; pCost->loadBlocks += 1; -// pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pQueryHandle, NULL); +// pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pTsdbReadHandle, NULL); // if (pBlock->pDataBlock == NULL) { // return terrno; // } @@ -3449,12 +3450,12 @@ void initCtxOutputBuffer(SQLFunctionCtx* pCtx, int32_t size) { } } -void setTaskStatus(STaskInfo *pTaskInfo, int8_t status) { - if (status == QUERY_NOT_COMPLETED) { +void setTaskStatus(SExecTaskInfo *pTaskInfo, int8_t status) { + if (status == TASK_NOT_COMPLETED) { pTaskInfo->status = status; } else { // QUERY_NOT_COMPLETED is not compatible with any other status, so clear its position first - CLEAR_QUERY_STATUS(pTaskInfo, QUERY_NOT_COMPLETED); + CLEAR_QUERY_STATUS(pTaskInfo, TASK_NOT_COMPLETED); pTaskInfo->status |= status; } } @@ -3476,6 +3477,10 @@ static void setupEnvForReverseScan(STableScanInfo *pTableScanInfo, SQLFunctionCt SWITCH_ORDER(pTableScanInfo->order); setupQueryRangeForReverseScan(pTableScanInfo); + + pTableScanInfo->times = 1; + pTableScanInfo->current = 0; + pTableScanInfo->reverseTimes = 0; } void finalizeQueryResult(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset) { @@ -3700,10 +3705,10 @@ void setCtxTagForJoin(STaskRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, SExprI // // int16_t tagType = pCtx[0].tag.nType; // if (tagType == TSDB_DATA_TYPE_BINARY || tagType == TSDB_DATA_TYPE_NCHAR) { -// //qDebug("QInfo:0x%"PRIx64" set tag value for join comparison, colId:%" PRId64 ", val:%s", GET_QID(pRuntimeEnv), +// //qDebug("QInfo:0x%"PRIx64" set tag value for join comparison, colId:%" PRId64 ", val:%s", GET_TASKID(pRuntimeEnv), //// pExprInfo->base.param[0].i, pCtx[0].tag.pz); // } else { -// //qDebug("QInfo:0x%"PRIx64" set tag value for join comparison, colId:%" PRId64 ", val:%" PRId64, GET_QID(pRuntimeEnv), +// //qDebug("QInfo:0x%"PRIx64" set tag value for join comparison, colId:%" PRId64 ", val:%" PRId64, GET_TASKID(pRuntimeEnv), //// pExprInfo->base.param[0].i, pCtx[0].tag.i); // } // } @@ -3723,9 +3728,9 @@ int32_t setTimestampListJoinInfo(STaskRuntimeEnv* pRuntimeEnv, SVariant* pTag, S // failed to find data with the specified tag value and vnodeId if (!tsBufIsValidElem(&elem)) { if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { - //qError("QInfo:0x%"PRIx64" failed to find tag:%s in ts_comp", GET_QID(pRuntimeEnv), pTag->pz); + //qError("QInfo:0x%"PRIx64" failed to find tag:%s in ts_comp", GET_TASKID(pRuntimeEnv), pTag->pz); } else { - //qError("QInfo:0x%"PRIx64" failed to find tag:%" PRId64 " in ts_comp", GET_QID(pRuntimeEnv), pTag->i); + //qError("QInfo:0x%"PRIx64" failed to find tag:%" PRId64 " in ts_comp", GET_TASKID(pRuntimeEnv), pTag->i); } return -1; @@ -3734,17 +3739,17 @@ int32_t setTimestampListJoinInfo(STaskRuntimeEnv* pRuntimeEnv, SVariant* pTag, S // Keep the cursor info of current table pTableQueryInfo->cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { - //qDebug("QInfo:0x%"PRIx64" find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_QID(pRuntimeEnv), pTag->pz, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); + //qDebug("QInfo:0x%"PRIx64" find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_TASKID(pRuntimeEnv), pTag->pz, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); } else { - //qDebug("QInfo:0x%"PRIx64" find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_QID(pRuntimeEnv), pTag->i, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); + //qDebug("QInfo:0x%"PRIx64" find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_TASKID(pRuntimeEnv), pTag->i, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); } } else { tsBufSetCursor(pRuntimeEnv->pTsBuf, &pTableQueryInfo->cur); if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { - //qDebug("QInfo:0x%"PRIx64" find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_QID(pRuntimeEnv), pTag->pz, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); + //qDebug("QInfo:0x%"PRIx64" find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_TASKID(pRuntimeEnv), pTag->pz, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); } else { - //qDebug("QInfo:0x%"PRIx64" find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_QID(pRuntimeEnv), pTag->i, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); + //qDebug("QInfo:0x%"PRIx64" find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_TASKID(pRuntimeEnv), pTag->i, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); } } @@ -3882,7 +3887,7 @@ static int32_t doCopyToSDataBlock(STaskRuntimeEnv* pRuntimeEnv, SGroupResInfo* p int32_t start = 0; int32_t step = -1; - //qDebug("QInfo:0x%"PRIx64" start to copy data from windowResInfo to output buf", GET_QID(pRuntimeEnv)); + //qDebug("QInfo:0x%"PRIx64" start to copy data from windowResInfo to output buf", GET_TASKID(pRuntimeEnv)); assert(orderType == TSDB_ORDER_ASC || orderType == TSDB_ORDER_DESC); if (orderType == TSDB_ORDER_ASC) { @@ -3927,7 +3932,7 @@ static int32_t doCopyToSDataBlock(STaskRuntimeEnv* pRuntimeEnv, SGroupResInfo* p } } - //qDebug("QInfo:0x%"PRIx64" copy data to query buf completed", GET_QID(pRuntimeEnv)); + //qDebug("QInfo:0x%"PRIx64" copy data to query buf completed", GET_TASKID(pRuntimeEnv)); pBlock->info.rows = numOfResult; return 0; } @@ -4055,7 +4060,7 @@ static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data //qDebug("QInfo:0x%"PRIx64" set %d subscribe info", pQInfo->qId, total); // Check if query is completed or not for stable query or normal table query respectively. - if (Q_STATUS_EQUAL(pRuntimeEnv->status, QUERY_COMPLETED) && pRuntimeEnv->proot->status == OP_EXEC_DONE) { + if (Q_STATUS_EQUAL(pRuntimeEnv->status, TASK_COMPLETED) && pRuntimeEnv->proot->status == OP_EXEC_DONE) { // setTaskStatus(pOperator->pTaskInfo, QUERY_OVER); } } @@ -4232,7 +4237,7 @@ void queryCostStatis(SQInfo *pQInfo) { // // assert(pQueryAttr->pos >= 0 && pQueryAttr->pos <= pBlockInfo->rows - 1); // -// SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); +// SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pTsdbReadHandle, NULL); // SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); // // // update the pQueryAttr->limit.offset value, and pQueryAttr->pos value @@ -4244,7 +4249,7 @@ void queryCostStatis(SQInfo *pQInfo) { // // int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, pBlockInfo, NULL, binarySearchForKey, pDataBlock); // -// //qDebug("QInfo:0x%"PRIx64" check data block, brange:%" PRId64 "-%" PRId64 ", numBlocksOfStep:%d, numOfRes:%d, lastKey:%"PRId64, GET_QID(pRuntimeEnv), +// //qDebug("QInfo:0x%"PRIx64" check data block, brange:%" PRId64 "-%" PRId64 ", numBlocksOfStep:%d, numOfRes:%d, lastKey:%"PRId64, GET_TASKID(pRuntimeEnv), // pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes, pQuery->current->lastKey); //} @@ -4259,22 +4264,22 @@ void queryCostStatis(SQInfo *pQInfo) { // int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); // // STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; -// TsdbQueryHandleT pQueryHandle = pRuntimeEnv->pQueryHandle; +// TsdbQueryHandleT pTsdbReadHandle = pRuntimeEnv->pTsdbReadHandle; // // SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; -// while (tsdbNextDataBlock(pQueryHandle)) { -// if (isQueryKilled(pRuntimeEnv->qinfo)) { +// while (tsdbNextDataBlock(pTsdbReadHandle)) { +// if (isTaskKilled(pRuntimeEnv->qinfo)) { // longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); // } // -// tsdbRetrieveDataBlockInfo(pQueryHandle, &blockInfo); +// tsdbRetrieveDataBlockInfo(pTsdbReadHandle, &blockInfo); // // if (pQueryAttr->limit.offset > blockInfo.rows) { // pQueryAttr->limit.offset -= blockInfo.rows; // pTableQueryInfo->lastKey = (QUERY_IS_ASC_QUERY(pQueryAttr)) ? blockInfo.window.ekey : blockInfo.window.skey; // pTableQueryInfo->lastKey += step; // -// //qDebug("QInfo:0x%"PRIx64" skip rows:%d, offset:%" PRId64, GET_QID(pRuntimeEnv), blockInfo.rows, +// //qDebug("QInfo:0x%"PRIx64" skip rows:%d, offset:%" PRId64, GET_TASKID(pRuntimeEnv), blockInfo.rows, // pQuery->limit.offset); // } else { // find the appropriated start position in current block // updateOffsetVal(pRuntimeEnv, &blockInfo); @@ -4300,7 +4305,7 @@ void queryCostStatis(SQInfo *pQInfo) { // // // load the data block and check data remaining in current data block // // TODO optimize performance -// SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); +// SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pTsdbReadHandle, NULL); // SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); // // tw = *win; @@ -4323,7 +4328,7 @@ void queryCostStatis(SQInfo *pQInfo) { // pRuntimeEnv->resultRowInfo.curIndex = index; // restore the window index // // //qDebug("QInfo:0x%"PRIx64" check data block, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, numOfRes:%d, lastKey:%" PRId64, -// GET_QID(pRuntimeEnv), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes, +// GET_TASKID(pRuntimeEnv), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes, // pQueryAttr->current->lastKey); // // return key; @@ -4365,8 +4370,8 @@ void queryCostStatis(SQInfo *pQInfo) { // STableQueryInfo *pTableQueryInfo = pRuntimeEnv->current; // // SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; -// while (tsdbNextDataBlock(pRuntimeEnv->pQueryHandle)) { -// tsdbRetrieveDataBlockInfo(pRuntimeEnv->pQueryHandle, &blockInfo); +// while (tsdbNextDataBlock(pRuntimeEnv->pTsdbReadHandle)) { +// tsdbRetrieveDataBlockInfo(pRuntimeEnv->pTsdbReadHandle, &blockInfo); // // if (QUERY_IS_ASC_QUERY(pQueryAttr)) { // if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) { @@ -4412,7 +4417,7 @@ void queryCostStatis(SQInfo *pQInfo) { // */ // if ((tw.skey <= blockInfo.window.ekey && ascQuery) || (tw.ekey >= blockInfo.window.skey && !ascQuery)) { // -// SArray *pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); +// SArray *pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pTsdbReadHandle, NULL); // SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); // // if ((win.ekey > blockInfo.window.ekey && ascQuery) || (win.ekey < blockInfo.window.skey && !ascQuery)) { @@ -4486,7 +4491,7 @@ static int32_t setupQueryHandle(void* tsdb, STaskRuntimeEnv* pRuntimeEnv, int64_ terrno = TSDB_CODE_SUCCESS; if (isFirstLastRowQuery(pQueryAttr)) { - pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); + pRuntimeEnv->pTsdbReadHandle = tsdbQueryLastRow(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); // update the query time window pQueryAttr->window = cond.twindow; @@ -4507,11 +4512,11 @@ static int32_t setupQueryHandle(void* tsdb, STaskRuntimeEnv* pRuntimeEnv, int64_ } } } else if (isCachedLastQuery(pQueryAttr)) { - pRuntimeEnv->pQueryHandle = tsdbQueryCacheLast(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); + pRuntimeEnv->pTsdbReadHandle = tsdbQueryCacheLast(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); } else if (pQueryAttr->pointInterpQuery) { - pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); + pRuntimeEnv->pTsdbReadHandle = tsdbQueryRowsInExternalWindow(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); } else { - pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); + pRuntimeEnv->pTsdbReadHandle = tsdbQueryTables(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); } #endif return terrno; @@ -4543,19 +4548,19 @@ int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr switch(tbScanner) { // case OP_TableBlockInfoScan: { -// pRuntimeEnv->proot = createTableBlockInfoScanOperator(pRuntimeEnv->pQueryHandle, pRuntimeEnv); +// pRuntimeEnv->proot = createTableBlockInfoScanOperator(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv); // break; // } // case OP_TableSeqScan: { -// pRuntimeEnv->proot = createTableSeqScanOperator(pRuntimeEnv->pQueryHandle, pRuntimeEnv); +// pRuntimeEnv->proot = createTableSeqScanOperator(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv); // break; // } // case OP_DataBlocksOptScan: { -// pRuntimeEnv->proot = createDataBlocksOptScanInfo(pRuntimeEnv->pQueryHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr), pQueryAttr->needReverseScan? 1:0); +// pRuntimeEnv->proot = createDataBlocksOptScanInfo(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr), pQueryAttr->needReverseScan? 1:0); // break; // } // case OP_TableScan: { -// pRuntimeEnv->proot = createTableScanOperator(pRuntimeEnv->pQueryHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr)); +// pRuntimeEnv->proot = createTableScanOperator(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr)); // break; // } default: { // do nothing @@ -4606,7 +4611,7 @@ int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr return TSDB_CODE_SUCCESS; } -static void doTableQueryInfoTimeWindowCheck(STaskInfo* pTaskInfo, STableQueryInfo* pTableQueryInfo, int32_t order) { +static void doTableQueryInfoTimeWindowCheck(SExecTaskInfo* pTaskInfo, STableQueryInfo* pTableQueryInfo, int32_t order) { if (order == TSDB_ORDER_ASC) { assert( (pTableQueryInfo->win.skey <= pTableQueryInfo->win.ekey) && @@ -4679,20 +4684,20 @@ static SSDataBlock* doTableScanImpl(void* param, bool* newgroup) { SOperatorInfo *pOperator = (SOperatorInfo*) param; STableScanInfo *pTableScanInfo = pOperator->info; - STaskInfo *pTaskInfo = pOperator->pTaskInfo; + SExecTaskInfo *pTaskInfo = pOperator->pTaskInfo; SSDataBlock *pBlock = &pTableScanInfo->block; STableGroupInfo *pTableGroupInfo = &pOperator->pTaskInfo->tableqinfoGroupInfo; *newgroup = false; - while (/*tsdbNextDataBlock(pTableScanInfo->pQueryHandle)*/1) { - if (isQueryKilled(pOperator->pRuntimeEnv->qinfo)) { - longjmp(pOperator->pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); + while (tsdbNextDataBlock(pTableScanInfo->pTsdbReadHandle)) { + if (isTaskKilled(pOperator->pTaskInfo)) { + longjmp(pOperator->pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED); } pTableScanInfo->numOfBlocks += 1; -// tsdbRetrieveDataBlockInfo(pTableScanInfo->pQueryHandle, &pBlock->info); + tsdbRetrieveDataBlockInfo(pTableScanInfo->pTsdbReadHandle, &pBlock->info); // todo opt // if (pTableGroupInfo->numOfTables > 1 || (pRuntimeEnv->current == NULL && pTableGroupInfo->numOfTables == 1)) { @@ -4711,7 +4716,7 @@ static SSDataBlock* doTableScanImpl(void* param, bool* newgroup) { int32_t code = loadDataBlock(pTaskInfo, pTableScanInfo, pBlock, &status); // int32_t code = loadDataBlockOnDemand(pOperator->pRuntimeEnv, pTableScanInfo, pBlock, &status); if (code != TSDB_CODE_SUCCESS) { - longjmp(pOperator->pRuntimeEnv->env, code); + longjmp(pOperator->pTaskInfo->env, code); } // current block is ignored according to filter result by block statistics data, continue load the next block @@ -4729,7 +4734,7 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { SOperatorInfo* pOperator = (SOperatorInfo*) param; STableScanInfo *pTableScanInfo = pOperator->info; - STaskInfo *pTaskInfo = pOperator->pTaskInfo; + SExecTaskInfo *pTaskInfo = pOperator->pTaskInfo; SResultRowInfo* pResultRowInfo = pTableScanInfo->pResultRowInfo; *newgroup = false; @@ -4741,7 +4746,7 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { } if (++pTableScanInfo->current >= pTableScanInfo->times) { - if (pTableScanInfo->reverseTimes <= 0/* || isTsdbCacheLastRow(pTableScanInfo->pQueryHandle)*/) { + if (pTableScanInfo->reverseTimes <= 0/* || isTsdbCacheLastRow(pTableScanInfo->pTsdbReadHandle)*/) { return NULL; } else { break; @@ -4750,9 +4755,9 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { // do prepare for the next round table scan operation // STsdbQueryCond cond = createTsdbQueryCond(pQueryAttr, &pQueryAttr->window); -// tsdbResetQueryHandle(pTableScanInfo->pQueryHandle, &cond); +// tsdbResetQueryHandle(pTableScanInfo->pTsdbReadHandle, &cond); - setTaskStatus(pTaskInfo, QUERY_NOT_COMPLETED); + setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); pTableScanInfo->scanFlag = REPEAT_SCAN; // if (pTaskInfo->pTsBuf) { @@ -4764,8 +4769,8 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { pResultRowInfo->curPos = 0; } - //qDebug("QInfo:0x%"PRIx64" start to repeat scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64, -// GET_QID(pRuntimeEnv), cond.twindow.skey, cond.twindow.ekey); + qDebug("QInfo:0x%"PRIx64" start to repeat scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64, + GET_TASKID(pTaskInfo), pTaskInfo->window.skey, pTaskInfo->window.ekey); } SSDataBlock *p = NULL; @@ -4773,15 +4778,10 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { if (pTableScanInfo->reverseTimes > 0) { setupEnvForReverseScan(pTableScanInfo, pTableScanInfo->pCtx, pTableScanInfo->numOfOutput); // STsdbQueryCond cond = createTsdbQueryCond(pQueryAttr, &pQueryAttr->window); -// tsdbResetQueryHandle(pTableScanInfo->pQueryHandle, &cond); +// tsdbResetQueryHandle(pTableScanInfo->pTsdbReadHandle, &cond); - //qDebug("QInfo:0x%"PRIx64" start to reverse scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64, -// GET_QID(pRuntimeEnv), cond.twindow.skey, cond.twindow.ekey); - - pTableScanInfo->times = 1; - pTableScanInfo->current = 0; - pTableScanInfo->reverseTimes = 0; -// pTableScanInfo->order = cond.order; + qDebug("QInfo:0x%"PRIx64" start to reverse scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64, + GET_TASKID(pTaskInfo), pTaskInfo->window.skey, pTaskInfo->window.ekey); if (pResultRowInfo->size > 0) { pResultRowInfo->curPos = pResultRowInfo->size - 1; @@ -4814,8 +4814,8 @@ static SSDataBlock* doBlockInfoScan(void* param, bool* newgroup) { tableBlockDist.maxRows = INT_MIN; tableBlockDist.minRows = INT_MAX; - tsdbGetFileBlocksDistInfo(pTableScanInfo->pQueryHandle, &tableBlockDist); - tableBlockDist.numOfRowsInMemTable = (int32_t) tsdbGetNumOfRowsInMemTable(pTableScanInfo->pQueryHandle); + tsdbGetFileBlocksDistInfo(pTableScanInfo->pTsdbReadHandle, &tableBlockDist); + tableBlockDist.numOfRowsInMemTable = (int32_t) tsdbGetNumOfRowsInMemTable(pTableScanInfo->pTsdbReadHandle); SSDataBlock* pBlock = &pTableScanInfo->block; pBlock->info.rows = 1; @@ -4842,11 +4842,11 @@ static SSDataBlock* doBlockInfoScan(void* param, bool* newgroup) { } -SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime) { +SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, SExecTaskInfo* pTaskInfo) { assert(repeatTime > 0 && numOfOutput > 0); STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - pInfo->pQueryHandle = pTsdbQueryHandle; + pInfo->pTsdbReadHandle = pTsdbQueryHandle; pInfo->times = repeatTime; pInfo->reverseTimes = 0; pInfo->order = order; @@ -4862,6 +4862,7 @@ SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, int32_t order, in pOperator->numOfOutput = numOfOutput; pOperator->pRuntimeEnv = NULL; pOperator->exec = doTableScan; + pOperator->pTaskInfo = pTaskInfo; return pOperator; } @@ -4869,7 +4870,7 @@ SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, int32_t order, in SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, STaskRuntimeEnv* pRuntimeEnv) { STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - pInfo->pQueryHandle = pTsdbQueryHandle; + pInfo->pTsdbReadHandle = pTsdbQueryHandle; pInfo->times = 1; pInfo->reverseTimes = 0; pInfo->order = pRuntimeEnv->pQueryAttr->order.order; @@ -4893,7 +4894,7 @@ SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, STaskRuntimeEn SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbQueryHandle, STaskRuntimeEnv* pRuntimeEnv) { STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - pInfo->pQueryHandle = pTsdbQueryHandle; + pInfo->pTsdbReadHandle = pTsdbQueryHandle; pInfo->block.pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); SColumnInfoData infoData = {{0}}; @@ -4977,7 +4978,7 @@ SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, STaskRuntimeE assert(repeatTime > 0); STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - pInfo->pQueryHandle = pTsdbQueryHandle; + pInfo->pTsdbReadHandle = pTsdbQueryHandle; pInfo->times = repeatTime; pInfo->reverseTimes = reverseTime; pInfo->current = 0; @@ -5145,7 +5146,7 @@ SOperatorInfo* createGlobalAggregateOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, S pOperator->numOfOutput = numOfOutput; pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->exec = doGlobalAggregate; +// pOperator->exec = doGlobalAggregate; pOperator->cleanup = destroyGlobalAggOperatorInfo; appendUpstream(pOperator, downstream); @@ -5188,7 +5189,7 @@ SOperatorInfo *createMultiwaySortOperatorInfo(STaskRuntimeEnv *pRuntimeEnv, SExp pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->numOfOutput = numOfOutput; pOperator->pExpr = pExpr; - pOperator->exec = doMultiwayMergeSort; +// pOperator->exec = doMultiwayMergeSort; pOperator->cleanup = destroyGlobalAggOperatorInfo; return pOperator; } @@ -5476,7 +5477,7 @@ static SSDataBlock* doProjectOperation(void* param, bool* newgroup) { assert(*newgroup == false); *newgroup = prevVal; - setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); + setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); break; } @@ -5644,7 +5645,7 @@ static SSDataBlock* doIntervalAgg(void* param, bool* newgroup) { pOperator->status = OP_RES_TO_RETURN; closeAllResultRows(&pIntervalInfo->resultRowInfo); - setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); + setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); finalizeQueryResult(pOperator, pIntervalInfo->pCtx, &pIntervalInfo->resultRowInfo, pIntervalInfo->rowCellInfoOffset); initGroupResInfo(&pRuntimeEnv->groupResInfo, &pIntervalInfo->resultRowInfo); @@ -5704,7 +5705,7 @@ static SSDataBlock* doAllIntervalAgg(void* param, bool* newgroup) { pOperator->status = OP_RES_TO_RETURN; closeAllResultRows(&pIntervalInfo->resultRowInfo); - setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); + setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); finalizeQueryResult(pOperator, pIntervalInfo->pCtx, &pIntervalInfo->resultRowInfo, pIntervalInfo->rowCellInfoOffset); initGroupResInfo(&pRuntimeEnv->groupResInfo, &pIntervalInfo->resultRowInfo); @@ -5767,7 +5768,7 @@ static SSDataBlock* doSTableIntervalAgg(void* param, bool* newgroup) { pOperator->status = OP_RES_TO_RETURN; pQueryAttr->order.order = order; // TODO : restore the order doCloseAllTimeWindow(pRuntimeEnv); - setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); + setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { @@ -5822,7 +5823,7 @@ static SSDataBlock* doAllSTableIntervalAgg(void* param, bool* newgroup) { pOperator->status = OP_RES_TO_RETURN; pQueryAttr->order.order = order; // TODO : restore the order doCloseAllTimeWindow(pRuntimeEnv); - setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); + setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); int64_t st = taosGetTimestampUs(); copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); @@ -5956,7 +5957,7 @@ static SSDataBlock* doStateWindowAgg(void *param, bool* newgroup) { pOperator->status = OP_RES_TO_RETURN; closeAllResultRows(&pBInfo->resultRowInfo); - setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); + setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); finalizeQueryResult(pOperator, pBInfo->pCtx, &pBInfo->resultRowInfo, pBInfo->rowCellInfoOffset); initGroupResInfo(&pRuntimeEnv->groupResInfo, &pBInfo->resultRowInfo); @@ -6094,7 +6095,7 @@ static SSDataBlock* hashGroupbyAggregate(void* param, bool* newgroup) { static void doHandleRemainBlockForNewGroupImpl(SFillOperatorInfo *pInfo, STaskRuntimeEnv* pRuntimeEnv, bool* newgroup) { pInfo->totalInputRows = pInfo->existNewGroupBlock->info.rows; - int64_t ekey = Q_STATUS_EQUAL(pRuntimeEnv->status, QUERY_COMPLETED)?pRuntimeEnv->pQueryAttr->window.ekey:pInfo->existNewGroupBlock->info.window.ekey; + int64_t ekey = Q_STATUS_EQUAL(pRuntimeEnv->status, TASK_COMPLETED)?pRuntimeEnv->pQueryAttr->window.ekey:pInfo->existNewGroupBlock->info.window.ekey; taosResetFillInfo(pInfo->pFillInfo, getFillInfoStart(pInfo->pFillInfo)); taosFillSetStartInfo(pInfo->pFillInfo, pInfo->existNewGroupBlock->info.rows, ekey); @@ -6422,7 +6423,7 @@ SOperatorInfo* createFilterOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI SFilterOperatorInfo* pInfo = calloc(1, sizeof(SFilterOperatorInfo)); assert(numOfFilter > 0 && pCols != NULL); - doCreateFilterInfo(pCols, numOfOutput, numOfFilter, &pInfo->pFilterInfo, 0); +// doCreateFilterInfo(pCols, numOfOutput, numOfFilter, &pInfo->pFilterInfo, 0); pInfo->numOfFilterCols = numOfFilter; SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); @@ -6714,10 +6715,10 @@ SOperatorInfo* createSLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "SLimitOperator"; -// pOperator->operatorType = OP_SLimit; + pOperator->operatorType = OP_SLimit; pOperator->blockingOptr = false; pOperator->status = OP_IN_EXECUTING; - pOperator->exec = doSLimit; +// pOperator->exec = doSLimit; pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->cleanup = destroySlimitOperatorInfo; @@ -6799,14 +6800,14 @@ static SSDataBlock* doTagScan(void* param, bool* newgroup) { count += 1; } - //qDebug("QInfo:0x%"PRIx64" create (tableId, tag) info completed, rows:%d", GET_QID(pRuntimeEnv), count); + //qDebug("QInfo:0x%"PRIx64" create (tableId, tag) info completed, rows:%d", GET_TASKID(pRuntimeEnv), count); } else if (functionId == FUNCTION_COUNT) {// handle the "count(tbname)" query SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, 0); *(int64_t*)pColInfo->pData = pInfo->totalTables; count = 1; pOperator->status = OP_EXEC_DONE; - //qDebug("QInfo:0x%"PRIx64" create count(tbname) query, res:%d rows:1", GET_QID(pRuntimeEnv), count); + //qDebug("QInfo:0x%"PRIx64" create count(tbname) query, res:%d rows:1", GET_TASKID(pRuntimeEnv), count); } else { // return only the tags|table name etc. SExprInfo* pExprInfo = &pOperator->pExpr[0]; // todo use the column list instead of exprinfo @@ -6845,11 +6846,11 @@ static SSDataBlock* doTagScan(void* param, bool* newgroup) { pOperator->status = OP_EXEC_DONE; } - //qDebug("QInfo:0x%"PRIx64" create tag values results completed, rows:%d", GET_QID(pRuntimeEnv), count); + //qDebug("QInfo:0x%"PRIx64" create tag values results completed, rows:%d", GET_TASKID(pRuntimeEnv), count); } if (pOperator->status == OP_EXEC_DONE) { - setTaskStatus(pOperator->pRuntimeEnv, QUERY_COMPLETED); + setTaskStatus(pOperator->pRuntimeEnv, TASK_COMPLETED); } pRes->info.rows = count; @@ -7174,90 +7175,52 @@ static int32_t deserializeColFilterInfo(SColumnFilterInfo* pColFilters, int16_t return TSDB_CODE_SUCCESS; } -/** - * { - "Id": { - "QueryId": 20, - "TemplateId": 0, - "SubplanId": 0 - }, - "Node": { - "Name": "TableScan", - "InputSchema": [{ - "Type": 9, - "ColId": 1, - "Bytes": 8 - }, { - "Type": 4, - "ColId": 2, - "Bytes": 4 - }, { - "Type": 8, - "ColId": 3, - "Bytes": 20 - }], - "TableScan": { - "TableId": 1, - "TableType": 3, - "Flag": 0, - "Window": { - "StartKey": 0, - "EndKey": 0 - } - } - }, - "DataSink": { - "Name": "Dispatch", - "Dispatch": { - } - } +static SExecTaskInfo* createExecTaskInfo(uint64_t queryId) { + SExecTaskInfo* pTaskInfo = calloc(1, sizeof(SExecTaskInfo)); + setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); + + pthread_mutex_init(&pTaskInfo->lock, NULL); + pTaskInfo->cost.created = taosGetTimestampMs(); + return pTaskInfo; } - */ -int32_t parseTaskInfo(const char* msg, int32_t len) { - cJSON* pJson = cJSON_Parse(msg); - if (NULL == pJson) { - return TSDB_CODE_INVALID_MSG; + +SOperatorInfo* doCreateOperatorTreeNode(SPhyNode* pPhyNode, SExecTaskInfo* pTaskInfo, void* param) { + if (pPhyNode->pChildren == NULL || taosArrayGetSize(pPhyNode->pChildren) == 0) { + if (pPhyNode->info.type == OP_TableScan) { + size_t numOfCols = taosArrayGetSize(pPhyNode->pTargets); + SOperatorInfo* pOperatorInfo = createTableScanOperator(param, TSDB_ORDER_ASC, numOfCols, 1, pTaskInfo); + pTaskInfo->pRoot = pOperatorInfo; + } } +} - cJSON* pSub = cJSON_GetObjectItem(pJson, "ID"); - if (NULL != pSub) { - printf("Id : %s\n", pSub->valuestring); - } +int32_t doCreateExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, void* readerHandle) { + STsdbQueryCond cond = {.order = TSDB_ORDER_ASC, .numOfCols = 2, .loadExternalRows = false}; + cond.twindow.skey = INT64_MIN; + cond.twindow.ekey = INT64_MAX; + cond.colList = calloc(cond.numOfCols, sizeof(SColumnInfo)); - cJSON* pNode = cJSON_GetObjectItem(pJson, "Node"); - if (pNode == NULL) { - return TSDB_CODE_INVALID_MSG; - } + // todo set the correct table column info + cond.colList[0].type = TSDB_DATA_TYPE_TIMESTAMP; + cond.colList[0].bytes = sizeof(uint64_t); + cond.colList[0].colId = 1; - cJSON* pNodeName = cJSON_GetObjectItem(pNode, "name"); - if (pNodeName == NULL) { - return TSDB_CODE_INVALID_MSG; - } + cond.colList[1].type = TSDB_DATA_TYPE_INT; + cond.colList[1].bytes = sizeof(int32_t); + cond.colList[1].colId = 2; - printf("node name is: %s\n", pNodeName->valuestring); - - cJSON* pNodeSchema = cJSON_GetObjectItem(pNode, "InputSchema"); - if (pNodeSchema == NULL) { - return TSDB_CODE_INVALID_MSG; - } - - cJSON* pOperator = cJSON_GetObjectItem(pNode, pNodeName->valuestring); - if (pOperator == NULL) { - return TSDB_CODE_INVALID_MSG; - } - - cJSON* pTableId = cJSON_GetObjectItem(pOperator, "tableId"); - if (pTableId == NULL) { - return TSDB_CODE_INVALID_MSG; - } - - cJSON* pTimeWindow = cJSON_GetObjectItem(pOperator, "window"); - if (pTimeWindow == NULL) { - return TSDB_CODE_INVALID_MSG; - } + STableGroupInfo group = {.numOfTables = 1, .pGroupList = taosArrayInit(1, POINTER_BYTES)}; + SArray* pa = taosArrayInit(1, sizeof(STableKeyInfo)); + STableKeyInfo info = {.pTable = NULL, .lastKey = 0, .uid = 1}; + taosArrayPush(pa, &info); + taosArrayPush(group.pGroupList, &pa); + *pTaskInfo = createExecTaskInfo((uint64_t)pPlan->id.queryId); + tsdbReadHandleT tsdbReadHandle = tsdbQueryTables(readerHandle, &cond, &group, (*pTaskInfo)->id.queryId, NULL); + doCreateOperatorTreeNode(pPlan->pNode, *pTaskInfo, tsdbReadHandle); + return TSDB_CODE_SUCCESS; } /** @@ -7704,7 +7667,7 @@ int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExp *pExprInfo = NULL; int32_t code = TSDB_CODE_SUCCESS; - code = initUdfInfo(pUdfInfo); +// code = initUdfInfo(pUdfInfo); if (code) { return code; } @@ -7996,8 +7959,8 @@ int32_t createFilterInfo(STaskAttr* pQueryAttr, uint64_t qId) { return TSDB_CODE_SUCCESS; } - doCreateFilterInfo(pQueryAttr->tableCols, pQueryAttr->numOfCols, pQueryAttr->numOfFilterCols, - &pQueryAttr->pFilterInfo, qId); +// doCreateFilterInfo(pQueryAttr->tableCols, pQueryAttr->numOfCols, pQueryAttr->numOfFilterCols, +// &pQueryAttr->pFilterInfo, qId); pQueryAttr->createFilterOperator = true; @@ -8505,7 +8468,7 @@ int32_t doDumpQueryResult(SQInfo *pQInfo, char *data, int8_t compressed, int32_t } // all data returned, set query over - if (Q_STATUS_EQUAL(pRuntimeEnv->status, QUERY_COMPLETED)) { + if (Q_STATUS_EQUAL(pRuntimeEnv->status, TASK_COMPLETED)) { // setTaskStatus(pOperator->pTaskInfo, QUERY_OVER); } } else { diff --git a/source/libs/executor/test/CMakeLists.txt b/source/libs/executor/test/CMakeLists.txt new file mode 100644 index 0000000000..ece84207c7 --- /dev/null +++ b/source/libs/executor/test/CMakeLists.txt @@ -0,0 +1,18 @@ + +MESSAGE(STATUS "build parser unit test") + +# GoogleTest requires at least C++11 +SET(CMAKE_CXX_STANDARD 11) +AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) + +ADD_EXECUTABLE(executorTest ${SOURCE_LIST}) +TARGET_LINK_LIBRARIES( + executorTest + PUBLIC os util common transport gtest taos qcom executor function planner +) + +TARGET_INCLUDE_DIRECTORIES( + executorTest + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/executor/" + PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/executor/inc" +) diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index e69de29bb2..5f5fddbe28 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -0,0 +1,221 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#pragma GCC diagnostic ignored "-Wwrite-strings" + +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" +#include "os.h" + +#include "taos.h" +#include "tdef.h" +#include "tvariant.h" +#include "tep.h" +#include "trpc.h" +#include "stub.h" +#include "executor.h" + +/** +{ + "Id": { + "QueryId": 1.3108161807422521e+19, + "TemplateId": 0, + "SubplanId": 0 + }, + "Node": { + "Name": "TableScan", + "Targets": [{ + "Base": { + "Schema": { + "Type": 9, + "ColId": 5000, + "Bytes": 8 + }, + "Columns": [{ + "TableId": 1, + "Flag": 0, + "Info": { + "ColId": 1, + "Type": 9, + "Bytes": 8 + } + }], + "InterBytes": 0 + }, + "Expr": { + "Type": 4, + "Column": { + "Type": 9, + "ColId": 1, + "Bytes": 8 + } + } + }, { + "Base": { + "Schema": { + "Type": 4, + "ColId": 5001, + "Bytes": 4 + }, + "Columns": [{ + "TableId": 1, + "Flag": 0, + "Info": { + "ColId": 2, + "Type": 4, + "Bytes": 4 + } + }], + "InterBytes": 0 + }, + "Expr": { + "Type": 4, + "Column": { + "Type": 4, + "ColId": 2, + "Bytes": 4 + } + } + }], + "InputSchema": [{ + "Type": 9, + "ColId": 5000, + "Bytes": 8 + }, { + "Type": 4, + "ColId": 5001, + "Bytes": 4 + }], + "TableScan": { + "TableId": 1, + "TableType": 2, + "Flag": 0, + "Window": { + "StartKey": -9.2233720368547758e+18, + "EndKey": 9.2233720368547758e+18 + } + } + }, + "DataSink": { + "Name": "Dispatch", + "Dispatch": { + } + } +} + */ + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +TEST(testCase, build_executor_tree_Test) { + + + const char* msg = "{\n" + "\t\"Id\":\t{\n" + "\t\t\"QueryId\":\t1.3108161807422521e+19,\n" + "\t\t\"TemplateId\":\t0,\n" + "\t\t\"SubplanId\":\t0\n" + "\t},\n" + "\t\"Node\":\t{\n" + "\t\t\"Name\":\t\"TableScan\",\n" + "\t\t\"Targets\":\t[{\n" + "\t\t\t\t\"Base\":\t{\n" + "\t\t\t\t\t\"Schema\":\t{\n" + "\t\t\t\t\t\t\"Type\":\t9,\n" + "\t\t\t\t\t\t\"ColId\":\t5000,\n" + "\t\t\t\t\t\t\"Bytes\":\t8\n" + "\t\t\t\t\t},\n" + "\t\t\t\t\t\"Columns\":\t[{\n" + "\t\t\t\t\t\t\t\"TableId\":\t1,\n" + "\t\t\t\t\t\t\t\"Flag\":\t0,\n" + "\t\t\t\t\t\t\t\"Info\":\t{\n" + "\t\t\t\t\t\t\t\t\"ColId\":\t1,\n" + "\t\t\t\t\t\t\t\t\"Type\":\t9,\n" + "\t\t\t\t\t\t\t\t\"Bytes\":\t8\n" + "\t\t\t\t\t\t\t}\n" + "\t\t\t\t\t\t}],\n" + "\t\t\t\t\t\"InterBytes\":\t0\n" + "\t\t\t\t},\n" + "\t\t\t\t\"Expr\":\t{\n" + "\t\t\t\t\t\"Type\":\t4,\n" + "\t\t\t\t\t\"Column\":\t{\n" + "\t\t\t\t\t\t\"Type\":\t9,\n" + "\t\t\t\t\t\t\"ColId\":\t1,\n" + "\t\t\t\t\t\t\"Bytes\":\t8\n" + "\t\t\t\t\t}\n" + "\t\t\t\t}\n" + "\t\t\t}, {\n" + "\t\t\t\t\"Base\":\t{\n" + "\t\t\t\t\t\"Schema\":\t{\n" + "\t\t\t\t\t\t\"Type\":\t4,\n" + "\t\t\t\t\t\t\"ColId\":\t5001,\n" + "\t\t\t\t\t\t\"Bytes\":\t4\n" + "\t\t\t\t\t},\n" + "\t\t\t\t\t\"Columns\":\t[{\n" + "\t\t\t\t\t\t\t\"TableId\":\t1,\n" + "\t\t\t\t\t\t\t\"Flag\":\t0,\n" + "\t\t\t\t\t\t\t\"Info\":\t{\n" + "\t\t\t\t\t\t\t\t\"ColId\":\t2,\n" + "\t\t\t\t\t\t\t\t\"Type\":\t4,\n" + "\t\t\t\t\t\t\t\t\"Bytes\":\t4\n" + "\t\t\t\t\t\t\t}\n" + "\t\t\t\t\t\t}],\n" + "\t\t\t\t\t\"InterBytes\":\t0\n" + "\t\t\t\t},\n" + "\t\t\t\t\"Expr\":\t{\n" + "\t\t\t\t\t\"Type\":\t4,\n" + "\t\t\t\t\t\"Column\":\t{\n" + "\t\t\t\t\t\t\"Type\":\t4,\n" + "\t\t\t\t\t\t\"ColId\":\t2,\n" + "\t\t\t\t\t\t\"Bytes\":\t4\n" + "\t\t\t\t\t}\n" + "\t\t\t\t}\n" + "\t\t\t}],\n" + "\t\t\"InputSchema\":\t[{\n" + "\t\t\t\t\"Type\":\t9,\n" + "\t\t\t\t\"ColId\":\t5000,\n" + "\t\t\t\t\"Bytes\":\t8\n" + "\t\t\t}, {\n" + "\t\t\t\t\"Type\":\t4,\n" + "\t\t\t\t\"ColId\":\t5001,\n" + "\t\t\t\t\"Bytes\":\t4\n" + "\t\t\t}],\n" + "\t\t\"TableScan\":\t{\n" + "\t\t\t\"TableId\":\t1,\n" + "\t\t\t\"TableType\":\t2,\n" + "\t\t\t\"Flag\":\t0,\n" + "\t\t\t\"Window\":\t{\n" + "\t\t\t\t\"StartKey\":\t-9.2233720368547758e+18,\n" + "\t\t\t\t\"EndKey\":\t9.2233720368547758e+18\n" + "\t\t\t}\n" + "\t\t}\n" + "\t},\n" + "\t\"DataSink\":\t{\n" + "\t\t\"Name\":\t\"Dispatch\",\n" + "\t\t\"Dispatch\":\t{\n" + "\t\t}\n" + "\t}\n" + "}"; + + SExecTaskInfo* pTaskInfo = nullptr; + int32_t code = qCreateExecTask((void*) 1, 2, NULL, (void**) &pTaskInfo); +} \ No newline at end of file diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index f440e6cdfe..5f17fcfaee 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -70,6 +70,9 @@ int32_t parseQuerySql(SParseContext* pCxt, SQueryNode** pQuery) { int32_t code = qParserValidateSqlNode(&pCxt->ctx, &info, pQueryInfo, pCxt->pMsg, pCxt->msgLen); if (code == TSDB_CODE_SUCCESS) { *pQuery = (SQueryNode*)pQueryInfo; + } else { + terrno = code; + return code; } } diff --git a/source/libs/planner/inc/plannerInt.h b/source/libs/planner/inc/plannerInt.h index 2a50752c88..be266bd415 100644 --- a/source/libs/planner/inc/plannerInt.h +++ b/source/libs/planner/inc/plannerInt.h @@ -62,7 +62,7 @@ typedef struct SQueryPlanNode { SSchema *pSchema; // the schema of the input SSDatablock int32_t numOfCols; // number of input columns SArray *pExpr; // the query functions or sql aggregations - int32_t numOfExpr; // number of result columns, which is also the number of pExprs + int32_t numOfExpr; // number of result columns, which is also the number of pExprs void *pExtInfo; // additional information // children operator to generated result for current node to process // in case of join, multiple prev nodes exist. diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index a38d110d5f..7ef434ccf5 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -88,16 +88,20 @@ static bool copySchema(SDataBlockSchema* dst, const SDataBlockSchema* src) { } static bool toDataBlockSchema(SQueryPlanNode* pPlanNode, SDataBlockSchema* dataBlockSchema) { - dataBlockSchema->numOfCols = pPlanNode->numOfCols; + dataBlockSchema->numOfCols = pPlanNode->numOfExpr; dataBlockSchema->pSchema = malloc(sizeof(SSlotSchema) * pPlanNode->numOfCols); if (NULL == dataBlockSchema->pSchema) { return false; } - memcpy(dataBlockSchema->pSchema, pPlanNode->pSchema, sizeof(SSlotSchema) * pPlanNode->numOfCols); + dataBlockSchema->resultRowSize = 0; - for (int32_t i = 0; i < dataBlockSchema->numOfCols; ++i) { + for (int32_t i = 0; i < pPlanNode->numOfExpr; ++i) { + SExprInfo* pExprInfo = taosArrayGetP(pPlanNode->pExpr, i); + memcpy(&dataBlockSchema->pSchema[i], &pExprInfo->base.resSchema, sizeof(SSlotSchema)); + dataBlockSchema->resultRowSize += dataBlockSchema->pSchema[i].bytes; } + return true; } @@ -284,7 +288,6 @@ static SPhyNode* createSingleTableScanNode(SQueryPlanNode* pPlanNode, SQueryTabl return createUserTableScanNode(pPlanNode, pTable, OP_TableScan); } - static SPhyNode* createTableScanNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { SQueryTableInfo* pTable = (SQueryTableInfo*)pPlanNode->pExtInfo; diff --git a/source/libs/planner/src/physicalPlanJson.c b/source/libs/planner/src/physicalPlanJson.c index d67ff956b9..5175845fee 100644 --- a/source/libs/planner/src/physicalPlanJson.c +++ b/source/libs/planner/src/physicalPlanJson.c @@ -154,9 +154,13 @@ static bool fromRawArrayWithAlloc(const cJSON* json, const char* name, FFromJson return fromItem(jArray, func, *array, itemSize, *size); } -static bool fromRawArray(const cJSON* json, const char* name, FFromJson func, void* array, int32_t itemSize, int32_t* size) { +static bool fromRawArray(const cJSON* json, const char* name, FFromJson func, void** array, int32_t itemSize, int32_t* size) { const cJSON* jArray = getArray(json, name, size); - return fromItem(jArray, func, array, itemSize, *size); + if (*array == NULL) { + *array = calloc(*size, itemSize); + } + + return fromItem(jArray, func, *array, itemSize, *size); } static char* getString(const cJSON* json, const char* name) { @@ -728,7 +732,7 @@ static bool phyNodeFromJson(const cJSON* json, void* obj) { res = fromArray(json, jkPnodeConditions, exprInfoFromJson, &node->pConditions, sizeof(SExprInfo)); } if (res) { - res = fromRawArray(json, jkPnodeSchema, schemaFromJson, node->targetSchema.pSchema, sizeof(SSlotSchema), &node->targetSchema.numOfCols); + res = fromRawArray(json, jkPnodeSchema, schemaFromJson, (void**)&node->targetSchema.pSchema, sizeof(SSlotSchema), &node->targetSchema.numOfCols); } if (res) { res = fromArray(json, jkPnodeChildren, phyNodeFromJson, &node->pChildren, sizeof(SSlotSchema)); @@ -889,6 +893,8 @@ int32_t subPlanToString(const SSubplan* subplan, char** str, int32_t* len) { } *str = cJSON_Print(json); + + printf("%s\n", *str); *len = strlen(*str) + 1; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index f80a631413..de3454ddbb 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -56,35 +56,43 @@ void qDestroyQueryDag(struct SQueryDag* pDag) { tfree(pDag); } -int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, uint64_t requestId) { - SQueryPlanNode* logicPlan; - int32_t code = createQueryPlan(pNode, &logicPlan); +int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, SSchema** pSchema, uint32_t* numOfResCols, uint64_t requestId) { + SQueryPlanNode* pLogicPlan; + int32_t code = createQueryPlan(pNode, &pLogicPlan); if (TSDB_CODE_SUCCESS != code) { - destroyQueryPlan(logicPlan); + destroyQueryPlan(pLogicPlan); return code; } - // - if (logicPlan->info.type != QNODE_MODIFY) { -// char* str = NULL; -// queryPlanToString(logicPlan, &str); -// printf("%s\n", str); + if (pLogicPlan->info.type != QNODE_MODIFY) { + char* str = NULL; + queryPlanToString(pLogicPlan, &str); + printf("%s\n", str); } - code = optimizeQueryPlan(logicPlan); + int32_t numOfOutput = pLogicPlan->numOfExpr; + *pSchema = calloc(numOfOutput, sizeof(SSchema)); + *numOfResCols = numOfOutput; + + for(int32_t i = 0; i < numOfOutput; ++i) { + SExprInfo* pExprInfo = taosArrayGetP(pLogicPlan->pExpr, i); + memcpy(&(*pSchema)[i], pExprInfo->pExpr->pSchema, sizeof(SSchema)); + } + + code = optimizeQueryPlan(pLogicPlan); if (TSDB_CODE_SUCCESS != code) { - destroyQueryPlan(logicPlan); + destroyQueryPlan(pLogicPlan); return code; } - code = createDag(logicPlan, NULL, pDag, requestId); + code = createDag(pLogicPlan, NULL, pDag, requestId); if (TSDB_CODE_SUCCESS != code) { - destroyQueryPlan(logicPlan); + destroyQueryPlan(pLogicPlan); qDestroyQueryDag(*pDag); return code; } - destroyQueryPlan(logicPlan); + destroyQueryPlan(pLogicPlan); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/test/phyPlanTests.cpp b/source/libs/planner/test/phyPlanTests.cpp index 3f6f5ab77b..0d3820cc77 100644 --- a/source/libs/planner/test/phyPlanTests.cpp +++ b/source/libs/planner/test/phyPlanTests.cpp @@ -62,7 +62,10 @@ protected: } SQueryDag* dag = nullptr; uint64_t requestId = 20; - code = qCreateQueryDag(query, &dag, requestId); + SSchema *schema = NULL; + uint32_t numOfOutput = 0; + + code = qCreateQueryDag(query, &dag, &schema, &numOfOutput, requestId); dag_.reset(dag); return code; } diff --git a/source/libs/qworker/CMakeLists.txt b/source/libs/qworker/CMakeLists.txt index ea3e97057b..a3db9c6992 100644 --- a/source/libs/qworker/CMakeLists.txt +++ b/source/libs/qworker/CMakeLists.txt @@ -1,15 +1,31 @@ aux_source_directory(src QWORKER_SRC) -add_library(qworker ${QWORKER_SRC}) +#add_library(qworker ${QWORKER_SRC}) +#target_include_directories( +# qworker +# PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/qworker" +# PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +#) +# +#target_link_libraries( +# qworker +# PRIVATE os util transport planner qcom executor +#) + +add_library(qworker STATIC ${QWORKER_SRC}) target_include_directories( qworker PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/qworker" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) -target_link_libraries( - qworker - PRIVATE os util transport planner qcom -) +#set_target_properties(qworker PROPERTIES +# IMPORTED_LOCATION "${CMAKE_CURRENT_SOURCE_DIR}/libqworker.a" +# INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_SOURCE_DIR}/include/libs/qworker" +# ) + +target_link_libraries(qworker + PRIVATE os util transport planner qcom executor + ) if(${BUILD_TEST}) ADD_SUBDIRECTORY(test) diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 808c1e19f9..56c60bc7f2 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -1,9 +1,10 @@ #include "qworker.h" -#include "tname.h" +#include "executor.h" #include "planner.h" #include "query.h" #include "qworkerInt.h" #include "tmsg.h" +#include "tname.h" int32_t qwValidateStatus(int8_t oriStatus, int8_t newStatus) { int32_t code = 0; @@ -1011,7 +1012,7 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { bool queryDone = false; bool queryRsped = false; bool needStop = false; - SSubplan *plan = NULL; + struct SSubplan *plan = NULL; QW_ERR_JRET(qwCheckTaskCancelDrop(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, &needStop)); if (needStop) { @@ -1025,8 +1026,8 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { QW_ERR_JRET(code); } - //TODO call executer to init subquery - code = 0; // return error directly + qTaskInfo_t pTaskInfo = NULL; + code = qCreateExecTask(node, 0, (struct SSubplan *)plan, &pTaskInfo); //TODO call executer to init subquery if (code) { @@ -1040,7 +1041,7 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { queryRsped = true; //TODO call executer to execute subquery - code = 0; + code = qExecTask(pTaskInfo); void *data = NULL; queryDone = false; //TODO call executer to execute subquery diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 3ac08b5c42..e4285dc444 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -181,7 +181,6 @@ void schFreeTask(SSchTask* pTask) { int32_t schValidateAndBuildJob(SQueryDag *pDag, SSchJob *pJob) { int32_t code = 0; - pJob->queryId = pDag->queryId; if (pDag->numOfSubplans <= 0) { @@ -256,7 +255,7 @@ int32_t schValidateAndBuildJob(SQueryDag *pDag, SSchJob *pJob) { SSchTask task = {0}; SSchTask *pTask = &task; - + schInitTask(pJob, &task, plan, pLevel); void *p = taosArrayPush(pLevel->subTasks, &task); @@ -279,7 +278,6 @@ int32_t schValidateAndBuildJob(SQueryDag *pDag, SSchJob *pJob) { SCH_ERR_JRET(schBuildTaskRalation(pJob, planToTask)); _return: - if (planToTask) { taosHashCleanup(planToTask); } @@ -592,60 +590,59 @@ int32_t schProcessRspMsg(SSchJob *job, SSchTask *task, int32_t msgType, char *ms break; } case TDMT_VND_SUBMIT_RSP: { - if (rspCode != TSDB_CODE_SUCCESS || NULL == msg) { - SCH_ERR_JRET(schProcessOnTaskFailure(job, task, rspCode)); - } else { - SShellSubmitRspMsg *rsp = (SShellSubmitRspMsg *)msg; - job->resNumOfRows += rsp->affectedRows; + if (rspCode != TSDB_CODE_SUCCESS || NULL == msg) { + SCH_ERR_JRET(schProcessOnTaskFailure(job, task, rspCode)); + } else { + SShellSubmitRspMsg *rsp = (SShellSubmitRspMsg *)msg; + job->resNumOfRows += rsp->affectedRows; - code = schProcessOnTaskSuccess(job, task); - if (code) { - goto _task_error; - } + code = schProcessOnTaskSuccess(job, task); + if (code) { + goto _task_error; } - break; } + break; + } case TDMT_VND_QUERY_RSP: { - SQueryTableRsp *rsp = (SQueryTableRsp *)msg; - - if (rsp->code != TSDB_CODE_SUCCESS || NULL == msg) { - SCH_ERR_JRET(schProcessOnTaskFailure(job, task, rsp->code)); - } else { - code = schBuildAndSendMsg(job, task, TDMT_VND_RES_READY); - if (code) { - goto _task_error; - } + SQueryTableRsp *rsp = (SQueryTableRsp *)msg; + + if (rsp->code != TSDB_CODE_SUCCESS || NULL == msg) { + SCH_ERR_JRET(schProcessOnTaskFailure(job, task, rsp->code)); + } else { + code = schBuildAndSendMsg(job, task, TDMT_VND_RES_READY); + if (code) { + goto _task_error; } - break; } + break; + } case TDMT_VND_RES_READY_RSP: { - SResReadyRsp *rsp = (SResReadyRsp *)msg; - - if (rsp->code != TSDB_CODE_SUCCESS || NULL == msg) { - SCH_ERR_JRET(schProcessOnTaskFailure(job, task, rsp->code)); - } else { - code = schProcessOnTaskSuccess(job, task); - if (code) { - goto _task_error; - } + SResReadyRsp *rsp = (SResReadyRsp *)msg; + + if (rsp->code != TSDB_CODE_SUCCESS || NULL == msg) { + SCH_ERR_JRET(schProcessOnTaskFailure(job, task, rsp->code)); + } else { + code = schProcessOnTaskSuccess(job, task); + if (code) { + goto _task_error; } - break; } + break; + } case TDMT_VND_FETCH_RSP: { - SCH_ERR_JRET(rspCode); - SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)msg; + SCH_ERR_JRET(rspCode); + SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)msg; - job->res = rsp; - if (rsp) { - job->resNumOfRows = rsp->numOfRows; - } - - SCH_ERR_JRET(schProcessOnDataFetched(job)); - break; + job->res = rsp; + if (rsp) { + job->resNumOfRows = rsp->numOfRows; } + + SCH_ERR_JRET(schProcessOnDataFetched(job)); + break; + } case TDMT_VND_DROP_TASK: { - - } + } default: qError("unknown msg type:%d received", msgType); return TSDB_CODE_QRY_INVALID_INPUT; @@ -1111,7 +1108,7 @@ int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, void } int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, void** pJob) { - if (NULL == transport || /*NULL == nodeList || */NULL == pDag || NULL == pDag->pSubplans || NULL == pJob) { + if (NULL == transport || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } From 358b5641aa4a20f164102bb736627765f5309839 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 10 Jan 2022 11:55:41 +0000 Subject: [PATCH 03/57] more tkv --- include/libs/tkv/tkv.h | 64 ---------------------------------- source/libs/tkv/inc/tDiskMgr.h | 11 +++--- source/libs/tkv/inc/tkv.h | 32 +++++++++++++++++ source/libs/tkv/src/tDiskMgr.c | 10 +++--- 4 files changed, 44 insertions(+), 73 deletions(-) delete mode 100644 include/libs/tkv/tkv.h create mode 100644 source/libs/tkv/inc/tkv.h diff --git a/include/libs/tkv/tkv.h b/include/libs/tkv/tkv.h deleted file mode 100644 index 98194f090c..0000000000 --- a/include/libs/tkv/tkv.h +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TKV_H_ -#define _TD_TKV_H_ - -#if 0 -#include "os.h" - -#ifdef __cplusplus -extern "C" { -#endif - -// Types exported -typedef struct STkvDb STkvDb; -typedef struct STkvOpts STkvOpts; -typedef struct STkvCache STkvCache; -typedef struct STkvReadOpts STkvReadOpts; -typedef struct STkvWriteOpts STkvWriteOpts; - -// DB operations -STkvDb *tkvOpen(const STkvOpts *options, const char *path); -void tkvClose(STkvDb *db); -void tkvPut(STkvDb *db, const STkvWriteOpts *, const char *key, size_t keylen, const char *val, size_t vallen); -char * tkvGet(STkvDb *db, const STkvReadOpts *, const char *key, size_t keylen, size_t *vallen); -void tkvCommit(STkvDb *db); - -// DB options -STkvOpts *tkvOptsCreate(); -void tkvOptsDestroy(STkvOpts *); -void tkvOptionsSetCache(STkvOpts *, STkvCache *); -void tkvOptsSetCreateIfMissing(STkvOpts *, unsigned char); - -// DB cache -typedef enum { TKV_LRU_CACHE = 0, TKV_LFU_CACHE = 1 } ETkvCacheType; -STkvCache *tkvCacheCreate(size_t capacity, ETkvCacheType type); -void tkvCacheDestroy(STkvCache *); - -// STkvReadOpts -STkvReadOpts *tkvReadOptsCreate(); -void tkvReadOptsDestroy(STkvReadOpts *); - -// STkvWriteOpts -STkvWriteOpts *tkvWriteOptsCreate(); -void tkvWriteOptsDestroy(STkvWriteOpts *); - -#ifdef __cplusplus -} -#endif - -#endif -#endif /*_TD_TKV_H_*/ \ No newline at end of file diff --git a/source/libs/tkv/inc/tDiskMgr.h b/source/libs/tkv/inc/tDiskMgr.h index 03622284f4..65ab4d7a3d 100644 --- a/source/libs/tkv/inc/tDiskMgr.h +++ b/source/libs/tkv/inc/tDiskMgr.h @@ -26,11 +26,12 @@ extern "C" { typedef struct SDiskMgr SDiskMgr; -int tdmOpen(SDiskMgr **ppDiskMgr, const char *fname, uint16_t pgsize); -int tdmClose(SDiskMgr *pDiskMgr); -int tdmReadPage(SDiskMgr *pDiskMgr, pgid_t pgid, void *pData); -int tdmWritePage(SDiskMgr *pDiskMgr, pgid_t pgid, const void *pData); -int32_t tdmAllocPage(SDiskMgr *pDiskMgr); +int tdmOpen(SDiskMgr **ppDiskMgr, const char *fname, uint16_t pgsize); +int tdmClose(SDiskMgr *pDiskMgr); +int tdmReadPage(SDiskMgr *pDiskMgr, pgid_t pgid, void *pData); +int tdmWritePage(SDiskMgr *pDiskMgr, pgid_t pgid, const void *pData); +int tdmFlush(SDiskMgr *pDiskMgr); +pgid_t tdmAllocPage(SDiskMgr *pDiskMgr); #ifdef __cplusplus } diff --git a/source/libs/tkv/inc/tkv.h b/source/libs/tkv/inc/tkv.h new file mode 100644 index 0000000000..3a08906033 --- /dev/null +++ b/source/libs/tkv/inc/tkv.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TKV_H_ +#define _TD_TKV_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +// SKey +typedef struct SKey { + void *bdata; +} SKey, SValue; + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_TKV_H_*/ \ No newline at end of file diff --git a/source/libs/tkv/src/tDiskMgr.c b/source/libs/tkv/src/tDiskMgr.c index c8ce3c6c2a..5774555ae0 100644 --- a/source/libs/tkv/src/tDiskMgr.c +++ b/source/libs/tkv/src/tDiskMgr.c @@ -16,10 +16,10 @@ #include "tDiskMgr.h" struct SDiskMgr { - const char *fname; - uint16_t pgsize; - FileFd fd; - int32_t npgid; + char * fname; + uint16_t pgsize; + FileFd fd; + pgid_t npgid; }; #define PAGE_OFFSET(PGID, PGSIZE) ((PGID) * (PGSIZE)) @@ -69,4 +69,6 @@ int tdmWritePage(SDiskMgr *pDiskMgr, pgid_t pgid, const void *pData) { return 0; } +int tdmFlush(SDiskMgr *pDiskMgr) { return taosFsyncFile(pDiskMgr->fd); } + int32_t tdmAllocPage(SDiskMgr *pDiskMgr) { return pDiskMgr->npgid++; } \ No newline at end of file From dca02a8566bb931178894502bbe35a0168abe102 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 10 Jan 2022 12:40:15 +0000 Subject: [PATCH 04/57] more tkv --- source/libs/tkv/CMakeLists.txt | 1 + source/libs/tkv/inc/tkvBufPool.h | 39 ++++++++++++++ source/libs/tkv/inc/tkvDef.h | 3 ++ .../libs/tkv/inc/{tDiskMgr.h => tkvDiskMgr.h} | 14 ++--- source/libs/tkv/inc/{tPage.h => tkvPage.h} | 20 ++++--- source/libs/tkv/src/tDiskMgr.c | 18 +++---- source/libs/tkv/src/tkvBufPool.c | 54 +++++++++++++++++++ 7 files changed, 127 insertions(+), 22 deletions(-) create mode 100644 source/libs/tkv/inc/tkvBufPool.h rename source/libs/tkv/inc/{tDiskMgr.h => tkvDiskMgr.h} (67%) rename source/libs/tkv/inc/{tPage.h => tkvPage.h} (71%) create mode 100644 source/libs/tkv/src/tkvBufPool.c diff --git a/source/libs/tkv/CMakeLists.txt b/source/libs/tkv/CMakeLists.txt index 0620e12f55..ec3259d1f2 100644 --- a/source/libs/tkv/CMakeLists.txt +++ b/source/libs/tkv/CMakeLists.txt @@ -8,4 +8,5 @@ target_include_directories( target_link_libraries( tkv PUBLIC os + PUBLIC util ) \ No newline at end of file diff --git a/source/libs/tkv/inc/tkvBufPool.h b/source/libs/tkv/inc/tkvBufPool.h new file mode 100644 index 0000000000..ec8d177a9a --- /dev/null +++ b/source/libs/tkv/inc/tkvBufPool.h @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TKV_BUF_POOL_H_ +#define _TD_TKV_BUF_POOL_H_ + +#include "tkvPage.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct STkvBufPool STkvBufPool; + +int tbpOpen(STkvBufPool **ppTkvBufPool); +int tbpClose(STkvBufPool *pTkvBufPool); +STkvPage *tbpNewPage(STkvBufPool *pTkvBufPool); +int tbpDelPage(STkvBufPool *pTkvBufPool); +STkvPage *tbpFetchPage(STkvBufPool *pTkvBufPool, pgid_t pgid); +int tbpUnpinPage(STkvBufPool *pTkvBufPool, pgid_t pgid); +void tbpFlushPages(STkvBufPool *pTkvBufPool); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_TKV_BUF_POOL_H_*/ \ No newline at end of file diff --git a/source/libs/tkv/inc/tkvDef.h b/source/libs/tkv/inc/tkvDef.h index 6f1072abd5..a80c395364 100644 --- a/source/libs/tkv/inc/tkvDef.h +++ b/source/libs/tkv/inc/tkvDef.h @@ -26,6 +26,9 @@ extern "C" { typedef int32_t pgid_t; #define TKV_IVLD_PGID ((pgid_t)-1) +// framd_id_t +typedef int32_t frame_id_t; + #ifdef __cplusplus } #endif diff --git a/source/libs/tkv/inc/tDiskMgr.h b/source/libs/tkv/inc/tkvDiskMgr.h similarity index 67% rename from source/libs/tkv/inc/tDiskMgr.h rename to source/libs/tkv/inc/tkvDiskMgr.h index 65ab4d7a3d..2ebe98ace2 100644 --- a/source/libs/tkv/inc/tDiskMgr.h +++ b/source/libs/tkv/inc/tkvDiskMgr.h @@ -24,14 +24,14 @@ extern "C" { #include "tkvDef.h" -typedef struct SDiskMgr SDiskMgr; +typedef struct STkvDiskMgr STkvDiskMgr; -int tdmOpen(SDiskMgr **ppDiskMgr, const char *fname, uint16_t pgsize); -int tdmClose(SDiskMgr *pDiskMgr); -int tdmReadPage(SDiskMgr *pDiskMgr, pgid_t pgid, void *pData); -int tdmWritePage(SDiskMgr *pDiskMgr, pgid_t pgid, const void *pData); -int tdmFlush(SDiskMgr *pDiskMgr); -pgid_t tdmAllocPage(SDiskMgr *pDiskMgr); +int tdmOpen(STkvDiskMgr **ppDiskMgr, const char *fname, uint16_t pgsize); +int tdmClose(STkvDiskMgr *pDiskMgr); +int tdmReadPage(STkvDiskMgr *pDiskMgr, pgid_t pgid, void *pData); +int tdmWritePage(STkvDiskMgr *pDiskMgr, pgid_t pgid, const void *pData); +int tdmFlush(STkvDiskMgr *pDiskMgr); +pgid_t tdmAllocPage(STkvDiskMgr *pDiskMgr); #ifdef __cplusplus } diff --git a/source/libs/tkv/inc/tPage.h b/source/libs/tkv/inc/tkvPage.h similarity index 71% rename from source/libs/tkv/inc/tPage.h rename to source/libs/tkv/inc/tkvPage.h index 0546f6184f..d596d215cd 100644 --- a/source/libs/tkv/inc/tPage.h +++ b/source/libs/tkv/inc/tkvPage.h @@ -17,22 +17,30 @@ #define _TD_TKV_PAGE_H_ #include "os.h" +#include "tkvDef.h" #ifdef __cplusplus extern "C" { #endif +typedef struct STkvPage { + pgid_t pgid; + int32_t pinCount; + bool idDirty; + char* pData; +} STkvPage; + typedef struct { uint16_t dbver; uint16_t pgsize; uint32_t cksm; -} SPgHdr; +} STkvPgHdr; -typedef struct { - SPgHdr chdr; - uint16_t used; // number of used slots - uint16_t loffset; // the offset of the starting location of the last slot used -} SSlottedPgHdr; +// typedef struct { +// SPgHdr chdr; +// uint16_t used; // number of used slots +// uint16_t loffset; // the offset of the starting location of the last slot used +// } SSlottedPgHdr; #ifdef __cplusplus } diff --git a/source/libs/tkv/src/tDiskMgr.c b/source/libs/tkv/src/tDiskMgr.c index 5774555ae0..fa8f6062d8 100644 --- a/source/libs/tkv/src/tDiskMgr.c +++ b/source/libs/tkv/src/tDiskMgr.c @@ -13,9 +13,9 @@ * along with this program. If not, see . */ -#include "tDiskMgr.h" +#include "tkvDiskMgr.h" -struct SDiskMgr { +struct STkvDiskMgr { char * fname; uint16_t pgsize; FileFd fd; @@ -24,8 +24,8 @@ struct SDiskMgr { #define PAGE_OFFSET(PGID, PGSIZE) ((PGID) * (PGSIZE)) -int tdmOpen(SDiskMgr **ppDiskMgr, const char *fname, uint16_t pgsize) { - SDiskMgr *pDiskMgr; +int tdmOpen(STkvDiskMgr **ppDiskMgr, const char *fname, uint16_t pgsize) { + STkvDiskMgr *pDiskMgr; pDiskMgr = malloc(sizeof(*pDiskMgr)); if (pDiskMgr == NULL) { @@ -50,25 +50,25 @@ int tdmOpen(SDiskMgr **ppDiskMgr, const char *fname, uint16_t pgsize) { return 0; } -int tdmClose(SDiskMgr *pDiskMgr) { +int tdmClose(STkvDiskMgr *pDiskMgr) { close(pDiskMgr->fd); free(pDiskMgr->fname); free(pDiskMgr); return 0; } -int tdmReadPage(SDiskMgr *pDiskMgr, pgid_t pgid, void *pData) { +int tdmReadPage(STkvDiskMgr *pDiskMgr, pgid_t pgid, void *pData) { taosLSeekFile(pDiskMgr->fd, PAGE_OFFSET(pgid, pDiskMgr->pgsize), SEEK_SET); taosReadFile(pDiskMgr->fd, pData, pDiskMgr->pgsize); return 0; } -int tdmWritePage(SDiskMgr *pDiskMgr, pgid_t pgid, const void *pData) { +int tdmWritePage(STkvDiskMgr *pDiskMgr, pgid_t pgid, const void *pData) { taosLSeekFile(pDiskMgr->fd, PAGE_OFFSET(pgid, pDiskMgr->pgsize), SEEK_SET); taosWriteFile(pDiskMgr->fd, pData, pDiskMgr->pgsize); return 0; } -int tdmFlush(SDiskMgr *pDiskMgr) { return taosFsyncFile(pDiskMgr->fd); } +int tdmFlush(STkvDiskMgr *pDiskMgr) { return taosFsyncFile(pDiskMgr->fd); } -int32_t tdmAllocPage(SDiskMgr *pDiskMgr) { return pDiskMgr->npgid++; } \ No newline at end of file +int32_t tdmAllocPage(STkvDiskMgr *pDiskMgr) { return pDiskMgr->npgid++; } \ No newline at end of file diff --git a/source/libs/tkv/src/tkvBufPool.c b/source/libs/tkv/src/tkvBufPool.c new file mode 100644 index 0000000000..86bfa0ba3e --- /dev/null +++ b/source/libs/tkv/src/tkvBufPool.c @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "thash.h" +#include "tlist.h" + +#include "tkvBufPool.h" +#include "tkvDiskMgr.h" +#include "tkvPage.h" + +struct SFrameIdWrapper { + TD_SLIST_NODE(SFrameIdWrapper); + frame_id_t id; +}; + +struct STkvBufPool { + STkvPage* pages; + STkvDiskMgr* pDiskMgr; + SHashObj* pgTb; // page_id_t --> frame_id_t + TD_SLIST(SFrameIdWrapper) freeList; + pthread_mutex_t mutex; +}; + +typedef struct STkvLRUReplacer { +} STkvLRUReplacer; + +typedef struct STkvLFUReplacer { +} STkvLFUReplacer; + +typedef struct STkvCLKReplacer { +} STkvCLKReplacer; + +typedef enum { TKV_LRU_REPLACER = 0, TKV_LFU_REPLACER, TVK_CLK_REPLACER } tkv_replacer_t; + +typedef struct STkvReplacer { + tkv_replacer_t type; + union { + STkvLRUReplacer lruRep; + STkvLFUReplacer lfuRep; + STkvCLKReplacer clkRep; + }; +} STkvReplacer; \ No newline at end of file From 3513390fb8828d3d9cfa443303891b0699bd1b58 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Mon, 10 Jan 2022 12:51:40 +0000 Subject: [PATCH 05/57] just have a try --- include/util/tarray.h | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/include/util/tarray.h b/include/util/tarray.h index f7c72add01..6d6120a49b 100644 --- a/include/util/tarray.h +++ b/include/util/tarray.h @@ -23,10 +23,23 @@ extern "C" { #include "os.h" #include "talgo.h" +#if 0 +#define TARRAY(TYPE) \ + struct { \ + int32_t tarray_size_; \ + int32_t tarray_neles_; \ + struct TYPE* td_array_data_; \ + } + +#define TARRAY_SIZE(ARRAY) (ARRAY)->tarray_size_ +#define TARRAY_NELES(ARRAY) (ARRAY)->tarray_neles_ +#define TARRAY_ELE_AT(ARRAY, IDX) ((ARRAY)->td_array_data_ + idx) +#endif + #define TARRAY_MIN_SIZE 8 #define TARRAY_GET_ELEM(array, index) ((void*)((char*)((array)->pData) + (index) * (array)->elemSize)) -#define TARRAY_ELEM_IDX(array, ele) (POINTER_DISTANCE(ele, (array)->pData) / (array)->elemSize) -#define TARRAY_GET_START(array) ((array)->pData) +#define TARRAY_ELEM_IDX(array, ele) (POINTER_DISTANCE(ele, (array)->pData) / (array)->elemSize) +#define TARRAY_GET_START(array) ((array)->pData) typedef struct SArray { size_t size; @@ -57,7 +70,7 @@ int32_t taosArrayEnsureCap(SArray* pArray, size_t tsize); * @param nEles * @return */ -void *taosArrayAddBatch(SArray *pArray, const void *pData, int nEles); +void* taosArrayAddBatch(SArray* pArray, const void* pData, int nEles); /** * @@ -65,7 +78,7 @@ void *taosArrayAddBatch(SArray *pArray, const void *pData, int nEles); * @param pData position array list * @param numOfElems the number of removed position */ -void taosArrayRemoveBatch(SArray *pArray, const int32_t* pData, int32_t numOfElems); +void taosArrayRemoveBatch(SArray* pArray, const int32_t* pData, int32_t numOfElems); /** * @@ -73,7 +86,7 @@ void taosArrayRemoveBatch(SArray *pArray, const int32_t* pData, int32_t numOfEle * @param comparFn * @param fp */ -void taosArrayRemoveDuplicate(SArray *pArray, __compar_fn_t comparFn, void (*fp)(void*)); +void taosArrayRemoveDuplicate(SArray* pArray, __compar_fn_t comparFn, void (*fp)(void*)); /** * add all element from the source array list into the destination @@ -242,19 +255,18 @@ int32_t taosArraySearchIdx(const SArray* pArray, const void* key, __compar_fn_t */ char* taosArraySearchString(const SArray* pArray, const char* key, __compar_fn_t comparFn, int flags); - /** * sort the pointer data in the array * @param pArray - * @param compar - * @param param + * @param compar + * @param param * @return */ -void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void *param); +void taosArraySortPWithExt(SArray* pArray, __ext_compar_fn_t fn, const void* param); #ifdef __cplusplus } #endif -#endif /*_TD_UTIL_ARRAY_H*/ +#endif /*_TD_UTIL_ARRAY_H*/ From 89dcffce889e362831701d46f225f77ac84cee74 Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 11 Jan 2022 10:14:06 +0800 Subject: [PATCH 06/57] feature/qnode --- include/libs/scheduler/scheduler.h | 2 +- source/libs/catalog/test/catalogTests.cpp | 1 + source/libs/scheduler/src/scheduler.c | 154 ++++++++++-------- source/libs/scheduler/test/schedulerTests.cpp | 93 +++++++---- source/os/src/osSysinfo.c | 2 +- 5 files changed, 156 insertions(+), 96 deletions(-) diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index 74b7813465..58ec3de32f 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -89,7 +89,7 @@ int32_t scheduleFetchRows(void *pJob, void **data); * @param pJob * @return */ -int32_t scheduleCancelJob(void *pJob); +//int32_t scheduleCancelJob(void *pJob); /** * Free the query job diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 87a3000d09..436593f9d6 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -1322,3 +1322,4 @@ int main(int argc, char** argv) { + \ No newline at end of file diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index d478e80af0..4a0d62a06f 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -87,68 +87,72 @@ int32_t schValidateTaskReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t m int32_t schCheckAndUpdateJobStatus(SSchJob *pJob, int8_t newStatus) { int32_t code = 0; - int8_t oriStatus = SCH_GET_JOB_STATUS(pJob); + int8_t oriStatus = 0; -/* - if (oriStatus == newStatus) { - SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); - } - - switch (oriStatus) { - case JOB_TASK_STATUS_NULL: - if (newStatus != JOB_TASK_STATUS_EXECUTING - && newStatus != JOB_TASK_STATUS_FAILED - && newStatus != JOB_TASK_STATUS_NOT_START) { - SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); - } - - break; - case JOB_TASK_STATUS_NOT_START: - if (newStatus != JOB_TASK_STATUS_CANCELLED) { - SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); - } - - break; - case JOB_TASK_STATUS_EXECUTING: - if (newStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED - && newStatus != JOB_TASK_STATUS_FAILED - && newStatus != JOB_TASK_STATUS_CANCELLING - && newStatus != JOB_TASK_STATUS_CANCELLED - && newStatus != JOB_TASK_STATUS_DROPPING) { - SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); - } - - break; - case JOB_TASK_STATUS_PARTIAL_SUCCEED: - if (newStatus != JOB_TASK_STATUS_EXECUTING - && newStatus != JOB_TASK_STATUS_SUCCEED - && newStatus != JOB_TASK_STATUS_CANCELLED) { - SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); - } - - break; - case JOB_TASK_STATUS_SUCCEED: - case JOB_TASK_STATUS_FAILED: - case JOB_TASK_STATUS_CANCELLING: - if (newStatus != JOB_TASK_STATUS_CANCELLED) { - SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); - } - - break; - case JOB_TASK_STATUS_CANCELLED: - case JOB_TASK_STATUS_DROPPING: + while (true) { + oriStatus = SCH_GET_JOB_STATUS(pJob); + + if (oriStatus == newStatus) { SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); - break; - - default: - qError("invalid task status:%d", oriStatus); - return TSDB_CODE_QRY_APP_ERROR; + } + + switch (oriStatus) { + case JOB_TASK_STATUS_NULL: + if (newStatus != JOB_TASK_STATUS_NOT_START) { + SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + } + + break; + case JOB_TASK_STATUS_NOT_START: + if (newStatus != JOB_TASK_STATUS_EXECUTING) { + SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + } + + break; + case JOB_TASK_STATUS_EXECUTING: + if (newStatus != JOB_TASK_STATUS_PARTIAL_SUCCEED + && newStatus != JOB_TASK_STATUS_FAILED + && newStatus != JOB_TASK_STATUS_CANCELLING + && newStatus != JOB_TASK_STATUS_CANCELLED + && newStatus != JOB_TASK_STATUS_DROPPING) { + SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + } + + break; + case JOB_TASK_STATUS_PARTIAL_SUCCEED: + if (newStatus != JOB_TASK_STATUS_FAILED + && newStatus != JOB_TASK_STATUS_SUCCEED + && newStatus != JOB_TASK_STATUS_DROPPING) { + SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + } + + break; + case JOB_TASK_STATUS_SUCCEED: + case JOB_TASK_STATUS_FAILED: + case JOB_TASK_STATUS_CANCELLING: + if (newStatus != JOB_TASK_STATUS_DROPPING) { + SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + } + + break; + case JOB_TASK_STATUS_CANCELLED: + case JOB_TASK_STATUS_DROPPING: + SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + break; + + default: + SCH_JOB_ELOG("invalid job status:%d", oriStatus); + SCH_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); + } + + if (oriStatus != atomic_val_compare_exchange_8(&pJob->status, oriStatus, newStatus)) { + continue; + } + + SCH_JOB_DLOG("job status updated from %d to %d", oriStatus, newStatus); + + break; } -*/ - - SCH_SET_JOB_STATUS(pJob, newStatus); - - SCH_JOB_DLOG("status updated from %d to %d", oriStatus, newStatus); return TSDB_CODE_SUCCESS; @@ -507,6 +511,7 @@ int32_t schTaskCheckAndSetRetry(SSchJob *job, SSchTask *task, int32_t errCode, b } +// Note: no more error processing, handled in function internal int32_t schFetchFromRemote(SSchJob *pJob) { int32_t code = 0; @@ -515,7 +520,13 @@ int32_t schFetchFromRemote(SSchJob *pJob) { return TSDB_CODE_SUCCESS; } - if (atomic_load_ptr(&pJob->res)) + void *res = atomic_load_ptr(&pJob->res); + if (res) { + atomic_val_compare_exchange_32(&pJob->remoteFetch, 1, 0); + + SCH_JOB_DLOG("res already fetched, res:%p", res); + return TSDB_CODE_SUCCESS; + } SCH_ERR_JRET(schBuildAndSendMsg(pJob, pJob->fetchTask, &pJob->resNode, TDMT_VND_FETCH)); @@ -525,12 +536,15 @@ _return: atomic_val_compare_exchange_32(&pJob->remoteFetch, 1, 0); + schProcessOnJobFailure(pJob, code); + return code; } // Note: no more error processing, handled in function internal int32_t schProcessOnJobFailure(SSchJob *pJob, int32_t errCode) { + // if already FAILED, no more processing SCH_ERR_RET(schCheckAndUpdateJobStatus(pJob, JOB_TASK_STATUS_FAILED)); if (errCode) { @@ -813,7 +827,7 @@ int32_t schHandleCallback(void* param, const SDataBuf* pMsg, int32_t msgType, in SSchJob **job = taosHashGet(schMgmt.jobs, &pParam->queryId, sizeof(pParam->queryId)); if (NULL == job || NULL == (*job)) { - qError("QID:%"PRIx64" taosHashGet queryId not exist", pParam->queryId); + qError("QID:%"PRIx64" taosHashGet queryId not exist, may be dropped", pParam->queryId); SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); } @@ -1147,7 +1161,7 @@ void schDropTaskOnExecutedNode(SSchJob *pJob, SSchTask *pTask) { int32_t size = (int32_t)taosArrayGetSize(pTask->execAddrs); if (size <= 0) { - SCH_TASK_DLOG("empty exec address, status:%d", SCH_GET_TASK_STATUS(pTask)); + SCH_TASK_DLOG("task has no exec address, no need to drop it, status:%d", SCH_GET_TASK_STATUS(pTask)); return; } @@ -1157,6 +1171,8 @@ void schDropTaskOnExecutedNode(SSchJob *pJob, SSchTask *pTask) { schBuildAndSendMsg(pJob, pTask, addr, TDMT_VND_DROP_TASK); } + + SCH_TASK_DLOG("task has %d exec address", size); } void schDropTaskInHashList(SSchJob *pJob, SHashObj *list) { @@ -1331,6 +1347,12 @@ int32_t scheduleFetchRows(void *job, void **data) { SSchJob *pJob = job; int32_t code = 0; + int8_t status = SCH_GET_JOB_STATUS(pJob); + if (status == JOB_TASK_STATUS_DROPPING) { + SCH_JOB_ELOG("job is dropping, status:%d", status); + return TSDB_CODE_SCH_STATUS_ERROR; + } + atomic_add_fetch_32(&pJob->ref, 1); if (!SCH_JOB_NEED_FETCH(&pJob->attr)) { @@ -1345,8 +1367,6 @@ int32_t scheduleFetchRows(void *job, void **data) { SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - int8_t status = SCH_GET_JOB_STATUS(pJob); - if (status == JOB_TASK_STATUS_FAILED) { *data = atomic_load_ptr(&pJob->res); atomic_store_ptr(&pJob->res, NULL); @@ -1414,6 +1434,10 @@ void scheduleFreeJob(void *job) { return; } + schCheckAndUpdateJobStatus(pJob, JOB_TASK_STATUS_DROPPING); + + SCH_JOB_DLOG("job removed from list, no further ref, ref:%d", atomic_load_32(&pJob->ref)); + while (true) { int32_t ref = atomic_load_32(&pJob->ref); if (0 == ref) { @@ -1425,6 +1449,8 @@ void scheduleFreeJob(void *job) { } } + SCH_JOB_DLOG("job no ref now, status:%d", SCH_GET_JOB_STATUS(pJob)); + if (pJob->status == JOB_TASK_STATUS_EXECUTING) { schCancelJob(pJob); } diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 70e49e6b45..f75f0ef263 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -38,52 +38,73 @@ namespace { extern "C" int32_t schHandleResponseMsg(SSchJob *job, SSchTask *task, int32_t msgType, char *msg, int32_t msgSize, int32_t rspCode); +void schtInitLogFile() { + const char *defaultLogFileNamePrefix = "taoslog"; + const int32_t maxLogFileNum = 10; + + tsAsyncLog = 0; + qDebugFlag = 159; + + char temp[128] = {0}; + sprintf(temp, "%s/%s", tsLogDir, defaultLogFileNamePrefix); + if (taosInitLog(temp, tsNumOfLogLines, maxLogFileNum) < 0) { + printf("failed to open log file in directory:%s\n", tsLogDir); + } + +} + + void schtBuildQueryDag(SQueryDag *dag) { uint64_t qId = 0x0000000000000001; dag->queryId = qId; dag->numOfSubplans = 2; dag->pSubplans = taosArrayInit(dag->numOfSubplans, POINTER_BYTES); - SArray *scan = taosArrayInit(1, sizeof(SSubplan)); - SArray *merge = taosArrayInit(1, sizeof(SSubplan)); + SArray *scan = taosArrayInit(1, POINTER_BYTES); + SArray *merge = taosArrayInit(1, POINTER_BYTES); - SSubplan scanPlan = {0}; - SSubplan mergePlan = {0}; + SSubplan *scanPlan = (SSubplan *)calloc(1, sizeof(SSubplan)); + SSubplan *mergePlan = (SSubplan *)calloc(1, sizeof(SSubplan)); - scanPlan.id.queryId = qId; - scanPlan.id.templateId = 0x0000000000000002; - scanPlan.id.subplanId = 0x0000000000000003; - scanPlan.type = QUERY_TYPE_SCAN; - scanPlan.execNode.numOfEps = 1; - scanPlan.execNode.nodeId = 1; - scanPlan.execNode.inUse = 0; - scanPlan.execNode.epAddr[0].port = 6030; - strcpy(scanPlan.execNode.epAddr[0].fqdn, "ep0"); - scanPlan.pChildren = NULL; - scanPlan.level = 1; - scanPlan.pParents = taosArrayInit(1, POINTER_BYTES); - scanPlan.pNode = (SPhyNode*)calloc(1, sizeof(SPhyNode)); + scanPlan->id.queryId = qId; + scanPlan->id.templateId = 0x0000000000000002; + scanPlan->id.subplanId = 0x0000000000000003; + scanPlan->type = QUERY_TYPE_SCAN; + scanPlan->execNode.numOfEps = 1; + scanPlan->execNode.nodeId = 1; + scanPlan->execNode.inUse = 0; + scanPlan->execNode.epAddr[0].port = 6030; + strcpy(scanPlan->execNode.epAddr[0].fqdn, "ep0"); + scanPlan->pChildren = NULL; + scanPlan->level = 1; + scanPlan->pParents = taosArrayInit(1, POINTER_BYTES); + scanPlan->pNode = (SPhyNode*)calloc(1, sizeof(SPhyNode)); - mergePlan.id.queryId = qId; - mergePlan.id.templateId = 0x4444444444; - mergePlan.id.subplanId = 0x5555555555; - mergePlan.type = QUERY_TYPE_MERGE; - mergePlan.level = 0; - mergePlan.execNode.numOfEps = 0; - mergePlan.pChildren = taosArrayInit(1, POINTER_BYTES); - mergePlan.pParents = NULL; - mergePlan.pNode = (SPhyNode*)calloc(1, sizeof(SPhyNode)); + mergePlan->id.queryId = qId; + mergePlan->id.templateId = 0x4444444444; + mergePlan->id.subplanId = 0x5555555555; + mergePlan->type = QUERY_TYPE_MERGE; + mergePlan->level = 0; + mergePlan->execNode.numOfEps = 0; + mergePlan->pChildren = taosArrayInit(1, POINTER_BYTES); + mergePlan->pParents = NULL; + mergePlan->pNode = (SPhyNode*)calloc(1, sizeof(SPhyNode)); SSubplan *mergePointer = (SSubplan *)taosArrayPush(merge, &mergePlan); SSubplan *scanPointer = (SSubplan *)taosArrayPush(scan, &scanPlan); - taosArrayPush(mergePointer->pChildren, &scanPointer); - taosArrayPush(scanPointer->pParents, &mergePointer); + taosArrayPush(mergePlan->pChildren, &scanPlan); + taosArrayPush(scanPlan->pParents, &mergePlan); taosArrayPush(dag->pSubplans, &merge); taosArrayPush(dag->pSubplans, &scan); } +void schtFreeQueryDag(SQueryDag *dag) { + +} + + void schtBuildInsertDag(SQueryDag *dag) { uint64_t qId = 0x0000000000000002; @@ -138,8 +159,8 @@ int32_t schtPlanToString(const SSubplan *subplan, char** str, int32_t* len) { return 0; } -int32_t schtExecNode(SSubplan* subplan, uint64_t templateId, SQueryNodeAddr* ep) { - return 0; +void schtExecNode(SSubplan* subplan, uint64_t templateId, SQueryNodeAddr* ep) { + } @@ -209,6 +230,9 @@ TEST(queryTest, normalCase) { SVgroupInfo vgInfo = {0}; void *pJob = NULL; SQueryDag dag = {0}; + + schtInitLogFile(); + SArray *qnodeList = taosArrayInit(1, sizeof(SEpAddr)); SEpAddr qnodeAddr = {0}; @@ -295,6 +319,8 @@ TEST(queryTest, normalCase) { ASSERT_EQ(data, (void*)NULL); scheduleFreeJob(pJob); + + schtFreeQueryDag(&dag); } @@ -308,6 +334,9 @@ TEST(insertTest, normalCase) { SVgroupInfo vgInfo = {0}; SQueryDag dag = {0}; uint64_t numOfRows = 0; + + schtInitLogFile(); + SArray *qnodeList = taosArrayInit(1, sizeof(SEpAddr)); SEpAddr qnodeAddr = {0}; @@ -336,7 +365,11 @@ TEST(insertTest, normalCase) { scheduleFreeJob(pInsertJob); } +TEST(multiThread, forceFree) { + schtInitLogFile(); + +} int main(int argc, char** argv) { diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index e235b0714e..0be17ca2b9 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -1087,7 +1087,7 @@ int32_t taosGetSystemUUID(char *uid, int32_t uidlen) { return 0; } - return -1; + return 0; } char *taosGetCmdlineByPID(int pid) { From d7ddb1589b309ff3c856c5ff9ab3f85c3b639d0d Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Mon, 10 Jan 2022 21:44:58 -0500 Subject: [PATCH 07/57] TD-12678 datasink interface adjust --- .../libs/executor/inc => include/libs/executor}/dataSinkMgt.h | 4 ++-- source/libs/executor/src/dataSinkMgt.c | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) rename {source/libs/executor/inc => include/libs/executor}/dataSinkMgt.h (96%) diff --git a/source/libs/executor/inc/dataSinkMgt.h b/include/libs/executor/dataSinkMgt.h similarity index 96% rename from source/libs/executor/inc/dataSinkMgt.h rename to include/libs/executor/dataSinkMgt.h index d13423b25d..6470f38ced 100644 --- a/source/libs/executor/inc/dataSinkMgt.h +++ b/include/libs/executor/dataSinkMgt.h @@ -73,9 +73,9 @@ void dsEndPut(DataSinkHandle handle); /** * Get the length of the data returned by the next call to dsGetDataBlock. * @param handle - * @return data length + * @param pLen data length */ -int32_t dsGetDataLength(DataSinkHandle handle, int32_t* pStatus); +void dsGetDataLength(DataSinkHandle handle, int32_t* pLen, int32_t* pStatus); /** * Get data, the caller needs to allocate data memory. diff --git a/source/libs/executor/src/dataSinkMgt.c b/source/libs/executor/src/dataSinkMgt.c index 8a96c5d05f..01cc84c4f7 100644 --- a/source/libs/executor/src/dataSinkMgt.c +++ b/source/libs/executor/src/dataSinkMgt.c @@ -41,9 +41,9 @@ void dsEndPut(DataSinkHandle handle) { return pHandleImpl->fEndPut(pHandleImpl); } -int32_t dsGetDataLength(DataSinkHandle handle, int32_t* pStatus) { +void dsGetDataLength(DataSinkHandle handle, int32_t* pLen, int32_t* pStatus) { SDataSinkHandle* pHandleImpl = (SDataSinkHandle*)handle; - return pHandleImpl->fGetLen(pHandleImpl, pStatus); + *pLen = pHandleImpl->fGetLen(pHandleImpl, pStatus); } int32_t dsGetDataBlock(DataSinkHandle handle, SOutPutData* pOutput, int32_t* pStatus) { From 52457838256af3b3fa4d9f3fabb4a0a83eaefd7b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 11 Jan 2022 10:51:23 +0800 Subject: [PATCH 08/57] [td-11818] add data into sink node --- include/libs/executor/executor.h | 12 ++-- source/libs/executor/inc/dataSinkMgt.h | 7 +- source/libs/executor/inc/executorInt.h | 1 + source/libs/executor/inc/executorimpl.h | 7 +- source/libs/executor/src/dataSinkMgt.c | 1 + source/libs/executor/src/executorMain.c | 88 +++++++++++++++---------- source/libs/executor/src/executorimpl.c | 1 + source/libs/qworker/src/qworker.c | 8 +-- src/inc/tsdb.h | 16 ++--- src/query/inc/qExecutor.h | 4 +- src/query/src/qExecutor.c | 66 +++++++++---------- 11 files changed, 120 insertions(+), 91 deletions(-) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 2356498bbe..71b014d025 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -21,6 +21,9 @@ extern "C" { #endif typedef void* qTaskInfo_t; +typedef void* DataSinkHandle; +struct SSubplan; + /** * Create the exec task object according to task json @@ -34,13 +37,14 @@ typedef void* qTaskInfo_t; int32_t qCreateExecTask(void* tsdb, int32_t vgId, struct SSubplan* pPlan, qTaskInfo_t* pTaskInfo); /** - * the main task execution function, including query on both table and multiple tables, + * The main task execution function, including query on both table and multiple tables, * which are decided according to the tag or table name query conditions * - * @param qinfo + * @param tinfo + * @param handle * @return */ -bool qExecTask(qTaskInfo_t qTask, SSDataBlock** pRes); +int32_t qExecTask(qTaskInfo_t tinfo, DataSinkHandle* handle); /** * Retrieve the produced results information, if current query is not paused or completed, @@ -62,7 +66,7 @@ int32_t qRetrieveQueryResultInfo(qTaskInfo_t qinfo, bool* buildRes, void* pRspCo * @param contLen payload length * @return */ -int32_t qDumpRetrieveResult(qTaskInfo_t qinfo, SRetrieveTableRsp** pRsp, int32_t* contLen, bool* continueExec); +//int32_t qDumpRetrieveResult(qTaskInfo_t qinfo, SRetrieveTableRsp** pRsp, int32_t* contLen, bool* continueExec); /** * return the transporter context (RPC) diff --git a/source/libs/executor/inc/dataSinkMgt.h b/source/libs/executor/inc/dataSinkMgt.h index d0057a213a..35a25f5104 100644 --- a/source/libs/executor/inc/dataSinkMgt.h +++ b/source/libs/executor/inc/dataSinkMgt.h @@ -21,6 +21,7 @@ extern "C" { #endif #include "os.h" +#include "executor.h" #include "executorimpl.h" #define DS_CAPACITY_ENOUGH 1 @@ -39,8 +40,6 @@ typedef struct SDataSinkMgtCfg { int32_t dsDataSinkMgtInit(SDataSinkMgtCfg *cfg); -typedef void* DataSinkHandle; - typedef struct SInputData { const SSDataBlock* pData; SHashObj* pTableRetrieveTsMap; @@ -68,6 +67,10 @@ int32_t dsCreateDataSinker(const struct SDataSink *pDataSink, DataSinkHandle* pH */ int32_t dsPutDataBlock(DataSinkHandle handle, const SInputData* pInput, int32_t* pStatus); +/** + * + * @param handle + */ void dsEndPut(DataSinkHandle handle); /** diff --git a/source/libs/executor/inc/executorInt.h b/source/libs/executor/inc/executorInt.h index ef66b3f247..59db66becc 100644 --- a/source/libs/executor/inc/executorInt.h +++ b/source/libs/executor/inc/executorInt.h @@ -20,6 +20,7 @@ extern "C" { #endif + #ifdef __cplusplus } #endif diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 33fafb4074..d76f270392 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -20,14 +20,16 @@ #include "ttszip.h" #include "tvariant.h" -#include "thash.h" +#include "dataSinkMgt.h" #include "executil.h" +#include "planner.h" #include "taosdef.h" #include "tarray.h" #include "tfilter.h" +#include "thash.h" #include "tlockfree.h" #include "tpagedfile.h" -#include "planner.h" +#include "executor.h" struct SColumnFilterElem; @@ -256,6 +258,7 @@ typedef struct SExecTaskInfo { // void* rspContext; // response context char *sql; // query sql string jmp_buf env; // + DataSinkHandle dsHandle; struct SOperatorInfo *pRoot; } SExecTaskInfo; diff --git a/source/libs/executor/src/dataSinkMgt.c b/source/libs/executor/src/dataSinkMgt.c index 8a96c5d05f..d9b122fbdd 100644 --- a/source/libs/executor/src/dataSinkMgt.c +++ b/source/libs/executor/src/dataSinkMgt.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include "tarray.h" #include "dataSinkMgt.h" #include "dataSinkInt.h" #include "planner.h" diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index ab9e7a5211..0c82bd1dac 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -13,9 +13,10 @@ * along with this program. If not, see . */ -#include -#include "exception.h" #include "os.h" +#include "tarray.h" +#include "dataSinkMgt.h" +#include "exception.h" #include "tcache.h" #include "tglobal.h" #include "tmsg.h" @@ -69,8 +70,9 @@ void freeParam(STaskParam *param) { int32_t qCreateExecTask(void* tsdb, int32_t vgId, SSubplan* pSubplan, qTaskInfo_t* pTaskInfo) { assert(tsdb != NULL && pSubplan != NULL); + SExecTaskInfo** pTask = (SExecTaskInfo**)pTaskInfo; - int32_t code = doCreateExecTaskInfo(pSubplan, (SExecTaskInfo**) pTaskInfo, tsdb); + int32_t code = doCreateExecTaskInfo(pSubplan, pTask, tsdb); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -81,8 +83,7 @@ int32_t qCreateExecTask(void* tsdb, int32_t vgId, SSubplan* pSubplan, qTaskInfo_ goto _error; } - DataSinkHandle pHandle = NULL; - code = dsCreateDataSinker(pSubplan->pDataSink, &pHandle); + code = dsCreateDataSinker(pSubplan->pDataSink, (*pTask)->dsHandle); _error: // if failed to add ref for all tables in this query, abort current query @@ -134,64 +135,79 @@ int waitMoment(SQInfo* pQInfo){ } #endif -bool qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes) { - SExecTaskInfo *pTaskInfo = (SExecTaskInfo *) tinfo; - int64_t threadId = taosGetSelfPthreadId(); +int32_t qExecTask(qTaskInfo_t tinfo, DataSinkHandle* handle) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + int64_t threadId = taosGetSelfPthreadId(); int64_t curOwner = 0; if ((curOwner = atomic_val_compare_exchange_64(&pTaskInfo->owner, 0, threadId)) != 0) { - qError("QInfo:0x%"PRIx64"-%p qhandle is now executed by thread:%p", GET_TASKID(pTaskInfo), pTaskInfo, (void*) curOwner); + qError("QInfo:0x%" PRIx64 "-%p qhandle is now executed by thread:%p", GET_TASKID(pTaskInfo), pTaskInfo, + (void*)curOwner); pTaskInfo->code = TSDB_CODE_QRY_IN_EXEC; - return false; + return pTaskInfo->code; } - if(pTaskInfo->cost.start == 0) { + if (pTaskInfo->cost.start == 0) { pTaskInfo->cost.start = taosGetTimestampMs(); } if (isTaskKilled(pTaskInfo)) { - qDebug("QInfo:0x%"PRIx64" it is already killed, abort", GET_TASKID(pTaskInfo)); -// return doBuildResCheck(pTaskInfo); + qDebug("QInfo:0x%" PRIx64 " it is already killed, abort", GET_TASKID(pTaskInfo)); + return pTaskInfo->code; } -// STaskRuntimeEnv* pRuntimeEnv = &pTaskInfo->runtimeEnv; -// if (pTaskInfo->tableqinfoGroupInfo.numOfTables == 0) { -// qDebug("QInfo:0x%"PRIx64" no table exists for query, abort", GET_TASKID(pTaskInfo)); -// setTaskStatus(pTaskInfo, TASK_COMPLETED); -// return doBuildResCheck(pTaskInfo); -// } + // STaskRuntimeEnv* pRuntimeEnv = &pTaskInfo->runtimeEnv; + // if (pTaskInfo->tableqinfoGroupInfo.numOfTables == 0) { + // qDebug("QInfo:0x%"PRIx64" no table exists for query, abort", GET_TASKID(pTaskInfo)); + // setTaskStatus(pTaskInfo, TASK_COMPLETED); + // return doBuildResCheck(pTaskInfo); + // } // error occurs, record the error code and return to client int32_t ret = setjmp(pTaskInfo->env); if (ret != TSDB_CODE_SUCCESS) { publishQueryAbortEvent(pTaskInfo, ret); pTaskInfo->code = ret; - qDebug("QInfo:0x%"PRIx64" query abort due to error/cancel occurs, code:%s", GET_TASKID(pTaskInfo), tstrerror(pTaskInfo->code)); -// return doBuildResCheck(pTaskInfo); + qDebug("QInfo:0x%" PRIx64 " query abort due to error/cancel occurs, code:%s", GET_TASKID(pTaskInfo), tstrerror(pTaskInfo->code)); + return pTaskInfo->code; } - qDebug("QInfo:0x%"PRIx64" query task is launched", GET_TASKID(pTaskInfo)); + qDebug("QInfo:0x%" PRIx64 " query task is launched", GET_TASKID(pTaskInfo)); bool newgroup = false; publishOperatorProfEvent(pTaskInfo->pRoot, QUERY_PROF_BEFORE_OPERATOR_EXEC); + int64_t st = 0; - int64_t st = taosGetTimestampUs(); - *pRes = pTaskInfo->pRoot->exec(pTaskInfo->pRoot, &newgroup); - // todo put the result into sink node. + handle = &pTaskInfo->dsHandle; - pTaskInfo->cost.elapsedTime += (taosGetTimestampUs() - st); - publishOperatorProfEvent(pTaskInfo->pRoot, QUERY_PROF_AFTER_OPERATOR_EXEC); + while(1) { + st = taosGetTimestampUs(); + SSDataBlock* pRes = pTaskInfo->pRoot->exec(pTaskInfo->pRoot, &newgroup); - if (isTaskKilled(pTaskInfo)) { - qDebug("QInfo:0x%"PRIx64" query is killed", GET_TASKID(pTaskInfo)); -// } else if (GET_NUM_OF_RESULTS(pRuntimeEnv) == 0) { -// qDebug("QInfo:0x%"PRIx64" over, %u tables queried, total %"PRId64" rows returned", pTaskInfo->qId, pRuntimeEnv->tableqinfoGroupInfo.numOfTables, -// pRuntimeEnv->resultInfo.total); - } else { -// qDebug("QInfo:0x%"PRIx64" query paused, %d rows returned, total:%" PRId64 " rows", pTaskInfo->qId, -// GET_NUM_OF_RESULTS(pRuntimeEnv), pRuntimeEnv->resultInfo.total); + pTaskInfo->cost.elapsedTime += (taosGetTimestampUs() - st); + publishOperatorProfEvent(pTaskInfo->pRoot, QUERY_PROF_AFTER_OPERATOR_EXEC); + + if (pRes == NULL) { // no results generated yet, abort + return pTaskInfo->code; + } + + int32_t status = 0; + SInputData inputData = {.pData = pRes, .pTableRetrieveTsMap = NULL}; + pTaskInfo->code = dsPutDataBlock(pTaskInfo->dsHandle, &inputData, &status); + + if (isTaskKilled(pTaskInfo)) { + qDebug("QInfo:0x%" PRIx64 " task is killed", GET_TASKID(pTaskInfo)); + // } else if (GET_NUM_OF_RESULTS(pRuntimeEnv) == 0) { + // qDebug("QInfo:0x%"PRIx64" over, %u tables queried, total %"PRId64" rows returned", pTaskInfo->qId, pRuntimeEnv->tableqinfoGroupInfo.numOfTables, + // pRuntimeEnv->resultInfo.total); + } + + if (status == DS_CAPACITY_FULL) { + qDebug("QInfo:0x%"PRIx64" query paused, %d rows returned, total:%" PRId64 " rows, in sinkNode:%d", GET_TASKID(pTaskInfo), + 0, 0L, 0); + return pTaskInfo->code; + } } -// return doBuildResCheck(pTaskInfo); } int32_t qRetrieveQueryResultInfo(qTaskInfo_t qinfo, bool* buildRes, void* pRspContext) { diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index db25e384be..9aeb979806 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -7181,6 +7181,7 @@ static SExecTaskInfo* createExecTaskInfo(uint64_t queryId) { pthread_mutex_init(&pTaskInfo->lock, NULL); pTaskInfo->cost.created = taosGetTimestampMs(); + pTaskInfo->id.queryId = queryId; return pTaskInfo; } diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 7351af83c5..ada534de7e 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -1038,9 +1038,9 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { QW_ERR_JRET(qwBuildAndSendQueryRsp(pMsg, TSDB_CODE_SUCCESS)); queryRsped = true; - - SSDataBlock* pRes = NULL; - code = qExecTask(pTaskInfo, &pRes); + + DataSinkHandle handle = NULL; + code = qExecTask(pTaskInfo, &handle); queryDone = false; //TODO call executer to execute subquery @@ -1048,7 +1048,7 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { if (code) { QW_ERR_JRET(code); } else { - QW_ERR_JRET(qwAddTaskResCache(qWorkerMgmt, msg->queryId, msg->taskId, pRes)); +// QW_ERR_JRET(qwAddTaskResCache(qWorkerMgmt, msg->queryId, msg->taskId, pRes)); QW_ERR_JRET(qwUpdateTaskStatus(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, JOB_TASK_STATUS_PARTIAL_SUCCEED)); } diff --git a/src/inc/tsdb.h b/src/inc/tsdb.h index 65c8a45d00..130628e799 100644 --- a/src/inc/tsdb.h +++ b/src/inc/tsdb.h @@ -274,7 +274,7 @@ TsdbQueryHandleT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STable TsdbQueryHandleT tsdbQueryCacheLast(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, SMemRef* pMemRef); -bool isTsdbCacheLastRow(TsdbQueryHandleT* pQueryHandle); +bool isTsdbCacheLastRow(TsdbQueryHandleT* pTsdbReadHandle); /** @@ -308,19 +308,19 @@ int64_t tsdbGetNumOfRowsInMemTable(TsdbQueryHandleT* pHandle); /** * move to next block if exists * - * @param pQueryHandle + * @param pTsdbReadHandle * @return */ -bool tsdbNextDataBlock(TsdbQueryHandleT pQueryHandle); +bool tsdbNextDataBlock(TsdbQueryHandleT pTsdbReadHandle); /** * Get current data block information * - * @param pQueryHandle + * @param pTsdbReadHandle * @param pBlockInfo * @return */ -void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT *pQueryHandle, SDataBlockInfo *pBlockInfo); +void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT *pTsdbReadHandle, SDataBlockInfo *pBlockInfo); /** * @@ -332,7 +332,7 @@ void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT *pQueryHandle, SDataBlockInfo *p * @pBlockStatis the pre-calculated value for current data blocks. if the block is a cache block, always return 0 * @return */ -int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT *pQueryHandle, SDataStatis **pBlockStatis); +int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT *pTsdbReadHandle, SDataStatis **pBlockStatis); /** * @@ -340,11 +340,11 @@ int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT *pQueryHandle, SDataSta * the returned data block must be satisfied with the time window condition in any cases, * which means the SData data block is not actually the completed disk data blocks. * - * @param pQueryHandle query handle + * @param pTsdbReadHandle query handle * @param pColumnIdList required data columns id list * @return */ -SArray *tsdbRetrieveDataBlock(TsdbQueryHandleT *pQueryHandle, SArray *pColumnIdList); +SArray *tsdbRetrieveDataBlock(TsdbQueryHandleT *pTsdbReadHandle, SArray *pColumnIdList); /** * Get the qualified table id for a super table according to the tag query expression. diff --git a/src/query/inc/qExecutor.h b/src/query/inc/qExecutor.h index 19ca8e7ed8..0c0e3363c8 100644 --- a/src/query/inc/qExecutor.h +++ b/src/query/inc/qExecutor.h @@ -284,7 +284,7 @@ typedef struct SQueryRuntimeEnv { uint32_t status; // query status void* qinfo; uint8_t scanFlag; // denotes reversed scan of data or not - void* pQueryHandle; + void* pTsdbReadHandle; int32_t prevGroupId; // previous executed group id bool enableGroupData; @@ -418,7 +418,7 @@ typedef struct SQueryParam { } SQueryParam; typedef struct STableScanInfo { - void *pQueryHandle; + void *pTsdbReadHandle; int32_t numOfBlocks; int32_t numOfSkipped; int32_t numOfBlockStatis; diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index be1bfb8143..a6a9115b2f 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -2326,8 +2326,8 @@ _clean: static void doFreeQueryHandle(SQueryRuntimeEnv* pRuntimeEnv) { SQueryAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; - tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle); - pRuntimeEnv->pQueryHandle = NULL; + tsdbCleanupQueryHandle(pRuntimeEnv->pTsdbReadHandle); + pRuntimeEnv->pTsdbReadHandle = NULL; SMemRef* pMemRef = &pQueryAttr->memRef; assert(pMemRef->ref == 0 && pMemRef->snapshot.imem == NULL && pMemRef->snapshot.mem == NULL); @@ -3148,10 +3148,10 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTa } else if ((*status) == BLK_DATA_STATIS_NEEDED) { // this function never returns error? pCost->loadBlockStatis += 1; - tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pQueryHandle, &pBlock->pBlockStatis); + tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pTsdbReadHandle, &pBlock->pBlockStatis); if (pBlock->pBlockStatis == NULL) { // data block statistics does not exist, load data block - pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pQueryHandle, NULL); + pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pTsdbReadHandle, NULL); pCost->totalCheckedRows += pBlock->info.rows; } } else { @@ -3159,7 +3159,7 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTa // load the data block statistics to perform further filter pCost->loadBlockStatis += 1; - tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pQueryHandle, &pBlock->pBlockStatis); + tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pTsdbReadHandle, &pBlock->pBlockStatis); if (pQueryAttr->topBotQuery && pBlock->pBlockStatis != NULL) { { // set previous window @@ -3205,7 +3205,7 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv* pRuntimeEnv, STableScanInfo* pTa pCost->totalCheckedRows += pBlockInfo->rows; pCost->loadBlocks += 1; - pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pQueryHandle, NULL); + pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pTsdbReadHandle, NULL); if (pBlock->pDataBlock == NULL) { return terrno; } @@ -4494,7 +4494,7 @@ void queryCostStatis(SQInfo *pQInfo) { // // assert(pQueryAttr->pos >= 0 && pQueryAttr->pos <= pBlockInfo->rows - 1); // -// SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); +// SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pTsdbReadHandle, NULL); // SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); // // // update the pQueryAttr->limit.offset value, and pQueryAttr->pos value @@ -4521,15 +4521,15 @@ void queryCostStatis(SQInfo *pQInfo) { // int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); // // STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; -// TsdbQueryHandleT pQueryHandle = pRuntimeEnv->pQueryHandle; +// TsdbQueryHandleT pTsdbReadHandle = pRuntimeEnv->pTsdbReadHandle; // // SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; -// while (tsdbNextDataBlock(pQueryHandle)) { +// while (tsdbNextDataBlock(pTsdbReadHandle)) { // if (isQueryKilled(pRuntimeEnv->qinfo)) { // longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); // } // -// tsdbRetrieveDataBlockInfo(pQueryHandle, &blockInfo); +// tsdbRetrieveDataBlockInfo(pTsdbReadHandle, &blockInfo); // // if (pQueryAttr->limit.offset > blockInfo.rows) { // pQueryAttr->limit.offset -= blockInfo.rows; @@ -4562,7 +4562,7 @@ void queryCostStatis(SQInfo *pQInfo) { // // // load the data block and check data remaining in current data block // // TODO optimize performance -// SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); +// SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pTsdbReadHandle, NULL); // SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); // // tw = *win; @@ -4627,8 +4627,8 @@ void queryCostStatis(SQInfo *pQInfo) { // STableQueryInfo *pTableQueryInfo = pRuntimeEnv->current; // // SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; -// while (tsdbNextDataBlock(pRuntimeEnv->pQueryHandle)) { -// tsdbRetrieveDataBlockInfo(pRuntimeEnv->pQueryHandle, &blockInfo); +// while (tsdbNextDataBlock(pRuntimeEnv->pTsdbReadHandle)) { +// tsdbRetrieveDataBlockInfo(pRuntimeEnv->pTsdbReadHandle, &blockInfo); // // if (QUERY_IS_ASC_QUERY(pQueryAttr)) { // if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) { @@ -4674,7 +4674,7 @@ void queryCostStatis(SQInfo *pQInfo) { // */ // if ((tw.skey <= blockInfo.window.ekey && ascQuery) || (tw.ekey >= blockInfo.window.skey && !ascQuery)) { // -// SArray *pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); +// SArray *pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pTsdbReadHandle, NULL); // SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); // // if ((win.ekey > blockInfo.window.ekey && ascQuery) || (win.ekey < blockInfo.window.skey && !ascQuery)) { @@ -4748,7 +4748,7 @@ static int32_t setupQueryHandle(void* tsdb, SQueryRuntimeEnv* pRuntimeEnv, int64 terrno = TSDB_CODE_SUCCESS; if (isFirstLastRowQuery(pQueryAttr)) { - pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); + pRuntimeEnv->pTsdbReadHandle = tsdbQueryLastRow(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); // update the query time window pQueryAttr->window = cond.twindow; @@ -4769,11 +4769,11 @@ static int32_t setupQueryHandle(void* tsdb, SQueryRuntimeEnv* pRuntimeEnv, int64 } } } else if (isCachedLastQuery(pQueryAttr)) { - pRuntimeEnv->pQueryHandle = tsdbQueryCacheLast(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); + pRuntimeEnv->pTsdbReadHandle = tsdbQueryCacheLast(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); } else if (pQueryAttr->pointInterpQuery) { - pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); + pRuntimeEnv->pTsdbReadHandle = tsdbQueryRowsInExternalWindow(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); } else { - pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); + pRuntimeEnv->pTsdbReadHandle = tsdbQueryTables(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); } return terrno; @@ -4831,19 +4831,19 @@ int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr switch(tbScanner) { case OP_TableBlockInfoScan: { - pRuntimeEnv->proot = createTableBlockInfoScanOperator(pRuntimeEnv->pQueryHandle, pRuntimeEnv); + pRuntimeEnv->proot = createTableBlockInfoScanOperator(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv); break; } case OP_TableSeqScan: { - pRuntimeEnv->proot = createTableSeqScanOperator(pRuntimeEnv->pQueryHandle, pRuntimeEnv); + pRuntimeEnv->proot = createTableSeqScanOperator(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv); break; } case OP_DataBlocksOptScan: { - pRuntimeEnv->proot = createDataBlocksOptScanInfo(pRuntimeEnv->pQueryHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr), pQueryAttr->needReverseScan? 1:0); + pRuntimeEnv->proot = createDataBlocksOptScanInfo(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr), pQueryAttr->needReverseScan? 1:0); break; } case OP_TableScan: { - pRuntimeEnv->proot = createTableScanOperator(pRuntimeEnv->pQueryHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr)); + pRuntimeEnv->proot = createTableScanOperator(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr)); break; } default: { // do nothing @@ -4974,13 +4974,13 @@ static SSDataBlock* doTableScanImpl(void* param, bool* newgroup) { *newgroup = false; - while (tsdbNextDataBlock(pTableScanInfo->pQueryHandle)) { + while (tsdbNextDataBlock(pTableScanInfo->pTsdbReadHandle)) { if (isQueryKilled(pOperator->pRuntimeEnv->qinfo)) { longjmp(pOperator->pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); } pTableScanInfo->numOfBlocks += 1; - tsdbRetrieveDataBlockInfo(pTableScanInfo->pQueryHandle, &pBlock->info); + tsdbRetrieveDataBlockInfo(pTableScanInfo->pTsdbReadHandle, &pBlock->info); // todo opt if (pTableGroupInfo->numOfTables > 1 || (pRuntimeEnv->current == NULL && pTableGroupInfo->numOfTables == 1)) { @@ -5037,7 +5037,7 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { } if (++pTableScanInfo->current >= pTableScanInfo->times) { - if (pTableScanInfo->reverseTimes <= 0 || isTsdbCacheLastRow(pTableScanInfo->pQueryHandle)) { + if (pTableScanInfo->reverseTimes <= 0 || isTsdbCacheLastRow(pTableScanInfo->pTsdbReadHandle)) { return NULL; } else { break; @@ -5046,7 +5046,7 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { // do prepare for the next round table scan operation STsdbQueryCond cond = createTsdbQueryCond(pQueryAttr, &pQueryAttr->window); - tsdbResetQueryHandle(pTableScanInfo->pQueryHandle, &cond); + tsdbResetQueryHandle(pTableScanInfo->pTsdbReadHandle, &cond); setQueryStatus(pRuntimeEnv, QUERY_NOT_COMPLETED); pRuntimeEnv->scanFlag = REPEAT_SCAN; @@ -5069,7 +5069,7 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { setupEnvForReverseScan(pRuntimeEnv, pTableScanInfo->pResultRowInfo, pTableScanInfo->pCtx, pTableScanInfo->numOfOutput); STsdbQueryCond cond = createTsdbQueryCond(pQueryAttr, &pQueryAttr->window); - tsdbResetQueryHandle(pTableScanInfo->pQueryHandle, &cond); + tsdbResetQueryHandle(pTableScanInfo->pTsdbReadHandle, &cond); qDebug("QInfo:0x%"PRIx64" start to reverse scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64, GET_QID(pRuntimeEnv), cond.twindow.skey, cond.twindow.ekey); @@ -5112,8 +5112,8 @@ static SSDataBlock* doBlockInfoScan(void* param, bool* newgroup) { tableBlockDist.maxRows = INT_MIN; tableBlockDist.minRows = INT_MAX; - tsdbGetFileBlocksDistInfo(pTableScanInfo->pQueryHandle, &tableBlockDist); - tableBlockDist.numOfRowsInMemTable = (int32_t) tsdbGetNumOfRowsInMemTable(pTableScanInfo->pQueryHandle); + tsdbGetFileBlocksDistInfo(pTableScanInfo->pTsdbReadHandle, &tableBlockDist); + tableBlockDist.numOfRowsInMemTable = (int32_t) tsdbGetNumOfRowsInMemTable(pTableScanInfo->pTsdbReadHandle); SSDataBlock* pBlock = &pTableScanInfo->block; pBlock->info.rows = 1; @@ -5142,7 +5142,7 @@ SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* assert(repeatTime > 0); STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - pInfo->pQueryHandle = pTsdbQueryHandle; + pInfo->pTsdbReadHandle = pTsdbQueryHandle; pInfo->times = repeatTime; pInfo->reverseTimes = 0; pInfo->order = pRuntimeEnv->pQueryAttr->order.order; @@ -5165,7 +5165,7 @@ SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv) { STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - pInfo->pQueryHandle = pTsdbQueryHandle; + pInfo->pTsdbReadHandle = pTsdbQueryHandle; pInfo->times = 1; pInfo->reverseTimes = 0; pInfo->order = pRuntimeEnv->pQueryAttr->order.order; @@ -5189,7 +5189,7 @@ SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, SQueryRuntimeE SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbQueryHandle, SQueryRuntimeEnv* pRuntimeEnv) { STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - pInfo->pQueryHandle = pTsdbQueryHandle; + pInfo->pTsdbReadHandle = pTsdbQueryHandle; pInfo->block.pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); SColumnInfoData infoData = {{0}}; @@ -5271,7 +5271,7 @@ SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, SQueryRuntime assert(repeatTime > 0); STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - pInfo->pQueryHandle = pTsdbQueryHandle; + pInfo->pTsdbReadHandle = pTsdbQueryHandle; pInfo->times = repeatTime; pInfo->reverseTimes = reverseTime; pInfo->current = 0; From 553cc16cbd32de3b33efbe4bb5edcf959dfe8573 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 10 Jan 2022 18:53:25 -0800 Subject: [PATCH 09/57] rename --- tests/script/{general/db/basic.sim => sim/db/basic6.sim} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/script/{general/db/basic.sim => sim/db/basic6.sim} (100%) diff --git a/tests/script/general/db/basic.sim b/tests/script/sim/db/basic6.sim similarity index 100% rename from tests/script/general/db/basic.sim rename to tests/script/sim/db/basic6.sim From 25a48e400643a35d656b389da7098a624221933c Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 11 Jan 2022 00:30:11 -0500 Subject: [PATCH 10/57] TD-12678 datasink interface adjust --- include/libs/executor/dataSinkMgt.h | 4 ++-- source/libs/executor/src/dataDispatcher.c | 7 ++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/include/libs/executor/dataSinkMgt.h b/include/libs/executor/dataSinkMgt.h index 6470f38ced..b44cad1668 100644 --- a/include/libs/executor/dataSinkMgt.h +++ b/include/libs/executor/dataSinkMgt.h @@ -24,9 +24,9 @@ extern "C" { #include "executorimpl.h" #define DS_CAPACITY_ENOUGH 1 -#define DS_CAPACITY_FULL 2 +#define DS_DATA_FULL 2 #define DS_NEED_SCHEDULE 3 -#define DS_END 4 +#define DS_QUERY_END 4 #define DS_IN_PROCESS 5 struct SDataSink; diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index 3d8e51d04d..83f180cd1c 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -124,7 +124,8 @@ static bool allocBuf(SDataDispatchHandle* pDispatcher, const SInputData* pInput, static int32_t updateStatus(SDataDispatchHandle* pDispatcher) { pthread_mutex_lock(&pDispatcher->mutex); - int32_t status = taosQueueSize(pDispatcher->pDataBlocks) < pDispatcher->pManager->cfg.maxDataBlockNumPerQuery ? DS_CAPACITY_ENOUGH : DS_CAPACITY_FULL; + int32_t status = taosQueueSize(pDispatcher->pDataBlocks) < pDispatcher->pManager->cfg.maxDataBlockNumPerQuery + ? DS_CAPACITY_ENOUGH : DS_DATA_FULL; pDispatcher->status = status; pthread_mutex_unlock(&pDispatcher->mutex); return status; @@ -152,14 +153,14 @@ static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, static void endPut(struct SDataSinkHandle* pHandle) { SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; pthread_mutex_lock(&pDispatcher->mutex); - pDispatcher->status = DS_END; + pDispatcher->status = DS_QUERY_END; pthread_mutex_unlock(&pDispatcher->mutex); } static int32_t getDataLength(SDataSinkHandle* pHandle, int32_t* pStatus) { SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; if (taosQueueEmpty(pDispatcher->pDataBlocks)) { - *pStatus = getStatus(pDispatcher) ? DS_END : DS_IN_PROCESS; + *pStatus = (DS_QUERY_END == getStatus(pDispatcher) ? DS_QUERY_END : DS_IN_PROCESS); return 0; } SDataDispatchBuf* pBuf = NULL; From 3d1c8dcb7d1cc872c9e54a63b0f058719e21263f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 11 Jan 2022 06:55:44 +0000 Subject: [PATCH 11/57] more --- source/libs/tkv/inc/tkvDef.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/source/libs/tkv/inc/tkvDef.h b/source/libs/tkv/inc/tkvDef.h index a80c395364..cd418019be 100644 --- a/source/libs/tkv/inc/tkvDef.h +++ b/source/libs/tkv/inc/tkvDef.h @@ -29,6 +29,12 @@ typedef int32_t pgid_t; // framd_id_t typedef int32_t frame_id_t; +// pgsize_t +typedef int32_t pgsize_t; +#define TKV_MIN_PGSIZE 512 +#define TKV_MAX_PGSIZE 16384 +#define TKV_IS_PGSIZE_VLD(s) (((s) >= TKV_MIN_PGSIZE) && (TKV_MAX_PGSIZE <= TKV_MAX_PGSIZE)) + #ifdef __cplusplus } #endif From e456c6391aac090f143fabe0eeaf5cd6e304024c Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Tue, 11 Jan 2022 07:12:38 +0000 Subject: [PATCH 12/57] refact --- source/libs/tkv/CMakeLists.txt | 9 +++++++-- source/libs/tkv/inc/tkv.h | 13 ++++++++++--- source/libs/tkv/{ => src}/inc/tkvBufPool.h | 0 source/libs/tkv/{ => src}/inc/tkvDef.h | 0 source/libs/tkv/{ => src}/inc/tkvDiskMgr.h | 0 source/libs/tkv/{ => src}/inc/tkvPage.h | 0 6 files changed, 17 insertions(+), 5 deletions(-) rename source/libs/tkv/{ => src}/inc/tkvBufPool.h (100%) rename source/libs/tkv/{ => src}/inc/tkvDef.h (100%) rename source/libs/tkv/{ => src}/inc/tkvDiskMgr.h (100%) rename source/libs/tkv/{ => src}/inc/tkvPage.h (100%) diff --git a/source/libs/tkv/CMakeLists.txt b/source/libs/tkv/CMakeLists.txt index ec3259d1f2..fec3f37cd5 100644 --- a/source/libs/tkv/CMakeLists.txt +++ b/source/libs/tkv/CMakeLists.txt @@ -1,9 +1,14 @@ aux_source_directory(src TKV_SRC) add_library(tkv STATIC ${TKV_SRC}) +# target_include_directories( +# tkv +# PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/tkv" +# PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +# ) target_include_directories( tkv - PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/tkv" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" + PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/inc" + PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/src/inc" ) target_link_libraries( tkv diff --git a/source/libs/tkv/inc/tkv.h b/source/libs/tkv/inc/tkv.h index 3a08906033..00534d2827 100644 --- a/source/libs/tkv/inc/tkv.h +++ b/source/libs/tkv/inc/tkv.h @@ -16,14 +16,21 @@ #ifndef _TD_TKV_H_ #define _TD_TKV_H_ +#include "os.h" + #ifdef __cplusplus extern "C" { #endif +// Forward declaration +typedef struct TDB TDB; +typedef struct TDB_ENV TDB_ENV; + // SKey -typedef struct SKey { - void *bdata; -} SKey, SValue; +typedef struct { + void * bdata; + uint32_t size; +} TDB_KEY, TDB_VALUE; #ifdef __cplusplus } diff --git a/source/libs/tkv/inc/tkvBufPool.h b/source/libs/tkv/src/inc/tkvBufPool.h similarity index 100% rename from source/libs/tkv/inc/tkvBufPool.h rename to source/libs/tkv/src/inc/tkvBufPool.h diff --git a/source/libs/tkv/inc/tkvDef.h b/source/libs/tkv/src/inc/tkvDef.h similarity index 100% rename from source/libs/tkv/inc/tkvDef.h rename to source/libs/tkv/src/inc/tkvDef.h diff --git a/source/libs/tkv/inc/tkvDiskMgr.h b/source/libs/tkv/src/inc/tkvDiskMgr.h similarity index 100% rename from source/libs/tkv/inc/tkvDiskMgr.h rename to source/libs/tkv/src/inc/tkvDiskMgr.h diff --git a/source/libs/tkv/inc/tkvPage.h b/source/libs/tkv/src/inc/tkvPage.h similarity index 100% rename from source/libs/tkv/inc/tkvPage.h rename to source/libs/tkv/src/inc/tkvPage.h From 3e40fa34277f2e609206c5870cb112bab962decb Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 11 Jan 2022 15:42:27 +0800 Subject: [PATCH 13/57] [td-11818] fix error in fetch data. --- include/libs/planner/planner.h | 2 +- include/libs/scheduler/scheduler.h | 10 ++-- source/client/inc/clientInt.h | 6 ++- source/client/src/clientImpl.c | 58 +++++++++++++++-------- source/client/test/clientTests.cpp | 2 +- source/libs/planner/src/physicalPlan.c | 4 +- source/libs/planner/src/planner.c | 11 +---- source/libs/planner/test/phyPlanTests.cpp | 2 +- source/libs/scheduler/inc/schedulerInt.h | 1 - source/libs/scheduler/src/scheduler.c | 50 ++++++++++--------- 10 files changed, 76 insertions(+), 70 deletions(-) diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index 91421a9174..da70f21498 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -150,7 +150,7 @@ struct SQueryNode; * @param requestId * @return */ -int32_t qCreateQueryDag(const struct SQueryNode* pQueryInfo, struct SQueryDag** pDag, SSchema** pSchema, uint32_t* numOfResCols, uint64_t requestId); +int32_t qCreateQueryDag(const struct SQueryNode* pQueryInfo, struct SQueryDag** pDag, uint64_t requestId); // Set datasource of this subplan, multiple calls may be made to a subplan. // @subplan subplan to be schedule diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index 74b7813465..ceb54eabef 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -20,8 +20,10 @@ extern "C" { #endif -#include "planner.h" #include "catalog.h" +#include "planner.h" + +struct SSchJob; typedef struct SSchedulerCfg { uint32_t maxJobNum; @@ -65,7 +67,7 @@ int32_t schedulerInit(SSchedulerCfg *cfg); * @param nodeList Qnode/Vnode address list, element is SQueryNodeAddr * @return */ -int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, void** pJob, SQueryResult *pRes); +int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob, SQueryResult *pRes); /** * Process the query job, generated according to the query physical plan. @@ -73,7 +75,7 @@ int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, void * @param nodeList Qnode/Vnode address list, element is SQueryNodeAddr * @return */ -int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, void** pJob); +int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob); /** * Fetch query result from the remote query executor @@ -81,7 +83,7 @@ int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, * @param data * @return */ -int32_t scheduleFetchRows(void *pJob, void **data); +int32_t scheduleFetchRows(struct SSchJob *pJob, void **data); /** diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 26afe237c9..7292375e7f 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -113,6 +113,7 @@ typedef struct SRequestSendRecvBody { tsem_t rspSem; // not used now void* fp; SShowReqInfo showInfo; // todo this attribute will be removed after the query framework being completed. + struct SSchJob *pQueryJob; // query job, created according to sql query DAG. SDataBuf requestMsg; SReqResultInfo resInfo; } SRequestSendRecvBody; @@ -129,7 +130,7 @@ typedef struct SRequestObj { char *msgBuf; void *pInfo; // sql parse info, generated by parser module int32_t code; - uint64_t affectedRows; + uint64_t affectedRows; // todo remove it SQueryExecMetric metric; SRequestSendRecvBody body; } SRequestObj; @@ -161,12 +162,13 @@ int taos_options_imp(TSDB_OPTION option, const char *str); void* openTransporter(const char *user, const char *auth, int32_t numOfThreads); void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet); + void initMsgHandleFp(); TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port); -TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen); void *doFetchRow(SRequestObj* pRequest); + void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows); #ifdef __cplusplus diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 26f97cc6ef..14cb35355f 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1,6 +1,10 @@ +#include "../../libs/scheduler/inc/schedulerInt.h" #include "clientInt.h" #include "clientLog.h" +#include "parser.h" +#include "planner.h" +#include "scheduler.h" #include "tdef.h" #include "tep.h" #include "tglobal.h" @@ -8,9 +12,6 @@ #include "tnote.h" #include "tpagedfile.h" #include "tref.h" -#include "parser.h" -#include "planner.h" -#include "scheduler.h" #define CHECK_CODE_GOTO(expr, lable) \ do { \ @@ -57,6 +58,7 @@ static char* getClusterKey(const char* user, const char* auth, const char* ip, i } static STscObj* taosConnectImpl(const char *ip, const char *user, const char *auth, const char *db, uint16_t port, __taos_async_fn_t fp, void *param, SAppInstInfo* pAppInfo); +static void setResSchemaInfo(SReqResultInfo* pResInfo, const SDataBlockSchema* pDataBlockSchema); TAOS *taos_connect_internal(const char *ip, const char *user, const char *pass, const char *auth, const char *db, uint16_t port) { if (taos_init() != TSDB_CODE_SUCCESS) { @@ -198,36 +200,48 @@ int32_t execDdlQuery(SRequestObj* pRequest, SQueryNode* pQuery) { int32_t getPlan(SRequestObj* pRequest, SQueryNode* pQueryNode, SQueryDag** pDag) { pRequest->type = pQueryNode->type; - SSchema *pSchema = NULL; SReqResultInfo* pResInfo = &pRequest->body.resInfo; - - int32_t code = qCreateQueryDag(pQueryNode, pDag, &pSchema, &pResInfo->numOfCols, pRequest->requestId); + int32_t code = qCreateQueryDag(pQueryNode, pDag, pRequest->requestId); if (code != 0) { return code; } if (pQueryNode->type == TSDB_SQL_SELECT) { - pResInfo->fields = calloc(1, sizeof(TAOS_FIELD)); - for (int32_t i = 0; i < pResInfo->numOfCols; ++i) { - pResInfo->fields[i].bytes = pSchema[i].bytes; - pResInfo->fields[i].type = pSchema[i].type; - tstrncpy(pResInfo->fields[i].name, pSchema[i].name, tListLen(pResInfo->fields[i].name)); - } + SArray* pa = taosArrayGetP((*pDag)->pSubplans, 0); + + SSubplan* pPlan = taosArrayGetP(pa, 0); + SDataBlockSchema* pDataBlockSchema = &(pPlan->pDataSink->schema); + setResSchemaInfo(pResInfo, pDataBlockSchema); + pRequest->type = TDMT_VND_QUERY; } return code; } -int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, void** pJob) { +void setResSchemaInfo(SReqResultInfo* pResInfo, const SDataBlockSchema* pDataBlockSchema) { + assert(pDataBlockSchema != NULL && pDataBlockSchema->numOfCols > 0); + + pResInfo->numOfCols = pDataBlockSchema->numOfCols; + pResInfo->fields = calloc(pDataBlockSchema->numOfCols, sizeof(pDataBlockSchema->pSchema[0])); + + for (int32_t i = 0; i < pResInfo->numOfCols; ++i) { + SSchema* pSchema = &pDataBlockSchema->pSchema[i]; + pResInfo->fields[i].bytes = pSchema->bytes; + pResInfo->fields[i].type = pSchema->type; + tstrncpy(pResInfo->fields[i].name, pSchema[i].name, tListLen(pResInfo->fields[i].name)); + } +} + +int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag) { if (TSDB_SQL_INSERT == pRequest->type || TSDB_SQL_CREATE_TABLE == pRequest->type) { SQueryResult res = {.code = 0, .numOfRows = 0, .msgSize = ERROR_MSG_BUF_DEFAULT_SIZE, .msg = pRequest->msgBuf}; - int32_t code = scheduleExecJob(pRequest->pTscObj->pTransporter, NULL, pDag, pJob, &res); + int32_t code = scheduleExecJob(pRequest->pTscObj->pTransporter, NULL, pDag, &pRequest->body.pQueryJob, &res); if (code != TSDB_CODE_SUCCESS) { // handle error and retry } else { - if (*pJob != NULL) { - scheduleFreeJob(*pJob); + if (pRequest->body.pQueryJob != NULL) { + scheduleFreeJob(pRequest->body.pQueryJob); } } @@ -235,7 +249,7 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, void** pJob) { return res.code; } - return scheduleAsyncExecJob(pRequest->pTscObj->pTransporter, NULL /*todo appInfo.xxx*/, pDag, pJob); + return scheduleAsyncExecJob(pRequest->pTscObj->pTransporter, NULL, pDag, &pRequest->body.pQueryJob); } TAOS_RES *tmq_create_topic(TAOS* taos, const char* name, const char* sql, int sqlLen) { @@ -312,7 +326,6 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) { SRequestObj *pRequest = NULL; SQueryNode *pQuery = NULL; SQueryDag *pDag = NULL; - void *pJob = NULL; terrno = TSDB_CODE_SUCCESS; CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return); @@ -322,9 +335,8 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) { CHECK_CODE_GOTO(execDdlQuery(pRequest, pQuery), _return); } else { CHECK_CODE_GOTO(getPlan(pRequest, pQuery, &pDag), _return); - CHECK_CODE_GOTO(scheduleQuery(pRequest, pDag, &pJob), _return); + CHECK_CODE_GOTO(scheduleQuery(pRequest, pDag), _return); pRequest->code = terrno; - return pRequest; } _return: @@ -333,6 +345,7 @@ _return: if (NULL != pRequest && TSDB_CODE_SUCCESS != terrno) { pRequest->code = terrno; } + return pRequest; } @@ -531,7 +544,10 @@ void* doFetchRow(SRequestObj* pRequest) { SReqResultInfo* pResultInfo = &pRequest->body.resInfo; if (pResultInfo->pData == NULL || pResultInfo->current >= pResultInfo->numOfRows) { - if (pRequest->type == TDMT_MND_SHOW) { + if (pRequest->type == TDMT_VND_QUERY) { + pRequest->type = TDMT_VND_FETCH; + scheduleFetchRows(pRequest->body.pQueryJob, &pRequest->body.resInfo.pData); + } else if (pRequest->type == TDMT_MND_SHOW) { pRequest->type = TDMT_MND_SHOW_RETRIEVE; } else if (pRequest->type == TDMT_VND_SHOW_TABLES) { pRequest->type = TDMT_VND_SHOW_TABLES_FETCH; diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index af5ad874f8..d507c565df 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -568,7 +568,7 @@ TEST(testCase, projection_query_tables) { TAOS_RES* pRes = taos_query(pConn, "use abc1"); // pRes = taos_query(pConn, "create table m1 (ts timestamp, k int) tags(a int)"); -// taos_free_result(pRes); + taos_free_result(pRes); // // pRes = taos_query(pConn, "create table tu using m1 tags(1)"); // taos_free_result(pRes); diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index 7ef434ccf5..01fb3c2513 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -89,7 +89,7 @@ static bool copySchema(SDataBlockSchema* dst, const SDataBlockSchema* src) { static bool toDataBlockSchema(SQueryPlanNode* pPlanNode, SDataBlockSchema* dataBlockSchema) { dataBlockSchema->numOfCols = pPlanNode->numOfExpr; - dataBlockSchema->pSchema = malloc(sizeof(SSlotSchema) * pPlanNode->numOfCols); + dataBlockSchema->pSchema = malloc(sizeof(SSlotSchema) * pPlanNode->numOfExpr); if (NULL == dataBlockSchema->pSchema) { return false; } @@ -306,8 +306,6 @@ static SPhyNode* createPhyNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { case QNODE_TABLESCAN: node = createTableScanNode(pCxt, pPlanNode); break; - case QNODE_PROJECT: -// node = create case QNODE_MODIFY: // Insert is not an operator in a physical plan. break; diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index de3454ddbb..3047ef4f5a 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -56,7 +56,7 @@ void qDestroyQueryDag(struct SQueryDag* pDag) { tfree(pDag); } -int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, SSchema** pSchema, uint32_t* numOfResCols, uint64_t requestId) { +int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, uint64_t requestId) { SQueryPlanNode* pLogicPlan; int32_t code = createQueryPlan(pNode, &pLogicPlan); if (TSDB_CODE_SUCCESS != code) { @@ -70,15 +70,6 @@ int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, printf("%s\n", str); } - int32_t numOfOutput = pLogicPlan->numOfExpr; - *pSchema = calloc(numOfOutput, sizeof(SSchema)); - *numOfResCols = numOfOutput; - - for(int32_t i = 0; i < numOfOutput; ++i) { - SExprInfo* pExprInfo = taosArrayGetP(pLogicPlan->pExpr, i); - memcpy(&(*pSchema)[i], pExprInfo->pExpr->pSchema, sizeof(SSchema)); - } - code = optimizeQueryPlan(pLogicPlan); if (TSDB_CODE_SUCCESS != code) { destroyQueryPlan(pLogicPlan); diff --git a/source/libs/planner/test/phyPlanTests.cpp b/source/libs/planner/test/phyPlanTests.cpp index 0d3820cc77..2733a73a3f 100644 --- a/source/libs/planner/test/phyPlanTests.cpp +++ b/source/libs/planner/test/phyPlanTests.cpp @@ -65,7 +65,7 @@ protected: SSchema *schema = NULL; uint32_t numOfOutput = 0; - code = qCreateQueryDag(query, &dag, &schema, &numOfOutput, requestId); + code = qCreateQueryDag(query, &dag, requestId); dag_.reset(dag); return code; } diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index ada9b247ce..b2115b5c6d 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -58,7 +58,6 @@ typedef struct SSchLevel { SArray *subTasks; // Element is SQueryTask } SSchLevel; - typedef struct SSchTask { uint64_t taskId; // task id SSchLevel *level; // level diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index f9406a04b6..2ce555525f 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -606,8 +606,8 @@ int32_t schProcessRspMsg(SSchJob *job, SSchTask *task, int32_t msgType, char *ms case TDMT_VND_QUERY_RSP: { SQueryTableRsp *rsp = (SQueryTableRsp *)msg; - if (rsp->code != TSDB_CODE_SUCCESS || NULL == msg) { - SCH_ERR_JRET(schProcessOnTaskFailure(job, task, rsp->code)); + if (rspCode != TSDB_CODE_SUCCESS || rsp->code != TSDB_CODE_SUCCESS || NULL == msg) { + SCH_ERR_JRET(schProcessOnTaskFailure(job, task, rspCode)); } else { code = schBuildAndSendMsg(job, task, TDMT_VND_RES_READY); if (code) { @@ -986,7 +986,7 @@ void schDropJobAllTasks(SSchJob *job) { } } -int32_t schExecJobImpl(void *transport, SArray *nodeList, SQueryDag* pDag, void** job, bool syncSchedule) { +int32_t schExecJobImpl(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** job, bool syncSchedule) { if (nodeList && taosArrayGetSize(nodeList) <= 0) { qInfo("QID:%"PRIx64" input nodeList is empty", pDag->queryId); } @@ -1092,7 +1092,7 @@ int32_t schedulerInit(SSchedulerCfg *cfg) { return TSDB_CODE_SUCCESS; } -int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, void** pJob, SQueryResult *pRes) { +int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob, SQueryResult *pRes) { if (NULL == transport || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob || NULL == pRes) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } @@ -1107,7 +1107,7 @@ int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, void return TSDB_CODE_SUCCESS; } -int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, void** pJob) { +int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, struct SSchJob** pJob) { if (NULL == transport || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } @@ -1116,53 +1116,51 @@ int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, } -int32_t scheduleFetchRows(void *pJob, void **data) { - if (NULL == pJob || NULL == data) { +int32_t scheduleFetchRows(SSchJob *pJob, void** pData) { + if (NULL == pJob || NULL == pData) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - SSchJob *job = pJob; int32_t code = 0; - - if (!SCH_JOB_NEED_FETCH(&job->attr)) { + if (!SCH_JOB_NEED_FETCH(&pJob->attr)) { qError("no need to fetch data"); SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - if (job->status == JOB_TASK_STATUS_FAILED) { - job->res = NULL; - SCH_RET(atomic_load_32(&job->errCode)); + if (pJob->status == JOB_TASK_STATUS_FAILED) { + pJob->res = NULL; + SCH_RET(atomic_load_32(&pJob->errCode)); } - if (job->status == JOB_TASK_STATUS_SUCCEED) { - job->res = NULL; + if (pJob->status == JOB_TASK_STATUS_SUCCEED) { + pJob->res = NULL; return TSDB_CODE_SUCCESS; } - if (atomic_val_compare_exchange_32(&job->userFetch, 0, 1) != 0) { + if (atomic_val_compare_exchange_32(&pJob->userFetch, 0, 1) != 0) { qError("prior fetching not finished"); SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - if (job->status == JOB_TASK_STATUS_PARTIAL_SUCCEED) { - SCH_ERR_JRET(schFetchFromRemote(job)); + if (pJob->status == JOB_TASK_STATUS_PARTIAL_SUCCEED) { + SCH_ERR_JRET(schFetchFromRemote(pJob)); } - tsem_wait(&job->rspSem); + tsem_wait(&pJob->rspSem); - if (job->status == JOB_TASK_STATUS_FAILED) { - code = atomic_load_32(&job->errCode); + if (pJob->status == JOB_TASK_STATUS_FAILED) { + code = atomic_load_32(&pJob->errCode); } - if (job->res && ((SRetrieveTableRsp *)job->res)->completed) { - job->status = JOB_TASK_STATUS_SUCCEED; + if (pJob->res && ((SRetrieveTableRsp *)pJob->res)->completed) { + pJob->status = JOB_TASK_STATUS_SUCCEED; } - *data = job->res; - job->res = NULL; + *pData = pJob->res; + pJob->res = NULL; _return: - atomic_val_compare_exchange_32(&job->userFetch, 1, 0); + atomic_val_compare_exchange_32(&pJob->userFetch, 1, 0); SCH_RET(code); } From e3ee88244e8fd08a0f90bf499613112ff425d3f3 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Mon, 10 Jan 2022 23:42:25 -0800 Subject: [PATCH 14/57] TD-12868 --- source/common/src/tglobal.c | 2 +- tests/script/sh/deploy.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 9a20fadbfb..463b107aa7 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -37,7 +37,7 @@ uint16_t tsServerPort = 6030; int32_t tsStatusInterval = 1; // second int8_t tsEnableTelemetryReporting = 0; char tsEmail[TSDB_FQDN_LEN] = {0}; -int32_t tsNumOfSupportVnodes = 16; +int32_t tsNumOfSupportVnodes = 128; // common int32_t tsRpcTimer = 300; diff --git a/tests/script/sh/deploy.sh b/tests/script/sh/deploy.sh index fcc11ca213..e00363b28f 100755 --- a/tests/script/sh/deploy.sh +++ b/tests/script/sh/deploy.sh @@ -120,7 +120,7 @@ echo "firstEp ${HOSTNAME}:7100" >> $TAOS_CFG echo "secondEp ${HOSTNAME}:7200" >> $TAOS_CFG echo "fqdn ${HOSTNAME}" >> $TAOS_CFG echo "serverPort ${NODE}" >> $TAOS_CFG -echo "supportVnodes 16" >> $TAOS_CFG +echo "supportVnodes 128" >> $TAOS_CFG echo "dataDir $DATA_DIR" >> $TAOS_CFG echo "logDir $LOG_DIR" >> $TAOS_CFG echo "debugFlag 0" >> $TAOS_CFG From 8a0da7bad296fbd3756c0de77edcec9c2d06aa02 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 11 Jan 2022 02:56:24 -0500 Subject: [PATCH 15/57] TD-12678 datasink interface adjust --- include/libs/executor/dataSinkMgt.h | 25 ++++++++------- source/libs/executor/inc/dataSinkInt.h | 8 ++--- source/libs/executor/src/dataDispatcher.c | 38 +++++++++++++++-------- source/libs/executor/src/dataSinkMgt.c | 16 +++++----- 4 files changed, 51 insertions(+), 36 deletions(-) diff --git a/include/libs/executor/dataSinkMgt.h b/include/libs/executor/dataSinkMgt.h index b44cad1668..733e6d665d 100644 --- a/include/libs/executor/dataSinkMgt.h +++ b/include/libs/executor/dataSinkMgt.h @@ -23,11 +23,9 @@ extern "C" { #include "os.h" #include "executorimpl.h" -#define DS_CAPACITY_ENOUGH 1 -#define DS_DATA_FULL 2 -#define DS_NEED_SCHEDULE 3 -#define DS_QUERY_END 4 -#define DS_IN_PROCESS 5 +#define DS_BUF_LOW 1 +#define DS_BUF_FULL 2 +#define DS_BUF_EMPTY 3 struct SDataSink; struct SSDataBlock; @@ -46,11 +44,16 @@ typedef struct SInputData { SHashObj* pTableRetrieveTsMap; } SInputData; -typedef struct SOutPutData { +typedef struct SOutputData { int32_t numOfRows; int8_t compressed; char* pData; -} SOutPutData; + bool queryEnd; + bool needSchedule; + int32_t bufStatus; + int64_t useconds; + int8_t precision; +} SOutputData; /** * Create a subplan's datasinker handle for all later operations. @@ -66,16 +69,16 @@ int32_t dsCreateDataSinker(const struct SDataSink *pDataSink, DataSinkHandle* pH * @param pRes * @return error code */ -int32_t dsPutDataBlock(DataSinkHandle handle, const SInputData* pInput, int32_t* pStatus); +int32_t dsPutDataBlock(DataSinkHandle handle, const SInputData* pInput, bool* pContinue); -void dsEndPut(DataSinkHandle handle); +void dsEndPut(DataSinkHandle handle, int64_t useconds); /** * Get the length of the data returned by the next call to dsGetDataBlock. * @param handle * @param pLen data length */ -void dsGetDataLength(DataSinkHandle handle, int32_t* pLen, int32_t* pStatus); +void dsGetDataLength(DataSinkHandle handle, int32_t* pLen, bool* pQueryEnd); /** * Get data, the caller needs to allocate data memory. @@ -84,7 +87,7 @@ void dsGetDataLength(DataSinkHandle handle, int32_t* pLen, int32_t* pStatus); * @param pStatus output * @return error code */ -int32_t dsGetDataBlock(DataSinkHandle handle, SOutPutData* pOutput, int32_t* pStatus); +int32_t dsGetDataBlock(DataSinkHandle handle, SOutputData* pOutput); /** * After dsGetStatus returns DS_NEED_SCHEDULE, the caller need to put this into the work queue. diff --git a/source/libs/executor/inc/dataSinkInt.h b/source/libs/executor/inc/dataSinkInt.h index 1bbf5494dd..69727626af 100644 --- a/source/libs/executor/inc/dataSinkInt.h +++ b/source/libs/executor/inc/dataSinkInt.h @@ -31,10 +31,10 @@ typedef struct SDataSinkManager { pthread_mutex_t mutex; } SDataSinkManager; -typedef int32_t (*FPutDataBlock)(struct SDataSinkHandle* pHandle, const SInputData* pInput, int32_t* pStatus); -typedef void (*FEndPut)(struct SDataSinkHandle* pHandle); -typedef int32_t (*FGetDataLength)(struct SDataSinkHandle* pHandle, int32_t* pStatus); -typedef int32_t (*FGetDataBlock)(struct SDataSinkHandle* pHandle, SOutPutData* pOutput, int32_t* pStatus); +typedef int32_t (*FPutDataBlock)(struct SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue); +typedef void (*FEndPut)(struct SDataSinkHandle* pHandle, int64_t useconds); +typedef void (*FGetDataLength)(struct SDataSinkHandle* pHandle, int32_t* pLen, bool* pQueryEnd); +typedef int32_t (*FGetDataBlock)(struct SDataSinkHandle* pHandle, SOutputData* pOutput); typedef int32_t (*FDestroyDataSinker)(struct SDataSinkHandle* pHandle); typedef struct SDataSinkHandle { diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index 83f180cd1c..954ed6daf1 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -42,6 +42,8 @@ typedef struct SDataDispatchHandle { STaosQueue* pDataBlocks; SDataDispatchBuf nextOutput; int32_t status; + bool queryEnd; + int64_t useconds; pthread_mutex_t mutex; } SDataDispatchHandle; @@ -124,8 +126,9 @@ static bool allocBuf(SDataDispatchHandle* pDispatcher, const SInputData* pInput, static int32_t updateStatus(SDataDispatchHandle* pDispatcher) { pthread_mutex_lock(&pDispatcher->mutex); - int32_t status = taosQueueSize(pDispatcher->pDataBlocks) < pDispatcher->pManager->cfg.maxDataBlockNumPerQuery - ? DS_CAPACITY_ENOUGH : DS_DATA_FULL; + int32_t blockNums = taosQueueSize(pDispatcher->pDataBlocks); + int32_t status = (0 == blockNums ? DS_BUF_EMPTY : + (blockNums < pDispatcher->pManager->cfg.maxDataBlockNumPerQuery ? DS_BUF_LOW : DS_BUF_FULL)); pDispatcher->status = status; pthread_mutex_unlock(&pDispatcher->mutex); return status; @@ -138,7 +141,7 @@ static int32_t getStatus(SDataDispatchHandle* pDispatcher) { return status; } -static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, int32_t* pStatus) { +static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue) { SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; SDataDispatchBuf* pBuf = taosAllocateQitem(sizeof(SDataDispatchBuf)); if (NULL == pBuf || !allocBuf(pDispatcher, pInput, pBuf)) { @@ -146,38 +149,46 @@ static int32_t putDataBlock(SDataSinkHandle* pHandle, const SInputData* pInput, } toDataCacheEntry(pDispatcher, pInput, pBuf); taosWriteQitem(pDispatcher->pDataBlocks, pBuf); - *pStatus = updateStatus(pDispatcher); + *pContinue = (DS_BUF_LOW == updateStatus(pDispatcher) ? true : false); return TSDB_CODE_SUCCESS; } -static void endPut(struct SDataSinkHandle* pHandle) { +static void endPut(struct SDataSinkHandle* pHandle, int64_t useconds) { SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; pthread_mutex_lock(&pDispatcher->mutex); - pDispatcher->status = DS_QUERY_END; + pDispatcher->queryEnd = true; + pDispatcher->useconds = useconds; pthread_mutex_unlock(&pDispatcher->mutex); } -static int32_t getDataLength(SDataSinkHandle* pHandle, int32_t* pStatus) { +static void getDataLength(SDataSinkHandle* pHandle, int32_t* pLen, bool* pQueryEnd) { SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; if (taosQueueEmpty(pDispatcher->pDataBlocks)) { - *pStatus = (DS_QUERY_END == getStatus(pDispatcher) ? DS_QUERY_END : DS_IN_PROCESS); - return 0; + *pQueryEnd = pDispatcher->queryEnd; + *pLen = 0; + return; } SDataDispatchBuf* pBuf = NULL; taosReadQitem(pDispatcher->pDataBlocks, (void**)&pBuf); memcpy(&pDispatcher->nextOutput, pBuf, sizeof(SDataDispatchBuf)); taosFreeQitem(pBuf); - return ((SDataCacheEntry*)(pDispatcher->nextOutput.pData))->dataLen; + *pLen = ((SDataCacheEntry*)(pDispatcher->nextOutput.pData))->dataLen; } -static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutPutData* pOutput, int32_t* pStatus) { +static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { SDataDispatchHandle* pDispatcher = (SDataDispatchHandle*)pHandle; SDataCacheEntry* pEntry = (SDataCacheEntry*)(pDispatcher->nextOutput.pData); memcpy(pOutput->pData, pEntry->data, pEntry->dataLen); pOutput->numOfRows = pEntry->numOfRows; pOutput->compressed = pEntry->compressed; tfree(pDispatcher->nextOutput.pData); // todo persistent - *pStatus = updateStatus(pDispatcher); + pOutput->bufStatus = updateStatus(pDispatcher); + pthread_mutex_lock(&pDispatcher->mutex); + pOutput->queryEnd = pDispatcher->queryEnd; + pOutput->needSchedule = false; + pOutput->useconds = pDispatcher->useconds; + pOutput->precision = pDispatcher->schema.precision; + pthread_mutex_unlock(&pDispatcher->mutex); return TSDB_CODE_SUCCESS; } @@ -206,7 +217,8 @@ int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSink* pDataS dispatcher->sink.fDestroy = destroyDataSinker; dispatcher->pManager = pManager; dispatcher->schema = pDataSink->schema; - dispatcher->status = DS_CAPACITY_ENOUGH; + dispatcher->status = DS_BUF_EMPTY; + dispatcher->queryEnd = false; dispatcher->pDataBlocks = taosOpenQueue(); pthread_mutex_init(&dispatcher->mutex, NULL); if (NULL == dispatcher->pDataBlocks) { diff --git a/source/libs/executor/src/dataSinkMgt.c b/source/libs/executor/src/dataSinkMgt.c index 01cc84c4f7..e3f0cd7eaa 100644 --- a/source/libs/executor/src/dataSinkMgt.c +++ b/source/libs/executor/src/dataSinkMgt.c @@ -31,24 +31,24 @@ int32_t dsCreateDataSinker(const struct SDataSink *pDataSink, DataSinkHandle* pH return TSDB_CODE_FAILED; } -int32_t dsPutDataBlock(DataSinkHandle handle, const SInputData* pInput, int32_t* pStatus) { +int32_t dsPutDataBlock(DataSinkHandle handle, const SInputData* pInput, bool* pContinue) { SDataSinkHandle* pHandleImpl = (SDataSinkHandle*)handle; - return pHandleImpl->fPut(pHandleImpl, pInput, pStatus); + return pHandleImpl->fPut(pHandleImpl, pInput, pContinue); } -void dsEndPut(DataSinkHandle handle) { +void dsEndPut(DataSinkHandle handle, int64_t useconds) { SDataSinkHandle* pHandleImpl = (SDataSinkHandle*)handle; - return pHandleImpl->fEndPut(pHandleImpl); + return pHandleImpl->fEndPut(pHandleImpl, useconds); } -void dsGetDataLength(DataSinkHandle handle, int32_t* pLen, int32_t* pStatus) { +void dsGetDataLength(DataSinkHandle handle, int32_t* pLen, bool* pQueryEnd) { SDataSinkHandle* pHandleImpl = (SDataSinkHandle*)handle; - *pLen = pHandleImpl->fGetLen(pHandleImpl, pStatus); + pHandleImpl->fGetLen(pHandleImpl, pLen, pQueryEnd); } -int32_t dsGetDataBlock(DataSinkHandle handle, SOutPutData* pOutput, int32_t* pStatus) { +int32_t dsGetDataBlock(DataSinkHandle handle, SOutputData* pOutput) { SDataSinkHandle* pHandleImpl = (SDataSinkHandle*)handle; - return pHandleImpl->fGetData(pHandleImpl, pOutput, pStatus); + return pHandleImpl->fGetData(pHandleImpl, pOutput); } void dsScheduleProcess(void* ahandle, void* pItem) { From 4d8cc7aedd682015b11fc19197ba530760f3eac1 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Tue, 11 Jan 2022 03:20:06 -0500 Subject: [PATCH 16/57] TD-12678 datasink interface adjust --- include/libs/executor/dataSinkMgt.h | 4 ++-- source/libs/executor/src/dataDispatcher.c | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/include/libs/executor/dataSinkMgt.h b/include/libs/executor/dataSinkMgt.h index 733e6d665d..a0819fcf85 100644 --- a/include/libs/executor/dataSinkMgt.h +++ b/include/libs/executor/dataSinkMgt.h @@ -21,7 +21,7 @@ extern "C" { #endif #include "os.h" -#include "executorimpl.h" +#include "thash.h" #define DS_BUF_LOW 1 #define DS_BUF_FULL 2 @@ -40,7 +40,7 @@ int32_t dsDataSinkMgtInit(SDataSinkMgtCfg *cfg); typedef void* DataSinkHandle; typedef struct SInputData { - const SSDataBlock* pData; + const struct SSDataBlock* pData; SHashObj* pTableRetrieveTsMap; } SInputData; diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index 954ed6daf1..e4b0557bff 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -19,6 +19,7 @@ #include "tcompression.h" #include "tglobal.h" #include "tqueue.h" +#include "executorimpl.h" #define DATA_META_LENGTH(tables) (sizeof(int32_t) + sizeof(STableIdInfo) * taosHashGetSize(tables) + sizeof(SRetrieveTableRsp)) From 93be707cdb0858e5db748a7cca57d08922bd244e Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 11 Jan 2022 16:23:23 +0800 Subject: [PATCH 17/57] feature/qnode --- source/client/src/clientImpl.c | 2 +- source/libs/executor/src/executorMain.c | 6 +- source/libs/qworker/src/qworker.c | 9 +-- source/libs/scheduler/src/scheduler.c | 84 +++++++++++++------------ 4 files changed, 50 insertions(+), 51 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 14cb35355f..616397f8e8 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -546,7 +546,7 @@ void* doFetchRow(SRequestObj* pRequest) { if (pResultInfo->pData == NULL || pResultInfo->current >= pResultInfo->numOfRows) { if (pRequest->type == TDMT_VND_QUERY) { pRequest->type = TDMT_VND_FETCH; - scheduleFetchRows(pRequest->body.pQueryJob, &pRequest->body.resInfo.pData); + scheduleFetchRows(pRequest->body.pQueryJob, (void **)&pRequest->body.resInfo.pData); } else if (pRequest->type == TDMT_MND_SHOW) { pRequest->type = TDMT_MND_SHOW_RETRIEVE; } else if (pRequest->type == TDMT_VND_SHOW_TABLES) { diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index e8799542b2..3f77e15382 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -191,9 +191,9 @@ int32_t qExecTask(qTaskInfo_t tinfo, DataSinkHandle* handle) { return pTaskInfo->code; } - int32_t status = 0; + bool qcontinue = false; SInputData inputData = {.pData = pRes, .pTableRetrieveTsMap = NULL}; - pTaskInfo->code = dsPutDataBlock(pTaskInfo->dsHandle, &inputData, &status); + pTaskInfo->code = dsPutDataBlock(pTaskInfo->dsHandle, &inputData, &qcontinue); if (isTaskKilled(pTaskInfo)) { qDebug("QInfo:0x%" PRIx64 " task is killed", GET_TASKID(pTaskInfo)); @@ -202,7 +202,7 @@ int32_t qExecTask(qTaskInfo_t tinfo, DataSinkHandle* handle) { // pRuntimeEnv->resultInfo.total); } - if (status == DS_DATA_FULL) { + if (!qcontinue) { qDebug("QInfo:0x%"PRIx64" query paused, %d rows returned, total:%" PRId64 " rows, in sinkNode:%d", GET_TASKID(pTaskInfo), 0, 0L, 0); return pTaskInfo->code; diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 91a27a831f..9401d38fc7 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -6,6 +6,7 @@ #include "qworkerInt.h" #include "tmsg.h" #include "tname.h" +#include "dataSinkMgt.h" int32_t qwValidateStatus(int8_t oriStatus, int8_t newStatus) { int32_t code = 0; @@ -891,14 +892,10 @@ int32_t qwHandleFetch(SQWorkerTaskHandlesCache *handles, SQWorkerMgmt *mgmt, uin QW_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } - code = dsGetDataLength(handles->sinkHandle, &dataLength, &queryEnd); - if (TSDB_CODE_SUCCESS != code) { - qError("dsGetDataLength failed, code:%x", code); - QW_ERR_JRET(code); - } + dsGetDataLength(handles->sinkHandle, &dataLength, &queryEnd); if (dataLength > 0) { - SOutPutData output = {0}; + SOutputData output = {0}; QW_ERR_JRET(qwInitFetchRsp(dataLength, &rsp)); output.pData = rsp->data; diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 9ce2e80951..37e061126b 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -509,6 +509,27 @@ int32_t schTaskCheckAndSetRetry(SSchJob *job, SSchTask *task, int32_t errCode, b } + +// Note: no more error processing, handled in function internal +int32_t schProcessOnJobFailure(SSchJob *pJob, int32_t errCode) { + // if already FAILED, no more processing + SCH_ERR_RET(schCheckAndUpdateJobStatus(pJob, JOB_TASK_STATUS_FAILED)); + + if (errCode) { + atomic_store_32(&pJob->errCode, errCode); + } + + if (atomic_load_8(&pJob->userFetch) || ((!SCH_JOB_NEED_FETCH(&pJob->attr)) && pJob->attr.syncSchedule)) { + tsem_post(&pJob->rspSem); + } + + SCH_ERR_RET(atomic_load_32(&pJob->errCode)); + + assert(0); +} + + + // Note: no more error processing, handled in function internal int32_t schFetchFromRemote(SSchJob *pJob) { int32_t code = 0; @@ -540,25 +561,6 @@ _return: } -// Note: no more error processing, handled in function internal -int32_t schProcessOnJobFailure(SSchJob *pJob, int32_t errCode) { - // if already FAILED, no more processing - SCH_ERR_RET(schCheckAndUpdateJobStatus(pJob, JOB_TASK_STATUS_FAILED)); - - if (errCode) { - atomic_store_32(&pJob->errCode, errCode); - } - - if (atomic_load_8(&pJob->userFetch) || ((!SCH_JOB_NEED_FETCH(&pJob->attr)) && pJob->attr.syncSchedule)) { - tsem_post(&pJob->rspSem); - } - - SCH_ERR_RET(atomic_load_32(&pJob->errCode)); - - assert(0); -} - - // Note: no more error processing, handled in function internal int32_t schProcessOnJobPartialSuccess(SSchJob *pJob) { int32_t code = 0; @@ -760,8 +762,6 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch break; } - break; - } case TDMT_VND_QUERY_RSP: { SQueryTableRsp *rsp = (SQueryTableRsp *)msg; @@ -784,8 +784,6 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch break; } - break; - } case TDMT_VND_FETCH_RSP: { SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)msg; @@ -1316,7 +1314,7 @@ int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, stru SSchJob *job = NULL; - SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, (void **)&job, true)); + SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, &job, true)); *pJob = job; @@ -1333,7 +1331,7 @@ int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, SSchJob *job = NULL; - SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, (void **)&job, false)); + SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, &job, false)); *pJob = job; @@ -1368,11 +1366,11 @@ int32_t scheduleFetchRows(SSchJob *pJob, void** pData) { } if (status == JOB_TASK_STATUS_FAILED) { - *data = atomic_load_ptr(&pJob->res); + *pData = atomic_load_ptr(&pJob->res); atomic_store_ptr(&pJob->res, NULL); SCH_ERR_JRET(atomic_load_32(&pJob->errCode)); } else if (status == JOB_TASK_STATUS_SUCCEED) { - *data = atomic_load_ptr(&pJob->res); + *pData = atomic_load_ptr(&pJob->res); atomic_store_ptr(&pJob->res, NULL); goto _return; } else if (status == JOB_TASK_STATUS_PARTIAL_SUCCEED) { @@ -1392,9 +1390,9 @@ int32_t scheduleFetchRows(SSchJob *pJob, void** pData) { } while (true) { - *data = atomic_load_ptr(&pJob->res); + *pData = atomic_load_ptr(&pJob->res); - if (*data != atomic_val_compare_exchange_ptr(&pJob->res, *data, NULL)) { + if (*pData != atomic_val_compare_exchange_ptr(&pJob->res, *pData, NULL)) { continue; } @@ -1472,17 +1470,21 @@ void scheduleFreeJob(void *job) { taosArrayDestroy(pLevel->subTasks); } - taosHashCleanup(pJob->execTasks); - taosHashCleanup(pJob->failTasks); - taosHashCleanup(pJob->succTasks); - - taosArrayDestroy(pJob->levels); + taosHashCleanup(pJob->execTasks); + taosHashCleanup(pJob->failTasks); + taosHashCleanup(pJob->succTasks); - tfree(pJob->res); - - tfree(pJob); + taosArrayDestroy(pJob->levels); + + tfree(pJob->res); + + tfree(pJob); +} + +void schedulerDestroy(void) { + if (schMgmt.jobs) { + taosHashCleanup(schMgmt.jobs); //TODO + schMgmt.jobs = NULL; } - - void schedulerDestroy(void) { - if (schMgmt.jobs) { - taosHashCleanup(sch +} + From e105d952bcbd2bdab369ba6ae22093b302e0d1fc Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 11 Jan 2022 18:15:37 +0800 Subject: [PATCH 18/57] feature/qnode --- include/libs/executor/dataSinkMgt.h | 1 + source/client/src/clientImpl.c | 9 ++ source/libs/executor/src/dataDispatcher.c | 7 +- source/libs/executor/src/executorMain.c | 7 +- source/libs/qworker/src/qworker.c | 126 +++++++++--------- source/libs/scheduler/src/scheduler.c | 42 +++++- source/libs/scheduler/test/schedulerTests.cpp | 4 +- 7 files changed, 127 insertions(+), 69 deletions(-) diff --git a/include/libs/executor/dataSinkMgt.h b/include/libs/executor/dataSinkMgt.h index 3ac16e9b29..f13ba5f87e 100644 --- a/include/libs/executor/dataSinkMgt.h +++ b/include/libs/executor/dataSinkMgt.h @@ -22,6 +22,7 @@ extern "C" { #include "os.h" #include "thash.h" +#include "executor.h" #define DS_BUF_LOW 1 #define DS_BUF_FULL 2 diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 616397f8e8..727b668f3a 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -547,6 +547,13 @@ void* doFetchRow(SRequestObj* pRequest) { if (pRequest->type == TDMT_VND_QUERY) { pRequest->type = TDMT_VND_FETCH; scheduleFetchRows(pRequest->body.pQueryJob, (void **)&pRequest->body.resInfo.pData); + + pResultInfo->current = 0; + if (pResultInfo->numOfRows <= pResultInfo->current) { + return NULL; + } + + goto _return; } else if (pRequest->type == TDMT_MND_SHOW) { pRequest->type = TDMT_MND_SHOW_RETRIEVE; } else if (pRequest->type == TDMT_VND_SHOW_TABLES) { @@ -590,6 +597,8 @@ void* doFetchRow(SRequestObj* pRequest) { } } +_return: + for(int32_t i = 0; i < pResultInfo->numOfCols; ++i) { pResultInfo->row[i] = pResultInfo->pCol[i] + pResultInfo->fields[i].bytes * pResultInfo->current; if (IS_VAR_DATA_TYPE(pResultInfo->fields[i].type)) { diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index e4b0557bff..a69084f3db 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -109,10 +109,14 @@ static void toDataCacheEntry(const SDataDispatchHandle* pHandle, const SInputDat SDataCacheEntry* pEntry = (SDataCacheEntry*)pBuf->pData; pEntry->compressed = (int8_t)needCompress(pInput->pData, &(pHandle->schema)); pEntry->numOfRows = pInput->pData->info.rows; + pEntry->dataLen = 0; pBuf->useSize = DATA_META_LENGTH(pInput->pTableRetrieveTsMap); copyData(pInput, &pHandle->schema, pEntry->data, pEntry->compressed, &pEntry->dataLen); - pBuf->useSize += (pEntry->compressed ? pEntry->dataLen : pHandle->schema.resultRowSize * pInput->pData->info.rows); + if (0 == pEntry->compressed) { + pEntry->dataLen = pHandle->schema.resultRowSize * pInput->pData->info.rows; + } + pBuf->useSize += pEntry->dataLen; // todo completed } @@ -213,6 +217,7 @@ int32_t createDataDispatcher(SDataSinkManager* pManager, const SDataSink* pDataS return TSDB_CODE_QRY_OUT_OF_MEMORY; } dispatcher->sink.fPut = putDataBlock; + dispatcher->sink.fEndPut = endPut; dispatcher->sink.fGetLen = getDataLength; dispatcher->sink.fGetData = getDataBlock; dispatcher->sink.fDestroy = destroyDataSinker; diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index 3f77e15382..50f69fb567 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -77,13 +77,13 @@ int32_t qCreateExecTask(void* tsdb, int32_t vgId, SSubplan* pSubplan, qTaskInfo_ goto _error; } - SDataSinkMgtCfg cfg = {.maxDataBlockNum = 1000}; + SDataSinkMgtCfg cfg = {.maxDataBlockNum = 1000, .maxDataBlockNumPerQuery = 100}; code = dsDataSinkMgtInit(&cfg); if (code != TSDB_CODE_SUCCESS) { goto _error; } - code = dsCreateDataSinker(pSubplan->pDataSink, (*pTask)->dsHandle); + code = dsCreateDataSinker(pSubplan->pDataSink, &(*pTask)->dsHandle); _error: // if failed to add ref for all tables in this query, abort current query @@ -178,7 +178,7 @@ int32_t qExecTask(qTaskInfo_t tinfo, DataSinkHandle* handle) { publishOperatorProfEvent(pTaskInfo->pRoot, QUERY_PROF_BEFORE_OPERATOR_EXEC); int64_t st = 0; - handle = &pTaskInfo->dsHandle; + *handle = pTaskInfo->dsHandle; while(1) { st = taosGetTimestampUs(); @@ -188,6 +188,7 @@ int32_t qExecTask(qTaskInfo_t tinfo, DataSinkHandle* handle) { publishOperatorProfEvent(pTaskInfo->pRoot, QUERY_PROF_AFTER_OPERATOR_EXEC); if (pRes == NULL) { // no results generated yet, abort + dsEndPut(pTaskInfo->dsHandle, pTaskInfo->cost.elapsedTime); return pTaskInfo->code; } diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 9401d38fc7..ad3e8073ee 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -617,6 +617,8 @@ int32_t qwInitFetchRsp(int32_t length, SRetrieveTableRsp **rsp) { memset(pRsp, 0, sizeof(SRetrieveTableRsp)); + *rsp = pRsp; + return TSDB_CODE_SUCCESS; } @@ -866,7 +868,61 @@ int32_t qwCheckTaskCancelDrop( SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryI return TSDB_CODE_SUCCESS; } -int32_t qwHandleFetch(SQWorkerTaskHandlesCache *handles, SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t taskId, SRpcMsg *pMsg) { + +int32_t qwQueryPostProcess(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int8_t status, int32_t errCode) { + SQWSchStatus *sch = NULL; + SQWTaskStatus *task = NULL; + int32_t code = 0; + int8_t newStatus = JOB_TASK_STATUS_CANCELLED; + + code = qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_ADD); + if (code) { + qError("sId:%"PRIx64" not in cache", sId); + QW_ERR_RET(code); + } + + code = qwAcquireTask(QW_READ, sch, qId, tId, &task); + if (code) { + qwReleaseScheduler(QW_READ, mgmt); + + if (JOB_TASK_STATUS_PARTIAL_SUCCEED == status || JOB_TASK_STATUS_SUCCEED == status) { + qError("sId:%"PRIx64" queryId:%"PRIx64" taskId:%"PRIx64" not in cache", sId, qId, tId); + QW_ERR_RET(code); + } + + QW_ERR_RET(qwAddTask(mgmt, sId, qId, tId, status, QW_EXIST_ACQUIRE, &sch, &task)); + } + + if (task->cancel) { + QW_LOCK(QW_WRITE, &task->lock); + qwUpdateTaskInfo(task, QW_TASK_INFO_STATUS, &newStatus); + QW_UNLOCK(QW_WRITE, &task->lock); + } + + if (task->drop) { + qwReleaseTask(QW_READ, sch); + qwReleaseScheduler(QW_READ, mgmt); + + qwDropTask(mgmt, sId, qId, tId); + + return TSDB_CODE_SUCCESS; + } + + if (!(task->cancel || task->drop)) { + QW_LOCK(QW_WRITE, &task->lock); + qwUpdateTaskInfo(task, QW_TASK_INFO_STATUS, &status); + task->code = errCode; + QW_UNLOCK(QW_WRITE, &task->lock); + } + + qwReleaseTask(QW_READ, sch); + qwReleaseScheduler(QW_READ, mgmt); + + return TSDB_CODE_SUCCESS; +} + + +int32_t qwHandleFetch(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64_t taskId, SRpcMsg *pMsg) { SQWSchStatus *sch = NULL; SQWTaskStatus *task = NULL; int32_t code = 0; @@ -876,6 +932,9 @@ int32_t qwHandleFetch(SQWorkerTaskHandlesCache *handles, SQWorkerMgmt *mgmt, uin int32_t dataLength = 0; SRetrieveTableRsp *rsp = NULL; bool queryEnd = false; + SQWorkerTaskHandlesCache *handles = NULL; + + QW_ERR_JRET(qwAcquireTaskHandles(QW_READ, mgmt, queryId, taskId, &handles)); QW_ERR_JRET(qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_RET_ERR)); QW_ERR_JRET(qwAcquireTask(QW_READ, sch, queryId, taskId, &task)); @@ -955,59 +1014,11 @@ _return: qwBuildAndSendFetchRsp(pMsg, rsp, dataLength, code); } - QW_RET(code); -} - -int32_t qwQueryPostProcess(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t qId, uint64_t tId, int8_t status, int32_t errCode) { - SQWSchStatus *sch = NULL; - SQWTaskStatus *task = NULL; - int32_t code = 0; - int8_t newStatus = JOB_TASK_STATUS_CANCELLED; - - code = qwAcquireScheduler(QW_READ, mgmt, sId, &sch, QW_NOT_EXIST_ADD); - if (code) { - qError("sId:%"PRIx64" not in cache", sId); - QW_ERR_RET(code); - } - - code = qwAcquireTask(QW_READ, sch, qId, tId, &task); - if (code) { - qwReleaseScheduler(QW_READ, mgmt); - - if (JOB_TASK_STATUS_PARTIAL_SUCCEED == status || JOB_TASK_STATUS_SUCCEED == status) { - qError("sId:%"PRIx64" queryId:%"PRIx64" taskId:%"PRIx64" not in cache", sId, qId, tId); - QW_ERR_RET(code); - } - - QW_ERR_RET(qwAddTask(mgmt, sId, qId, tId, status, QW_EXIST_ACQUIRE, &sch, &task)); - } - - if (task->cancel) { - QW_LOCK(QW_WRITE, &task->lock); - qwUpdateTaskInfo(task, QW_TASK_INFO_STATUS, &newStatus); - QW_UNLOCK(QW_WRITE, &task->lock); - } - - if (task->drop) { - qwReleaseTask(QW_READ, sch); - qwReleaseScheduler(QW_READ, mgmt); - - qwDropTask(mgmt, sId, qId, tId); - - return TSDB_CODE_SUCCESS; - } - - if (!(task->cancel || task->drop)) { - QW_LOCK(QW_WRITE, &task->lock); - qwUpdateTaskInfo(task, QW_TASK_INFO_STATUS, &status); - task->code = errCode; - QW_UNLOCK(QW_WRITE, &task->lock); + if (handles) { + qwReleaseTaskResCache(QW_READ, mgmt); } - qwReleaseTask(QW_READ, sch); - qwReleaseScheduler(QW_READ, mgmt); - - return TSDB_CODE_SUCCESS; + QW_RET(code); } int32_t qWorkerInit(SQWorkerCfg *cfg, void **qWorkerMgmt) { @@ -1081,6 +1092,7 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { qTaskInfo_t pTaskInfo = NULL; code = qCreateExecTask(node, 0, (struct SSubplan *)plan, &pTaskInfo); if (code) { + qError("qCreateExecTask failed, code:%x", code); QW_ERR_JRET(code); } else { QW_ERR_JRET(qwAddTask(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, JOB_TASK_STATUS_EXECUTING, QW_EXIST_RET_ERR, NULL, NULL)); @@ -1094,6 +1106,7 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { code = qExecTask(pTaskInfo, &sinkHandle); if (code) { + qError("qExecTask failed, code:%x", code); QW_ERR_JRET(code); } else { QW_ERR_JRET(qwAddTaskHandlesToCache(qWorkerMgmt, msg->queryId, msg->taskId, pTaskInfo, sinkHandle)); @@ -1225,17 +1238,10 @@ int32_t qWorkerProcessFetchMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { QW_ERR_RET(qwUpdateSchLastAccess(qWorkerMgmt, msg->sId)); void *data = NULL; - SQWorkerTaskHandlesCache *handles = NULL; int32_t code = 0; - QW_ERR_RET(qwAcquireTaskHandles(QW_READ, qWorkerMgmt, msg->queryId, msg->taskId, &handles)); + QW_ERR_RET(qwHandleFetch(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, pMsg)); - QW_ERR_JRET(qwHandleFetch(handles, qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, pMsg)); - -_return: - - qwReleaseTaskResCache(QW_READ, qWorkerMgmt); - QW_RET(code); } diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 37e061126b..ed12408844 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -68,7 +68,7 @@ int32_t schValidateTaskReceivedMsgType(SSchJob *pJob, SSchTask *pTask, int32_t m SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } - if (SCH_GET_TASK_STATUS(pTask) != JOB_TASK_STATUS_EXECUTING) { + if (SCH_GET_TASK_STATUS(pTask) != JOB_TASK_STATUS_EXECUTING && SCH_GET_TASK_STATUS(pTask) != JOB_TASK_STATUS_PARTIAL_SUCCEED) { SCH_TASK_ELOG("rsp msg conflicted with task status, status:%d, rspType:%d", SCH_GET_TASK_STATUS(pTask), msgType); SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); } @@ -497,6 +497,33 @@ int32_t schMoveTaskToFailList(SSchJob *pJob, SSchTask *pTask, bool *moved) { return TSDB_CODE_SUCCESS; } + +int32_t schMoveTaskToExecList(SSchJob *pJob, SSchTask *pTask, bool *moved) { + if (0 != taosHashRemove(pJob->succTasks, &pTask->taskId, sizeof(pTask->taskId))) { + SCH_TASK_WLOG("remove task from succTask list failed, may not exist, status:%d", SCH_GET_TASK_STATUS(pTask)); + } + + int32_t code = taosHashPut(pJob->execTasks, &pTask->taskId, sizeof(pTask->taskId), &pTask, POINTER_BYTES); + if (0 != code) { + if (HASH_NODE_EXIST(code)) { + *moved = true; + + SCH_TASK_ELOG("task already in execTask list, status:%d", SCH_GET_TASK_STATUS(pTask)); + SCH_ERR_RET(TSDB_CODE_SCH_STATUS_ERROR); + } + + SCH_TASK_ELOG("taosHashPut task to execTask list failed, errno:%d", errno); + SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + *moved = true; + + SCH_TASK_DLOG("task moved to execTask list, numOfTasks:%d", taosHashGetSize(pJob->execTasks)); + + return TSDB_CODE_SUCCESS; +} + + int32_t schTaskCheckAndSetRetry(SSchJob *job, SSchTask *task, int32_t errCode, bool *needRetry) { // TODO set retry or not based on task type/errCode/retry times/job status/available eps... // TODO if needRetry, set task retry info @@ -657,7 +684,7 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { SCH_ERR_RET(code); } - SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_SUCCEED); + SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_PARTIAL_SUCCEED); SCH_ERR_JRET(schRecordTaskSucceedNode(pTask)); @@ -689,6 +716,11 @@ int32_t schProcessOnTaskSuccess(SSchJob *pJob, SSchTask *pTask) { } pJob->fetchTask = pTask; + + code = schMoveTaskToExecList(pJob, pTask, &moved); + if (code && moved) { + SCH_ERR_RET(code); + } SCH_ERR_RET(schProcessOnJobPartialSuccess(pJob)); @@ -836,7 +868,10 @@ int32_t schHandleCallback(void* param, const SDataBuf* pMsg, int32_t msgType, in atomic_add_fetch_32(&pJob->ref, 1); int32_t s = taosHashGetSize(pJob->execTasks); - assert(s != 0); + if (s <= 0) { + qError("QID:%"PRIx64",TID:%"PRIx64" no task in execTask list", pParam->queryId, pParam->taskId); + SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR); + } SSchTask **task = taosHashGet(pJob->execTasks, &pParam->taskId, sizeof(pParam->taskId)); if (NULL == task || NULL == (*task)) { @@ -1383,6 +1418,7 @@ int32_t scheduleFetchRows(SSchJob *pJob, void** pData) { if (status == JOB_TASK_STATUS_FAILED) { code = atomic_load_32(&pJob->errCode); + SCH_ERR_JRET(code); } if (pJob->res && ((SRetrieveTableRsp *)pJob->res)->completed) { diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index f75f0ef263..b438521234 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -217,7 +217,7 @@ void *schtSendRsp(void *param) { return NULL; } -void *pInsertJob = NULL; +struct SSchJob *pInsertJob = NULL; } @@ -228,7 +228,7 @@ TEST(queryTest, normalCase) { char *dbname = "1.db1"; char *tablename = "table1"; SVgroupInfo vgInfo = {0}; - void *pJob = NULL; + SSchJob *pJob = NULL; SQueryDag dag = {0}; schtInitLogFile(); From 59d23065fd469447717b2bc7da46ce2ec012c01e Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 11 Jan 2022 02:54:11 -0800 Subject: [PATCH 19/57] fix invalid write in mnode --- source/dnode/mnode/impl/src/mndDb.c | 2 +- source/dnode/mnode/impl/src/mndDnode.c | 2 +- source/dnode/mnode/impl/src/mndVgroup.c | 8 ++++ tests/script/sh/exec.sh | 4 +- tests/script/sim/db/basic6.sim | 56 ++++++++++--------------- 5 files changed, 33 insertions(+), 39 deletions(-) diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 85b7fbbb42..bc4d890257 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -740,7 +740,7 @@ static int32_t mndBuildDropVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj * if (pReq == NULL) return -1; action.pCont = pReq; - action.contLen = sizeof(SCreateVnodeReq); + action.contLen = sizeof(SDropVnodeReq); action.msgType = TDMT_DND_DROP_VNODE; action.acceptableCode = TSDB_CODE_DND_VNODE_NOT_DEPLOYED; if (mndTransAppendRedoAction(pTrans, &action) != 0) { diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 4bc570c11d..2a67e6a0f9 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -235,7 +235,7 @@ int32_t mndGetDnodeSize(SMnode *pMnode) { bool mndIsDnodeOnline(SMnode *pMnode, SDnodeObj *pDnode, int64_t curMs) { int64_t interval = ABS(pDnode->lastAccessTime - curMs); - if (interval > 3500 * pMnode->cfg.statusInterval) { + if (interval > 10000 * pMnode->cfg.statusInterval) { if (pDnode->rebootTime > 0) { pDnode->offlineReason = DND_REASON_STATUS_MSG_TIMEOUT; } diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index e0d6d3dd42..93d6d104ff 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -319,6 +319,14 @@ static int32_t mndGetAvailableDnode(SMnode *pMnode, SVgObj *pVgroup, SArray *pAr taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes); + int32_t size = taosArrayGetSize(pArray); + if (size < pVgroup->replica) { + mError("db:%s, vgId:%d, no enough online dnodes:%d to alloc %d replica", pVgroup->dbName, pVgroup->vgId, size, + pVgroup->replica); + terrno = TSDB_CODE_MND_NO_ENOUGH_DNODES; + return -1; + } + for (int32_t v = 0; v < pVgroup->replica; ++v) { SVnodeGid *pVgid = &pVgroup->vnodeGid[v]; SDnodeObj *pDnode = taosArrayGet(pArray, v); diff --git a/tests/script/sh/exec.sh b/tests/script/sh/exec.sh index 2e95a740d0..05f756ebb6 100755 --- a/tests/script/sh/exec.sh +++ b/tests/script/sh/exec.sh @@ -30,7 +30,7 @@ do CLEAR_OPTION="clear" ;; v) - SHELL_OPTION="true" + VALGRIND_OPTION="true" ;; u) USERS=$OPTARG @@ -99,7 +99,7 @@ fi if [ "$EXEC_OPTON" = "start" ]; then echo "ExcuteCmd:" $EXE_DIR/taosd -c $CFG_DIR - if [ "$SHELL_OPTION" = "true" ]; then + if [ "$VALGRIND_OPTION" = "true" ]; then TT=`date +%s` mkdir ${LOG_DIR}/${TT} nohup valgrind --log-file=${LOG_DIR}/${TT}/valgrind.log --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 & diff --git a/tests/script/sim/db/basic6.sim b/tests/script/sim/db/basic6.sim index 684ce825fe..8e7d4f5ec1 100644 --- a/tests/script/sim/db/basic6.sim +++ b/tests/script/sim/db/basic6.sim @@ -1,29 +1,24 @@ system sh/stop_dnodes.sh system sh/deploy.sh -n dnode1 -i 1 -system sh/cfg.sh -n dnode1 -c wallevel -v 0 -system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 4 -system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 1000 - system sh/exec.sh -n dnode1 -s start - -sleep 2000 sql connect + print ============================ dnode1 start $i = 0 -$dbPrefix = ob_db_db -$tbPrefix = ob_db_tb +$dbPrefix = db +$tbPrefix = tb $db = $dbPrefix . $i $tb = $tbPrefix . $i print =============== step1 -sql create database $db replica 1 days 20 keep 2000 cache 16 +sql create database $db replica 1 days 20 keep 2000 cache 16 vgroups 4 sql show databases print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 if $data00 != $db then return -1 endi -if $data02 != 0 then +if $data02 != 4 then return -1 endi if $data03 != 0 then @@ -63,9 +58,6 @@ print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 if $data00 != $db then return -1 endi -if $data02 != 0 then - return -1 -endi if $data03 != 0 then return -1 endi @@ -77,30 +69,14 @@ if $data06 != 15 then endi print =============== step6 -sql use $db -sql create table $tb (ts timestamp, speed int) -$i = 1 -while $i < 4 +$i = $i + 1 +while $i < 5 $db = $dbPrefix . $i - $tb = $tbPrefix . $i sql create database $db sql use $db - sql create table $tb (ts timestamp, speed int) $i = $i + 1 endw -sql show databases -if $rows != 4 then - return -1 -endi - -$i = 4 -$db = $dbPrefix . $i -$tb = $tbPrefix . $i -sql create database $db -sql use $db -sql create table $tb (ts timestamp, speed int) - print =============== step7 $i = 0 while $i < 5 @@ -115,7 +91,8 @@ $db = $dbPrefix . $i $tb = $tbPrefix . $i sql create database $db sql use $db -sql create table $tb (ts timestamp, speed int) +sql create table st (ts timestamp, i int) tags (j int) +sql create table $tb using st tags(1) sql show tables if $rows != 1 then return -1 @@ -133,7 +110,8 @@ if $rows != 0 then endi print =============== step11 -sql create table $tb (ts timestamp, speed int) +sql create table st (ts timestamp, i int) tags (j int) +sql create table $tb using st tags(1) sql show tables if $rows != 1 then return -1 @@ -149,16 +127,23 @@ sql show tables if $rows != 0 then return -1 endi -sql create table $tb (ts timestamp, speed int) + +sql create table st (ts timestamp, i int) tags (j int) +sql create table $tb using st tags(1) sql show tables if $rows != 1 then return -1 endi + sql insert into $tb values (now+1a, 0) sql insert into $tb values (now+2a, 1) sql insert into $tb values (now+3a, 2) sql insert into $tb values (now+4a, 3) sql insert into $tb values (now+5a, 4) + + +return + sql select * from $tb if $rows != 5 then return -1 @@ -176,7 +161,8 @@ if $rows != 0 then endi print =============== step16 -sql create table $tb (ts timestamp, speed int) +sql create table st (ts timestamp, i int) tags (j int) +sql create table $tb using st tags(1) sql show tables if $rows != 1 then return -1 From 0920bb05763b6f8d1e6b0cc508b6190d332779a5 Mon Sep 17 00:00:00 2001 From: lihui Date: Tue, 11 Jan 2022 18:55:09 +0800 Subject: [PATCH 20/57] [modify] --- tests/test/c/create_table.c | 117 ++++++++++++++++++++++++++---------- 1 file changed, 84 insertions(+), 33 deletions(-) diff --git a/tests/test/c/create_table.c b/tests/test/c/create_table.c index f2db9d0a0c..a5a55bbc91 100644 --- a/tests/test/c/create_table.c +++ b/tests/test/c/create_table.c @@ -40,6 +40,8 @@ typedef struct { float createTableSpeed; float insertDataSpeed; int64_t startMs; + int64_t maxDelay; + int64_t minDelay; pthread_t thread; } SThreadInfo; @@ -58,12 +60,30 @@ int32_t main(int32_t argc, char *argv[]) { pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); SThreadInfo *pInfo = (SThreadInfo *)calloc(numOfThreads, sizeof(SThreadInfo)); - int64_t numOfTablesPerThread = numOfTables / numOfThreads; - numOfTables = numOfTablesPerThread * numOfThreads; + //int64_t numOfTablesPerThread = numOfTables / numOfThreads; + //numOfTables = numOfTablesPerThread * numOfThreads; + + + if (numOfThreads < 1) { + numOfThreads = 1; + } + + int64_t a = numOfTables / numOfThreads; + if (a < 1) { + numOfThreads = numOfTables; + a = 1; + } + + int64_t b = 0; + b = numOfTables % numOfThreads; + + int64_t tableFrom = 0; for (int32_t i = 0; i < numOfThreads; ++i) { - pInfo[i].tableBeginIndex = i * numOfTablesPerThread; - pInfo[i].tableEndIndex = (i + 1) * numOfTablesPerThread; + pInfo[i].tableBeginIndex = tableFrom; + pInfo[i].tableEndIndex = i < b ? tableFrom + a : tableFrom + a - 1; + tableFrom = pInfo[i].tableEndIndex + 1; pInfo[i].threadIndex = i; + pInfo[i].minDelay = INT64_MAX; strcpy(pInfo[i].dbName, dbName); strcpy(pInfo[i].stbName, stbName); pthread_create(&(pInfo[i].thread), &thattr, threadFunc, (void *)(pInfo + i)); @@ -74,9 +94,15 @@ int32_t main(int32_t argc, char *argv[]) { pthread_join(pInfo[i].thread, NULL); } + int64_t maxDelay = 0; + int64_t minDelay = INT64_MAX; + float createTableSpeed = 0; for (int32_t i = 0; i < numOfThreads; ++i) { createTableSpeed += pInfo[i].createTableSpeed; + + if (pInfo[i].maxDelay > maxDelay) maxDelay = pInfo[i].maxDelay; + if (pInfo[i].minDelay < minDelay) minDelay = pInfo[i].minDelay; } float insertDataSpeed = 0; @@ -84,10 +110,19 @@ int32_t main(int32_t argc, char *argv[]) { insertDataSpeed += pInfo[i].insertDataSpeed; } - pPrint("%s total %" PRId64 " tables, %.1f tables/second, threads:%d %s", GREEN, numOfTables, createTableSpeed, - numOfThreads, NC); - pPrint("%s total %" PRId64 " tables, %.1f rows/second, threads:%d %s", GREEN, numOfTables, insertDataSpeed, + pPrint("%s total %" PRId64 " tables, %.1f tables/second, threads:%d, maxDelay: %" PRId64 "us, minDelay: %" PRId64 "us %s", + GREEN, + numOfTables, + createTableSpeed, + numOfThreads, + maxDelay, + minDelay, + NC); + + if (insertData) { + pPrint("%s total %" PRId64 " tables, %.1f rows/second, threads:%d %s", GREEN, numOfTables, insertDataSpeed, numOfThreads, NC); + } pthread_attr_destroy(&thattr); free(pInfo); @@ -99,36 +134,36 @@ void createDbAndStb() { TAOS *con = taos_connect(NULL, "root", "taosdata", NULL, 0); if (con == NULL) { - pError("failed to connect to DB, reason:%s", taos_errstr(con)); + pError("failed to connect to DB, reason:%s", taos_errstr(NULL)); exit(1); } sprintf(qstr, "create database if not exists %s vgroups %d", dbName, numOfVgroups); - TAOS_RES *pSql = taos_query(con, qstr); - int32_t code = taos_errno(pSql); + TAOS_RES *pRes = taos_query(con, qstr); + int32_t code = taos_errno(pRes); if (code != 0) { - pError("failed to create database:%s, sql:%s, code:%d reason:%s", dbName, qstr, taos_errno(con), taos_errstr(con)); + pError("failed to create database:%s, sql:%s, code:%d reason:%s", dbName, qstr, taos_errno(pRes), taos_errstr(pRes)); exit(0); } - taos_free_result(pSql); + taos_free_result(pRes); sprintf(qstr, "use %s", dbName); - pSql = taos_query(con, qstr); - code = taos_errno(pSql); + pRes = taos_query(con, qstr); + code = taos_errno(pRes); if (code != 0) { - pError("failed to use db, code:%d reason:%s", taos_errno(con), taos_errstr(con)); + pError("failed to use db, code:%d reason:%s", taos_errno(pRes), taos_errstr(pRes)); exit(0); } - taos_free_result(pSql); + taos_free_result(pRes); sprintf(qstr, "create table %s (ts timestamp, i int) tags (j int)", stbName); - pSql = taos_query(con, qstr); - code = taos_errno(pSql); + pRes = taos_query(con, qstr); + code = taos_errno(pRes); if (code != 0) { - pError("failed to use db, code:%d reason:%s", taos_errno(con), taos_errstr(con)); + pError("failed to use db, code:%d reason:%s", taos_errno(pRes), taos_errstr(pRes)); exit(0); } - taos_free_result(pSql); + taos_free_result(pRes); taos_close(con); } @@ -160,16 +195,20 @@ void *threadFunc(void *param) { TAOS *con = taos_connect(NULL, "root", "taosdata", NULL, 0); if (con == NULL) { - pError("index:%d, failed to connect to DB, reason:%s", pInfo->threadIndex, taos_errstr(con)); + pError("index:%d, failed to connect to DB, reason:%s", pInfo->threadIndex, taos_errstr(NULL)); exit(1); } + //printf("thread:%d, table range: %"PRId64 " - %"PRId64 "\n", pInfo->threadIndex, pInfo->tableBeginIndex, pInfo->tableEndIndex); sprintf(qstr, "use %s", pInfo->dbName); - TAOS_RES *pSql = taos_query(con, qstr); - taos_free_result(pSql); + TAOS_RES *pRes = taos_query(con, qstr); + taos_free_result(pRes); + if (createTable) { - pInfo->startMs = taosGetTimestampMs(); + int64_t curMs = 0; + int64_t beginMs = taosGetTimestampMs(); + pInfo->startMs = beginMs; for (int64_t t = pInfo->tableBeginIndex; t < pInfo->tableEndIndex; ++t) { int64_t batch = (pInfo->tableEndIndex - t); batch = MIN(batch, batchNum); @@ -179,14 +218,22 @@ void *threadFunc(void *param) { len += sprintf(qstr + len, " t%" PRId64 " using %s tags(%" PRId64 ")", t + i, stbName, t + i); } - TAOS_RES *pSql = taos_query(con, qstr); - code = taos_errno(pSql); + int64_t startTs = taosGetTimestampUs(); + TAOS_RES *pRes = taos_query(con, qstr); + code = taos_errno(pRes); if (code != 0) { pError("failed to create table t%" PRId64 ", reason:%s", t, tstrerror(code)); } - taos_free_result(pSql); + taos_free_result(pRes); + int64_t endTs = taosGetTimestampUs(); + int64_t delay = endTs - startTs; + //printf("==== %"PRId64" - %"PRId64", %"PRId64"\n", startTs, endTs, delay); + if (delay > pInfo->maxDelay) pInfo->maxDelay = delay; + if (delay < pInfo->minDelay) pInfo->minDelay = delay; - if (t % 100000 == 0) { + curMs = taosGetTimestampMs(); + if (curMs - beginMs > 10000) { + beginMs = curMs; printCreateProgress(pInfo, t); } t += (batch - 1); @@ -195,6 +242,9 @@ void *threadFunc(void *param) { } if (insertData) { + int64_t curMs = 0; + int64_t beginMs = taosGetTimestampMs();; + pInfo->startMs = taosGetTimestampMs(); for (int64_t t = pInfo->tableBeginIndex; t < pInfo->tableEndIndex; ++t) { int64_t batch = (pInfo->tableEndIndex - t); @@ -205,14 +255,15 @@ void *threadFunc(void *param) { len += sprintf(qstr + len, " t%" PRId64 " values(now, %" PRId64 ")", t + i, t + i); } - TAOS_RES *pSql = taos_query(con, qstr); - code = taos_errno(pSql); + TAOS_RES *pRes = taos_query(con, qstr); + code = taos_errno(pRes); if (code != 0) { pError("failed to insert table t%" PRId64 ", reason:%s", t, tstrerror(code)); } - taos_free_result(pSql); + taos_free_result(pRes); - if (t % 100000 == 0) { + curMs = taosGetTimestampMs(); + if (curMs - beginMs > 10000) { printInsertProgress(pInfo, t); } t += (batch - 1); @@ -266,7 +317,7 @@ void parseArgument(int32_t argc, char *argv[]) { numOfThreads = atoi(argv[++i]); } else if (strcmp(argv[i], "-n") == 0) { numOfTables = atoll(argv[++i]); - } else if (strcmp(argv[i], "-n") == 0) { + } else if (strcmp(argv[i], "-v") == 0) { numOfVgroups = atoi(argv[++i]); } else if (strcmp(argv[i], "-a") == 0) { createTable = atoi(argv[++i]); From 6677dfba943e176ddde55ca373fb6dfe26541fef Mon Sep 17 00:00:00 2001 From: dapan1121 Date: Tue, 11 Jan 2022 18:57:19 +0800 Subject: [PATCH 21/57] feature/qnode --- source/libs/qworker/src/qworker.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index ad3e8073ee..2a395fcfe1 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -964,6 +964,13 @@ int32_t qwHandleFetch(SQWorkerMgmt *mgmt, uint64_t sId, uint64_t queryId, uint64 qError("dsGetDataBlock failed, code:%x", code); QW_ERR_JRET(code); } + + rsp->useconds = htobe64(output.useconds); + rsp->completed = 0; + rsp->precision = output.precision; + rsp->compressed = output.compressed; + rsp->compLen = htonl(dataLength); + rsp->numOfRows = htonl(output.numOfRows); if (DS_BUF_EMPTY == output.bufStatus && output.queryEnd) { rsp->completed = 1; From f94f8aadc0b00e15a0869eb7291ae53b8bec8d49 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 11 Jan 2022 03:13:26 -0800 Subject: [PATCH 22/57] dnode may offline while create vnode --- source/dnode/mgmt/impl/inc/dndInt.h | 1 + source/dnode/mgmt/impl/src/dndMgmt.c | 13 ++++++++++++- source/dnode/mnode/impl/src/mndDnode.c | 2 +- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/source/dnode/mgmt/impl/inc/dndInt.h b/source/dnode/mgmt/impl/inc/dndInt.h index afdd678213..e637b38815 100644 --- a/source/dnode/mgmt/impl/inc/dndInt.h +++ b/source/dnode/mgmt/impl/inc/dndInt.h @@ -94,6 +94,7 @@ typedef struct { pthread_t *threadId; SRWLatch latch; SDnodeWorker mgmtWorker; + SDnodeWorker statusWorker; } SDnodeMgmt; typedef struct { diff --git a/source/dnode/mgmt/impl/src/dndMgmt.c b/source/dnode/mgmt/impl/src/dndMgmt.c index f252bffbbf..d9edf39b73 100644 --- a/source/dnode/mgmt/impl/src/dndMgmt.c +++ b/source/dnode/mgmt/impl/src/dndMgmt.c @@ -536,6 +536,11 @@ int32_t dndInitMgmt(SDnode *pDnode) { return -1; } + if (dndInitWorker(pDnode, &pMgmt->statusWorker, DND_WORKER_SINGLE, "dnode-status", 1, 1, dndProcessMgmtQueue) != 0) { + dError("failed to start dnode mgmt worker since %s", terrstr()); + return -1; + } + pMgmt->threadId = taosCreateThread(dnodeThreadRoutine, pDnode); if (pMgmt->threadId == NULL) { dError("failed to init dnode thread"); @@ -550,6 +555,7 @@ int32_t dndInitMgmt(SDnode *pDnode) { void dndStopMgmt(SDnode *pDnode) { SDnodeMgmt *pMgmt = &pDnode->dmgmt; dndCleanupWorker(&pMgmt->mgmtWorker); + dndCleanupWorker(&pMgmt->statusWorker); if (pMgmt->threadId != NULL) { taosDestoryThread(pMgmt->threadId); @@ -587,7 +593,12 @@ void dndProcessMgmtMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { dndUpdateMnodeEpSet(pDnode, pEpSet); } - if (dndWriteMsgToWorker(&pMgmt->mgmtWorker, pMsg, sizeof(SRpcMsg)) != 0) { + SDnodeWorker *pWorker = &pMgmt->mgmtWorker; + if (pMsg->msgType == TDMT_MND_STATUS_RSP) { + pWorker = &pMgmt->statusWorker; + } + + if (dndWriteMsgToWorker(pWorker, pMsg, sizeof(SRpcMsg)) != 0) { if (pMsg->msgType & 1u) { SRpcMsg rsp = {.handle = pMsg->handle, .code = TSDB_CODE_OUT_OF_MEMORY}; rpcSendResponse(&rsp); diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 2a67e6a0f9..4bc570c11d 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -235,7 +235,7 @@ int32_t mndGetDnodeSize(SMnode *pMnode) { bool mndIsDnodeOnline(SMnode *pMnode, SDnodeObj *pDnode, int64_t curMs) { int64_t interval = ABS(pDnode->lastAccessTime - curMs); - if (interval > 10000 * pMnode->cfg.statusInterval) { + if (interval > 3500 * pMnode->cfg.statusInterval) { if (pDnode->rebootTime > 0) { pDnode->offlineReason = DND_REASON_STATUS_MSG_TIMEOUT; } From 07a1b4acf21d25321819781d1ec9a2214d98e175 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 11 Jan 2022 03:41:38 -0800 Subject: [PATCH 23/57] minor changes --- source/dnode/mnode/impl/test/profile/profile.cpp | 10 +++++----- tests/script/jenkins/basic.txt | 5 +++-- tests/script/sim/db/basic6.sim | 4 ++++ 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/source/dnode/mnode/impl/test/profile/profile.cpp b/source/dnode/mnode/impl/test/profile/profile.cpp index bdffb6c72a..bf047517d3 100644 --- a/source/dnode/mnode/impl/test/profile/profile.cpp +++ b/source/dnode/mnode/impl/test/profile/profile.cpp @@ -13,7 +13,7 @@ class MndTestProfile : public ::testing::Test { protected: - static void SetUpTestSuite() { test.Init("/tmp/mnode_test_profile", 9022); } + static void SetUpTestSuite() { test.Init("/tmp/mnode_test_profile", 9031); } static void TearDownTestSuite() { test.Cleanup(); } static Testbase test; @@ -53,7 +53,7 @@ TEST_F(MndTestProfile, 01_ConnectMsg) { EXPECT_EQ(pRsp->epSet.inUse, 0); EXPECT_EQ(pRsp->epSet.numOfEps, 1); - EXPECT_EQ(pRsp->epSet.port[0], 9022); + EXPECT_EQ(pRsp->epSet.port[0], 9031); EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost"); connId = pRsp->connId; @@ -127,7 +127,7 @@ TEST_F(MndTestProfile, 04_HeartBeatMsg) { EXPECT_EQ(pRsp->epSet.inUse, 0); EXPECT_EQ(pRsp->epSet.numOfEps, 1); - EXPECT_EQ(pRsp->epSet.port[0], 9022); + EXPECT_EQ(pRsp->epSet.port[0], 9031); EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost"); } @@ -185,7 +185,7 @@ TEST_F(MndTestProfile, 05_KillConnMsg) { EXPECT_EQ(pRsp->epSet.inUse, 0); EXPECT_EQ(pRsp->epSet.numOfEps, 1); - EXPECT_EQ(pRsp->epSet.port[0], 9022); + EXPECT_EQ(pRsp->epSet.port[0], 9031); EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost"); connId = pRsp->connId; @@ -249,7 +249,7 @@ TEST_F(MndTestProfile, 07_KillQueryMsg) { EXPECT_EQ(pRsp->epSet.inUse, 0); EXPECT_EQ(pRsp->epSet.numOfEps, 1); - EXPECT_EQ(pRsp->epSet.port[0], 9022); + EXPECT_EQ(pRsp->epSet.port[0], 9031); EXPECT_STREQ(pRsp->epSet.fqdn[0], "localhost"); } } diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index bc0c3a4f64..a84e473d9f 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -5,8 +5,9 @@ ./test.sh -f sim/user/basic1.sim # ---- db -./test.sh -f sim/db/basic1.sim -./test.sh -f sim/db/error1.sim +./test.sh -f sim/db/basic1.sim +./test.sh -f sim/db/error6.sim +./test.sh -f sim/db/error1.sim # ---- table ./test.sh -f sim/table/basic1.sim diff --git a/tests/script/sim/db/basic6.sim b/tests/script/sim/db/basic6.sim index 8e7d4f5ec1..a688b4c2f3 100644 --- a/tests/script/sim/db/basic6.sim +++ b/tests/script/sim/db/basic6.sim @@ -93,6 +93,10 @@ sql create database $db sql use $db sql create table st (ts timestamp, i int) tags (j int) sql create table $tb using st tags(1) + +return +system sh/exec.sh -n dnode1 -s stop -x SIGINT + sql show tables if $rows != 1 then return -1 From 530ffb6cbc6fc4b14a07c96388bc6e7fcf22c1fc Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 11 Jan 2022 03:46:05 -0800 Subject: [PATCH 24/57] minor changes --- tests/script/jenkins/basic.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index a84e473d9f..bb6569deb0 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -6,7 +6,7 @@ # ---- db ./test.sh -f sim/db/basic1.sim -./test.sh -f sim/db/error6.sim +./test.sh -f sim/db/basic6.sim ./test.sh -f sim/db/error1.sim # ---- table From b94500077956a634a8928d89e30d81f0b0856df7 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 11 Jan 2022 23:06:36 +0800 Subject: [PATCH 25/57] add libuv --- cmake/libuv_CMakeLists.txt.in | 8 +- source/libs/transport/CMakeLists.txt | 17 +- source/libs/transport/inc/rpcCache.h | 2 + source/libs/transport/inc/rpcHead.h | 60 +- source/libs/transport/inc/rpcTcp.h | 13 +- source/libs/transport/inc/transportInt.h | 7 +- source/libs/transport/src/rpcCache.c | 53 +- source/libs/transport/src/rpcMain.c | 883 +++++++++++++---------- source/libs/transport/src/rpcTcp.c | 140 ++-- source/libs/transport/src/rpcUdp.c | 81 ++- source/libs/transport/src/transport.c | 2 +- 11 files changed, 720 insertions(+), 546 deletions(-) diff --git a/cmake/libuv_CMakeLists.txt.in b/cmake/libuv_CMakeLists.txt.in index ed406e089e..6c7ab79ed2 100644 --- a/cmake/libuv_CMakeLists.txt.in +++ b/cmake/libuv_CMakeLists.txt.in @@ -4,9 +4,9 @@ ExternalProject_Add(libuv GIT_REPOSITORY https://github.com/libuv/libuv.git GIT_TAG v1.42.0 SOURCE_DIR "${CMAKE_CONTRIB_DIR}/libuv" - BINARY_DIR "" - CONFIGURE_COMMAND "" - BUILD_COMMAND "" + BINARY_DIR "${CMAKE_CONTRIB_DIR}/libuv" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" INSTALL_COMMAND "" TEST_COMMAND "" - ) \ No newline at end of file + ) diff --git a/source/libs/transport/CMakeLists.txt b/source/libs/transport/CMakeLists.txt index 98a380dc8f..c4eeef5df2 100644 --- a/source/libs/transport/CMakeLists.txt +++ b/source/libs/transport/CMakeLists.txt @@ -12,4 +12,19 @@ target_link_libraries( PUBLIC os PUBLIC util PUBLIC common -) \ No newline at end of file +) +if (${BUILD_WITH_UV}) + target_include_directories( + transport + PUBLIC "${CMAKE_SOURCE_DIR}/contrib/libuv/include" + ) + +#LINK_DIRECTORIES("${CMAKE_SOURCE_DIR}/debug/contrib/libuv") + target_link_libraries( + transport + PUBLIC uv_a + ) + add_definitions(-DUSE_UV) +endif(${BUILD_WITH_UV}) + + diff --git a/source/libs/transport/inc/rpcCache.h b/source/libs/transport/inc/rpcCache.h index 3a996aab7c..5148f5e23c 100644 --- a/source/libs/transport/inc/rpcCache.h +++ b/source/libs/transport/inc/rpcCache.h @@ -16,6 +16,8 @@ #ifndef TDENGINE_RPC_CACHE_H #define TDENGINE_RPC_CACHE_H +#include + #ifdef __cplusplus extern "C" { #endif diff --git a/source/libs/transport/inc/rpcHead.h b/source/libs/transport/inc/rpcHead.h index 6e98bbd563..7317d84af1 100644 --- a/source/libs/transport/inc/rpcHead.h +++ b/source/libs/transport/inc/rpcHead.h @@ -16,52 +16,57 @@ #ifndef TDENGINE_RPCHEAD_H #define TDENGINE_RPCHEAD_H +#include #ifdef __cplusplus extern "C" { #endif -#define RPC_CONN_TCP 2 +#ifdef USE_UV + +#else + +#define RPC_CONN_TCP 2 extern int tsRpcOverhead; typedef struct { - void *msg; + void* msg; int msgLen; - uint32_t ip; + uint32_t ip; uint16_t port; int connType; - void *shandle; - void *thandle; - void *chandle; + void* shandle; + void* thandle; + void* chandle; } SRecvInfo; #pragma pack(push, 1) typedef struct { - char version:4; // RPC version - char comp:4; // compression algorithm, 0:no compression 1:lz4 - char resflag:2; // reserved bits - char spi:3; // security parameter index - char encrypt:3; // encrypt algorithm, 0: no encryption - uint16_t tranId; // transcation ID - uint32_t linkUid; // for unique connection ID assigned by client - uint64_t ahandle; // ahandle assigned by client - uint32_t sourceId; // source ID, an index for connection list - uint32_t destId; // destination ID, an index for connection list - uint32_t destIp; // destination IP address, for NAT scenario - char user[TSDB_UNI_LEN]; // user ID - uint16_t port; // for UDP only, port may be changed - char empty[1]; // reserved - uint16_t msgType; // message type - int32_t msgLen; // message length including the header iteslf + char version : 4; // RPC version + char comp : 4; // compression algorithm, 0:no compression 1:lz4 + char resflag : 2; // reserved bits + char spi : 3; // security parameter index + char encrypt : 3; // encrypt algorithm, 0: no encryption + uint16_t tranId; // transcation ID + uint32_t linkUid; // for unique connection ID assigned by client + uint64_t ahandle; // ahandle assigned by client + uint32_t sourceId; // source ID, an index for connection list + uint32_t destId; // destination ID, an index for connection list + uint32_t destIp; // destination IP address, for NAT scenario + char user[TSDB_UNI_LEN]; // user ID + uint16_t port; // for UDP only, port may be changed + char empty[1]; // reserved + uint16_t msgType; // message type + int32_t msgLen; // message length including the header iteslf uint32_t msgVer; - int32_t code; // code in response message - uint8_t content[0]; // message body starts from here + int32_t code; // code in response message + uint8_t content[0]; // message body starts from here } SRpcHead; typedef struct { - int32_t reserved; - int32_t contLen; + int32_t reserved; + int32_t contLen; } SRpcComp; typedef struct { @@ -70,11 +75,10 @@ typedef struct { } SRpcDigest; #pragma pack(pop) - +#endif #ifdef __cplusplus } #endif #endif // TDENGINE_RPCHEAD_H - diff --git a/source/libs/transport/inc/rpcTcp.h b/source/libs/transport/inc/rpcTcp.h index 6ef8fc2d92..5e5c43a1db 100644 --- a/source/libs/transport/inc/rpcTcp.h +++ b/source/libs/transport/inc/rpcTcp.h @@ -15,23 +15,28 @@ #ifndef _rpc_tcp_header_ #define _rpc_tcp_header_ +#include #ifdef __cplusplus extern "C" { #endif +#ifdef USE_UV +#else void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle); -void taosStopTcpServer(void *param); -void taosCleanUpTcpServer(void *param); +void taosStopTcpServer(void *param); +void taosCleanUpTcpServer(void *param); void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int num, void *fp, void *shandle); -void taosStopTcpClient(void *chandle); -void taosCleanUpTcpClient(void *chandle); +void taosStopTcpClient(void *chandle); +void taosCleanUpTcpClient(void *chandle); void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port); void taosCloseTcpConnection(void *chandle); int taosSendTcpData(uint32_t ip, uint16_t port, void *data, int len, void *chandle); +#endif + #ifdef __cplusplus } #endif diff --git a/source/libs/transport/inc/transportInt.h b/source/libs/transport/inc/transportInt.h index 24a2e0ef89..9809f7ee1a 100644 --- a/source/libs/transport/inc/transportInt.h +++ b/source/libs/transport/inc/transportInt.h @@ -20,8 +20,13 @@ extern "C" { #endif +#ifdef USE_UV + +#else + +#endif #ifdef __cplusplus } #endif -#endif /*_TD_TRANSPORT_INT_H_*/ \ No newline at end of file +#endif /*_TD_TRANSPORT_INT_H_*/ diff --git a/source/libs/transport/src/rpcCache.c b/source/libs/transport/src/rpcCache.c index 7de7aa341b..40767d2ba5 100644 --- a/source/libs/transport/src/rpcCache.c +++ b/source/libs/transport/src/rpcCache.c @@ -13,37 +13,40 @@ * along with this program. If not, see . */ +#include "rpcCache.h" #include "os.h" +#include "rpcLog.h" #include "taosdef.h" #include "tglobal.h" #include "tmempool.h" #include "ttimer.h" #include "tutil.h" -#include "rpcLog.h" -#include "rpcCache.h" +#ifdef USE_UV + +#else typedef struct SConnHash { char fqdn[TSDB_FQDN_LEN]; uint16_t port; char connType; struct SConnHash *prev; struct SConnHash *next; - void *data; + void * data; uint64_t time; } SConnHash; typedef struct { - SConnHash **connHashList; + SConnHash ** connHashList; mpool_h connHashMemPool; int maxSessions; int total; int * count; int64_t keepTimer; pthread_mutex_t mutex; - void (*cleanFp)(void *); - void *tmrCtrl; - void *pTimer; - int64_t *lockedBy; + void (*cleanFp)(void *); + void * tmrCtrl; + void * pTimer; + int64_t *lockedBy; } SConnCache; static int rpcHashConn(void *handle, char *fqdn, uint16_t port, int8_t connType); @@ -122,7 +125,7 @@ void rpcAddConnIntoCache(void *handle, void *data, char *fqdn, uint16_t port, in uint64_t time = taosGetTimestampMs(); pCache = (SConnCache *)handle; - assert(pCache); + assert(pCache); assert(data); hash = rpcHashConn(pCache, fqdn, port, connType); @@ -134,7 +137,7 @@ void rpcAddConnIntoCache(void *handle, void *data, char *fqdn, uint16_t port, in pNode->prev = NULL; pNode->time = time; - rpcLockCache(pCache->lockedBy+hash); + rpcLockCache(pCache->lockedBy + hash); pNode->next = pCache->connHashList[hash]; if (pCache->connHashList[hash] != NULL) (pCache->connHashList[hash])->prev = pNode; @@ -143,10 +146,11 @@ void rpcAddConnIntoCache(void *handle, void *data, char *fqdn, uint16_t port, in pCache->count[hash]++; rpcRemoveExpiredNodes(pCache, pNode->next, hash, time); - rpcUnlockCache(pCache->lockedBy+hash); + rpcUnlockCache(pCache->lockedBy + hash); pCache->total++; - // tTrace("%p %s:%hu:%d:%d:%p added into cache, connections:%d", data, fqdn, port, connType, hash, pNode, pCache->count[hash]); + // tTrace("%p %s:%hu:%d:%d:%p added into cache, connections:%d", data, fqdn, port, connType, hash, pNode, + // pCache->count[hash]); return; } @@ -158,12 +162,12 @@ void *rpcGetConnFromCache(void *handle, char *fqdn, uint16_t port, int8_t connTy void * pData = NULL; pCache = (SConnCache *)handle; - assert(pCache); + assert(pCache); uint64_t time = taosGetTimestampMs(); hash = rpcHashConn(pCache, fqdn, port, connType); - rpcLockCache(pCache->lockedBy+hash); + rpcLockCache(pCache->lockedBy + hash); pNode = pCache->connHashList[hash]; while (pNode) { @@ -197,12 +201,14 @@ void *rpcGetConnFromCache(void *handle, char *fqdn, uint16_t port, int8_t connTy pCache->count[hash]--; } - rpcUnlockCache(pCache->lockedBy+hash); + rpcUnlockCache(pCache->lockedBy + hash); if (pData) { - //tTrace("%p %s:%hu:%d:%d:%p retrieved from cache, connections:%d", pData, fqdn, port, connType, hash, pNode, pCache->count[hash]); + // tTrace("%p %s:%hu:%d:%d:%p retrieved from cache, connections:%d", pData, fqdn, port, connType, hash, pNode, + // pCache->count[hash]); } else { - //tTrace("%s:%hu:%d:%d failed to retrieve conn from cache, connections:%d", fqdn, port, connType, hash, pCache->count[hash]); + // tTrace("%s:%hu:%d:%d failed to retrieve conn from cache, connections:%d", fqdn, port, connType, hash, + // pCache->count[hash]); } return pData; @@ -221,10 +227,10 @@ static void rpcCleanConnCache(void *handle, void *tmrId) { uint64_t time = taosGetTimestampMs(); for (hash = 0; hash < pCache->maxSessions; ++hash) { - rpcLockCache(pCache->lockedBy+hash); + rpcLockCache(pCache->lockedBy + hash); pNode = pCache->connHashList[hash]; rpcRemoveExpiredNodes(pCache, pNode, hash, time); - rpcUnlockCache(pCache->lockedBy+hash); + rpcUnlockCache(pCache->lockedBy + hash); } // tTrace("timer, total connections in cache:%d", pCache->total); @@ -233,7 +239,7 @@ static void rpcCleanConnCache(void *handle, void *tmrId) { } static void rpcRemoveExpiredNodes(SConnCache *pCache, SConnHash *pNode, int hash, uint64_t time) { - if (pNode == NULL || (time < pCache->keepTimer + pNode->time) ) return; + if (pNode == NULL || (time < pCache->keepTimer + pNode->time)) return; SConnHash *pPrev = pNode->prev, *pNext; @@ -242,7 +248,8 @@ static void rpcRemoveExpiredNodes(SConnCache *pCache, SConnHash *pNode, int hash pNext = pNode->next; pCache->total--; pCache->count[hash]--; - //tTrace("%p %s:%hu:%d:%d:%p removed from cache, connections:%d", pNode->data, pNode->fqdn, pNode->port, pNode->connType, hash, pNode, + // tTrace("%p %s:%hu:%d:%d:%p removed from cache, connections:%d", pNode->data, pNode->fqdn, pNode->port, + // pNode->connType, hash, pNode, // pCache->count[hash]); taosMemPoolFree(pCache->connHashMemPool, (char *)pNode); pNode = pNext; @@ -257,7 +264,7 @@ static void rpcRemoveExpiredNodes(SConnCache *pCache, SConnHash *pNode, int hash static int rpcHashConn(void *handle, char *fqdn, uint16_t port, int8_t connType) { SConnCache *pCache = (SConnCache *)handle; int hash = 0; - char *temp = fqdn; + char * temp = fqdn; while (*temp) { hash += *temp; @@ -288,4 +295,4 @@ static void rpcUnlockCache(int64_t *lockedBy) { assert(false); } } - +#endif diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index d58ea63c4f..9bba0dca0a 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -13,117 +13,265 @@ * along with this program. If not, see . */ +#include +#include "lz4.h" #include "os.h" +#include "rpcCache.h" +#include "rpcHead.h" +#include "rpcLog.h" +#include "rpcTcp.h" +#include "rpcUdp.h" +#include "taoserror.h" +#include "tglobal.h" +#include "thash.h" #include "tidpool.h" #include "tmd5.h" #include "tmempool.h" +#include "tmsg.h" +#include "tref.h" +#include "trpc.h" #include "ttimer.h" #include "tutil.h" -#include "lz4.h" -#include "tref.h" -#include "taoserror.h" -#include "tglobal.h" -#include "tmsg.h" -#include "trpc.h" -#include "thash.h" -#include "rpcLog.h" -#include "rpcUdp.h" -#include "rpcCache.h" -#include "rpcTcp.h" -#include "rpcHead.h" -#define RPC_MSG_OVERHEAD (sizeof(SRpcReqContext) + sizeof(SRpcHead) + sizeof(SRpcDigest)) -#define rpcHeadFromCont(cont) ((SRpcHead *) ((char*)cont - sizeof(SRpcHead))) +#ifdef USE_UV + +#define container_of(ptr, type, member) ((type*)((char*)(ptr)-offsetof(type, member))) + +int32_t rpcInit() { return -1; } +void rpcCleanup() { return; }; +void* rpcOpen(const SRpcInit* pRpc) { return NULL; } +void rpcClose(void* arg) { return; } +void* rpcMallocCont(int contLen) { return NULL; } +void rpcFreeCont(void* cont) { return; } +void* rpcReallocCont(void* ptr, int contLen) { return NULL; } + +void rpcSendRequest(void* thandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* rid) { return; } + +void rpcSendResponse(const SRpcMsg* pMsg) {} + +void rpcSendRedirectRsp(void* pConn, const SEpSet* pEpSet) {} +int rpcGetConnInfo(void* thandle, SRpcConnInfo* pInfo) { return -1; } +void rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pReq, SRpcMsg* pRsp) { return; } +int rpcReportProgress(void* pConn, char* pCont, int contLen) { return -1; } +void rpcCancelRequest(int64_t rid) { return; } + +typedef struct SThreadObj { + pthread_t thread; + uv_pipe_t* pipe; + uv_loop_t* loop; + uv_async_t* workerAsync; // + int fd; +} SThreadObj; + +typedef struct SServerObj { + uv_tcp_t server; + uv_loop_t* loop; + int workerIdx; + int numOfThread; + SThreadObj** pThreadObj; + uv_pipe_t** pipe; +} SServerObj; + +typedef struct SConnCtx { + uv_tcp_t* pClient; + uv_timer_t* pTimer; + uv_async_t* pWorkerAsync; + int ref; +} SConnCtx; + +void allocBuffer(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { + buf->base = malloc(suggested_size); + buf->len = suggested_size; +} + +void onTimeout(uv_timer_t* handle) { + // opt + tDebug("time out"); +} +void onRead(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf) { + // opt + tDebug("data already was read on a stream"); +} + +void onWrite(uv_write_t* req, int status) { + // opt + if (req) tDebug("data already was written on stream"); +} + +static void workerAsyncCB(uv_async_t* handle) { + // opt + SThreadObj* pObj = container_of(handle, SThreadObj, workerAsync); +} +void onAccept(uv_stream_t* stream, int status) { + if (status == -1) { + return; + } + SServerObj* pObj = container_of(stream, SServerObj, server); + tDebug("new conntion accepted by main server, dispatch to one worker thread"); + + uv_tcp_t* cli = (uv_tcp_t*)malloc(sizeof(uv_tcp_t)); + uv_tcp_init(pObj->loop, cli); + if (uv_accept(stream, (uv_stream_t*)cli) == 0) { + uv_write_t* wr = (uv_write_t*)malloc(sizeof(uv_write_t)); + + uv_buf_t buf = uv_buf_init("a", 1); + // despatch to worker thread + pObj->workerIdx = (pObj->workerIdx + 1) % pObj->numOfThread; + uv_write2(wr, (uv_stream_t*)&(pObj->pipe[pObj->workerIdx][0]), &buf, 1, (uv_stream_t*)cli, onWrite); + } else { + uv_close((uv_handle_t*)cli, NULL); + } +} +void onConnection(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf) { + if (nread < 0) { + if (nread != UV_EOF) { + tError("read error %s", uv_err_name(nread)); + } + // TODO(log other failure reason) + uv_close((uv_handle_t*)q, NULL); + return; + } + SThreadObj* pObj = (SThreadObj*)container_of(q, struct SThreadObj, pipe); + + uv_pipe_t* pipe = (uv_pipe_t*)q; + if (!uv_pipe_pending_count(pipe)) { + tError("No pending count"); + return; + } + uv_handle_type pending = uv_pipe_pending_type(pipe); + assert(pending == UV_TCP); + + SConnCtx* pConn = malloc(sizeof(SConnCtx)); + /* init conn timer*/ + pConn->pTimer = malloc(sizeof(uv_timer_t)); + uv_timer_init(pObj->loop, pConn->pTimer); + + pConn->pClient = (uv_tcp_t*)malloc(sizeof(uv_tcp_t)); + pConn->pWorkerAsync = pObj->workerAsync; // thread safty + uv_tcp_init(pObj->loop, pConn->pClient); + + if (uv_accept(q, (uv_stream_t*)(pConn->pClient)) == 0) { + uv_os_fd_t fd; + uv_fileno((const uv_handle_t*)pConn->pClient, &fd); + tDebug("new connection created: %d", fd); + uv_timer_start(pConn->pTimer, onTimeout, 10, 0); + uv_read_start((uv_stream_t*)(pConn->pClient), allocBuffer, onRead); + } else { + uv_timer_stop(pConn->pTimer); + free(pConn->pTimer); + uv_close((uv_handle_t*)pConn->pClient, NULL); + free(pConn->pClient); + free(pConn); + } +} + +void* workerThread(void* arg) { + SThreadObj* pObj = (SThreadObj*)arg; + int fd = pObj->fd; + pObj->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); + uv_loop_init(pObj->loop); + + uv_pipe_init(pObj->loop, pObj->pipe, 1); + uv_pipe_open(pObj->pipe, fd); + + pObj->workerAsync = malloc(sizeof(uv_async_t)); + uv_async_init(pObj->loop, pObj->workerAsync, workerAsyncCB); + uv_read_start((uv_stream_t*)pObj->pipe, allocBuffer, onConnection); +} +#else + +#define RPC_MSG_OVERHEAD (sizeof(SRpcReqContext) + sizeof(SRpcHead) + sizeof(SRpcDigest)) +#define rpcHeadFromCont(cont) ((SRpcHead*)((char*)cont - sizeof(SRpcHead))) #define rpcContFromHead(msg) (msg + sizeof(SRpcHead)) #define rpcMsgLenFromCont(contLen) (contLen + sizeof(SRpcHead)) #define rpcContLenFromMsg(msgLen) (msgLen - sizeof(SRpcHead)) #define rpcIsReq(type) (type & 1U) typedef struct { - int sessions; // number of sessions allowed - int numOfThreads; // number of threads to process incoming messages - int idleTime; // milliseconds; + int sessions; // number of sessions allowed + int numOfThreads; // number of threads to process incoming messages + int idleTime; // milliseconds; uint16_t localPort; int8_t connType; - int index; // for UDP server only, round robin for multiple threads + int index; // for UDP server only, round robin for multiple threads char label[TSDB_LABEL_LEN]; - char user[TSDB_UNI_LEN]; // meter ID - char spi; // security parameter index - char encrypt; // encrypt algorithm - char secret[TSDB_PASSWORD_LEN]; // secret for the link - char ckey[TSDB_PASSWORD_LEN]; // ciphering key + char user[TSDB_UNI_LEN]; // meter ID + char spi; // security parameter index + char encrypt; // encrypt algorithm + char secret[TSDB_PASSWORD_LEN]; // secret for the link + char ckey[TSDB_PASSWORD_LEN]; // ciphering key - void (*cfp)(void *parent, SRpcMsg *, SEpSet *); - int (*afp)(void *parent, char *user, char *spi, char *encrypt, char *secret, char *ckey); + void (*cfp)(void *parent, SRpcMsg *, SEpSet *); + int (*afp)(void *parent, char *user, char *spi, char *encrypt, char *secret, char *ckey); - int32_t refCount; - void *parent; - void *idPool; // handle to ID pool - void *tmrCtrl; // handle to timer - SHashObj *hash; // handle returned by hash utility - void *tcphandle;// returned handle from TCP initialization - void *udphandle;// returned handle from UDP initialization - void *pCache; // connection cache + int32_t refCount; + void * parent; + void * idPool; // handle to ID pool + void * tmrCtrl; // handle to timer + SHashObj * hash; // handle returned by hash utility + void * tcphandle; // returned handle from TCP initialization + void * udphandle; // returned handle from UDP initialization + void * pCache; // connection cache pthread_mutex_t mutex; struct SRpcConn *connList; // connection list } SRpcInfo; typedef struct { - SRpcInfo *pRpc; // associated SRpcInfo - SEpSet epSet; // ip list provided by app - void *ahandle; // handle provided by app - struct SRpcConn *pConn; // pConn allocated - tmsg_t msgType; // message type - uint8_t *pCont; // content provided by app - int32_t contLen; // content length - int32_t code; // error code - int16_t numOfTry; // number of try for different servers - int8_t oldInUse; // server EP inUse passed by app - int8_t redirect; // flag to indicate redirect - int8_t connType; // connection type - int64_t rid; // refId returned by taosAddRef - SRpcMsg *pRsp; // for synchronous API - tsem_t *pSem; // for synchronous API - SEpSet *pSet; // for synchronous API - char msg[0]; // RpcHead starts from here + SRpcInfo * pRpc; // associated SRpcInfo + SEpSet epSet; // ip list provided by app + void * ahandle; // handle provided by app + struct SRpcConn *pConn; // pConn allocated + tmsg_t msgType; // message type + uint8_t * pCont; // content provided by app + int32_t contLen; // content length + int32_t code; // error code + int16_t numOfTry; // number of try for different servers + int8_t oldInUse; // server EP inUse passed by app + int8_t redirect; // flag to indicate redirect + int8_t connType; // connection type + int64_t rid; // refId returned by taosAddRef + SRpcMsg * pRsp; // for synchronous API + tsem_t * pSem; // for synchronous API + SEpSet * pSet; // for synchronous API + char msg[0]; // RpcHead starts from here } SRpcReqContext; typedef struct SRpcConn { - char info[48];// debug info: label + pConn + ahandle - int sid; // session ID - uint32_t ownId; // own link ID - uint32_t peerId; // peer link ID - char user[TSDB_UNI_LEN]; // user ID for the link - char spi; // security parameter index - char encrypt; // encryption, 0:1 - char secret[TSDB_PASSWORD_LEN]; // secret for the link - char ckey[TSDB_PASSWORD_LEN]; // ciphering key - char secured; // if set to 1, no authentication - uint16_t localPort; // for UDP only - uint32_t linkUid; // connection unique ID assigned by client - uint32_t peerIp; // peer IP - uint16_t peerPort; // peer port - char peerFqdn[TSDB_FQDN_LEN]; // peer FQDN or ip string - uint16_t tranId; // outgoing transcation ID, for build message - uint16_t outTranId; // outgoing transcation ID - uint16_t inTranId; // transcation ID for incoming msg - tmsg_t outType; // message type for outgoing request - tmsg_t inType; // message type for incoming request - void *chandle; // handle passed by TCP/UDP connection layer - void *ahandle; // handle provided by upper app layter - int retry; // number of retry for sending request - int tretry; // total retry - void *pTimer; // retry timer to monitor the response - void *pIdleTimer; // idle timer - char *pRspMsg; // response message including header - int rspMsgLen; // response messag length - char *pReqMsg; // request message including header - int reqMsgLen; // request message length - SRpcInfo *pRpc; // the associated SRpcInfo - int8_t connType; // connection type - int64_t lockedBy; // lock for connection - SRpcReqContext *pContext; // request context + char info[48]; // debug info: label + pConn + ahandle + int sid; // session ID + uint32_t ownId; // own link ID + uint32_t peerId; // peer link ID + char user[TSDB_UNI_LEN]; // user ID for the link + char spi; // security parameter index + char encrypt; // encryption, 0:1 + char secret[TSDB_PASSWORD_LEN]; // secret for the link + char ckey[TSDB_PASSWORD_LEN]; // ciphering key + char secured; // if set to 1, no authentication + uint16_t localPort; // for UDP only + uint32_t linkUid; // connection unique ID assigned by client + uint32_t peerIp; // peer IP + uint16_t peerPort; // peer port + char peerFqdn[TSDB_FQDN_LEN]; // peer FQDN or ip string + uint16_t tranId; // outgoing transcation ID, for build message + uint16_t outTranId; // outgoing transcation ID + uint16_t inTranId; // transcation ID for incoming msg + tmsg_t outType; // message type for outgoing request + tmsg_t inType; // message type for incoming request + void * chandle; // handle passed by TCP/UDP connection layer + void * ahandle; // handle provided by upper app layter + int retry; // number of retry for sending request + int tretry; // total retry + void * pTimer; // retry timer to monitor the response + void * pIdleTimer; // idle timer + char * pRspMsg; // response message including header + int rspMsgLen; // response messag length + char * pReqMsg; // request message including header + int reqMsgLen; // request message length + SRpcInfo * pRpc; // the associated SRpcInfo + int8_t connType; // connection type + int64_t lockedBy; // lock for connection + SRpcReqContext *pContext; // request context } SRpcConn; static pthread_once_t tsRpcInitOnce = PTHREAD_ONCE_INIT; @@ -137,41 +285,29 @@ int tsRpcOverhead; static int tsRpcRefId = -1; static int32_t tsRpcNum = 0; -//static pthread_once_t tsRpcInit = PTHREAD_ONCE_INIT; +// static pthread_once_t tsRpcInit = PTHREAD_ONCE_INIT; // server:0 client:1 tcp:2 udp:0 -#define RPC_CONN_UDPS 0 -#define RPC_CONN_UDPC 1 -#define RPC_CONN_TCPS 2 -#define RPC_CONN_TCPC 3 +#define RPC_CONN_UDPS 0 +#define RPC_CONN_UDPC 1 +#define RPC_CONN_TCPS 2 +#define RPC_CONN_TCPC 3 void *(*taosInitConn[])(uint32_t ip, uint16_t port, char *label, int threads, void *fp, void *shandle) = { - taosInitUdpConnection, - taosInitUdpConnection, - taosInitTcpServer, - taosInitTcpClient -}; + taosInitUdpConnection, taosInitUdpConnection, taosInitTcpServer, taosInitTcpClient}; -void (*taosCleanUpConn[])(void *thandle) = { - taosCleanUpUdpConnection, - taosCleanUpUdpConnection, - taosCleanUpTcpServer, - taosCleanUpTcpClient -}; +void (*taosCleanUpConn[])(void *thandle) = {taosCleanUpUdpConnection, taosCleanUpUdpConnection, taosCleanUpTcpServer, + taosCleanUpTcpClient}; void (*taosStopConn[])(void *thandle) = { - taosStopUdpConnection, - taosStopUdpConnection, + taosStopUdpConnection, + taosStopUdpConnection, taosStopTcpServer, taosStopTcpClient, }; int (*taosSendData[])(uint32_t ip, uint16_t port, void *data, int len, void *chandle) = { - taosSendUdpData, - taosSendUdpData, - taosSendTcpData, - taosSendTcpData -}; + taosSendUdpData, taosSendUdpData, taosSendTcpData, taosSendTcpData}; void *(*taosOpenConn[])(void *shandle, void *thandle, uint32_t ip, uint16_t port) = { taosOpenUdpConnection, @@ -180,12 +316,7 @@ void *(*taosOpenConn[])(void *shandle, void *thandle, uint32_t ip, uint16_t port taosOpenTcpClientConnection, }; -void (*taosCloseConn[])(void *chandle) = { - NULL, - NULL, - taosCloseTcpConnection, - taosCloseTcpConnection -}; +void (*taosCloseConn[])(void *chandle) = {NULL, NULL, taosCloseTcpConnection, taosCloseTcpConnection}; static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerFqdn, uint16_t peerPort, int8_t connType); static void rpcCloseConn(void *thandle); @@ -194,11 +325,11 @@ static SRpcConn *rpcAllocateClientConn(SRpcInfo *pRpc); static SRpcConn *rpcAllocateServerConn(SRpcInfo *pRpc, SRecvInfo *pRecv); static SRpcConn *rpcGetConnObj(SRpcInfo *pRpc, int sid, SRecvInfo *pRecv); -static void rpcSendReqToServer(SRpcInfo *pRpc, SRpcReqContext *pContext); -static void rpcSendQuickRsp(SRpcConn *pConn, int32_t code); -static void rpcSendErrorMsgToPeer(SRecvInfo *pRecv, int32_t code); -static void rpcSendMsgToPeer(SRpcConn *pConn, void *data, int dataLen); -static void rpcSendReqHead(SRpcConn *pConn); +static void rpcSendReqToServer(SRpcInfo *pRpc, SRpcReqContext *pContext); +static void rpcSendQuickRsp(SRpcConn *pConn, int32_t code); +static void rpcSendErrorMsgToPeer(SRecvInfo *pRecv, int32_t code); +static void rpcSendMsgToPeer(SRpcConn *pConn, void *data, int dataLen); +static void rpcSendReqHead(SRpcConn *pConn); static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv); static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead, SRpcReqContext *pContext); @@ -207,15 +338,15 @@ static void rpcProcessRetryTimer(void *, void *); static void rpcProcessIdleTimer(void *param, void *tmrId); static void rpcProcessProgressTimer(void *param, void *tmrId); -static void rpcFreeMsg(void *msg); -static int32_t rpcCompressRpcMsg(char* pCont, int32_t contLen); +static void rpcFreeMsg(void *msg); +static int32_t rpcCompressRpcMsg(char *pCont, int32_t contLen); static SRpcHead *rpcDecompressRpcMsg(SRpcHead *pHead); -static int rpcAddAuthPart(SRpcConn *pConn, char *msg, int msgLen); -static int rpcCheckAuthentication(SRpcConn *pConn, char *msg, int msgLen); -static void rpcLockConn(SRpcConn *pConn); -static void rpcUnlockConn(SRpcConn *pConn); -static void rpcAddRef(SRpcInfo *pRpc); -static void rpcDecRef(SRpcInfo *pRpc); +static int rpcAddAuthPart(SRpcConn *pConn, char *msg, int msgLen); +static int rpcCheckAuthentication(SRpcConn *pConn, char *msg, int msgLen); +static void rpcLockConn(SRpcConn *pConn); +static void rpcUnlockConn(SRpcConn *pConn); +static void rpcAddRef(SRpcInfo *pRpc); +static void rpcDecRef(SRpcInfo *pRpc); static void rpcFree(void *p) { tTrace("free mem: %p", p); @@ -240,26 +371,26 @@ void rpcCleanup(void) { taosCloseRef(tsRpcRefId); tsRpcRefId = -1; } - + void *rpcOpen(const SRpcInit *pInit) { SRpcInfo *pRpc; - //pthread_once(&tsRpcInit, rpcInit); + // pthread_once(&tsRpcInit, rpcInit); pRpc = (SRpcInfo *)calloc(1, sizeof(SRpcInfo)); if (pRpc == NULL) return NULL; - if(pInit->label) tstrncpy(pRpc->label, pInit->label, sizeof(pRpc->label)); + if (pInit->label) tstrncpy(pRpc->label, pInit->label, sizeof(pRpc->label)); pRpc->connType = pInit->connType; if (pRpc->connType == TAOS_CONN_CLIENT) { pRpc->numOfThreads = pInit->numOfThreads; } else { - pRpc->numOfThreads = pInit->numOfThreads>TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS:pInit->numOfThreads; + pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; } pRpc->idleTime = pInit->idleTime; pRpc->localPort = pInit->localPort; pRpc->afp = pInit->afp; - pRpc->sessions = pInit->sessions+1; + pRpc->sessions = pInit->sessions + 1; if (pInit->user) tstrncpy(pRpc->user, pInit->user, sizeof(pRpc->user)); if (pInit->secret) memcpy(pRpc->secret, pInit->secret, sizeof(pRpc->secret)); if (pInit->ckey) tstrncpy(pRpc->ckey, pInit->ckey, sizeof(pRpc->ckey)); @@ -279,14 +410,14 @@ void *rpcOpen(const SRpcInit *pInit) { return NULL; } - pRpc->idPool = taosInitIdPool(pRpc->sessions-1); + pRpc->idPool = taosInitIdPool(pRpc->sessions - 1); if (pRpc->idPool == NULL) { tError("%s failed to init ID pool", pRpc->label); rpcClose(pRpc); return NULL; } - pRpc->tmrCtrl = taosTmrInit(pRpc->sessions*2 + 1, 50, 10000, pRpc->label); + pRpc->tmrCtrl = taosTmrInit(pRpc->sessions * 2 + 1, 50, 10000, pRpc->label); if (pRpc->tmrCtrl == NULL) { tError("%s failed to init timers", pRpc->label); rpcClose(pRpc); @@ -301,8 +432,8 @@ void *rpcOpen(const SRpcInit *pInit) { return NULL; } } else { - pRpc->pCache = rpcOpenConnCache(pRpc->sessions, rpcCloseConn, pRpc->tmrCtrl, pRpc->idleTime * 20); - if ( pRpc->pCache == NULL ) { + pRpc->pCache = rpcOpenConnCache(pRpc->sessions, rpcCloseConn, pRpc->tmrCtrl, pRpc->idleTime * 20); + if (pRpc->pCache == NULL) { tError("%s failed to init connection cache", pRpc->label); rpcClose(pRpc); return NULL; @@ -311,10 +442,10 @@ void *rpcOpen(const SRpcInit *pInit) { pthread_mutex_init(&pRpc->mutex, NULL); - pRpc->tcphandle = (*taosInitConn[pRpc->connType|RPC_CONN_TCP])(0, pRpc->localPort, pRpc->label, - pRpc->numOfThreads, rpcProcessMsgFromPeer, pRpc); - pRpc->udphandle = (*taosInitConn[pRpc->connType])(0, pRpc->localPort, pRpc->label, - pRpc->numOfThreads, rpcProcessMsgFromPeer, pRpc); + pRpc->tcphandle = (*taosInitConn[pRpc->connType | RPC_CONN_TCP])(0, pRpc->localPort, pRpc->label, pRpc->numOfThreads, + rpcProcessMsgFromPeer, pRpc); + pRpc->udphandle = + (*taosInitConn[pRpc->connType])(0, pRpc->localPort, pRpc->label, pRpc->numOfThreads, rpcProcessMsgFromPeer, pRpc); if (pRpc->tcphandle == NULL || pRpc->udphandle == NULL) { tError("%s failed to init network, port:%d", pRpc->label, pRpc->localPort); @@ -334,7 +465,7 @@ void rpcClose(void *param) { (*taosStopConn[pRpc->connType | RPC_CONN_TCP])(pRpc->tcphandle); (*taosStopConn[pRpc->connType])(pRpc->udphandle); - // close all connections + // close all connections for (int i = 0; i < pRpc->sessions; ++i) { if (pRpc->connList && pRpc->connList[i].user[0]) { rpcCloseConn((void *)(pRpc->connList + i)); @@ -375,8 +506,8 @@ void *rpcReallocCont(void *ptr, int contLen) { if (ptr == NULL) return rpcMallocCont(contLen); char *start = ((char *)ptr) - sizeof(SRpcReqContext) - sizeof(SRpcHead); - if (contLen == 0 ) { - free(start); + if (contLen == 0) { + free(start); return NULL; } @@ -385,17 +516,17 @@ void *rpcReallocCont(void *ptr, int contLen) { if (start == NULL) { tError("failed to realloc cont, size:%d", size); return NULL; - } + } return start + sizeof(SRpcReqContext) + sizeof(SRpcHead); } void rpcSendRequest(void *shandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *pRid) { - SRpcInfo *pRpc = (SRpcInfo *)shandle; + SRpcInfo * pRpc = (SRpcInfo *)shandle; SRpcReqContext *pContext; int contLen = rpcCompressRpcMsg(pMsg->pCont, pMsg->contLen); - pContext = (SRpcReqContext *) ((char*)pMsg->pCont-sizeof(SRpcHead)-sizeof(SRpcReqContext)); + pContext = (SRpcReqContext *)((char *)pMsg->pCont - sizeof(SRpcHead) - sizeof(SRpcReqContext)); pContext->ahandle = pMsg->ahandle; pContext->pRpc = (SRpcInfo *)shandle; pContext->epSet = *pEpSet; @@ -404,16 +535,15 @@ void rpcSendRequest(void *shandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t pContext->msgType = pMsg->msgType; pContext->oldInUse = pEpSet->inUse; - pContext->connType = RPC_CONN_UDPC; - if (contLen > tsRpcMaxUdpSize || tsRpcForceTcp ) pContext->connType = RPC_CONN_TCPC; + pContext->connType = RPC_CONN_UDPC; + if (contLen > tsRpcMaxUdpSize || tsRpcForceTcp) pContext->connType = RPC_CONN_TCPC; - // connection type is application specific. + // connection type is application specific. // for TDengine, all the query, show commands shall have TCP connection tmsg_t type = pMsg->msgType; - if (type == TDMT_VND_QUERY || type == TDMT_MND_SHOW_RETRIEVE - || type == TDMT_VND_FETCH || type == TDMT_MND_VGROUP_LIST - || type == TDMT_VND_TABLES_META || type == TDMT_VND_TABLE_META - || type == TDMT_MND_SHOW || type == TDMT_MND_STATUS || type == TDMT_VND_ALTER_TABLE) + if (type == TDMT_VND_QUERY || type == TDMT_MND_SHOW_RETRIEVE || type == TDMT_VND_FETCH || + type == TDMT_MND_VGROUP_LIST || type == TDMT_VND_TABLES_META || type == TDMT_VND_TABLE_META || + type == TDMT_MND_SHOW || type == TDMT_MND_STATUS || type == TDMT_VND_ALTER_TABLE) pContext->connType = RPC_CONN_TCPC; pContext->rid = taosAddRef(tsRpcRefId, pContext); @@ -425,26 +555,26 @@ void rpcSendRequest(void *shandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t void rpcSendResponse(const SRpcMsg *pRsp) { if (pRsp->handle == NULL) return; - int msgLen = 0; - SRpcConn *pConn = (SRpcConn *)pRsp->handle; - SRpcMsg rpcMsg = *pRsp; - SRpcMsg *pMsg = &rpcMsg; - SRpcInfo *pRpc = pConn->pRpc; + int msgLen = 0; + SRpcConn *pConn = (SRpcConn *)pRsp->handle; + SRpcMsg rpcMsg = *pRsp; + SRpcMsg * pMsg = &rpcMsg; + SRpcInfo *pRpc = pConn->pRpc; - if ( pMsg->pCont == NULL ) { + if (pMsg->pCont == NULL) { pMsg->pCont = rpcMallocCont(0); pMsg->contLen = 0; } - SRpcHead *pHead = rpcHeadFromCont(pMsg->pCont); - char *msg = (char *)pHead; + SRpcHead *pHead = rpcHeadFromCont(pMsg->pCont); + char * msg = (char *)pHead; pMsg->contLen = rpcCompressRpcMsg(pMsg->pCont, pMsg->contLen); msgLen = rpcMsgLenFromCont(pMsg->contLen); rpcLockConn(pConn); - if ( pConn->inType == 0 || pConn->user[0] == 0 ) { + if (pConn->inType == 0 || pConn->user[0] == 0) { tError("%s, connection is already released, rsp wont be sent", pConn->info); rpcUnlockConn(pConn); rpcFreeCont(pMsg->pCont); @@ -454,7 +584,7 @@ void rpcSendResponse(const SRpcMsg *pRsp) { // set msg header pHead->version = 1; - pHead->msgType = pConn->inType+1; + pHead->msgType = pConn->inType + 1; pHead->spi = pConn->spi; pHead->encrypt = pConn->encrypt; pHead->tranId = pConn->inTranId; @@ -463,13 +593,13 @@ void rpcSendResponse(const SRpcMsg *pRsp) { pHead->linkUid = pConn->linkUid; pHead->port = htons(pConn->localPort); pHead->code = htonl(pMsg->code); - pHead->ahandle = (uint64_t) pConn->ahandle; - + pHead->ahandle = (uint64_t)pConn->ahandle; + // set pConn parameters pConn->inType = 0; // response message is released until new response is sent - rpcFreeMsg(pConn->pRspMsg); + rpcFreeMsg(pConn->pRspMsg); pConn->pRspMsg = msg; pConn->rspMsgLen = msgLen; if (pMsg->code == TSDB_CODE_RPC_ACTION_IN_PROGRESS) pConn->inTranId--; @@ -482,23 +612,22 @@ void rpcSendResponse(const SRpcMsg *pRsp) { rpcSendMsgToPeer(pConn, msg, msgLen); // if not set to secured, set it expcet NOT_READY case, since client wont treat it as secured - if (pConn->secured == 0 && pMsg->code != TSDB_CODE_RPC_NOT_READY) - pConn->secured = 1; // connection shall be secured + if (pConn->secured == 0 && pMsg->code != TSDB_CODE_RPC_NOT_READY) pConn->secured = 1; // connection shall be secured if (pConn->pReqMsg) rpcFreeCont(pConn->pReqMsg); pConn->pReqMsg = NULL; pConn->reqMsgLen = 0; rpcUnlockConn(pConn); - rpcDecRef(pRpc); // decrease the referene count + rpcDecRef(pRpc); // decrease the referene count return; } void rpcSendRedirectRsp(void *thandle, const SEpSet *pEpSet) { - SRpcMsg rpcMsg; + SRpcMsg rpcMsg; memset(&rpcMsg, 0, sizeof(rpcMsg)); - + rpcMsg.contLen = sizeof(SEpSet); rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); if (rpcMsg.pCont == NULL) return; @@ -514,24 +643,24 @@ void rpcSendRedirectRsp(void *thandle, const SEpSet *pEpSet) { } int rpcGetConnInfo(void *thandle, SRpcConnInfo *pInfo) { - SRpcConn *pConn = (SRpcConn *)thandle; + SRpcConn *pConn = (SRpcConn *)thandle; if (pConn->user[0] == 0) return -1; pInfo->clientIp = pConn->peerIp; pInfo->clientPort = pConn->peerPort; // pInfo->serverIp = pConn->destIp; - + tstrncpy(pInfo->user, pConn->user, sizeof(pInfo->user)); return 0; } void rpcSendRecv(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { SRpcReqContext *pContext; - pContext = (SRpcReqContext *) ((char*)pMsg->pCont-sizeof(SRpcHead)-sizeof(SRpcReqContext)); + pContext = (SRpcReqContext *)((char *)pMsg->pCont - sizeof(SRpcHead) - sizeof(SRpcReqContext)); memset(pRsp, 0, sizeof(SRpcMsg)); - - tsem_t sem; + + tsem_t sem; tsem_init(&sem, 0, 0); pContext->pSem = &sem; pContext->pRsp = pRsp; @@ -548,13 +677,13 @@ void rpcSendRecv(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { // this API is used by server app to keep an APP context in case connection is broken int rpcReportProgress(void *handle, char *pCont, int contLen) { SRpcConn *pConn = (SRpcConn *)handle; - int code = 0; + int code = 0; rpcLockConn(pConn); if (pConn->user[0]) { // pReqMsg and reqMsgLen is re-used to store the context from app server - pConn->pReqMsg = pCont; + pConn->pReqMsg = pCont; pConn->reqMsgLen = contLen; } else { tDebug("%s, rpc connection is already released", pConn->info); @@ -567,7 +696,6 @@ int rpcReportProgress(void *handle, char *pCont, int contLen) { } void rpcCancelRequest(int64_t rid) { - SRpcReqContext *pContext = taosAcquireRef(tsRpcRefId, rid); if (pContext == NULL) return; @@ -577,7 +705,7 @@ void rpcCancelRequest(int64_t rid) { } static void rpcFreeMsg(void *msg) { - if ( msg ) { + if (msg) { char *temp = (char *)msg - sizeof(SRpcReqContext); free(temp); tTrace("free mem: %p", temp); @@ -589,14 +717,14 @@ static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerFqdn, uint16_t peerPort, uint32_t peerIp = taosGetIpv4FromFqdn(peerFqdn); if (peerIp == 0xFFFFFFFF) { - tError("%s, failed to resolve FQDN:%s", pRpc->label, peerFqdn); - terrno = TSDB_CODE_RPC_FQDN_ERROR; + tError("%s, failed to resolve FQDN:%s", pRpc->label, peerFqdn); + terrno = TSDB_CODE_RPC_FQDN_ERROR; return NULL; } - pConn = rpcAllocateClientConn(pRpc); + pConn = rpcAllocateClientConn(pRpc); - if (pConn) { + if (pConn) { tstrncpy(pConn->peerFqdn, peerFqdn, sizeof(pConn->peerFqdn)); pConn->peerIp = peerIp; pConn->peerPort = peerPort; @@ -604,7 +732,7 @@ static SRpcConn *rpcOpenConn(SRpcInfo *pRpc, char *peerFqdn, uint16_t peerPort, pConn->connType = connType; if (taosOpenConn[connType]) { - void *shandle = (connType & RPC_CONN_TCP)? pRpc->tcphandle:pRpc->udphandle; + void *shandle = (connType & RPC_CONN_TCP) ? pRpc->tcphandle : pRpc->udphandle; pConn->chandle = (*taosOpenConn[connType])(shandle, pConn, pConn->peerIp, pConn->peerPort); if (pConn->chandle == NULL) { tError("failed to connect to:%s:%d", taosIpStr(pConn->peerIp), pConn->peerPort); @@ -629,13 +757,14 @@ static void rpcReleaseConn(SRpcConn *pConn) { taosTmrStopA(&pConn->pTimer); taosTmrStopA(&pConn->pIdleTimer); - if ( pRpc->connType == TAOS_CONN_SERVER) { - char hashstr[40] = {0}; - size_t size = snprintf(hashstr, sizeof(hashstr), "%x:%x:%x:%d", pConn->peerIp, pConn->linkUid, pConn->peerId, pConn->connType); + if (pRpc->connType == TAOS_CONN_SERVER) { + char hashstr[40] = {0}; + size_t size = snprintf(hashstr, sizeof(hashstr), "%x:%x:%x:%d", pConn->peerIp, pConn->linkUid, pConn->peerId, + pConn->connType); taosHashRemove(pRpc->hash, hashstr, size); - rpcFreeMsg(pConn->pRspMsg); // it may have a response msg saved, but not request msg + rpcFreeMsg(pConn->pRspMsg); // it may have a response msg saved, but not request msg pConn->pRspMsg = NULL; - + // if server has ever reported progress, free content if (pConn->pReqMsg) rpcFreeCont(pConn->pReqMsg); // do not use rpcFreeMsg } else { @@ -643,17 +772,17 @@ static void rpcReleaseConn(SRpcConn *pConn) { if (pConn->outType && pConn->pReqMsg) { SRpcReqContext *pContext = pConn->pContext; if (pContext) { - if (pContext->pRsp) { - // for synchronous API, post semaphore to unblock app + if (pContext->pRsp) { + // for synchronous API, post semaphore to unblock app pContext->pRsp->code = TSDB_CODE_RPC_APP_ERROR; pContext->pRsp->pCont = NULL; pContext->pRsp->contLen = 0; tsem_post(pContext->pSem); } - pContext->pConn = NULL; + pContext->pConn = NULL; taosRemoveRef(tsRpcRefId, pContext->rid); } else { - assert(0); + assert(0); } } } @@ -682,8 +811,7 @@ static void rpcCloseConn(void *thandle) { rpcLockConn(pConn); - if (pConn->user[0]) - rpcReleaseConn(pConn); + if (pConn->user[0]) rpcReleaseConn(pConn); rpcUnlockConn(pConn); } @@ -717,8 +845,9 @@ static SRpcConn *rpcAllocateServerConn(SRpcInfo *pRpc, SRecvInfo *pRecv) { char hashstr[40] = {0}; SRpcHead *pHead = (SRpcHead *)pRecv->msg; - size_t size = snprintf(hashstr, sizeof(hashstr), "%x:%x:%x:%d", pRecv->ip, pHead->linkUid, pHead->sourceId, pRecv->connType); - + size_t size = + snprintf(hashstr, sizeof(hashstr), "%x:%x:%x:%d", pRecv->ip, pHead->linkUid, pHead->sourceId, pRecv->connType); + // check if it is already allocated SRpcConn **ppConn = (SRpcConn **)(taosHashGet(pRpc->hash, hashstr, size)); if (ppConn) pConn = *ppConn; @@ -767,22 +896,23 @@ static SRpcConn *rpcAllocateServerConn(SRpcInfo *pRpc, SRecvInfo *pRecv) { } taosHashPut(pRpc->hash, hashstr, size, (char *)&pConn, POINTER_BYTES); - tDebug("%s %p server connection is allocated, uid:0x%x sid:%d key:%s", pRpc->label, pConn, pConn->linkUid, sid, hashstr); + tDebug("%s %p server connection is allocated, uid:0x%x sid:%d key:%s", pRpc->label, pConn, pConn->linkUid, sid, + hashstr); } return pConn; } static SRpcConn *rpcGetConnObj(SRpcInfo *pRpc, int sid, SRecvInfo *pRecv) { - SRpcConn *pConn = NULL; + SRpcConn *pConn = NULL; SRpcHead *pHead = (SRpcHead *)pRecv->msg; if (sid) { pConn = pRpc->connList + sid; if (pConn->user[0] == 0) pConn = NULL; - } + } - if (pConn == NULL) { + if (pConn == NULL) { if (pRpc->connType == TAOS_CONN_SERVER) { pConn = rpcAllocateServerConn(pRpc, pRecv); } else { @@ -805,12 +935,13 @@ static SRpcConn *rpcGetConnObj(SRpcInfo *pRpc, int sid, SRecvInfo *pRecv) { static SRpcConn *rpcSetupConnToServer(SRpcReqContext *pContext) { SRpcConn *pConn; SRpcInfo *pRpc = pContext->pRpc; - SEpSet *pEpSet = &pContext->epSet; + SEpSet * pEpSet = &pContext->epSet; - pConn = rpcGetConnFromCache(pRpc->pCache, pEpSet->fqdn[pEpSet->inUse], pEpSet->port[pEpSet->inUse], pContext->connType); - if ( pConn == NULL || pConn->user[0] == 0) { + pConn = + rpcGetConnFromCache(pRpc->pCache, pEpSet->fqdn[pEpSet->inUse], pEpSet->port[pEpSet->inUse], pContext->connType); + if (pConn == NULL || pConn->user[0] == 0) { pConn = rpcOpenConn(pRpc, pEpSet->fqdn[pEpSet->inUse], pEpSet->port[pEpSet->inUse], pContext->connType); - } + } if (pConn) { pConn->tretry = 0; @@ -825,55 +956,52 @@ static SRpcConn *rpcSetupConnToServer(SRpcReqContext *pContext) { } static int rpcProcessReqHead(SRpcConn *pConn, SRpcHead *pHead) { - - if (pConn->peerId == 0) { - pConn->peerId = pHead->sourceId; - } else { - if (pConn->peerId != pHead->sourceId) { - tDebug("%s, source Id is changed, old:0x%08x new:0x%08x", pConn->info, - pConn->peerId, pHead->sourceId); - return TSDB_CODE_RPC_INVALID_VALUE; - } + if (pConn->peerId == 0) { + pConn->peerId = pHead->sourceId; + } else { + if (pConn->peerId != pHead->sourceId) { + tDebug("%s, source Id is changed, old:0x%08x new:0x%08x", pConn->info, pConn->peerId, pHead->sourceId); + return TSDB_CODE_RPC_INVALID_VALUE; } + } - if (pConn->inTranId == pHead->tranId) { - if (pConn->inType == pHead->msgType) { - if (pHead->code == 0) { - tDebug("%s, %s is retransmitted", pConn->info, TMSG_INFO(pHead->msgType)); - rpcSendQuickRsp(pConn, TSDB_CODE_RPC_ACTION_IN_PROGRESS); - } else { - // do nothing, it is heart beat from client - } - } else if (pConn->inType == 0) { - tDebug("%s, %s is already processed, tranId:%d", pConn->info, TMSG_INFO(pHead->msgType), pConn->inTranId); - rpcSendMsgToPeer(pConn, pConn->pRspMsg, pConn->rspMsgLen); // resend the response + if (pConn->inTranId == pHead->tranId) { + if (pConn->inType == pHead->msgType) { + if (pHead->code == 0) { + tDebug("%s, %s is retransmitted", pConn->info, TMSG_INFO(pHead->msgType)); + rpcSendQuickRsp(pConn, TSDB_CODE_RPC_ACTION_IN_PROGRESS); } else { - tDebug("%s, mismatched message %s and tranId", pConn->info, TMSG_INFO(pHead->msgType)); + // do nothing, it is heart beat from client } - - // do not reply any message - return TSDB_CODE_RPC_ALREADY_PROCESSED; + } else if (pConn->inType == 0) { + tDebug("%s, %s is already processed, tranId:%d", pConn->info, TMSG_INFO(pHead->msgType), pConn->inTranId); + rpcSendMsgToPeer(pConn, pConn->pRspMsg, pConn->rspMsgLen); // resend the response + } else { + tDebug("%s, mismatched message %s and tranId", pConn->info, TMSG_INFO(pHead->msgType)); } - if (pConn->inType != 0) { - tDebug("%s, last session is not finished, inTranId:%d tranId:%d", pConn->info, - pConn->inTranId, pHead->tranId); - return TSDB_CODE_RPC_LAST_SESSION_NOT_FINISHED; - } + // do not reply any message + return TSDB_CODE_RPC_ALREADY_PROCESSED; + } - if (rpcContLenFromMsg(pHead->msgLen) <= 0) { - tDebug("%s, message body is empty, ignore", pConn->info); - return TSDB_CODE_RPC_APP_ERROR; - } + if (pConn->inType != 0) { + tDebug("%s, last session is not finished, inTranId:%d tranId:%d", pConn->info, pConn->inTranId, pHead->tranId); + return TSDB_CODE_RPC_LAST_SESSION_NOT_FINISHED; + } - pConn->inTranId = pHead->tranId; - pConn->inType = pHead->msgType; + if (rpcContLenFromMsg(pHead->msgLen) <= 0) { + tDebug("%s, message body is empty, ignore", pConn->info); + return TSDB_CODE_RPC_APP_ERROR; + } - // start the progress timer to monitor the response from server app - if (pConn->connType != RPC_CONN_TCPS) - pConn->pTimer = taosTmrStart(rpcProcessProgressTimer, tsProgressTimer, pConn, pConn->pRpc->tmrCtrl); - - return 0; + pConn->inTranId = pHead->tranId; + pConn->inType = pHead->msgType; + + // start the progress timer to monitor the response from server app + if (pConn->connType != RPC_CONN_TCPS) + pConn->pTimer = taosTmrStart(rpcProcessProgressTimer, tsProgressTimer, pConn, pConn->pRpc->tmrCtrl); + + return 0; } static int rpcProcessRspHead(SRpcConn *pConn, SRpcHead *pHead) { @@ -898,7 +1026,7 @@ static int rpcProcessRspHead(SRpcConn *pConn, SRpcHead *pHead) { if (pHead->code == TSDB_CODE_RPC_AUTH_REQUIRED && pRpc->spi) { tDebug("%s, authentication shall be restarted", pConn->info); pConn->secured = 0; - rpcSendMsgToPeer(pConn, pConn->pReqMsg, pConn->reqMsgLen); + rpcSendMsgToPeer(pConn, pConn->pReqMsg, pConn->reqMsgLen); if (pConn->connType != RPC_CONN_TCPC) pConn->pTimer = taosTmrStart(rpcProcessRetryTimer, tsRpcTimer, pConn, pRpc->tmrCtrl); return TSDB_CODE_RPC_ALREADY_PROCESSED; @@ -908,7 +1036,7 @@ static int rpcProcessRspHead(SRpcConn *pConn, SRpcHead *pHead) { tDebug("%s, mismatched linkUid, link shall be restarted", pConn->info); pConn->secured = 0; ((SRpcHead *)pConn->pReqMsg)->destId = 0; - rpcSendMsgToPeer(pConn, pConn->pReqMsg, pConn->reqMsgLen); + rpcSendMsgToPeer(pConn, pConn->pReqMsg, pConn->reqMsgLen); if (pConn->connType != RPC_CONN_TCPC) pConn->pTimer = taosTmrStart(rpcProcessRetryTimer, tsRpcTimer, pConn, pRpc->tmrCtrl); return TSDB_CODE_RPC_ALREADY_PROCESSED; @@ -934,25 +1062,25 @@ static int rpcProcessRspHead(SRpcConn *pConn, SRpcHead *pHead) { pConn->reqMsgLen = 0; SRpcReqContext *pContext = pConn->pContext; - if (pHead->code == TSDB_CODE_RPC_REDIRECT) { + if (pHead->code == TSDB_CODE_RPC_REDIRECT) { if (rpcContLenFromMsg(pHead->msgLen) < sizeof(SEpSet)) { // if EpSet is not included in the msg, treat it as NOT_READY - pHead->code = TSDB_CODE_RPC_NOT_READY; + pHead->code = TSDB_CODE_RPC_NOT_READY; } else { pContext->redirect++; if (pContext->redirect > TSDB_MAX_REPLICA) { - pHead->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; + pHead->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; tWarn("%s, too many redirects, quit", pConn->info); } } - } + } return TSDB_CODE_SUCCESS; } static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv, SRpcReqContext **ppContext) { - int32_t sid; - SRpcConn *pConn = NULL; + int32_t sid; + SRpcConn *pConn = NULL; SRpcHead *pHead = (SRpcHead *)pRecv->msg; @@ -961,25 +1089,29 @@ static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv, SRpcReqCont if (TMSG_INDEX(pHead->msgType) >= TDMT_MAX || TMSG_INDEX(pHead->msgType) <= 0) { tDebug("%s sid:%d, invalid message type:%d", pRpc->label, sid, pHead->msgType); - terrno = TSDB_CODE_RPC_INVALID_MSG_TYPE; return NULL; + terrno = TSDB_CODE_RPC_INVALID_MSG_TYPE; + return NULL; } if (sid < 0 || sid >= pRpc->sessions) { - tDebug("%s sid:%d, sid is out of range, max sid:%d, %s discarded", pRpc->label, sid, - pRpc->sessions, TMSG_INFO(pHead->msgType)); - terrno = TSDB_CODE_RPC_INVALID_SESSION_ID; return NULL; + tDebug("%s sid:%d, sid is out of range, max sid:%d, %s discarded", pRpc->label, sid, pRpc->sessions, + TMSG_INFO(pHead->msgType)); + terrno = TSDB_CODE_RPC_INVALID_SESSION_ID; + return NULL; } if (rpcIsReq(pHead->msgType) && htonl(pHead->msgVer) != tsVersion >> 8) { - tDebug("%s sid:%d, invalid client version:%x/%x %s", pRpc->label, sid, htonl(pHead->msgVer), tsVersion, TMSG_INFO(pHead->msgType)); - terrno = TSDB_CODE_RPC_INVALID_VERSION; return NULL; + tDebug("%s sid:%d, invalid client version:%x/%x %s", pRpc->label, sid, htonl(pHead->msgVer), tsVersion, + TMSG_INFO(pHead->msgType)); + terrno = TSDB_CODE_RPC_INVALID_VERSION; + return NULL; } pConn = rpcGetConnObj(pRpc, sid, pRecv); if (pConn == NULL) { - tDebug("%s %p, failed to get connection obj(%s)", pRpc->label, (void *)pHead->ahandle, tstrerror(terrno)); + tDebug("%s %p, failed to get connection obj(%s)", pRpc->label, (void *)pHead->ahandle, tstrerror(terrno)); return NULL; - } + } rpcLockConn(pConn); @@ -990,9 +1122,9 @@ static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv, SRpcReqCont sid = pConn->sid; if (pConn->chandle == NULL) pConn->chandle = pRecv->chandle; - pConn->peerIp = pRecv->ip; + pConn->peerIp = pRecv->ip; pConn->peerPort = pRecv->port; - if (pHead->port) pConn->peerPort = htons(pHead->port); + if (pHead->port) pConn->peerPort = htons(pHead->port); terrno = rpcCheckAuthentication(pConn, (char *)pHead, pRecv->msgLen); @@ -1004,16 +1136,16 @@ static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv, SRpcReqCont // decrypt here } - if ( rpcIsReq(pHead->msgType) ) { + if (rpcIsReq(pHead->msgType)) { pConn->connType = pRecv->connType; terrno = rpcProcessReqHead(pConn, pHead); // stop idle timer - taosTmrStopA(&pConn->pIdleTimer); + taosTmrStopA(&pConn->pIdleTimer); - // client shall send the request within tsRpcTime again for UDP, double it + // client shall send the request within tsRpcTime again for UDP, double it if (pConn->connType != RPC_CONN_TCPS) - pConn->pIdleTimer = taosTmrStart(rpcProcessIdleTimer, tsRpcTimer*2, pConn, pRpc->tmrCtrl); + pConn->pIdleTimer = taosTmrStart(rpcProcessIdleTimer, tsRpcTimer * 2, pConn, pRpc->tmrCtrl); } else { terrno = rpcProcessRspHead(pConn, pHead); *ppContext = pConn->pContext; @@ -1026,9 +1158,9 @@ static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv, SRpcReqCont } static void doRpcReportBrokenLinkToServer(void *param, void *id) { - SRpcMsg *pRpcMsg = (SRpcMsg *)(param); - SRpcConn *pConn = (SRpcConn *)(pRpcMsg->handle); - SRpcInfo *pRpc = pConn->pRpc; + SRpcMsg * pRpcMsg = (SRpcMsg *)(param); + SRpcConn *pConn = (SRpcConn *)(pRpcMsg->handle); + SRpcInfo *pRpc = pConn->pRpc; (*(pRpc->cfp))(pRpc->parent, pRpcMsg, NULL); free(pRpcMsg); } @@ -1041,12 +1173,12 @@ static void rpcReportBrokenLinkToServer(SRpcConn *pConn) { tDebug("%s, notify the server app, connection is gone", pConn->info); SRpcMsg *rpcMsg = malloc(sizeof(SRpcMsg)); - rpcMsg->pCont = pConn->pReqMsg; // pReqMsg is re-used to store the APP context from server - rpcMsg->contLen = pConn->reqMsgLen; // reqMsgLen is re-used to store the APP context length + rpcMsg->pCont = pConn->pReqMsg; // pReqMsg is re-used to store the APP context from server + rpcMsg->contLen = pConn->reqMsgLen; // reqMsgLen is re-used to store the APP context length rpcMsg->ahandle = pConn->ahandle; rpcMsg->handle = pConn; rpcMsg->msgType = pConn->inType; - rpcMsg->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; + rpcMsg->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; pConn->pReqMsg = NULL; pConn->reqMsgLen = 0; if (pRpc->cfp) { @@ -1070,22 +1202,22 @@ static void rpcProcessBrokenLink(SRpcConn *pConn) { pConn->pReqMsg = NULL; taosTmrStart(rpcProcessConnError, 0, pContext, pRpc->tmrCtrl); } - - if (pConn->inType) rpcReportBrokenLinkToServer(pConn); + + if (pConn->inType) rpcReportBrokenLinkToServer(pConn); rpcReleaseConn(pConn); rpcUnlockConn(pConn); } static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) { - SRpcHead *pHead = (SRpcHead *)pRecv->msg; - SRpcInfo *pRpc = (SRpcInfo *)pRecv->shandle; - SRpcConn *pConn = (SRpcConn *)pRecv->thandle; + SRpcHead *pHead = (SRpcHead *)pRecv->msg; + SRpcInfo *pRpc = (SRpcInfo *)pRecv->shandle; + SRpcConn *pConn = (SRpcConn *)pRecv->thandle; tDump(pRecv->msg, pRecv->msgLen); // underlying UDP layer does not know it is server or client - pRecv->connType = pRecv->connType | pRpc->connType; + pRecv->connType = pRecv->connType | pRpc->connType; if (pRecv->msg == NULL) { rpcProcessBrokenLink(pConn); @@ -1100,62 +1232,62 @@ static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) { taosIpPort2String(pRecv->ip, pRecv->port, ipstr); if (TMSG_INDEX(pHead->msgType) >= 1 && TMSG_INDEX(pHead->msgType) < TDMT_MAX) { - tDebug("%s %p %p, %s received from %s, parse code:0x%x len:%d sig:0x%08x:0x%08x:%d code:0x%x", pRpc->label, - pConn, (void *)pHead->ahandle, TMSG_INFO(pHead->msgType), ipstr, terrno, pRecv->msgLen, - pHead->sourceId, pHead->destId, pHead->tranId, pHead->code); + tDebug("%s %p %p, %s received from %s, parse code:0x%x len:%d sig:0x%08x:0x%08x:%d code:0x%x", pRpc->label, pConn, + (void *)pHead->ahandle, TMSG_INFO(pHead->msgType), ipstr, terrno, pRecv->msgLen, pHead->sourceId, + pHead->destId, pHead->tranId, pHead->code); } else { - tDebug("%s %p %p, %d received from %s, parse code:0x%x len:%d sig:0x%08x:0x%08x:%d code:0x%x", pRpc->label, - pConn, (void *)pHead->ahandle, pHead->msgType, ipstr, terrno, pRecv->msgLen, - pHead->sourceId, pHead->destId, pHead->tranId, pHead->code); + tDebug("%s %p %p, %d received from %s, parse code:0x%x len:%d sig:0x%08x:0x%08x:%d code:0x%x", pRpc->label, pConn, + (void *)pHead->ahandle, pHead->msgType, ipstr, terrno, pRecv->msgLen, pHead->sourceId, pHead->destId, + pHead->tranId, pHead->code); } int32_t code = terrno; if (code != TSDB_CODE_RPC_ALREADY_PROCESSED) { - if (code != 0) { // parsing error + if (code != 0) { // parsing error if (rpcIsReq(pHead->msgType)) { rpcSendErrorMsgToPeer(pRecv, code); if (code == TSDB_CODE_RPC_INVALID_TIME_STAMP || code == TSDB_CODE_RPC_AUTH_FAILURE) { rpcCloseConn(pConn); } if (TMSG_INDEX(pHead->msgType) + 1 > 1 && TMSG_INDEX(pHead->msgType) + 1 < TDMT_MAX) { - tDebug("%s %p %p, %s is sent with error code:0x%x", pRpc->label, pConn, (void *)pHead->ahandle, TMSG_INFO(pHead->msgType+1), code); + tDebug("%s %p %p, %s is sent with error code:0x%x", pRpc->label, pConn, (void *)pHead->ahandle, + TMSG_INFO(pHead->msgType + 1), code); } else { - tError("%s %p %p, %s is sent with error code:0x%x", pRpc->label, pConn, (void *)pHead->ahandle, TMSG_INFO(pHead->msgType), code); + tError("%s %p %p, %s is sent with error code:0x%x", pRpc->label, pConn, (void *)pHead->ahandle, + TMSG_INFO(pHead->msgType), code); } - } - } else { // msg is passed to app only parsing is ok + } + } else { // msg is passed to app only parsing is ok rpcProcessIncomingMsg(pConn, pHead, pContext); } } - if (code) rpcFreeMsg(pRecv->msg); // parsing failed, msg shall be freed + if (code) rpcFreeMsg(pRecv->msg); // parsing failed, msg shall be freed return pConn; } static void rpcNotifyClient(SRpcReqContext *pContext, SRpcMsg *pMsg) { - SRpcInfo *pRpc = pContext->pRpc; + SRpcInfo *pRpc = pContext->pRpc; pContext->pConn = NULL; - if (pContext->pRsp) { + if (pContext->pRsp) { // for synchronous API memcpy(pContext->pSet, &pContext->epSet, sizeof(SEpSet)); memcpy(pContext->pRsp, pMsg, sizeof(SRpcMsg)); tsem_post(pContext->pSem); } else { - // for asynchronous API + // for asynchronous API SEpSet *pEpSet = NULL; - if (pContext->epSet.inUse != pContext->oldInUse || pContext->redirect) - pEpSet = &pContext->epSet; + if (pContext->epSet.inUse != pContext->oldInUse || pContext->redirect) pEpSet = &pContext->epSet; (*pRpc->cfp)(pRpc->parent, pMsg, pEpSet); } // free the request message - taosRemoveRef(tsRpcRefId, pContext->rid); + taosRemoveRef(tsRpcRefId, pContext->rid); } static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead, SRpcReqContext *pContext) { - SRpcInfo *pRpc = pConn->pRpc; SRpcMsg rpcMsg; @@ -1180,14 +1312,15 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead, SRpcReqConte // for UDP, port may be changed by server, the port in epSet shall be used for cache if (pHead->code != TSDB_CODE_RPC_TOO_SLOW) { - rpcAddConnIntoCache(pRpc->pCache, pConn, pConn->peerFqdn, pContext->epSet.port[pContext->epSet.inUse], pConn->connType); + rpcAddConnIntoCache(pRpc->pCache, pConn, pConn->peerFqdn, pContext->epSet.port[pContext->epSet.inUse], + pConn->connType); } else { rpcCloseConn(pConn); } if (pHead->code == TSDB_CODE_RPC_REDIRECT) { pContext->numOfTry = 0; - SEpSet *pEpSet = (SEpSet*)pHead->content; + SEpSet *pEpSet = (SEpSet *)pHead->content; if (pEpSet->numOfEps > 0) { memcpy(&pContext->epSet, pHead->content, sizeof(pContext->epSet)); tDebug("%s, redirect is received, numOfEps:%d inUse:%d", pConn->info, pContext->epSet.numOfEps, @@ -1200,7 +1333,8 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead, SRpcReqConte } rpcSendReqToServer(pRpc, pContext); rpcFreeCont(rpcMsg.pCont); - } else if (pHead->code == TSDB_CODE_RPC_NOT_READY || pHead->code == TSDB_CODE_APP_NOT_READY || pHead->code == TSDB_CODE_DND_OFFLINE) { + } else if (pHead->code == TSDB_CODE_RPC_NOT_READY || pHead->code == TSDB_CODE_APP_NOT_READY || + pHead->code == TSDB_CODE_DND_OFFLINE) { pContext->code = pHead->code; rpcProcessConnError(pContext, NULL); rpcFreeCont(rpcMsg.pCont); @@ -1211,14 +1345,14 @@ static void rpcProcessIncomingMsg(SRpcConn *pConn, SRpcHead *pHead, SRpcReqConte } static void rpcSendQuickRsp(SRpcConn *pConn, int32_t code) { - char msg[RPC_MSG_OVERHEAD]; - SRpcHead *pHead; + char msg[RPC_MSG_OVERHEAD]; + SRpcHead *pHead; // set msg header memset(msg, 0, sizeof(SRpcHead)); pHead = (SRpcHead *)msg; pHead->version = 1; - pHead->msgType = pConn->inType+1; + pHead->msgType = pConn->inType + 1; pHead->spi = pConn->spi; pHead->encrypt = 0; pHead->tranId = pConn->inTranId; @@ -1230,12 +1364,12 @@ static void rpcSendQuickRsp(SRpcConn *pConn, int32_t code) { pHead->code = htonl(code); rpcSendMsgToPeer(pConn, msg, sizeof(SRpcHead)); - pConn->secured = 1; // connection shall be secured + pConn->secured = 1; // connection shall be secured } static void rpcSendReqHead(SRpcConn *pConn) { - char msg[RPC_MSG_OVERHEAD]; - SRpcHead *pHead; + char msg[RPC_MSG_OVERHEAD]; + SRpcHead *pHead; // set msg header memset(msg, 0, sizeof(SRpcHead)); @@ -1257,10 +1391,10 @@ static void rpcSendReqHead(SRpcConn *pConn) { } static void rpcSendErrorMsgToPeer(SRecvInfo *pRecv, int32_t code) { - SRpcHead *pRecvHead, *pReplyHead; - char msg[sizeof(SRpcHead) + sizeof(SRpcDigest) + sizeof(uint32_t) ]; - uint32_t timeStamp; - int msgLen; + SRpcHead *pRecvHead, *pReplyHead; + char msg[sizeof(SRpcHead) + sizeof(SRpcDigest) + sizeof(uint32_t)]; + uint32_t timeStamp; + int msgLen; pRecvHead = (SRpcHead *)pRecv->msg; pReplyHead = (SRpcHead *)msg; @@ -1290,14 +1424,14 @@ static void rpcSendErrorMsgToPeer(SRecvInfo *pRecv, int32_t code) { pReplyHead->msgLen = (int32_t)htonl((uint32_t)msgLen); (*taosSendData[pRecv->connType])(pRecv->ip, pRecv->port, msg, msgLen, pRecv->chandle); - return; + return; } static void rpcSendReqToServer(SRpcInfo *pRpc, SRpcReqContext *pContext) { - SRpcHead *pHead = rpcHeadFromCont(pContext->pCont); - char *msg = (char *)pHead; - int msgLen = rpcMsgLenFromCont(pContext->contLen); - tmsg_t msgType = pContext->msgType; + SRpcHead *pHead = rpcHeadFromCont(pContext->pCont); + char * msg = (char *)pHead; + int msgLen = rpcMsgLenFromCont(pContext->contLen); + tmsg_t msgType = pContext->msgType; pContext->numOfTry++; SRpcConn *pConn = rpcSetupConnToServer(pContext); @@ -1311,13 +1445,13 @@ static void rpcSendReqToServer(SRpcInfo *pRpc, SRpcReqContext *pContext) { pConn->ahandle = pContext->ahandle; rpcLockConn(pConn); - // set the message header + // set the message header pHead->version = 1; pHead->msgVer = htonl(tsVersion >> 8); pHead->msgType = msgType; pHead->encrypt = 0; pConn->tranId++; - if ( pConn->tranId == 0 ) pConn->tranId++; + if (pConn->tranId == 0) pConn->tranId++; pHead->tranId = pConn->tranId; pHead->sourceId = pConn->ownId; pHead->destId = pConn->peerId; @@ -1341,45 +1475,43 @@ static void rpcSendReqToServer(SRpcInfo *pRpc, SRpcReqContext *pContext) { } static void rpcSendMsgToPeer(SRpcConn *pConn, void *msg, int msgLen) { - int writtenLen = 0; - SRpcHead *pHead = (SRpcHead *)msg; + int writtenLen = 0; + SRpcHead *pHead = (SRpcHead *)msg; msgLen = rpcAddAuthPart(pConn, msg, msgLen); - if ( rpcIsReq(pHead->msgType)) { - tDebug("%s, %s is sent to %s:%hu, len:%d sig:0x%08x:0x%08x:%d", - pConn->info, TMSG_INFO(pHead->msgType), pConn->peerFqdn, pConn->peerPort, - msgLen, pHead->sourceId, pHead->destId, pHead->tranId); + if (rpcIsReq(pHead->msgType)) { + tDebug("%s, %s is sent to %s:%hu, len:%d sig:0x%08x:0x%08x:%d", pConn->info, TMSG_INFO(pHead->msgType), + pConn->peerFqdn, pConn->peerPort, msgLen, pHead->sourceId, pHead->destId, pHead->tranId); } else { - if (pHead->code == 0) pConn->secured = 1; // for success response, set link as secured - tDebug("%s, %s is sent to 0x%x:%hu, code:0x%x len:%d sig:0x%08x:0x%08x:%d", - pConn->info, TMSG_INFO(pHead->msgType), pConn->peerIp, pConn->peerPort, - htonl(pHead->code), msgLen, pHead->sourceId, pHead->destId, pHead->tranId); + if (pHead->code == 0) pConn->secured = 1; // for success response, set link as secured + tDebug("%s, %s is sent to 0x%x:%hu, code:0x%x len:%d sig:0x%08x:0x%08x:%d", pConn->info, TMSG_INFO(pHead->msgType), + pConn->peerIp, pConn->peerPort, htonl(pHead->code), msgLen, pHead->sourceId, pHead->destId, pHead->tranId); } - //tTrace("connection type is: %d", pConn->connType); + // tTrace("connection type is: %d", pConn->connType); writtenLen = (*taosSendData[pConn->connType])(pConn->peerIp, pConn->peerPort, pHead, msgLen, pConn->chandle); if (writtenLen != msgLen) { tError("%s, failed to send, msgLen:%d written:%d, reason:%s", pConn->info, msgLen, writtenLen, strerror(errno)); } - + tDump(msg, msgLen); } static void rpcProcessConnError(void *param, void *id) { SRpcReqContext *pContext = (SRpcReqContext *)param; - SRpcInfo *pRpc = pContext->pRpc; + SRpcInfo * pRpc = pContext->pRpc; SRpcMsg rpcMsg; - + if (pRpc == NULL) { return; } - + tDebug("%s %p, connection error happens", pRpc->label, pContext->ahandle); if (pContext->numOfTry >= pContext->epSet.numOfEps || pContext->msgType == TDMT_VND_FETCH) { - rpcMsg.msgType = pContext->msgType+1; + rpcMsg.msgType = pContext->msgType + 1; rpcMsg.ahandle = pContext->ahandle; rpcMsg.code = pContext->code; rpcMsg.pCont = NULL; @@ -1387,7 +1519,7 @@ static void rpcProcessConnError(void *param, void *id) { rpcNotifyClient(pContext, &rpcMsg); } else { - // move to next IP + // move to next IP pContext->epSet.inUse++; pContext->epSet.inUse = pContext->epSet.inUse % pContext->epSet.numOfEps; rpcSendReqToServer(pRpc, pContext); @@ -1407,11 +1539,12 @@ static void rpcProcessRetryTimer(void *param, void *tmrId) { if (pConn->retry < 4) { tDebug("%s, re-send msg:%s to %s:%hu", pConn->info, TMSG_INFO(pConn->outType), pConn->peerFqdn, pConn->peerPort); - rpcSendMsgToPeer(pConn, pConn->pReqMsg, pConn->reqMsgLen); + rpcSendMsgToPeer(pConn, pConn->pReqMsg, pConn->reqMsgLen); pConn->pTimer = taosTmrStart(rpcProcessRetryTimer, tsRpcTimer, pConn, pRpc->tmrCtrl); } else { // close the connection - tDebug("%s, failed to send msg:%s to %s:%hu", pConn->info, TMSG_INFO(pConn->outType), pConn->peerFqdn, pConn->peerPort); + tDebug("%s, failed to send msg:%s to %s:%hu", pConn->info, TMSG_INFO(pConn->outType), pConn->peerFqdn, + pConn->peerPort); if (pConn->pContext) { pConn->pContext->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; pConn->pContext->pConn = NULL; @@ -1434,7 +1567,7 @@ static void rpcProcessIdleTimer(void *param, void *tmrId) { if (pConn->user[0]) { tDebug("%s, close the connection since no activity", pConn->info); - if (pConn->inType) rpcReportBrokenLinkToServer(pConn); + if (pConn->inType) rpcReportBrokenLinkToServer(pConn); rpcReleaseConn(pConn); } else { tDebug("%s, idle timer:%p not processed", pConn->info, tmrId); @@ -1460,34 +1593,34 @@ static void rpcProcessProgressTimer(void *param, void *tmrId) { rpcUnlockConn(pConn); } -static int32_t rpcCompressRpcMsg(char* pCont, int32_t contLen) { - SRpcHead *pHead = rpcHeadFromCont(pCont); - int32_t finalLen = 0; - int overhead = sizeof(SRpcComp); - +static int32_t rpcCompressRpcMsg(char *pCont, int32_t contLen) { + SRpcHead *pHead = rpcHeadFromCont(pCont); + int32_t finalLen = 0; + int overhead = sizeof(SRpcComp); + if (!NEEDTO_COMPRESSS_MSG(contLen)) { return contLen; } - - char *buf = malloc (contLen + overhead + 8); // 8 extra bytes + + char *buf = malloc(contLen + overhead + 8); // 8 extra bytes if (buf == NULL) { tError("failed to allocate memory for rpc msg compression, contLen:%d", contLen); return contLen; } - + int32_t compLen = LZ4_compress_default(pCont, buf, contLen, contLen + overhead); tDebug("compress rpc msg, before:%d, after:%d, overhead:%d", contLen, compLen, overhead); - + /* * only the compressed size is less than the value of contLen - overhead, the compression is applied * The first four bytes is set to 0, the second four bytes are utilized to keep the original length of message */ if (compLen > 0 && compLen < contLen - overhead) { SRpcComp *pComp = (SRpcComp *)pCont; - pComp->reserved = 0; - pComp->contLen = htonl(contLen); + pComp->reserved = 0; + pComp->contLen = htonl(contLen); memcpy(pCont + overhead, buf, compLen); - + pHead->comp = 1; tDebug("compress rpc msg, before:%d, after:%d", contLen, compLen); finalLen = compLen + overhead; @@ -1500,29 +1633,29 @@ static int32_t rpcCompressRpcMsg(char* pCont, int32_t contLen) { } static SRpcHead *rpcDecompressRpcMsg(SRpcHead *pHead) { - int overhead = sizeof(SRpcComp); - SRpcHead *pNewHead = NULL; - uint8_t *pCont = pHead->content; - SRpcComp *pComp = (SRpcComp *)pHead->content; + int overhead = sizeof(SRpcComp); + SRpcHead *pNewHead = NULL; + uint8_t * pCont = pHead->content; + SRpcComp *pComp = (SRpcComp *)pHead->content; if (pHead->comp) { // decompress the content assert(pComp->reserved == 0); int contLen = htonl(pComp->contLen); - + // prepare the temporary buffer to decompress message char *temp = (char *)malloc(contLen + RPC_MSG_OVERHEAD); - pNewHead = (SRpcHead *)(temp + sizeof(SRpcReqContext)); // reserve SRpcReqContext - + pNewHead = (SRpcHead *)(temp + sizeof(SRpcReqContext)); // reserve SRpcReqContext + if (pNewHead) { int compLen = rpcContLenFromMsg(pHead->msgLen) - overhead; - int origLen = LZ4_decompress_safe((char*)(pCont + overhead), (char *)pNewHead->content, compLen, contLen); + int origLen = LZ4_decompress_safe((char *)(pCont + overhead), (char *)pNewHead->content, compLen, contLen); assert(origLen == contLen); - + memcpy(pNewHead, pHead, sizeof(SRpcHead)); pNewHead->msgLen = rpcMsgLenFromCont(origLen); - rpcFreeMsg(pHead); // free the compressed message buffer - pHead = pNewHead; + rpcFreeMsg(pHead); // free the compressed message buffer + pHead = pNewHead; tTrace("decomp malloc mem:%p", temp); } else { tError("failed to allocate memory to decompress msg, contLen:%d", contLen); @@ -1534,7 +1667,7 @@ static SRpcHead *rpcDecompressRpcMsg(SRpcHead *pHead) { static int rpcAuthenticateMsg(void *pMsg, int msgLen, void *pAuth, void *pKey) { T_MD5_CTX context; - int ret = -1; + int ret = -1; tMD5Init(&context); tMD5Update(&context, (uint8_t *)pKey, TSDB_PASSWORD_LEN); @@ -1582,25 +1715,25 @@ static int rpcCheckAuthentication(SRpcConn *pConn, char *msg, int msgLen) { SRpcHead *pHead = (SRpcHead *)msg; int code = 0; - if ((pConn->secured && pHead->spi == 0) || (pHead->spi == 0 && pConn->spi == 0)){ - // secured link, or no authentication + if ((pConn->secured && pHead->spi == 0) || (pHead->spi == 0 && pConn->spi == 0)) { + // secured link, or no authentication pHead->msgLen = (int32_t)htonl((uint32_t)pHead->msgLen); // tTrace("%s, secured link, no auth is required", pConn->info); return 0; } - if ( !rpcIsReq(pHead->msgType) ) { + if (!rpcIsReq(pHead->msgType)) { // for response, if code is auth failure, it shall bypass the auth process code = htonl(pHead->code); if (code == TSDB_CODE_RPC_INVALID_TIME_STAMP || code == TSDB_CODE_RPC_AUTH_FAILURE || - code == TSDB_CODE_RPC_INVALID_VERSION || - code == TSDB_CODE_RPC_AUTH_REQUIRED || code == TSDB_CODE_MND_USER_NOT_EXIST || code == TSDB_CODE_RPC_NOT_READY) { + code == TSDB_CODE_RPC_INVALID_VERSION || code == TSDB_CODE_RPC_AUTH_REQUIRED || + code == TSDB_CODE_MND_USER_NOT_EXIST || code == TSDB_CODE_RPC_NOT_READY) { pHead->msgLen = (int32_t)htonl((uint32_t)pHead->msgLen); // tTrace("%s, dont check authentication since code is:0x%x", pConn->info, code); return 0; } } - + code = 0; if (pHead->spi == pConn->spi) { // authentication @@ -1613,12 +1746,12 @@ static int rpcCheckAuthentication(SRpcConn *pConn, char *msg, int msgLen) { tWarn("%s, time diff:%d is too big, msg discarded", pConn->info, delta); code = TSDB_CODE_RPC_INVALID_TIME_STAMP; } else { - if (rpcAuthenticateMsg(pHead, msgLen-TSDB_AUTH_LEN, pDigest->auth, pConn->secret) < 0) { + if (rpcAuthenticateMsg(pHead, msgLen - TSDB_AUTH_LEN, pDigest->auth, pConn->secret) < 0) { tDebug("%s, authentication failed, msg discarded", pConn->info); code = TSDB_CODE_RPC_AUTH_FAILURE; } else { pHead->msgLen = (int32_t)htonl((uint32_t)pHead->msgLen) - sizeof(SRpcDigest); - if ( !rpcIsReq(pHead->msgType) ) pConn->secured = 1; // link is secured for client + if (!rpcIsReq(pHead->msgType)) pConn->secured = 1; // link is secured for client // tTrace("%s, message is authenticated", pConn->info); } } @@ -1647,13 +1780,9 @@ static void rpcUnlockConn(SRpcConn *pConn) { } } -static void rpcAddRef(SRpcInfo *pRpc) -{ - atomic_add_fetch_32(&pRpc->refCount, 1); -} +static void rpcAddRef(SRpcInfo *pRpc) { atomic_add_fetch_32(&pRpc->refCount, 1); } -static void rpcDecRef(SRpcInfo *pRpc) -{ +static void rpcDecRef(SRpcInfo *pRpc) { if (atomic_sub_fetch_32(&pRpc->refCount, 1) == 0) { rpcCloseConnCache(pRpc->pCache); taosHashCleanup(pRpc->hash); @@ -1668,4 +1797,4 @@ static void rpcDecRef(SRpcInfo *pRpc) atomic_sub_fetch_32(&tsRpcNum, 1); } } - +#endif diff --git a/source/libs/transport/src/rpcTcp.c b/source/libs/transport/src/rpcTcp.c index d0710c883f..a3e1f2434b 100644 --- a/source/libs/transport/src/rpcTcp.c +++ b/source/libs/transport/src/rpcTcp.c @@ -13,24 +13,28 @@ * along with this program. If not, see . */ +#include "rpcTcp.h" +#include #include "os.h" -#include "tutil.h" +#include "rpcHead.h" +#include "rpcLog.h" #include "taosdef.h" #include "taoserror.h" -#include "rpcLog.h" -#include "rpcHead.h" -#include "rpcTcp.h" +#include "tutil.h" +#ifdef USE_UV + +#else typedef struct SFdObj { - void *signature; - SOCKET fd; // TCP socket FD - void *thandle; // handle from upper layer, like TAOS + void * signature; + SOCKET fd; // TCP socket FD + void * thandle; // handle from upper layer, like TAOS uint32_t ip; uint16_t port; - int16_t closedByApp; // 1: already closed by App + int16_t closedByApp; // 1: already closed by App struct SThreadObj *pThreadObj; - struct SFdObj *prev; - struct SFdObj *next; + struct SFdObj * prev; + struct SFdObj * next; } SFdObj; typedef struct SThreadObj { @@ -43,35 +47,35 @@ typedef struct SThreadObj { int numOfFds; int threadId; char label[TSDB_LABEL_LEN]; - void *shandle; // handle passed by upper layer during server initialization - void *(*processData)(SRecvInfo *pPacket); + void * shandle; // handle passed by upper layer during server initialization + void *(*processData)(SRecvInfo *pPacket); } SThreadObj; typedef struct { - char label[TSDB_LABEL_LEN]; - int32_t index; - int numOfThreads; + char label[TSDB_LABEL_LEN]; + int32_t index; + int numOfThreads; SThreadObj **pThreadObj; } SClientObj; typedef struct { - SOCKET fd; - uint32_t ip; - uint16_t port; - int8_t stop; - int8_t reserve; - char label[TSDB_LABEL_LEN]; - int numOfThreads; - void * shandle; + SOCKET fd; + uint32_t ip; + uint16_t port; + int8_t stop; + int8_t reserve; + char label[TSDB_LABEL_LEN]; + int numOfThreads; + void * shandle; SThreadObj **pThreadObj; - pthread_t thread; + pthread_t thread; } SServerObj; -static void *taosProcessTcpData(void *param); +static void * taosProcessTcpData(void *param); static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, SOCKET fd); static void taosFreeFdObj(SFdObj *pFdObj); static void taosReportBrokenLink(SFdObj *pFdObj); -static void *taosAcceptTcpConnection(void *arg); +static void * taosAcceptTcpConnection(void *arg); void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle) { SServerObj *pServerObj; @@ -99,7 +103,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread return NULL; } - int code = 0; + int code = 0; pthread_attr_t thattr; pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); @@ -110,7 +114,7 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread if (pThreadObj == NULL) { tError("TCP:%s no enough memory", label); terrno = TAOS_SYSTEM_ERROR(errno); - for (int j=0; jpThreadObj[j]); + for (int j = 0; j < i; ++j) free(pServerObj->pThreadObj[j]); free(pServerObj->pThreadObj); free(pServerObj); return NULL; @@ -172,8 +176,10 @@ void *taosInitTcpServer(uint32_t ip, uint16_t port, char *label, int numOfThread return (void *)pServerObj; } -static void taosStopTcpThread(SThreadObj* pThreadObj) { - if (pThreadObj == NULL) { return;} +static void taosStopTcpThread(SThreadObj *pThreadObj) { + if (pThreadObj == NULL) { + return; + } // save thread into local variable and signal thread to stop pthread_t thread = pThreadObj->thread; if (!taosCheckPthreadValid(thread)) { @@ -194,7 +200,7 @@ void taosStopTcpServer(void *handle) { pServerObj->stop = 1; if (pServerObj->fd >= 0) { - taosShutDownSocketRD(pServerObj->fd); + taosShutDownSocketRD(pServerObj->fd); } if (taosCheckPthreadValid(pServerObj->thread)) { if (taosComparePthread(pServerObj->thread, pthread_self())) { @@ -227,8 +233,8 @@ static void *taosAcceptTcpConnection(void *arg) { SOCKET connFd = -1; struct sockaddr_in caddr; int threadId = 0; - SThreadObj *pThreadObj; - SServerObj *pServerObj; + SThreadObj * pThreadObj; + SServerObj * pServerObj; pServerObj = (SServerObj *)arg; tDebug("%s TCP server is ready, ip:0x%x:%hu", pServerObj->label, pServerObj->ip, pServerObj->port); @@ -253,8 +259,8 @@ static void *taosAcceptTcpConnection(void *arg) { } taosKeepTcpAlive(connFd); - struct timeval to={5, 0}; - int32_t ret = taosSetSockOpt(connFd, SOL_SOCKET, SO_RCVTIMEO, &to, sizeof(to)); + struct timeval to = {5, 0}; + int32_t ret = taosSetSockOpt(connFd, SOL_SOCKET, SO_RCVTIMEO, &to, sizeof(to)); if (ret != 0) { taosCloseSocket(connFd); tError("%s failed to set recv timeout fd(%s)for connection from:%s:%hu", pServerObj->label, strerror(errno), @@ -262,7 +268,6 @@ static void *taosAcceptTcpConnection(void *arg) { continue; } - // pick up the thread to handle this connection pThreadObj = pServerObj->pThreadObj[threadId]; @@ -271,7 +276,7 @@ static void *taosAcceptTcpConnection(void *arg) { pFdObj->ip = caddr.sin_addr.s_addr; pFdObj->port = htons(caddr.sin_port); tDebug("%s new TCP connection from %s:%hu, fd:%d FD:%p numOfFds:%d", pServerObj->label, - taosInetNtoa(caddr.sin_addr), pFdObj->port, connFd, pFdObj, pThreadObj->numOfFds); + taosInetNtoa(caddr.sin_addr), pFdObj->port, connFd, pFdObj, pThreadObj->numOfFds); } else { taosCloseSocket(connFd); tError("%s failed to malloc FdObj(%s) for connection from:%s:%hu", pServerObj->label, strerror(errno), @@ -297,14 +302,14 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int numOfThread tstrncpy(pClientObj->label, label, sizeof(pClientObj->label)); pClientObj->numOfThreads = numOfThreads; - pClientObj->pThreadObj = (SThreadObj **)calloc(numOfThreads, sizeof(SThreadObj*)); + pClientObj->pThreadObj = (SThreadObj **)calloc(numOfThreads, sizeof(SThreadObj *)); if (pClientObj->pThreadObj == NULL) { tError("TCP:%s no enough memory", label); tfree(pClientObj); terrno = TAOS_SYSTEM_ERROR(errno); } - int code = 0; + int code = 0; pthread_attr_t thattr; pthread_attr_init(&thattr); pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); @@ -314,15 +319,15 @@ void *taosInitTcpClient(uint32_t ip, uint16_t port, char *label, int numOfThread if (pThreadObj == NULL) { tError("TCP:%s no enough memory", label); terrno = TAOS_SYSTEM_ERROR(errno); - for (int j=0; jpThreadObj[j]); + for (int j = 0; j < i; ++j) free(pClientObj->pThreadObj[j]); free(pClientObj); pthread_attr_destroy(&thattr); return NULL; } pClientObj->pThreadObj[i] = pThreadObj; taosResetPthread(&pThreadObj->thread); - pThreadObj->ip = ip; - pThreadObj->stop = false; + pThreadObj->ip = ip; + pThreadObj->stop = false; tstrncpy(pThreadObj->label, label, sizeof(pThreadObj->label)); pThreadObj->shandle = shandle; pThreadObj->processData = fp; @@ -364,14 +369,14 @@ void taosStopTcpClient(void *chandle) { if (pClientObj == NULL) return; - tDebug ("%s TCP client is stopped", pClientObj->label); + tDebug("%s TCP client is stopped", pClientObj->label); } void taosCleanUpTcpClient(void *chandle) { SClientObj *pClientObj = chandle; if (pClientObj == NULL) return; for (int i = 0; i < pClientObj->numOfThreads; ++i) { - SThreadObj *pThreadObj= pClientObj->pThreadObj[i]; + SThreadObj *pThreadObj = pClientObj->pThreadObj[i]; taosStopTcpThread(pThreadObj); } @@ -381,9 +386,9 @@ void taosCleanUpTcpClient(void *chandle) { } void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uint16_t port) { - SClientObj * pClientObj = shandle; - int32_t index = atomic_load_32(&pClientObj->index) % pClientObj->numOfThreads; - atomic_store_32(&pClientObj->index, index + 1); + SClientObj *pClientObj = shandle; + int32_t index = atomic_load_32(&pClientObj->index) % pClientObj->numOfThreads; + atomic_store_32(&pClientObj->index, index + 1); SThreadObj *pThreadObj = pClientObj->pThreadObj[index]; SOCKET fd = taosOpenTcpClientSocket(ip, port, pThreadObj->ip); @@ -394,10 +399,9 @@ void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uin #endif struct sockaddr_in sin; - uint16_t localPort = 0; - unsigned int addrlen = sizeof(sin); - if (getsockname(fd, (struct sockaddr *)&sin, &addrlen) == 0 && - sin.sin_family == AF_INET && addrlen == sizeof(sin)) { + uint16_t localPort = 0; + unsigned int addrlen = sizeof(sin); + if (getsockname(fd, (struct sockaddr *)&sin, &addrlen) == 0 && sin.sin_family == AF_INET && addrlen == sizeof(sin)) { localPort = (uint16_t)ntohs(sin.sin_port); } @@ -407,8 +411,8 @@ void *taosOpenTcpClientConnection(void *shandle, void *thandle, uint32_t ip, uin pFdObj->thandle = thandle; pFdObj->port = port; pFdObj->ip = ip; - tDebug("%s %p TCP connection to 0x%x:%hu is created, localPort:%hu FD:%p numOfFds:%d", - pThreadObj->label, thandle, ip, port, localPort, pFdObj, pThreadObj->numOfFds); + tDebug("%s %p TCP connection to 0x%x:%hu is created, localPort:%hu FD:%p numOfFds:%d", pThreadObj->label, thandle, + ip, port, localPort, pFdObj, pThreadObj->numOfFds); } else { tError("%s failed to malloc client FdObj(%s)", pThreadObj->label, strerror(errno)); taosCloseSocket(fd); @@ -441,7 +445,6 @@ int taosSendTcpData(uint32_t ip, uint16_t port, void *data, int len, void *chand } static void taosReportBrokenLink(SFdObj *pFdObj) { - SThreadObj *pThreadObj = pFdObj->pThreadObj; // notify the upper layer, so it will clean the associated context @@ -464,9 +467,9 @@ static void taosReportBrokenLink(SFdObj *pFdObj) { } static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) { - SRpcHead rpcHead; - int32_t msgLen, leftLen, retLen, headLen; - char *buffer, *msg; + SRpcHead rpcHead; + int32_t msgLen, leftLen, retLen, headLen; + char * buffer, *msg; SThreadObj *pThreadObj = pFdObj->pThreadObj; @@ -483,7 +486,8 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) { tError("%s %p TCP malloc(size:%d) fail", pThreadObj->label, pFdObj->thandle, msgLen); return -1; } else { - tTrace("%s %p read data, FD:%p fd:%d TCP malloc mem:%p", pThreadObj->label, pFdObj->thandle, pFdObj, pFdObj->fd, buffer); + tTrace("%s %p read data, FD:%p fd:%d TCP malloc mem:%p", pThreadObj->label, pFdObj->thandle, pFdObj, pFdObj->fd, + buffer); } msg = buffer + tsRpcOverhead; @@ -491,8 +495,7 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) { retLen = taosReadMsg(pFdObj->fd, msg + headLen, leftLen); if (leftLen != retLen) { - tError("%s %p read error, leftLen:%d retLen:%d FD:%p", - pThreadObj->label, pFdObj->thandle, leftLen, retLen, pFdObj); + tError("%s %p read error, leftLen:%d retLen:%d FD:%p", pThreadObj->label, pFdObj->thandle, leftLen, retLen, pFdObj); free(buffer); return -1; } @@ -519,8 +522,8 @@ static int taosReadTcpData(SFdObj *pFdObj, SRecvInfo *pInfo) { #define maxEvents 10 static void *taosProcessTcpData(void *param) { - SThreadObj *pThreadObj = param; - SFdObj *pFdObj; + SThreadObj * pThreadObj = param; + SFdObj * pFdObj; struct epoll_event events[maxEvents]; SRecvInfo recvInfo; @@ -569,7 +572,7 @@ static void *taosProcessTcpData(void *param) { if (pThreadObj->stop) break; } - if (pThreadObj->pollFd >=0) { + if (pThreadObj->pollFd >= 0) { EpollClose(pThreadObj->pollFd); pThreadObj->pollFd = -1; } @@ -620,7 +623,6 @@ static SFdObj *taosMallocFdObj(SThreadObj *pThreadObj, SOCKET fd) { } static void taosFreeFdObj(SFdObj *pFdObj) { - if (pFdObj == NULL) return; if (pFdObj->signature != pFdObj) return; @@ -638,8 +640,8 @@ static void taosFreeFdObj(SFdObj *pFdObj) { pThreadObj->numOfFds--; if (pThreadObj->numOfFds < 0) - tError("%s %p TCP thread:%d, number of FDs is negative!!!", - pThreadObj->label, pFdObj->thandle, pThreadObj->threadId); + tError("%s %p TCP thread:%d, number of FDs is negative!!!", pThreadObj->label, pFdObj->thandle, + pThreadObj->threadId); if (pFdObj->prev) { (pFdObj->prev)->next = pFdObj->next; @@ -653,8 +655,10 @@ static void taosFreeFdObj(SFdObj *pFdObj) { pthread_mutex_unlock(&pThreadObj->mutex); - tDebug("%s %p TCP connection is closed, FD:%p fd:%d numOfFds:%d", - pThreadObj->label, pFdObj->thandle, pFdObj, pFdObj->fd, pThreadObj->numOfFds); + tDebug("%s %p TCP connection is closed, FD:%p fd:%d numOfFds:%d", pThreadObj->label, pFdObj->thandle, pFdObj, + pFdObj->fd, pThreadObj->numOfFds); tfree(pFdObj); } + +#endif diff --git a/source/libs/transport/src/rpcUdp.c b/source/libs/transport/src/rpcUdp.c index 5bc31c189a..79956cc98d 100644 --- a/source/libs/transport/src/rpcUdp.c +++ b/source/libs/transport/src/rpcUdp.c @@ -13,50 +13,53 @@ * along with this program. If not, see . */ +#include "rpcUdp.h" #include "os.h" -#include "ttimer.h" -#include "tutil.h" +#include "rpcHead.h" +#include "rpcLog.h" #include "taosdef.h" #include "taoserror.h" -#include "rpcLog.h" -#include "rpcUdp.h" -#include "rpcHead.h" +#include "ttimer.h" +#include "tutil.h" +#ifdef USE_UV +// no support upd currently +#else #define RPC_MAX_UDP_CONNS 256 #define RPC_MAX_UDP_PKTS 1000 #define RPC_UDP_BUF_TIME 5 // mseconds #define RPC_MAX_UDP_SIZE 65480 typedef struct { - int index; - SOCKET fd; - uint16_t port; // peer port - uint16_t localPort; // local port - char label[TSDB_LABEL_LEN]; // copy from udpConnSet; - pthread_t thread; - void *hash; - void *shandle; // handle passed by upper layer during server initialization - void *pSet; - void *(*processData)(SRecvInfo *pRecv); - char *buffer; // buffer to receive data + int index; + SOCKET fd; + uint16_t port; // peer port + uint16_t localPort; // local port + char label[TSDB_LABEL_LEN]; // copy from udpConnSet; + pthread_t thread; + void * hash; + void * shandle; // handle passed by upper layer during server initialization + void * pSet; + void *(*processData)(SRecvInfo *pRecv); + char *buffer; // buffer to receive data } SUdpConn; typedef struct { - int index; - int server; - uint32_t ip; // local IP - uint16_t port; // local Port - void *shandle; // handle passed by upper layer during server initialization - int threads; - char label[TSDB_LABEL_LEN]; - void *(*fp)(SRecvInfo *pPacket); - SUdpConn udpConn[]; + int index; + int server; + uint32_t ip; // local IP + uint16_t port; // local Port + void * shandle; // handle passed by upper layer during server initialization + int threads; + char label[TSDB_LABEL_LEN]; + void *(*fp)(SRecvInfo *pPacket); + SUdpConn udpConn[]; } SUdpConnSet; static void *taosRecvUdpData(void *param); void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads, void *fp, void *shandle) { - SUdpConn *pConn; + SUdpConn * pConn; SUdpConnSet *pSet; int size = (int)sizeof(SUdpConnSet) + threads * (int)sizeof(SUdpConn); @@ -79,7 +82,7 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads pthread_attr_init(&thAttr); pthread_attr_setdetachstate(&thAttr, PTHREAD_CREATE_JOINABLE); - int i; + int i; uint16_t ownPort; for (i = 0; i < threads; ++i) { pConn = pSet->udpConn + i; @@ -97,9 +100,9 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads } struct sockaddr_in sin; - unsigned int addrlen = sizeof(sin); - if (getsockname(pConn->fd, (struct sockaddr *)&sin, &addrlen) == 0 && - sin.sin_family == AF_INET && addrlen == sizeof(sin)) { + unsigned int addrlen = sizeof(sin); + if (getsockname(pConn->fd, (struct sockaddr *)&sin, &addrlen) == 0 && sin.sin_family == AF_INET && + addrlen == sizeof(sin)) { pConn->localPort = (uint16_t)ntohs(sin.sin_port); } @@ -118,7 +121,7 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads pthread_attr_destroy(&thAttr); - if (i != threads) { + if (i != threads) { terrno = TAOS_SYSTEM_ERROR(errno); taosCleanUpUdpConnection(pSet); return NULL; @@ -130,14 +133,14 @@ void *taosInitUdpConnection(uint32_t ip, uint16_t port, char *label, int threads void taosStopUdpConnection(void *handle) { SUdpConnSet *pSet = (SUdpConnSet *)handle; - SUdpConn *pConn; + SUdpConn * pConn; if (pSet == NULL) return; for (int i = 0; i < pSet->threads; ++i) { pConn = pSet->udpConn + i; - if (pConn->fd >=0) shutdown(pConn->fd, SHUT_RDWR); - if (pConn->fd >=0) taosCloseSocket(pConn->fd); + if (pConn->fd >= 0) shutdown(pConn->fd, SHUT_RDWR); + if (pConn->fd >= 0) taosCloseSocket(pConn->fd); pConn->fd = -1; } @@ -155,13 +158,13 @@ void taosStopUdpConnection(void *handle) { void taosCleanUpUdpConnection(void *handle) { SUdpConnSet *pSet = (SUdpConnSet *)handle; - SUdpConn *pConn; + SUdpConn * pConn; if (pSet == NULL) return; for (int i = 0; i < pSet->threads; ++i) { pConn = pSet->udpConn + i; - if (pConn->fd >=0) taosCloseSocket(pConn->fd); + if (pConn->fd >= 0) taosCloseSocket(pConn->fd); } tDebug("%s UDP is cleaned up", pSet->label); @@ -182,7 +185,7 @@ void *taosOpenUdpConnection(void *shandle, void *thandle, uint32_t ip, uint16_t } static void *taosRecvUdpData(void *param) { - SUdpConn *pConn = param; + SUdpConn * pConn = param; struct sockaddr_in sourceAdd; ssize_t dataLen; unsigned int addLen; @@ -218,7 +221,7 @@ static void *taosRecvUdpData(void *param) { } int32_t size = dataLen + tsRpcOverhead; - char *tmsg = malloc(size); + char * tmsg = malloc(size); if (NULL == tmsg) { tError("%s failed to allocate memory, size:%" PRId64, pConn->label, (int64_t)dataLen); continue; @@ -257,4 +260,4 @@ int taosSendUdpData(uint32_t ip, uint16_t port, void *data, int dataLen, void *c return ret; } - +#endif diff --git a/source/libs/transport/src/transport.c b/source/libs/transport/src/transport.c index 6dea4a4e57..f2f48bbc8a 100644 --- a/source/libs/transport/src/transport.c +++ b/source/libs/transport/src/transport.c @@ -11,4 +11,4 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . - */ \ No newline at end of file + */ From 4aa8be83339472d639ec24543c4671dd9f1dfabf Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Tue, 11 Jan 2022 23:16:45 +0800 Subject: [PATCH 26/57] [td-11818] Support the select *. --- include/libs/planner/planner.h | 4 +- source/client/src/clientImpl.c | 26 ++++++++++++- source/client/src/clientMsgHandler.c | 3 -- source/libs/executor/src/executorimpl.c | 39 +++++++++++++------ source/libs/planner/src/physicalPlan.c | 19 +++++---- source/libs/planner/src/physicalPlanJson.c | 29 ++++++++++---- source/libs/scheduler/src/scheduler.c | 10 +---- source/libs/scheduler/test/schedulerTests.cpp | 6 +-- 8 files changed, 92 insertions(+), 44 deletions(-) diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index da70f21498..d98c1ca595 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -91,8 +91,10 @@ typedef struct SPhyNode { typedef struct SScanPhyNode { SPhyNode node; - uint64_t uid; // unique id of the table + uint64_t uid; // unique id of the table int8_t tableType; + int32_t order; // scan order: TSDB_ORDER_ASC|TSDB_ORDER_DESC + int32_t count; // repeat count } SScanPhyNode; typedef SScanPhyNode SSystemTableScanPhyNode; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 727b668f3a..d476670c78 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -25,8 +25,9 @@ static int32_t initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet); static SMsgSendInfo* buildConnectMsg(SRequestObj *pRequest); static void destroySendMsgInfo(SMsgSendInfo* pMsgBody); +static void setQueryResultByRsp(SReqResultInfo* pResultInfo, SRetrieveTableRsp* pRsp); -static bool stringLengthCheck(const char* str, size_t maxsize) { + static bool stringLengthCheck(const char* str, size_t maxsize) { if (str == NULL) { return false; } @@ -212,6 +213,7 @@ int32_t getPlan(SRequestObj* pRequest, SQueryNode* pQueryNode, SQueryDag** pDag) SSubplan* pPlan = taosArrayGetP(pa, 0); SDataBlockSchema* pDataBlockSchema = &(pPlan->pDataSink->schema); setResSchemaInfo(pResInfo, pDataBlockSchema); + pRequest->type = TDMT_VND_QUERY; } @@ -547,7 +549,8 @@ void* doFetchRow(SRequestObj* pRequest) { if (pRequest->type == TDMT_VND_QUERY) { pRequest->type = TDMT_VND_FETCH; scheduleFetchRows(pRequest->body.pQueryJob, (void **)&pRequest->body.resInfo.pData); - + setQueryResultByRsp(&pRequest->body.resInfo, (SRetrieveTableRsp*)pRequest->body.resInfo.pData); + pResultInfo->current = 0; if (pResultInfo->numOfRows <= pResultInfo->current) { return NULL; @@ -611,12 +614,23 @@ _return: return pResultInfo->row; } +static void doPrepareResPtr(SReqResultInfo* pResInfo) { + if (pResInfo->row == NULL) { + pResInfo->row = calloc(pResInfo->numOfCols, POINTER_BYTES); + pResInfo->pCol = calloc(pResInfo->numOfCols, POINTER_BYTES); + pResInfo->length = calloc(pResInfo->numOfCols, sizeof(int32_t)); + } +} + void setResultDataPtr(SReqResultInfo* pResultInfo, TAOS_FIELD* pFields, int32_t numOfCols, int32_t numOfRows) { assert(numOfCols > 0 && pFields != NULL && pResultInfo != NULL); if (numOfRows == 0) { return; } + // todo check for the failure of malloc + doPrepareResPtr(pResultInfo); + int32_t offset = 0; for (int32_t i = 0; i < numOfCols; ++i) { pResultInfo->length[i] = pResultInfo->fields[i].bytes; @@ -642,3 +656,11 @@ void setConnectionDB(STscObj* pTscObj, const char* db) { pthread_mutex_unlock(&pTscObj->mutex); } +void setQueryResultByRsp(SReqResultInfo* pResultInfo, SRetrieveTableRsp* pRsp) { + assert(pRsp != NULL && pResultInfo != NULL); + pResultInfo->pRspMsg = pRsp; + pResultInfo->pData = (void*) pRsp->data; + pResultInfo->numOfRows = htonl(pRsp->numOfRows); + + setResultDataPtr(pResultInfo, pResultInfo->fields, pResultInfo->numOfCols, pResultInfo->numOfRows); +} \ No newline at end of file diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index 85f3fb06a7..4f1a96ad29 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -154,9 +154,6 @@ int32_t processShowRsp(void* param, const SDataBuf* pMsg, int32_t code) { pResInfo->fields = pFields; pResInfo->numOfCols = pMetaMsg->numOfColumns; - pResInfo->row = calloc(pResInfo->numOfCols, POINTER_BYTES); - pResInfo->pCol = calloc(pResInfo->numOfCols, POINTER_BYTES); - pResInfo->length = calloc(pResInfo->numOfCols, sizeof(int32_t)); pRequest->body.showInfo.execId = pShow->showId; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 9aeb979806..0dfcc9b1d8 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -18,7 +18,6 @@ #include "ttime.h" #include "exception.h" -#include "../../../../contrib/cJson/cJSON.h" #include "executorimpl.h" #include "function.h" #include "tcompare.h" @@ -7188,31 +7187,47 @@ static SExecTaskInfo* createExecTaskInfo(uint64_t queryId) { SOperatorInfo* doCreateOperatorTreeNode(SPhyNode* pPhyNode, SExecTaskInfo* pTaskInfo, void* param) { if (pPhyNode->pChildren == NULL || taosArrayGetSize(pPhyNode->pChildren) == 0) { if (pPhyNode->info.type == OP_TableScan) { + SScanPhyNode* pScanPhyNode = (SScanPhyNode*) pPhyNode; size_t numOfCols = taosArrayGetSize(pPhyNode->pTargets); - SOperatorInfo* pOperatorInfo = createTableScanOperator(param, TSDB_ORDER_ASC, numOfCols, 1, pTaskInfo); + SOperatorInfo* pOperatorInfo = createTableScanOperator(param, pScanPhyNode->order, numOfCols, pScanPhyNode->count, pTaskInfo); pTaskInfo->pRoot = pOperatorInfo; + } else { + assert(0); } } } int32_t doCreateExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, void* readerHandle) { - STsdbQueryCond cond = {.order = TSDB_ORDER_ASC, .numOfCols = 2, .loadExternalRows = false}; + STsdbQueryCond cond = {.loadExternalRows = false}; cond.twindow.skey = INT64_MIN; cond.twindow.ekey = INT64_MAX; - cond.colList = calloc(cond.numOfCols, sizeof(SColumnInfo)); - // todo set the correct table column info - cond.colList[0].type = TSDB_DATA_TYPE_TIMESTAMP; - cond.colList[0].bytes = sizeof(uint64_t); - cond.colList[0].colId = 1; + uint64_t uid = 0; + SPhyNode* pPhyNode = pPlan->pNode; + if (pPhyNode->info.type == OP_TableScan) { - cond.colList[1].type = TSDB_DATA_TYPE_INT; - cond.colList[1].bytes = sizeof(int32_t); - cond.colList[1].colId = 2; + SScanPhyNode* pScanNode = (SScanPhyNode*) pPhyNode; + uid = pScanNode->uid; + cond.order = pScanNode->order; + cond.numOfCols = taosArrayGetSize(pScanNode->node.pTargets); + cond.colList = calloc(cond.numOfCols, sizeof(SColumnInfo)); + + for(int32_t i = 0; i < cond.numOfCols; ++i) { + SExprInfo* pExprInfo = taosArrayGetP(pScanNode->node.pTargets, i); + assert(pExprInfo->pExpr->nodeType == TEXPR_COL_NODE); + + SSchema* pSchema = pExprInfo->pExpr->pSchema; + cond.colList[i].type = pSchema->type; + cond.colList[i].bytes = pSchema->bytes; + cond.colList[i].colId = pSchema->colId; + } + } else { + assert(0); + } STableGroupInfo group = {.numOfTables = 1, .pGroupList = taosArrayInit(1, POINTER_BYTES)}; SArray* pa = taosArrayInit(1, sizeof(STableKeyInfo)); - STableKeyInfo info = {.pTable = NULL, .lastKey = 0, .uid = 1}; + STableKeyInfo info = {.pTable = NULL, .lastKey = 0, .uid = uid}; taosArrayPush(pa, &info); taosArrayPush(group.pGroupList, &pa); diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index 01fb3c2513..0604ba3920 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -156,9 +156,14 @@ static SPhyNode* initPhyNode(SQueryPlanNode* pPlanNode, int32_t type, int32_t si } static SPhyNode* initScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable, int32_t type, int32_t size) { - SScanPhyNode* node = (SScanPhyNode*)initPhyNode(pPlanNode, type, size); - node->uid = pTable->pMeta->pTableMeta->uid; - node->tableType = pTable->pMeta->pTableMeta->tableType; + SScanPhyNode* node = (SScanPhyNode*) initPhyNode(pPlanNode, type, size); + + STableMeta *pTableMeta = pTable->pMeta->pTableMeta; + node->uid = pTableMeta->uid; + node->count = 1; + node->order = TSDB_ORDER_ASC; + node->tableType = pTableMeta->tableType; + return (SPhyNode*)node; } @@ -176,10 +181,10 @@ static uint8_t getScanFlag(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable) { return MAIN_SCAN; } -static SPhyNode* createUserTableScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pTable, int32_t op) { - STableScanPhyNode* node = (STableScanPhyNode*)initScanNode(pPlanNode, pTable, op, sizeof(STableScanPhyNode)); - node->scanFlag = getScanFlag(pPlanNode, pTable); - node->window = pTable->window; +static SPhyNode* createUserTableScanNode(SQueryPlanNode* pPlanNode, SQueryTableInfo* pQueryTableInfo, int32_t op) { + STableScanPhyNode* node = (STableScanPhyNode*)initScanNode(pPlanNode, pQueryTableInfo, op, sizeof(STableScanPhyNode)); + node->scanFlag = getScanFlag(pPlanNode, pQueryTableInfo); + node->window = pQueryTableInfo->window; // todo tag cond return (SPhyNode*)node; } diff --git a/source/libs/planner/src/physicalPlanJson.c b/source/libs/planner/src/physicalPlanJson.c index 895788c550..64490932fa 100644 --- a/source/libs/planner/src/physicalPlanJson.c +++ b/source/libs/planner/src/physicalPlanJson.c @@ -136,7 +136,7 @@ static const cJSON* getArray(const cJSON* json, const char* name, int32_t* size) static bool fromItem(const cJSON* jArray, FFromJson func, void* array, int32_t itemSize, int32_t size) { for (int32_t i = 0; i < size; ++i) { - if (!func(cJSON_GetArrayItem(jArray, i), (char*)array + itemSize)) { + if (!func(cJSON_GetArrayItem(jArray, i), (char*)array + itemSize * i)) { return false; } } @@ -165,9 +165,7 @@ static bool fromRawArray(const cJSON* json, const char* name, FFromJson func, vo static char* getString(const cJSON* json, const char* name) { char* p = cJSON_GetStringValue(cJSON_GetObjectItem(json, name)); - char* res = calloc(1, strlen(p) + 1); - strcpy(res, p); - return res; + return strdup(p); } static void copyString(const cJSON* json, const char* name, char* dst) { @@ -285,7 +283,7 @@ static bool columnInfoToJson(const void* obj, cJSON* jCol) { static bool columnInfoFromJson(const cJSON* json, void* obj) { SColumnInfo* col = (SColumnInfo*)obj; col->colId = getNumber(json, jkColumnInfoColId); - col->type = getNumber(json, jkColumnInfoType); + col->type = getNumber(json, jkColumnInfoType); col->bytes = getNumber(json, jkColumnInfoBytes); int32_t size = 0; bool res = fromRawArrayWithAlloc(json, jkColumnInfoFilterList, columnFilterInfoFromJson, (void**)&col->flist.filterInfo, sizeof(SColumnFilterInfo), &size); @@ -530,13 +528,25 @@ static bool timeWindowFromJson(const cJSON* json, void* obj) { static const char* jkScanNodeTableId = "TableId"; static const char* jkScanNodeTableType = "TableType"; +static const char* jkScanNodeTableOrder = "Order"; +static const char* jkScanNodeTableCount = "Count"; static bool scanNodeToJson(const void* obj, cJSON* json) { const SScanPhyNode* scan = (const SScanPhyNode*)obj; bool res = cJSON_AddNumberToObject(json, jkScanNodeTableId, scan->uid); + if (res) { res = cJSON_AddNumberToObject(json, jkScanNodeTableType, scan->tableType); } + + if (res) { + res = cJSON_AddNumberToObject(json, jkScanNodeTableOrder, scan->order); + } + + if (res) { + res = cJSON_AddNumberToObject(json, jkScanNodeTableCount, scan->count); + } + return res; } @@ -544,6 +554,8 @@ static bool scanNodeFromJson(const cJSON* json, void* obj) { SScanPhyNode* scan = (SScanPhyNode*)obj; scan->uid = getNumber(json, jkScanNodeTableId); scan->tableType = getNumber(json, jkScanNodeTableType); + scan->count = getNumber(json, jkScanNodeTableCount); + scan->order = getNumber(json, jkScanNodeTableOrder); return true; } @@ -748,9 +760,11 @@ static bool phyNodeToJson(const void* obj, cJSON* jNode) { } static bool phyNodeFromJson(const cJSON* json, void* obj) { - SPhyNode* node = (SPhyNode*)obj; + SPhyNode* node = (SPhyNode*) obj; + node->info.name = getString(json, jkPnodeName); node->info.type = opNameToOpType(node->info.name); + bool res = fromArray(json, jkPnodeTargets, exprInfoFromJson, &node->pTargets, sizeof(SExprInfo)); if (res) { res = fromArray(json, jkPnodeConditions, exprInfoFromJson, &node->pConditions, sizeof(SExprInfo)); @@ -896,7 +910,8 @@ static SSubplan* subplanFromJson(const cJSON* json) { } bool res = fromObject(json, jkSubplanId, subplanIdFromJson, &subplan->id, true); if (res) { - res = fromObjectWithAlloc(json, jkSubplanNode, phyNodeFromJson, (void**)&subplan->pNode, sizeof(SPhyNode), false); + size_t size = MAX(sizeof(SPhyNode), sizeof(SScanPhyNode)); + res = fromObjectWithAlloc(json, jkSubplanNode, phyNodeFromJson, (void**)&subplan->pNode, size, false); } if (res) { res = fromObjectWithAlloc(json, jkSubplanDataSink, dataSinkFromJson, (void**)&subplan->pDataSink, sizeof(SDataSink), false); diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index ed12408844..525d6ebe87 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -798,7 +798,7 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch SQueryTableRsp *rsp = (SQueryTableRsp *)msg; if (rspCode != TSDB_CODE_SUCCESS || NULL == msg || rsp->code != TSDB_CODE_SUCCESS) { - SCH_ERR_RET(schProcessOnTaskFailure(pJob, pTask, rsp->code)); + SCH_ERR_RET(schProcessOnTaskFailure(pJob, pTask, rspCode)); } SCH_ERR_JRET(schBuildAndSendMsg(pJob, pTask, NULL, TDMT_VND_RES_READY)); @@ -1364,16 +1364,10 @@ int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } - SSchJob *job = NULL; - - SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, &job, false)); - - *pJob = job; - + SCH_ERR_RET(schExecJobImpl(transport, nodeList, pDag, pJob, false)); return TSDB_CODE_SUCCESS; } - int32_t scheduleFetchRows(SSchJob *pJob, void** pData) { if (NULL == pJob || NULL == pData) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index b438521234..ccf22173cb 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -160,7 +160,7 @@ int32_t schtPlanToString(const SSubplan *subplan, char** str, int32_t* len) { } void schtExecNode(SSubplan* subplan, uint64_t templateId, SQueryNodeAddr* ep) { - + } @@ -218,8 +218,6 @@ void *schtSendRsp(void *param) { } struct SSchJob *pInsertJob = NULL; - - } TEST(queryTest, normalCase) { @@ -336,7 +334,7 @@ TEST(insertTest, normalCase) { uint64_t numOfRows = 0; schtInitLogFile(); - + SArray *qnodeList = taosArrayInit(1, sizeof(SEpAddr)); SEpAddr qnodeAddr = {0}; From f1f666e780238cbd39373ddb98fc6a07df6d6b3a Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 11 Jan 2022 23:22:08 +0800 Subject: [PATCH 27/57] add message queue for vnode --- include/dnode/vnode/vnode.h | 17 ++++++++++++++--- source/dnode/mgmt/impl/src/dnode.c | 15 +++++++++++++-- source/dnode/vnode/impl/inc/vnodeDef.h | 4 ++++ source/dnode/vnode/impl/src/vnodeMgr.c | 18 +++++++++++++----- 4 files changed, 44 insertions(+), 10 deletions(-) diff --git a/include/dnode/vnode/vnode.h b/include/dnode/vnode/vnode.h index 2212a8c29a..b83ca53859 100644 --- a/include/dnode/vnode/vnode.h +++ b/include/dnode/vnode/vnode.h @@ -66,15 +66,26 @@ typedef struct SVnodeCfg { SWalCfg walCfg; } SVnodeCfg; +typedef struct SDnode SDnode; +typedef void (*PutReqToVQueryQFp)(SDnode *pDnode, struct SRpcMsg *pReq); +typedef struct { + int32_t sver; + SDnode *pDnode; + char *timezone; + char *locale; + char *charset; + PutReqToVQueryQFp putReqToVQueryQFp; + uint16_t nthreads; // number of commit threads. 0 for no threads and a schedule queue should be given (TODO) +} SVnodeOpt; + /* ------------------------ SVnode ------------------------ */ /** * @brief Initialize the vnode module * - * @param nthreads number of commit threads. 0 for no threads and - * a schedule queue should be given (TODO) + * @param pOption Option of the vnode mnodule * @return int 0 for success and -1 for failure */ -int vnodeInit(uint16_t nthreads); +int vnodeInit(const SVnodeOpt *pOption); /** * @brief clear a vnode diff --git a/source/dnode/mgmt/impl/src/dnode.c b/source/dnode/mgmt/impl/src/dnode.c index cd27781df3..f3016feda5 100644 --- a/source/dnode/mgmt/impl/src/dnode.c +++ b/source/dnode/mgmt/impl/src/dnode.c @@ -22,8 +22,8 @@ #include "dndTransport.h" #include "dndVnodes.h" #include "sync.h" -#include "wal.h" #include "tfs.h" +#include "wal.h" EStat dndGetStat(SDnode *pDnode) { return pDnode->stat; } @@ -153,6 +153,8 @@ static void dndCleanupEnv(SDnode *pDnode) { taosStopCacheRefreshWorker(); } +static void dndPutMsgToVQueryQ(SDnode *pDnode, SRpcMsg *pRpcMsg) { dndProcessVnodeQueryMsg(pDnode, pRpcMsg, NULL); } + SDnode *dndInit(SDnodeOpt *pOption) { taosIgnSIGPIPE(); taosBlockSIGPIPE(); @@ -196,7 +198,16 @@ SDnode *dndInit(SDnodeOpt *pOption) { return NULL; } - if (vnodeInit(pDnode->opt.numOfCommitThreads) != 0) { + SVnodeOpt vnodeOpt = { + .sver = pDnode->opt.sver, + .pDnode = pDnode, + .timezone = pDnode->opt.timezone, + .locale = pDnode->opt.locale, + .charset = pDnode->opt.charset, + .putReqToVQueryQFp = dndPutMsgToVQueryQ, + .nthreads = pDnode->opt.numOfCommitThreads, + }; + if (vnodeInit(&vnodeOpt) != 0) { dError("failed to init vnode env"); dndCleanup(pDnode); return NULL; diff --git a/source/dnode/vnode/impl/inc/vnodeDef.h b/source/dnode/vnode/impl/inc/vnodeDef.h index e70e891794..361fdd10e0 100644 --- a/source/dnode/vnode/impl/inc/vnodeDef.h +++ b/source/dnode/vnode/impl/inc/vnodeDef.h @@ -57,6 +57,8 @@ typedef struct SVnodeMgr { pthread_cond_t hasTask; TD_DLIST(SVnodeTask) queue; // For vnode Mgmt + SDnode* pDnode; + PutReqToVQueryQFp putReqToVQueryQFp; } SVnodeMgr; extern SVnodeMgr vnodeMgr; @@ -79,6 +81,8 @@ struct SVnode { int vnodeScheduleTask(SVnodeTask* task); +void vnodePutReqToVQueryQ(struct SRpcMsg *pReq); + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/impl/src/vnodeMgr.c b/source/dnode/vnode/impl/src/vnodeMgr.c index 784d1abb60..43527c13fe 100644 --- a/source/dnode/vnode/impl/src/vnodeMgr.c +++ b/source/dnode/vnode/impl/src/vnodeMgr.c @@ -19,17 +19,19 @@ SVnodeMgr vnodeMgr = {.vnodeInitFlag = TD_MOD_UNINITIALIZED}; static void* loop(void* arg); -int vnodeInit(uint16_t nthreads) { +int vnodeInit(const SVnodeOpt *pOption) { if (TD_CHECK_AND_SET_MODE_INIT(&(vnodeMgr.vnodeInitFlag)) == TD_MOD_INITIALIZED) { return 0; } vnodeMgr.stop = false; + vnodeMgr.putReqToVQueryQFp = pOption->putReqToVQueryQFp; + vnodeMgr.pDnode = pOption->pDnode; // Start commit handers - if (nthreads > 0) { - vnodeMgr.nthreads = nthreads; - vnodeMgr.threads = (pthread_t*)calloc(nthreads, sizeof(pthread_t)); + if (pOption->nthreads > 0) { + vnodeMgr.nthreads = pOption->nthreads; + vnodeMgr.threads = (pthread_t*)calloc(pOption->nthreads, sizeof(pthread_t)); if (vnodeMgr.threads == NULL) { return -1; } @@ -38,7 +40,7 @@ int vnodeInit(uint16_t nthreads) { pthread_cond_init(&(vnodeMgr.hasTask), NULL); TD_DLIST_INIT(&(vnodeMgr.queue)); - for (uint16_t i = 0; i < nthreads; i++) { + for (uint16_t i = 0; i < pOption->nthreads; i++) { pthread_create(&(vnodeMgr.threads[i]), NULL, loop, NULL); pthread_setname_np(vnodeMgr.threads[i], "VND Commit Thread"); } @@ -89,6 +91,12 @@ int vnodeScheduleTask(SVnodeTask* pTask) { return 0; } +void vnodePutReqToVQueryQ(struct SRpcMsg* pReq) { + if (vnodeMgr.putReqToVQueryQFp) { + (*vnodeMgr.putReqToVQueryQFp)(vnodeMgr.pDnode, pReq); + } +} + /* ------------------------ STATIC METHODS ------------------------ */ static void* loop(void* arg) { SVnodeTask* pTask; From dde3dfd7bd9fc58075f58ecce04ab6311e31c502 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Tue, 11 Jan 2022 23:53:18 +0800 Subject: [PATCH 28/57] add libuv --- source/libs/transport/src/rpcMain.c | 158 +++++++++++++++++++--------- 1 file changed, 109 insertions(+), 49 deletions(-) diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index 9bba0dca0a..802c5ae4ea 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -33,28 +33,40 @@ #include "ttimer.h" #include "tutil.h" +typedef struct { + int sessions; // number of sessions allowed + int numOfThreads; // number of threads to process incoming messages + int idleTime; // milliseconds; + uint16_t localPort; + int8_t connType; + int index; // for UDP server only, round robin for multiple threads + char label[TSDB_LABEL_LEN]; + + char user[TSDB_UNI_LEN]; // meter ID + char spi; // security parameter index + char encrypt; // encrypt algorithm + char secret[TSDB_PASSWORD_LEN]; // secret for the link + char ckey[TSDB_PASSWORD_LEN]; // ciphering key + + void (*cfp)(void* parent, SRpcMsg*, SEpSet*); + int (*afp)(void* parent, char* user, char* spi, char* encrypt, char* secret, char* ckey); + + int32_t refCount; + void* parent; + void* idPool; // handle to ID pool + void* tmrCtrl; // handle to timer + SHashObj* hash; // handle returned by hash utility + void* tcphandle; // returned handle from TCP initialization + void* udphandle; // returned handle from UDP initialization + void* pCache; // connection cache + pthread_mutex_t mutex; + struct SRpcConn* connList; // connection list +} SRpcInfo; + #ifdef USE_UV #define container_of(ptr, type, member) ((type*)((char*)(ptr)-offsetof(type, member))) -int32_t rpcInit() { return -1; } -void rpcCleanup() { return; }; -void* rpcOpen(const SRpcInit* pRpc) { return NULL; } -void rpcClose(void* arg) { return; } -void* rpcMallocCont(int contLen) { return NULL; } -void rpcFreeCont(void* cont) { return; } -void* rpcReallocCont(void* ptr, int contLen) { return NULL; } - -void rpcSendRequest(void* thandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* rid) { return; } - -void rpcSendResponse(const SRpcMsg* pMsg) {} - -void rpcSendRedirectRsp(void* pConn, const SEpSet* pEpSet) {} -int rpcGetConnInfo(void* thandle, SRpcConnInfo* pInfo) { return -1; } -void rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pReq, SRpcMsg* pRsp) { return; } -int rpcReportProgress(void* pConn, char* pCont, int contLen) { return -1; } -void rpcCancelRequest(int64_t rid) { return; } - typedef struct SThreadObj { pthread_t thread; uv_pipe_t* pipe; @@ -79,6 +91,84 @@ typedef struct SConnCtx { int ref; } SConnCtx; +static void allocBuffer(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); +static void onTimeout(uv_timer_t* handle); +static void onRead(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf); +static void onWrite(uv_write_t* req, int status); +static void onAccept(uv_stream_t* stream, int status); +void onConnection(uv_stream_t* q, ssize_t nread, const uv_buf_t* buf); +static void workerAsyncCB(uv_async_t* handle); +static void* workerThread(void* arg); + +int32_t rpcInit() { return -1; } +void rpcCleanup() { return; }; +void* rpcOpen(const SRpcInit* pInit) { + SRpcInfo* pRpc = calloc(1, sizeof(SRpcInfo)); + if (pRpc == NULL) { + return NULL; + } + if (pInit->label) { + tstrncpy(pRpc->label, pInit->label, sizeof(pRpc->label)); + } + pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; + + SServerObj* srv = calloc(1, sizeof(SServerObj)); + srv->loop = (uv_loop_t*)malloc(sizeof(uv_loop_t)); + srv->numOfThread = pRpc->numOfThreads; + srv->workerIdx = 0; + srv->pThreadObj = (SThreadObj**)calloc(srv->numOfThread, sizeof(SThreadObj*)); + srv->pipe = (uv_pipe_t**)calloc(srv->numOfThread, sizeof(uv_pipe_t*)); + uv_loop_init(srv->loop); + + for (int i = 0; i < srv->numOfThread; i++) { + srv->pThreadObj[i] = (SThreadObj*)calloc(1, sizeof(SThreadObj)); + srv->pipe[i] = (uv_pipe_t*)calloc(2, sizeof(uv_pipe_t)); + int fds[2]; + if (uv_socketpair(AF_UNIX, SOCK_STREAM, fds, UV_NONBLOCK_PIPE, UV_NONBLOCK_PIPE) != 0) { + return NULL; + } + uv_pipe_init(srv->loop, &(srv->pipe[i][0]), 1); + uv_pipe_open(&(srv->pipe[i][0]), fds[1]); // init write + + srv->pThreadObj[i]->fd = fds[0]; + srv->pThreadObj[i]->pipe = &(srv->pipe[i][1]); // init read + int err = pthread_create(&(srv->pThreadObj[i]->thread), NULL, workerThread, (void*)(srv->pThreadObj[i])); + if (err == 0) { + tError("sucess to create worker thread %d", i); + // printf("thread %d create\n", i); + } else { + tError("failed to create worker thread %d", i); + return NULL; + } + } + uv_tcp_init(srv->loop, &srv->server); + struct sockaddr_in bind_addr; + uv_ip4_addr("0.0.0.0", pInit->localPort, &bind_addr); + uv_tcp_bind(&srv->server, (const struct sockaddr*)&bind_addr, 0); + int err = 0; + if ((err = uv_listen((uv_stream_t*)&srv->server, 128, onAccept)) != 0) { + tError("Listen error %s\n", uv_err_name(err)); + return NULL; + } + uv_run(srv->loop, UV_RUN_DEFAULT); + + return pRpc; +} +void rpcClose(void* arg) { return; } +void* rpcMallocCont(int contLen) { return NULL; } +void rpcFreeCont(void* cont) { return; } +void* rpcReallocCont(void* ptr, int contLen) { return NULL; } + +void rpcSendRequest(void* thandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* rid) { return; } + +void rpcSendResponse(const SRpcMsg* pMsg) {} + +void rpcSendRedirectRsp(void* pConn, const SEpSet* pEpSet) {} +int rpcGetConnInfo(void* thandle, SRpcConnInfo* pInfo) { return -1; } +void rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pReq, SRpcMsg* pRsp) { return; } +int rpcReportProgress(void* pConn, char* pCont, int contLen) { return -1; } +void rpcCancelRequest(int64_t rid) { return; } + void allocBuffer(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { buf->base = malloc(suggested_size); buf->len = suggested_size; @@ -98,7 +188,7 @@ void onWrite(uv_write_t* req, int status) { if (req) tDebug("data already was written on stream"); } -static void workerAsyncCB(uv_async_t* handle) { +void workerAsyncCB(uv_async_t* handle) { // opt SThreadObj* pObj = container_of(handle, SThreadObj, workerAsync); } @@ -187,36 +277,6 @@ void* workerThread(void* arg) { #define rpcContLenFromMsg(msgLen) (msgLen - sizeof(SRpcHead)) #define rpcIsReq(type) (type & 1U) -typedef struct { - int sessions; // number of sessions allowed - int numOfThreads; // number of threads to process incoming messages - int idleTime; // milliseconds; - uint16_t localPort; - int8_t connType; - int index; // for UDP server only, round robin for multiple threads - char label[TSDB_LABEL_LEN]; - - char user[TSDB_UNI_LEN]; // meter ID - char spi; // security parameter index - char encrypt; // encrypt algorithm - char secret[TSDB_PASSWORD_LEN]; // secret for the link - char ckey[TSDB_PASSWORD_LEN]; // ciphering key - - void (*cfp)(void *parent, SRpcMsg *, SEpSet *); - int (*afp)(void *parent, char *user, char *spi, char *encrypt, char *secret, char *ckey); - - int32_t refCount; - void * parent; - void * idPool; // handle to ID pool - void * tmrCtrl; // handle to timer - SHashObj * hash; // handle returned by hash utility - void * tcphandle; // returned handle from TCP initialization - void * udphandle; // returned handle from UDP initialization - void * pCache; // connection cache - pthread_mutex_t mutex; - struct SRpcConn *connList; // connection list -} SRpcInfo; - typedef struct { SRpcInfo * pRpc; // associated SRpcInfo SEpSet epSet; // ip list provided by app From ab3378e090100a98451f4fa606ca85c2a8fc9ec8 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 11 Jan 2022 16:44:11 -0800 Subject: [PATCH 29/57] add msg queue --- include/dnode/vnode/vnode.h | 11 ++++++----- source/dnode/mgmt/impl/src/dndVnodes.c | 6 ++++-- source/dnode/mgmt/impl/src/dnode.c | 3 +-- source/dnode/vnode/impl/inc/vnodeDef.h | 3 ++- source/dnode/vnode/impl/src/vnodeMain.c | 21 ++++++++++++--------- source/dnode/vnode/impl/src/vnodeMgr.c | 9 ++++----- 6 files changed, 29 insertions(+), 24 deletions(-) diff --git a/include/dnode/vnode/vnode.h b/include/dnode/vnode/vnode.h index b83ca53859..03a59b01e6 100644 --- a/include/dnode/vnode/vnode.h +++ b/include/dnode/vnode/vnode.h @@ -31,8 +31,12 @@ extern "C" { /* ------------------------ TYPES EXPOSED ------------------------ */ typedef struct SVnode SVnode; +typedef struct SDnode SDnode; +typedef void (*PutReqToVQueryQFp)(SDnode *pDnode, struct SRpcMsg *pReq); + typedef struct SVnodeCfg { int32_t vgId; + SDnode *pDnode; /** vnode buffer pool options */ struct { @@ -66,16 +70,13 @@ typedef struct SVnodeCfg { SWalCfg walCfg; } SVnodeCfg; -typedef struct SDnode SDnode; -typedef void (*PutReqToVQueryQFp)(SDnode *pDnode, struct SRpcMsg *pReq); typedef struct { int32_t sver; - SDnode *pDnode; char *timezone; char *locale; char *charset; - PutReqToVQueryQFp putReqToVQueryQFp; uint16_t nthreads; // number of commit threads. 0 for no threads and a schedule queue should be given (TODO) + PutReqToVQueryQFp putReqToVQueryQFp; } SVnodeOpt; /* ------------------------ SVnode ------------------------ */ @@ -100,7 +101,7 @@ void vnodeClear(); * @param pVnodeCfg options of the vnode * @return SVnode* The vnode object */ -SVnode *vnodeOpen(const char *path, const SVnodeCfg *pVnodeCfg, int32_t vid); +SVnode *vnodeOpen(const char *path, const SVnodeCfg *pVnodeCfg); /** * @brief Close a VNODE diff --git a/source/dnode/mgmt/impl/src/dndVnodes.c b/source/dnode/mgmt/impl/src/dndVnodes.c index bf27a542ae..9f585859a8 100644 --- a/source/dnode/mgmt/impl/src/dndVnodes.c +++ b/source/dnode/mgmt/impl/src/dndVnodes.c @@ -381,7 +381,8 @@ static void *dnodeOpenVnodeFunc(void *param) { pMgmt->openVnodes, pMgmt->totalVnodes); dndReportStartup(pDnode, "open-vnodes", stepDesc); - SVnode *pImpl = vnodeOpen(pCfg->path, NULL, pCfg->vgId); + SVnodeCfg cfg = {.pDnode = pDnode, .vgId = pCfg->vgId}; + SVnode *pImpl = vnodeOpen(pCfg->path, &cfg); if (pImpl == NULL) { dError("vgId:%d, failed to open vnode by thread:%d", pCfg->vgId, pThread->threadIndex); pThread->failed++; @@ -581,7 +582,8 @@ int32_t dndProcessCreateVnodeReq(SDnode *pDnode, SRpcMsg *pReq) { return -1; } - SVnode *pImpl = vnodeOpen(wrapperCfg.path, NULL /*pCfg*/, pCreate->vgId); + vnodeCfg.pDnode = pDnode; + SVnode *pImpl = vnodeOpen(wrapperCfg.path, &vnodeCfg); if (pImpl == NULL) { dError("vgId:%d, failed to create vnode since %s", pCreate->vgId, terrstr()); return -1; diff --git a/source/dnode/mgmt/impl/src/dnode.c b/source/dnode/mgmt/impl/src/dnode.c index f3016feda5..ea42db96ab 100644 --- a/source/dnode/mgmt/impl/src/dnode.c +++ b/source/dnode/mgmt/impl/src/dnode.c @@ -200,12 +200,11 @@ SDnode *dndInit(SDnodeOpt *pOption) { SVnodeOpt vnodeOpt = { .sver = pDnode->opt.sver, - .pDnode = pDnode, .timezone = pDnode->opt.timezone, .locale = pDnode->opt.locale, .charset = pDnode->opt.charset, - .putReqToVQueryQFp = dndPutMsgToVQueryQ, .nthreads = pDnode->opt.numOfCommitThreads, + .putReqToVQueryQFp = dndPutMsgToVQueryQ, }; if (vnodeInit(&vnodeOpt) != 0) { dError("failed to init vnode env"); diff --git a/source/dnode/vnode/impl/inc/vnodeDef.h b/source/dnode/vnode/impl/inc/vnodeDef.h index 361fdd10e0..4f53dcd899 100644 --- a/source/dnode/vnode/impl/inc/vnodeDef.h +++ b/source/dnode/vnode/impl/inc/vnodeDef.h @@ -77,11 +77,12 @@ struct SVnode { SVnodeFS* pFs; tsem_t canCommit; SQHandle* pQuery; + SDnode* pDnode; }; int vnodeScheduleTask(SVnodeTask* task); -void vnodePutReqToVQueryQ(struct SRpcMsg *pReq); +void vnodePutReqToVQueryQ(SVnode *pVnode, struct SRpcMsg *pReq); #ifdef __cplusplus } diff --git a/source/dnode/vnode/impl/src/vnodeMain.c b/source/dnode/vnode/impl/src/vnodeMain.c index eb4b45bc20..85ccc9879e 100644 --- a/source/dnode/vnode/impl/src/vnodeMain.c +++ b/source/dnode/vnode/impl/src/vnodeMain.c @@ -15,27 +15,29 @@ #include "vnodeDef.h" -static SVnode *vnodeNew(const char *path, const SVnodeCfg *pVnodeCfg, int32_t vid); +static SVnode *vnodeNew(const char *path, const SVnodeCfg *pVnodeCfg); static void vnodeFree(SVnode *pVnode); static int vnodeOpenImpl(SVnode *pVnode); static void vnodeCloseImpl(SVnode *pVnode); -SVnode *vnodeOpen(const char *path, const SVnodeCfg *pVnodeCfg, int32_t vid) { +SVnode *vnodeOpen(const char *path, const SVnodeCfg *pVnodeCfg) { SVnode *pVnode = NULL; // Set default options - //if (pVnodeCfg == NULL) { - pVnodeCfg = &defaultVnodeOptions; - //} + SVnodeCfg cfg = defaultVnodeOptions; + if (pVnodeCfg != NULL) { + cfg.vgId = pVnodeCfg->vgId; + cfg.pDnode = pVnodeCfg->pDnode; + } // Validate options - if (vnodeValidateOptions(pVnodeCfg) < 0) { + if (vnodeValidateOptions(&cfg) < 0) { // TODO return NULL; } // Create the handle - pVnode = vnodeNew(path, pVnodeCfg, vid); + pVnode = vnodeNew(path, &cfg); if (pVnode == NULL) { // TODO: handle error return NULL; @@ -62,7 +64,7 @@ void vnodeClose(SVnode *pVnode) { void vnodeDestroy(const char *path) { taosRemoveDir(path); } /* ------------------------ STATIC METHODS ------------------------ */ -static SVnode *vnodeNew(const char *path, const SVnodeCfg *pVnodeCfg, int32_t vid) { +static SVnode *vnodeNew(const char *path, const SVnodeCfg *pVnodeCfg) { SVnode *pVnode = NULL; pVnode = (SVnode *)calloc(1, sizeof(*pVnode)); @@ -71,7 +73,8 @@ static SVnode *vnodeNew(const char *path, const SVnodeCfg *pVnodeCfg, int32_t vi return NULL; } - pVnode->vgId = vid; + pVnode->vgId = pVnodeCfg->vgId; + pVnode->pDnode = pVnodeCfg->pDnode; pVnode->path = strdup(path); vnodeOptionsCopy(&(pVnode->config), pVnodeCfg); diff --git a/source/dnode/vnode/impl/src/vnodeMgr.c b/source/dnode/vnode/impl/src/vnodeMgr.c index 43527c13fe..51f33031ac 100644 --- a/source/dnode/vnode/impl/src/vnodeMgr.c +++ b/source/dnode/vnode/impl/src/vnodeMgr.c @@ -26,7 +26,6 @@ int vnodeInit(const SVnodeOpt *pOption) { vnodeMgr.stop = false; vnodeMgr.putReqToVQueryQFp = pOption->putReqToVQueryQFp; - vnodeMgr.pDnode = pOption->pDnode; // Start commit handers if (pOption->nthreads > 0) { @@ -91,10 +90,10 @@ int vnodeScheduleTask(SVnodeTask* pTask) { return 0; } -void vnodePutReqToVQueryQ(struct SRpcMsg* pReq) { - if (vnodeMgr.putReqToVQueryQFp) { - (*vnodeMgr.putReqToVQueryQFp)(vnodeMgr.pDnode, pReq); - } +void vnodePutReqToVQueryQ(SVnode* pVnode, struct SRpcMsg* pReq) { + assert(vnodeMgr.putReqToVQueryQFp); + assert(pVnode->pDnode); + (*vnodeMgr.putReqToVQueryQFp)(pVnode->pDnode, pReq); } /* ------------------------ STATIC METHODS ------------------------ */ From 53b4c06d9b1442c0063dc6520fa8982d913f8da3 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 11 Jan 2022 17:07:34 -0800 Subject: [PATCH 30/57] add msg queue --- include/dnode/vnode/vnode.h | 2 +- source/dnode/mgmt/impl/inc/dndVnodes.h | 2 ++ source/dnode/mgmt/impl/src/dndVnodes.c | 26 ++++++++++++++++++++------ source/dnode/mgmt/impl/src/dnode.c | 4 +--- source/dnode/vnode/impl/inc/vnodeDef.h | 2 +- source/dnode/vnode/impl/src/vnodeMgr.c | 10 ++++++---- 6 files changed, 31 insertions(+), 15 deletions(-) diff --git a/include/dnode/vnode/vnode.h b/include/dnode/vnode/vnode.h index 03a59b01e6..4332fc8e58 100644 --- a/include/dnode/vnode/vnode.h +++ b/include/dnode/vnode/vnode.h @@ -32,7 +32,7 @@ extern "C" { /* ------------------------ TYPES EXPOSED ------------------------ */ typedef struct SVnode SVnode; typedef struct SDnode SDnode; -typedef void (*PutReqToVQueryQFp)(SDnode *pDnode, struct SRpcMsg *pReq); +typedef int32_t (*PutReqToVQueryQFp)(SDnode *pDnode, struct SRpcMsg *pReq); typedef struct SVnodeCfg { int32_t vgId; diff --git a/source/dnode/mgmt/impl/inc/dndVnodes.h b/source/dnode/mgmt/impl/inc/dndVnodes.h index b5fae62959..a78db602fe 100644 --- a/source/dnode/mgmt/impl/inc/dndVnodes.h +++ b/source/dnode/mgmt/impl/inc/dndVnodes.h @@ -36,6 +36,8 @@ int32_t dndProcessAuthVnodeReq(SDnode *pDnode, SRpcMsg *pReq); int32_t dndProcessSyncVnodeReq(SDnode *pDnode, SRpcMsg *pReq); int32_t dndProcessCompactVnodeReq(SDnode *pDnode, SRpcMsg *pReq); +int32_t dndPutReqToVQueryQ(SDnode *pDnode, SRpcMsg *pReq); + #ifdef __cplusplus } #endif diff --git a/source/dnode/mgmt/impl/src/dndVnodes.c b/source/dnode/mgmt/impl/src/dndVnodes.c index 9f585859a8..2d7999fe5a 100644 --- a/source/dnode/mgmt/impl/src/dndVnodes.c +++ b/source/dnode/mgmt/impl/src/dndVnodes.c @@ -802,7 +802,7 @@ static void dndProcessVnodeSyncQueue(SVnodeObj *pVnode, STaosQall *qall, int32_t } } -static int32_t dndWriteRpcMsgToVnodeQueue(STaosQueue *pQueue, SRpcMsg *pRpcMsg) { +static int32_t dndWriteRpcMsgToVnodeQueue(STaosQueue *pQueue, SRpcMsg *pRpcMsg, bool sendRsp) { int32_t code = 0; if (pQueue == NULL) { @@ -819,13 +819,15 @@ static int32_t dndWriteRpcMsgToVnodeQueue(STaosQueue *pQueue, SRpcMsg *pRpcMsg) } } - if (code != TSDB_CODE_SUCCESS) { + if (code != TSDB_CODE_SUCCESS && sendRsp) { if (pRpcMsg->msgType & 1u) { SRpcMsg rsp = {.handle = pRpcMsg->handle, .code = code}; rpcSendResponse(&rsp); } rpcFreeCont(pRpcMsg->pCont); } + + return code; } static SVnodeObj *dndAcquireVnodeFromMsg(SDnode *pDnode, SRpcMsg *pMsg) { @@ -848,7 +850,7 @@ static SVnodeObj *dndAcquireVnodeFromMsg(SDnode *pDnode, SRpcMsg *pMsg) { void dndProcessVnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { SVnodeObj *pVnode = dndAcquireVnodeFromMsg(pDnode, pMsg); if (pVnode != NULL) { - dndWriteRpcMsgToVnodeQueue(pVnode->pWriteQ, pMsg); + (void)dndWriteRpcMsgToVnodeQueue(pVnode->pWriteQ, pMsg, true); dndReleaseVnode(pDnode, pVnode); } } @@ -856,7 +858,7 @@ void dndProcessVnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { void dndProcessVnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { SVnodeObj *pVnode = dndAcquireVnodeFromMsg(pDnode, pMsg); if (pVnode != NULL) { - dndWriteRpcMsgToVnodeQueue(pVnode->pSyncQ, pMsg); + (void)dndWriteRpcMsgToVnodeQueue(pVnode->pSyncQ, pMsg, true); dndReleaseVnode(pDnode, pVnode); } } @@ -864,7 +866,7 @@ void dndProcessVnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { void dndProcessVnodeQueryMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { SVnodeObj *pVnode = dndAcquireVnodeFromMsg(pDnode, pMsg); if (pVnode != NULL) { - dndWriteRpcMsgToVnodeQueue(pVnode->pQueryQ, pMsg); + (void)dndWriteRpcMsgToVnodeQueue(pVnode->pQueryQ, pMsg, true); dndReleaseVnode(pDnode, pVnode); } } @@ -872,11 +874,23 @@ void dndProcessVnodeQueryMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { void dndProcessVnodeFetchMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) { SVnodeObj *pVnode = dndAcquireVnodeFromMsg(pDnode, pMsg); if (pVnode != NULL) { - dndWriteRpcMsgToVnodeQueue(pVnode->pFetchQ, pMsg); + (void)dndWriteRpcMsgToVnodeQueue(pVnode->pFetchQ, pMsg, true); dndReleaseVnode(pDnode, pVnode); } } +int32_t dndPutReqToVQueryQ(SDnode *pDnode, SRpcMsg *pMsg) { + SMsgHead *pHead = pMsg->pCont; + // pHead->vgId = htonl(pHead->vgId); + + SVnodeObj *pVnode = dndAcquireVnode(pDnode, pHead->vgId); + if (pVnode == NULL) return -1; + + int32_t code = dndWriteRpcMsgToVnodeQueue(pVnode->pFetchQ, pMsg, false); + dndReleaseVnode(pDnode, pVnode); + return code; +} + static int32_t dndPutMsgIntoVnodeApplyQueue(SDnode *pDnode, int32_t vgId, SRpcMsg *pMsg) { SVnodeObj *pVnode = dndAcquireVnode(pDnode, vgId); if (pVnode == NULL) return -1; diff --git a/source/dnode/mgmt/impl/src/dnode.c b/source/dnode/mgmt/impl/src/dnode.c index ea42db96ab..f3d5e09564 100644 --- a/source/dnode/mgmt/impl/src/dnode.c +++ b/source/dnode/mgmt/impl/src/dnode.c @@ -153,8 +153,6 @@ static void dndCleanupEnv(SDnode *pDnode) { taosStopCacheRefreshWorker(); } -static void dndPutMsgToVQueryQ(SDnode *pDnode, SRpcMsg *pRpcMsg) { dndProcessVnodeQueryMsg(pDnode, pRpcMsg, NULL); } - SDnode *dndInit(SDnodeOpt *pOption) { taosIgnSIGPIPE(); taosBlockSIGPIPE(); @@ -204,7 +202,7 @@ SDnode *dndInit(SDnodeOpt *pOption) { .locale = pDnode->opt.locale, .charset = pDnode->opt.charset, .nthreads = pDnode->opt.numOfCommitThreads, - .putReqToVQueryQFp = dndPutMsgToVQueryQ, + .putReqToVQueryQFp = dndPutReqToVQueryQ, }; if (vnodeInit(&vnodeOpt) != 0) { dError("failed to init vnode env"); diff --git a/source/dnode/vnode/impl/inc/vnodeDef.h b/source/dnode/vnode/impl/inc/vnodeDef.h index 4f53dcd899..f9172dd351 100644 --- a/source/dnode/vnode/impl/inc/vnodeDef.h +++ b/source/dnode/vnode/impl/inc/vnodeDef.h @@ -82,7 +82,7 @@ struct SVnode { int vnodeScheduleTask(SVnodeTask* task); -void vnodePutReqToVQueryQ(SVnode *pVnode, struct SRpcMsg *pReq); +int32_t vnodePutReqToVQueryQ(SVnode *pVnode, struct SRpcMsg *pReq); #ifdef __cplusplus } diff --git a/source/dnode/vnode/impl/src/vnodeMgr.c b/source/dnode/vnode/impl/src/vnodeMgr.c index 51f33031ac..cc369a0d15 100644 --- a/source/dnode/vnode/impl/src/vnodeMgr.c +++ b/source/dnode/vnode/impl/src/vnodeMgr.c @@ -90,10 +90,12 @@ int vnodeScheduleTask(SVnodeTask* pTask) { return 0; } -void vnodePutReqToVQueryQ(SVnode* pVnode, struct SRpcMsg* pReq) { - assert(vnodeMgr.putReqToVQueryQFp); - assert(pVnode->pDnode); - (*vnodeMgr.putReqToVQueryQFp)(pVnode->pDnode, pReq); +int32_t vnodePutReqToVQueryQ(SVnode* pVnode, struct SRpcMsg* pReq) { + if (pVnode == NULL || pVnode->pDnode == NULL || vnodeMgr.putReqToVQueryQFp == NULL) { + terrno = TSDB_CODE_VND_APP_ERROR; + return -1; + } + return (*vnodeMgr.putReqToVQueryQFp)(pVnode->pDnode, pReq); } /* ------------------------ STATIC METHODS ------------------------ */ From a5df21beab302198b655ecd6951ac0bbf5505ead Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 12 Jan 2022 01:26:00 +0000 Subject: [PATCH 31/57] more --- source/libs/tkv/src/inc/tkvDB.h | 31 +++++++++++++++++++++++++++++++ source/libs/tkv/src/inc/tkvEnv.h | 31 +++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100644 source/libs/tkv/src/inc/tkvDB.h create mode 100644 source/libs/tkv/src/inc/tkvEnv.h diff --git a/source/libs/tkv/src/inc/tkvDB.h b/source/libs/tkv/src/inc/tkvDB.h new file mode 100644 index 0000000000..1a45702540 --- /dev/null +++ b/source/libs/tkv/src/inc/tkvDB.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TKV_DB_H_ +#define _TD_TKV_DB_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +struct TDB { + // TODO +}; + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_TKV_DB_H_*/ \ No newline at end of file diff --git a/source/libs/tkv/src/inc/tkvEnv.h b/source/libs/tkv/src/inc/tkvEnv.h new file mode 100644 index 0000000000..eba442e5a5 --- /dev/null +++ b/source/libs/tkv/src/inc/tkvEnv.h @@ -0,0 +1,31 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_TKV_ENV_H_ +#define _TD_TKV_ENV_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +struct TDB_ENV { + char *homeDir; +}; + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_TKV_ENV_H_*/ \ No newline at end of file From 6d10ef5e69678b0d7e3ad350e6991df95925b1f3 Mon Sep 17 00:00:00 2001 From: yihaoDeng Date: Wed, 12 Jan 2022 09:44:23 +0800 Subject: [PATCH 32/57] add libuv --- source/libs/transport/src/rpcMain.c | 2 ++ source/libs/transport/src/rpcTcp.c | 2 ++ 2 files changed, 4 insertions(+) diff --git a/source/libs/transport/src/rpcMain.c b/source/libs/transport/src/rpcMain.c index 802c5ae4ea..3095ddb9d2 100644 --- a/source/libs/transport/src/rpcMain.c +++ b/source/libs/transport/src/rpcMain.c @@ -13,7 +13,9 @@ * along with this program. If not, see . */ +#ifdef USE_UV #include +#endif #include "lz4.h" #include "os.h" #include "rpcCache.h" diff --git a/source/libs/transport/src/rpcTcp.c b/source/libs/transport/src/rpcTcp.c index a3e1f2434b..9fa51a6fdc 100644 --- a/source/libs/transport/src/rpcTcp.c +++ b/source/libs/transport/src/rpcTcp.c @@ -14,7 +14,9 @@ */ #include "rpcTcp.h" +#ifdef USE_UV #include +#endif #include "os.h" #include "rpcHead.h" #include "rpcLog.h" From 25bc2b3a35496adb62d79de1e033193a00261db5 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 11 Jan 2022 17:44:55 -0800 Subject: [PATCH 33/57] rename files --- source/dnode/mgmt/impl/inc/dndBnode.h | 2 +- source/dnode/mgmt/impl/inc/dndEnv.h | 30 +++++++++++++++++++ source/dnode/mgmt/impl/inc/dndMgmt.h | 2 +- source/dnode/mgmt/impl/inc/dndMnode.h | 2 +- source/dnode/mgmt/impl/inc/dndQnode.h | 2 +- source/dnode/mgmt/impl/inc/dndSnode.h | 2 +- source/dnode/mgmt/impl/inc/dndTransport.h | 2 +- source/dnode/mgmt/impl/inc/dndVnodes.h | 2 +- source/dnode/mgmt/impl/inc/dndWorker.h | 2 +- .../dnode/mgmt/impl/src/{dnode.c => dndEnv.c} | 0 10 files changed, 38 insertions(+), 8 deletions(-) create mode 100644 source/dnode/mgmt/impl/inc/dndEnv.h rename source/dnode/mgmt/impl/src/{dnode.c => dndEnv.c} (100%) diff --git a/source/dnode/mgmt/impl/inc/dndBnode.h b/source/dnode/mgmt/impl/inc/dndBnode.h index 853b54ff69..080cd2e487 100644 --- a/source/dnode/mgmt/impl/inc/dndBnode.h +++ b/source/dnode/mgmt/impl/inc/dndBnode.h @@ -19,7 +19,7 @@ #ifdef __cplusplus extern "C" { #endif -#include "dndInt.h" +#include "dndEnv.h" int32_t dndInitBnode(SDnode *pDnode); void dndCleanupBnode(SDnode *pDnode); diff --git a/source/dnode/mgmt/impl/inc/dndEnv.h b/source/dnode/mgmt/impl/inc/dndEnv.h new file mode 100644 index 0000000000..61d9dd93be --- /dev/null +++ b/source/dnode/mgmt/impl/inc/dndEnv.h @@ -0,0 +1,30 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_DND_ENV_H_ +#define _TD_DND_ENV_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "dndEnv.h" + + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_DND_ENV_H_*/ \ No newline at end of file diff --git a/source/dnode/mgmt/impl/inc/dndMgmt.h b/source/dnode/mgmt/impl/inc/dndMgmt.h index 0df3af23d5..9cc0c4ae66 100644 --- a/source/dnode/mgmt/impl/inc/dndMgmt.h +++ b/source/dnode/mgmt/impl/inc/dndMgmt.h @@ -19,7 +19,7 @@ #ifdef __cplusplus extern "C" { #endif -#include "dndInt.h" +#include "dndEnv.h" int32_t dndInitMgmt(SDnode *pDnode); void dndStopMgmt(SDnode *pDnode); diff --git a/source/dnode/mgmt/impl/inc/dndMnode.h b/source/dnode/mgmt/impl/inc/dndMnode.h index 0f7cec230e..dafbae10ad 100644 --- a/source/dnode/mgmt/impl/inc/dndMnode.h +++ b/source/dnode/mgmt/impl/inc/dndMnode.h @@ -19,7 +19,7 @@ #ifdef __cplusplus extern "C" { #endif -#include "dndInt.h" +#include "dndEnv.h" int32_t dndInitMnode(SDnode *pDnode); void dndCleanupMnode(SDnode *pDnode); diff --git a/source/dnode/mgmt/impl/inc/dndQnode.h b/source/dnode/mgmt/impl/inc/dndQnode.h index 677c234679..2a25dca1c6 100644 --- a/source/dnode/mgmt/impl/inc/dndQnode.h +++ b/source/dnode/mgmt/impl/inc/dndQnode.h @@ -19,7 +19,7 @@ #ifdef __cplusplus extern "C" { #endif -#include "dndInt.h" +#include "dndEnv.h" int32_t dndInitQnode(SDnode *pDnode); void dndCleanupQnode(SDnode *pDnode); diff --git a/source/dnode/mgmt/impl/inc/dndSnode.h b/source/dnode/mgmt/impl/inc/dndSnode.h index 8cb883794d..b21e9191e8 100644 --- a/source/dnode/mgmt/impl/inc/dndSnode.h +++ b/source/dnode/mgmt/impl/inc/dndSnode.h @@ -19,7 +19,7 @@ #ifdef __cplusplus extern "C" { #endif -#include "dndInt.h" +#include "dndEnv.h" int32_t dndInitSnode(SDnode *pDnode); void dndCleanupSnode(SDnode *pDnode); diff --git a/source/dnode/mgmt/impl/inc/dndTransport.h b/source/dnode/mgmt/impl/inc/dndTransport.h index 42fb379fc1..e0ea21cba8 100644 --- a/source/dnode/mgmt/impl/inc/dndTransport.h +++ b/source/dnode/mgmt/impl/inc/dndTransport.h @@ -19,7 +19,7 @@ #ifdef __cplusplus extern "C" { #endif -#include "dndInt.h" +#include "dndEnv.h" int32_t dndInitTrans(SDnode *pDnode); void dndCleanupTrans(SDnode *pDnode); diff --git a/source/dnode/mgmt/impl/inc/dndVnodes.h b/source/dnode/mgmt/impl/inc/dndVnodes.h index a78db602fe..f5177778ec 100644 --- a/source/dnode/mgmt/impl/inc/dndVnodes.h +++ b/source/dnode/mgmt/impl/inc/dndVnodes.h @@ -19,7 +19,7 @@ #ifdef __cplusplus extern "C" { #endif -#include "dndInt.h" +#include "dndEnv.h" int32_t dndInitVnodes(SDnode *pDnode); void dndCleanupVnodes(SDnode *pDnode); diff --git a/source/dnode/mgmt/impl/inc/dndWorker.h b/source/dnode/mgmt/impl/inc/dndWorker.h index 49ef88e67d..9c037d91c7 100644 --- a/source/dnode/mgmt/impl/inc/dndWorker.h +++ b/source/dnode/mgmt/impl/inc/dndWorker.h @@ -19,7 +19,7 @@ #ifdef __cplusplus extern "C" { #endif -#include "dndInt.h" +#include "dndEnv.h" int32_t dndInitWorker(SDnode *pDnode, SDnodeWorker *pWorker, EWorkerType type, const char *name, int32_t minNum, int32_t maxNum, void *queueFp); diff --git a/source/dnode/mgmt/impl/src/dnode.c b/source/dnode/mgmt/impl/src/dndEnv.c similarity index 100% rename from source/dnode/mgmt/impl/src/dnode.c rename to source/dnode/mgmt/impl/src/dndEnv.c From 8b5e6b689555f586b43b21b5d4169f0476fe79f9 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 12 Jan 2022 01:55:37 +0000 Subject: [PATCH 34/57] fix memory leakage --- source/dnode/vnode/impl/src/vnodeWrite.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/impl/src/vnodeWrite.c b/source/dnode/vnode/impl/src/vnodeWrite.c index ddcb93863a..185487757f 100644 --- a/source/dnode/vnode/impl/src/vnodeWrite.c +++ b/source/dnode/vnode/impl/src/vnodeWrite.c @@ -83,8 +83,18 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { if (metaCreateTable(pVnode->pMeta, pCreateTbReq) < 0) { // TODO: handle error } + if (pCreateTbReq->type == TD_SUPER_TABLE) { + free(pCreateTbReq->stbCfg.pSchema); + free(pCreateTbReq->stbCfg.pTagSchema); + } else if (pCreateTbReq->type == TD_CHILD_TABLE) { + free(pCreateTbReq->ctbCfg.pTag); + } else { + free(pCreateTbReq->ntbCfg.pSchema); + } } - + taosArrayDestroy(vCreateTbBatchReq.pArray); + break; + case TDMT_VND_DROP_STB: case TDMT_VND_DROP_TABLE: // if (metaDropTable(pVnode->pMeta, vReq.dtReq.uid) < 0) { From 90e25132e1e006797861b4b42ed1ef55335106ca Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 12 Jan 2022 10:16:59 +0800 Subject: [PATCH 35/57] [td-11818] support select * from child_table. --- source/client/inc/clientInt.h | 3 +-- source/client/src/clientImpl.c | 20 +++++++++++++------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 7292375e7f..ebb8502d6c 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -93,13 +93,12 @@ typedef struct SReqResultInfo { const char *pData; TAOS_FIELD *fields; uint32_t numOfCols; - int32_t *length; TAOS_ROW row; char **pCol; - uint32_t numOfRows; uint32_t current; + bool completed; } SReqResultInfo; typedef struct SShowReqInfo { diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index d476670c78..40354998ee 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -25,7 +25,7 @@ static int32_t initEpSetFromCfg(const char *firstEp, const char *secondEp, SCorEpSet *pEpSet); static SMsgSendInfo* buildConnectMsg(SRequestObj *pRequest); static void destroySendMsgInfo(SMsgSendInfo* pMsgBody); -static void setQueryResultByRsp(SReqResultInfo* pResultInfo, SRetrieveTableRsp* pRsp); +static void setQueryResultByRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp); static bool stringLengthCheck(const char* str, size_t maxsize) { if (str == NULL) { @@ -547,12 +547,15 @@ void* doFetchRow(SRequestObj* pRequest) { if (pResultInfo->pData == NULL || pResultInfo->current >= pResultInfo->numOfRows) { if (pRequest->type == TDMT_VND_QUERY) { - pRequest->type = TDMT_VND_FETCH; + // All data has returned to App already, no need to try again + if (pResultInfo->completed) { + return NULL; + } + scheduleFetchRows(pRequest->body.pQueryJob, (void **)&pRequest->body.resInfo.pData); setQueryResultByRsp(&pRequest->body.resInfo, (SRetrieveTableRsp*)pRequest->body.resInfo.pData); - pResultInfo->current = 0; - if (pResultInfo->numOfRows <= pResultInfo->current) { + if (pResultInfo->numOfRows == 0) { return NULL; } @@ -656,11 +659,14 @@ void setConnectionDB(STscObj* pTscObj, const char* db) { pthread_mutex_unlock(&pTscObj->mutex); } -void setQueryResultByRsp(SReqResultInfo* pResultInfo, SRetrieveTableRsp* pRsp) { - assert(pRsp != NULL && pResultInfo != NULL); - pResultInfo->pRspMsg = pRsp; +void setQueryResultByRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp) { + assert(pResultInfo != NULL && pRsp != NULL); + + pResultInfo->pRspMsg = (const char*) pRsp; pResultInfo->pData = (void*) pRsp->data; pResultInfo->numOfRows = htonl(pRsp->numOfRows); + pResultInfo->current = 0; + pResultInfo->completed = (pRsp->completed == 1); setResultDataPtr(pResultInfo, pResultInfo->fields, pResultInfo->numOfCols, pResultInfo->numOfRows); } \ No newline at end of file From 027109727089329cd327dfeb8e4e63a88e6b93dd Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 12 Jan 2022 10:26:00 +0800 Subject: [PATCH 36/57] [td-11818] disable print query plan. --- source/client/src/clientImpl.c | 2 +- source/libs/parser/src/queryInfoUtil.c | 3 +++ source/libs/planner/src/physicalPlanJson.c | 4 ++-- source/libs/planner/src/planner.c | 6 +++--- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 40354998ee..6b2d89c9be 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -230,7 +230,7 @@ void setResSchemaInfo(SReqResultInfo* pResInfo, const SDataBlockSchema* pDataBlo SSchema* pSchema = &pDataBlockSchema->pSchema[i]; pResInfo->fields[i].bytes = pSchema->bytes; pResInfo->fields[i].type = pSchema->type; - tstrncpy(pResInfo->fields[i].name, pSchema[i].name, tListLen(pResInfo->fields[i].name)); + tstrncpy(pResInfo->fields[i].name, pSchema->name, tListLen(pResInfo->fields[i].name)); } } diff --git a/source/libs/parser/src/queryInfoUtil.c b/source/libs/parser/src/queryInfoUtil.c index d7aa758576..1aa8836cb2 100644 --- a/source/libs/parser/src/queryInfoUtil.c +++ b/source/libs/parser/src/queryInfoUtil.c @@ -160,11 +160,14 @@ void addExprInfo(SArray* pExprList, int32_t index, SExprInfo* pExprInfo, int32_t taosArrayInsert(pExprList, index, &pExprInfo); } +#if 0 if (pExprInfo->pExpr->nodeType == TEXPR_FUNCTION_NODE) { printf("add function: %s, level:%d, total:%ld\n", pExprInfo->pExpr->_function.functionName, level, taosArrayGetSize(pExprList)); } else { printf("add operator: %s, level:%d, total:%ld\n", pExprInfo->base.resSchema.name, level, taosArrayGetSize(pExprList)); } +#endif + } void updateExprInfo(SExprInfo* pExprInfo, int16_t functionId, int32_t colId, int16_t srcColumnIndex, int16_t resType, int16_t resSize) { diff --git a/source/libs/planner/src/physicalPlanJson.c b/source/libs/planner/src/physicalPlanJson.c index 64490932fa..7e36e0e124 100644 --- a/source/libs/planner/src/physicalPlanJson.c +++ b/source/libs/planner/src/physicalPlanJson.c @@ -940,8 +940,8 @@ int32_t subPlanToString(const SSubplan* subplan, char** str, int32_t* len) { } *str = cJSON_Print(json); - - printf("%s\n", *str); +// printf("====Physical plan:====\n") +// printf("%s\n", *str); *len = strlen(*str) + 1; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 3047ef4f5a..21f57d95d5 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -65,9 +65,9 @@ int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, } if (pLogicPlan->info.type != QNODE_MODIFY) { - char* str = NULL; - queryPlanToString(pLogicPlan, &str); - printf("%s\n", str); +// char* str = NULL; +// queryPlanToString(pLogicPlan, &str); +// printf("%s\n", str); } code = optimizeQueryPlan(pLogicPlan); From d66067404d92a6794b15ea14313622dbc90f13e0 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 11 Jan 2022 18:46:52 -0800 Subject: [PATCH 37/57] refact dnode module --- include/dnode/mgmt/dnode.h | 48 ++++-- include/util/taoserror.h | 2 + source/dnode/mgmt/daemon/src/daemon.c | 93 +++++++----- source/dnode/mgmt/impl/inc/dndEnv.h | 121 +++++++++++++++- source/dnode/mgmt/impl/inc/dndInt.h | 121 +--------------- source/dnode/mgmt/impl/src/dndBnode.c | 2 +- source/dnode/mgmt/impl/src/dndEnv.c | 137 +++++++++++------- source/dnode/mgmt/impl/src/dndMgmt.c | 26 ++-- source/dnode/mgmt/impl/src/dndMnode.c | 24 +-- source/dnode/mgmt/impl/src/dndQnode.c | 2 +- source/dnode/mgmt/impl/src/dndSnode.c | 2 +- source/dnode/mgmt/impl/src/dndTransport.c | 10 +- source/dnode/mgmt/impl/src/dndVnodes.c | 10 +- source/dnode/mgmt/impl/test/sut/inc/server.h | 2 +- .../dnode/mgmt/impl/test/sut/src/server.cpp | 38 +++-- source/dnode/mgmt/impl/test/sut/src/sut.cpp | 4 + source/util/src/terror.c | 2 + 17 files changed, 356 insertions(+), 288 deletions(-) diff --git a/include/dnode/mgmt/dnode.h b/include/dnode/mgmt/dnode.h index 2e3863c3a1..3f1277a2f9 100644 --- a/include/dnode/mgmt/dnode.h +++ b/include/dnode/mgmt/dnode.h @@ -22,15 +22,39 @@ extern "C" { #endif -/* ------------------------ TYPES EXPOSED ------------------------ */ +/* ------------------------ TYPES EXPOSED ---------------- */ typedef struct SDnode SDnode; +/* ------------------------ Environment ------------------ */ +typedef struct { + int32_t sver; + int32_t numOfCores; + int8_t enableTelem; + char timezone[TSDB_TIMEZONE_LEN]; + char locale[TSDB_LOCALE_LEN]; + char charset[TSDB_LOCALE_LEN]; + char buildinfo[64]; + char gitinfo[48]; +} SDnodeEnvCfg; + +/** + * @brief Initialize the environment + * + * @param pOption Option of the environment + * @return int32_t 0 for success and -1 for failure + */ +int32_t dndInit(const SDnodeEnvCfg *pCfg); + +/** + * @brief clear the environment + * + */ +void dndCleanup(); + +/* ------------------------ SDnode ----------------------- */ typedef struct { - int32_t sver; - int32_t numOfCores; - int32_t numOfSupportVnodes; int16_t numOfCommitThreads; - int8_t enableTelem; + int32_t numOfSupportVnodes; int32_t statusInterval; float numOfThreadsPerCore; float ratioOfQueryCores; @@ -41,28 +65,22 @@ typedef struct { char localEp[TSDB_EP_LEN]; char localFqdn[TSDB_FQDN_LEN]; char firstEp[TSDB_EP_LEN]; - char timezone[TSDB_TIMEZONE_LEN]; - char locale[TSDB_LOCALE_LEN]; - char charset[TSDB_LOCALE_LEN]; - char buildinfo[64]; - char gitinfo[48]; -} SDnodeOpt; +} SDnodeObjCfg; -/* ------------------------ SDnode ------------------------ */ /** * @brief Initialize and start the dnode. * - * @param pOption Option of the dnode. + * @param pCfg Config of the dnode. * @return SDnode* The dnode object. */ -SDnode *dndInit(SDnodeOpt *pOption); +SDnode *dndCreate(SDnodeObjCfg *pCfg); /** * @brief Stop and cleanup the dnode. * * @param pDnode The dnode object to close. */ -void dndCleanup(SDnode *pDnode); +void dndClose(SDnode *pDnode); #ifdef __cplusplus } diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 80241405a6..854d16f67d 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -70,6 +70,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_INVALID_MSG TAOS_DEF_ERROR_CODE(0, 0x0108) #define TSDB_CODE_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0109) #define TSDB_CODE_INVALID_PARA TAOS_DEF_ERROR_CODE(0, 0x010A) +#define TSDB_CODE_REPEAT_INIT TAOS_DEF_ERROR_CODE(0, 0x010B) + #define TSDB_CODE_REF_NO_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0110) #define TSDB_CODE_REF_FULL TAOS_DEF_ERROR_CODE(0, 0x0111) #define TSDB_CODE_REF_ID_REMOVED TAOS_DEF_ERROR_CODE(0, 0x0112) diff --git a/source/dnode/mgmt/daemon/src/daemon.c b/source/dnode/mgmt/daemon/src/daemon.c index cedab6266e..4e98a35483 100644 --- a/source/dnode/mgmt/daemon/src/daemon.c +++ b/source/dnode/mgmt/daemon/src/daemon.c @@ -28,11 +28,11 @@ static struct { bool printAuth; bool printVersion; char configDir[PATH_MAX]; -} global = {0}; +} dmn = {0}; void dmnSigintHandle(int signum, void *info, void *ctx) { uInfo("singal:%d is received", signum); - global.stop = true; + dmn.stop = true; } void dmnSetSignalHandle() { @@ -44,7 +44,7 @@ void dmnSetSignalHandle() { } int dmnParseOption(int argc, char const *argv[]) { - tstrncpy(global.configDir, "/etc/taos", PATH_MAX); + tstrncpy(dmn.configDir, "/etc/taos", PATH_MAX); for (int i = 1; i < argc; ++i) { if (strcmp(argv[i], "-c") == 0) { @@ -53,19 +53,19 @@ int dmnParseOption(int argc, char const *argv[]) { printf("config file path overflow"); return -1; } - tstrncpy(global.configDir, argv[i], PATH_MAX); + tstrncpy(dmn.configDir, argv[i], PATH_MAX); } else { printf("'-c' requires a parameter, default is %s\n", configDir); return -1; } } else if (strcmp(argv[i], "-C") == 0) { - global.dumpConfig = true; + dmn.dumpConfig = true; } else if (strcmp(argv[i], "-k") == 0) { - global.generateGrant = true; + dmn.generateGrant = true; } else if (strcmp(argv[i], "-A") == 0) { - global.printAuth = true; + dmn.printAuth = true; } else if (strcmp(argv[i], "-V") == 0) { - global.printVersion = true; + dmn.printVersion = true; } else { } } @@ -92,7 +92,7 @@ void dmnPrintVersion() { } int dmnReadConfig(const char *path) { - tstrncpy(configDir, global.configDir, PATH_MAX); + tstrncpy(configDir, dmn.configDir, PATH_MAX); taosInitGlobalCfg(); taosReadGlobalLogCfg(); @@ -114,12 +114,12 @@ int dmnReadConfig(const char *path) { } if (taosReadCfgFromFile() != 0) { - uError("failed to read global config"); + uError("failed to read config"); return -1; } if (taosCheckAndPrintCfg() != 0) { - uError("failed to check global config"); + uError("failed to check config"); return -1; } @@ -131,38 +131,50 @@ void dmnDumpConfig() { taosDumpGlobalCfg(); } void dmnWaitSignal() { dmnSetSignalHandle(); - while (!global.stop) { + while (!dmn.stop) { taosMsleep(100); } } -void dmnInitOption(SDnodeOpt *pOption) { - pOption->sver = 30000000; //3.0.0.0 - pOption->numOfCores = tsNumOfCores; - pOption->numOfSupportVnodes = tsNumOfSupportVnodes; - pOption->numOfCommitThreads = tsNumOfCommitThreads; - pOption->statusInterval = tsStatusInterval; - pOption->numOfThreadsPerCore = tsNumOfThreadsPerCore; - pOption->ratioOfQueryCores = tsRatioOfQueryCores; - pOption->maxShellConns = tsMaxShellConns; - pOption->shellActivityTimer = tsShellActivityTimer; - pOption->serverPort = tsServerPort; - tstrncpy(pOption->dataDir, tsDataDir, TSDB_FILENAME_LEN); - tstrncpy(pOption->localEp, tsLocalEp, TSDB_EP_LEN); - tstrncpy(pOption->localFqdn, tsLocalFqdn, TSDB_FQDN_LEN); - tstrncpy(pOption->firstEp, tsFirst, TSDB_EP_LEN); - tstrncpy(pOption->timezone, tsTimezone, TSDB_TIMEZONE_LEN); - tstrncpy(pOption->locale, tsLocale, TSDB_LOCALE_LEN); - tstrncpy(pOption->charset, tsCharset, TSDB_LOCALE_LEN); - tstrncpy(pOption->buildinfo, buildinfo, 64); - tstrncpy(pOption->gitinfo, gitinfo, 48); +void dnmInitEnvCfg(SDnodeEnvCfg *pCfg) { + pCfg->sver = 30000000; // 3.0.0.0 + pCfg->numOfCores = tsNumOfCores; + pCfg->enableTelem = 0; + tstrncpy(pCfg->timezone, tsTimezone, TSDB_TIMEZONE_LEN); + tstrncpy(pCfg->locale, tsLocale, TSDB_LOCALE_LEN); + tstrncpy(pCfg->charset, tsCharset, TSDB_LOCALE_LEN); + tstrncpy(pCfg->buildinfo, buildinfo, 64); + tstrncpy(pCfg->gitinfo, gitinfo, 48); +} + +void dmnInitObjCfg(SDnodeObjCfg *pCfg) { + pCfg->numOfSupportVnodes = tsNumOfSupportVnodes; + pCfg->numOfCommitThreads = tsNumOfCommitThreads; + pCfg->statusInterval = tsStatusInterval; + pCfg->numOfThreadsPerCore = tsNumOfThreadsPerCore; + pCfg->ratioOfQueryCores = tsRatioOfQueryCores; + pCfg->maxShellConns = tsMaxShellConns; + pCfg->shellActivityTimer = tsShellActivityTimer; + pCfg->serverPort = tsServerPort; + tstrncpy(pCfg->dataDir, tsDataDir, TSDB_FILENAME_LEN); + tstrncpy(pCfg->localEp, tsLocalEp, TSDB_EP_LEN); + tstrncpy(pCfg->localFqdn, tsLocalFqdn, TSDB_FQDN_LEN); + tstrncpy(pCfg->firstEp, tsFirst, TSDB_EP_LEN); } int dmnRunDnode() { - SDnodeOpt option = {0}; - dmnInitOption(&option); + SDnodeEnvCfg envCfg = {0}; + SDnodeObjCfg objCfg = {0}; - SDnode *pDnode = dndInit(&option); + dnmInitEnvCfg(&envCfg); + dmnInitObjCfg(&objCfg); + + if (dndInit(&envCfg) != 0) { + uInfo("Failed to start TDengine, please check the log at %s", tsLogDir); + return -1; + } + + SDnode *pDnode = dndCreate(&objCfg); if (pDnode == NULL) { uInfo("Failed to start TDengine, please check the log at %s", tsLogDir); return -1; @@ -172,7 +184,8 @@ int dmnRunDnode() { dmnWaitSignal(); uInfo("TDengine is shut down!"); - dndCleanup(pDnode); + dndClose(pDnode); + dndCleanup(); taosCloseLog(); return 0; } @@ -182,21 +195,21 @@ int main(int argc, char const *argv[]) { return -1; } - if (global.generateGrant) { + if (dmn.generateGrant) { dmnGenerateGrant(); return 0; } - if (global.printVersion) { + if (dmn.printVersion) { dmnPrintVersion(); return 0; } - if (dmnReadConfig(global.configDir) != 0) { + if (dmnReadConfig(dmn.configDir) != 0) { return -1; } - if (global.dumpConfig) { + if (dmn.dumpConfig) { dmnDumpConfig(); return 0; } diff --git a/source/dnode/mgmt/impl/inc/dndEnv.h b/source/dnode/mgmt/impl/inc/dndEnv.h index 61d9dd93be..e80237bb9c 100644 --- a/source/dnode/mgmt/impl/inc/dndEnv.h +++ b/source/dnode/mgmt/impl/inc/dndEnv.h @@ -20,8 +20,127 @@ extern "C" { #endif -#include "dndEnv.h" +#include "dndInt.h" +typedef struct { + EWorkerType type; + const char *name; + int32_t minNum; + int32_t maxNum; + void *queueFp; + SDnode *pDnode; + STaosQueue *queue; + union { + SWorkerPool pool; + SMWorkerPool mpool; + }; +} SDnodeWorker; + +typedef struct { + char *dnode; + char *mnode; + char *snode; + char *bnode; + char *vnodes; +} SDnodeDir; + +typedef struct { + int32_t dnodeId; + int32_t dropped; + int64_t clusterId; + int64_t dver; + int64_t rebootTime; + int64_t updateTime; + int8_t statusSent; + SEpSet mnodeEpSet; + char *file; + SHashObj *dnodeHash; + SDnodeEps *dnodeEps; + pthread_t *threadId; + SRWLatch latch; + SDnodeWorker mgmtWorker; + SDnodeWorker statusWorker; +} SDnodeMgmt; + +typedef struct { + int32_t refCount; + int8_t deployed; + int8_t dropped; + SMnode *pMnode; + SRWLatch latch; + SDnodeWorker readWorker; + SDnodeWorker writeWorker; + SDnodeWorker syncWorker; + int8_t replica; + int8_t selfIndex; + SReplica replicas[TSDB_MAX_REPLICA]; +} SMnodeMgmt; + +typedef struct { + int32_t refCount; + int8_t deployed; + int8_t dropped; + SQnode *pQnode; + SRWLatch latch; + SDnodeWorker queryWorker; + SDnodeWorker fetchWorker; +} SQnodeMgmt; + +typedef struct { + int32_t refCount; + int8_t deployed; + int8_t dropped; + SSnode *pSnode; + SRWLatch latch; + SDnodeWorker writeWorker; +} SSnodeMgmt; + +typedef struct { + int32_t refCount; + int8_t deployed; + int8_t dropped; + SBnode *pBnode; + SRWLatch latch; + SDnodeWorker writeWorker; +} SBnodeMgmt; + +typedef struct { + SHashObj *hash; + int32_t openVnodes; + int32_t totalVnodes; + SRWLatch latch; + SWorkerPool queryPool; + SWorkerPool fetchPool; + SMWorkerPool syncPool; + SMWorkerPool writePool; +} SVnodesMgmt; + +typedef struct { + void *serverRpc; + void *clientRpc; + DndMsgFp msgFp[TDMT_MAX]; +} STransMgmt; + +typedef struct SDnode { + EStat stat; + SDnodeObjCfg cfg; + SDnodeEnvCfg env; + SDnodeDir dir; + FileFd lockFd; + SDnodeMgmt dmgmt; + SMnodeMgmt mmgmt; + SQnodeMgmt qmgmt; + SSnodeMgmt smgmt; + SBnodeMgmt bmgmt; + SVnodesMgmt vmgmt; + STransMgmt tmgmt; + SStartupReq startup; +} SDnode; + +typedef struct { + int8_t once; + SDnodeEnvCfg cfg; +} SDnodeEnv; #ifdef __cplusplus } diff --git a/source/dnode/mgmt/impl/inc/dndInt.h b/source/dnode/mgmt/impl/inc/dndInt.h index e637b38815..c5ba494e5e 100644 --- a/source/dnode/mgmt/impl/inc/dndInt.h +++ b/source/dnode/mgmt/impl/inc/dndInt.h @@ -55,125 +55,12 @@ extern int32_t dDebugFlag; typedef enum { DND_STAT_INIT, DND_STAT_RUNNING, DND_STAT_STOPPED } EStat; typedef enum { DND_WORKER_SINGLE, DND_WORKER_MULTI } EWorkerType; +typedef enum { DND_ENV_INIT = 0, DND_ENV_READY = 1, DND_ENV_CLEANUP = 2 } EEnvStat; typedef void (*DndMsgFp)(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEps); -typedef struct { - EWorkerType type; - const char *name; - int32_t minNum; - int32_t maxNum; - void *queueFp; - SDnode *pDnode; - STaosQueue *queue; - union { - SWorkerPool pool; - SMWorkerPool mpool; - }; -} SDnodeWorker; - -typedef struct { - char *dnode; - char *mnode; - char *snode; - char *bnode; - char *vnodes; -} SDnodeDir; - -typedef struct { - int32_t dnodeId; - int32_t dropped; - int64_t clusterId; - int64_t dver; - int64_t rebootTime; - int64_t updateTime; - int8_t statusSent; - SEpSet mnodeEpSet; - char *file; - SHashObj *dnodeHash; - SDnodeEps *dnodeEps; - pthread_t *threadId; - SRWLatch latch; - SDnodeWorker mgmtWorker; - SDnodeWorker statusWorker; -} SDnodeMgmt; - -typedef struct { - int32_t refCount; - int8_t deployed; - int8_t dropped; - SMnode *pMnode; - SRWLatch latch; - SDnodeWorker readWorker; - SDnodeWorker writeWorker; - SDnodeWorker syncWorker; - int8_t replica; - int8_t selfIndex; - SReplica replicas[TSDB_MAX_REPLICA]; -} SMnodeMgmt; - -typedef struct { - int32_t refCount; - int8_t deployed; - int8_t dropped; - SQnode *pQnode; - SRWLatch latch; - SDnodeWorker queryWorker; - SDnodeWorker fetchWorker; -} SQnodeMgmt; - -typedef struct { - int32_t refCount; - int8_t deployed; - int8_t dropped; - SSnode *pSnode; - SRWLatch latch; - SDnodeWorker writeWorker; -} SSnodeMgmt; - -typedef struct { - int32_t refCount; - int8_t deployed; - int8_t dropped; - SBnode *pBnode; - SRWLatch latch; - SDnodeWorker writeWorker; -} SBnodeMgmt; - -typedef struct { - SHashObj *hash; - int32_t openVnodes; - int32_t totalVnodes; - SRWLatch latch; - SWorkerPool queryPool; - SWorkerPool fetchPool; - SMWorkerPool syncPool; - SMWorkerPool writePool; -} SVnodesMgmt; - -typedef struct { - void *serverRpc; - void *clientRpc; - DndMsgFp msgFp[TDMT_MAX]; -} STransMgmt; - -typedef struct SDnode { - EStat stat; - SDnodeOpt opt; - SDnodeDir dir; - FileFd lockFd; - SDnodeMgmt dmgmt; - SMnodeMgmt mmgmt; - SQnodeMgmt qmgmt; - SSnodeMgmt smgmt; - SBnodeMgmt bmgmt; - SVnodesMgmt vmgmt; - STransMgmt tmgmt; - SStartupReq startup; -} SDnode; - -EStat dndGetStat(SDnode *pDnode); -void dndSetStat(SDnode *pDnode, EStat stat); -char *dndStatStr(EStat stat); +EStat dndGetStat(SDnode *pDnode); +void dndSetStat(SDnode *pDnode, EStat stat); +const char *dndStatStr(EStat stat); void dndReportStartup(SDnode *pDnode, char *pName, char *pDesc); void dndGetStartup(SDnode *pDnode, SStartupReq *pStartup); diff --git a/source/dnode/mgmt/impl/src/dndBnode.c b/source/dnode/mgmt/impl/src/dndBnode.c index 15be59a419..ce058ebba4 100644 --- a/source/dnode/mgmt/impl/src/dndBnode.c +++ b/source/dnode/mgmt/impl/src/dndBnode.c @@ -179,7 +179,7 @@ static void dndBuildBnodeOption(SDnode *pDnode, SBnodeOpt *pOption) { pOption->sendRedirectRspFp = dndSendRedirectRsp; pOption->dnodeId = dndGetDnodeId(pDnode); pOption->clusterId = dndGetClusterId(pDnode); - pOption->cfg.sver = pDnode->opt.sver; + pOption->cfg.sver = pDnode->env.sver; } static int32_t dndOpenBnode(SDnode *pDnode) { diff --git a/source/dnode/mgmt/impl/src/dndEnv.c b/source/dnode/mgmt/impl/src/dndEnv.c index f3d5e09564..44a01049ba 100644 --- a/source/dnode/mgmt/impl/src/dndEnv.c +++ b/source/dnode/mgmt/impl/src/dndEnv.c @@ -25,6 +25,8 @@ #include "tfs.h" #include "wal.h" +static SDnodeEnv dndEnv = {0}; + EStat dndGetStat(SDnode *pDnode) { return pDnode->stat; } void dndSetStat(SDnode *pDnode, EStat stat) { @@ -32,7 +34,7 @@ void dndSetStat(SDnode *pDnode, EStat stat) { pDnode->stat = stat; } -char *dndStatStr(EStat stat) { +const char *dndStatStr(EStat stat) { switch (stat) { case DND_STAT_INIT: return "init"; @@ -79,25 +81,26 @@ static FileFd dndCheckRunning(char *dataDir) { return fd; } -static int32_t dndInitEnv(SDnode *pDnode, SDnodeOpt *pOption) { - pDnode->lockFd = dndCheckRunning(pOption->dataDir); +static int32_t dndCreateImp(SDnode *pDnode, SDnodeObjCfg *pCfg) { + pDnode->lockFd = dndCheckRunning(pCfg->dataDir); if (pDnode->lockFd < 0) { return -1; } char path[PATH_MAX + 100]; - snprintf(path, sizeof(path), "%s%smnode", pOption->dataDir, TD_DIRSEP); + snprintf(path, sizeof(path), "%s%smnode", pCfg->dataDir, TD_DIRSEP); pDnode->dir.mnode = tstrdup(path); - snprintf(path, sizeof(path), "%s%svnode", pOption->dataDir, TD_DIRSEP); + snprintf(path, sizeof(path), "%s%svnode", pCfg->dataDir, TD_DIRSEP); pDnode->dir.vnodes = tstrdup(path); - snprintf(path, sizeof(path), "%s%sdnode", pOption->dataDir, TD_DIRSEP); + snprintf(path, sizeof(path), "%s%sdnode", pCfg->dataDir, TD_DIRSEP); pDnode->dir.dnode = tstrdup(path); - snprintf(path, sizeof(path), "%s%ssnode", pOption->dataDir, TD_DIRSEP); + snprintf(path, sizeof(path), "%s%ssnode", pCfg->dataDir, TD_DIRSEP); pDnode->dir.snode = tstrdup(path); - snprintf(path, sizeof(path), "%s%sbnode", pOption->dataDir, TD_DIRSEP); + snprintf(path, sizeof(path), "%s%sbnode", pCfg->dataDir, TD_DIRSEP); pDnode->dir.bnode = tstrdup(path); - if (pDnode->dir.mnode == NULL || pDnode->dir.vnodes == NULL || pDnode->dir.dnode == NULL) { + if (pDnode->dir.mnode == NULL || pDnode->dir.vnodes == NULL || pDnode->dir.dnode == NULL || + pDnode->dir.snode == NULL || pDnode->dir.bnode == NULL) { dError("failed to malloc dir object"); terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -133,11 +136,12 @@ static int32_t dndInitEnv(SDnode *pDnode, SDnodeOpt *pOption) { return -1; } - memcpy(&pDnode->opt, pOption, sizeof(SDnodeOpt)); + memcpy(&pDnode->cfg, pCfg, sizeof(SDnodeObjCfg)); + memcpy(&pDnode->env, &dndEnv.cfg, sizeof(SDnodeEnvCfg)); return 0; } -static void dndCleanupEnv(SDnode *pDnode) { +static void dndCloseImp(SDnode *pDnode) { tfree(pDnode->dir.mnode); tfree(pDnode->dir.vnodes); tfree(pDnode->dir.dnode); @@ -149,126 +153,121 @@ static void dndCleanupEnv(SDnode *pDnode) { taosCloseFile(pDnode->lockFd); pDnode->lockFd = 0; } - - taosStopCacheRefreshWorker(); } -SDnode *dndInit(SDnodeOpt *pOption) { - taosIgnSIGPIPE(); - taosBlockSIGPIPE(); - taosResolveCRC(); +SDnode *dndCreate(SDnodeObjCfg *pCfg) { + dInfo("start to create dnode object"); SDnode *pDnode = calloc(1, sizeof(SDnode)); if (pDnode == NULL) { - dError("failed to create dnode object"); terrno = TSDB_CODE_OUT_OF_MEMORY; + dError("failed to create dnode object since %s", terrstr()); return NULL; } - dInfo("start to initialize TDengine"); dndSetStat(pDnode, DND_STAT_INIT); - if (dndInitEnv(pDnode, pOption) != 0) { - dError("failed to init env"); - dndCleanup(pDnode); + if (dndCreateImp(pDnode, pCfg) != 0) { + dError("failed to init dnode dir since %s", terrstr()); + dndClose(pDnode); return NULL; } if (rpcInit() != 0) { - dError("failed to init rpc env"); - dndCleanup(pDnode); + dError("failed to init rpc since %s", terrstr()); + dndClose(pDnode); return NULL; } if (walInit() != 0) { - dError("failed to init wal env"); - dndCleanup(pDnode); + dError("failed to init wal since %s", terrstr()); + dndClose(pDnode); return NULL; } SDiskCfg dCfg; - strcpy(dCfg.dir, pDnode->opt.dataDir); + strcpy(dCfg.dir, pDnode->cfg.dataDir); dCfg.level = 0; dCfg.primary = 1; if (tfsInit(&dCfg, 1) != 0) { - dError("failed to init tfs env"); - dndCleanup(pDnode); + dError("failed to init tfs since %s", terrstr()); + dndClose(pDnode); return NULL; } SVnodeOpt vnodeOpt = { - .sver = pDnode->opt.sver, - .timezone = pDnode->opt.timezone, - .locale = pDnode->opt.locale, - .charset = pDnode->opt.charset, - .nthreads = pDnode->opt.numOfCommitThreads, + .sver = pDnode->env.sver, + .timezone = pDnode->env.timezone, + .locale = pDnode->env.locale, + .charset = pDnode->env.charset, + .nthreads = pDnode->cfg.numOfCommitThreads, .putReqToVQueryQFp = dndPutReqToVQueryQ, }; if (vnodeInit(&vnodeOpt) != 0) { - dError("failed to init vnode env"); - dndCleanup(pDnode); + dError("failed to init vnode since %s", terrstr()); + dndClose(pDnode); return NULL; } if (dndInitMgmt(pDnode) != 0) { - dError("failed to init dnode"); - dndCleanup(pDnode); + dError("failed to init mgmt since %s", terrstr()); + dndClose(pDnode); return NULL; } if (dndInitVnodes(pDnode) != 0) { - dError("failed to init vnodes"); - dndCleanup(pDnode); + dError("failed to init vnodes since %s", terrstr()); + dndClose(pDnode); return NULL; } if (dndInitQnode(pDnode) != 0) { - dError("failed to init qnode"); - dndCleanup(pDnode); + dError("failed to init qnode since %s", terrstr()); + dndClose(pDnode); return NULL; } if (dndInitSnode(pDnode) != 0) { - dError("failed to init snode"); - dndCleanup(pDnode); + dError("failed to init snode since %s", terrstr()); + dndClose(pDnode); return NULL; } if (dndInitBnode(pDnode) != 0) { - dError("failed to init bnode"); - dndCleanup(pDnode); + dError("failed to init bnode since %s", terrstr()); + dndClose(pDnode); return NULL; } if (dndInitMnode(pDnode) != 0) { - dError("failed to init mnode"); - dndCleanup(pDnode); + dError("failed to init mnode since %s", terrstr()); + dndClose(pDnode); return NULL; } if (dndInitTrans(pDnode) != 0) { - dError("failed to init transport"); - dndCleanup(pDnode); + dError("failed to init transport since %s", terrstr()); + dndClose(pDnode); return NULL; } dndSetStat(pDnode, DND_STAT_RUNNING); dndSendStatusReq(pDnode); dndReportStartup(pDnode, "TDengine", "initialized successfully"); - dInfo("TDengine is initialized successfully, pDnode:%p", pDnode); + dInfo("dnode object is created, data:%p", pDnode); return pDnode; } -void dndCleanup(SDnode *pDnode) { +void dndClose(SDnode *pDnode) { if (pDnode == NULL) return; if (dndGetStat(pDnode) == DND_STAT_STOPPED) { - dError("dnode is shutting down"); + dError("dnode is shutting down, data:%p", pDnode); return; } - dInfo("start to cleanup TDengine"); + dInfo("start to close dnode, data:%p", pDnode); dndSetStat(pDnode, DND_STAT_STOPPED); dndCleanupTrans(pDnode); dndStopMgmt(pDnode); @@ -283,7 +282,33 @@ void dndCleanup(SDnode *pDnode) { walCleanUp(); rpcCleanup(); - dndCleanupEnv(pDnode); + dndCloseImp(pDnode); free(pDnode); - dInfo("TDengine is cleaned up successfully"); + dInfo("dnode object is closed, data:%p", pDnode); } + +int32_t dndInit(const SDnodeEnvCfg *pCfg) { + if (atomic_val_compare_exchange_8(&dndEnv.once, DND_ENV_INIT, DND_ENV_READY) != DND_ENV_INIT) { + terrno = TSDB_CODE_REPEAT_INIT; + dError("failed to init dnode env since %s", terrstr()); + return -1; + } + + taosIgnSIGPIPE(); + taosBlockSIGPIPE(); + taosResolveCRC(); + + memcpy(&dndEnv.cfg, pCfg, sizeof(SDnodeEnvCfg)); + dInfo("dnode env is initialized"); + return 0; +} + +void dndCleanup() { + if (atomic_val_compare_exchange_8(&dndEnv.once, DND_ENV_READY, DND_ENV_CLEANUP) != DND_ENV_READY) { + dError("dnode env is already cleaned up"); + return; + } + + taosStopCacheRefreshWorker(); + dInfo("dnode env is cleaned up"); +} \ No newline at end of file diff --git a/source/dnode/mgmt/impl/src/dndMgmt.c b/source/dnode/mgmt/impl/src/dndMgmt.c index d9edf39b73..507f9a2644 100644 --- a/source/dnode/mgmt/impl/src/dndMgmt.c +++ b/source/dnode/mgmt/impl/src/dndMgmt.c @@ -86,7 +86,7 @@ void dndSendRedirectRsp(SDnode *pDnode, SRpcMsg *pReq) { dDebug("RPC %p, req:%s is redirected, num:%d use:%d", pReq->handle, TMSG_INFO(msgType), epSet.numOfEps, epSet.inUse); for (int32_t i = 0; i < epSet.numOfEps; ++i) { dDebug("mnode index:%d %s:%u", i, epSet.fqdn[i], epSet.port[i]); - if (strcmp(epSet.fqdn[i], pDnode->opt.localFqdn) == 0 && epSet.port[i] == pDnode->opt.serverPort) { + if (strcmp(epSet.fqdn[i], pDnode->cfg.localFqdn) == 0 && epSet.port[i] == pDnode->cfg.serverPort) { epSet.inUse = (i + 1) % epSet.numOfEps; } @@ -289,8 +289,8 @@ PRASE_DNODE_OVER: if (root != NULL) cJSON_Delete(root); if (fp != NULL) fclose(fp); - if (dndIsEpChanged(pDnode, pMgmt->dnodeId, pDnode->opt.localEp)) { - dError("localEp %s different with %s and need reconfigured", pDnode->opt.localEp, pMgmt->file); + if (dndIsEpChanged(pDnode, pMgmt->dnodeId, pDnode->cfg.localEp)) { + dError("localEp %s different with %s and need reconfigured", pDnode->cfg.localEp, pMgmt->file); return -1; } @@ -298,7 +298,7 @@ PRASE_DNODE_OVER: pMgmt->dnodeEps = calloc(1, sizeof(SDnodeEps) + sizeof(SDnodeEp)); pMgmt->dnodeEps->num = 1; pMgmt->dnodeEps->eps[0].isMnode = 1; - taosGetFqdnPortFromEp(pDnode->opt.firstEp, pMgmt->dnodeEps->eps[0].fqdn, &pMgmt->dnodeEps->eps[0].port); + taosGetFqdnPortFromEp(pDnode->cfg.firstEp, pMgmt->dnodeEps->eps[0].fqdn, &pMgmt->dnodeEps->eps[0].port); } dndResetDnodes(pDnode, pMgmt->dnodeEps); @@ -362,24 +362,24 @@ void dndSendStatusReq(SDnode *pDnode) { SDnodeMgmt *pMgmt = &pDnode->dmgmt; taosRLockLatch(&pMgmt->latch); - pStatus->sver = htonl(pDnode->opt.sver); + pStatus->sver = htonl(pDnode->env.sver); pStatus->dver = htobe64(pMgmt->dver); pStatus->dnodeId = htonl(pMgmt->dnodeId); pStatus->clusterId = htobe64(pMgmt->clusterId); pStatus->rebootTime = htobe64(pMgmt->rebootTime); pStatus->updateTime = htobe64(pMgmt->updateTime); - pStatus->numOfCores = htonl(pDnode->opt.numOfCores); - pStatus->numOfSupportVnodes = htonl(pDnode->opt.numOfSupportVnodes); - tstrncpy(pStatus->dnodeEp, pDnode->opt.localEp, TSDB_EP_LEN); + pStatus->numOfCores = htonl(pDnode->env.numOfCores); + pStatus->numOfSupportVnodes = htonl(pDnode->cfg.numOfSupportVnodes); + tstrncpy(pStatus->dnodeEp, pDnode->cfg.localEp, TSDB_EP_LEN); - pStatus->clusterCfg.statusInterval = htonl(pDnode->opt.statusInterval); + pStatus->clusterCfg.statusInterval = htonl(pDnode->cfg.statusInterval); pStatus->clusterCfg.checkTime = 0; char timestr[32] = "1970-01-01 00:00:00.00"; (void)taosParseTime(timestr, &pStatus->clusterCfg.checkTime, (int32_t)strlen(timestr), TSDB_TIME_PRECISION_MILLI, 0); pStatus->clusterCfg.checkTime = htonl(pStatus->clusterCfg.checkTime); - tstrncpy(pStatus->clusterCfg.timezone, pDnode->opt.timezone, TSDB_TIMEZONE_LEN); - tstrncpy(pStatus->clusterCfg.locale, pDnode->opt.locale, TSDB_LOCALE_LEN); - tstrncpy(pStatus->clusterCfg.charset, pDnode->opt.charset, TSDB_LOCALE_LEN); + tstrncpy(pStatus->clusterCfg.timezone, pDnode->env.timezone, TSDB_TIMEZONE_LEN); + tstrncpy(pStatus->clusterCfg.locale, pDnode->env.locale, TSDB_LOCALE_LEN); + tstrncpy(pStatus->clusterCfg.charset, pDnode->env.charset, TSDB_LOCALE_LEN); taosRUnLockLatch(&pMgmt->latch); dndGetVnodeLoads(pDnode, &pStatus->vnodeLoads); @@ -485,7 +485,7 @@ void dndProcessStartupReq(SDnode *pDnode, SRpcMsg *pReq) { static void *dnodeThreadRoutine(void *param) { SDnode *pDnode = param; SDnodeMgmt *pMgmt = &pDnode->dmgmt; - int32_t ms = pDnode->opt.statusInterval * 1000; + int32_t ms = pDnode->cfg.statusInterval * 1000; while (true) { pthread_testcancel(); diff --git a/source/dnode/mgmt/impl/src/dndMnode.c b/source/dnode/mgmt/impl/src/dndMnode.c index 6c23af7f00..32730fc1e3 100644 --- a/source/dnode/mgmt/impl/src/dndMnode.c +++ b/source/dnode/mgmt/impl/src/dndMnode.c @@ -247,7 +247,7 @@ static bool dndNeedDeployMnode(SDnode *pDnode) { return false; } - if (strcmp(pDnode->opt.localEp, pDnode->opt.firstEp) != 0) { + if (strcmp(pDnode->cfg.localEp, pDnode->cfg.firstEp) != 0) { return false; } @@ -266,15 +266,15 @@ static void dndInitMnodeOption(SDnode *pDnode, SMnodeOpt *pOption) { pOption->putReqToMWriteQFp = dndPutMsgToMWriteQ; pOption->dnodeId = dndGetDnodeId(pDnode); pOption->clusterId = dndGetClusterId(pDnode); - pOption->cfg.sver = pDnode->opt.sver; - pOption->cfg.enableTelem = pDnode->opt.enableTelem; - pOption->cfg.statusInterval = pDnode->opt.statusInterval; - pOption->cfg.shellActivityTimer = pDnode->opt.shellActivityTimer; - pOption->cfg.timezone = pDnode->opt.timezone; - pOption->cfg.charset = pDnode->opt.charset; - pOption->cfg.locale = pDnode->opt.locale; - pOption->cfg.gitinfo = pDnode->opt.gitinfo; - pOption->cfg.buildinfo = pDnode->opt.buildinfo; + pOption->cfg.sver = pDnode->env.sver; + pOption->cfg.enableTelem = pDnode->env.enableTelem; + pOption->cfg.statusInterval = pDnode->cfg.statusInterval; + pOption->cfg.shellActivityTimer = pDnode->cfg.shellActivityTimer; + pOption->cfg.timezone = pDnode->env.timezone; + pOption->cfg.charset = pDnode->env.charset; + pOption->cfg.locale = pDnode->env.locale; + pOption->cfg.gitinfo = pDnode->env.gitinfo; + pOption->cfg.buildinfo = pDnode->env.buildinfo; } static void dndBuildMnodeDeployOption(SDnode *pDnode, SMnodeOpt *pOption) { @@ -283,8 +283,8 @@ static void dndBuildMnodeDeployOption(SDnode *pDnode, SMnodeOpt *pOption) { pOption->selfIndex = 0; SReplica *pReplica = &pOption->replicas[0]; pReplica->id = 1; - pReplica->port = pDnode->opt.serverPort; - memcpy(pReplica->fqdn, pDnode->opt.localFqdn, TSDB_FQDN_LEN); + pReplica->port = pDnode->cfg.serverPort; + memcpy(pReplica->fqdn, pDnode->cfg.localFqdn, TSDB_FQDN_LEN); SMnodeMgmt *pMgmt = &pDnode->mmgmt; pMgmt->selfIndex = pOption->selfIndex; diff --git a/source/dnode/mgmt/impl/src/dndQnode.c b/source/dnode/mgmt/impl/src/dndQnode.c index 9d2f623c45..9a8d49010c 100644 --- a/source/dnode/mgmt/impl/src/dndQnode.c +++ b/source/dnode/mgmt/impl/src/dndQnode.c @@ -185,7 +185,7 @@ static void dndBuildQnodeOption(SDnode *pDnode, SQnodeOpt *pOption) { pOption->sendRedirectRspFp = dndSendRedirectRsp; pOption->dnodeId = dndGetDnodeId(pDnode); pOption->clusterId = dndGetClusterId(pDnode); - pOption->cfg.sver = pDnode->opt.sver; + pOption->cfg.sver = pDnode->env.sver; } static int32_t dndOpenQnode(SDnode *pDnode) { diff --git a/source/dnode/mgmt/impl/src/dndSnode.c b/source/dnode/mgmt/impl/src/dndSnode.c index 00435d4c3e..a07f02e761 100644 --- a/source/dnode/mgmt/impl/src/dndSnode.c +++ b/source/dnode/mgmt/impl/src/dndSnode.c @@ -179,7 +179,7 @@ static void dndBuildSnodeOption(SDnode *pDnode, SSnodeOpt *pOption) { pOption->sendRedirectRspFp = dndSendRedirectRsp; pOption->dnodeId = dndGetDnodeId(pDnode); pOption->clusterId = dndGetClusterId(pDnode); - pOption->cfg.sver = pDnode->opt.sver; + pOption->cfg.sver = pDnode->env.sver; } static int32_t dndOpenSnode(SDnode *pDnode) { diff --git a/source/dnode/mgmt/impl/src/dndTransport.c b/source/dnode/mgmt/impl/src/dndTransport.c index 509e8f4cab..ab4ae4ac53 100644 --- a/source/dnode/mgmt/impl/src/dndTransport.c +++ b/source/dnode/mgmt/impl/src/dndTransport.c @@ -176,7 +176,7 @@ static int32_t dndInitClient(SDnode *pDnode) { rpcInit.cfp = dndProcessResponse; rpcInit.sessions = 1024; rpcInit.connType = TAOS_CONN_CLIENT; - rpcInit.idleTime = pDnode->opt.shellActivityTimer * 1000; + rpcInit.idleTime = pDnode->cfg.shellActivityTimer * 1000; rpcInit.user = INTERNAL_USER; rpcInit.ckey = INTERNAL_CKEY; rpcInit.secret = INTERNAL_SECRET; @@ -325,20 +325,20 @@ static int32_t dndInitServer(SDnode *pDnode) { STransMgmt *pMgmt = &pDnode->tmgmt; dndInitMsgFp(pMgmt); - int32_t numOfThreads = (int32_t)((pDnode->opt.numOfCores * pDnode->opt.numOfThreadsPerCore) / 2.0); + int32_t numOfThreads = (int32_t)((pDnode->env.numOfCores * pDnode->cfg.numOfThreadsPerCore) / 2.0); if (numOfThreads < 1) { numOfThreads = 1; } SRpcInit rpcInit; memset(&rpcInit, 0, sizeof(rpcInit)); - rpcInit.localPort = pDnode->opt.serverPort; + rpcInit.localPort = pDnode->cfg.serverPort; rpcInit.label = "DND-S"; rpcInit.numOfThreads = numOfThreads; rpcInit.cfp = dndProcessRequest; - rpcInit.sessions = pDnode->opt.maxShellConns; + rpcInit.sessions = pDnode->cfg.maxShellConns; rpcInit.connType = TAOS_CONN_SERVER; - rpcInit.idleTime = pDnode->opt.shellActivityTimer * 1000; + rpcInit.idleTime = pDnode->cfg.shellActivityTimer * 1000; rpcInit.afp = dndRetrieveUserAuthInfo; rpcInit.parent = pDnode; diff --git a/source/dnode/mgmt/impl/src/dndVnodes.c b/source/dnode/mgmt/impl/src/dndVnodes.c index 2d7999fe5a..156009fa68 100644 --- a/source/dnode/mgmt/impl/src/dndVnodes.c +++ b/source/dnode/mgmt/impl/src/dndVnodes.c @@ -420,7 +420,7 @@ static int32_t dndOpenVnodes(SDnode *pDnode) { pMgmt->totalVnodes = numOfVnodes; - int32_t threadNum = pDnode->opt.numOfCores; + int32_t threadNum = pDnode->env.numOfCores; int32_t vnodesPerThread = numOfVnodes / threadNum + 1; SVnodeThread *threads = calloc(threadNum, sizeof(SVnodeThread)); @@ -904,11 +904,11 @@ static int32_t dndInitVnodeWorkers(SDnode *pDnode) { SVnodesMgmt *pMgmt = &pDnode->vmgmt; int32_t maxFetchThreads = 4; - int32_t minFetchThreads = MIN(maxFetchThreads, pDnode->opt.numOfCores); - int32_t minQueryThreads = MAX((int32_t)(pDnode->opt.numOfCores * pDnode->opt.ratioOfQueryCores), 1); + int32_t minFetchThreads = MIN(maxFetchThreads, pDnode->env.numOfCores); + int32_t minQueryThreads = MAX((int32_t)(pDnode->env.numOfCores * pDnode->cfg.ratioOfQueryCores), 1); int32_t maxQueryThreads = minQueryThreads; - int32_t maxWriteThreads = MAX(pDnode->opt.numOfCores, 1); - int32_t maxSyncThreads = MAX(pDnode->opt.numOfCores / 2, 1); + int32_t maxWriteThreads = MAX(pDnode->env.numOfCores, 1); + int32_t maxSyncThreads = MAX(pDnode->env.numOfCores / 2, 1); SWorkerPool *pPool = &pMgmt->queryPool; pPool->name = "vnode-query"; diff --git a/source/dnode/mgmt/impl/test/sut/inc/server.h b/source/dnode/mgmt/impl/test/sut/inc/server.h index 5f9e4846a7..99554a7aa7 100644 --- a/source/dnode/mgmt/impl/test/sut/inc/server.h +++ b/source/dnode/mgmt/impl/test/sut/inc/server.h @@ -24,7 +24,7 @@ class TestServer { bool DoStart(); private: - SDnodeOpt BuildOption(const char* path, const char* fqdn, uint16_t port, const char* firstEp); + SDnodeObjCfg BuildOption(const char* path, const char* fqdn, uint16_t port, const char* firstEp); private: SDnode* pDnode; diff --git a/source/dnode/mgmt/impl/test/sut/src/server.cpp b/source/dnode/mgmt/impl/test/sut/src/server.cpp index fb2974294c..f7007bbbab 100644 --- a/source/dnode/mgmt/impl/test/sut/src/server.cpp +++ b/source/dnode/mgmt/impl/test/sut/src/server.cpp @@ -22,30 +22,28 @@ void* serverLoop(void* param) { } } -SDnodeOpt TestServer::BuildOption(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { - SDnodeOpt option = {0}; - option.sver = 1; - option.numOfCores = 1; - option.numOfSupportVnodes = 16; - option.numOfCommitThreads = 1; - option.statusInterval = 1; - option.numOfThreadsPerCore = 1; - option.ratioOfQueryCores = 1; - option.maxShellConns = 1000; - option.shellActivityTimer = 30; - option.serverPort = port; - strcpy(option.dataDir, path); - snprintf(option.localEp, TSDB_EP_LEN, "%s:%u", fqdn, port); - snprintf(option.localFqdn, TSDB_FQDN_LEN, "%s", fqdn); - snprintf(option.firstEp, TSDB_EP_LEN, "%s", firstEp); - return option; +SDnodeObjCfg TestServer::BuildOption(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { + SDnodeObjCfg cfg = {0}; + cfg.numOfSupportVnodes = 16; + cfg.numOfCommitThreads = 1; + cfg.statusInterval = 1; + cfg.numOfThreadsPerCore = 1; + cfg.ratioOfQueryCores = 1; + cfg.maxShellConns = 1000; + cfg.shellActivityTimer = 30; + cfg.serverPort = port; + strcpy(cfg.dataDir, path); + snprintf(cfg.localEp, TSDB_EP_LEN, "%s:%u", fqdn, port); + snprintf(cfg.localFqdn, TSDB_FQDN_LEN, "%s", fqdn); + snprintf(cfg.firstEp, TSDB_EP_LEN, "%s", firstEp); + return cfg; } bool TestServer::DoStart() { - SDnodeOpt option = BuildOption(path, fqdn, port, firstEp); + SDnodeObjCfg cfg = BuildOption(path, fqdn, port, firstEp); taosMkDir(path); - pDnode = dndInit(&option); + pDnode = dndCreate(&cfg); if (pDnode != NULL) { return false; } @@ -81,7 +79,7 @@ void TestServer::Stop() { } if (pDnode != NULL) { - dndCleanup(pDnode); + dndClose(pDnode); pDnode = NULL; } } diff --git a/source/dnode/mgmt/impl/test/sut/src/sut.cpp b/source/dnode/mgmt/impl/test/sut/src/sut.cpp index ba530dbdd0..5912da5bdf 100644 --- a/source/dnode/mgmt/impl/test/sut/src/sut.cpp +++ b/source/dnode/mgmt/impl/test/sut/src/sut.cpp @@ -43,6 +43,9 @@ void Testbase::InitLog(const char* path) { } void Testbase::Init(const char* path, int16_t port) { + SDnodeEnvCfg cfg = {0}; + dndInit(&cfg); + char fqdn[] = "localhost"; char firstEp[TSDB_EP_LEN] = {0}; snprintf(firstEp, TSDB_EP_LEN, "%s:%u", fqdn, port); @@ -56,6 +59,7 @@ void Testbase::Init(const char* path, int16_t port) { void Testbase::Cleanup() { server.Stop(); client.Cleanup(); + dndCleanup(); } void Testbase::Restart() { server.Restart(); } diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 3ea564722b..8294bca959 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -80,6 +80,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_CHECKSUM_ERROR, "Checksum error") TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_MSG, "Invalid config message") TAOS_DEFINE_ERROR(TSDB_CODE_MSG_NOT_PROCESSED, "Message not processed") TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_PARA, "Invalid parameters") +TAOS_DEFINE_ERROR(TSDB_CODE_REPEAT_INIT, "Repeat initialization") + TAOS_DEFINE_ERROR(TSDB_CODE_REF_NO_MEMORY, "Ref out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_REF_FULL, "too many Ref Objs") TAOS_DEFINE_ERROR(TSDB_CODE_REF_ID_REMOVED, "Ref ID is removed") From 35b0ee1a10a6930c09e7b81abd287d94d15868ea Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 12 Jan 2022 10:53:53 +0800 Subject: [PATCH 38/57] [td-11818]remove SParseBasicCtx --- include/libs/parser/parsenodes.h | 9 -------- include/libs/parser/parser.h | 8 ++++++- source/client/src/clientImpl.c | 13 +++++++----- source/libs/parser/inc/astToMsg.h | 8 +++---- source/libs/parser/inc/parserInt.h | 8 +++---- source/libs/parser/inc/parserUtil.h | 2 +- source/libs/parser/src/astToMsg.c | 8 +++---- source/libs/parser/src/astValidate.c | 4 ++-- source/libs/parser/src/dCDAstProcess.c | 10 ++++----- source/libs/parser/src/insertParser.c | 8 +++---- source/libs/parser/src/parser.c | 14 ++++++------ source/libs/parser/src/parserUtil.c | 2 +- source/libs/parser/test/parserTests.cpp | 26 +++++++++++------------ source/libs/parser/test/plannerTest.cpp | 4 ++-- source/libs/parser/test/tokenizerTest.cpp | 2 +- 15 files changed, 63 insertions(+), 63 deletions(-) diff --git a/include/libs/parser/parsenodes.h b/include/libs/parser/parsenodes.h index ac8a10067d..374ae02e4a 100644 --- a/include/libs/parser/parsenodes.h +++ b/include/libs/parser/parsenodes.h @@ -43,15 +43,6 @@ typedef struct SField { int32_t bytes; } SField; -typedef struct SParseBasicCtx { - uint64_t requestId; - int32_t acctId; - const char *db; - void *pTransporter; - SEpSet mgmtEpSet; - struct SCatalog *pCatalog; -} SParseBasicCtx; - typedef struct SFieldInfo { int16_t numOfOutput; // number of column in result SField *final; diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index edf9cf461f..0cdbfdb07f 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -23,11 +23,17 @@ extern "C" { #include "parsenodes.h" typedef struct SParseContext { - SParseBasicCtx ctx; + uint64_t requestId; + int32_t acctId; + const char *db; + void *pTransporter; + SEpSet mgmtEpSet; const char *pSql; // sql string size_t sqlLen; // length of the sql string char *pMsg; // extended error message if exists to help identifying the problem in sql statement. int32_t msgLen; // max length of the msg + + struct SCatalog *pCatalog; } SParseContext; /** diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 6b2d89c9be..a614d2c3c1 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -151,23 +151,26 @@ int32_t parseSql(SRequestObj* pRequest, SQueryNode** pQuery) { STscObj* pTscObj = pRequest->pTscObj; SParseContext cxt = { - .ctx = {.requestId = pRequest->requestId, .acctId = pTscObj->acctId, .db = getConnectionDB(pTscObj), .pTransporter = pTscObj->pTransporter}, + .requestId = pRequest->requestId, + .acctId = pTscObj->acctId, + .db = getConnectionDB(pTscObj), + .pTransporter = pTscObj->pTransporter, .pSql = pRequest->sqlstr, .sqlLen = pRequest->sqlLen, .pMsg = pRequest->msgBuf, .msgLen = ERROR_MSG_BUF_DEFAULT_SIZE }; - cxt.ctx.mgmtEpSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); - int32_t code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &cxt.ctx.pCatalog); + cxt.mgmtEpSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp); + int32_t code = catalogGetHandle(pTscObj->pAppInfo->clusterId, &cxt.pCatalog); if (code != TSDB_CODE_SUCCESS) { - tfree(cxt.ctx.db); + tfree(cxt.db); return code; } code = qParseQuerySql(&cxt, pQuery); - tfree(cxt.ctx.db); + tfree(cxt.db); return code; } diff --git a/source/libs/parser/inc/astToMsg.h b/source/libs/parser/inc/astToMsg.h index 83683c2bfc..8acbc6bc11 100644 --- a/source/libs/parser/inc/astToMsg.h +++ b/source/libs/parser/inc/astToMsg.h @@ -8,10 +8,10 @@ SCreateUserReq* buildUserManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen); SCreateAcctReq* buildAcctManipulationMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen); SDropUserReq* buildDropUserMsg(SSqlInfo* pInfo, int32_t* outputLen, int64_t id, char* msgBuf, int32_t msgLen); -SShowReq* buildShowMsg(SShowInfo* pShowInfo, SParseBasicCtx* pParseCtx, char* msgBuf, int32_t msgLen); -SCreateDbReq* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, SParseBasicCtx *pCtx, SMsgBuf* pMsgBuf); -SMCreateStbReq* buildCreateStbMsg(SCreateTableSql* pCreateTableSql, int32_t* len, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf); -SMDropStbReq* buildDropStableMsg(SSqlInfo* pInfo, int32_t* len, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf); +SShowReq* buildShowMsg(SShowInfo* pShowInfo, SParseContext* pParseCtx, char* msgBuf, int32_t msgLen); +SCreateDbReq* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, SParseContext *pCtx, SMsgBuf* pMsgBuf); +SMCreateStbReq* buildCreateStbMsg(SCreateTableSql* pCreateTableSql, int32_t* len, SParseContext* pParseCtx, SMsgBuf* pMsgBuf); +SMDropStbReq* buildDropStableMsg(SSqlInfo* pInfo, int32_t* len, SParseContext* pParseCtx, SMsgBuf* pMsgBuf); SCreateDnodeReq *buildCreateDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf); SDropDnodeReq *buildDropDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf); diff --git a/source/libs/parser/inc/parserInt.h b/source/libs/parser/inc/parserInt.h index 10ec335fc8..abb19b3ab6 100644 --- a/source/libs/parser/inc/parserInt.h +++ b/source/libs/parser/inc/parserInt.h @@ -51,7 +51,7 @@ void clearAllTableMetaInfo(SQueryStmtInfo* pQueryInfo, bool removeMeta, uint64_t * @param msgBufLen * @return */ -int32_t qParserValidateSqlNode(SParseBasicCtx *pCtx, SSqlInfo* pInfo, SQueryStmtInfo* pQueryInfo, char* msgBuf, int32_t msgBufLen); +int32_t qParserValidateSqlNode(SParseContext *pCtx, SSqlInfo* pInfo, SQueryStmtInfo* pQueryInfo, char* msgBuf, int32_t msgBufLen); /** * validate the ddl ast, and convert the ast to the corresponding message format @@ -60,7 +60,7 @@ int32_t qParserValidateSqlNode(SParseBasicCtx *pCtx, SSqlInfo* pInfo, SQueryStmt * @param type * @return */ -SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, char* msgBuf, int32_t msgBufLen); +SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseContext* pCtx, char* msgBuf, int32_t msgBufLen); /** * @@ -70,7 +70,7 @@ SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, c * @param msgBufLen * @return */ -SVnodeModifOpStmtInfo* qParserValidateCreateTbSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, char* msgBuf, int32_t msgBufLen); +SVnodeModifOpStmtInfo* qParserValidateCreateTbSqlNode(SSqlInfo* pInfo, SParseContext* pCtx, char* msgBuf, int32_t msgBufLen); /** * Evaluate the numeric and timestamp arithmetic expression in the WHERE clause. @@ -98,7 +98,7 @@ int32_t checkForInvalidExpr(SQueryStmtInfo* pQueryInfo, SMsgBuf* pMsgBuf); * @param msgBufLen * @return */ -int32_t qParserExtractRequestedMetaInfo(const SSqlInfo* pSqlInfo, SCatalogReq* pMetaInfo, SParseBasicCtx *pCtx, char* msg, int32_t msgBufLen); +int32_t qParserExtractRequestedMetaInfo(const SSqlInfo* pSqlInfo, SCatalogReq* pMetaInfo, SParseContext *pCtx, char* msg, int32_t msgBufLen); /** * Destroy the meta data request structure. diff --git a/source/libs/parser/inc/parserUtil.h b/source/libs/parser/inc/parserUtil.h index 764b363394..d660d36d3f 100644 --- a/source/libs/parser/inc/parserUtil.h +++ b/source/libs/parser/inc/parserUtil.h @@ -81,7 +81,7 @@ int32_t KvRowAppend(const void *value, int32_t len, void *param); typedef int32_t (*_row_append_fn_t)(const void *value, int32_t len, void *param); int32_t parseValueToken(char** end, SToken* pToken, SSchema* pSchema, int16_t timePrec, char* tmpTokenBuf, _row_append_fn_t func, void* param, SMsgBuf* pMsgBuf); -int32_t createSName(SName* pName, SToken* pTableName, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf); +int32_t createSName(SName* pName, SToken* pTableName, SParseContext* pParseCtx, SMsgBuf* pMsgBuf); #ifdef __cplusplus } diff --git a/source/libs/parser/src/astToMsg.c b/source/libs/parser/src/astToMsg.c index 5f45ce824e..5b841594e0 100644 --- a/source/libs/parser/src/astToMsg.c +++ b/source/libs/parser/src/astToMsg.c @@ -85,7 +85,7 @@ SDropUserReq* buildDropUserMsg(SSqlInfo* pInfo, int32_t *msgLen, int64_t id, cha return pMsg; } -SShowReq* buildShowMsg(SShowInfo* pShowInfo, SParseBasicCtx *pCtx, char* msgBuf, int32_t msgLen) { +SShowReq* buildShowMsg(SShowInfo* pShowInfo, SParseContext *pCtx, char* msgBuf, int32_t msgLen) { SShowReq* pShowMsg = calloc(1, sizeof(SShowReq)); pShowMsg->type = pShowInfo->showType; @@ -210,7 +210,7 @@ int32_t setDbOptions(SCreateDbReq* pCreateDbMsg, const SCreateDbInfo* pCreateDbS return TSDB_CODE_SUCCESS; } -SCreateDbReq* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, SParseBasicCtx *pCtx, SMsgBuf* pMsgBuf) { +SCreateDbReq* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, SParseContext *pCtx, SMsgBuf* pMsgBuf) { SCreateDbReq* pCreateMsg = calloc(1, sizeof(SCreateDbReq)); if (setDbOptions(pCreateMsg, pCreateDbInfo, pMsgBuf) != TSDB_CODE_SUCCESS) { tfree(pCreateMsg); @@ -230,7 +230,7 @@ SCreateDbReq* buildCreateDbMsg(SCreateDbInfo* pCreateDbInfo, SParseBasicCtx *pCt return pCreateMsg; } -SMCreateStbReq* buildCreateStbMsg(SCreateTableSql* pCreateTableSql, int32_t* len, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf) { +SMCreateStbReq* buildCreateStbMsg(SCreateTableSql* pCreateTableSql, int32_t* len, SParseContext* pParseCtx, SMsgBuf* pMsgBuf) { SSchema* pSchema; int32_t numOfTags = 0; @@ -315,7 +315,7 @@ SMCreateStbReq* buildCreateStbMsg(SCreateTableSql* pCreateTableSql, int32_t* len return pCreateStbMsg; } -SMDropStbReq* buildDropStableMsg(SSqlInfo* pInfo, int32_t* len, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf) { +SMDropStbReq* buildDropStableMsg(SSqlInfo* pInfo, int32_t* len, SParseContext* pParseCtx, SMsgBuf* pMsgBuf) { SToken* tableName = taosArrayGet(pInfo->pMiscInfo->a, 0); SName name = {0}; diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c index faa8c526a0..5d56a88c14 100644 --- a/source/libs/parser/src/astValidate.c +++ b/source/libs/parser/src/astValidate.c @@ -3628,7 +3628,7 @@ int32_t evaluateSqlNode(SSqlNode* pNode, int32_t tsPrecision, SMsgBuf* pMsgBuf) return TSDB_CODE_SUCCESS; } -int32_t setTableVgroupList(SParseBasicCtx *pCtx, SName* name, SVgroupsInfo **pVgList) { +int32_t setTableVgroupList(SParseContext *pCtx, SName* name, SVgroupsInfo **pVgList) { SArray* vgroupList = NULL; int32_t code = catalogGetTableDistVgroup(pCtx->pCatalog, pCtx->pTransporter, &pCtx->mgmtEpSet, name, &vgroupList); if (code != TSDB_CODE_SUCCESS) { @@ -3655,7 +3655,7 @@ int32_t setTableVgroupList(SParseBasicCtx *pCtx, SName* name, SVgroupsInfo **pVg return TSDB_CODE_SUCCESS; } -int32_t qParserValidateSqlNode(SParseBasicCtx *pCtx, SSqlInfo* pInfo, SQueryStmtInfo* pQueryInfo, char* msgBuf, int32_t msgBufLen) { +int32_t qParserValidateSqlNode(SParseContext *pCtx, SSqlInfo* pInfo, SQueryStmtInfo* pQueryInfo, char* msgBuf, int32_t msgBufLen) { assert(pCtx != NULL && pInfo != NULL); int32_t code = 0; diff --git a/source/libs/parser/src/dCDAstProcess.c b/source/libs/parser/src/dCDAstProcess.c index 6676e1ebf6..4d4d68e962 100644 --- a/source/libs/parser/src/dCDAstProcess.c +++ b/source/libs/parser/src/dCDAstProcess.c @@ -18,7 +18,7 @@ static bool has(SArray* pFieldList, int32_t startIndex, const char* name) { return false; } -static int32_t setShowInfo(SShowInfo* pShowInfo, SParseBasicCtx* pCtx, void** output, int32_t* outputLen, +static int32_t setShowInfo(SShowInfo* pShowInfo, SParseContext* pCtx, void** output, int32_t* outputLen, SEpSet* pEpSet, void** pExtension, SMsgBuf* pMsgBuf) { const char* msg1 = "invalid name"; const char* msg2 = "wildcard string should be less than %d characters"; @@ -396,7 +396,7 @@ static void destroyCreateTbReqBatch(SVgroupTablesBatch* pTbBatch) { taosArrayDestroy(pTbBatch->req.pArray); } -static int32_t doCheckAndBuildCreateCTableReq(SCreateTableSql* pCreateTable, SParseBasicCtx* pCtx, SMsgBuf* pMsgBuf, SArray** pBufArray) { +static int32_t doCheckAndBuildCreateCTableReq(SCreateTableSql* pCreateTable, SParseContext* pCtx, SMsgBuf* pMsgBuf, SArray** pBufArray) { const char* msg1 = "invalid table name"; const char* msg2 = "tags number not matched"; const char* msg3 = "tag value too long"; @@ -634,7 +634,7 @@ static int32_t doBuildSingleTableBatchReq(SName* pTableName, SArray* pColumns, S return TSDB_CODE_SUCCESS; } -int32_t doCheckAndBuildCreateTableReq(SCreateTableSql* pCreateTable, SParseBasicCtx* pCtx, SMsgBuf* pMsgBuf, char** pOutput, int32_t* len) { +int32_t doCheckAndBuildCreateTableReq(SCreateTableSql* pCreateTable, SParseContext* pCtx, SMsgBuf* pMsgBuf, char** pOutput, int32_t* len) { SArray* pBufArray = NULL; int32_t code = 0; @@ -703,7 +703,7 @@ SArray* doSerializeVgroupCreateTableInfo(SHashObj* pVgroupHashmap) { return pBufArray; } -SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, char* msgBuf, int32_t msgBufLen) { +SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseContext* pCtx, char* msgBuf, int32_t msgBufLen) { int32_t code = 0; SDclStmtInfo* pDcl = calloc(1, sizeof(SDclStmtInfo)); @@ -961,7 +961,7 @@ SDclStmtInfo* qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, c return NULL; } -SVnodeModifOpStmtInfo* qParserValidateCreateTbSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, char* msgBuf, int32_t msgBufLen) { +SVnodeModifOpStmtInfo* qParserValidateCreateTbSqlNode(SSqlInfo* pInfo, SParseContext* pCtx, char* msgBuf, int32_t msgBufLen) { SCreateTableSql* pCreateTable = pInfo->pCreateTableInfo; assert(pCreateTable->type == TSDB_SQL_CREATE_TABLE); diff --git a/source/libs/parser/src/insertParser.c b/source/libs/parser/src/insertParser.c index 04c287baf1..8b1506bad1 100644 --- a/source/libs/parser/src/insertParser.c +++ b/source/libs/parser/src/insertParser.c @@ -84,11 +84,11 @@ static int32_t buildName(SInsertParseContext* pCxt, SToken* pStname, char* fullD char* p = strnchr(pStname->z, TS_PATH_DELIMITER[0], pStname->n, false); if (NULL != p) { // db.table - int32_t n = sprintf(fullDbName, "%d.", pCxt->pComCxt->ctx.acctId); + int32_t n = sprintf(fullDbName, "%d.", pCxt->pComCxt->acctId); strncpy(fullDbName + n, pStname->z, p - pStname->z); strncpy(tableName, p + 1, pStname->n - (p - pStname->z) - 1); } else { - snprintf(fullDbName, TSDB_DB_FNAME_LEN, "%d.%s", pCxt->pComCxt->ctx.acctId, pCxt->pComCxt->ctx.db); + snprintf(fullDbName, TSDB_DB_FNAME_LEN, "%d.%s", pCxt->pComCxt->acctId, pCxt->pComCxt->db); strncpy(tableName, pStname->z, pStname->n); } @@ -97,11 +97,11 @@ static int32_t buildName(SInsertParseContext* pCxt, SToken* pStname, char* fullD static int32_t getTableMeta(SInsertParseContext* pCxt, SToken* pTname) { SName name = {0}; - createSName(&name, pTname, &pCxt->pComCxt->ctx, &pCxt->msg); + createSName(&name, pTname, pCxt->pComCxt, &pCxt->msg); char tableName[TSDB_TABLE_FNAME_LEN] = {0}; tNameExtractFullName(&name, tableName); - SParseBasicCtx* pBasicCtx = &pCxt->pComCxt->ctx; + SParseContext* pBasicCtx = pCxt->pComCxt; CHECK_CODE(catalogGetTableMeta(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, &name, &pCxt->pTableMeta)); SVgroupInfo vg; CHECK_CODE(catalogGetTableHashVgroup(pBasicCtx->pCatalog, pBasicCtx->pTransporter, &pBasicCtx->mgmtEpSet, &name, &vg)); diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 5f17fcfaee..fea215238c 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -45,14 +45,14 @@ int32_t parseQuerySql(SParseContext* pCxt, SQueryNode** pQuery) { if (!isDqlSqlStatement(&info)) { if (info.type == TSDB_SQL_CREATE_TABLE) { - SVnodeModifOpStmtInfo * pModifStmtInfo = qParserValidateCreateTbSqlNode(&info, &pCxt->ctx, pCxt->pMsg, pCxt->msgLen); + SVnodeModifOpStmtInfo * pModifStmtInfo = qParserValidateCreateTbSqlNode(&info, pCxt, pCxt->pMsg, pCxt->msgLen); if (pModifStmtInfo == NULL) { return terrno; } *pQuery = (SQueryNode*)pModifStmtInfo; } else { - SDclStmtInfo* pDcl = qParserValidateDclSqlNode(&info, &pCxt->ctx, pCxt->pMsg, pCxt->msgLen); + SDclStmtInfo* pDcl = qParserValidateDclSqlNode(&info, pCxt, pCxt->pMsg, pCxt->msgLen); if (pDcl == NULL) { return terrno; } @@ -67,7 +67,7 @@ int32_t parseQuerySql(SParseContext* pCxt, SQueryNode** pQuery) { return terrno; } - int32_t code = qParserValidateSqlNode(&pCxt->ctx, &info, pQueryInfo, pCxt->pMsg, pCxt->msgLen); + int32_t code = qParserValidateSqlNode(pCxt, &info, pQueryInfo, pCxt->pMsg, pCxt->msgLen); if (code == TSDB_CODE_SUCCESS) { *pQuery = (SQueryNode*)pQueryInfo; } else { @@ -92,7 +92,7 @@ int32_t qParserConvertSql(const char* pStr, size_t length, char** pConvertSql) { return 0; } -static int32_t getTableNameFromSqlNode(SSqlNode* pSqlNode, SArray* tableNameList, SParseBasicCtx *pCtx, SMsgBuf* pMsgBuf); +static int32_t getTableNameFromSqlNode(SSqlNode* pSqlNode, SArray* tableNameList, SParseContext *pCtx, SMsgBuf* pMsgBuf); static int32_t tnameComparFn(const void* p1, const void* p2) { SName* pn1 = (SName*)p1; @@ -116,7 +116,7 @@ static int32_t tnameComparFn(const void* p1, const void* p2) { } } -static int32_t getTableNameFromSubquery(SSqlNode* pSqlNode, SArray* tableNameList, SParseBasicCtx *pCtx, SMsgBuf* pMsgBuf) { +static int32_t getTableNameFromSubquery(SSqlNode* pSqlNode, SArray* tableNameList, SParseContext *pCtx, SMsgBuf* pMsgBuf) { int32_t numOfSub = (int32_t)taosArrayGetSize(pSqlNode->from->list); for (int32_t j = 0; j < numOfSub; ++j) { @@ -139,7 +139,7 @@ static int32_t getTableNameFromSubquery(SSqlNode* pSqlNode, SArray* tableNameLis return TSDB_CODE_SUCCESS; } -int32_t getTableNameFromSqlNode(SSqlNode* pSqlNode, SArray* tableNameList, SParseBasicCtx *pParseCtx, SMsgBuf* pMsgBuf) { +int32_t getTableNameFromSqlNode(SSqlNode* pSqlNode, SArray* tableNameList, SParseContext *pParseCtx, SMsgBuf* pMsgBuf) { const char* msg1 = "invalid table name"; int32_t numOfTables = (int32_t) taosArrayGetSize(pSqlNode->from->list); @@ -173,7 +173,7 @@ static void freePtrElem(void* p) { tfree(*(char**)p); } -int32_t qParserExtractRequestedMetaInfo(const SSqlInfo* pSqlInfo, SCatalogReq* pMetaInfo, SParseBasicCtx *pCtx, char* msg, int32_t msgBufLen) { +int32_t qParserExtractRequestedMetaInfo(const SSqlInfo* pSqlInfo, SCatalogReq* pMetaInfo, SParseContext *pCtx, char* msg, int32_t msgBufLen) { int32_t code = TSDB_CODE_SUCCESS; SMsgBuf msgBuf = {.buf = msg, .len = msgBufLen}; diff --git a/source/libs/parser/src/parserUtil.c b/source/libs/parser/src/parserUtil.c index 22545255a3..7bb0f8ee83 100644 --- a/source/libs/parser/src/parserUtil.c +++ b/source/libs/parser/src/parserUtil.c @@ -1943,7 +1943,7 @@ int32_t KvRowAppend(const void *value, int32_t len, void *param) { return TSDB_CODE_SUCCESS; } -int32_t createSName(SName* pName, SToken* pTableName, SParseBasicCtx* pParseCtx, SMsgBuf* pMsgBuf) { +int32_t createSName(SName* pName, SToken* pTableName, SParseContext* pParseCtx, SMsgBuf* pMsgBuf) { const char* msg1 = "name too long"; int32_t code = TSDB_CODE_SUCCESS; diff --git a/source/libs/parser/test/parserTests.cpp b/source/libs/parser/test/parserTests.cpp index 8758fdbc71..4847b50082 100644 --- a/source/libs/parser/test/parserTests.cpp +++ b/source/libs/parser/test/parserTests.cpp @@ -77,7 +77,7 @@ void sqlCheck(const char* sql, bool valid) { buf.len = 128; buf.buf = msg; - SParseBasicCtx ctx = {0}; + SParseContext ctx = {0}; ctx.db = "db1"; ctx.acctId = 1; SSqlNode* pNode = (SSqlNode*)taosArrayGetP(((SArray*)info1.sub.node), 0); @@ -122,7 +122,7 @@ TEST(testCase, validateAST_test) { ASSERT_EQ(code, 0); SCatalogReq req = {0}; - SParseBasicCtx ctx = {0}; + SParseContext ctx = {0}; ctx.db = "db1"; ctx.acctId = 1; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, &ctx, msg, 128); @@ -184,7 +184,7 @@ TEST(testCase, function_Test) { ASSERT_EQ(code, 0); SCatalogReq req = {0}; - SParseBasicCtx ctx = {0}; + SParseContext ctx = {0}; ctx.db = "db1"; ctx.acctId = 1; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, &ctx, msg, 128); @@ -234,7 +234,7 @@ TEST(testCase, function_Test2) { ASSERT_EQ(code, 0); SCatalogReq req = {0}; - SParseBasicCtx ctx = {0}; + SParseContext ctx = {0}; ctx.db = "db1"; ctx.acctId = 1; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, &ctx, msg, 128); @@ -284,7 +284,7 @@ TEST(testCase, function_Test3) { ASSERT_EQ(code, 0); SCatalogReq req = {0}; - SParseBasicCtx ctx = {0}; + SParseContext ctx = {0}; ctx.db = "db1"; ctx.acctId = 1; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, &ctx, msg, 128); @@ -333,7 +333,7 @@ TEST(testCase, function_Test4) { ASSERT_EQ(code, 0); SCatalogReq req = {0}; - SParseBasicCtx ctx = {0}; + SParseContext ctx = {0}; ctx.db = "db1"; ctx.acctId = 1; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, &ctx, msg, 128); @@ -385,7 +385,7 @@ TEST(testCase, function_Test5) { ASSERT_EQ(code, 0); SCatalogReq req = {0}; - SParseBasicCtx ctx = {0}; + SParseContext ctx = {0}; ctx.db = "db1"; ctx.acctId = 1; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, &ctx, msg, 128); @@ -474,7 +474,7 @@ TEST(testCase, function_Test6) { ASSERT_EQ(code, 0); SCatalogReq req = {0}; - SParseBasicCtx ctx = {0}; + SParseContext ctx = {0}; ctx.db = "db1"; ctx.acctId = 1; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, &ctx, msg, 128); @@ -556,7 +556,7 @@ TEST(testCase, function_Test6) { ASSERT_EQ(code, 0); SCatalogReq req = {0}; - SParseBasicCtx ctx = {0}; + SParseContext ctx = {0}; ctx.db = "db1"; ctx.acctId = 1; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, &ctx, msg, 128); @@ -622,7 +622,7 @@ TEST(testCase, function_Test6) { ASSERT_EQ(code, 0); SCatalogReq req = {0}; - SParseBasicCtx ctx = {0}; + SParseContext ctx = {0}; ctx.db = "db1"; ctx.acctId = 1; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, &ctx, msg, 128); @@ -705,7 +705,7 @@ TEST(testCase, function_Test6) { ASSERT_EQ(code, 0); SCatalogReq req = {0}; - SParseBasicCtx ctx = {0}; + SParseContext ctx = {0}; ctx.db = "db1"; ctx.acctId = 1; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, &ctx, msg, 128); @@ -756,7 +756,7 @@ TEST(testCase, show_user_Test) { SSqlInfo info1 = doGenerateAST(sql1); ASSERT_EQ(info1.valid, true); - SParseBasicCtx ct= {.requestId = 1, .acctId = 1, .db = "abc", .pTransporter = NULL}; + SParseContext ct= {.requestId = 1, .acctId = 1, .db = "abc", .pTransporter = NULL}; SDclStmtInfo* output = qParserValidateDclSqlNode(&info1, &ct, msg, buf.len); ASSERT_NE(output, nullptr); @@ -776,7 +776,7 @@ TEST(testCase, create_user_Test) { ASSERT_EQ(info1.valid, true); ASSERT_EQ(isDclSqlStatement(&info1), true); - SParseBasicCtx ct= {.requestId = 1, .acctId = 1, .db = "abc"}; + SParseContext ct= {.requestId = 1, .acctId = 1, .db = "abc"}; SDclStmtInfo* output = qParserValidateDclSqlNode(&info1, &ct, msg, buf.len); ASSERT_NE(output, nullptr); diff --git a/source/libs/parser/test/plannerTest.cpp b/source/libs/parser/test/plannerTest.cpp index 8d9fbadfad..a278e0053c 100644 --- a/source/libs/parser/test/plannerTest.cpp +++ b/source/libs/parser/test/plannerTest.cpp @@ -81,7 +81,7 @@ void generateLogicplan(const char* sql) { ASSERT_EQ(code, 0); SCatalogReq req = {0}; - SParseBasicCtx ctx = {0}; + SParseContext ctx = {0}; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, &ctx, msg, 128); ASSERT_EQ(ret, 0); ASSERT_EQ(taosArrayGetSize(req.pTableName), 1); @@ -122,7 +122,7 @@ TEST(testCase, planner_test) { ASSERT_EQ(code, 0); SCatalogReq req = {0}; - SParseBasicCtx ctx = {0}; + SParseContext ctx = {0}; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, &ctx, msg, 128); ASSERT_EQ(ret, 0); diff --git a/source/libs/parser/test/tokenizerTest.cpp b/source/libs/parser/test/tokenizerTest.cpp index ee01a50148..7ed052e8cc 100644 --- a/source/libs/parser/test/tokenizerTest.cpp +++ b/source/libs/parser/test/tokenizerTest.cpp @@ -711,7 +711,7 @@ TEST(testCase, extractMeta_test) { char msg[128] = {0}; SCatalogReq req = {0}; - SParseBasicCtx ctx = {0}; + SParseContext ctx = {0}; ctx.db = "db1"; ctx.acctId = 1; int32_t ret = qParserExtractRequestedMetaInfo(&info1, &req, &ctx, msg, 128); From b55da7758e98d18044bb4eaa7d50ac1471bffb9a Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 12 Jan 2022 10:55:01 +0800 Subject: [PATCH 39/57] [td-11818]remove SParseBasicCtx --- source/libs/parser/test/insertParserTest.cpp | 4 ++-- source/libs/planner/test/phyPlanTests.cpp | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/source/libs/parser/test/insertParserTest.cpp b/source/libs/parser/test/insertParserTest.cpp index 9b007fb36d..86e8b1d7aa 100644 --- a/source/libs/parser/test/insertParserTest.cpp +++ b/source/libs/parser/test/insertParserTest.cpp @@ -43,8 +43,8 @@ protected: void bind(const char* sql) { reset(); - cxt_.ctx.acctId = atoi(acctId_.c_str()); - cxt_.ctx.db = (char*) db_.c_str(); + cxt_.acctId = atoi(acctId_.c_str()); + cxt_.db = (char*) db_.c_str(); strcpy(sqlBuf_, sql); cxt_.sqlLen = strlen(sql); sqlBuf_[cxt_.sqlLen] = '\0'; diff --git a/source/libs/planner/test/phyPlanTests.cpp b/source/libs/planner/test/phyPlanTests.cpp index 2733a73a3f..29f6e48dc7 100644 --- a/source/libs/planner/test/phyPlanTests.cpp +++ b/source/libs/planner/test/phyPlanTests.cpp @@ -135,9 +135,9 @@ private: _sql = sql; memset(_msg, 0, _msgMaxLen); - pCxt->ctx.acctId = 1; - pCxt->ctx.db = _db.c_str(); - pCxt->ctx.requestId = 1; + pCxt->acctId = 1; + pCxt->db = _db.c_str(); + pCxt->requestId = 1; pCxt->pSql = _sql.c_str(); pCxt->sqlLen = _sql.length(); pCxt->pMsg = _msg; From 6c9ad20bafeae5b6e3b0f08155c37d95b6a3240e Mon Sep 17 00:00:00 2001 From: lihui Date: Wed, 12 Jan 2022 11:01:47 +0800 Subject: [PATCH 40/57] [add show tables] --- tests/test/c/create_table.c | 233 +++++++++++++++++++++++------------- 1 file changed, 151 insertions(+), 82 deletions(-) diff --git a/tests/test/c/create_table.c b/tests/test/c/create_table.c index a5a55bbc91..aae4dc7074 100644 --- a/tests/test/c/create_table.c +++ b/tests/test/c/create_table.c @@ -30,6 +30,7 @@ int32_t createTable = 1; int32_t insertData = 0; int32_t batchNum = 100; int32_t numOfVgroups = 2; +int32_t showTablesFlag = 0; typedef struct { int64_t tableBeginIndex; @@ -45,88 +46,9 @@ typedef struct { pthread_t thread; } SThreadInfo; -void parseArgument(int32_t argc, char *argv[]); -void *threadFunc(void *param); -void createDbAndStb(); - -int32_t main(int32_t argc, char *argv[]) { - parseArgument(argc, argv); - createDbAndStb(); - - pPrint("%d threads are spawned to create %d tables", numOfThreads, numOfThreads); - - pthread_attr_t thattr; - pthread_attr_init(&thattr); - pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); - SThreadInfo *pInfo = (SThreadInfo *)calloc(numOfThreads, sizeof(SThreadInfo)); - - //int64_t numOfTablesPerThread = numOfTables / numOfThreads; - //numOfTables = numOfTablesPerThread * numOfThreads; - - - if (numOfThreads < 1) { - numOfThreads = 1; - } - - int64_t a = numOfTables / numOfThreads; - if (a < 1) { - numOfThreads = numOfTables; - a = 1; - } - - int64_t b = 0; - b = numOfTables % numOfThreads; - - int64_t tableFrom = 0; - for (int32_t i = 0; i < numOfThreads; ++i) { - pInfo[i].tableBeginIndex = tableFrom; - pInfo[i].tableEndIndex = i < b ? tableFrom + a : tableFrom + a - 1; - tableFrom = pInfo[i].tableEndIndex + 1; - pInfo[i].threadIndex = i; - pInfo[i].minDelay = INT64_MAX; - strcpy(pInfo[i].dbName, dbName); - strcpy(pInfo[i].stbName, stbName); - pthread_create(&(pInfo[i].thread), &thattr, threadFunc, (void *)(pInfo + i)); - } - - taosMsleep(300); - for (int32_t i = 0; i < numOfThreads; i++) { - pthread_join(pInfo[i].thread, NULL); - } - - int64_t maxDelay = 0; - int64_t minDelay = INT64_MAX; - - float createTableSpeed = 0; - for (int32_t i = 0; i < numOfThreads; ++i) { - createTableSpeed += pInfo[i].createTableSpeed; - - if (pInfo[i].maxDelay > maxDelay) maxDelay = pInfo[i].maxDelay; - if (pInfo[i].minDelay < minDelay) minDelay = pInfo[i].minDelay; - } - - float insertDataSpeed = 0; - for (int32_t i = 0; i < numOfThreads; ++i) { - insertDataSpeed += pInfo[i].insertDataSpeed; - } - - pPrint("%s total %" PRId64 " tables, %.1f tables/second, threads:%d, maxDelay: %" PRId64 "us, minDelay: %" PRId64 "us %s", - GREEN, - numOfTables, - createTableSpeed, - numOfThreads, - maxDelay, - minDelay, - NC); - - if (insertData) { - pPrint("%s total %" PRId64 " tables, %.1f rows/second, threads:%d %s", GREEN, numOfTables, insertDataSpeed, - numOfThreads, NC); - } - - pthread_attr_destroy(&thattr); - free(pInfo); -} +//void parseArgument(int32_t argc, char *argv[]); +//void *threadFunc(void *param); +//void createDbAndStb(); void createDbAndStb() { pPrint("start to create db and stable"); @@ -188,6 +110,62 @@ void printInsertProgress(SThreadInfo *pInfo, int64_t t) { totalTables, seconds, speed); } +static int64_t getResult(TAOS_RES *tres) { + TAOS_ROW row = taos_fetch_row(tres); + if (row == NULL) { + return 0; + } + + int num_fields = taos_num_fields(tres); + TAOS_FIELD *fields = taos_fetch_fields(tres); + int precision = taos_result_precision(tres); + + int64_t numOfRows = 0; + do { + numOfRows++; + row = taos_fetch_row(tres); + } while (row != NULL); + + return numOfRows; +} + + +void showTables() { + pPrint("start to show tables"); + char qstr[32]; + + TAOS *con = taos_connect(NULL, "root", "taosdata", NULL, 0); + if (con == NULL) { + pError("failed to connect to DB, reason:%s", taos_errstr(NULL)); + exit(1); + } + + sprintf(qstr, "use %s", dbName); + TAOS_RES *pRes = taos_query(con, qstr); + int code = taos_errno(pRes); + if (code != 0) { + pError("failed to use db, code:%d reason:%s", taos_errno(pRes), taos_errstr(pRes)); + exit(1); + } + taos_free_result(pRes); + + sprintf(qstr, "show tables"); + pRes = taos_query(con, qstr); + code = taos_errno(pRes); + if (code != 0) { + pError("failed to show tables, code:%d reason:%s", taos_errno(pRes), taos_errstr(pRes)); + exit(0); + } + + int64_t totalTableNum = getResult(pRes); + taos_free_result(pRes); + + pPrint("%s database: %s, total %" PRId64 " tables %s", GREEN, dbName, totalTableNum, NC); + + taos_close(con); +} + + void *threadFunc(void *param) { SThreadInfo *pInfo = (SThreadInfo *)param; char *qstr = malloc(2000 * 1000); @@ -298,6 +276,8 @@ void printHelp() { printf("%s%s%s%d\n", indent, indent, "insertData, default is ", insertData); printf("%s%s\n", indent, "-b"); printf("%s%s%s%d\n", indent, indent, "batchNum, default is ", batchNum); + printf("%s%s\n", indent, "-w"); + printf("%s%s%s%d\n", indent, indent, "showTablesFlag, default is ", showTablesFlag); exit(EXIT_SUCCESS); } @@ -325,6 +305,8 @@ void parseArgument(int32_t argc, char *argv[]) { insertData = atoi(argv[++i]); } else if (strcmp(argv[i], "-b") == 0) { batchNum = atoi(argv[++i]); + } else if (strcmp(argv[i], "-w") == 0) { + showTablesFlag = atoi(argv[++i]); } else { } } @@ -338,6 +320,93 @@ void parseArgument(int32_t argc, char *argv[]) { pPrint("%s createTable:%d %s", GREEN, createTable, NC); pPrint("%s insertData:%d %s", GREEN, insertData, NC); pPrint("%s batchNum:%d %s", GREEN, batchNum, NC); + pPrint("%s showTablesFlag:%d %s", GREEN, showTablesFlag, NC); pPrint("%s start create table performace test %s", GREEN, NC); } + +int32_t main(int32_t argc, char *argv[]) { + parseArgument(argc, argv); + + if (showTablesFlag) { + showTables(); + return 0; + } + + createDbAndStb(); + + pPrint("%d threads are spawned to create %d tables", numOfThreads, numOfThreads); + + pthread_attr_t thattr; + pthread_attr_init(&thattr); + pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE); + SThreadInfo *pInfo = (SThreadInfo *)calloc(numOfThreads, sizeof(SThreadInfo)); + + //int64_t numOfTablesPerThread = numOfTables / numOfThreads; + //numOfTables = numOfTablesPerThread * numOfThreads; + + + if (numOfThreads < 1) { + numOfThreads = 1; + } + + int64_t a = numOfTables / numOfThreads; + if (a < 1) { + numOfThreads = numOfTables; + a = 1; + } + + int64_t b = 0; + b = numOfTables % numOfThreads; + + int64_t tableFrom = 0; + for (int32_t i = 0; i < numOfThreads; ++i) { + pInfo[i].tableBeginIndex = tableFrom; + pInfo[i].tableEndIndex = i < b ? tableFrom + a : tableFrom + a - 1; + tableFrom = pInfo[i].tableEndIndex + 1; + pInfo[i].threadIndex = i; + pInfo[i].minDelay = INT64_MAX; + strcpy(pInfo[i].dbName, dbName); + strcpy(pInfo[i].stbName, stbName); + pthread_create(&(pInfo[i].thread), &thattr, threadFunc, (void *)(pInfo + i)); + } + + taosMsleep(300); + for (int32_t i = 0; i < numOfThreads; i++) { + pthread_join(pInfo[i].thread, NULL); + } + + int64_t maxDelay = 0; + int64_t minDelay = INT64_MAX; + + float createTableSpeed = 0; + for (int32_t i = 0; i < numOfThreads; ++i) { + createTableSpeed += pInfo[i].createTableSpeed; + + if (pInfo[i].maxDelay > maxDelay) maxDelay = pInfo[i].maxDelay; + if (pInfo[i].minDelay < minDelay) minDelay = pInfo[i].minDelay; + } + + float insertDataSpeed = 0; + for (int32_t i = 0; i < numOfThreads; ++i) { + insertDataSpeed += pInfo[i].insertDataSpeed; + } + + pPrint("%s total %" PRId64 " tables, %.1f tables/second, threads:%d, maxDelay: %" PRId64 "us, minDelay: %" PRId64 "us %s", + GREEN, + numOfTables, + createTableSpeed, + numOfThreads, + maxDelay, + minDelay, + NC); + + if (insertData) { + pPrint("%s total %" PRId64 " tables, %.1f rows/second, threads:%d %s", GREEN, numOfTables, insertDataSpeed, + numOfThreads, NC); + } + + pthread_attr_destroy(&thattr); + free(pInfo); +} + From aac15d18224a51417ec98e6585bbb3ad6a6ab63f Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 11 Jan 2022 19:24:39 -0800 Subject: [PATCH 41/57] minor changes --- include/dnode/bnode/bnode.h | 12 ++++-------- include/dnode/qnode/qnode.h | 12 ++++-------- include/dnode/snode/snode.h | 12 ++++-------- source/dnode/bnode/inc/bndInt.h | 7 +------ source/dnode/mgmt/impl/src/dndBnode.c | 2 +- source/dnode/mgmt/impl/src/dndQnode.c | 2 +- source/dnode/mgmt/impl/src/dndSnode.c | 2 +- source/dnode/qnode/inc/qndInt.h | 7 +------ source/dnode/snode/inc/sndInt.h | 7 +------ 9 files changed, 18 insertions(+), 45 deletions(-) diff --git a/include/dnode/bnode/bnode.h b/include/dnode/bnode/bnode.h index 3cc26861ab..7f7b8a8d53 100644 --- a/include/dnode/bnode/bnode.h +++ b/include/dnode/bnode/bnode.h @@ -23,22 +23,18 @@ extern "C" { /* ------------------------ TYPES EXPOSED ------------------------ */ typedef struct SDnode SDnode; typedef struct SBnode SBnode; -typedef int32_t (*SendReqToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *rpcMsg); -typedef int32_t (*SendReqToMnodeFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg); -typedef void (*SendRedirectRspFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg); +typedef int32_t (*SendReqToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *pMsg); +typedef int32_t (*SendReqToMnodeFp)(SDnode *pDnode, struct SRpcMsg *pMsg); +typedef void (*SendRedirectRspFp)(SDnode *pDnode, struct SRpcMsg *pMsg); typedef struct { int64_t numOfErrors; } SBnodeLoad; typedef struct { - int32_t sver; -} SBnodeCfg; - -typedef struct { + int32_t sver; int32_t dnodeId; int64_t clusterId; - SBnodeCfg cfg; SDnode *pDnode; SendReqToDnodeFp sendReqToDnodeFp; SendReqToMnodeFp sendReqToMnodeFp; diff --git a/include/dnode/qnode/qnode.h b/include/dnode/qnode/qnode.h index 554c57a045..3de2986047 100644 --- a/include/dnode/qnode/qnode.h +++ b/include/dnode/qnode/qnode.h @@ -23,9 +23,9 @@ extern "C" { /* ------------------------ TYPES EXPOSED ------------------------ */ typedef struct SDnode SDnode; typedef struct SQnode SQnode; -typedef int32_t (*SendReqToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *rpcMsg); -typedef int32_t (*SendReqToMnodeFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg); -typedef void (*SendRedirectRspFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg); +typedef int32_t (*SendReqToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *pMsg); +typedef int32_t (*SendReqToMnodeFp)(SDnode *pDnode, struct SRpcMsg *pMsg); +typedef void (*SendRedirectRspFp)(SDnode *pDnode, struct SRpcMsg *pMsg); typedef struct { int64_t numOfStartTask; @@ -39,13 +39,9 @@ typedef struct { } SQnodeLoad; typedef struct { - int32_t sver; -} SQnodeCfg; - -typedef struct { + int32_t sver; int32_t dnodeId; int64_t clusterId; - SQnodeCfg cfg; SDnode *pDnode; SendReqToDnodeFp sendReqToDnodeFp; SendReqToMnodeFp sendReqToMnodeFp; diff --git a/include/dnode/snode/snode.h b/include/dnode/snode/snode.h index 43d3dd9b4b..319708decc 100644 --- a/include/dnode/snode/snode.h +++ b/include/dnode/snode/snode.h @@ -23,22 +23,18 @@ extern "C" { /* ------------------------ TYPES EXPOSED ------------------------ */ typedef struct SDnode SDnode; typedef struct SSnode SSnode; -typedef int32_t (*SendReqToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *rpcMsg); -typedef int32_t (*SendReqToMnodeFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg); -typedef void (*SendRedirectRspFp)(SDnode *pDnode, struct SRpcMsg *rpcMsg); +typedef int32_t (*SendReqToDnodeFp)(SDnode *pDnode, struct SEpSet *epSet, struct SRpcMsg *pMsg); +typedef int32_t (*SendReqToMnodeFp)(SDnode *pDnode, struct SRpcMsg *pMsg); +typedef void (*SendRedirectRspFp)(SDnode *pDnode, struct SRpcMsg *pMsg); typedef struct { int64_t numOfErrors; } SSnodeLoad; typedef struct { - int32_t sver; -} SSnodeCfg; - -typedef struct { + int32_t sver; int32_t dnodeId; int64_t clusterId; - SSnodeCfg cfg; SDnode *pDnode; SendReqToDnodeFp sendReqToDnodeFp; SendReqToMnodeFp sendReqToMnodeFp; diff --git a/source/dnode/bnode/inc/bndInt.h b/source/dnode/bnode/inc/bndInt.h index cddb1e50f1..4a0d16a749 100644 --- a/source/dnode/bnode/inc/bndInt.h +++ b/source/dnode/bnode/inc/bndInt.h @@ -30,12 +30,7 @@ extern "C" { #endif typedef struct SBnode { - int32_t dnodeId; - int64_t clusterId; - SBnodeCfg cfg; - SendReqToDnodeFp sendReqToDnodeFp; - SendReqToMnodeFp sendReqToMnodeFp; - SendRedirectRspFp sendRedirectRspFp; + SBnodeOpt opt; } SBnode; #ifdef __cplusplus diff --git a/source/dnode/mgmt/impl/src/dndBnode.c b/source/dnode/mgmt/impl/src/dndBnode.c index ce058ebba4..f26ec72f1d 100644 --- a/source/dnode/mgmt/impl/src/dndBnode.c +++ b/source/dnode/mgmt/impl/src/dndBnode.c @@ -179,7 +179,7 @@ static void dndBuildBnodeOption(SDnode *pDnode, SBnodeOpt *pOption) { pOption->sendRedirectRspFp = dndSendRedirectRsp; pOption->dnodeId = dndGetDnodeId(pDnode); pOption->clusterId = dndGetClusterId(pDnode); - pOption->cfg.sver = pDnode->env.sver; + pOption->sver = pDnode->env.sver; } static int32_t dndOpenBnode(SDnode *pDnode) { diff --git a/source/dnode/mgmt/impl/src/dndQnode.c b/source/dnode/mgmt/impl/src/dndQnode.c index 9a8d49010c..fa53375381 100644 --- a/source/dnode/mgmt/impl/src/dndQnode.c +++ b/source/dnode/mgmt/impl/src/dndQnode.c @@ -185,7 +185,7 @@ static void dndBuildQnodeOption(SDnode *pDnode, SQnodeOpt *pOption) { pOption->sendRedirectRspFp = dndSendRedirectRsp; pOption->dnodeId = dndGetDnodeId(pDnode); pOption->clusterId = dndGetClusterId(pDnode); - pOption->cfg.sver = pDnode->env.sver; + pOption->sver = pDnode->env.sver; } static int32_t dndOpenQnode(SDnode *pDnode) { diff --git a/source/dnode/mgmt/impl/src/dndSnode.c b/source/dnode/mgmt/impl/src/dndSnode.c index a07f02e761..6f22e5b00c 100644 --- a/source/dnode/mgmt/impl/src/dndSnode.c +++ b/source/dnode/mgmt/impl/src/dndSnode.c @@ -179,7 +179,7 @@ static void dndBuildSnodeOption(SDnode *pDnode, SSnodeOpt *pOption) { pOption->sendRedirectRspFp = dndSendRedirectRsp; pOption->dnodeId = dndGetDnodeId(pDnode); pOption->clusterId = dndGetClusterId(pDnode); - pOption->cfg.sver = pDnode->env.sver; + pOption->sver = pDnode->env.sver; } static int32_t dndOpenSnode(SDnode *pDnode) { diff --git a/source/dnode/qnode/inc/qndInt.h b/source/dnode/qnode/inc/qndInt.h index 529c407efa..0f8e65de7e 100644 --- a/source/dnode/qnode/inc/qndInt.h +++ b/source/dnode/qnode/inc/qndInt.h @@ -29,12 +29,7 @@ extern "C" { #endif typedef struct SQnode { - int32_t dnodeId; - int64_t clusterId; - SQnodeCfg cfg; - SendReqToDnodeFp sendReqToDnodeFp; - SendReqToMnodeFp sendReqToMnodeFp; - SendRedirectRspFp sendRedirectRspFp; + SQnodeOpt opt; } SQnode; #ifdef __cplusplus diff --git a/source/dnode/snode/inc/sndInt.h b/source/dnode/snode/inc/sndInt.h index 3b41c7f4b1..aff82e4ae7 100644 --- a/source/dnode/snode/inc/sndInt.h +++ b/source/dnode/snode/inc/sndInt.h @@ -29,12 +29,7 @@ extern "C" { #endif typedef struct SSnode { - int32_t dnodeId; - int64_t clusterId; - SSnodeCfg cfg; - SendReqToDnodeFp sendReqToDnodeFp; - SendReqToMnodeFp sendReqToMnodeFp; - SendRedirectRspFp sendRedirectRspFp; + SSnodeOpt cfg; } SSnode; #ifdef __cplusplus From 57281f6e8f544997fe73dc5475ec66f7211f1fd9 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 12 Jan 2022 11:26:23 +0800 Subject: [PATCH 42/57] [td-11818]refactor. --- source/client/inc/clientInt.h | 1 - source/client/src/clientImpl.c | 5 +++-- source/client/src/clientMain.c | 8 +++++++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index ebb8502d6c..39f84ffd86 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -129,7 +129,6 @@ typedef struct SRequestObj { char *msgBuf; void *pInfo; // sql parse info, generated by parser module int32_t code; - uint64_t affectedRows; // todo remove it SQueryExecMetric metric; SRequestSendRecvBody body; } SRequestObj; diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index a614d2c3c1..ba08e67127 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -250,8 +250,9 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag) { } } - pRequest->affectedRows = res.numOfRows; - return res.code; + pRequest->body.resInfo.numOfRows = res.numOfRows; + pRequest->code = res.code; + return pRequest->code; } return scheduleAsyncExecJob(pRequest->pTscObj->pTransporter, NULL, pDag, &pRequest->body.pQueryJob); diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 1238976b97..ba2a21d7ea 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -265,7 +265,13 @@ const char *taos_data_type(int type) { const char *taos_get_client_info() { return version; } int taos_affected_rows(TAOS_RES *res) { - return ((SRequestObj*)res)->affectedRows; + if (res == NULL) { + return 0; + } + + SRequestObj* pRequest = (SRequestObj*) res; + SReqResultInfo* pResInfo = &pRequest->body.resInfo; + return pResInfo->numOfRows; } int taos_result_precision(TAOS_RES *res) { return TSDB_TIME_PRECISION_MILLI; } From 0b6d32a3d92ac53f8653e0a641b6dab60629e8c0 Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 11 Jan 2022 19:35:40 -0800 Subject: [PATCH 43/57] rename file --- include/dnode/vnode/vnode.h | 4 +-- include/libs/tfs/tfs.h | 2 +- source/dnode/mgmt/impl/src/dndEnv.c | 33 ++++++++++--------- source/dnode/vnode/impl/src/vnodeMgr.c | 2 +- .../dnode/vnode/impl/test/vnodeApiTests.cpp | 4 +-- source/libs/tfs/src/tfs.c | 8 ++--- src/inc/tfs.h | 2 +- 7 files changed, 28 insertions(+), 27 deletions(-) diff --git a/include/dnode/vnode/vnode.h b/include/dnode/vnode/vnode.h index 4332fc8e58..6113d0a536 100644 --- a/include/dnode/vnode/vnode.h +++ b/include/dnode/vnode/vnode.h @@ -89,10 +89,10 @@ typedef struct { int vnodeInit(const SVnodeOpt *pOption); /** - * @brief clear a vnode + * @brief Cleanup the vnode module * */ -void vnodeClear(); +void vnodeCleanup(); /** * @brief Open a VNODE. diff --git a/include/libs/tfs/tfs.h b/include/libs/tfs/tfs.h index 6c850d1016..793c861363 100644 --- a/include/libs/tfs/tfs.h +++ b/include/libs/tfs/tfs.h @@ -49,7 +49,7 @@ typedef struct { } STierMeta; int tfsInit(SDiskCfg *pDiskCfg, int ndisk); -void tfsDestroy(); +void tfsCleanup(); void tfsUpdateInfo(SFSMeta *pFSMeta, STierMeta *tierMetas, int8_t numLevels); void tfsGetMeta(SFSMeta *pMeta); void tfsAllocDisk(int expLevel, int *level, int *id); diff --git a/source/dnode/mgmt/impl/src/dndEnv.c b/source/dnode/mgmt/impl/src/dndEnv.c index 44a01049ba..88e38daf1c 100644 --- a/source/dnode/mgmt/impl/src/dndEnv.c +++ b/source/dnode/mgmt/impl/src/dndEnv.c @@ -173,18 +173,6 @@ SDnode *dndCreate(SDnodeObjCfg *pCfg) { return NULL; } - if (rpcInit() != 0) { - dError("failed to init rpc since %s", terrstr()); - dndClose(pDnode); - return NULL; - } - - if (walInit() != 0) { - dError("failed to init wal since %s", terrstr()); - dndClose(pDnode); - return NULL; - } - SDiskCfg dCfg; strcpy(dCfg.dir, pDnode->cfg.dataDir); dCfg.level = 0; @@ -277,10 +265,8 @@ void dndClose(SDnode *pDnode) { dndCleanupQnode(pDnode); dndCleanupVnodes(pDnode); dndCleanupMgmt(pDnode); - vnodeClear(); - tfsDestroy(); - walCleanUp(); - rpcCleanup(); + vnodeCleanup(); + tfsCleanup(); dndCloseImp(pDnode); free(pDnode); @@ -298,6 +284,18 @@ int32_t dndInit(const SDnodeEnvCfg *pCfg) { taosBlockSIGPIPE(); taosResolveCRC(); + if (rpcInit() != 0) { + dError("failed to init rpc since %s", terrstr()); + dndCleanup(); + return -1; + } + + if (walInit() != 0) { + dError("failed to init wal since %s", terrstr()); + dndCleanup(); + return -1; + } + memcpy(&dndEnv.cfg, pCfg, sizeof(SDnodeEnvCfg)); dInfo("dnode env is initialized"); return 0; @@ -309,6 +307,9 @@ void dndCleanup() { return; } + walCleanUp(); + rpcCleanup(); + taosStopCacheRefreshWorker(); dInfo("dnode env is cleaned up"); } \ No newline at end of file diff --git a/source/dnode/vnode/impl/src/vnodeMgr.c b/source/dnode/vnode/impl/src/vnodeMgr.c index cc369a0d15..15550a899b 100644 --- a/source/dnode/vnode/impl/src/vnodeMgr.c +++ b/source/dnode/vnode/impl/src/vnodeMgr.c @@ -56,7 +56,7 @@ int vnodeInit(const SVnodeOpt *pOption) { return 0; } -void vnodeClear() { +void vnodeCleanup() { if (TD_CHECK_AND_SET_MOD_CLEAR(&(vnodeMgr.vnodeInitFlag)) == TD_MOD_UNINITIALIZED) { return; } diff --git a/source/dnode/vnode/impl/test/vnodeApiTests.cpp b/source/dnode/vnode/impl/test/vnodeApiTests.cpp index 71ffdd1d0f..6ec58a1e9d 100644 --- a/source/dnode/vnode/impl/test/vnodeApiTests.cpp +++ b/source/dnode/vnode/impl/test/vnodeApiTests.cpp @@ -227,7 +227,7 @@ TEST(vnodeApiTest, vnode_simple_create_table_test) { // CLOSE THE VNODE vnodeClose(pVnode); - vnodeClear(); + vnodeCleanup(); taosArrayDestroy(pMsgArr); } @@ -279,7 +279,7 @@ TEST(vnodeApiTest, vnode_simple_insert_test) { // Close the vnode vnodeClose(pVnode); - vnodeClear(); + vnodeCleanup(); taosArrayDestroy(pMsgArr); } \ No newline at end of file diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index 83d9905511..88d6d587a7 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -85,19 +85,19 @@ int tfsInit(SDiskCfg *pDiskCfg, int ndisk) { taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); if (pfs->map == NULL) { terrno = TSDB_CODE_FS_OUT_OF_MEMORY; - tfsDestroy(); + tfsCleanup(); return -1; } for (int idisk = 0; idisk < ndisk; idisk++) { if (tfsMount(pDiskCfg + idisk) < 0) { - tfsDestroy(); + tfsCleanup(); return -1; } } if (tfsCheck() < 0) { - tfsDestroy(); + tfsCleanup(); return -1; } @@ -109,7 +109,7 @@ int tfsInit(SDiskCfg *pDiskCfg, int ndisk) { return 0; } -void tfsDestroy() { +void tfsCleanup() { taosHashCleanup(pfs->map); pfs->map = NULL; diff --git a/src/inc/tfs.h b/src/inc/tfs.h index 11e33a3af7..9ad7a8f66e 100644 --- a/src/inc/tfs.h +++ b/src/inc/tfs.h @@ -49,7 +49,7 @@ typedef struct { } STierMeta; int tfsInit(SDiskCfg *pDiskCfg, int ndisk); -void tfsDestroy(); +void tfsCleanup(); void tfsUpdateInfo(SFSMeta *pFSMeta, STierMeta *tierMetas, int8_t numLevels); void tfsGetMeta(SFSMeta *pMeta); void tfsAllocDisk(int expLevel, int *level, int *id); From 21c69b8bf6557322c4430bf3c02c75a32bf79def Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 11 Jan 2022 19:44:42 -0800 Subject: [PATCH 44/57] minor changes --- include/dnode/mgmt/dnode.h | 2 +- source/dnode/mgmt/daemon/src/daemon.c | 2 +- source/dnode/mgmt/impl/src/dndEnv.c | 29 ++++++++++--------- .../dnode/mgmt/impl/test/sut/src/server.cpp | 1 - source/dnode/mgmt/impl/test/sut/src/sut.cpp | 1 + 5 files changed, 18 insertions(+), 17 deletions(-) diff --git a/include/dnode/mgmt/dnode.h b/include/dnode/mgmt/dnode.h index 3f1277a2f9..e64d358de4 100644 --- a/include/dnode/mgmt/dnode.h +++ b/include/dnode/mgmt/dnode.h @@ -29,6 +29,7 @@ typedef struct SDnode SDnode; typedef struct { int32_t sver; int32_t numOfCores; + int16_t numOfCommitThreads; int8_t enableTelem; char timezone[TSDB_TIMEZONE_LEN]; char locale[TSDB_LOCALE_LEN]; @@ -53,7 +54,6 @@ void dndCleanup(); /* ------------------------ SDnode ----------------------- */ typedef struct { - int16_t numOfCommitThreads; int32_t numOfSupportVnodes; int32_t statusInterval; float numOfThreadsPerCore; diff --git a/source/dnode/mgmt/daemon/src/daemon.c b/source/dnode/mgmt/daemon/src/daemon.c index 4e98a35483..ce672bf7b8 100644 --- a/source/dnode/mgmt/daemon/src/daemon.c +++ b/source/dnode/mgmt/daemon/src/daemon.c @@ -139,6 +139,7 @@ void dmnWaitSignal() { void dnmInitEnvCfg(SDnodeEnvCfg *pCfg) { pCfg->sver = 30000000; // 3.0.0.0 pCfg->numOfCores = tsNumOfCores; + pCfg->numOfCommitThreads = tsNumOfCommitThreads; pCfg->enableTelem = 0; tstrncpy(pCfg->timezone, tsTimezone, TSDB_TIMEZONE_LEN); tstrncpy(pCfg->locale, tsLocale, TSDB_LOCALE_LEN); @@ -149,7 +150,6 @@ void dnmInitEnvCfg(SDnodeEnvCfg *pCfg) { void dmnInitObjCfg(SDnodeObjCfg *pCfg) { pCfg->numOfSupportVnodes = tsNumOfSupportVnodes; - pCfg->numOfCommitThreads = tsNumOfCommitThreads; pCfg->statusInterval = tsStatusInterval; pCfg->numOfThreadsPerCore = tsNumOfThreadsPerCore; pCfg->ratioOfQueryCores = tsRatioOfQueryCores; diff --git a/source/dnode/mgmt/impl/src/dndEnv.c b/source/dnode/mgmt/impl/src/dndEnv.c index 88e38daf1c..d6f6bdee06 100644 --- a/source/dnode/mgmt/impl/src/dndEnv.c +++ b/source/dnode/mgmt/impl/src/dndEnv.c @@ -183,20 +183,6 @@ SDnode *dndCreate(SDnodeObjCfg *pCfg) { return NULL; } - SVnodeOpt vnodeOpt = { - .sver = pDnode->env.sver, - .timezone = pDnode->env.timezone, - .locale = pDnode->env.locale, - .charset = pDnode->env.charset, - .nthreads = pDnode->cfg.numOfCommitThreads, - .putReqToVQueryQFp = dndPutReqToVQueryQ, - }; - if (vnodeInit(&vnodeOpt) != 0) { - dError("failed to init vnode since %s", terrstr()); - dndClose(pDnode); - return NULL; - } - if (dndInitMgmt(pDnode) != 0) { dError("failed to init mgmt since %s", terrstr()); dndClose(pDnode); @@ -296,6 +282,21 @@ int32_t dndInit(const SDnodeEnvCfg *pCfg) { return -1; } + SVnodeOpt vnodeOpt = { + .sver = pCfg->sver, + .timezone = pCfg->timezone, + .locale = pCfg->locale, + .charset = pCfg->charset, + .nthreads = pCfg->numOfCommitThreads, + .putReqToVQueryQFp = dndPutReqToVQueryQ, + }; + + if (vnodeInit(&vnodeOpt) != 0) { + dError("failed to init vnode since %s", terrstr()); + dndCleanup(); + return NULL; + } + memcpy(&dndEnv.cfg, pCfg, sizeof(SDnodeEnvCfg)); dInfo("dnode env is initialized"); return 0; diff --git a/source/dnode/mgmt/impl/test/sut/src/server.cpp b/source/dnode/mgmt/impl/test/sut/src/server.cpp index f7007bbbab..f7b5ab4599 100644 --- a/source/dnode/mgmt/impl/test/sut/src/server.cpp +++ b/source/dnode/mgmt/impl/test/sut/src/server.cpp @@ -25,7 +25,6 @@ void* serverLoop(void* param) { SDnodeObjCfg TestServer::BuildOption(const char* path, const char* fqdn, uint16_t port, const char* firstEp) { SDnodeObjCfg cfg = {0}; cfg.numOfSupportVnodes = 16; - cfg.numOfCommitThreads = 1; cfg.statusInterval = 1; cfg.numOfThreadsPerCore = 1; cfg.ratioOfQueryCores = 1; diff --git a/source/dnode/mgmt/impl/test/sut/src/sut.cpp b/source/dnode/mgmt/impl/test/sut/src/sut.cpp index 5912da5bdf..972b4bb332 100644 --- a/source/dnode/mgmt/impl/test/sut/src/sut.cpp +++ b/source/dnode/mgmt/impl/test/sut/src/sut.cpp @@ -44,6 +44,7 @@ void Testbase::InitLog(const char* path) { void Testbase::Init(const char* path, int16_t port) { SDnodeEnvCfg cfg = {0}; + cfg.numOfCommitThreads = 1; dndInit(&cfg); char fqdn[] = "localhost"; From eadb5c27356837036e0508772327bac7bbf8d761 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 12 Jan 2022 12:06:19 +0800 Subject: [PATCH 45/57] [td-11818] fix memory leak. --- source/client/test/clientTests.cpp | 120 ++++++++++++------------- source/libs/planner/src/logicPlan.c | 6 +- source/libs/planner/src/physicalPlan.c | 4 +- source/libs/scheduler/src/scheduler.c | 4 + 4 files changed, 69 insertions(+), 65 deletions(-) diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index d507c565df..d1093bb1a6 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -279,7 +279,7 @@ TEST(testCase, connect_Test) { // taos_free_result(pRes); // taos_close(pConn); //} -// + //TEST(testCase, create_table_Test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // assert(pConn != NULL); @@ -292,26 +292,26 @@ TEST(testCase, connect_Test) { // // taos_close(pConn); //} -// -//TEST(testCase, create_ctable_Test) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// assert(pConn != NULL); -// -// TAOS_RES* pRes = taos_query(pConn, "use abc1"); -// if (taos_errno(pRes) != 0) { -// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "create table tm0 using st1 tags(1)"); -// if (taos_errno(pRes) != 0) { -// printf("failed to create child table tm0, reason:%s\n", taos_errstr(pRes)); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} -// + +TEST(testCase, create_ctable_Test) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + assert(pConn != NULL); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("failed to use db, reason:%s\n", taos_errstr(pRes)); + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "create table tm0 using st1 tags(1)"); + if (taos_errno(pRes) != 0) { + printf("failed to create child table tm0, reason:%s\n", taos_errstr(pRes)); + } + + taos_free_result(pRes); + taos_close(pConn); +} + //TEST(testCase, show_stable_Test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // assert(pConn != NULL); @@ -533,6 +533,7 @@ TEST(testCase, connect_Test) { // tmq_create_topic(pConn, "test_topic_1", sql, strlen(sql)); // taos_close(pConn); //} + //TEST(testCase, insert_test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // ASSERT_EQ(pConn, nullptr); @@ -550,49 +551,48 @@ TEST(testCase, connect_Test) { // taos_free_result(pRes); // taos_close(pConn); //} -//#endif -TEST(testCase, projection_query_tables) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - ASSERT_NE(pConn, nullptr); - -// TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); +//TEST(testCase, projection_query_tables) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// ASSERT_NE(pConn, nullptr); +// +//// TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); +//// if (taos_errno(pRes) != 0) { +//// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); +//// taos_free_result(pRes); +//// return; +//// } +// +//// taos_free_result(pRes); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// +//// pRes = taos_query(pConn, "create table m1 (ts timestamp, k int) tags(a int)"); +// taos_free_result(pRes); +//// +//// pRes = taos_query(pConn, "create table tu using m1 tags(1)"); +//// taos_free_result(pRes); +//// +//// pRes = taos_query(pConn, "insert into tu values(now, 1)"); +//// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "select * from tu"); // if (taos_errno(pRes) != 0) { -// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); +// printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); // taos_free_result(pRes); -// return; +// ASSERT_TRUE(false); // } - -// taos_free_result(pRes); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - -// pRes = taos_query(pConn, "create table m1 (ts timestamp, k int) tags(a int)"); - taos_free_result(pRes); // -// pRes = taos_query(pConn, "create table tu using m1 tags(1)"); -// taos_free_result(pRes); +// TAOS_ROW pRow = NULL; +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } // -// pRes = taos_query(pConn, "insert into tu values(now, 1)"); // taos_free_result(pRes); - - pRes = taos_query(pConn, "select * from tu"); - if (taos_errno(pRes) != 0) { - printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); - } - - TAOS_ROW pRow = NULL; - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); - - char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } - - taos_free_result(pRes); - taos_close(pConn); -} +// taos_close(pConn); +//} diff --git a/source/libs/planner/src/logicPlan.c b/source/libs/planner/src/logicPlan.c index 9a9b40473b..93f72bba95 100644 --- a/source/libs/planner/src/logicPlan.c +++ b/source/libs/planner/src/logicPlan.c @@ -38,10 +38,10 @@ int32_t optimizeQueryPlan(struct SQueryPlanNode* pQueryNode) { } static int32_t createModificationOpPlan(const SQueryNode* pNode, SQueryPlanNode** pQueryPlan) { - SVnodeModifOpStmtInfo* pInsert = (SVnodeModifOpStmtInfo*)pNode; + SVnodeModifOpStmtInfo* pModifStmtInfo = (SVnodeModifOpStmtInfo*)pNode; *pQueryPlan = calloc(1, sizeof(SQueryPlanNode)); - SArray* blocks = taosArrayInit(taosArrayGetSize(pInsert->pDataBlocks), POINTER_BYTES); + SArray* blocks = taosArrayInit(taosArrayGetSize(pModifStmtInfo->pDataBlocks), POINTER_BYTES); SDataPayloadInfo* pPayload = calloc(1, sizeof(SDataPayloadInfo)); if (NULL == *pQueryPlan || NULL == blocks || NULL == pPayload) { @@ -49,7 +49,7 @@ static int32_t createModificationOpPlan(const SQueryNode* pNode, SQueryPlanNode* } (*pQueryPlan)->info.type = QNODE_MODIFY; - taosArrayAddAll(blocks, pInsert->pDataBlocks); + taosArrayAddAll(blocks, pModifStmtInfo->pDataBlocks); if (pNode->type == TSDB_SQL_INSERT) { pPayload->msgType = TDMT_VND_SUBMIT; diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index 0604ba3920..71e69c67d2 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -369,9 +369,9 @@ int32_t createDag(SQueryPlanNode* pQueryNode, struct SCatalog* pCatalog, SQueryD TRY(TSDB_MAX_TAG_CONDITIONS) { SPlanContext context = { .pCatalog = pCatalog, - .pDag = validPointer(calloc(1, sizeof(SQueryDag))), + .pDag = validPointer(calloc(1, sizeof(SQueryDag))), .pCurrentSubplan = NULL, - .nextId = {.queryId = requestId}, + .nextId = {.queryId = requestId}, }; *pDag = context.pDag; diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 525d6ebe87..09e5291c85 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -50,6 +50,10 @@ void schFreeTask(SSchTask* pTask) { if (pTask->parents) { taosArrayDestroy(pTask->parents); } + + if (pTask->execAddrs) { + taosArrayDestroy(pTask->execAddrs); + } } From a24087b8d0806fdc0c8fa4059d4e32cf4bdf0ade Mon Sep 17 00:00:00 2001 From: Shengliang Guan Date: Tue, 11 Jan 2022 19:52:02 -0800 Subject: [PATCH 46/57] minor changes --- source/dnode/mgmt/impl/src/dndEnv.c | 2 +- source/dnode/mgmt/impl/test/sut/src/sut.cpp | 1 + source/dnode/vnode/impl/src/vnodeMgr.c | 2 -- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/source/dnode/mgmt/impl/src/dndEnv.c b/source/dnode/mgmt/impl/src/dndEnv.c index d6f6bdee06..1bf1ea2b92 100644 --- a/source/dnode/mgmt/impl/src/dndEnv.c +++ b/source/dnode/mgmt/impl/src/dndEnv.c @@ -251,7 +251,6 @@ void dndClose(SDnode *pDnode) { dndCleanupQnode(pDnode); dndCleanupVnodes(pDnode); dndCleanupMgmt(pDnode); - vnodeCleanup(); tfsCleanup(); dndCloseImp(pDnode); @@ -309,6 +308,7 @@ void dndCleanup() { } walCleanUp(); + vnodeCleanup(); rpcCleanup(); taosStopCacheRefreshWorker(); diff --git a/source/dnode/mgmt/impl/test/sut/src/sut.cpp b/source/dnode/mgmt/impl/test/sut/src/sut.cpp index 972b4bb332..c6e6c0006d 100644 --- a/source/dnode/mgmt/impl/test/sut/src/sut.cpp +++ b/source/dnode/mgmt/impl/test/sut/src/sut.cpp @@ -45,6 +45,7 @@ void Testbase::InitLog(const char* path) { void Testbase::Init(const char* path, int16_t port) { SDnodeEnvCfg cfg = {0}; cfg.numOfCommitThreads = 1; + cfg.numOfCores = 1; dndInit(&cfg); char fqdn[] = "localhost"; diff --git a/source/dnode/vnode/impl/src/vnodeMgr.c b/source/dnode/vnode/impl/src/vnodeMgr.c index 15550a899b..fdb96e52e2 100644 --- a/source/dnode/vnode/impl/src/vnodeMgr.c +++ b/source/dnode/vnode/impl/src/vnodeMgr.c @@ -61,8 +61,6 @@ void vnodeCleanup() { return; } - walCleanUp(); - // Stop commit handler pthread_mutex_lock(&(vnodeMgr.mutex)); vnodeMgr.stop = true; From c78f3e357f3f80acb3025e692eb6a70ad8455402 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Wed, 12 Jan 2022 09:26:42 +0000 Subject: [PATCH 47/57] refact vnode code --- source/dnode/vnode/CMakeLists.txt | 37 +- source/dnode/vnode/impl/test/CMakeLists.txt | 12 - .../dnode/vnode/impl/test/vBenchmarkTest.cpp | 2 - .../dnode/vnode/impl/test/vnodeApiTests.cpp | 285 ------- .../meta => source/dnode/vnode/inc}/meta.h | 0 .../vnode/tq => source/dnode/vnode/inc}/tq.h | 0 .../tsdb => source/dnode/vnode/inc}/tsdb.h | 0 .../vnode => source/dnode/vnode/inc}/vnode.h | 0 source/dnode/vnode/meta/src/metaSQLiteImpl.c | 212 ----- source/dnode/vnode/meta/test/CMakeLists.txt | 24 - source/dnode/vnode/meta/test/metaAPITest.cpp | 105 --- .../dnode/vnode/{meta => src}/inc/metaCache.h | 0 .../dnode/vnode/{meta => src}/inc/metaCfg.h | 0 source/dnode/vnode/{meta => src}/inc/metaDB.h | 0 .../dnode/vnode/{meta => src}/inc/metaDef.h | 0 .../dnode/vnode/{meta => src}/inc/metaIdx.h | 0 .../dnode/vnode/{meta => src}/inc/metaQuery.h | 0 .../dnode/vnode/{meta => src}/inc/metaTbCfg.h | 0 .../dnode/vnode/{meta => src}/inc/metaTbTag.h | 0 .../dnode/vnode/{meta => src}/inc/metaTbUid.h | 0 source/dnode/vnode/{tq => src}/inc/tqCommit.h | 0 source/dnode/vnode/{tq => src}/inc/tqInt.h | 0 .../dnode/vnode/{tq => src}/inc/tqMetaStore.h | 0 .../vnode/{tsdb => src}/inc/tsdbCommit.h | 0 .../vnode/{tsdb => src}/inc/tsdbCompact.h | 0 .../dnode/vnode/{tsdb => src}/inc/tsdbDef.h | 0 source/dnode/vnode/{tsdb => src}/inc/tsdbFS.h | 0 .../dnode/vnode/{tsdb => src}/inc/tsdbFile.h | 0 .../vnode/{tsdb => src}/inc/tsdbHealth.h | 0 .../dnode/vnode/{tsdb => src}/inc/tsdbLog.h | 0 .../vnode/{tsdb => src}/inc/tsdbMemTable.h | 0 .../vnode/{tsdb => src}/inc/tsdbMemory.h | 0 .../vnode/{tsdb => src}/inc/tsdbOptions.h | 0 .../vnode/{tsdb => src}/inc/tsdbReadImpl.h | 0 .../vnode/{tsdb => src}/inc/tsdbRowMergeBuf.h | 0 .../vnode/{impl => src}/inc/vnodeBufferPool.h | 0 .../dnode/vnode/{impl => src}/inc/vnodeCfg.h | 0 .../vnode/{impl => src}/inc/vnodeCommit.h | 0 .../dnode/vnode/{impl => src}/inc/vnodeDef.h | 2 +- .../dnode/vnode/{impl => src}/inc/vnodeFS.h | 0 .../dnode/vnode/{impl => src}/inc/vnodeInt.h | 2 +- .../dnode/vnode/{impl => src}/inc/vnodeMAF.h | 0 .../{impl => src}/inc/vnodeMemAllocator.h | 0 .../vnode/{impl => src}/inc/vnodeQuery.h | 0 .../dnode/vnode/{impl => src}/inc/vnodeRead.h | 0 .../vnode/{impl => src}/inc/vnodeRequest.h | 0 .../vnode/{impl => src}/inc/vnodeStateMgr.h | 0 .../dnode/vnode/{impl => src}/inc/vnodeSync.h | 2 +- .../vnode/{impl => src}/inc/vnodeWrite.h | 0 .../dnode/vnode/{ => src}/meta/CMakeLists.txt | 0 .../{meta/src => src/meta}/metaBDBImpl.c | 0 .../vnode/{meta/src => src/meta}/metaCache.c | 0 .../vnode/{meta/src => src/meta}/metaCfg.c | 0 .../vnode/{meta/src => src/meta}/metaCommit.c | 0 .../vnode/{meta/src => src/meta}/metaIdx.c | 2 + .../vnode/{meta/src => src/meta}/metaMain.c | 0 .../vnode/{meta/src => src/meta}/metaQuery.c | 0 .../vnode/{meta/src => src/meta}/metaTable.c | 0 .../vnode/{meta/src => src/meta}/metaTbCfg.c | 0 .../vnode/{meta/src => src/meta}/metaTbTag.c | 0 .../vnode/{meta/src => src/meta}/metaTbUid.c | 0 .../dnode/vnode/{ => src}/tq/CMakeLists.txt | 0 source/dnode/vnode/{tq/src => src/tq}/tq.c | 0 .../dnode/vnode/{tq/src => src/tq}/tqCommit.c | 0 .../vnode/{tq/src => src/tq}/tqMetaStore.c | 0 .../dnode/vnode/{ => src}/tsdb/CMakeLists.txt | 0 .../vnode/{tsdb/src => src/tsdb}/tsdbCommit.c | 0 .../{tsdb/src => src/tsdb}/tsdbCompact.c | 0 .../vnode/{tsdb/src => src/tsdb}/tsdbFS.c | 0 .../vnode/{tsdb/src => src/tsdb}/tsdbFile.c | 0 source/dnode/vnode/src/tsdb/tsdbHealth.c | 98 +++ .../vnode/{tsdb/src => src/tsdb}/tsdbMain.c | 0 .../{tsdb/src => src/tsdb}/tsdbMemTable.c | 0 .../{tsdb/src => src/tsdb}/tsdbOptions.c | 0 .../vnode/{tsdb/src => src/tsdb}/tsdbRead.c | 2 +- .../{tsdb/src => src/tsdb}/tsdbReadImpl.c | 0 .../{tsdb/src => src/tsdb}/tsdbRowMergeBuf.c | 26 +- .../vnode/{tsdb/src => src/tsdb}/tsdbScan.c | 2 +- .../vnode/{tsdb/src => src/tsdb}/tsdbWrite.c | 0 .../vnode/{impl => src/vnd}/CMakeLists.txt | 0 .../{impl/src => src/vnd}/vnodeArenaMAImpl.c | 0 .../{impl/src => src/vnd}/vnodeBufferPool.c | 0 .../vnode/{impl/src => src/vnd}/vnodeCfg.c | 0 .../vnode/{impl/src => src/vnd}/vnodeCommit.c | 0 .../vnode/{impl/src => src/vnd}/vnodeFS.c | 0 .../vnode/{impl/src => src/vnd}/vnodeInt.c | 0 .../vnode/{impl/src => src/vnd}/vnodeMain.c | 0 .../vnode/{impl/src => src/vnd}/vnodeMgr.c | 0 .../vnode/{impl/src => src/vnd}/vnodeQuery.c | 0 .../vnode/{impl/src => src/vnd}/vnodeRead.c | 0 .../{impl/src => src/vnd}/vnodeRequest.c | 0 .../{impl/src => src/vnd}/vnodeStateMgr.c | 0 .../vnode/{impl/src => src/vnd}/vnodeSync.c | 0 .../vnode/{impl/src => src/vnd}/vnodeWrite.c | 0 .../dnode/vnode/{tq => }/test/CMakeLists.txt | 0 .../dnode/vnode/{tq => }/test/tqMetaTest.cpp | 0 .../vnode/{tq => }/test/tqSerializerTest.cpp | 0 source/dnode/vnode/tsdb/inc/tsdbint.h | 149 ---- source/dnode/vnode/tsdb/src/tsdbHealth.c | 98 --- source/dnode/vnode/tsdb/src/tsdbSync.c | 724 ------------------ source/dnode/vnode/tsdb/test/tsdbTests.cpp | 0 source/libs/executor/CMakeLists.txt | 2 +- 102 files changed, 152 insertions(+), 1634 deletions(-) delete mode 100644 source/dnode/vnode/impl/test/CMakeLists.txt delete mode 100644 source/dnode/vnode/impl/test/vBenchmarkTest.cpp delete mode 100644 source/dnode/vnode/impl/test/vnodeApiTests.cpp rename {include/dnode/vnode/meta => source/dnode/vnode/inc}/meta.h (100%) rename {include/dnode/vnode/tq => source/dnode/vnode/inc}/tq.h (100%) rename {include/dnode/vnode/tsdb => source/dnode/vnode/inc}/tsdb.h (100%) rename {include/dnode/vnode => source/dnode/vnode/inc}/vnode.h (100%) delete mode 100644 source/dnode/vnode/meta/src/metaSQLiteImpl.c delete mode 100644 source/dnode/vnode/meta/test/CMakeLists.txt delete mode 100644 source/dnode/vnode/meta/test/metaAPITest.cpp rename source/dnode/vnode/{meta => src}/inc/metaCache.h (100%) rename source/dnode/vnode/{meta => src}/inc/metaCfg.h (100%) rename source/dnode/vnode/{meta => src}/inc/metaDB.h (100%) rename source/dnode/vnode/{meta => src}/inc/metaDef.h (100%) rename source/dnode/vnode/{meta => src}/inc/metaIdx.h (100%) rename source/dnode/vnode/{meta => src}/inc/metaQuery.h (100%) rename source/dnode/vnode/{meta => src}/inc/metaTbCfg.h (100%) rename source/dnode/vnode/{meta => src}/inc/metaTbTag.h (100%) rename source/dnode/vnode/{meta => src}/inc/metaTbUid.h (100%) rename source/dnode/vnode/{tq => src}/inc/tqCommit.h (100%) rename source/dnode/vnode/{tq => src}/inc/tqInt.h (100%) rename source/dnode/vnode/{tq => src}/inc/tqMetaStore.h (100%) rename source/dnode/vnode/{tsdb => src}/inc/tsdbCommit.h (100%) rename source/dnode/vnode/{tsdb => src}/inc/tsdbCompact.h (100%) rename source/dnode/vnode/{tsdb => src}/inc/tsdbDef.h (100%) rename source/dnode/vnode/{tsdb => src}/inc/tsdbFS.h (100%) rename source/dnode/vnode/{tsdb => src}/inc/tsdbFile.h (100%) rename source/dnode/vnode/{tsdb => src}/inc/tsdbHealth.h (100%) rename source/dnode/vnode/{tsdb => src}/inc/tsdbLog.h (100%) rename source/dnode/vnode/{tsdb => src}/inc/tsdbMemTable.h (100%) rename source/dnode/vnode/{tsdb => src}/inc/tsdbMemory.h (100%) rename source/dnode/vnode/{tsdb => src}/inc/tsdbOptions.h (100%) rename source/dnode/vnode/{tsdb => src}/inc/tsdbReadImpl.h (100%) rename source/dnode/vnode/{tsdb => src}/inc/tsdbRowMergeBuf.h (100%) rename source/dnode/vnode/{impl => src}/inc/vnodeBufferPool.h (100%) rename source/dnode/vnode/{impl => src}/inc/vnodeCfg.h (100%) rename source/dnode/vnode/{impl => src}/inc/vnodeCommit.h (100%) rename source/dnode/vnode/{impl => src}/inc/vnodeDef.h (98%) rename source/dnode/vnode/{impl => src}/inc/vnodeFS.h (100%) rename source/dnode/vnode/{impl => src}/inc/vnodeInt.h (98%) rename source/dnode/vnode/{impl => src}/inc/vnodeMAF.h (100%) rename source/dnode/vnode/{impl => src}/inc/vnodeMemAllocator.h (100%) rename source/dnode/vnode/{impl => src}/inc/vnodeQuery.h (100%) rename source/dnode/vnode/{impl => src}/inc/vnodeRead.h (100%) rename source/dnode/vnode/{impl => src}/inc/vnodeRequest.h (100%) rename source/dnode/vnode/{impl => src}/inc/vnodeStateMgr.h (100%) rename source/dnode/vnode/{impl => src}/inc/vnodeSync.h (97%) rename source/dnode/vnode/{impl => src}/inc/vnodeWrite.h (100%) rename source/dnode/vnode/{ => src}/meta/CMakeLists.txt (100%) rename source/dnode/vnode/{meta/src => src/meta}/metaBDBImpl.c (100%) rename source/dnode/vnode/{meta/src => src/meta}/metaCache.c (100%) rename source/dnode/vnode/{meta/src => src/meta}/metaCfg.c (100%) rename source/dnode/vnode/{meta/src => src/meta}/metaCommit.c (100%) rename source/dnode/vnode/{meta/src => src/meta}/metaIdx.c (98%) rename source/dnode/vnode/{meta/src => src/meta}/metaMain.c (100%) rename source/dnode/vnode/{meta/src => src/meta}/metaQuery.c (100%) rename source/dnode/vnode/{meta/src => src/meta}/metaTable.c (100%) rename source/dnode/vnode/{meta/src => src/meta}/metaTbCfg.c (100%) rename source/dnode/vnode/{meta/src => src/meta}/metaTbTag.c (100%) rename source/dnode/vnode/{meta/src => src/meta}/metaTbUid.c (100%) rename source/dnode/vnode/{ => src}/tq/CMakeLists.txt (100%) rename source/dnode/vnode/{tq/src => src/tq}/tq.c (100%) rename source/dnode/vnode/{tq/src => src/tq}/tqCommit.c (100%) rename source/dnode/vnode/{tq/src => src/tq}/tqMetaStore.c (100%) rename source/dnode/vnode/{ => src}/tsdb/CMakeLists.txt (100%) rename source/dnode/vnode/{tsdb/src => src/tsdb}/tsdbCommit.c (100%) rename source/dnode/vnode/{tsdb/src => src/tsdb}/tsdbCompact.c (100%) rename source/dnode/vnode/{tsdb/src => src/tsdb}/tsdbFS.c (100%) rename source/dnode/vnode/{tsdb/src => src/tsdb}/tsdbFile.c (100%) create mode 100644 source/dnode/vnode/src/tsdb/tsdbHealth.c rename source/dnode/vnode/{tsdb/src => src/tsdb}/tsdbMain.c (100%) rename source/dnode/vnode/{tsdb/src => src/tsdb}/tsdbMemTable.c (100%) rename source/dnode/vnode/{tsdb/src => src/tsdb}/tsdbOptions.c (100%) rename source/dnode/vnode/{tsdb/src => src/tsdb}/tsdbRead.c (99%) rename source/dnode/vnode/{tsdb/src => src/tsdb}/tsdbReadImpl.c (100%) rename source/dnode/vnode/{tsdb/src => src/tsdb}/tsdbRowMergeBuf.c (53%) rename source/dnode/vnode/{tsdb/src => src/tsdb}/tsdbScan.c (100%) rename source/dnode/vnode/{tsdb/src => src/tsdb}/tsdbWrite.c (100%) rename source/dnode/vnode/{impl => src/vnd}/CMakeLists.txt (100%) rename source/dnode/vnode/{impl/src => src/vnd}/vnodeArenaMAImpl.c (100%) rename source/dnode/vnode/{impl/src => src/vnd}/vnodeBufferPool.c (100%) rename source/dnode/vnode/{impl/src => src/vnd}/vnodeCfg.c (100%) rename source/dnode/vnode/{impl/src => src/vnd}/vnodeCommit.c (100%) rename source/dnode/vnode/{impl/src => src/vnd}/vnodeFS.c (100%) rename source/dnode/vnode/{impl/src => src/vnd}/vnodeInt.c (100%) rename source/dnode/vnode/{impl/src => src/vnd}/vnodeMain.c (100%) rename source/dnode/vnode/{impl/src => src/vnd}/vnodeMgr.c (100%) rename source/dnode/vnode/{impl/src => src/vnd}/vnodeQuery.c (100%) rename source/dnode/vnode/{impl/src => src/vnd}/vnodeRead.c (100%) rename source/dnode/vnode/{impl/src => src/vnd}/vnodeRequest.c (100%) rename source/dnode/vnode/{impl/src => src/vnd}/vnodeStateMgr.c (100%) rename source/dnode/vnode/{impl/src => src/vnd}/vnodeSync.c (100%) rename source/dnode/vnode/{impl/src => src/vnd}/vnodeWrite.c (100%) rename source/dnode/vnode/{tq => }/test/CMakeLists.txt (100%) rename source/dnode/vnode/{tq => }/test/tqMetaTest.cpp (100%) rename source/dnode/vnode/{tq => }/test/tqSerializerTest.cpp (100%) delete mode 100644 source/dnode/vnode/tsdb/inc/tsdbint.h delete mode 100644 source/dnode/vnode/tsdb/src/tsdbHealth.c delete mode 100644 source/dnode/vnode/tsdb/src/tsdbSync.c delete mode 100644 source/dnode/vnode/tsdb/test/tsdbTests.cpp diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index a4a9cff002..9dc4bb1873 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -1,4 +1,33 @@ -add_subdirectory(meta) -add_subdirectory(tq) -add_subdirectory(tsdb) -add_subdirectory(impl) \ No newline at end of file +aux_source_directory(src/meta META_SRC) +aux_source_directory(src/tq TQ_SRC) +aux_source_directory(src/tsdb TSDB_SRC) +aux_source_directory(src/vnd VND_SRC) +list(APPEND + VNODE_SRC + ${META_SRC} + ${TQ_SRC} + ${TSDB_SRC} + ${VND_SRC} +) + +add_library(vnode STATIC ${VNODE_SRC}) +target_include_directories( + vnode + PUBLIC inc + PRIVATE src/inc +) +target_link_libraries( + vnode + PUBLIC os + PUBLIC util + PUBLIC common + PUBLIC transport + PUBLIC bdb + PUBLIC tfs + PUBLIC wal + PUBLIC qworker +) + +if(${BUILD_TEST}) + # add_subdirectory(test) +endif(${BUILD_TEST}) diff --git a/source/dnode/vnode/impl/test/CMakeLists.txt b/source/dnode/vnode/impl/test/CMakeLists.txt deleted file mode 100644 index e1226331e9..0000000000 --- a/source/dnode/vnode/impl/test/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -# Vnode API test -add_executable(vnodeApiTests "") -target_sources(vnodeApiTests - PRIVATE - "vnodeApiTests.cpp" -) -target_link_libraries(vnodeApiTests vnode gtest gtest_main) - -add_test( - NAME vnode_api_tests - COMMAND ${CMAKE_CURRENT_BINARY_DIR}/vnodeApiTests - ) \ No newline at end of file diff --git a/source/dnode/vnode/impl/test/vBenchmarkTest.cpp b/source/dnode/vnode/impl/test/vBenchmarkTest.cpp deleted file mode 100644 index e218886231..0000000000 --- a/source/dnode/vnode/impl/test/vBenchmarkTest.cpp +++ /dev/null @@ -1,2 +0,0 @@ -// https://stackoverflow.com/questions/8565666/benchmarking-with-googletest -// https://github.com/google/benchmark \ No newline at end of file diff --git a/source/dnode/vnode/impl/test/vnodeApiTests.cpp b/source/dnode/vnode/impl/test/vnodeApiTests.cpp deleted file mode 100644 index 6ec58a1e9d..0000000000 --- a/source/dnode/vnode/impl/test/vnodeApiTests.cpp +++ /dev/null @@ -1,285 +0,0 @@ -/** - * @file vnodeApiTests.cpp - * @author hzcheng (hzcheng@taosdata.com) - * @brief VNODE module API tests - * @version 0.1 - * @date 2021-12-13 - * - * @copyright Copyright (c) 2021 - * - */ - -#include -#include - -#include "vnode.h" - -static STSchema *vtCreateBasicSchema() { - STSchemaBuilder sb; - STSchema * pSchema = NULL; - - tdInitTSchemaBuilder(&sb, 0); - - tdAddColToSchema(&sb, TSDB_DATA_TYPE_TIMESTAMP, 0, 0); - for (int i = 1; i < 10; i++) { - tdAddColToSchema(&sb, TSDB_DATA_TYPE_INT, i, 0); - } - - pSchema = tdGetSchemaFromBuilder(&sb); - - tdDestroyTSchemaBuilder(&sb); - - return pSchema; -} - -static STSchema *vtCreateBasicTagSchema() { - STSchemaBuilder sb; - STSchema * pSchema = NULL; - - tdInitTSchemaBuilder(&sb, 0); - - tdAddColToSchema(&sb, TSDB_DATA_TYPE_TIMESTAMP, 0, 0); - for (int i = 10; i < 12; i++) { - tdAddColToSchema(&sb, TSDB_DATA_TYPE_BINARY, i, 20); - } - - pSchema = tdGetSchemaFromBuilder(&sb); - - tdDestroyTSchemaBuilder(&sb); - - return pSchema; -} - -static SKVRow vtCreateBasicTag() { - SKVRowBuilder rb; - SKVRow pTag; - - tdInitKVRowBuilder(&rb); - - for (int i = 0; i < 2; i++) { - void *pVal = malloc(sizeof(VarDataLenT) + strlen("foo")); - varDataLen(pVal) = strlen("foo"); - memcpy(varDataVal(pVal), "foo", strlen("foo")); - - tdAddColToKVRow(&rb, i, TSDB_DATA_TYPE_BINARY, pVal); - free(pVal); - } - - pTag = tdGetKVRowFromBuilder(&rb); - tdDestroyKVRowBuilder(&rb); - - return pTag; -} - -static void vtBuildCreateStbReq(tb_uid_t suid, char *tbname, SRpcMsg **ppMsg) { - SRpcMsg * pMsg; - STSchema *pSchema; - STSchema *pTagSchema; - int zs; - void * pBuf; - - pSchema = vtCreateBasicSchema(); - pTagSchema = vtCreateBasicTagSchema(); - - SVnodeReq vCreateSTbReq; - vnodeSetCreateStbReq(&vCreateSTbReq, tbname, UINT32_MAX, UINT32_MAX, suid, pSchema, pTagSchema); - - zs = vnodeBuildReq(NULL, &vCreateSTbReq, TDMT_VND_CREATE_STB); - pMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg) + zs); - pMsg->msgType = TDMT_VND_CREATE_STB; - pMsg->contLen = zs; - pMsg->pCont = POINTER_SHIFT(pMsg, sizeof(SRpcMsg)); - - pBuf = pMsg->pCont; - vnodeBuildReq(&pBuf, &vCreateSTbReq, TDMT_VND_CREATE_STB); - META_CLEAR_TB_CFG(&vCreateSTbReq); - - tdFreeSchema(pSchema); - tdFreeSchema(pTagSchema); - - *ppMsg = pMsg; -} - -static void vtBuildCreateCtbReq(tb_uid_t suid, char *tbname, SRpcMsg **ppMsg) { - SRpcMsg *pMsg; - int tz; - SKVRow pTag = vtCreateBasicTag(); - - SVnodeReq vCreateCTbReq; - vnodeSetCreateCtbReq(&vCreateCTbReq, tbname, UINT32_MAX, UINT32_MAX, suid, pTag); - - tz = vnodeBuildReq(NULL, &vCreateCTbReq, TDMT_VND_CREATE_TABLE); - pMsg = (SRpcMsg *)malloc(sizeof(SRpcMsg) + tz); - pMsg->msgType = TDMT_VND_CREATE_TABLE; - pMsg->contLen = tz; - pMsg->pCont = POINTER_SHIFT(pMsg, sizeof(*pMsg)); - void *pBuf = pMsg->pCont; - - vnodeBuildReq(&pBuf, &vCreateCTbReq, TDMT_VND_CREATE_TABLE); - META_CLEAR_TB_CFG(&vCreateCTbReq); - free(pTag); - - *ppMsg = pMsg; -} - -static void vtBuildCreateNtbReq(char *tbname, SRpcMsg **ppMsg) { - // TODO -} - -static void vtBuildSubmitReq(SRpcMsg **ppMsg) { - SRpcMsg * pMsg; - SSubmitMsg *pSubmitMsg; - SSubmitBlk *pSubmitBlk; - int tz = 1024; // TODO - - pMsg = (SRpcMsg *)malloc(sizeof(*pMsg) + tz); - pMsg->msgType = TDMT_VND_SUBMIT; - pMsg->contLen = tz; - pMsg->pCont = POINTER_SHIFT(pMsg, sizeof(*pMsg)); - - // For submit msg header - pSubmitMsg = (SSubmitMsg *)(pMsg->pCont); - // pSubmitMsg->header.contLen = 0; - // pSubmitMsg->header.vgId = 0; - // pSubmitMsg->length = 0; - pSubmitMsg->numOfBlocks = 1; - - // For submit blk - pSubmitBlk = (SSubmitBlk *)(pSubmitMsg->blocks); - pSubmitBlk->uid = 0; - pSubmitBlk->tid = 0; - pSubmitBlk->padding = 0; - pSubmitBlk->sversion = 0; - pSubmitBlk->dataLen = 0; - pSubmitBlk->numOfRows = 0; - - // For row batch - - *ppMsg = pMsg; -} - -static void vtClearMsgBatch(SArray *pMsgArr) { - SRpcMsg *pMsg; - for (size_t i = 0; i < taosArrayGetSize(pMsgArr); i++) { - pMsg = *(SRpcMsg **)taosArrayGet(pMsgArr, i); - free(pMsg); - } - - taosArrayClear(pMsgArr); -} - -static void vtProcessAndApplyReqs(SVnode *pVnode, SArray *pMsgArr) { - int rcode; - SRpcMsg *pReq; - SRpcMsg *pRsp; - - rcode = vnodeProcessWMsgs(pVnode, pMsgArr); - GTEST_ASSERT_EQ(rcode, 0); - - for (size_t i = 0; i < taosArrayGetSize(pMsgArr); i++) { - pReq = *(SRpcMsg **)taosArrayGet(pMsgArr, i); - rcode = vnodeApplyWMsg(pVnode, pReq, NULL); - GTEST_ASSERT_EQ(rcode, 0); - } -} - -TEST(vnodeApiTest, vnode_simple_create_table_test) { - tb_uid_t suid = 1638166374163; - SRpcMsg *pMsg; - SArray * pMsgArr = NULL; - SVnode * pVnode; - int rcode; - int ntables = 1000000; - int batch = 10; - char tbname[128]; - - pMsgArr = (SArray *)taosArrayInit(batch, sizeof(pMsg)); - - vnodeDestroy("vnode1"); - GTEST_ASSERT_GE(vnodeInit(2), 0); - - // CREATE AND OPEN A VNODE - pVnode = vnodeOpen("vnode1", NULL); - ASSERT_NE(pVnode, nullptr); - - // CREATE A SUPER TABLE - sprintf(tbname, "st"); - vtBuildCreateStbReq(suid, tbname, &pMsg); - taosArrayPush(pMsgArr, &pMsg); - vtProcessAndApplyReqs(pVnode, pMsgArr); - vtClearMsgBatch(pMsgArr); - - // CREATE A LOT OF CHILD TABLES - for (int i = 0; i < ntables / batch; i++) { - // Build request batch - for (int j = 0; j < batch; j++) { - sprintf(tbname, "ct%d", i * batch + j + 1); - vtBuildCreateCtbReq(suid, tbname, &pMsg); - taosArrayPush(pMsgArr, &pMsg); - } - - // Process request batch - vtProcessAndApplyReqs(pVnode, pMsgArr); - - // Clear request batch - vtClearMsgBatch(pMsgArr); - } - - // CLOSE THE VNODE - vnodeClose(pVnode); - vnodeCleanup(); - - taosArrayDestroy(pMsgArr); -} - -TEST(vnodeApiTest, vnode_simple_insert_test) { - const char *vname = "vnode2"; - char tbname[128]; - tb_uid_t suid = 1638166374163; - SRpcMsg * pMsg; - SArray * pMsgArr; - int rcode; - SVnode * pVnode; - int batch = 1; - int loop = 1000000; - - pMsgArr = (SArray *)taosArrayInit(0, sizeof(pMsg)); - - vnodeDestroy(vname); - - GTEST_ASSERT_GE(vnodeInit(2), 0); - - // Open a vnode - pVnode = vnodeOpen(vname, NULL); - GTEST_ASSERT_NE(pVnode, nullptr); - - // 1. CREATE A SUPER TABLE - sprintf(tbname, "st"); - vtBuildCreateStbReq(suid, tbname, &pMsg); - taosArrayPush(pMsgArr, &pMsg); - vtProcessAndApplyReqs(pVnode, pMsgArr); - vtClearMsgBatch(pMsgArr); - - // 2. CREATE A CHILD TABLE - sprintf(tbname, "t0"); - vtBuildCreateCtbReq(suid, tbname, &pMsg); - taosArrayPush(pMsgArr, &pMsg); - vtProcessAndApplyReqs(pVnode, pMsgArr); - vtClearMsgBatch(pMsgArr); - - // 3. WRITE A LOT OF TIME-SERIES DATA - for (int j = 0; j < loop; j++) { - for (int i = 0; i < batch; i++) { - vtBuildSubmitReq(&pMsg); - taosArrayPush(pMsgArr, &pMsg); - } - vtProcessAndApplyReqs(pVnode, pMsgArr); - vtClearMsgBatch(pMsgArr); - } - - // Close the vnode - vnodeClose(pVnode); - vnodeCleanup(); - - taosArrayDestroy(pMsgArr); -} \ No newline at end of file diff --git a/include/dnode/vnode/meta/meta.h b/source/dnode/vnode/inc/meta.h similarity index 100% rename from include/dnode/vnode/meta/meta.h rename to source/dnode/vnode/inc/meta.h diff --git a/include/dnode/vnode/tq/tq.h b/source/dnode/vnode/inc/tq.h similarity index 100% rename from include/dnode/vnode/tq/tq.h rename to source/dnode/vnode/inc/tq.h diff --git a/include/dnode/vnode/tsdb/tsdb.h b/source/dnode/vnode/inc/tsdb.h similarity index 100% rename from include/dnode/vnode/tsdb/tsdb.h rename to source/dnode/vnode/inc/tsdb.h diff --git a/include/dnode/vnode/vnode.h b/source/dnode/vnode/inc/vnode.h similarity index 100% rename from include/dnode/vnode/vnode.h rename to source/dnode/vnode/inc/vnode.h diff --git a/source/dnode/vnode/meta/src/metaSQLiteImpl.c b/source/dnode/vnode/meta/src/metaSQLiteImpl.c deleted file mode 100644 index fe9ef22fb3..0000000000 --- a/source/dnode/vnode/meta/src/metaSQLiteImpl.c +++ /dev/null @@ -1,212 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "metaDef.h" -#include "sqlite3.h" - -struct SMetaDB { - sqlite3 *pDB; -}; - -int metaOpenDB(SMeta *pMeta) { - char dir[128]; - int rc; - char *err = NULL; - - pMeta->pDB = (SMetaDB *)calloc(1, sizeof(SMetaDB)); - if (pMeta->pDB == NULL) { - // TODO: handle error - return -1; - } - - sprintf(dir, "%s/meta.db", pMeta->path); - rc = sqlite3_open(dir, &(pMeta->pDB->pDB)); - if (rc != SQLITE_OK) { - // TODO: handle error - printf("failed to open meta.db\n"); - } - - // For all tables - rc = sqlite3_exec(pMeta->pDB->pDB, - "CREATE TABLE IF NOT EXISTS tb (" - " tbname VARCHAR(256) NOT NULL UNIQUE," - " tb_uid INTEGER NOT NULL UNIQUE " - ");", - NULL, NULL, &err); - if (rc != SQLITE_OK) { - // TODO: handle error - printf("failed to create meta table tb since %s\n", err); - } - - // For super tables - rc = sqlite3_exec(pMeta->pDB->pDB, - "CREATE TABLE IF NOT EXISTS stb (" - " tb_uid INTEGER NOT NULL UNIQUE," - " tbname VARCHAR(256) NOT NULL UNIQUE," - " tb_schema BLOB NOT NULL," - " tag_schema BLOB NOT NULL" - ");", - NULL, NULL, &err); - if (rc != SQLITE_OK) { - // TODO: handle error - printf("failed to create meta table stb since %s\n", err); - } - - // For normal tables - rc = sqlite3_exec(pMeta->pDB->pDB, - "CREATE TABLE IF NOT EXISTS ntb (" - " tb_uid INTEGER NOT NULL UNIQUE," - " tbname VARCHAR(256) NOT NULL," - " tb_schema BLOB NOT NULL" - ");", - NULL, NULL, &err); - if (rc != SQLITE_OK) { - // TODO: handle error - printf("failed to create meta table ntb since %s\n", err); - } - - sqlite3_exec(pMeta->pDB->pDB, "BEGIN;", NULL, NULL, &err); - - tfree(err); - - return 0; -} - -void metaCloseDB(SMeta *pMeta) { - if (pMeta->pDB) { - sqlite3_exec(pMeta->pDB->pDB, "COMMIT;", NULL, NULL, NULL); - sqlite3_close(pMeta->pDB->pDB); - free(pMeta->pDB); - pMeta->pDB = NULL; - } - - // TODO -} - -int metaSaveTableToDB(SMeta *pMeta, const STbCfg *pTbCfg) { - char sql[256]; - char * err = NULL; - int rc; - tb_uid_t uid; - sqlite3_stmt *stmt; - char buf[256]; - void * pBuf; - - switch (pTbCfg->type) { - case META_SUPER_TABLE: - uid = pTbCfg->stbCfg.suid; - sprintf(sql, - "INSERT INTO tb VALUES (\'%s\', %" PRIu64 - ");" - "CREATE TABLE IF NOT EXISTS stb_%" PRIu64 - " (" - " tb_uid INTEGER NOT NULL UNIQUE," - " tbname VARCHAR(256)," - " tag1 INTEGER);", - pTbCfg->name, uid, uid); - rc = sqlite3_exec(pMeta->pDB->pDB, sql, NULL, NULL, &err); - if (rc != SQLITE_OK) { - printf("failed to create normal table since %s\n", err); - } - - sprintf(sql, "INSERT INTO stb VALUES (%" PRIu64 ", %s, ?, ?)", uid, pTbCfg->name); - sqlite3_prepare_v2(pMeta->pDB->pDB, sql, -1, &stmt, NULL); - - pBuf = buf; - tdEncodeSchema(&pBuf, pTbCfg->stbCfg.pSchema); - sqlite3_bind_blob(stmt, 1, buf, POINTER_DISTANCE(pBuf, buf), NULL); - pBuf = buf; - tdEncodeSchema(&pBuf, pTbCfg->stbCfg.pTagSchema); - sqlite3_bind_blob(stmt, 2, buf, POINTER_DISTANCE(pBuf, buf), NULL); - - sqlite3_step(stmt); - - sqlite3_finalize(stmt); - -#if 0 - sprintf(sql, - "INSERT INTO tb VALUES (?, ?);" - // "INSERT INTO stb VALUES (?, ?, ?, ?);" - // "CREATE TABLE IF NOT EXISTS stb_%" PRIu64 - // " (" - // " tb_uid INTEGER NOT NULL UNIQUE," - // " tbname VARCHAR(256)," - // " tag1 INTEGER);" - , - uid); - rc = sqlite3_prepare_v2(pMeta->pDB->pDB, sql, -1, &stmt, NULL); - if (rc != SQLITE_OK) { - return -1; - } - sqlite3_bind_text(stmt, 1, pTbCfg->name, -1, SQLITE_TRANSIENT); - sqlite3_bind_int64(stmt, 2, uid); - sqlite3_step(stmt); - sqlite3_finalize(stmt); - - - // sqlite3_bind_int64(stmt, 3, uid); - // sqlite3_bind_text(stmt, 4, pTbCfg->name, -1, SQLITE_TRANSIENT); - // pBuf = buf; - // tdEncodeSchema(&pBuf, pTbCfg->stbCfg.pSchema); - // sqlite3_bind_blob(stmt, 5, buf, POINTER_DISTANCE(pBuf, buf), NULL); - // pBuf = buf; - // tdEncodeSchema(&pBuf, pTbCfg->stbCfg.pTagSchema); - // sqlite3_bind_blob(stmt, 6, buf, POINTER_DISTANCE(pBuf, buf), NULL); - - rc = sqliteVjj3_step(stmt); - if (rc != SQLITE_OK) { - printf("failed to create normal table since %s\n", sqlite3_errmsg(pMeta->pDB->pDB)); - } - sqlite3_finalize(stmt); -#endif - break; - case META_NORMAL_TABLE: - // uid = metaGenerateUid(pMeta); - // sprintf(sql, - // "INSERT INTO tb VALUES (\'%s\', %" PRIu64 - // ");" - // "INSERT INTO ntb VALUES (%" PRIu64 ", \'%s\', );", - // pTbCfg->name, uid, uid, pTbCfg->name, ); - - // rc = sqlite3_exec(pMeta->pDB->pDB, sql, NULL, NULL, &err); - // if (rc != SQLITE_OK) { - // printf("failed to create normal table since %s\n", err); - // } - break; - case META_CHILD_TABLE: -#if 0 - uid = metaGenerateUid(pMeta); - // sprintf(sql, "INSERT INTO tb VALUES (\'%s\', %" PRIu64 - // ");" - // "INSERT INTO stb_%" PRIu64 " VALUES (%" PRIu64 ", \'%s\', );"); - rc = sqlite3_exec(pMeta->pDB->pDB, sql, NULL, NULL, &err); - if (rc != SQLITE_OK) { - printf("failed to create child table since %s\n", err); - } -#endif - break; - default: - break; - } - - tfree(err); - - return 0; -} - -int metaRemoveTableFromDb(SMeta *pMeta, tb_uid_t uid) { - /* TODO */ - return 0; -} \ No newline at end of file diff --git a/source/dnode/vnode/meta/test/CMakeLists.txt b/source/dnode/vnode/meta/test/CMakeLists.txt deleted file mode 100644 index 625c07ad56..0000000000 --- a/source/dnode/vnode/meta/test/CMakeLists.txt +++ /dev/null @@ -1,24 +0,0 @@ -# add_executable(metaTest "") -# target_sources(metaTest -# PRIVATE -# "../src/metaMain.c" -# "../src/metaUid.c" -# "metaTests.cpp" -# ) -# target_include_directories(metaTest -# PUBLIC -# "${CMAKE_SOURCE_DIR}/include/server/vnode/meta" -# "${CMAKE_CURRENT_SOURCE_DIR}/../inc" -# ) -# target_link_libraries(metaTest -# os -# util -# common -# gtest_main -# tkv -# ) -# enable_testing() -# add_test( -# NAME meta_test -# COMMAND metaTest -# ) diff --git a/source/dnode/vnode/meta/test/metaAPITest.cpp b/source/dnode/vnode/meta/test/metaAPITest.cpp deleted file mode 100644 index 0d79882018..0000000000 --- a/source/dnode/vnode/meta/test/metaAPITest.cpp +++ /dev/null @@ -1,105 +0,0 @@ -#if 0 -#include -#include -#include - -#include "meta.h" - -static STSchema *metaGetSimpleSchema() { - STSchema * pSchema = NULL; - STSchemaBuilder sb = {0}; - - tdInitTSchemaBuilder(&sb, 0); - tdAddColToSchema(&sb, TSDB_DATA_TYPE_TIMESTAMP, 0, 8); - tdAddColToSchema(&sb, TSDB_DATA_TYPE_INT, 1, 4); - - pSchema = tdGetSchemaFromBuilder(&sb); - tdDestroyTSchemaBuilder(&sb); - - return pSchema; -} - -static SKVRow metaGetSimpleTags() { - SKVRowBuilder kvrb = {0}; - SKVRow row; - - tdInitKVRowBuilder(&kvrb); - int64_t ts = 1634287978000; - int32_t a = 10; - - tdAddColToKVRow(&kvrb, 0, TSDB_DATA_TYPE_TIMESTAMP, (void *)(&ts)); - tdAddColToKVRow(&kvrb, 0, TSDB_DATA_TYPE_INT, (void *)(&a)); - - row = tdGetKVRowFromBuilder(&kvrb); - - tdDestroyKVRowBuilder(&kvrb); - - return row; -} - -TEST(MetaTest, DISABLED_meta_create_1m_normal_tables_test) { - // Open Meta - SMeta *meta = metaOpen(NULL, NULL); - std::cout << "Meta is opened!" << std::endl; - - // Create 1000000 normal tables - META_TABLE_OPTS_DECLARE(tbOpts); - STSchema *pSchema = metaGetSimpleSchema(); - char tbname[128]; - - for (size_t i = 0; i < 1000000; i++) { - sprintf(tbname, "ntb%ld", i); - metaNormalTableOptsInit(&tbOpts, tbname, pSchema); - metaCreateTable(meta, &tbOpts); - metaTableOptsClear(&tbOpts); - } - - tdFreeSchema(pSchema); - - // Close Meta - metaClose(meta); - std::cout << "Meta is closed!" << std::endl; - - // Destroy Meta - metaDestroy("meta"); - std::cout << "Meta is destroyed!" << std::endl; -} - -TEST(MetaTest, meta_create_1m_child_tables_test) { - // Open Meta - SMeta *meta = metaOpen(NULL); - std::cout << "Meta is opened!" << std::endl; - - // Create a super tables - tb_uid_t uid = 477529885843758ul; - META_TABLE_OPTS_DECLARE(tbOpts); - STSchema *pSchema = metaGetSimpleSchema(); - STSchema *pTagSchema = metaGetSimpleSchema(); - - metaSuperTableOptsInit(&tbOpts, "st", uid, pSchema, pTagSchema); - metaCreateTable(meta, &tbOpts); - metaTableOptsClear(&tbOpts); - - tdFreeSchema(pSchema); - tdFreeSchema(pTagSchema); - - // Create 1000000 child tables - char name[128]; - SKVRow row = metaGetSimpleTags(); - for (size_t i = 0; i < 1000000; i++) { - sprintf(name, "ctb%ld", i); - metaChildTableOptsInit(&tbOpts, name, uid, row); - metaCreateTable(meta, &tbOpts); - metaTableOptsClear(&tbOpts); - } - kvRowFree(row); - - // Close Meta - metaClose(meta); - std::cout << "Meta is closed!" << std::endl; - - // Destroy Meta - metaDestroy("meta"); - std::cout << "Meta is destroyed!" << std::endl; -} -#endif \ No newline at end of file diff --git a/source/dnode/vnode/meta/inc/metaCache.h b/source/dnode/vnode/src/inc/metaCache.h similarity index 100% rename from source/dnode/vnode/meta/inc/metaCache.h rename to source/dnode/vnode/src/inc/metaCache.h diff --git a/source/dnode/vnode/meta/inc/metaCfg.h b/source/dnode/vnode/src/inc/metaCfg.h similarity index 100% rename from source/dnode/vnode/meta/inc/metaCfg.h rename to source/dnode/vnode/src/inc/metaCfg.h diff --git a/source/dnode/vnode/meta/inc/metaDB.h b/source/dnode/vnode/src/inc/metaDB.h similarity index 100% rename from source/dnode/vnode/meta/inc/metaDB.h rename to source/dnode/vnode/src/inc/metaDB.h diff --git a/source/dnode/vnode/meta/inc/metaDef.h b/source/dnode/vnode/src/inc/metaDef.h similarity index 100% rename from source/dnode/vnode/meta/inc/metaDef.h rename to source/dnode/vnode/src/inc/metaDef.h diff --git a/source/dnode/vnode/meta/inc/metaIdx.h b/source/dnode/vnode/src/inc/metaIdx.h similarity index 100% rename from source/dnode/vnode/meta/inc/metaIdx.h rename to source/dnode/vnode/src/inc/metaIdx.h diff --git a/source/dnode/vnode/meta/inc/metaQuery.h b/source/dnode/vnode/src/inc/metaQuery.h similarity index 100% rename from source/dnode/vnode/meta/inc/metaQuery.h rename to source/dnode/vnode/src/inc/metaQuery.h diff --git a/source/dnode/vnode/meta/inc/metaTbCfg.h b/source/dnode/vnode/src/inc/metaTbCfg.h similarity index 100% rename from source/dnode/vnode/meta/inc/metaTbCfg.h rename to source/dnode/vnode/src/inc/metaTbCfg.h diff --git a/source/dnode/vnode/meta/inc/metaTbTag.h b/source/dnode/vnode/src/inc/metaTbTag.h similarity index 100% rename from source/dnode/vnode/meta/inc/metaTbTag.h rename to source/dnode/vnode/src/inc/metaTbTag.h diff --git a/source/dnode/vnode/meta/inc/metaTbUid.h b/source/dnode/vnode/src/inc/metaTbUid.h similarity index 100% rename from source/dnode/vnode/meta/inc/metaTbUid.h rename to source/dnode/vnode/src/inc/metaTbUid.h diff --git a/source/dnode/vnode/tq/inc/tqCommit.h b/source/dnode/vnode/src/inc/tqCommit.h similarity index 100% rename from source/dnode/vnode/tq/inc/tqCommit.h rename to source/dnode/vnode/src/inc/tqCommit.h diff --git a/source/dnode/vnode/tq/inc/tqInt.h b/source/dnode/vnode/src/inc/tqInt.h similarity index 100% rename from source/dnode/vnode/tq/inc/tqInt.h rename to source/dnode/vnode/src/inc/tqInt.h diff --git a/source/dnode/vnode/tq/inc/tqMetaStore.h b/source/dnode/vnode/src/inc/tqMetaStore.h similarity index 100% rename from source/dnode/vnode/tq/inc/tqMetaStore.h rename to source/dnode/vnode/src/inc/tqMetaStore.h diff --git a/source/dnode/vnode/tsdb/inc/tsdbCommit.h b/source/dnode/vnode/src/inc/tsdbCommit.h similarity index 100% rename from source/dnode/vnode/tsdb/inc/tsdbCommit.h rename to source/dnode/vnode/src/inc/tsdbCommit.h diff --git a/source/dnode/vnode/tsdb/inc/tsdbCompact.h b/source/dnode/vnode/src/inc/tsdbCompact.h similarity index 100% rename from source/dnode/vnode/tsdb/inc/tsdbCompact.h rename to source/dnode/vnode/src/inc/tsdbCompact.h diff --git a/source/dnode/vnode/tsdb/inc/tsdbDef.h b/source/dnode/vnode/src/inc/tsdbDef.h similarity index 100% rename from source/dnode/vnode/tsdb/inc/tsdbDef.h rename to source/dnode/vnode/src/inc/tsdbDef.h diff --git a/source/dnode/vnode/tsdb/inc/tsdbFS.h b/source/dnode/vnode/src/inc/tsdbFS.h similarity index 100% rename from source/dnode/vnode/tsdb/inc/tsdbFS.h rename to source/dnode/vnode/src/inc/tsdbFS.h diff --git a/source/dnode/vnode/tsdb/inc/tsdbFile.h b/source/dnode/vnode/src/inc/tsdbFile.h similarity index 100% rename from source/dnode/vnode/tsdb/inc/tsdbFile.h rename to source/dnode/vnode/src/inc/tsdbFile.h diff --git a/source/dnode/vnode/tsdb/inc/tsdbHealth.h b/source/dnode/vnode/src/inc/tsdbHealth.h similarity index 100% rename from source/dnode/vnode/tsdb/inc/tsdbHealth.h rename to source/dnode/vnode/src/inc/tsdbHealth.h diff --git a/source/dnode/vnode/tsdb/inc/tsdbLog.h b/source/dnode/vnode/src/inc/tsdbLog.h similarity index 100% rename from source/dnode/vnode/tsdb/inc/tsdbLog.h rename to source/dnode/vnode/src/inc/tsdbLog.h diff --git a/source/dnode/vnode/tsdb/inc/tsdbMemTable.h b/source/dnode/vnode/src/inc/tsdbMemTable.h similarity index 100% rename from source/dnode/vnode/tsdb/inc/tsdbMemTable.h rename to source/dnode/vnode/src/inc/tsdbMemTable.h diff --git a/source/dnode/vnode/tsdb/inc/tsdbMemory.h b/source/dnode/vnode/src/inc/tsdbMemory.h similarity index 100% rename from source/dnode/vnode/tsdb/inc/tsdbMemory.h rename to source/dnode/vnode/src/inc/tsdbMemory.h diff --git a/source/dnode/vnode/tsdb/inc/tsdbOptions.h b/source/dnode/vnode/src/inc/tsdbOptions.h similarity index 100% rename from source/dnode/vnode/tsdb/inc/tsdbOptions.h rename to source/dnode/vnode/src/inc/tsdbOptions.h diff --git a/source/dnode/vnode/tsdb/inc/tsdbReadImpl.h b/source/dnode/vnode/src/inc/tsdbReadImpl.h similarity index 100% rename from source/dnode/vnode/tsdb/inc/tsdbReadImpl.h rename to source/dnode/vnode/src/inc/tsdbReadImpl.h diff --git a/source/dnode/vnode/tsdb/inc/tsdbRowMergeBuf.h b/source/dnode/vnode/src/inc/tsdbRowMergeBuf.h similarity index 100% rename from source/dnode/vnode/tsdb/inc/tsdbRowMergeBuf.h rename to source/dnode/vnode/src/inc/tsdbRowMergeBuf.h diff --git a/source/dnode/vnode/impl/inc/vnodeBufferPool.h b/source/dnode/vnode/src/inc/vnodeBufferPool.h similarity index 100% rename from source/dnode/vnode/impl/inc/vnodeBufferPool.h rename to source/dnode/vnode/src/inc/vnodeBufferPool.h diff --git a/source/dnode/vnode/impl/inc/vnodeCfg.h b/source/dnode/vnode/src/inc/vnodeCfg.h similarity index 100% rename from source/dnode/vnode/impl/inc/vnodeCfg.h rename to source/dnode/vnode/src/inc/vnodeCfg.h diff --git a/source/dnode/vnode/impl/inc/vnodeCommit.h b/source/dnode/vnode/src/inc/vnodeCommit.h similarity index 100% rename from source/dnode/vnode/impl/inc/vnodeCommit.h rename to source/dnode/vnode/src/inc/vnodeCommit.h diff --git a/source/dnode/vnode/impl/inc/vnodeDef.h b/source/dnode/vnode/src/inc/vnodeDef.h similarity index 98% rename from source/dnode/vnode/impl/inc/vnodeDef.h rename to source/dnode/vnode/src/inc/vnodeDef.h index f9172dd351..1c534a8aeb 100644 --- a/source/dnode/vnode/impl/inc/vnodeDef.h +++ b/source/dnode/vnode/src/inc/vnodeDef.h @@ -17,7 +17,7 @@ #define _TD_VNODE_DEF_H_ #include "mallocator.h" -#include "sync.h" +// #include "sync.h" #include "tcoding.h" #include "tlist.h" #include "tlockfree.h" diff --git a/source/dnode/vnode/impl/inc/vnodeFS.h b/source/dnode/vnode/src/inc/vnodeFS.h similarity index 100% rename from source/dnode/vnode/impl/inc/vnodeFS.h rename to source/dnode/vnode/src/inc/vnodeFS.h diff --git a/source/dnode/vnode/impl/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h similarity index 98% rename from source/dnode/vnode/impl/inc/vnodeInt.h rename to source/dnode/vnode/src/inc/vnodeInt.h index 48977ff046..028798bc3e 100644 --- a/source/dnode/vnode/impl/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -19,7 +19,7 @@ #include "vnode.h" #include "meta.h" -#include "sync.h" +// #include "sync.h" #include "tlog.h" #include "tq.h" #include "tsdb.h" diff --git a/source/dnode/vnode/impl/inc/vnodeMAF.h b/source/dnode/vnode/src/inc/vnodeMAF.h similarity index 100% rename from source/dnode/vnode/impl/inc/vnodeMAF.h rename to source/dnode/vnode/src/inc/vnodeMAF.h diff --git a/source/dnode/vnode/impl/inc/vnodeMemAllocator.h b/source/dnode/vnode/src/inc/vnodeMemAllocator.h similarity index 100% rename from source/dnode/vnode/impl/inc/vnodeMemAllocator.h rename to source/dnode/vnode/src/inc/vnodeMemAllocator.h diff --git a/source/dnode/vnode/impl/inc/vnodeQuery.h b/source/dnode/vnode/src/inc/vnodeQuery.h similarity index 100% rename from source/dnode/vnode/impl/inc/vnodeQuery.h rename to source/dnode/vnode/src/inc/vnodeQuery.h diff --git a/source/dnode/vnode/impl/inc/vnodeRead.h b/source/dnode/vnode/src/inc/vnodeRead.h similarity index 100% rename from source/dnode/vnode/impl/inc/vnodeRead.h rename to source/dnode/vnode/src/inc/vnodeRead.h diff --git a/source/dnode/vnode/impl/inc/vnodeRequest.h b/source/dnode/vnode/src/inc/vnodeRequest.h similarity index 100% rename from source/dnode/vnode/impl/inc/vnodeRequest.h rename to source/dnode/vnode/src/inc/vnodeRequest.h diff --git a/source/dnode/vnode/impl/inc/vnodeStateMgr.h b/source/dnode/vnode/src/inc/vnodeStateMgr.h similarity index 100% rename from source/dnode/vnode/impl/inc/vnodeStateMgr.h rename to source/dnode/vnode/src/inc/vnodeStateMgr.h diff --git a/source/dnode/vnode/impl/inc/vnodeSync.h b/source/dnode/vnode/src/inc/vnodeSync.h similarity index 97% rename from source/dnode/vnode/impl/inc/vnodeSync.h rename to source/dnode/vnode/src/inc/vnodeSync.h index a3eb004dfa..e82979551d 100644 --- a/source/dnode/vnode/impl/inc/vnodeSync.h +++ b/source/dnode/vnode/src/inc/vnodeSync.h @@ -16,7 +16,7 @@ #ifndef _TD_VNODE_SYNC_H_ #define _TD_VNODE_SYNC_H_ -#include "sync.h" +// #include "sync.h" #ifdef __cplusplus extern "C" { diff --git a/source/dnode/vnode/impl/inc/vnodeWrite.h b/source/dnode/vnode/src/inc/vnodeWrite.h similarity index 100% rename from source/dnode/vnode/impl/inc/vnodeWrite.h rename to source/dnode/vnode/src/inc/vnodeWrite.h diff --git a/source/dnode/vnode/meta/CMakeLists.txt b/source/dnode/vnode/src/meta/CMakeLists.txt similarity index 100% rename from source/dnode/vnode/meta/CMakeLists.txt rename to source/dnode/vnode/src/meta/CMakeLists.txt diff --git a/source/dnode/vnode/meta/src/metaBDBImpl.c b/source/dnode/vnode/src/meta/metaBDBImpl.c similarity index 100% rename from source/dnode/vnode/meta/src/metaBDBImpl.c rename to source/dnode/vnode/src/meta/metaBDBImpl.c diff --git a/source/dnode/vnode/meta/src/metaCache.c b/source/dnode/vnode/src/meta/metaCache.c similarity index 100% rename from source/dnode/vnode/meta/src/metaCache.c rename to source/dnode/vnode/src/meta/metaCache.c diff --git a/source/dnode/vnode/meta/src/metaCfg.c b/source/dnode/vnode/src/meta/metaCfg.c similarity index 100% rename from source/dnode/vnode/meta/src/metaCfg.c rename to source/dnode/vnode/src/meta/metaCfg.c diff --git a/source/dnode/vnode/meta/src/metaCommit.c b/source/dnode/vnode/src/meta/metaCommit.c similarity index 100% rename from source/dnode/vnode/meta/src/metaCommit.c rename to source/dnode/vnode/src/meta/metaCommit.c diff --git a/source/dnode/vnode/meta/src/metaIdx.c b/source/dnode/vnode/src/meta/metaIdx.c similarity index 98% rename from source/dnode/vnode/meta/src/metaIdx.c rename to source/dnode/vnode/src/meta/metaIdx.c index 3da56fc394..d9abb4bb7b 100644 --- a/source/dnode/vnode/meta/src/metaIdx.c +++ b/source/dnode/vnode/src/meta/metaIdx.c @@ -13,7 +13,9 @@ * along with this program. If not, see . */ +#ifdef USE_INVERTED_INDEX #include "index.h" +#endif #include "metaDef.h" struct SMetaIdx { diff --git a/source/dnode/vnode/meta/src/metaMain.c b/source/dnode/vnode/src/meta/metaMain.c similarity index 100% rename from source/dnode/vnode/meta/src/metaMain.c rename to source/dnode/vnode/src/meta/metaMain.c diff --git a/source/dnode/vnode/meta/src/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c similarity index 100% rename from source/dnode/vnode/meta/src/metaQuery.c rename to source/dnode/vnode/src/meta/metaQuery.c diff --git a/source/dnode/vnode/meta/src/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c similarity index 100% rename from source/dnode/vnode/meta/src/metaTable.c rename to source/dnode/vnode/src/meta/metaTable.c diff --git a/source/dnode/vnode/meta/src/metaTbCfg.c b/source/dnode/vnode/src/meta/metaTbCfg.c similarity index 100% rename from source/dnode/vnode/meta/src/metaTbCfg.c rename to source/dnode/vnode/src/meta/metaTbCfg.c diff --git a/source/dnode/vnode/meta/src/metaTbTag.c b/source/dnode/vnode/src/meta/metaTbTag.c similarity index 100% rename from source/dnode/vnode/meta/src/metaTbTag.c rename to source/dnode/vnode/src/meta/metaTbTag.c diff --git a/source/dnode/vnode/meta/src/metaTbUid.c b/source/dnode/vnode/src/meta/metaTbUid.c similarity index 100% rename from source/dnode/vnode/meta/src/metaTbUid.c rename to source/dnode/vnode/src/meta/metaTbUid.c diff --git a/source/dnode/vnode/tq/CMakeLists.txt b/source/dnode/vnode/src/tq/CMakeLists.txt similarity index 100% rename from source/dnode/vnode/tq/CMakeLists.txt rename to source/dnode/vnode/src/tq/CMakeLists.txt diff --git a/source/dnode/vnode/tq/src/tq.c b/source/dnode/vnode/src/tq/tq.c similarity index 100% rename from source/dnode/vnode/tq/src/tq.c rename to source/dnode/vnode/src/tq/tq.c diff --git a/source/dnode/vnode/tq/src/tqCommit.c b/source/dnode/vnode/src/tq/tqCommit.c similarity index 100% rename from source/dnode/vnode/tq/src/tqCommit.c rename to source/dnode/vnode/src/tq/tqCommit.c diff --git a/source/dnode/vnode/tq/src/tqMetaStore.c b/source/dnode/vnode/src/tq/tqMetaStore.c similarity index 100% rename from source/dnode/vnode/tq/src/tqMetaStore.c rename to source/dnode/vnode/src/tq/tqMetaStore.c diff --git a/source/dnode/vnode/tsdb/CMakeLists.txt b/source/dnode/vnode/src/tsdb/CMakeLists.txt similarity index 100% rename from source/dnode/vnode/tsdb/CMakeLists.txt rename to source/dnode/vnode/src/tsdb/CMakeLists.txt diff --git a/source/dnode/vnode/tsdb/src/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c similarity index 100% rename from source/dnode/vnode/tsdb/src/tsdbCommit.c rename to source/dnode/vnode/src/tsdb/tsdbCommit.c diff --git a/source/dnode/vnode/tsdb/src/tsdbCompact.c b/source/dnode/vnode/src/tsdb/tsdbCompact.c similarity index 100% rename from source/dnode/vnode/tsdb/src/tsdbCompact.c rename to source/dnode/vnode/src/tsdb/tsdbCompact.c diff --git a/source/dnode/vnode/tsdb/src/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c similarity index 100% rename from source/dnode/vnode/tsdb/src/tsdbFS.c rename to source/dnode/vnode/src/tsdb/tsdbFS.c diff --git a/source/dnode/vnode/tsdb/src/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c similarity index 100% rename from source/dnode/vnode/tsdb/src/tsdbFile.c rename to source/dnode/vnode/src/tsdb/tsdbFile.c diff --git a/source/dnode/vnode/src/tsdb/tsdbHealth.c b/source/dnode/vnode/src/tsdb/tsdbHealth.c new file mode 100644 index 0000000000..99c1b925b0 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbHealth.c @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// #include "os.h" +// #include "tmsg.h" +// #include "tarray.h" +// #include "query.h" +// #include "tglobal.h" +// #include "tlist.h" +// #include "tsdbint.h" +// #include "tsdbBuffer.h" +// #include "tsdbLog.h" +// #include "tsdbHealth.h" +// #include "ttimer.h" +// #include "tthread.h" + + +// // return malloc new block count +// int32_t tsdbInsertNewBlock(STsdbRepo * pRepo) { +// STsdbBufPool *pPool = pRepo->pPool; +// int32_t cnt = 0; + +// if(tsdbAllowNewBlock(pRepo)) { +// STsdbBufBlock *pBufBlock = tsdbNewBufBlock(pPool->bufBlockSize); +// if (pBufBlock) { +// if (tdListAppend(pPool->bufBlockList, (void *)(&pBufBlock)) < 0) { +// // append error +// tsdbFreeBufBlock(pBufBlock); +// } else { +// pPool->nElasticBlocks ++; +// cnt ++ ; +// } +// } +// } +// return cnt; +// } + +// // switch anther thread to run +// void* cbKillQueryFree(void* param) { +// STsdbRepo* pRepo = (STsdbRepo*)param; +// // vnode +// if(pRepo->appH.notifyStatus) { +// pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_NOBLOCK, TSDB_CODE_SUCCESS); +// } + +// // free +// if(pRepo->pthread){ +// void* p = pRepo->pthread; +// pRepo->pthread = NULL; +// free(p); +// } + +// return NULL; +// } + +// // return true do free , false do nothing +// bool tsdbUrgeQueryFree(STsdbRepo * pRepo) { +// // check previous running +// if(pRepo->pthread && taosThreadRunning(pRepo->pthread)) { +// tsdbWarn("vgId:%d pre urge thread is runing. nBlocks=%d nElasticBlocks=%d", REPO_ID(pRepo), pRepo->pPool->nBufBlocks, pRepo->pPool->nElasticBlocks); +// return false; +// } +// // create new +// pRepo->pthread = taosCreateThread(cbKillQueryFree, pRepo); +// if(pRepo->pthread == NULL) { +// tsdbError("vgId:%d create urge thread error.", REPO_ID(pRepo)); +// return false; +// } +// return true; +// } + +// bool tsdbAllowNewBlock(STsdbRepo* pRepo) { +// int32_t nMaxElastic = pRepo->config.totalBlocks/3; +// STsdbBufPool* pPool = pRepo->pPool; +// if(pPool->nElasticBlocks >= nMaxElastic) { +// tsdbWarn("vgId:%d tsdbAllowNewBlock return fasle. nElasticBlock(%d) >= MaxElasticBlocks(%d)", REPO_ID(pRepo), pPool->nElasticBlocks, nMaxElastic); +// return false; +// } +// return true; +// } + +// bool tsdbNoProblem(STsdbRepo* pRepo) { +// if(listNEles(pRepo->pPool->bufBlockList) == 0) +// return false; +// return true; +// } \ No newline at end of file diff --git a/source/dnode/vnode/tsdb/src/tsdbMain.c b/source/dnode/vnode/src/tsdb/tsdbMain.c similarity index 100% rename from source/dnode/vnode/tsdb/src/tsdbMain.c rename to source/dnode/vnode/src/tsdb/tsdbMain.c diff --git a/source/dnode/vnode/tsdb/src/tsdbMemTable.c b/source/dnode/vnode/src/tsdb/tsdbMemTable.c similarity index 100% rename from source/dnode/vnode/tsdb/src/tsdbMemTable.c rename to source/dnode/vnode/src/tsdb/tsdbMemTable.c diff --git a/source/dnode/vnode/tsdb/src/tsdbOptions.c b/source/dnode/vnode/src/tsdb/tsdbOptions.c similarity index 100% rename from source/dnode/vnode/tsdb/src/tsdbOptions.c rename to source/dnode/vnode/src/tsdb/tsdbOptions.c diff --git a/source/dnode/vnode/tsdb/src/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c similarity index 99% rename from source/dnode/vnode/tsdb/src/tsdbRead.c rename to source/dnode/vnode/src/tsdb/tsdbRead.c index 906046ed9a..a8296e8f8e 100644 --- a/source/dnode/vnode/tsdb/src/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -28,7 +28,7 @@ #include "taosdef.h" #include "tlosertree.h" -#include "tsdbint.h" +#include "tsdbDef.h" #include "tmsg.h" #define EXTRA_BYTES 2 diff --git a/source/dnode/vnode/tsdb/src/tsdbReadImpl.c b/source/dnode/vnode/src/tsdb/tsdbReadImpl.c similarity index 100% rename from source/dnode/vnode/tsdb/src/tsdbReadImpl.c rename to source/dnode/vnode/src/tsdb/tsdbReadImpl.c diff --git a/source/dnode/vnode/tsdb/src/tsdbRowMergeBuf.c b/source/dnode/vnode/src/tsdb/tsdbRowMergeBuf.c similarity index 53% rename from source/dnode/vnode/tsdb/src/tsdbRowMergeBuf.c rename to source/dnode/vnode/src/tsdb/tsdbRowMergeBuf.c index 5ce580f70f..1eebea22d4 100644 --- a/source/dnode/vnode/tsdb/src/tsdbRowMergeBuf.c +++ b/source/dnode/vnode/src/tsdb/tsdbRowMergeBuf.c @@ -13,18 +13,18 @@ * along with this program. If not, see . */ -#include "tsdbRowMergeBuf.h" -#include "tdataformat.h" +// #include "tsdbRowMergeBuf.h" +// #include "tdataformat.h" -// row1 has higher priority -SMemRow tsdbMergeTwoRows(SMergeBuf *pBuf, SMemRow row1, SMemRow row2, STSchema *pSchema1, STSchema *pSchema2) { - if(row2 == NULL) return row1; - if(row1 == NULL) return row2; - ASSERT(pSchema1->version == memRowVersion(row1)); - ASSERT(pSchema2->version == memRowVersion(row2)); +// // row1 has higher priority +// SMemRow tsdbMergeTwoRows(SMergeBuf *pBuf, SMemRow row1, SMemRow row2, STSchema *pSchema1, STSchema *pSchema2) { +// if(row2 == NULL) return row1; +// if(row1 == NULL) return row2; +// ASSERT(pSchema1->version == memRowVersion(row1)); +// ASSERT(pSchema2->version == memRowVersion(row2)); - if(tsdbMergeBufMakeSureRoom(pBuf, pSchema1, pSchema2) < 0) { - return NULL; - } - return mergeTwoMemRows(*pBuf, row1, row2, pSchema1, pSchema2); -} +// if(tsdbMergeBufMakeSureRoom(pBuf, pSchema1, pSchema2) < 0) { +// return NULL; +// } +// return mergeTwoMemRows(*pBuf, row1, row2, pSchema1, pSchema2); +// } diff --git a/source/dnode/vnode/tsdb/src/tsdbScan.c b/source/dnode/vnode/src/tsdb/tsdbScan.c similarity index 100% rename from source/dnode/vnode/tsdb/src/tsdbScan.c rename to source/dnode/vnode/src/tsdb/tsdbScan.c index 382f7b11ae..c0e468e640 100644 --- a/source/dnode/vnode/tsdb/src/tsdbScan.c +++ b/source/dnode/vnode/src/tsdb/tsdbScan.c @@ -13,9 +13,9 @@ * along with this program. If not, see . */ -#include "tsdbint.h" #if 0 +#include "tsdbint.h" #ifndef _TSDB_PLUGINS int tsdbScanFGroup(STsdbScanHandle* pScanHandle, char* rootDir, int fid) { return 0; } diff --git a/source/dnode/vnode/tsdb/src/tsdbWrite.c b/source/dnode/vnode/src/tsdb/tsdbWrite.c similarity index 100% rename from source/dnode/vnode/tsdb/src/tsdbWrite.c rename to source/dnode/vnode/src/tsdb/tsdbWrite.c diff --git a/source/dnode/vnode/impl/CMakeLists.txt b/source/dnode/vnode/src/vnd/CMakeLists.txt similarity index 100% rename from source/dnode/vnode/impl/CMakeLists.txt rename to source/dnode/vnode/src/vnd/CMakeLists.txt diff --git a/source/dnode/vnode/impl/src/vnodeArenaMAImpl.c b/source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c similarity index 100% rename from source/dnode/vnode/impl/src/vnodeArenaMAImpl.c rename to source/dnode/vnode/src/vnd/vnodeArenaMAImpl.c diff --git a/source/dnode/vnode/impl/src/vnodeBufferPool.c b/source/dnode/vnode/src/vnd/vnodeBufferPool.c similarity index 100% rename from source/dnode/vnode/impl/src/vnodeBufferPool.c rename to source/dnode/vnode/src/vnd/vnodeBufferPool.c diff --git a/source/dnode/vnode/impl/src/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c similarity index 100% rename from source/dnode/vnode/impl/src/vnodeCfg.c rename to source/dnode/vnode/src/vnd/vnodeCfg.c diff --git a/source/dnode/vnode/impl/src/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c similarity index 100% rename from source/dnode/vnode/impl/src/vnodeCommit.c rename to source/dnode/vnode/src/vnd/vnodeCommit.c diff --git a/source/dnode/vnode/impl/src/vnodeFS.c b/source/dnode/vnode/src/vnd/vnodeFS.c similarity index 100% rename from source/dnode/vnode/impl/src/vnodeFS.c rename to source/dnode/vnode/src/vnd/vnodeFS.c diff --git a/source/dnode/vnode/impl/src/vnodeInt.c b/source/dnode/vnode/src/vnd/vnodeInt.c similarity index 100% rename from source/dnode/vnode/impl/src/vnodeInt.c rename to source/dnode/vnode/src/vnd/vnodeInt.c diff --git a/source/dnode/vnode/impl/src/vnodeMain.c b/source/dnode/vnode/src/vnd/vnodeMain.c similarity index 100% rename from source/dnode/vnode/impl/src/vnodeMain.c rename to source/dnode/vnode/src/vnd/vnodeMain.c diff --git a/source/dnode/vnode/impl/src/vnodeMgr.c b/source/dnode/vnode/src/vnd/vnodeMgr.c similarity index 100% rename from source/dnode/vnode/impl/src/vnodeMgr.c rename to source/dnode/vnode/src/vnd/vnodeMgr.c diff --git a/source/dnode/vnode/impl/src/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c similarity index 100% rename from source/dnode/vnode/impl/src/vnodeQuery.c rename to source/dnode/vnode/src/vnd/vnodeQuery.c diff --git a/source/dnode/vnode/impl/src/vnodeRead.c b/source/dnode/vnode/src/vnd/vnodeRead.c similarity index 100% rename from source/dnode/vnode/impl/src/vnodeRead.c rename to source/dnode/vnode/src/vnd/vnodeRead.c diff --git a/source/dnode/vnode/impl/src/vnodeRequest.c b/source/dnode/vnode/src/vnd/vnodeRequest.c similarity index 100% rename from source/dnode/vnode/impl/src/vnodeRequest.c rename to source/dnode/vnode/src/vnd/vnodeRequest.c diff --git a/source/dnode/vnode/impl/src/vnodeStateMgr.c b/source/dnode/vnode/src/vnd/vnodeStateMgr.c similarity index 100% rename from source/dnode/vnode/impl/src/vnodeStateMgr.c rename to source/dnode/vnode/src/vnd/vnodeStateMgr.c diff --git a/source/dnode/vnode/impl/src/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c similarity index 100% rename from source/dnode/vnode/impl/src/vnodeSync.c rename to source/dnode/vnode/src/vnd/vnodeSync.c diff --git a/source/dnode/vnode/impl/src/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c similarity index 100% rename from source/dnode/vnode/impl/src/vnodeWrite.c rename to source/dnode/vnode/src/vnd/vnodeWrite.c diff --git a/source/dnode/vnode/tq/test/CMakeLists.txt b/source/dnode/vnode/test/CMakeLists.txt similarity index 100% rename from source/dnode/vnode/tq/test/CMakeLists.txt rename to source/dnode/vnode/test/CMakeLists.txt diff --git a/source/dnode/vnode/tq/test/tqMetaTest.cpp b/source/dnode/vnode/test/tqMetaTest.cpp similarity index 100% rename from source/dnode/vnode/tq/test/tqMetaTest.cpp rename to source/dnode/vnode/test/tqMetaTest.cpp diff --git a/source/dnode/vnode/tq/test/tqSerializerTest.cpp b/source/dnode/vnode/test/tqSerializerTest.cpp similarity index 100% rename from source/dnode/vnode/tq/test/tqSerializerTest.cpp rename to source/dnode/vnode/test/tqSerializerTest.cpp diff --git a/source/dnode/vnode/tsdb/inc/tsdbint.h b/source/dnode/vnode/tsdb/inc/tsdbint.h deleted file mode 100644 index bdd1bd6f71..0000000000 --- a/source/dnode/vnode/tsdb/inc/tsdbint.h +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TSDB_INT_H_ -#define _TD_TSDB_INT_H_ - -#if 0 -// // TODO: remove the include -// #include -// #include -// #include -// #include -// #include -// #include -// #include -// #include - -#include "hash.h" -#include "os.h" -#include "taosdef.h" -#include "taoserror.h" -#include "tarray.h" -#include "tchecksum.h" -#include "tcoding.h" -#include "tcompression.h" -#include "tdataformat.h" -#include "tfs.h" -#include "tlist.h" -#include "tlockfree.h" -#include "tlog.h" -#include "tskiplist.h" -#include "tsocket.h" - -#include "tsdb.h" - -#ifdef __cplusplus -extern "C" { -#endif - -// Log -#include "tsdbLog.h" -// Meta -#include "tsdbMeta.h" -// Buffer -#include "tsdbBuffer.h" -// MemTable -#include "tsdbMemTable.h" -// File -#include "tsdbFile.h" -// FS -#include "tsdbFS.h" -// ReadImpl -#include "tsdbReadImpl.h" -// Commit -#include "tsdbCommit.h" -// Compact -#include "tsdbCompact.h" -// Commit Queue -#include "tsdbCommitQueue.h" - -#include "tsdbRowMergeBuf.h" -// Main definitions -struct STsdbRepo { - uint8_t state; - - STsdbCfg config; - - STsdbCfg save_config; // save apply config - bool config_changed; // config changed flag - pthread_mutex_t save_mutex; // protect save config - - uint8_t hasCachedLastColumn; - - STsdbAppH appH; - STsdbStat stat; - STsdbMeta* tsdbMeta; - STsdbBufPool* pPool; - SMemTable* mem; - SMemTable* imem; - STsdbFS* fs; - SRtn rtn; - tsem_t readyToCommit; - pthread_mutex_t mutex; - bool repoLocked; - int32_t code; // Commit code - - SMergeBuf mergeBuf; //used when update=2 - int8_t compactState; // compact state: inCompact/noCompact/waitingCompact? - pthread_t* pthread; -}; - -#define REPO_ID(r) (r)->config.tsdbId -#define REPO_CFG(r) (&((r)->config)) -#define REPO_FS(r) ((r)->fs) -#define IS_REPO_LOCKED(r) (r)->repoLocked -#define TSDB_SUBMIT_MSG_HEAD_SIZE sizeof(SSubmitMsg) - -int tsdbLockRepo(STsdbRepo* pRepo); -int tsdbUnlockRepo(STsdbRepo* pRepo); -STsdbMeta* tsdbGetMeta(STsdbRepo* pRepo); -int tsdbCheckCommit(STsdbRepo* pRepo); -int tsdbRestoreInfo(STsdbRepo* pRepo); -int tsdbCacheLastData(STsdbRepo *pRepo, STsdbCfg* oldCfg); -void tsdbGetRootDir(int repoid, char dirName[]); -void tsdbGetDataDir(int repoid, char dirName[]); - -static FORCE_INLINE STsdbBufBlock* tsdbGetCurrBufBlock(STsdbRepo* pRepo) { - ASSERT(pRepo != NULL); - if (pRepo->mem == NULL) return NULL; - - SListNode* pNode = listTail(pRepo->mem->bufBlockList); - if (pNode == NULL) return NULL; - - STsdbBufBlock* pBufBlock = NULL; - tdListNodeGetData(pRepo->mem->bufBlockList, pNode, (void*)(&pBufBlock)); - - return pBufBlock; -} - -static FORCE_INLINE int tsdbGetNextMaxTables(int tid) { - ASSERT(tid >= 1 && tid <= TSDB_MAX_TABLES); - int maxTables = TSDB_INIT_NTABLES; - while (true) { - maxTables = MIN(maxTables, TSDB_MAX_TABLES); - if (tid <= maxTables) break; - maxTables *= 2; - } - - return maxTables + 1; -} - -#ifdef __cplusplus -} -#endif - -#endif -#endif /* _TD_TSDB_INT_H_ */ diff --git a/source/dnode/vnode/tsdb/src/tsdbHealth.c b/source/dnode/vnode/tsdb/src/tsdbHealth.c deleted file mode 100644 index 4205f3e90f..0000000000 --- a/source/dnode/vnode/tsdb/src/tsdbHealth.c +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "os.h" -#include "tmsg.h" -#include "tarray.h" -#include "query.h" -#include "tglobal.h" -#include "tlist.h" -#include "tsdbint.h" -#include "tsdbBuffer.h" -#include "tsdbLog.h" -#include "tsdbHealth.h" -#include "ttimer.h" -#include "tthread.h" - - -// return malloc new block count -int32_t tsdbInsertNewBlock(STsdbRepo * pRepo) { - STsdbBufPool *pPool = pRepo->pPool; - int32_t cnt = 0; - - if(tsdbAllowNewBlock(pRepo)) { - STsdbBufBlock *pBufBlock = tsdbNewBufBlock(pPool->bufBlockSize); - if (pBufBlock) { - if (tdListAppend(pPool->bufBlockList, (void *)(&pBufBlock)) < 0) { - // append error - tsdbFreeBufBlock(pBufBlock); - } else { - pPool->nElasticBlocks ++; - cnt ++ ; - } - } - } - return cnt; -} - -// switch anther thread to run -void* cbKillQueryFree(void* param) { - STsdbRepo* pRepo = (STsdbRepo*)param; - // vnode - if(pRepo->appH.notifyStatus) { - pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_NOBLOCK, TSDB_CODE_SUCCESS); - } - - // free - if(pRepo->pthread){ - void* p = pRepo->pthread; - pRepo->pthread = NULL; - free(p); - } - - return NULL; -} - -// return true do free , false do nothing -bool tsdbUrgeQueryFree(STsdbRepo * pRepo) { - // check previous running - if(pRepo->pthread && taosThreadRunning(pRepo->pthread)) { - tsdbWarn("vgId:%d pre urge thread is runing. nBlocks=%d nElasticBlocks=%d", REPO_ID(pRepo), pRepo->pPool->nBufBlocks, pRepo->pPool->nElasticBlocks); - return false; - } - // create new - pRepo->pthread = taosCreateThread(cbKillQueryFree, pRepo); - if(pRepo->pthread == NULL) { - tsdbError("vgId:%d create urge thread error.", REPO_ID(pRepo)); - return false; - } - return true; -} - -bool tsdbAllowNewBlock(STsdbRepo* pRepo) { - int32_t nMaxElastic = pRepo->config.totalBlocks/3; - STsdbBufPool* pPool = pRepo->pPool; - if(pPool->nElasticBlocks >= nMaxElastic) { - tsdbWarn("vgId:%d tsdbAllowNewBlock return fasle. nElasticBlock(%d) >= MaxElasticBlocks(%d)", REPO_ID(pRepo), pPool->nElasticBlocks, nMaxElastic); - return false; - } - return true; -} - -bool tsdbNoProblem(STsdbRepo* pRepo) { - if(listNEles(pRepo->pPool->bufBlockList) == 0) - return false; - return true; -} \ No newline at end of file diff --git a/source/dnode/vnode/tsdb/src/tsdbSync.c b/source/dnode/vnode/tsdb/src/tsdbSync.c deleted file mode 100644 index edcb84d091..0000000000 --- a/source/dnode/vnode/tsdb/src/tsdbSync.c +++ /dev/null @@ -1,724 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#define _DEFAULT_SOURCE -#include "os.h" -#include "taoserror.h" -#include "tsdbint.h" - -// Sync handle -typedef struct { - STsdbRepo *pRepo; - SRtn rtn; - SOCKET socketFd; - void * pBuf; - bool mfChanged; - SMFile * pmf; - SMFile mf; - SDFileSet df; - SDFileSet *pdf; -} SSyncH; - -#define SYNC_BUFFER(sh) ((sh)->pBuf) - -static void tsdbInitSyncH(SSyncH *pSyncH, STsdbRepo *pRepo, SOCKET socketFd); -static void tsdbDestroySyncH(SSyncH *pSyncH); -static int32_t tsdbSyncSendMeta(SSyncH *pSynch); -static int32_t tsdbSyncRecvMeta(SSyncH *pSynch); -static int32_t tsdbSendMetaInfo(SSyncH *pSynch); -static int32_t tsdbRecvMetaInfo(SSyncH *pSynch); -static int32_t tsdbSendDecision(SSyncH *pSynch, bool toSend); -static int32_t tsdbRecvDecision(SSyncH *pSynch, bool *toSend); -static int32_t tsdbSyncSendDFileSetArray(SSyncH *pSynch); -static int32_t tsdbSyncRecvDFileSetArray(SSyncH *pSynch); -static bool tsdbIsTowFSetSame(SDFileSet *pSet1, SDFileSet *pSet2); -static int32_t tsdbSyncSendDFileSet(SSyncH *pSynch, SDFileSet *pSet); -static int32_t tsdbSendDFileSetInfo(SSyncH *pSynch, SDFileSet *pSet); -static int32_t tsdbRecvDFileSetInfo(SSyncH *pSynch); -static int tsdbReload(STsdbRepo *pRepo, bool isMfChanged); - -int32_t tsdbSyncSend(void *tsdb, SOCKET socketFd) { - STsdbRepo *pRepo = (STsdbRepo *)tsdb; - SSyncH synch = {0}; - - tsdbInitSyncH(&synch, pRepo, socketFd); - // Disable TSDB commit - tsem_wait(&(pRepo->readyToCommit)); - - if (tsdbSyncSendMeta(&synch) < 0) { - tsdbError("vgId:%d, failed to send metafile since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _err; - } - - if (tsdbSyncSendDFileSetArray(&synch) < 0) { - tsdbError("vgId:%d, failed to send filesets since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _err; - } - - // Enable TSDB commit - tsem_post(&(pRepo->readyToCommit)); - tsdbDestroySyncH(&synch); - return 0; - -_err: - tsem_post(&(pRepo->readyToCommit)); - tsdbDestroySyncH(&synch); - return -1; -} - -int32_t tsdbSyncRecv(void *tsdb, SOCKET socketFd) { - STsdbRepo *pRepo = (STsdbRepo *)tsdb; - SSyncH synch = {0}; - - pRepo->state = TSDB_STATE_OK; - - tsdbInitSyncH(&synch, pRepo, socketFd); - tsem_wait(&(pRepo->readyToCommit)); - tsdbStartFSTxn(pRepo, 0, 0); - - if (tsdbSyncRecvMeta(&synch) < 0) { - tsdbError("vgId:%d, failed to recv metafile since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _err; - } - - if (tsdbSyncRecvDFileSetArray(&synch) < 0) { - tsdbError("vgId:%d, failed to recv filesets since %s", REPO_ID(pRepo), tstrerror(terrno)); - goto _err; - } - - tsdbEndFSTxn(pRepo); - tsem_post(&(pRepo->readyToCommit)); - tsdbDestroySyncH(&synch); - - // Reload file change - tsdbReload(pRepo, synch.mfChanged); - - return 0; - -_err: - tsdbEndFSTxnWithError(REPO_FS(pRepo)); - tsem_post(&(pRepo->readyToCommit)); - tsdbDestroySyncH(&synch); - return -1; -} - -static void tsdbInitSyncH(SSyncH *pSyncH, STsdbRepo *pRepo, SOCKET socketFd) { - pSyncH->pRepo = pRepo; - pSyncH->socketFd = socketFd; - tsdbGetRtnSnap(pRepo, &(pSyncH->rtn)); -} - -static void tsdbDestroySyncH(SSyncH *pSyncH) { taosTZfree(pSyncH->pBuf); } - -static int32_t tsdbSyncSendMeta(SSyncH *pSynch) { - STsdbRepo *pRepo = pSynch->pRepo; - bool toSendMeta = false; - SMFile mf; - - // Send meta info to remote - tsdbInfo("vgId:%d, metainfo will be sent", REPO_ID(pRepo)); - if (tsdbSendMetaInfo(pSynch) < 0) { - tsdbError("vgId:%d, failed to send metainfo since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - if (pRepo->fs->cstatus->pmf == NULL) { - // No meta file, not need to wait to retrieve meta file - tsdbInfo("vgId:%d, metafile not exist, no need to send", REPO_ID(pRepo)); - return 0; - } - - if (tsdbRecvDecision(pSynch, &toSendMeta) < 0) { - tsdbError("vgId:%d, failed to recv decision while send meta since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - if (toSendMeta) { - tsdbInitMFileEx(&mf, pRepo->fs->cstatus->pmf); - if (tsdbOpenMFile(&mf, O_RDONLY) < 0) { - tsdbError("vgId:%d, failed to open file while send metafile since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - int64_t writeLen = mf.info.size; - tsdbInfo("vgId:%d, metafile:%s will be sent, size:%" PRId64, REPO_ID(pRepo), mf.f.aname, writeLen); - - int64_t ret = taosSendFile(pSynch->socketFd, TSDB_FILE_FD(&mf), 0, writeLen); - if (ret != writeLen) { - terrno = TAOS_SYSTEM_ERROR(errno); - tsdbError("vgId:%d, failed to send metafile since %s, ret:%" PRId64 " writeLen:%" PRId64, REPO_ID(pRepo), - tstrerror(terrno), ret, writeLen); - tsdbCloseMFile(&mf); - return -1; - } - - tsdbCloseMFile(&mf); - tsdbInfo("vgId:%d, metafile is sent", REPO_ID(pRepo)); - } else { - tsdbInfo("vgId:%d, metafile is same, no need to send", REPO_ID(pRepo)); - } - - return 0; -} - -static int32_t tsdbSyncRecvMeta(SSyncH *pSynch) { - STsdbRepo *pRepo = pSynch->pRepo; - SMFile * pLMFile = pRepo->fs->cstatus->pmf; - - // Recv meta info from remote - if (tsdbRecvMetaInfo(pSynch) < 0) { - tsdbError("vgId:%d, failed to recv metainfo since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - // No meta file, do nothing (rm local meta file) - if (pSynch->pmf == NULL) { - if (pLMFile == NULL) { - pSynch->mfChanged = false; - } else { - pSynch->mfChanged = true; - } - tsdbInfo("vgId:%d, metafile not exist in remote, no need to recv", REPO_ID(pRepo)); - return 0; - } - - if (pLMFile == NULL || pSynch->pmf->info.size != pLMFile->info.size || - pSynch->pmf->info.magic != pLMFile->info.magic || TSDB_FILE_IS_BAD(pLMFile)) { - // Local has no meta file or has a different meta file, need to copy from remote - pSynch->mfChanged = true; - - if (tsdbSendDecision(pSynch, true) < 0) { - tsdbError("vgId:%d, failed to send decision while recv metafile since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - tsdbInfo("vgId:%d, metafile will be received", REPO_ID(pRepo)); - - // Recv from remote - SMFile mf; - SDiskID did = {.level = TFS_PRIMARY_LEVEL, .id = TFS_PRIMARY_ID}; - tsdbInitMFile(&mf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo))); - if (tsdbCreateMFile(&mf, false) < 0) { - tsdbError("vgId:%d, failed to create file while recv metafile since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - tsdbInfo("vgId:%d, metafile:%s is created", REPO_ID(pRepo), mf.f.aname); - - int64_t readLen = pSynch->pmf->info.size; - int64_t ret = taosCopyFds(pSynch->socketFd, TSDB_FILE_FD(&mf), readLen); - if (ret != readLen) { - terrno = TAOS_SYSTEM_ERROR(errno); - tsdbError("vgId:%d, failed to recv metafile since %s, ret:%" PRId64 " readLen:%" PRId64, REPO_ID(pRepo), - tstrerror(terrno), ret, readLen); - tsdbCloseMFile(&mf); - tsdbRemoveMFile(&mf); - return -1; - } - - tsdbInfo("vgId:%d, metafile is received, size:%" PRId64, REPO_ID(pRepo), readLen); - - mf.info = pSynch->pmf->info; - tsdbCloseMFile(&mf); - tsdbUpdateMFile(REPO_FS(pRepo), &mf); - } else { - pSynch->mfChanged = false; - tsdbInfo("vgId:%d, metafile is same, no need to recv", REPO_ID(pRepo)); - if (tsdbSendDecision(pSynch, false) < 0) { - tsdbError("vgId:%d, failed to send decision while recv metafile since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - tsdbUpdateMFile(REPO_FS(pRepo), pLMFile); - } - - return 0; -} - -static int32_t tsdbSendMetaInfo(SSyncH *pSynch) { - STsdbRepo *pRepo = pSynch->pRepo; - uint32_t tlen = 0; - SMFile * pMFile = pRepo->fs->cstatus->pmf; - - if (pMFile) { - tlen = tlen + tsdbEncodeSMFileEx(NULL, pMFile) + sizeof(TSCKSUM); - } - - if (tsdbMakeRoom((void **)(&SYNC_BUFFER(pSynch)), tlen + sizeof(tlen)) < 0) { - tsdbError("vgId:%d, failed to makeroom while send metainfo since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - void *ptr = SYNC_BUFFER(pSynch); - taosEncodeFixedU32(&ptr, tlen); - void *tptr = ptr; - if (pMFile) { - tsdbEncodeSMFileEx(&ptr, pMFile); - taosCalcChecksumAppend(0, (uint8_t *)tptr, tlen); - } - - int32_t writeLen = tlen + sizeof(uint32_t); - int32_t ret = taosWriteMsg(pSynch->socketFd, SYNC_BUFFER(pSynch), writeLen); - if (ret != writeLen) { - terrno = TAOS_SYSTEM_ERROR(errno); - tsdbError("vgId:%d, failed to send metainfo since %s, ret:%d writeLen:%d", REPO_ID(pRepo), tstrerror(terrno), ret, - writeLen); - return -1; - } - - tsdbInfo("vgId:%d, metainfo is sent, tlen:%d, writeLen:%d", REPO_ID(pRepo), tlen, writeLen); - return 0; -} - -static int32_t tsdbRecvMetaInfo(SSyncH *pSynch) { - STsdbRepo *pRepo = pSynch->pRepo; - uint32_t tlen = 0; - char buf[64] = {0}; - - int32_t readLen = sizeof(uint32_t); - int32_t ret = taosReadMsg(pSynch->socketFd, buf, readLen); - if (ret != readLen) { - terrno = TAOS_SYSTEM_ERROR(errno); - tsdbError("vgId:%d, failed to recv metalen, ret:%d readLen:%d", REPO_ID(pRepo), ret, readLen); - return -1; - } - - taosDecodeFixedU32(buf, &tlen); - - tsdbInfo("vgId:%d, metalen is received, readLen:%d, tlen:%d", REPO_ID(pRepo), readLen, tlen); - if (tlen == 0) { - pSynch->pmf = NULL; - return 0; - } - - if (tsdbMakeRoom((void **)(&SYNC_BUFFER(pSynch)), tlen) < 0) { - tsdbError("vgId:%d, failed to makeroom while recv metainfo since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - ret = taosReadMsg(pSynch->socketFd, SYNC_BUFFER(pSynch), tlen); - if (ret != tlen) { - terrno = TAOS_SYSTEM_ERROR(errno); - tsdbError("vgId:%d, failed to recv metainfo, ret:%d tlen:%d", REPO_ID(pRepo), ret, tlen); - return -1; - } - - tsdbInfo("vgId:%d, metainfo is received, tlen:%d", REPO_ID(pRepo), tlen); - if (!taosCheckChecksumWhole((uint8_t *)SYNC_BUFFER(pSynch), tlen)) { - terrno = TSDB_CODE_TDB_MESSED_MSG; - tsdbError("vgId:%d, failed to checksum while recv metainfo since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - pSynch->pmf = &(pSynch->mf); - tsdbDecodeSMFileEx(SYNC_BUFFER(pSynch), pSynch->pmf); - - return 0; -} - -static int32_t tsdbSendDecision(SSyncH *pSynch, bool toSend) { - STsdbRepo *pRepo = pSynch->pRepo; - uint8_t decision = toSend; - - int32_t writeLen = sizeof(uint8_t); - int32_t ret = taosWriteMsg(pSynch->socketFd, (void *)(&decision), writeLen); - if (ret != writeLen) { - terrno = TAOS_SYSTEM_ERROR(errno); - tsdbError("vgId:%d, failed to send decison, ret:%d writeLen:%d", REPO_ID(pRepo), ret, writeLen); - return -1; - } - - return 0; -} - -static int32_t tsdbRecvDecision(SSyncH *pSynch, bool *toSend) { - STsdbRepo *pRepo = pSynch->pRepo; - uint8_t decision = 0; - - int32_t readLen = sizeof(uint8_t); - int32_t ret = taosReadMsg(pSynch->socketFd, (void *)(&decision), readLen); - if (ret != readLen) { - terrno = TAOS_SYSTEM_ERROR(errno); - tsdbError("vgId:%d, failed to recv decison, ret:%d readLen:%d", REPO_ID(pRepo), ret, readLen); - return -1; - } - - *toSend = decision; - return 0; -} - -static int32_t tsdbSyncSendDFileSetArray(SSyncH *pSynch) { - STsdbRepo *pRepo = pSynch->pRepo; - STsdbFS * pfs = REPO_FS(pRepo); - SFSIter fsiter; - SDFileSet *pSet; - - tsdbFSIterInit(&fsiter, pfs, TSDB_FS_ITER_FORWARD); - - do { - pSet = tsdbFSIterNext(&fsiter); - if (tsdbSyncSendDFileSet(pSynch, pSet) < 0) { - tsdbError("vgId:%d, failed to send fileset:%d since %s", REPO_ID(pRepo), pSet ? pSet->fid : -1, - tstrerror(terrno)); - return -1; - } - - // No more file set to send, jut break - if (pSet == NULL) { - tsdbInfo("vgId:%d, no filesets any more", REPO_ID(pRepo)); - break; - } - } while (true); - - return 0; -} - -static int32_t tsdbSyncRecvDFileSetArray(SSyncH *pSynch) { - STsdbRepo *pRepo = pSynch->pRepo; - STsdbFS * pfs = REPO_FS(pRepo); - SFSIter fsiter; - SDFileSet *pLSet; // Local file set - - tsdbFSIterInit(&fsiter, pfs, TSDB_FS_ITER_FORWARD); - - pLSet = tsdbFSIterNext(&fsiter); - if (tsdbRecvDFileSetInfo(pSynch) < 0) { - tsdbError("vgId:%d, failed to recv fileset since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - while (true) { - if (pLSet == NULL && pSynch->pdf == NULL) { - tsdbInfo("vgId:%d, all filesets is disposed", REPO_ID(pRepo)); - break; - } else { - tsdbInfo("vgId:%d, fileset local:%d remote:%d, will be disposed", REPO_ID(pRepo), pLSet != NULL ? pLSet->fid : -1, - pSynch->pdf != NULL ? pSynch->pdf->fid : -1); - } - - if (pLSet && (pSynch->pdf == NULL || pLSet->fid < pSynch->pdf->fid)) { - // remote not has pLSet->fid set, just remove local (do nothing to remote the fset) - tsdbInfo("vgId:%d, fileset:%d smaller than remote:%d, remove it", REPO_ID(pRepo), pLSet->fid, - pSynch->pdf != NULL ? pSynch->pdf->fid : -1); - pLSet = tsdbFSIterNext(&fsiter); - } else { - if (pLSet && pSynch->pdf && pLSet->fid == pSynch->pdf->fid && tsdbIsTowFSetSame(pLSet, pSynch->pdf) && - tsdbFSetIsOk(pLSet)) { - // Just keep local files and notify remote not to send - tsdbInfo("vgId:%d, fileset:%d is same and no need to recv", REPO_ID(pRepo), pLSet->fid); - - if (tsdbUpdateDFileSet(pfs, pLSet) < 0) { - tsdbError("vgId:%d, failed to update fileset since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - if (tsdbSendDecision(pSynch, false) < 0) { - tsdbError("vgId:%d, failed to send decision since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - } else { - // Need to copy from remote - int fidLevel = tsdbGetFidLevel(pSynch->pdf->fid, &(pSynch->rtn)); - if (fidLevel < 0) { // expired fileset - tsdbInfo("vgId:%d, fileset:%d will be skipped as expired", REPO_ID(pRepo), pSynch->pdf->fid); - if (tsdbSendDecision(pSynch, false) < 0) { - tsdbError("vgId:%d, failed to send decision since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - // Move forward - if (tsdbRecvDFileSetInfo(pSynch) < 0) { - tsdbError("vgId:%d, failed to recv fileset since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - if (pLSet) { - pLSet = tsdbFSIterNext(&fsiter); - } - // Next loop - continue; - } else { - tsdbInfo("vgId:%d, fileset:%d will be received", REPO_ID(pRepo), pSynch->pdf->fid); - // Notify remote to send there file here - if (tsdbSendDecision(pSynch, true) < 0) { - tsdbError("vgId:%d, failed to send decision since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - } - - // Create local files and copy from remote - SDiskID did; - SDFileSet fset; - - tfsAllocDisk(fidLevel, &(did.level), &(did.id)); - if (did.level == TFS_UNDECIDED_LEVEL) { - terrno = TSDB_CODE_TDB_NO_AVAIL_DISK; - tsdbError("vgId:%d, failed allc disk since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - tsdbInitDFileSet(&fset, did, REPO_ID(pRepo), pSynch->pdf->fid, FS_TXN_VERSION(pfs)); - - // Create new FSET - if (tsdbCreateDFileSet(&fset, false) < 0) { - tsdbError("vgId:%d, failed to create fileset since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { - SDFile *pDFile = TSDB_DFILE_IN_SET(&fset, ftype); // local file - SDFile *pRDFile = TSDB_DFILE_IN_SET(pSynch->pdf, ftype); // remote file - - tsdbInfo("vgId:%d, file:%s will be received, osize:%" PRIu64 " rsize:%" PRIu64, REPO_ID(pRepo), - pDFile->f.aname, pDFile->info.size, pRDFile->info.size); - - int64_t writeLen = pRDFile->info.size; - int64_t ret = taosCopyFds(pSynch->socketFd, pDFile->fd, writeLen); - if (ret != writeLen) { - terrno = TAOS_SYSTEM_ERROR(errno); - tsdbError("vgId:%d, failed to recv file:%s since %s, ret:%" PRId64 " writeLen:%" PRId64, REPO_ID(pRepo), - pDFile->f.aname, tstrerror(terrno), ret, writeLen); - tsdbCloseDFileSet(&fset); - tsdbRemoveDFileSet(&fset); - return -1; - } - - // Update new file info - pDFile->info = pRDFile->info; - tsdbInfo("vgId:%d, file:%s is received, size:%" PRId64, REPO_ID(pRepo), pDFile->f.aname, writeLen); - } - - tsdbCloseDFileSet(&fset); - if (tsdbUpdateDFileSet(pfs, &fset) < 0) { - tsdbInfo("vgId:%d, fileset:%d failed to update since %s", REPO_ID(pRepo), fset.fid, tstrerror(terrno)); - return -1; - } - - tsdbInfo("vgId:%d, fileset:%d is received", REPO_ID(pRepo), pSynch->pdf->fid); - } - - // Move forward - if (tsdbRecvDFileSetInfo(pSynch) < 0) { - tsdbError("vgId:%d, failed to recv fileset since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - if (pLSet) { - pLSet = tsdbFSIterNext(&fsiter); - } - } - -#if 0 - if (pLSet == NULL) { - // Copy from remote >>>>>>>>>>> - } else { - if (pSynch->pdf == NULL) { - // Remove local file, just ignore ++++++++++++++ - pLSet = tsdbFSIterNext(&fsiter); - } else { - if (pLSet->fid < pSynch->pdf->fid) { - // Remove local file, just ignore ++++++++++++ - pLSet = tsdbFSIterNext(&fsiter); - } else if (pLSet->fid > pSynch->pdf->fid){ - // Copy from remote >>>>>>>>>>>>>> - if (tsdbRecvDFileSetInfo(pSynch) < 0) { - // TODO - return -1; - } - } else { - if (true/*TODO: is same fset*/) { - // No need to copy --------------------- - } else { - // copy from remote >>>>>>>>>>>>>. - } - } - } - } -#endif - } - - return 0; -} - -static bool tsdbIsTowFSetSame(SDFileSet *pSet1, SDFileSet *pSet2) { - for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { - SDFile *pDFile1 = TSDB_DFILE_IN_SET(pSet1, ftype); - SDFile *pDFile2 = TSDB_DFILE_IN_SET(pSet2, ftype); - - if (pDFile1->info.size != pDFile2->info.size || pDFile1->info.magic != pDFile2->info.magic) { - return false; - } - } - - return true; -} - -static int32_t tsdbSyncSendDFileSet(SSyncH *pSynch, SDFileSet *pSet) { - STsdbRepo *pRepo = pSynch->pRepo; - bool toSend = false; - - // skip expired fileset - if (pSet && tsdbGetFidLevel(pSet->fid, &(pSynch->rtn)) < 0) { - tsdbInfo("vgId:%d, don't sync send since fileset:%d smaller than minFid:%d", REPO_ID(pRepo), pSet->fid, - pSynch->rtn.minFid); - return 0; - } - - if (tsdbSendDFileSetInfo(pSynch, pSet) < 0) { - tsdbError("vgId:%d, failed to send fileset:%d info since %s", REPO_ID(pRepo), pSet ? pSet->fid : -1, tstrerror(terrno)); - return -1; - } - - // No file any more, no need to send file, just return - if (pSet == NULL) { - return 0; - } - - if (tsdbRecvDecision(pSynch, &toSend) < 0) { - tsdbError("vgId:%d, failed to recv decision while send fileset:%d since %s", REPO_ID(pRepo), pSet->fid, - tstrerror(terrno)); - return -1; - } - - if (toSend) { - tsdbInfo("vgId:%d, fileset:%d will be sent", REPO_ID(pRepo), pSet->fid); - - for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) { - SDFile df = *TSDB_DFILE_IN_SET(pSet, ftype); - - if (tsdbOpenDFile(&df, O_RDONLY) < 0) { - tsdbError("vgId:%d, failed to file:%s since %s", REPO_ID(pRepo), df.f.aname, tstrerror(terrno)); - return -1; - } - - int64_t writeLen = df.info.size; - tsdbInfo("vgId:%d, file:%s will be sent, size:%" PRId64, REPO_ID(pRepo), df.f.aname, writeLen); - - int64_t ret = taosSendFile(pSynch->socketFd, TSDB_FILE_FD(&df), 0, writeLen); - if (ret != writeLen) { - terrno = TAOS_SYSTEM_ERROR(errno); - tsdbError("vgId:%d, failed to send file:%s since %s, ret:%" PRId64 " writeLen:%" PRId64, REPO_ID(pRepo), - df.f.aname, tstrerror(terrno), ret, writeLen); - tsdbCloseDFile(&df); - return -1; - } - - tsdbInfo("vgId:%d, file:%s is sent", REPO_ID(pRepo), df.f.aname); - tsdbCloseDFile(&df); - } - - tsdbInfo("vgId:%d, fileset:%d is sent", REPO_ID(pRepo), pSet->fid); - } else { - tsdbInfo("vgId:%d, fileset:%d is same, no need to send", REPO_ID(pRepo), pSet->fid); - } - - return 0; -} - -static int32_t tsdbSendDFileSetInfo(SSyncH *pSynch, SDFileSet *pSet) { - STsdbRepo *pRepo = pSynch->pRepo; - uint32_t tlen = 0; - - if (pSet) { - tlen = tsdbEncodeDFileSetEx(NULL, pSet) + sizeof(TSCKSUM); - } - - if (tsdbMakeRoom((void **)(&SYNC_BUFFER(pSynch)), tlen + sizeof(tlen)) < 0) { - tsdbError("vgId:%d, failed to makeroom while send fileinfo since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - void *ptr = SYNC_BUFFER(pSynch); - taosEncodeFixedU32(&ptr, tlen); - void *tptr = ptr; - if (pSet) { - tsdbEncodeDFileSetEx(&ptr, pSet); - taosCalcChecksumAppend(0, (uint8_t *)tptr, tlen); - } - - int32_t writeLen = tlen + sizeof(uint32_t); - int32_t ret = taosWriteMsg(pSynch->socketFd, SYNC_BUFFER(pSynch), writeLen); - if (ret != writeLen) { - terrno = TAOS_SYSTEM_ERROR(errno); - tsdbError("vgId:%d, failed to send fileinfo, ret:%d writeLen:%d", REPO_ID(pRepo), ret, writeLen); - return -1; - } - - return 0; -} - -static int32_t tsdbRecvDFileSetInfo(SSyncH *pSynch) { - STsdbRepo *pRepo = pSynch->pRepo; - uint32_t tlen; - char buf[64] = {0}; - - int32_t readLen = sizeof(uint32_t); - int32_t ret = taosReadMsg(pSynch->socketFd, buf, readLen); - if (ret != readLen) { - terrno = TAOS_SYSTEM_ERROR(errno); - return -1; - } - - taosDecodeFixedU32(buf, &tlen); - - tsdbInfo("vgId:%d, fileinfo len:%d is received", REPO_ID(pRepo), tlen); - if (tlen == 0) { - pSynch->pdf = NULL; - return 0; - } - - if (tsdbMakeRoom((void **)(&SYNC_BUFFER(pSynch)), tlen) < 0) { - tsdbError("vgId:%d, failed to makeroom while recv fileinfo since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - ret = taosReadMsg(pSynch->socketFd, SYNC_BUFFER(pSynch), tlen); - if (ret != tlen) { - terrno = TAOS_SYSTEM_ERROR(errno); - tsdbError("vgId:%d, failed to recv fileinfo, ret:%d readLen:%d", REPO_ID(pRepo), ret, tlen); - return -1; - } - - if (!taosCheckChecksumWhole((uint8_t *)SYNC_BUFFER(pSynch), tlen)) { - terrno = TSDB_CODE_TDB_MESSED_MSG; - tsdbError("vgId:%d, failed to checksum while recv fileinfo since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - pSynch->pdf = &(pSynch->df); - tsdbDecodeDFileSetEx(SYNC_BUFFER(pSynch), pSynch->pdf); - - return 0; -} - -static int tsdbReload(STsdbRepo *pRepo, bool isMfChanged) { - // TODO: may need to stop and restart stream - // if (isMfChanged) { - tsdbCloseMeta(pRepo); - tsdbFreeMeta(pRepo->tsdbMeta); - pRepo->tsdbMeta = tsdbNewMeta(REPO_CFG(pRepo)); - tsdbOpenMeta(pRepo); - tsdbLoadMetaCache(pRepo, true); - // } - - tsdbUnRefMemTable(pRepo, pRepo->mem); - tsdbUnRefMemTable(pRepo, pRepo->imem); - pRepo->mem = NULL; - pRepo->imem = NULL; - - if (tsdbRestoreInfo(pRepo) < 0) { - tsdbError("vgId:%d failed to restore info from file since %s", REPO_ID(pRepo), tstrerror(terrno)); - return -1; - } - - return 0; -} \ No newline at end of file diff --git a/source/dnode/vnode/tsdb/test/tsdbTests.cpp b/source/dnode/vnode/tsdb/test/tsdbTests.cpp deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/source/libs/executor/CMakeLists.txt b/source/libs/executor/CMakeLists.txt index 04b5fab4bf..9b53cc1fbb 100644 --- a/source/libs/executor/CMakeLists.txt +++ b/source/libs/executor/CMakeLists.txt @@ -13,7 +13,7 @@ add_library(executor STATIC ${EXECUTOR_SRC}) # INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_SOURCE_DIR}/include/libs/executor" # ) target_link_libraries(executor - PRIVATE os util common function parser planner qcom tsdb + PRIVATE os util common function parser planner qcom vnode ) target_include_directories( From 0db6b12a47e16d16d1768cd9de5e24e1d0d5869a Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 13 Jan 2022 01:40:33 +0000 Subject: [PATCH 48/57] more tkv --- .../tkv/src/inc/tkvBtree.h} | 19 ++++++++++++++++++- source/libs/tkv/src/inc/tkvDB.h | 16 +++++++++++++++- .../tkv/src/inc/tkvHash.h} | 13 ++++++++----- 3 files changed, 41 insertions(+), 7 deletions(-) rename source/{dnode/vnode/src/vnd/vnodeRead.c => libs/tkv/src/inc/tkvBtree.h} (72%) rename source/{dnode/vnode/src/inc/vnodeRead.h => libs/tkv/src/inc/tkvHash.h} (81%) diff --git a/source/dnode/vnode/src/vnd/vnodeRead.c b/source/libs/tkv/src/inc/tkvBtree.h similarity index 72% rename from source/dnode/vnode/src/vnd/vnodeRead.c rename to source/libs/tkv/src/inc/tkvBtree.h index 5e9f89ccd5..6360e62c79 100644 --- a/source/dnode/vnode/src/vnd/vnodeRead.c +++ b/source/libs/tkv/src/inc/tkvBtree.h @@ -13,4 +13,21 @@ * along with this program. If not, see . */ -#include "vnodeDef.h" \ No newline at end of file +#ifndef _TD_TKV_BTREE_H_ +#define _TD_TKV_BTREE_H_ + +#include "tkvDef.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct { + pgid_t root; // root page number +} STkvBtree; + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_TKV_BTREE_H_*/ \ No newline at end of file diff --git a/source/libs/tkv/src/inc/tkvDB.h b/source/libs/tkv/src/inc/tkvDB.h index 1a45702540..fc8219db3f 100644 --- a/source/libs/tkv/src/inc/tkvDB.h +++ b/source/libs/tkv/src/inc/tkvDB.h @@ -16,12 +16,26 @@ #ifndef _TD_TKV_DB_H_ #define _TD_TKV_DB_H_ +#include "tkvBtree.h" + #ifdef __cplusplus extern "C" { #endif +typedef enum { + TDB_BTREE = 0, + TDB_HASH, + TDB_HEAP, +} tdb_db_t; + struct TDB { - // TODO + pgsize_t pageSize; + + tdb_db_t type; // DB type + + union { + STkvBtree btree; + } dbimpl; }; #ifdef __cplusplus diff --git a/source/dnode/vnode/src/inc/vnodeRead.h b/source/libs/tkv/src/inc/tkvHash.h similarity index 81% rename from source/dnode/vnode/src/inc/vnodeRead.h rename to source/libs/tkv/src/inc/tkvHash.h index 5ce84b2ebf..06343b7ac9 100644 --- a/source/dnode/vnode/src/inc/vnodeRead.h +++ b/source/libs/tkv/src/inc/tkvHash.h @@ -13,18 +13,21 @@ * along with this program. If not, see . */ -#ifndef _TD_VNODE_READ_H_ -#define _TD_VNODE_READ_H_ +#ifndef _TD_TKV_HAHS_H_ +#define _TD_TKV_HAHS_H_ + +#include "tkvDef.h" #ifdef __cplusplus extern "C" { #endif -#include "vnodeInt.h" -void vnodeProcessReadMsg(SVnode *pVnode, SVnodeMsg *pMsg); +typedef struct STkvHash { + // TODO +} STkvHash; #ifdef __cplusplus } #endif -#endif /*_TD_VNODE_READ_H_*/ +#endif /*_TD_TKV_HAHS_H_*/ \ No newline at end of file From c31134216c9da4c07898b8bda25e9ff14419b1d4 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 13 Jan 2022 01:50:42 +0000 Subject: [PATCH 49/57] more --- source/libs/tkv/src/inc/tkvDB.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/source/libs/tkv/src/inc/tkvDB.h b/source/libs/tkv/src/inc/tkvDB.h index fc8219db3f..6fe9dabf05 100644 --- a/source/libs/tkv/src/inc/tkvDB.h +++ b/source/libs/tkv/src/inc/tkvDB.h @@ -17,6 +17,7 @@ #define _TD_TKV_DB_H_ #include "tkvBtree.h" +#include "tkvHash.h" #ifdef __cplusplus extern "C" { @@ -30,11 +31,10 @@ typedef enum { struct TDB { pgsize_t pageSize; - - tdb_db_t type; // DB type - + tdb_db_t type; union { STkvBtree btree; + STkvhash hash; } dbimpl; }; From c8c09c8a8cbee993c3c944a62f47fa96d64067aa Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 13 Jan 2022 02:12:24 +0000 Subject: [PATCH 50/57] refact --- source/libs/CMakeLists.txt | 2 +- source/libs/{tkv => tdb}/CMakeLists.txt | 8 ++++---- source/libs/{tkv/inc/tkv.h => tdb/inc/tdb.h} | 8 ++++---- .../src/inc/tkvBtree.h => tdb/src/inc/tdbBtree.h} | 8 ++++---- .../inc/tkvBufPool.h => tdb/src/inc/tdbBufPool.h} | 12 ++++++------ .../{tkv/src/inc/tkvDB.h => tdb/src/inc/tdbDB.h} | 10 +++++----- .../{tkv/src/inc/tkvDef.h => tdb/src/inc/tdbDef.h} | 6 +++--- .../inc/tkvDiskMgr.h => tdb/src/inc/tdbDiskMgr.h} | 2 +- .../{tkv/src/inc/tkvEnv.h => tdb/src/inc/tdbEnv.h} | 6 +++--- .../src/inc/tkvHash.h => tdb/src/inc/tdbHash.h} | 6 +++--- .../src/inc/tkvPage.h => tdb/src/inc/tdbPage.h} | 14 ++++---------- .../{tkv/src/tkvBufPool.c => tdb/src/tdbBufPool.c} | 2 +- .../{tkv/src/tDiskMgr.c => tdb/src/tdbDiskMgr.c} | 2 +- source/libs/{tkv => tdb}/test/tDiskMgrTest.cpp | 0 source/libs/{tkv => tdb}/test/tkvTests.cpp | 0 15 files changed, 40 insertions(+), 46 deletions(-) rename source/libs/{tkv => tdb}/CMakeLists.txt (79%) rename source/libs/{tkv/inc/tkv.h => tdb/inc/tdb.h} (91%) rename source/libs/{tkv/src/inc/tkvBtree.h => tdb/src/inc/tdbBtree.h} (89%) rename source/libs/{tkv/src/inc/tkvBufPool.h => tdb/src/inc/tdbBufPool.h} (82%) rename source/libs/{tkv/src/inc/tkvDB.h => tdb/src/inc/tdbDB.h} (88%) rename source/libs/{tkv/src/inc/tkvDef.h => tdb/src/inc/tdbDef.h} (93%) rename source/libs/{tkv/src/inc/tkvDiskMgr.h => tdb/src/inc/tdbDiskMgr.h} (98%) rename source/libs/{tkv/src/inc/tkvEnv.h => tdb/src/inc/tdbEnv.h} (91%) rename source/libs/{tkv/src/inc/tkvHash.h => tdb/src/inc/tdbHash.h} (93%) rename source/libs/{tkv/src/inc/tkvPage.h => tdb/src/inc/tdbPage.h} (77%) rename source/libs/{tkv/src/tkvBufPool.c => tdb/src/tdbBufPool.c} (98%) rename source/libs/{tkv/src/tDiskMgr.c => tdb/src/tdbDiskMgr.c} (98%) rename source/libs/{tkv => tdb}/test/tDiskMgrTest.cpp (100%) rename source/libs/{tkv => tdb}/test/tkvTests.cpp (100%) diff --git a/source/libs/CMakeLists.txt b/source/libs/CMakeLists.txt index 1dc16c74f7..1d23f333b2 100644 --- a/source/libs/CMakeLists.txt +++ b/source/libs/CMakeLists.txt @@ -1,6 +1,6 @@ add_subdirectory(transport) add_subdirectory(sync) -add_subdirectory(tkv) +add_subdirectory(tdb) add_subdirectory(index) add_subdirectory(wal) add_subdirectory(parser) diff --git a/source/libs/tkv/CMakeLists.txt b/source/libs/tdb/CMakeLists.txt similarity index 79% rename from source/libs/tkv/CMakeLists.txt rename to source/libs/tdb/CMakeLists.txt index fec3f37cd5..1832ea0b5d 100644 --- a/source/libs/tkv/CMakeLists.txt +++ b/source/libs/tdb/CMakeLists.txt @@ -1,17 +1,17 @@ -aux_source_directory(src TKV_SRC) -add_library(tkv STATIC ${TKV_SRC}) +aux_source_directory(src TDB_SRC) +add_library(tdb STATIC ${TDB_SRC}) # target_include_directories( # tkv # PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/tkv" # PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" # ) target_include_directories( - tkv + tdb PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/inc" PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/src/inc" ) target_link_libraries( - tkv + tdb PUBLIC os PUBLIC util ) \ No newline at end of file diff --git a/source/libs/tkv/inc/tkv.h b/source/libs/tdb/inc/tdb.h similarity index 91% rename from source/libs/tkv/inc/tkv.h rename to source/libs/tdb/inc/tdb.h index 00534d2827..2f47f545b1 100644 --- a/source/libs/tkv/inc/tkv.h +++ b/source/libs/tdb/inc/tdb.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef _TD_TKV_H_ -#define _TD_TKV_H_ +#ifndef _TD_TDB_H_ +#define _TD_TDB_H_ #include "os.h" @@ -28,7 +28,7 @@ typedef struct TDB_ENV TDB_ENV; // SKey typedef struct { - void * bdata; + void* bdata; uint32_t size; } TDB_KEY, TDB_VALUE; @@ -36,4 +36,4 @@ typedef struct { } #endif -#endif /*_TD_TKV_H_*/ \ No newline at end of file +#endif /*_TD_TDB_H_*/ \ No newline at end of file diff --git a/source/libs/tkv/src/inc/tkvBtree.h b/source/libs/tdb/src/inc/tdbBtree.h similarity index 89% rename from source/libs/tkv/src/inc/tkvBtree.h rename to source/libs/tdb/src/inc/tdbBtree.h index 6360e62c79..c68f94bb48 100644 --- a/source/libs/tkv/src/inc/tkvBtree.h +++ b/source/libs/tdb/src/inc/tdbBtree.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef _TD_TKV_BTREE_H_ -#define _TD_TKV_BTREE_H_ +#ifndef _TD_TDB_BTREE_H_ +#define _TD_TDB_BTREE_H_ #include "tkvDef.h" @@ -24,10 +24,10 @@ extern "C" { typedef struct { pgid_t root; // root page number -} STkvBtree; +} TDB_BTREE; #ifdef __cplusplus } #endif -#endif /*_TD_TKV_BTREE_H_*/ \ No newline at end of file +#endif /*_TD_TDB_BTREE_H_*/ \ No newline at end of file diff --git a/source/libs/tkv/src/inc/tkvBufPool.h b/source/libs/tdb/src/inc/tdbBufPool.h similarity index 82% rename from source/libs/tkv/src/inc/tkvBufPool.h rename to source/libs/tdb/src/inc/tdbBufPool.h index ec8d177a9a..7fda5bf2e6 100644 --- a/source/libs/tkv/src/inc/tkvBufPool.h +++ b/source/libs/tdb/src/inc/tdbBufPool.h @@ -13,10 +13,10 @@ * along with this program. If not, see . */ -#ifndef _TD_TKV_BUF_POOL_H_ -#define _TD_TKV_BUF_POOL_H_ +#ifndef _TD_TDB_BUF_POOL_H_ +#define _TD_TDB_BUF_POOL_H_ -#include "tkvPage.h" +#include "tdbPage.h" #ifdef __cplusplus extern "C" { @@ -26,9 +26,9 @@ typedef struct STkvBufPool STkvBufPool; int tbpOpen(STkvBufPool **ppTkvBufPool); int tbpClose(STkvBufPool *pTkvBufPool); -STkvPage *tbpNewPage(STkvBufPool *pTkvBufPool); +STdbPage *tbpNewPage(STkvBufPool *pTkvBufPool); int tbpDelPage(STkvBufPool *pTkvBufPool); -STkvPage *tbpFetchPage(STkvBufPool *pTkvBufPool, pgid_t pgid); +STdbPage *tbpFetchPage(STkvBufPool *pTkvBufPool, pgid_t pgid); int tbpUnpinPage(STkvBufPool *pTkvBufPool, pgid_t pgid); void tbpFlushPages(STkvBufPool *pTkvBufPool); @@ -36,4 +36,4 @@ void tbpFlushPages(STkvBufPool *pTkvBufPool); } #endif -#endif /*_TD_TKV_BUF_POOL_H_*/ \ No newline at end of file +#endif /*_TD_TDB_BUF_POOL_H_*/ \ No newline at end of file diff --git a/source/libs/tkv/src/inc/tkvDB.h b/source/libs/tdb/src/inc/tdbDB.h similarity index 88% rename from source/libs/tkv/src/inc/tkvDB.h rename to source/libs/tdb/src/inc/tdbDB.h index 6fe9dabf05..40ddb1eb31 100644 --- a/source/libs/tkv/src/inc/tkvDB.h +++ b/source/libs/tdb/src/inc/tdbDB.h @@ -13,11 +13,11 @@ * along with this program. If not, see . */ -#ifndef _TD_TKV_DB_H_ -#define _TD_TKV_DB_H_ +#ifndef _TD_TDB_DB_H_ +#define _TD_TDB_DB_H_ -#include "tkvBtree.h" -#include "tkvHash.h" +#include "tdbBtree.h" +#include "tdbHash.h" #ifdef __cplusplus extern "C" { @@ -42,4 +42,4 @@ struct TDB { } #endif -#endif /*_TD_TKV_DB_H_*/ \ No newline at end of file +#endif /*_TD_TDB_DB_H_*/ \ No newline at end of file diff --git a/source/libs/tkv/src/inc/tkvDef.h b/source/libs/tdb/src/inc/tdbDef.h similarity index 93% rename from source/libs/tkv/src/inc/tkvDef.h rename to source/libs/tdb/src/inc/tdbDef.h index cd418019be..a04b8cc402 100644 --- a/source/libs/tkv/src/inc/tkvDef.h +++ b/source/libs/tdb/src/inc/tdbDef.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef _TD_TKV_DEF_H_ -#define _TD_TKV_DEF_H_ +#ifndef _TD_TDB_DEF_H_ +#define _TD_TDB_DEF_H_ #include "os.h" @@ -39,4 +39,4 @@ typedef int32_t pgsize_t; } #endif -#endif /*_TD_TKV_DEF_H_*/ \ No newline at end of file +#endif /*_TD_TDB_DEF_H_*/ \ No newline at end of file diff --git a/source/libs/tkv/src/inc/tkvDiskMgr.h b/source/libs/tdb/src/inc/tdbDiskMgr.h similarity index 98% rename from source/libs/tkv/src/inc/tkvDiskMgr.h rename to source/libs/tdb/src/inc/tdbDiskMgr.h index 2ebe98ace2..b83a147437 100644 --- a/source/libs/tkv/src/inc/tkvDiskMgr.h +++ b/source/libs/tdb/src/inc/tdbDiskMgr.h @@ -22,7 +22,7 @@ extern "C" { #include "os.h" -#include "tkvDef.h" +#include "tdbDef.h" typedef struct STkvDiskMgr STkvDiskMgr; diff --git a/source/libs/tkv/src/inc/tkvEnv.h b/source/libs/tdb/src/inc/tdbEnv.h similarity index 91% rename from source/libs/tkv/src/inc/tkvEnv.h rename to source/libs/tdb/src/inc/tdbEnv.h index eba442e5a5..f3e4ef5888 100644 --- a/source/libs/tkv/src/inc/tkvEnv.h +++ b/source/libs/tdb/src/inc/tdbEnv.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#ifndef _TD_TKV_ENV_H_ -#define _TD_TKV_ENV_H_ +#ifndef _TD_TDB_ENV_H_ +#define _TD_TDB_ENV_H_ #ifdef __cplusplus extern "C" { @@ -28,4 +28,4 @@ struct TDB_ENV { } #endif -#endif /*_TD_TKV_ENV_H_*/ \ No newline at end of file +#endif /*_TD_TDB_ENV_H_*/ \ No newline at end of file diff --git a/source/libs/tkv/src/inc/tkvHash.h b/source/libs/tdb/src/inc/tdbHash.h similarity index 93% rename from source/libs/tkv/src/inc/tkvHash.h rename to source/libs/tdb/src/inc/tdbHash.h index 06343b7ac9..fca19035f1 100644 --- a/source/libs/tkv/src/inc/tkvHash.h +++ b/source/libs/tdb/src/inc/tdbHash.h @@ -16,15 +16,15 @@ #ifndef _TD_TKV_HAHS_H_ #define _TD_TKV_HAHS_H_ -#include "tkvDef.h" +#include "tdbDef.h" #ifdef __cplusplus extern "C" { #endif -typedef struct STkvHash { +typedef struct { // TODO -} STkvHash; +} TDB_HASH; #ifdef __cplusplus } diff --git a/source/libs/tkv/src/inc/tkvPage.h b/source/libs/tdb/src/inc/tdbPage.h similarity index 77% rename from source/libs/tkv/src/inc/tkvPage.h rename to source/libs/tdb/src/inc/tdbPage.h index d596d215cd..e7245b6c39 100644 --- a/source/libs/tkv/src/inc/tkvPage.h +++ b/source/libs/tdb/src/inc/tdbPage.h @@ -17,30 +17,24 @@ #define _TD_TKV_PAGE_H_ #include "os.h" -#include "tkvDef.h" +#include "tdbDef.h" #ifdef __cplusplus extern "C" { #endif -typedef struct STkvPage { +typedef struct { pgid_t pgid; int32_t pinCount; bool idDirty; char* pData; -} STkvPage; +} STdbPage; typedef struct { uint16_t dbver; uint16_t pgsize; uint32_t cksm; -} STkvPgHdr; - -// typedef struct { -// SPgHdr chdr; -// uint16_t used; // number of used slots -// uint16_t loffset; // the offset of the starting location of the last slot used -// } SSlottedPgHdr; +} STdbPgHdr; #ifdef __cplusplus } diff --git a/source/libs/tkv/src/tkvBufPool.c b/source/libs/tdb/src/tdbBufPool.c similarity index 98% rename from source/libs/tkv/src/tkvBufPool.c rename to source/libs/tdb/src/tdbBufPool.c index 86bfa0ba3e..6ae3499064 100644 --- a/source/libs/tkv/src/tkvBufPool.c +++ b/source/libs/tdb/src/tdbBufPool.c @@ -26,7 +26,7 @@ struct SFrameIdWrapper { }; struct STkvBufPool { - STkvPage* pages; + STdbPage* pages; STkvDiskMgr* pDiskMgr; SHashObj* pgTb; // page_id_t --> frame_id_t TD_SLIST(SFrameIdWrapper) freeList; diff --git a/source/libs/tkv/src/tDiskMgr.c b/source/libs/tdb/src/tdbDiskMgr.c similarity index 98% rename from source/libs/tkv/src/tDiskMgr.c rename to source/libs/tdb/src/tdbDiskMgr.c index fa8f6062d8..71ab5f2589 100644 --- a/source/libs/tkv/src/tDiskMgr.c +++ b/source/libs/tdb/src/tdbDiskMgr.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include "tkvDiskMgr.h" +#include "tdbDiskMgr.h" struct STkvDiskMgr { char * fname; diff --git a/source/libs/tkv/test/tDiskMgrTest.cpp b/source/libs/tdb/test/tDiskMgrTest.cpp similarity index 100% rename from source/libs/tkv/test/tDiskMgrTest.cpp rename to source/libs/tdb/test/tDiskMgrTest.cpp diff --git a/source/libs/tkv/test/tkvTests.cpp b/source/libs/tdb/test/tkvTests.cpp similarity index 100% rename from source/libs/tkv/test/tkvTests.cpp rename to source/libs/tdb/test/tkvTests.cpp From 3aae62f5dc55153df9081b01e54b9dfef3670294 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 13 Jan 2022 02:39:12 +0000 Subject: [PATCH 51/57] more --- source/libs/tdb/src/inc/tdbBufPool.h | 16 ++++++++-------- source/libs/tdb/src/tdbBufPool.c | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/source/libs/tdb/src/inc/tdbBufPool.h b/source/libs/tdb/src/inc/tdbBufPool.h index 7fda5bf2e6..5200d22faa 100644 --- a/source/libs/tdb/src/inc/tdbBufPool.h +++ b/source/libs/tdb/src/inc/tdbBufPool.h @@ -22,15 +22,15 @@ extern "C" { #endif -typedef struct STkvBufPool STkvBufPool; +typedef struct STdbBufPool STdbBufPool; -int tbpOpen(STkvBufPool **ppTkvBufPool); -int tbpClose(STkvBufPool *pTkvBufPool); -STdbPage *tbpNewPage(STkvBufPool *pTkvBufPool); -int tbpDelPage(STkvBufPool *pTkvBufPool); -STdbPage *tbpFetchPage(STkvBufPool *pTkvBufPool, pgid_t pgid); -int tbpUnpinPage(STkvBufPool *pTkvBufPool, pgid_t pgid); -void tbpFlushPages(STkvBufPool *pTkvBufPool); +int tbpOpen(STdbBufPool **ppTkvBufPool); +int tbpClose(STdbBufPool *pTkvBufPool); +STdbPage *tbpNewPage(STdbBufPool *pTkvBufPool); +int tbpDelPage(STdbBufPool *pTkvBufPool); +STdbPage *tbpFetchPage(STdbBufPool *pTkvBufPool, pgid_t pgid); +int tbpUnpinPage(STdbBufPool *pTkvBufPool, pgid_t pgid); +void tbpFlushPages(STdbBufPool *pTkvBufPool); #ifdef __cplusplus } diff --git a/source/libs/tdb/src/tdbBufPool.c b/source/libs/tdb/src/tdbBufPool.c index 6ae3499064..3eed100566 100644 --- a/source/libs/tdb/src/tdbBufPool.c +++ b/source/libs/tdb/src/tdbBufPool.c @@ -25,7 +25,7 @@ struct SFrameIdWrapper { frame_id_t id; }; -struct STkvBufPool { +struct STdbBufPool { STdbPage* pages; STkvDiskMgr* pDiskMgr; SHashObj* pgTb; // page_id_t --> frame_id_t From f1df5eaabfe8df2f9e527f39a9d14c537f46409f Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 13 Jan 2022 05:37:12 +0000 Subject: [PATCH 52/57] more refact --- source/dnode/vnode/src/inc/vnodeDef.h | 13 ++- source/dnode/vnode/src/inc/vnodeMAF.h | 32 ------ source/dnode/vnode/src/inc/vnodeRequest.h | 33 ------ source/dnode/vnode/src/inc/vnodeSync.h | 33 ------ source/dnode/vnode/src/meta/CMakeLists.txt | 46 -------- source/dnode/vnode/src/tq/CMakeLists.txt | 20 ---- source/dnode/vnode/src/tsdb/CMakeLists.txt | 34 ------ source/dnode/vnode/src/vnd/CMakeLists.txt | 24 ----- source/dnode/vnode/src/vnd/vnodeMgr.c | 2 +- source/dnode/vnode/src/vnd/vnodeRequest.c | 119 --------------------- source/dnode/vnode/src/vnd/vnodeSync.c | 14 --- 11 files changed, 11 insertions(+), 359 deletions(-) delete mode 100644 source/dnode/vnode/src/inc/vnodeMAF.h delete mode 100644 source/dnode/vnode/src/inc/vnodeRequest.h delete mode 100644 source/dnode/vnode/src/inc/vnodeSync.h delete mode 100644 source/dnode/vnode/src/meta/CMakeLists.txt delete mode 100644 source/dnode/vnode/src/tq/CMakeLists.txt delete mode 100644 source/dnode/vnode/src/tsdb/CMakeLists.txt delete mode 100644 source/dnode/vnode/src/vnd/CMakeLists.txt delete mode 100644 source/dnode/vnode/src/vnd/vnodeRequest.c delete mode 100644 source/dnode/vnode/src/vnd/vnodeSync.c diff --git a/source/dnode/vnode/src/inc/vnodeDef.h b/source/dnode/vnode/src/inc/vnodeDef.h index 1c534a8aeb..7054d5bc11 100644 --- a/source/dnode/vnode/src/inc/vnodeDef.h +++ b/source/dnode/vnode/src/inc/vnodeDef.h @@ -33,9 +33,7 @@ #include "vnodeFS.h" #include "vnodeMemAllocator.h" #include "vnodeQuery.h" -#include "vnodeRequest.h" #include "vnodeStateMgr.h" -#include "vnodeSync.h" #ifdef __cplusplus extern "C" { @@ -73,7 +71,6 @@ struct SVnode { STsdb* pTsdb; STQ* pTq; SWal* pWal; - SVnodeSync* pSync; SVnodeFS* pFs; tsem_t canCommit; SQHandle* pQuery; @@ -84,6 +81,16 @@ int vnodeScheduleTask(SVnodeTask* task); int32_t vnodePutReqToVQueryQ(SVnode *pVnode, struct SRpcMsg *pReq); +// For Log +extern int32_t vDebugFlag; + +#define vFatal(...) do { if (vDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", 255, __VA_ARGS__); }} while(0) +#define vError(...) do { if (vDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", 255, __VA_ARGS__); }} while(0) +#define vWarn(...) do { if (vDebugFlag & DEBUG_WARN) { taosPrintLog("TDB WARN ", 255, __VA_ARGS__); }} while(0) +#define vInfo(...) do { if (vDebugFlag & DEBUG_INFO) { taosPrintLog("TDB ", 255, __VA_ARGS__); }} while(0) +#define vDebug(...) do { if (vDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0) +#define vTrace(...) do { if (vDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0) + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/inc/vnodeMAF.h b/source/dnode/vnode/src/inc/vnodeMAF.h deleted file mode 100644 index 7aa405103c..0000000000 --- a/source/dnode/vnode/src/inc/vnodeMAF.h +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_VNODE_MAF_H_ -#define _TD_VNODE_MAF_H_ - -#include "vnode.h" - -#ifdef __cplusplus -extern "C" { -#endif - -int vnodeOpenMAF(SVnode *pVnode); -void vnodeCloseMAF(SVnode *pVnode); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_VNODE_MAF_H_*/ \ No newline at end of file diff --git a/source/dnode/vnode/src/inc/vnodeRequest.h b/source/dnode/vnode/src/inc/vnodeRequest.h deleted file mode 100644 index 52f4281eea..0000000000 --- a/source/dnode/vnode/src/inc/vnodeRequest.h +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_VNODE_REQUEST_H_ -#define _TD_VNODE_REQUEST_H_ - -#include "vnode.h" - -#ifdef __cplusplus -extern "C" { -#endif - -// SVDropTbReq -// int vnodeBuildDropTableReq(void **buf, const SVDropTbReq *pReq); -// void *vnodeParseDropTableReq(void *buf, SVDropTbReq *pReq); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_VNODE_REQUEST_H_*/ \ No newline at end of file diff --git a/source/dnode/vnode/src/inc/vnodeSync.h b/source/dnode/vnode/src/inc/vnodeSync.h deleted file mode 100644 index e82979551d..0000000000 --- a/source/dnode/vnode/src/inc/vnodeSync.h +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_VNODE_SYNC_H_ -#define _TD_VNODE_SYNC_H_ - -// #include "sync.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct { - /* data */ -} SVnodeSync; - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_VNODE_SYNC_H_*/ \ No newline at end of file diff --git a/source/dnode/vnode/src/meta/CMakeLists.txt b/source/dnode/vnode/src/meta/CMakeLists.txt deleted file mode 100644 index 7041811617..0000000000 --- a/source/dnode/vnode/src/meta/CMakeLists.txt +++ /dev/null @@ -1,46 +0,0 @@ -set(META_DB_IMPL_LIST "BDB" "SQLITE") -set(META_DB_IMPL "BDB" CACHE STRING "Use BDB as the default META implementation") -set_property(CACHE META_DB_IMPL PROPERTY STRINGS ${META_DB_IMPL_LIST}) - -if(META_DB_IMPL IN_LIST META_DB_IMPL_LIST) - message(STATUS "META DB Impl: ${META_DB_IMPL}==============") -else() - message(FATAL_ERROR "Invalid META DB IMPL: ${META_DB_IMPL}==============") -endif() - -aux_source_directory(src META_SRC) -if(${META_DB_IMPL} STREQUAL "BDB") - list(REMOVE_ITEM META_SRC "src/metaSQLiteImpl.c") -elseif(${META_DB_IMPL} STREQUAL "SQLITE") - list(REMOVE_ITEM META_SRC "src/metaBDBImpl.c") -endif() - -add_library(meta STATIC ${META_SRC}) -target_include_directories( - meta - PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/vnode/meta" - PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/index" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" -) -target_link_libraries( - meta - PUBLIC common - PUBLIC index -) - -if(${META_DB_IMPL} STREQUAL "BDB") - target_link_libraries( - meta - PUBLIC bdb - ) -elseif(${META_DB_IMPL} STREQUAL "SQLITE") - target_link_libraries( - meta - PUBLIC sqlite - ) -endif() - - -if(${BUILD_TEST}) - add_subdirectory(test) -endif(${BUILD_TEST}) diff --git a/source/dnode/vnode/src/tq/CMakeLists.txt b/source/dnode/vnode/src/tq/CMakeLists.txt deleted file mode 100644 index 7cb7499d64..0000000000 --- a/source/dnode/vnode/src/tq/CMakeLists.txt +++ /dev/null @@ -1,20 +0,0 @@ -aux_source_directory(src TQ_SRC) -add_library(tq ${TQ_SRC}) -target_include_directories( - tq - PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/vnode/tq" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" -) - -target_link_libraries( - tq - PUBLIC wal - PUBLIC os - PUBLIC util - PUBLIC common - PUBLIC transport -) - -if(${BUILD_TEST}) - add_subdirectory(test) -endif(${BUILD_TEST}) diff --git a/source/dnode/vnode/src/tsdb/CMakeLists.txt b/source/dnode/vnode/src/tsdb/CMakeLists.txt deleted file mode 100644 index e38ba1c466..0000000000 --- a/source/dnode/vnode/src/tsdb/CMakeLists.txt +++ /dev/null @@ -1,34 +0,0 @@ -aux_source_directory(src TSDB_SRC) -if(0) - add_library(tsdb ${TSDB_SRC}) -else(0) - add_library(tsdb STATIC "") - target_sources(tsdb - PRIVATE - "src/tsdbCommit.c" - "src/tsdbMain.c" - "src/tsdbMemTable.c" - "src/tsdbOptions.c" - "src/tsdbWrite.c" - "src/tsdbReadImpl.c" - "src/tsdbFile.c" - "src/tsdbFS.c" - "src/tsdbRead.c" - ) -endif(0) - -target_include_directories( - tsdb - PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/vnode/tsdb" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" -) - -target_link_libraries( - tsdb - PUBLIC os - PUBLIC util - PUBLIC common - PUBLIC tkv - PUBLIC tfs - PUBLIC meta -) \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/CMakeLists.txt b/source/dnode/vnode/src/vnd/CMakeLists.txt deleted file mode 100644 index 944a4276db..0000000000 --- a/source/dnode/vnode/src/vnd/CMakeLists.txt +++ /dev/null @@ -1,24 +0,0 @@ -aux_source_directory(src VNODE_SRC) -add_library(vnode STATIC ${VNODE_SRC}) -target_include_directories( - vnode - PUBLIC "${CMAKE_SOURCE_DIR}/include/dnode/vnode" - private "${CMAKE_CURRENT_SOURCE_DIR}/inc" -) -target_link_libraries( - vnode - PUBLIC os - PUBLIC transport - PUBLIC meta - PUBLIC tq - PUBLIC tsdb - PUBLIC wal - PUBLIC sync - PUBLIC cjson - PUBLIC qworker -) - -# test -if(${BUILD_TEST}) -# add_subdirectory(test) -endif(${BUILD_TEST}) diff --git a/source/dnode/vnode/src/vnd/vnodeMgr.c b/source/dnode/vnode/src/vnd/vnodeMgr.c index fdb96e52e2..ce9e487076 100644 --- a/source/dnode/vnode/src/vnd/vnodeMgr.c +++ b/source/dnode/vnode/src/vnd/vnodeMgr.c @@ -41,7 +41,7 @@ int vnodeInit(const SVnodeOpt *pOption) { for (uint16_t i = 0; i < pOption->nthreads; i++) { pthread_create(&(vnodeMgr.threads[i]), NULL, loop, NULL); - pthread_setname_np(vnodeMgr.threads[i], "VND Commit Thread"); + // pthread_setname_np(vnodeMgr.threads[i], "VND Commit Thread"); } } else { // TODO: if no commit thread is set, then another mechanism should be diff --git a/source/dnode/vnode/src/vnd/vnodeRequest.c b/source/dnode/vnode/src/vnd/vnodeRequest.c deleted file mode 100644 index 5367c9e091..0000000000 --- a/source/dnode/vnode/src/vnd/vnodeRequest.c +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "vnodeDef.h" - -#if 0 - -static int vnodeBuildCreateTableReq(void **buf, const SVCreateTableReq *pReq); -static void *vnodeParseCreateTableReq(void *buf, SVCreateTableReq *pReq); - -int vnodeBuildReq(void **buf, const SVnodeReq *pReq, tmsg_t type) { - int tsize = 0; - - tsize += taosEncodeFixedU64(buf, pReq->ver); - switch (type) { - case TDMT_VND_CREATE_STB: - tsize += vnodeBuildCreateTableReq(buf, &(pReq->ctReq)); - break; - case TDMT_VND_SUBMIT: - /* code */ - break; - default: - break; - } - /* TODO */ - return tsize; -} - -void *vnodeParseReq(void *buf, SVnodeReq *pReq, tmsg_t type) { - buf = taosDecodeFixedU64(buf, &(pReq->ver)); - - switch (type) { - case TDMT_VND_CREATE_STB: - buf = vnodeParseCreateTableReq(buf, &(pReq->ctReq)); - break; - - default: - break; - } - - // TODO - return buf; -} - -static int vnodeBuildCreateTableReq(void **buf, const SVCreateTableReq *pReq) { - int tsize = 0; - - tsize += taosEncodeString(buf, pReq->name); - tsize += taosEncodeFixedU32(buf, pReq->ttl); - tsize += taosEncodeFixedU32(buf, pReq->keep); - tsize += taosEncodeFixedU8(buf, pReq->type); - - switch (pReq->type) { - case META_SUPER_TABLE: - tsize += taosEncodeFixedU64(buf, pReq->stbCfg.suid); - tsize += tdEncodeSchema(buf, pReq->stbCfg.pSchema); - tsize += tdEncodeSchema(buf, pReq->stbCfg.pTagSchema); - break; - case META_CHILD_TABLE: - tsize += taosEncodeFixedU64(buf, pReq->ctbCfg.suid); - tsize += tdEncodeKVRow(buf, pReq->ctbCfg.pTag); - break; - case META_NORMAL_TABLE: - tsize += tdEncodeSchema(buf, pReq->ntbCfg.pSchema); - break; - default: - break; - } - - return tsize; -} - -static void *vnodeParseCreateTableReq(void *buf, SVCreateTableReq *pReq) { - buf = taosDecodeString(buf, &(pReq->name)); - buf = taosDecodeFixedU32(buf, &(pReq->ttl)); - buf = taosDecodeFixedU32(buf, &(pReq->keep)); - buf = taosDecodeFixedU8(buf, &(pReq->type)); - - switch (pReq->type) { - case META_SUPER_TABLE: - buf = taosDecodeFixedU64(buf, &(pReq->stbCfg.suid)); - buf = tdDecodeSchema(buf, &(pReq->stbCfg.pSchema)); - buf = tdDecodeSchema(buf, &(pReq->stbCfg.pTagSchema)); - break; - case META_CHILD_TABLE: - buf = taosDecodeFixedU64(buf, &(pReq->ctbCfg.suid)); - buf = tdDecodeKVRow(buf, &(pReq->ctbCfg.pTag)); - break; - case META_NORMAL_TABLE: - buf = tdDecodeSchema(buf, &(pReq->ntbCfg.pSchema)); - break; - default: - break; - } - - return buf; -} - -int vnodeBuildDropTableReq(void **buf, const SVDropTbReq *pReq) { - // TODO - return 0; -} - -void *vnodeParseDropTableReq(void *buf, SVDropTbReq *pReq) { - // TODO -} -#endif \ No newline at end of file diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c deleted file mode 100644 index 6dea4a4e57..0000000000 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ \ No newline at end of file From 093b982fa01b4d13349ed2315118f115de6b2550 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 13 Jan 2022 05:45:03 +0000 Subject: [PATCH 53/57] more refact --- source/dnode/vnode/src/inc/vnodeDef.h | 2 -- source/dnode/vnode/src/inc/vnodeFS.h | 47 ------------------------- source/dnode/vnode/src/inc/vnodeQuery.h | 3 +- source/dnode/vnode/src/vnd/vnodeFS.c | 16 --------- source/libs/tdb/src/tdbBufPool.c | 6 ++-- 5 files changed, 4 insertions(+), 70 deletions(-) delete mode 100644 source/dnode/vnode/src/inc/vnodeFS.h delete mode 100644 source/dnode/vnode/src/vnd/vnodeFS.c diff --git a/source/dnode/vnode/src/inc/vnodeDef.h b/source/dnode/vnode/src/inc/vnodeDef.h index 7054d5bc11..85563890fa 100644 --- a/source/dnode/vnode/src/inc/vnodeDef.h +++ b/source/dnode/vnode/src/inc/vnodeDef.h @@ -30,7 +30,6 @@ #include "vnodeBufferPool.h" #include "vnodeCfg.h" #include "vnodeCommit.h" -#include "vnodeFS.h" #include "vnodeMemAllocator.h" #include "vnodeQuery.h" #include "vnodeStateMgr.h" @@ -71,7 +70,6 @@ struct SVnode { STsdb* pTsdb; STQ* pTq; SWal* pWal; - SVnodeFS* pFs; tsem_t canCommit; SQHandle* pQuery; SDnode* pDnode; diff --git a/source/dnode/vnode/src/inc/vnodeFS.h b/source/dnode/vnode/src/inc/vnodeFS.h deleted file mode 100644 index dbec985695..0000000000 --- a/source/dnode/vnode/src/inc/vnodeFS.h +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_VNODE_FS_H_ -#define _TD_VNODE_FS_H_ - -#include "vnode.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct { -} SDir; - -typedef struct { -} SFile; - -typedef struct SFS { - void *pImpl; - int (*startEdit)(struct SFS *); - int (*endEdit)(struct SFS *); -} SFS; - -typedef struct { -} SVnodeFS; - -int vnodeOpenFS(SVnode *pVnode); -void vnodeCloseFS(SVnode *pVnode); - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_VNODE_FS_H_*/ \ No newline at end of file diff --git a/source/dnode/vnode/src/inc/vnodeQuery.h b/source/dnode/vnode/src/inc/vnodeQuery.h index d43f5b1cf1..9d40c34f00 100644 --- a/source/dnode/vnode/src/inc/vnodeQuery.h +++ b/source/dnode/vnode/src/inc/vnodeQuery.h @@ -19,12 +19,11 @@ #ifdef __cplusplus extern "C" { #endif -#include "vnodeInt.h" #include "qworker.h" +#include "vnode.h" typedef struct SQWorkerMgmt SQHandle; - int vnodeQueryOpen(SVnode *pVnode); #ifdef __cplusplus diff --git a/source/dnode/vnode/src/vnd/vnodeFS.c b/source/dnode/vnode/src/vnd/vnodeFS.c deleted file mode 100644 index 5e9f89ccd5..0000000000 --- a/source/dnode/vnode/src/vnd/vnodeFS.c +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#include "vnodeDef.h" \ No newline at end of file diff --git a/source/libs/tdb/src/tdbBufPool.c b/source/libs/tdb/src/tdbBufPool.c index 3eed100566..bc3c386b0f 100644 --- a/source/libs/tdb/src/tdbBufPool.c +++ b/source/libs/tdb/src/tdbBufPool.c @@ -16,9 +16,9 @@ #include "thash.h" #include "tlist.h" -#include "tkvBufPool.h" -#include "tkvDiskMgr.h" -#include "tkvPage.h" +#include "tdbBufPool.h" +#include "tdbDiskMgr.h" +#include "tdbPage.h" struct SFrameIdWrapper { TD_SLIST_NODE(SFrameIdWrapper); From c9118a52e6776db03c854afadc9d4c77f0c9fa82 Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 13 Jan 2022 06:13:10 +0000 Subject: [PATCH 54/57] more tkv and refact --- source/dnode/mgmt/impl/src/dndEnv.c | 2 +- source/dnode/vnode/inc/vnode.h | 18 ---------------- source/libs/tdb/inc/tdb.h | 15 +++++++++++-- source/libs/tdb/src/inc/tdbDB.h | 11 +++------- source/libs/tdb/src/inc/tdbEnv.h | 31 --------------------------- source/libs/tdb/test/tDiskMgrTest.cpp | 10 --------- source/libs/tdb/test/tdbTest.cpp | 5 +++++ source/libs/tdb/test/tkvTests.cpp | 0 8 files changed, 22 insertions(+), 70 deletions(-) delete mode 100644 source/libs/tdb/src/inc/tdbEnv.h delete mode 100644 source/libs/tdb/test/tDiskMgrTest.cpp create mode 100644 source/libs/tdb/test/tdbTest.cpp delete mode 100644 source/libs/tdb/test/tkvTests.cpp diff --git a/source/dnode/mgmt/impl/src/dndEnv.c b/source/dnode/mgmt/impl/src/dndEnv.c index 1bf1ea2b92..23fc643abe 100644 --- a/source/dnode/mgmt/impl/src/dndEnv.c +++ b/source/dnode/mgmt/impl/src/dndEnv.c @@ -293,7 +293,7 @@ int32_t dndInit(const SDnodeEnvCfg *pCfg) { if (vnodeInit(&vnodeOpt) != 0) { dError("failed to init vnode since %s", terrstr()); dndCleanup(); - return NULL; + return -1; } memcpy(&dndEnv.cfg, pCfg, sizeof(SDnodeEnvCfg)); diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 6113d0a536..7ff02309ec 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -37,36 +37,18 @@ typedef int32_t (*PutReqToVQueryQFp)(SDnode *pDnode, struct SRpcMsg *pReq); typedef struct SVnodeCfg { int32_t vgId; SDnode *pDnode; - - /** vnode buffer pool options */ struct { - /** write buffer size */ uint64_t wsize; uint64_t ssize; uint64_t lsize; - /** use heap allocator or arena allocator */ bool isHeapAllocator; }; - - /** time to live of tables in this vnode */ uint32_t ttl; - - /** data to keep in this vnode */ uint32_t keep; - - /** if TS data is eventually consistency */ bool isWeak; - - /** TSDB config */ STsdbCfg tsdbCfg; - - /** META config */ SMetaCfg metaCfg; - - /** TQ config */ STqCfg tqCfg; - - /** WAL config */ SWalCfg walCfg; } SVnodeCfg; diff --git a/source/libs/tdb/inc/tdb.h b/source/libs/tdb/inc/tdb.h index 2f47f545b1..40d79de821 100644 --- a/source/libs/tdb/inc/tdb.h +++ b/source/libs/tdb/inc/tdb.h @@ -22,9 +22,15 @@ extern "C" { #endif +typedef enum { + TDB_BTREE = 0, + TDB_HASH, + TDB_HEAP, +} tdb_db_t; + // Forward declaration -typedef struct TDB TDB; -typedef struct TDB_ENV TDB_ENV; +typedef struct TDB TDB; +typedef struct TDB_CURSOR TDB_CURSOR; // SKey typedef struct { @@ -32,6 +38,11 @@ typedef struct { uint32_t size; } TDB_KEY, TDB_VALUE; +// TDB Operations +int tdbCreateDB(TDB** dbpp); +int tdbOpenDB(TDB* dbp, tdb_db_t type, uint32_t flags); +int tdbCloseDB(TDB* dbp, uint32_t flags); + #ifdef __cplusplus } #endif diff --git a/source/libs/tdb/src/inc/tdbDB.h b/source/libs/tdb/src/inc/tdbDB.h index 40ddb1eb31..479ef77711 100644 --- a/source/libs/tdb/src/inc/tdbDB.h +++ b/source/libs/tdb/src/inc/tdbDB.h @@ -23,19 +23,14 @@ extern "C" { #endif -typedef enum { - TDB_BTREE = 0, - TDB_HASH, - TDB_HEAP, -} tdb_db_t; struct TDB { pgsize_t pageSize; tdb_db_t type; union { - STkvBtree btree; - STkvhash hash; - } dbimpl; + TDB_BTREE btree; + TDB_HASH hash; + } dbam; // Different access methods }; #ifdef __cplusplus diff --git a/source/libs/tdb/src/inc/tdbEnv.h b/source/libs/tdb/src/inc/tdbEnv.h deleted file mode 100644 index f3e4ef5888..0000000000 --- a/source/libs/tdb/src/inc/tdbEnv.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) 2019 TAOS Data, Inc. - * - * This program is free software: you can use, redistribute, and/or modify - * it under the terms of the GNU Affero General Public License, version 3 - * or later ("AGPL"), as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -#ifndef _TD_TDB_ENV_H_ -#define _TD_TDB_ENV_H_ - -#ifdef __cplusplus -extern "C" { -#endif - -struct TDB_ENV { - char *homeDir; -}; - -#ifdef __cplusplus -} -#endif - -#endif /*_TD_TDB_ENV_H_*/ \ No newline at end of file diff --git a/source/libs/tdb/test/tDiskMgrTest.cpp b/source/libs/tdb/test/tDiskMgrTest.cpp deleted file mode 100644 index a735fdb33d..0000000000 --- a/source/libs/tdb/test/tDiskMgrTest.cpp +++ /dev/null @@ -1,10 +0,0 @@ -#include "gtest/gtest.h" - -#include "iostream" - -#include "tDiskMgr.h" - -TEST(tDiskMgrTest, simple_test) { - // TODO - std::cout << "This is in tDiskMgrTest::simple_test" << std::endl; -} \ No newline at end of file diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp new file mode 100644 index 0000000000..ee91c7669b --- /dev/null +++ b/source/libs/tdb/test/tdbTest.cpp @@ -0,0 +1,5 @@ +#include "gtest/gtest.h" + +TEST(tdb_api_test, tdb_create_open_close_db_test) { + +} \ No newline at end of file diff --git a/source/libs/tdb/test/tkvTests.cpp b/source/libs/tdb/test/tkvTests.cpp deleted file mode 100644 index e69de29bb2..0000000000 From ac87dcf9661c1b2c2793add1219a15dbec2b8ffa Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 13 Jan 2022 08:04:43 +0000 Subject: [PATCH 55/57] more --- source/libs/tdb/CMakeLists.txt | 6 +++++- source/libs/tdb/test/CMakeLists.txt | 0 source/libs/tdb/test/tdbTest.cpp | 11 ++++++++++- 3 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 source/libs/tdb/test/CMakeLists.txt diff --git a/source/libs/tdb/CMakeLists.txt b/source/libs/tdb/CMakeLists.txt index 1832ea0b5d..68aa40c072 100644 --- a/source/libs/tdb/CMakeLists.txt +++ b/source/libs/tdb/CMakeLists.txt @@ -14,4 +14,8 @@ target_link_libraries( tdb PUBLIC os PUBLIC util -) \ No newline at end of file +) + +if(${BUILD_TEST}) + add_subdirectory(test) +endif(${BUILD_TEST}) diff --git a/source/libs/tdb/test/CMakeLists.txt b/source/libs/tdb/test/CMakeLists.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/source/libs/tdb/test/tdbTest.cpp b/source/libs/tdb/test/tdbTest.cpp index ee91c7669b..3ff43fdd69 100644 --- a/source/libs/tdb/test/tdbTest.cpp +++ b/source/libs/tdb/test/tdbTest.cpp @@ -1,5 +1,14 @@ #include "gtest/gtest.h" -TEST(tdb_api_test, tdb_create_open_close_db_test) { +#include "tdb.h" +TEST(tdb_api_test, tdb_create_open_close_db_test) { + int ret; + TDB *dbp; + + tdbCreateDB(&dbp); + + tdbOpenDB(dbp, TDB_BTREE, 0); + + tdbCloseDB(dbp, 0); } \ No newline at end of file From a20724f46eda6f4dcb9bfeedf5e9c8c93a2370fc Mon Sep 17 00:00:00 2001 From: Hongze Cheng Date: Thu, 13 Jan 2022 08:15:28 +0000 Subject: [PATCH 56/57] more --- source/dnode/vnode/src/vnd/vnodeWrite.c | 1 + source/libs/tdb/CMakeLists.txt | 2 +- source/libs/tdb/test/CMakeLists.txt | 3 +++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/source/dnode/vnode/src/vnd/vnodeWrite.c b/source/dnode/vnode/src/vnd/vnodeWrite.c index 185487757f..01619b5c77 100644 --- a/source/dnode/vnode/src/vnd/vnodeWrite.c +++ b/source/dnode/vnode/src/vnd/vnodeWrite.c @@ -83,6 +83,7 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { if (metaCreateTable(pVnode->pMeta, pCreateTbReq) < 0) { // TODO: handle error } + vTrace("vgId:%d process create table %s", pVnode->vgId, pCreateTbReq->name); if (pCreateTbReq->type == TD_SUPER_TABLE) { free(pCreateTbReq->stbCfg.pSchema); free(pCreateTbReq->stbCfg.pTagSchema); diff --git a/source/libs/tdb/CMakeLists.txt b/source/libs/tdb/CMakeLists.txt index 68aa40c072..647771fd2d 100644 --- a/source/libs/tdb/CMakeLists.txt +++ b/source/libs/tdb/CMakeLists.txt @@ -17,5 +17,5 @@ target_link_libraries( ) if(${BUILD_TEST}) - add_subdirectory(test) + # add_subdirectory(test) endif(${BUILD_TEST}) diff --git a/source/libs/tdb/test/CMakeLists.txt b/source/libs/tdb/test/CMakeLists.txt index e69de29bb2..2d77c1f4e9 100644 --- a/source/libs/tdb/test/CMakeLists.txt +++ b/source/libs/tdb/test/CMakeLists.txt @@ -0,0 +1,3 @@ +# tdbTest +add_executable(tdbTest "tdbTest.cpp") +target_link_libraries(tdbTest tdb gtest gtest_main) \ No newline at end of file From 6a0b6671cdb576ac40b2c49217d2b2893357a83a Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Thu, 13 Jan 2022 06:24:22 -0500 Subject: [PATCH 57/57] TD-12678 single table aggregation physical plan code --- include/libs/planner/planner.h | 19 +++ include/libs/planner/plannerOp.h | 2 +- source/libs/planner/src/physicalPlan.c | 25 +++- source/libs/planner/src/physicalPlanJson.c | 153 ++++++++++++++++----- source/libs/planner/test/phyPlanTests.cpp | 32 ++++- 5 files changed, 192 insertions(+), 39 deletions(-) diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index d98c1ca595..4f1db37f79 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -119,6 +119,25 @@ typedef struct SExchangePhyNode { SArray *pSrcEndPoints; // SEpAddr, scheduler fill by calling qSetSuplanExecutionNode } SExchangePhyNode; +typedef enum EAggAlgo { + AGG_ALGO_PLAIN = 1, // simple agg across all input rows + AGG_ALGO_SORTED, // grouped agg, input must be sorted + AGG_ALGO_HASHED // grouped agg, use internal hashtable +} EAggAlgo; + +typedef enum EAggSplit { + AGG_SPLIT_PRE = 1, // first level agg, maybe don't need calculate the final result + AGG_SPLIT_FINAL // second level agg, must calculate the final result +} EAggSplit; + +typedef struct SAggPhyNode { + SPhyNode node; + EAggAlgo aggAlgo; // algorithm used by agg operator + EAggSplit aggSplit; // distributed splitting mode + SArray *pExprs; // SExprInfo list, these are expression list of group_by_clause and parameter expression of aggregate function + SArray *pGroupByList; // SColIndex list, but these must be column node +} SAggPhyNode; + typedef struct SSubplanId { uint64_t queryId; uint64_t templateId; diff --git a/include/libs/planner/plannerOp.h b/include/libs/planner/plannerOp.h index 41d6e028cf..5cc896f1c2 100644 --- a/include/libs/planner/plannerOp.h +++ b/include/libs/planner/plannerOp.h @@ -30,7 +30,7 @@ OP_ENUM_MACRO(TagScan) OP_ENUM_MACRO(SystemTableScan) OP_ENUM_MACRO(Aggregate) OP_ENUM_MACRO(Project) -OP_ENUM_MACRO(Groupby) +// OP_ENUM_MACRO(Groupby) OP_ENUM_MACRO(Limit) OP_ENUM_MACRO(SLimit) OP_ENUM_MACRO(TimeWindow) diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index 71e69c67d2..65c97110ce 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -189,7 +189,6 @@ static SPhyNode* createUserTableScanNode(SQueryPlanNode* pPlanNode, SQueryTableI return (SPhyNode*)node; } - static bool isSystemTable(SQueryTableInfo* pTable) { // todo return false; @@ -295,13 +294,31 @@ static SPhyNode* createSingleTableScanNode(SQueryPlanNode* pPlanNode, SQueryTabl static SPhyNode* createTableScanNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { SQueryTableInfo* pTable = (SQueryTableInfo*)pPlanNode->pExtInfo; - if (needMultiNodeScan(pTable)) { return createExchangeNode(pCxt, pPlanNode, splitSubplanByTable(pCxt, pPlanNode, pTable)); } return createSingleTableScanNode(pPlanNode, pTable, pCxt->pCurrentSubplan); } +static SPhyNode* createSingleTableAgg(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { + SAggPhyNode* node = (SAggPhyNode*)initPhyNode(pPlanNode, OP_Aggregate, sizeof(SAggPhyNode)); + SGroupbyExpr* pGroupBy = (SGroupbyExpr*)pPlanNode->pExtInfo; + node->aggAlgo = AGG_ALGO_PLAIN; + node->aggSplit = AGG_SPLIT_FINAL; + if (NULL != pGroupBy) { + node->aggAlgo = AGG_ALGO_HASHED; + node->pGroupByList = validPointer(taosArrayDup(pGroupBy->columnInfo)); + } + return (SPhyNode*)node; +} + +static SPhyNode* createAggNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { + // if (needMultiNodeAgg(pPlanNode)) { + + // } + return createSingleTableAgg(pCxt, pPlanNode); +} + static SPhyNode* createPhyNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { SPhyNode* node = NULL; switch (pPlanNode->info.type) { @@ -311,6 +328,10 @@ static SPhyNode* createPhyNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { case QNODE_TABLESCAN: node = createTableScanNode(pCxt, pPlanNode); break; + case QNODE_AGGREGATE: + case QNODE_GROUPBY: + node = createAggNode(pCxt, pPlanNode); + break; case QNODE_MODIFY: // Insert is not an operator in a physical plan. break; diff --git a/source/libs/planner/src/physicalPlanJson.c b/source/libs/planner/src/physicalPlanJson.c index 7e36e0e124..123364134d 100644 --- a/source/libs/planner/src/physicalPlanJson.c +++ b/source/libs/planner/src/physicalPlanJson.c @@ -20,6 +20,19 @@ typedef bool (*FToJson)(const void* obj, cJSON* json); typedef bool (*FFromJson)(const cJSON* json, void* obj); +static char* getString(const cJSON* json, const char* name) { + char* p = cJSON_GetStringValue(cJSON_GetObjectItem(json, name)); + return strdup(p); +} + +static void copyString(const cJSON* json, const char* name, char* dst) { + strcpy(dst, cJSON_GetStringValue(cJSON_GetObjectItem(json, name))); +} + +static int64_t getNumber(const cJSON* json, const char* name) { + return cJSON_GetNumberValue(cJSON_GetObjectItem(json, name)); +} + static bool addObject(cJSON* json, const char* name, FToJson func, const void* obj) { if (NULL == obj) { return true; @@ -62,6 +75,39 @@ static bool fromObjectWithAlloc(const cJSON* json, const char* name, FFromJson f return func(jObj, *obj); } +static const char* jkPnodeType = "Type"; +static int32_t getPnodeTypeSize(cJSON* json) { + switch (getNumber(json, jkPnodeType)) { + case OP_TableScan: + case OP_DataBlocksOptScan: + case OP_TableSeqScan: + return sizeof(STableScanPhyNode); + case OP_TagScan: + return sizeof(STagScanPhyNode); + case OP_SystemTableScan: + return sizeof(SSystemTableScanPhyNode); + case OP_Aggregate: + return sizeof(SAggPhyNode); + case OP_Exchange: + return sizeof(SExchangePhyNode); + default: + break; + }; + return -1; +} + +static bool fromPnode(const cJSON* json, const char* name, FFromJson func, void** obj) { + cJSON* jObj = cJSON_GetObjectItem(json, name); + if (NULL == jObj) { + return true; + } + *obj = calloc(1, getPnodeTypeSize(jObj)); + if (NULL == *obj) { + return false; + } + return func(jObj, *obj); +} + static bool addTarray(cJSON* json, const char* name, FToJson func, const SArray* array, bool isPoint) { size_t size = (NULL == array) ? 0 : taosArrayGetSize(array); if (size > 0) { @@ -154,26 +200,9 @@ static bool fromRawArrayWithAlloc(const cJSON* json, const char* name, FFromJson return fromItem(jArray, func, *array, itemSize, *size); } -static bool fromRawArray(const cJSON* json, const char* name, FFromJson func, void** array, int32_t itemSize, int32_t* size) { +static bool fromRawArray(const cJSON* json, const char* name, FFromJson func, void* array, int32_t itemSize, int32_t* size) { const cJSON* jArray = getArray(json, name, size); - if (*array == NULL) { - *array = calloc(*size, itemSize); - } - - return fromItem(jArray, func, *array, itemSize, *size); -} - -static char* getString(const cJSON* json, const char* name) { - char* p = cJSON_GetStringValue(cJSON_GetObjectItem(json, name)); - return strdup(p); -} - -static void copyString(const cJSON* json, const char* name, char* dst) { - strcpy(dst, cJSON_GetStringValue(cJSON_GetObjectItem(json, name))); -} - -static int64_t getNumber(const cJSON* json, const char* name) { - return cJSON_GetNumberValue(cJSON_GetObjectItem(json, name)); + return fromItem(jArray, func, array, itemSize, *size); } static const char* jkSchemaType = "Type"; @@ -221,7 +250,7 @@ static bool dataBlockSchemaFromJson(const cJSON* json, void* obj) { schema->resultRowSize = getNumber(json, jkDataBlockSchemaResultRowSize); schema->precision = getNumber(json, jkDataBlockSchemaPrecision); - return fromRawArray(json, jkDataBlockSchemaSlotSchema, schemaFromJson, (void**) &(schema->pSchema), sizeof(SSlotSchema), &schema->numOfCols); + return fromRawArrayWithAlloc(json, jkDataBlockSchemaSlotSchema, schemaFromJson, (void**)&(schema->pSchema), sizeof(SSlotSchema), &schema->numOfCols); } static const char* jkColumnFilterInfoLowerRelOptr = "LowerRelOptr"; @@ -534,19 +563,15 @@ static const char* jkScanNodeTableCount = "Count"; static bool scanNodeToJson(const void* obj, cJSON* json) { const SScanPhyNode* scan = (const SScanPhyNode*)obj; bool res = cJSON_AddNumberToObject(json, jkScanNodeTableId, scan->uid); - if (res) { res = cJSON_AddNumberToObject(json, jkScanNodeTableType, scan->tableType); } - if (res) { res = cJSON_AddNumberToObject(json, jkScanNodeTableOrder, scan->order); } - if (res) { res = cJSON_AddNumberToObject(json, jkScanNodeTableCount, scan->count); } - return res; } @@ -559,6 +584,66 @@ static bool scanNodeFromJson(const cJSON* json, void* obj) { return true; } +static const char* jkColIndexColId = "ColId"; +static const char* jkColIndexColIndex = "ColIndex"; +static const char* jkColIndexFlag = "Flag"; +static const char* jkColIndexName = "Name"; + +static bool colIndexToJson(const void* obj, cJSON* json) { + const SColIndex* col = (const SColIndex*)obj; + bool res = cJSON_AddNumberToObject(json, jkColIndexColId, col->colId); + if (res) { + res = cJSON_AddNumberToObject(json, jkColIndexColIndex, col->colIndex); + } + if (res) { + res = cJSON_AddNumberToObject(json, jkColIndexFlag, col->flag); + } + if (res) { + res = cJSON_AddStringToObject(json, jkColIndexName, col->name); + } + return res; +} + +static bool colIndexFromJson(const cJSON* json, void* obj) { + SColIndex* col = (SColIndex*)obj; + col->colId = getNumber(json, jkColIndexColId); + col->colIndex = getNumber(json, jkColIndexColIndex); + col->flag = getNumber(json, jkColIndexFlag); + copyString(json, jkColIndexName, col->name); + return true; +} + +static const char* jkAggNodeAggAlgo = "AggAlgo"; +static const char* jkAggNodeAggSplit = "AggSplit"; +static const char* jkAggNodeExprs = "Exprs"; +static const char* jkAggNodeGroupByList = "GroupByList"; + +static bool aggNodeToJson(const void* obj, cJSON* json) { + const SAggPhyNode* agg = (const SAggPhyNode*)obj; + bool res = cJSON_AddNumberToObject(json, jkAggNodeAggAlgo, agg->aggAlgo); + if (res) { + res = cJSON_AddNumberToObject(json, jkAggNodeAggSplit, agg->aggSplit); + } + if (res) { + res = addArray(json, jkAggNodeExprs, exprInfoToJson, agg->pExprs); + } + if (res) { + res = addArray(json, jkAggNodeGroupByList, colIndexToJson, agg->pGroupByList); + } + return res; +} + +static bool aggNodeFromJson(const cJSON* json, void* obj) { + SAggPhyNode* agg = (SAggPhyNode*)obj; + agg->aggAlgo = getNumber(json, jkAggNodeAggAlgo); + agg->aggSplit = getNumber(json, jkAggNodeAggSplit); + bool res = fromArray(json, jkAggNodeExprs, exprInfoFromJson, &agg->pExprs, sizeof(SExprInfo)); + if (res) { + res = fromArray(json, jkAggNodeGroupByList, colIndexFromJson, &agg->pGroupByList, sizeof(SExprInfo)); + } + return res; +} + static const char* jkTableScanNodeFlag = "Flag"; static const char* jkTableScanNodeWindow = "Window"; static const char* jkTableScanNodeTagsConditions = "TagsConditions"; @@ -667,10 +752,10 @@ static bool specificPhyNodeToJson(const void* obj, cJSON* json) { case OP_SystemTableScan: return scanNodeToJson(obj, json); case OP_Aggregate: - break; // todo + return aggNodeToJson(obj, json); case OP_Project: return true; - case OP_Groupby: + // case OP_Groupby: case OP_Limit: case OP_SLimit: case OP_TimeWindow: @@ -708,7 +793,7 @@ static bool specificPhyNodeFromJson(const cJSON* json, void* obj) { break; // todo case OP_Project: return true; - case OP_Groupby: + // case OP_Groupby: case OP_Limit: case OP_SLimit: case OP_TimeWindow: @@ -735,12 +820,15 @@ static bool specificPhyNodeFromJson(const cJSON* json, void* obj) { static const char* jkPnodeName = "Name"; static const char* jkPnodeTargets = "Targets"; static const char* jkPnodeConditions = "Conditions"; -static const char* jkPnodeSchema = "InputSchema"; +static const char* jkPnodeSchema = "TargetSchema"; static const char* jkPnodeChildren = "Children"; // The 'pParent' field do not need to be serialized. static bool phyNodeToJson(const void* obj, cJSON* jNode) { const SPhyNode* phyNode = (const SPhyNode*)obj; - bool res = cJSON_AddStringToObject(jNode, jkPnodeName, phyNode->info.name); + bool res = cJSON_AddNumberToObject(jNode, jkPnodeType, phyNode->info.type); + if (res) { + res = cJSON_AddStringToObject(jNode, jkPnodeName, phyNode->info.name); + } if (res) { res = addArray(jNode, jkPnodeTargets, exprInfoToJson, phyNode->pTargets); } @@ -762,8 +850,8 @@ static bool phyNodeToJson(const void* obj, cJSON* jNode) { static bool phyNodeFromJson(const cJSON* json, void* obj) { SPhyNode* node = (SPhyNode*) obj; - node->info.name = getString(json, jkPnodeName); - node->info.type = opNameToOpType(node->info.name); + node->info.type = getNumber(json, jkPnodeType); + node->info.name = opTypeToOpName(node->info.type); bool res = fromArray(json, jkPnodeTargets, exprInfoFromJson, &node->pTargets, sizeof(SExprInfo)); if (res) { @@ -910,8 +998,7 @@ static SSubplan* subplanFromJson(const cJSON* json) { } bool res = fromObject(json, jkSubplanId, subplanIdFromJson, &subplan->id, true); if (res) { - size_t size = MAX(sizeof(SPhyNode), sizeof(SScanPhyNode)); - res = fromObjectWithAlloc(json, jkSubplanNode, phyNodeFromJson, (void**)&subplan->pNode, size, false); + res = fromPnode(json, jkSubplanNode, phyNodeFromJson, (void**)&subplan->pNode); } if (res) { res = fromObjectWithAlloc(json, jkSubplanDataSink, dataSinkFromJson, (void**)&subplan->pDataSink, sizeof(SDataSink), false); diff --git a/source/libs/planner/test/phyPlanTests.cpp b/source/libs/planner/test/phyPlanTests.cpp index 29f6e48dc7..6d9e08e829 100644 --- a/source/libs/planner/test/phyPlanTests.cpp +++ b/source/libs/planner/test/phyPlanTests.cpp @@ -30,6 +30,21 @@ void* myCalloc(size_t nmemb, size_t size) { class PhyPlanTest : public Test { protected: + void pushAgg(int32_t aggOp) { + unique_ptr agg((SQueryPlanNode*)myCalloc(1, sizeof(SQueryPlanNode))); + agg->info.type = aggOp; + agg->pExpr = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); + unique_ptr expr((SExprInfo*)myCalloc(1, sizeof(SExprInfo))); + expr->base.resSchema.type = TSDB_DATA_TYPE_INT; + expr->base.resSchema.bytes = tDataTypes[TSDB_DATA_TYPE_INT].bytes; + expr->pExpr = (tExprNode*)myCalloc(1, sizeof(tExprNode)); + expr->pExpr->nodeType = TEXPR_FUNCTION_NODE; + strcpy(expr->pExpr->_function.functionName, "Count"); + SExprInfo* item = expr.release(); + taosArrayPush(agg->pExpr, &item); + pushNode(agg.release()); + } + void pushScan(const string& db, const string& table, int32_t scanOp) { shared_ptr meta = mockCatalogService->getTableMeta(db, table); EXPECT_TRUE(meta); @@ -95,10 +110,11 @@ protected: private: void pushNode(SQueryPlanNode* node) { if (logicPlan_) { - // todo - } else { - logicPlan_.reset(node); + node->pChildren = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES); + SQueryPlanNode* child = logicPlan_.release(); + taosArrayPush(node->pChildren, &child); } + logicPlan_.reset(node); } void copySchemaMeta(STableMeta** dst, const STableMeta* src) { @@ -174,6 +190,16 @@ TEST_F(PhyPlanTest, superTableScanTest) { // todo check } +// select count(*) from table +TEST_F(PhyPlanTest, simpleAggTest) { + pushScan("test", "t1", QNODE_TABLESCAN); + pushAgg(QNODE_AGGREGATE); + ASSERT_EQ(run(), TSDB_CODE_SUCCESS); + explain(); + SQueryDag* dag = result(); + // todo check +} + // insert into t values(...) TEST_F(PhyPlanTest, insertTest) { ASSERT_EQ(run("test", "insert into t1 values (now, 1, \"beijing\")"), TSDB_CODE_SUCCESS);