diff --git a/include/common/common.h b/include/common/common.h index 0ae06e5a7f..9b8a465442 100644 --- a/include/common/common.h +++ b/include/common/common.h @@ -62,6 +62,12 @@ typedef struct SConstantItem { SVariant value; } SConstantItem; +typedef struct { + uint32_t numOfTables; + SArray *pGroupList; + SHashObj *map; // speedup acquire the tableQueryInfo by table uid +} STableGroupInfo; + typedef struct SSDataBlock { SColumnDataAgg *pBlockAgg; SArray *pDataBlock; // SArray diff --git a/include/dnode/vnode/tsdb/tsdb.h b/include/dnode/vnode/tsdb/tsdb.h index 3aeef4d013..f3d9f71346 100644 --- a/include/dnode/vnode/tsdb/tsdb.h +++ b/include/dnode/vnode/tsdb/tsdb.h @@ -18,6 +18,7 @@ #include "mallocator.h" #include "meta.h" +#include "common.h" #ifdef __cplusplus extern "C" { @@ -39,6 +40,10 @@ typedef struct STable { STSchema *pSchema; } STable; +#define BLOCK_LOAD_OFFSET_SEQ_ORDER 1 +#define BLOCK_LOAD_TABLE_SEQ_ORDER 2 +#define BLOCK_LOAD_TABLE_RR_ORDER 3 + #define TABLE_TID(t) (t)->tid #define TABLE_UID(t) (t)->uid @@ -58,6 +63,22 @@ typedef struct STsdbCfg { int8_t compression; } STsdbCfg; +// query condition to build multi-table data block iterator +typedef struct STsdbQueryCond { + STimeWindow twindow; + int32_t order; // desc|asc order to iterate the data block + int32_t numOfCols; + SColumnInfo *colList; + bool loadExternalRows; // load external rows or not + int32_t type; // data block load type: +} STsdbQueryCond; + +typedef struct { + void *pTable; + TSKEY lastKey; + uint64_t uid; +} STableKeyInfo; + // STsdb STsdb *tsdbOpen(const char *path, int32_t vgId, const STsdbCfg *pTsdbCfg, SMemAllocatorFactory *pMAF, SMeta *pMeta); void tsdbClose(STsdb *); @@ -70,6 +91,119 @@ int tsdbCommit(STsdb *pTsdb); int tsdbOptionsInit(STsdbCfg *); void tsdbOptionsClear(STsdbCfg *); +typedef void* tsdbReadHandleT; +/** + * Get the data block iterator, starting from position according to the query condition + * + * @param tsdb tsdb handle + * @param pCond query condition, including time window, result set order, and basic required columns for each block + * @param tableInfoGroup table object list in the form of set, grouped into different sets according to the + * group by condition + * @param qinfo query info handle from query processor + * @return + */ +tsdbReadHandleT *tsdbQueryTables(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, uint64_t qId, + void *pRef); + +/** + * Get the last row of the given query time window for all the tables in STableGroupInfo object. + * Note that only one data block with only row will be returned while invoking retrieve data block function for + * all tables in this group. + * + * @param tsdb tsdb handle + * @param pCond query condition, including time window, result set order, and basic required columns for each block + * @param tableInfo table list. + * @return + */ +//tsdbReadHandleT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfo, uint64_t qId, +// SMemRef *pRef); + + +tsdbReadHandleT tsdbQueryCacheLast(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, void* pMemRef); + +bool isTsdbCacheLastRow(tsdbReadHandleT* pTsdbReadHandle); + +/** + * get num of rows in mem table + * + * @param pHandle + * @return row size + */ + +int64_t tsdbGetNumOfRowsInMemTable(tsdbReadHandleT* pHandle); + +/** + * move to next block if exists + * + * @param pTsdbReadHandle + * @return + */ +bool tsdbNextDataBlock(tsdbReadHandleT pTsdbReadHandle); + +/** + * Get current data block information + * + * @param pTsdbReadHandle + * @param pBlockInfo + * @return + */ +void tsdbRetrieveDataBlockInfo(tsdbReadHandleT *pTsdbReadHandle, SDataBlockInfo *pBlockInfo); + +/** + * + * Get the pre-calculated information w.r.t. current data block. + * + * In case of data block in cache, the pBlockStatis will always be NULL. + * If a block is not completed loaded from disk, the pBlockStatis will be NULL. + + * @pBlockStatis the pre-calculated value for current data blocks. if the block is a cache block, always return 0 + * @return + */ +int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReadHandleT *pTsdbReadHandle, SDataStatis **pBlockStatis); + +/** + * + * The query condition with primary timestamp is passed to iterator during its constructor function, + * the returned data block must be satisfied with the time window condition in any cases, + * which means the SData data block is not actually the completed disk data blocks. + * + * @param pTsdbReadHandle query handle + * @param pColumnIdList required data columns id list + * @return + */ +SArray *tsdbRetrieveDataBlock(tsdbReadHandleT *pTsdbReadHandle, SArray *pColumnIdList); + +/** + * destroy the created table group list, which is generated by tag query + * @param pGroupList + */ +void tsdbDestroyTableGroup(STableGroupInfo *pGroupList); + +/** + * create the table group result including only one table, used to handle the normal table query + * + * @param tsdb tsdbHandle + * @param uid table uid + * @param pGroupInfo the generated result + * @return + */ +int32_t tsdbGetOneTableGroup(STsdb *tsdb, uint64_t uid, TSKEY startKey, STableGroupInfo *pGroupInfo); + +/** + * + * @param tsdb + * @param pTableIdList + * @param pGroupInfo + * @return + */ +int32_t tsdbGetTableGroupFromIdList(STsdb *tsdb, SArray *pTableIdList, STableGroupInfo *pGroupInfo); + +/** + * clean up the query handle + * @param queryHandle + */ +void tsdbCleanupQueryHandle(tsdbReadHandleT queryHandle); + #ifdef __cplusplus } #endif diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 17c11b5d09..2356498bbe 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -22,23 +22,25 @@ extern "C" { typedef void* qTaskInfo_t; -/** - * create the qinfo object according to QueryTableMsg - * @param tsdb - * @param pQueryTableMsg - * @param pTaskInfo - * @return - */ -int32_t qCreateTask(void* tsdb, int32_t vgId, void* pQueryTableMsg, qTaskInfo_t* pTaskInfo, uint64_t qId); + /** + * Create the exec task object according to task json + * @param tsdb + * @param vgId + * @param pTaskInfoMsg + * @param pTaskInfo + * @param qId + * @return + */ +int32_t qCreateExecTask(void* tsdb, int32_t vgId, struct SSubplan* pPlan, qTaskInfo_t* pTaskInfo); /** - * the main query execution function, including query on both table and multiple tables, + * the main task execution function, including query on both table and multiple tables, * which are decided according to the tag or table name query conditions * * @param qinfo * @return */ -bool qExecTask(qTaskInfo_t qinfo, uint64_t *qId); +bool qExecTask(qTaskInfo_t qTask, SSDataBlock** pRes); /** * Retrieve the produced results information, if current query is not paused or completed, @@ -81,7 +83,7 @@ int32_t qKillTask(qTaskInfo_t qinfo); * @param qinfo * @return */ -int32_t qIsQueryCompleted(qTaskInfo_t qinfo); +int32_t qIsTaskCompleted(qTaskInfo_t qinfo); /** * destroy query info structure @@ -113,7 +115,7 @@ int32_t qGetQualifiedTableIdList(void* pTableList, const char* tagCond, int32_t * @param numOfIndex * @return */ -int32_t qCreateTableGroupByGroupExpr(SArray* pTableIdList, TSKEY skey, STableGroupInfo groupInfo, SColIndex* groupByIndex, int32_t numOfIndex); +//int32_t qCreateTableGroupByGroupExpr(SArray* pTableIdList, TSKEY skey, STableGroupInfo groupInfo, SColIndex* groupByIndex, int32_t numOfIndex); /** * Update the table id list of a given query. diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index da70f21498..91421a9174 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -150,7 +150,7 @@ struct SQueryNode; * @param requestId * @return */ -int32_t qCreateQueryDag(const struct SQueryNode* pQueryInfo, struct SQueryDag** pDag, uint64_t requestId); +int32_t qCreateQueryDag(const struct SQueryNode* pQueryInfo, struct SQueryDag** pDag, SSchema** pSchema, uint32_t* numOfResCols, uint64_t requestId); // Set datasource of this subplan, multiple calls may be made to a subplan. // @subplan subplan to be schedule diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index d18142cebf..26f97cc6ef 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -197,7 +197,25 @@ int32_t execDdlQuery(SRequestObj* pRequest, SQueryNode* pQuery) { int32_t getPlan(SRequestObj* pRequest, SQueryNode* pQueryNode, SQueryDag** pDag) { pRequest->type = pQueryNode->type; - return qCreateQueryDag(pQueryNode, pDag, pRequest->requestId); + + SSchema *pSchema = NULL; + SReqResultInfo* pResInfo = &pRequest->body.resInfo; + + int32_t code = qCreateQueryDag(pQueryNode, pDag, &pSchema, &pResInfo->numOfCols, pRequest->requestId); + if (code != 0) { + return code; + } + + if (pQueryNode->type == TSDB_SQL_SELECT) { + pResInfo->fields = calloc(1, sizeof(TAOS_FIELD)); + for (int32_t i = 0; i < pResInfo->numOfCols; ++i) { + pResInfo->fields[i].bytes = pSchema[i].bytes; + pResInfo->fields[i].type = pSchema[i].type; + tstrncpy(pResInfo->fields[i].name, pSchema[i].name, tListLen(pResInfo->fields[i].name)); + } + } + + return code; } int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, void** pJob) { diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 73c9fc5e9f..af5ad874f8 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -24,7 +24,6 @@ #include "../inc/clientInt.h" #include "taos.h" -#include "tglobal.h" namespace { void showDB(TAOS* pConn) { @@ -57,449 +56,449 @@ TEST(testCase, connect_Test) { taos_close(pConn); } -TEST(testCase, create_user_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "create user abc pass 'abc'"); - if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { - printf("failed to create user, reason:%s\n", taos_errstr(pRes)); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, create_account_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "create account aabc pass 'abc'"); - if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { - printf("failed to create user, reason:%s\n", taos_errstr(pRes)); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, drop_account_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "drop account aabc"); - if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { - printf("failed to create user, reason:%s\n", taos_errstr(pRes)); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, show_user_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "show users"); - TAOS_ROW pRow = NULL; - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); - - char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, drop_user_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "drop user abc"); - if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { - printf("failed to create user, reason:%s\n", taos_errstr(pRes)); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, show_db_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "show databases"); - TAOS_ROW pRow = NULL; - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); - - char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } - - taos_close(pConn); -} - -TEST(testCase, create_db_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); - if (taos_errno(pRes) != 0) { - printf("error in create db, reason:%s\n", taos_errstr(pRes)); - } - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - ASSERT_TRUE(pFields == NULL); - - int32_t numOfFields = taos_num_fields(pRes); - ASSERT_EQ(numOfFields, 0); - - taos_free_result(pRes); - - pRes = taos_query(pConn, "create database abc1 vgroups 4"); - if (taos_errno(pRes) != 0) { - printf("error in create db, reason:%s\n", taos_errstr(pRes)); - } - taos_close(pConn); -} - -TEST(testCase, create_dnode_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "create dnode abc1 port 7000"); - if (taos_errno(pRes) != 0) { - printf("error in create dnode, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create dnode 1.1.1.1 port 9000"); - if (taos_errno(pRes) != 0) { - printf("failed to create dnode, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - taos_close(pConn); -} - -TEST(testCase, drop_dnode_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "drop dnode 2"); - if (taos_errno(pRes) != 0) { - printf("error in drop dnode, reason:%s\n", taos_errstr(pRes)); - } - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - ASSERT_TRUE(pFields == NULL); - - int32_t numOfFields = taos_num_fields(pRes); - ASSERT_EQ(numOfFields, 0); - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, use_db_test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("error in use db, reason:%s\n", taos_errstr(pRes)); - } - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - ASSERT_TRUE(pFields == NULL); - - int32_t numOfFields = taos_num_fields(pRes); - ASSERT_EQ(numOfFields, 0); - - taos_close(pConn); -} - -// TEST(testCase, drop_db_test) { +//TEST(testCase, create_user_Test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); // assert(pConn != NULL); // -// showDB(pConn); -// -// TAOS_RES* pRes = taos_query(pConn, "drop database abc1"); -// if (taos_errno(pRes) != 0) { -// printf("failed to drop db, reason:%s\n", taos_errstr(pRes)); +// TAOS_RES* pRes = taos_query(pConn, "create user abc pass 'abc'"); +// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { +// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); // } -// taos_free_result(pRes); // -// showDB(pConn); -// -// pRes = taos_query(pConn, "create database abc1"); -// if (taos_errno(pRes) != 0) { -// printf("create to drop db, reason:%s\n", taos_errstr(pRes)); -// } // taos_free_result(pRes); // taos_close(pConn); //} - -TEST(testCase, create_stable_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); - if (taos_errno(pRes) != 0) { - printf("error in create db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("error in use db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create stable st1(ts timestamp, k int) tags(a int)"); - if (taos_errno(pRes) != 0) { - printf("error in create stable, reason:%s\n", taos_errstr(pRes)); - } - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - ASSERT_TRUE(pFields == NULL); - - int32_t numOfFields = taos_num_fields(pRes); - ASSERT_EQ(numOfFields, 0); - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, create_table_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table tm0(ts timestamp, k int)"); - taos_free_result(pRes); - - taos_close(pConn); -} - -TEST(testCase, create_ctable_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("failed to use db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table tm0 using st1 tags(1)"); - if (taos_errno(pRes) != 0) { - printf("failed to create child table tm0, reason:%s\n", taos_errstr(pRes)); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, show_stable_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("failed to use db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "show stables"); - if (taos_errno(pRes) != 0) { - printf("failed to show stables, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); - } - - TAOS_ROW pRow = NULL; - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); - - char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, show_vgroup_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("failed to use db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "show vgroups"); - if (taos_errno(pRes) != 0) { - printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); - } - - TAOS_ROW pRow = NULL; - - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); - - char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, create_multiple_tables) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - ASSERT_NE(pConn, nullptr); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("failed to use db, reason:%s", taos_errstr(pRes)); - taos_free_result(pRes); - taos_close(pConn); - return; - } - - taos_free_result(pRes); - - pRes = taos_query(pConn, "create table t_2 using st1 tags(1)"); - if (taos_errno(pRes) != 0) { - printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); - } - - taos_free_result(pRes); - pRes = taos_query(pConn, "create table t_3 using st1 tags(2)"); - if (taos_errno(pRes) != 0) { - printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); - } - - TAOS_ROW pRow = NULL; - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); - - char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } - - taos_free_result(pRes); - - for (int32_t i = 0; i < 20; ++i) { - char sql[512] = {0}; - snprintf(sql, tListLen(sql), - "create table t_x_%d using st1 tags(2) t_x_%d using st1 tags(5) t_x_%d using st1 tags(911)", i, - (i + 1) * 30, (i + 2) * 40); - TAOS_RES* pres = taos_query(pConn, sql); - if (taos_errno(pres) != 0) { - printf("failed to create table %d\n, reason:%s", i, taos_errstr(pres)); - } - taos_free_result(pres); - } - - taos_close(pConn); -} - -TEST(testCase, show_table_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "use abc1"); - taos_free_result(pRes); - - pRes = taos_query(pConn, "show tables"); - if (taos_errno(pRes) != 0) { - printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes)); - taos_free_result(pRes); - ASSERT_TRUE(false); - } - - TAOS_ROW pRow = NULL; - TAOS_FIELD* pFields = taos_fetch_fields(pRes); - int32_t numOfFields = taos_num_fields(pRes); - - char str[512] = {0}; - while ((pRow = taos_fetch_row(pRes)) != NULL) { - int32_t code = taos_print_row(str, pRow, pFields, numOfFields); - printf("%s\n", str); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, drop_stable_Test) { - TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); - assert(pConn != NULL); - - TAOS_RES* pRes = taos_query(pConn, "create database abc1"); - if (taos_errno(pRes) != 0) { - printf("error in creating db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "use abc1"); - if (taos_errno(pRes) != 0) { - printf("error in using db, reason:%s\n", taos_errstr(pRes)); - } - taos_free_result(pRes); - - pRes = taos_query(pConn, "drop stable st1"); - if (taos_errno(pRes) != 0) { - printf("failed to drop stable, reason:%s\n", taos_errstr(pRes)); - } - - taos_free_result(pRes); - taos_close(pConn); -} - -TEST(testCase, generated_request_id_test) { - SHashObj* phash = taosHashInit(10000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); - - for (int32_t i = 0; i < 50000; ++i) { - uint64_t v = generateRequestId(); - void* result = taosHashGet(phash, &v, sizeof(v)); - if (result != nullptr) { - printf("0x%lx, index:%d\n", v, i); - } - assert(result == nullptr); - taosHashPut(phash, &v, sizeof(v), NULL, 0); - } - - taosHashCleanup(phash); -} +// +//TEST(testCase, create_account_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "create account aabc pass 'abc'"); +// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { +// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, drop_account_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "drop account aabc"); +// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { +// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, show_user_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "show users"); +// TAOS_ROW pRow = NULL; +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, drop_user_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "drop user abc"); +// if (taos_errno(pRes) != TSDB_CODE_SUCCESS) { +// printf("failed to create user, reason:%s\n", taos_errstr(pRes)); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, show_db_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "show databases"); +// TAOS_ROW pRow = NULL; +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// +// taos_close(pConn); +//} +// +//TEST(testCase, create_db_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); +// if (taos_errno(pRes) != 0) { +// printf("error in create db, reason:%s\n", taos_errstr(pRes)); +// } +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// ASSERT_TRUE(pFields == NULL); +// +// int32_t numOfFields = taos_num_fields(pRes); +// ASSERT_EQ(numOfFields, 0); +// +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create database abc1 vgroups 4"); +// if (taos_errno(pRes) != 0) { +// printf("error in create db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_close(pConn); +//} +// +//TEST(testCase, create_dnode_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "create dnode abc1 port 7000"); +// if (taos_errno(pRes) != 0) { +// printf("error in create dnode, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create dnode 1.1.1.1 port 9000"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create dnode, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// taos_close(pConn); +//} +// +//TEST(testCase, drop_dnode_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "drop dnode 2"); +// if (taos_errno(pRes) != 0) { +// printf("error in drop dnode, reason:%s\n", taos_errstr(pRes)); +// } +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// ASSERT_TRUE(pFields == NULL); +// +// int32_t numOfFields = taos_num_fields(pRes); +// ASSERT_EQ(numOfFields, 0); +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, use_db_test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("error in use db, reason:%s\n", taos_errstr(pRes)); +// } +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// ASSERT_TRUE(pFields == NULL); +// +// int32_t numOfFields = taos_num_fields(pRes); +// ASSERT_EQ(numOfFields, 0); +// +// taos_close(pConn); +//} +// +//// TEST(testCase, drop_db_test) { +//// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +//// assert(pConn != NULL); +//// +//// showDB(pConn); +//// +//// TAOS_RES* pRes = taos_query(pConn, "drop database abc1"); +//// if (taos_errno(pRes) != 0) { +//// printf("failed to drop db, reason:%s\n", taos_errstr(pRes)); +//// } +//// taos_free_result(pRes); +//// +//// showDB(pConn); +//// +//// pRes = taos_query(pConn, "create database abc1"); +//// if (taos_errno(pRes) != 0) { +//// printf("create to drop db, reason:%s\n", taos_errstr(pRes)); +//// } +//// taos_free_result(pRes); +//// taos_close(pConn); +////} +// +//TEST(testCase, create_stable_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); +// if (taos_errno(pRes) != 0) { +// printf("error in create db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("error in use db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create stable st1(ts timestamp, k int) tags(a int)"); +// if (taos_errno(pRes) != 0) { +// printf("error in create stable, reason:%s\n", taos_errstr(pRes)); +// } +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// ASSERT_TRUE(pFields == NULL); +// +// int32_t numOfFields = taos_num_fields(pRes); +// ASSERT_EQ(numOfFields, 0); +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, create_table_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create table tm0(ts timestamp, k int)"); +// taos_free_result(pRes); +// +// taos_close(pConn); +//} +// +//TEST(testCase, create_ctable_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create table tm0 using st1 tags(1)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create child table tm0, reason:%s\n", taos_errstr(pRes)); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, show_stable_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "show stables"); +// if (taos_errno(pRes) != 0) { +// printf("failed to show stables, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// ASSERT_TRUE(false); +// } +// +// TAOS_ROW pRow = NULL; +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, show_vgroup_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "show vgroups"); +// if (taos_errno(pRes) != 0) { +// printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// ASSERT_TRUE(false); +// } +// +// TAOS_ROW pRow = NULL; +// +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, create_multiple_tables) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// ASSERT_NE(pConn, nullptr); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// taos_close(pConn); +// return; +// } +// +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create table t_2 using st1 tags(1)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// ASSERT_TRUE(false); +// } +// +// taos_free_result(pRes); +// pRes = taos_query(pConn, "create table t_3 using st1 tags(2)"); +// if (taos_errno(pRes) != 0) { +// printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// ASSERT_TRUE(false); +// } +// +// TAOS_ROW pRow = NULL; +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// +// taos_free_result(pRes); +// +// for (int32_t i = 0; i < 20; ++i) { +// char sql[512] = {0}; +// snprintf(sql, tListLen(sql), +// "create table t_x_%d using st1 tags(2) t_x_%d using st1 tags(5) t_x_%d using st1 tags(911)", i, +// (i + 1) * 30, (i + 2) * 40); +// TAOS_RES* pres = taos_query(pConn, sql); +// if (taos_errno(pres) != 0) { +// printf("failed to create table %d\n, reason:%s", i, taos_errstr(pres)); +// } +// taos_free_result(pres); +// } +// +// taos_close(pConn); +//} +// +//TEST(testCase, show_table_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "use abc1"); +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "show tables"); +// if (taos_errno(pRes) != 0) { +// printf("failed to show vgroups, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// ASSERT_TRUE(false); +// } +// +// TAOS_ROW pRow = NULL; +// TAOS_FIELD* pFields = taos_fetch_fields(pRes); +// int32_t numOfFields = taos_num_fields(pRes); +// +// char str[512] = {0}; +// while ((pRow = taos_fetch_row(pRes)) != NULL) { +// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); +// printf("%s\n", str); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, drop_stable_Test) { +// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); +// assert(pConn != NULL); +// +// TAOS_RES* pRes = taos_query(pConn, "create database abc1"); +// if (taos_errno(pRes) != 0) { +// printf("error in creating db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "use abc1"); +// if (taos_errno(pRes) != 0) { +// printf("error in using db, reason:%s\n", taos_errstr(pRes)); +// } +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "drop stable st1"); +// if (taos_errno(pRes) != 0) { +// printf("failed to drop stable, reason:%s\n", taos_errstr(pRes)); +// } +// +// taos_free_result(pRes); +// taos_close(pConn); +//} +// +//TEST(testCase, generated_request_id_test) { +// SHashObj* phash = taosHashInit(10000, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); +// +// for (int32_t i = 0; i < 50000; ++i) { +// uint64_t v = generateRequestId(); +// void* result = taosHashGet(phash, &v, sizeof(v)); +// if (result != nullptr) { +// printf("0x%lx, index:%d\n", v, i); +// } +// assert(result == nullptr); +// taosHashPut(phash, &v, sizeof(v), NULL, 0); +// } +// +// taosHashCleanup(phash); +//} // TEST(testCase, create_topic_Test) { // TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -557,18 +556,29 @@ TEST(testCase, projection_query_tables) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(pConn, nullptr); - TAOS_RES* pRes = taos_query(pConn, "use test1"); - if (taos_errno(pRes) != 0) { - printf("failed to use db, reason:%s", taos_errstr(pRes)); - taos_free_result(pRes); - return; - } +// TAOS_RES* pRes = taos_query(pConn, "create database abc1 vgroups 2"); +// if (taos_errno(pRes) != 0) { +// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); +// taos_free_result(pRes); +// return; +// } - taos_free_result(pRes); +// taos_free_result(pRes); - pRes = taos_query(pConn, "select * from tm0"); + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + +// pRes = taos_query(pConn, "create table m1 (ts timestamp, k int) tags(a int)"); +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "create table tu using m1 tags(1)"); +// taos_free_result(pRes); +// +// pRes = taos_query(pConn, "insert into tu values(now, 1)"); +// taos_free_result(pRes); + + pRes = taos_query(pConn, "select * from tu"); if (taos_errno(pRes) != 0) { - printf("failed to create multiple tables, reason:%s\n", taos_errstr(pRes)); + printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); taos_free_result(pRes); ASSERT_TRUE(false); } diff --git a/source/dnode/vnode/impl/src/vnodeQuery.c b/source/dnode/vnode/impl/src/vnodeQuery.c index 29b6984937..909b233efb 100644 --- a/source/dnode/vnode/impl/src/vnodeQuery.c +++ b/source/dnode/vnode/impl/src/vnodeQuery.c @@ -23,7 +23,7 @@ int vnodeQueryOpen(SVnode *pVnode) { return qWorkerInit(NULL, &pVnode->pQuery); int vnodeProcessQueryReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { vTrace("query message is processed"); - return qWorkerProcessQueryMsg(pVnode, pVnode->pQuery, pMsg); + return qWorkerProcessQueryMsg(pVnode->pTsdb, pVnode->pQuery, pMsg); } int vnodeProcessFetchReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) { diff --git a/source/dnode/vnode/tsdb/CMakeLists.txt b/source/dnode/vnode/tsdb/CMakeLists.txt index 65dc95a9d8..e38ba1c466 100644 --- a/source/dnode/vnode/tsdb/CMakeLists.txt +++ b/source/dnode/vnode/tsdb/CMakeLists.txt @@ -13,6 +13,7 @@ else(0) "src/tsdbReadImpl.c" "src/tsdbFile.c" "src/tsdbFS.c" + "src/tsdbRead.c" ) endif(0) diff --git a/source/dnode/vnode/tsdb/src/tsdbCommit.c b/source/dnode/vnode/tsdb/src/tsdbCommit.c index 2be032a335..5aab343c4b 100644 --- a/source/dnode/vnode/tsdb/src/tsdbCommit.c +++ b/source/dnode/vnode/tsdb/src/tsdbCommit.c @@ -1253,7 +1253,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDataCols * pBlock->keyFirst = dataColsKeyFirst(pDataCols); pBlock->keyLast = dataColsKeyLast(pDataCols); - tsdbDebug("vgId:%d tid:%d a block of data is written to file %s, offset %" PRId64 + tsdbDebug("vgId:%d uid:%"PRId64" a block of data is written to file %s, offset %" PRId64 " numOfRows %d len %d numOfCols %" PRId16 " keyFirst %" PRId64 " keyLast %" PRId64, REPO_ID(pRepo), TABLE_TID(pTable), TSDB_FILE_FULL_NAME(pDFile), offset, rowsToWrite, pBlock->len, pBlock->numOfCols, pBlock->keyFirst, pBlock->keyLast); diff --git a/source/dnode/vnode/tsdb/src/tsdbRead.c b/source/dnode/vnode/tsdb/src/tsdbRead.c index da5481dae1..906046ed9a 100644 --- a/source/dnode/vnode/tsdb/src/tsdbRead.c +++ b/source/dnode/vnode/tsdb/src/tsdbRead.c @@ -13,18 +13,23 @@ * along with this program. If not, see . */ +#include "tsdb.h" +#include "tsdbDef.h" +#include "tsdbFS.h" +#include "tsdbLog.h" +#include "tsdbReadImpl.h" +#include "ttime.h" +#include "exception.h" #include "os.h" -#include "tdataformat.h" -#include "tskiplist.h" -#include "tulog.h" #include "talgo.h" #include "tcompare.h" -#include "exception.h" +#include "tdataformat.h" +#include "tskiplist.h" #include "taosdef.h" #include "tlosertree.h" #include "tsdbint.h" -#include "texpr.h" +#include "tmsg.h" #define EXTRA_BYTES 2 #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) @@ -34,8 +39,7 @@ ((SDataBlockInfo){.window = {.skey = (_block)->keyFirst, .ekey = (_block)->keyLast}, \ .numOfCols = (_block)->numOfCols, \ .rows = (_block)->numOfRows, \ - .tid = (_checkInfo)->tableId.tid, \ - .uid = (_checkInfo)->tableId.uid}) + .uid = (_checkInfo)->tableId}) enum { TSDB_QUERY_TYPE_ALL = 1, @@ -62,7 +66,7 @@ typedef struct SQueryFilePos { typedef struct SDataBlockLoadInfo { SDFileSet* fileGroup; int32_t slot; - int32_t tid; + uint64_t uid; SArray* pLoadedCols; } SDataBlockLoadInfo; @@ -79,13 +83,13 @@ enum { typedef struct STableCheckInfo { - STableId tableId; + uint64_t tableId; TSKEY lastKey; STable* pTableObj; SBlockInfo* pCompInfo; int32_t compSize; int32_t numOfBlocks:29; // number of qualified data blocks not the original blocks - uint8_t chosen:2; // indicate which iterator should move forward + uint8_t chosen:2; // indicate which iterator should move forward bool initBuf; // whether to initialize the in-memory skip list iterator or not SSkipListIterator* iter; // mem buffer skip list iterator SSkipListIterator* iiter; // imem buffer skip list iterator @@ -111,8 +115,8 @@ typedef struct SIOCostSummary { int64_t headFileLoadTime; } SIOCostSummary; -typedef struct STsdbQueryHandle { - STsdbRepo* pTsdb; +typedef struct STsdbReadHandle { + STsdb* pTsdb; SQueryFilePos cur; // current position int16_t order; STimeWindow window; // the primary query time window that applies to all queries @@ -137,7 +141,8 @@ typedef struct STsdbQueryHandle { STableBlockInfo* pDataBlockInfo; SDataCols *pDataCols; // in order to hold current file data block int32_t allocSize; // allocated data block size - SMemRef *pMemRef; +// STsdb +// STsdbMemTable * pMemTable; SArray *defaultLoadColumn;// default load column SDataBlockLoadInfo dataBlockLoadInfo; /* record current block load information */ SLoadCompBlockInfo compBlockLoadInfo; /* record current compblock information in SQueryAttr */ @@ -145,7 +150,7 @@ typedef struct STsdbQueryHandle { SArray *prev; // previous row which is before than time window SArray *next; // next row which is after the query time window SIOCostSummary cost; -} STsdbQueryHandle; +} STsdbReadHandle; typedef struct STableGroupSupporter { int32_t numOfCols; @@ -154,23 +159,23 @@ typedef struct STableGroupSupporter { } STableGroupSupporter; static STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList); -static int32_t checkForCachedLastRow(STsdbQueryHandle* pQueryHandle, STableGroupInfo *groupList); -static int32_t checkForCachedLast(STsdbQueryHandle* pQueryHandle); -static int32_t tsdbGetCachedLastRow(STable* pTable, SMemRow* pRes, TSKEY* lastKey); +static int32_t checkForCachedLastRow(STsdbReadHandle* pTsdbReadHandle, STableGroupInfo *groupList); +static int32_t checkForCachedLast(STsdbReadHandle* pTsdbReadHandle); +//static int32_t tsdbGetCachedLastRow(STable* pTable, SMemRow* pRes, TSKEY* lastKey); -static void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle); -static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SBlock* pBlock); +static void changeQueryHandleForInterpQuery(tsdbReadHandleT pHandle); +static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInfo* pCheckInfo, SBlock* pBlock); static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); -static int32_t tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win, STsdbQueryHandle* pQueryHandle); +static int32_t tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win, STsdbReadHandle* pTsdbReadHandle); static int32_t tsdbCheckInfoCompar(const void* key1, const void* key2); -static int32_t doGetExternalRow(STsdbQueryHandle* pQueryHandle, int16_t type, SMemRef* pMemRef); -static void* doFreeColumnInfoData(SArray* pColumnInfoData); -static void* destroyTableCheckInfo(SArray* pTableCheckInfo); -static bool tsdbGetExternalRow(TsdbQueryHandleT pHandle); +//static int32_t doGetExternalRow(STsdbReadHandle* pTsdbReadHandle, int16_t type, void* pMemRef); +//static void* doFreeColumnInfoData(SArray* pColumnInfoData); +//static void* destroyTableCheckInfo(SArray* pTableCheckInfo); +static bool tsdbGetExternalRow(tsdbReadHandleT pHandle); static void tsdbInitDataBlockLoadInfo(SDataBlockLoadInfo* pBlockLoadInfo) { pBlockLoadInfo->slot = -1; - pBlockLoadInfo->tid = -1; + pBlockLoadInfo->uid = 0; pBlockLoadInfo->fileGroup = NULL; } @@ -179,21 +184,21 @@ static void tsdbInitCompBlockLoadInfo(SLoadCompBlockInfo* pCompBlockLoadInfo) { pCompBlockLoadInfo->fileId = -1; } -static SArray* getColumnIdList(STsdbQueryHandle* pQueryHandle) { - size_t numOfCols = QH_GET_NUM_OF_COLS(pQueryHandle); +static SArray* getColumnIdList(STsdbReadHandle* pTsdbReadHandle) { + size_t numOfCols = QH_GET_NUM_OF_COLS(pTsdbReadHandle); assert(numOfCols <= TSDB_MAX_COLUMNS); SArray* pIdList = taosArrayInit(numOfCols, sizeof(int16_t)); for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i); + SColumnInfoData* pCol = taosArrayGet(pTsdbReadHandle->pColumns, i); taosArrayPush(pIdList, &pCol->info.colId); } return pIdList; } -static SArray* getDefaultLoadColumns(STsdbQueryHandle* pQueryHandle, bool loadTS) { - SArray* pLocalIdList = getColumnIdList(pQueryHandle); +static SArray* getDefaultLoadColumns(STsdbReadHandle* pTsdbReadHandle, bool loadTS) { + SArray* pLocalIdList = getColumnIdList(pTsdbReadHandle); // check if the primary time stamp column needs to load int16_t colId = *(int16_t*)taosArrayGet(pLocalIdList, 0); @@ -207,63 +212,63 @@ static SArray* getDefaultLoadColumns(STsdbQueryHandle* pQueryHandle, bool loadTS return pLocalIdList; } -static void tsdbMayTakeMemSnapshot(STsdbQueryHandle* pQueryHandle, SArray* psTable) { - assert(pQueryHandle != NULL && pQueryHandle->pMemRef != NULL); - - SMemRef* pMemRef = pQueryHandle->pMemRef; - if (pQueryHandle->pMemRef->ref++ == 0) { - tsdbTakeMemSnapshot(pQueryHandle->pTsdb, &(pMemRef->snapshot), psTable); - } - - taosArrayDestroy(psTable); +static void tsdbMayTakeMemSnapshot(STsdbReadHandle* pTsdbReadHandle, SArray* psTable) { +// assert(pTsdbReadHandle != NULL && pTsdbReadHandle->pMemRef != NULL); +// +// STsdbMemTable* pMemRef = pTsdbReadHandle->pMemRef; +// if (pTsdbReadHandle->pMemRef->ref++ == 0) { +// tsdbTakeMemSnapshot(pTsdbReadHandle->pTsdb, &(pMemRef->snapshot), psTable); +// } +// +// taosArrayDestroy(psTable); } -static void tsdbMayUnTakeMemSnapshot(STsdbQueryHandle* pQueryHandle) { - assert(pQueryHandle != NULL); - SMemRef* pMemRef = pQueryHandle->pMemRef; - if (pMemRef == NULL) { // it has been freed - return; - } - - if (--pMemRef->ref == 0) { - tsdbUnTakeMemSnapShot(pQueryHandle->pTsdb, &(pMemRef->snapshot)); - } - - pQueryHandle->pMemRef = NULL; +static void tsdbMayUnTakeMemSnapshot(STsdbReadHandle* pTsdbReadHandle) { +// assert(pTsdbReadHandle != NULL); +// STsdbMemTable* pMemRef = pTsdbReadHandle->pMemRef; +// if (pMemRef == NULL) { // it has been freed +// return; +// } +// +// if (--pMemRef->ref == 0) { +// tsdbUnTakeMemSnapShot(pTsdbReadHandle->pTsdb, &(pMemRef->snapshot)); +// } +// +// pTsdbReadHandle->pMemRef = NULL; } -int64_t tsdbGetNumOfRowsInMemTable(TsdbQueryHandleT* pHandle) { - STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; +//int64_t tsdbGetNumOfRowsInMemTable(tsdbReadHandleT* pHandle) { +// STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) pHandle; +// +// int64_t rows = 0; +// STsdbMemTable* pMemTable = pTsdbReadHandle->pMemTable; +// if (pMemTable == NULL) { return rows; } +// +//// STableData* pMem = NULL; +//// STableData* pIMem = NULL; +// +//// SMemTable* pMemT = pMemRef->snapshot.mem; +//// SMemTable* pIMemT = pMemRef->snapshot.imem; +// +// size_t size = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); +// for (int32_t i = 0; i < size; ++i) { +// STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, i); +// +//// if (pMemT && pCheckInfo->tableId < pMemT->maxTables) { +//// pMem = pMemT->tData[pCheckInfo->tableId]; +//// rows += (pMem && pMem->uid == pCheckInfo->tableId) ? pMem->numOfRows : 0; +//// } +//// if (pIMemT && pCheckInfo->tableId < pIMemT->maxTables) { +//// pIMem = pIMemT->tData[pCheckInfo->tableId]; +//// rows += (pIMem && pIMem->uid == pCheckInfo->tableId) ? pIMem->numOfRows : 0; +//// } +// } +// return rows; +//} - int64_t rows = 0; - SMemRef* pMemRef = pQueryHandle->pMemRef; - if (pMemRef == NULL) { return rows; } - - STableData* pMem = NULL; - STableData* pIMem = NULL; - - SMemTable* pMemT = pMemRef->snapshot.mem; - SMemTable* pIMemT = pMemRef->snapshot.imem; - - size_t size = taosArrayGetSize(pQueryHandle->pTableCheckInfo); - for (int32_t i = 0; i < size; ++i) { - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); - - if (pMemT && pCheckInfo->tableId.tid < pMemT->maxTables) { - pMem = pMemT->tData[pCheckInfo->tableId.tid]; - rows += (pMem && pMem->uid == pCheckInfo->tableId.uid) ? pMem->numOfRows : 0; - } - if (pIMemT && pCheckInfo->tableId.tid < pIMemT->maxTables) { - pIMem = pIMemT->tData[pCheckInfo->tableId.tid]; - rows += (pIMem && pIMem->uid == pCheckInfo->tableId.uid) ? pIMem->numOfRows : 0; - } - } - return rows; -} - -static SArray* createCheckInfoFromTableGroup(STsdbQueryHandle* pQueryHandle, STableGroupInfo* pGroupList, STsdbMeta* pMeta, SArray** psTable) { +static SArray* createCheckInfoFromTableGroup(STsdbReadHandle* pTsdbReadHandle, STableGroupInfo* pGroupList, SArray** psTable) { size_t sizeOfGroup = taosArrayGetSize(pGroupList->pGroupList); - assert(sizeOfGroup >= 1 && pMeta != NULL); + assert(sizeOfGroup >= 1); // allocate buffer in order to load data blocks from file SArray* pTableCheckInfo = taosArrayInit(pGroupList->numOfTables, sizeof(STableCheckInfo)); @@ -288,32 +293,28 @@ static SArray* createCheckInfoFromTableGroup(STsdbQueryHandle* pQueryHandle, STa STableKeyInfo* pKeyInfo = (STableKeyInfo*) taosArrayGet(group, j); STableCheckInfo info = { .lastKey = pKeyInfo->lastKey, .pTableObj = pKeyInfo->pTable }; - assert(info.pTableObj != NULL && (info.pTableObj->type == TSDB_NORMAL_TABLE || - info.pTableObj->type == TSDB_CHILD_TABLE || info.pTableObj->type == TSDB_STREAM_TABLE)); +// assert(info.pTableObj != NULL && (info.pTableObj->type == TSDB_NORMAL_TABLE || +// info.pTableObj->type == TSDB_CHILD_TABLE || info.pTableObj->type == TSDB_STREAM_TABLE)); - info.tableId.tid = info.pTableObj->tableId.tid; - info.tableId.uid = info.pTableObj->tableId.uid; + info.tableId = pKeyInfo->uid; - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { - if (info.lastKey == INT64_MIN || info.lastKey < pQueryHandle->window.skey) { - info.lastKey = pQueryHandle->window.skey; + if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { + if (info.lastKey == INT64_MIN || info.lastKey < pTsdbReadHandle->window.skey) { + info.lastKey = pTsdbReadHandle->window.skey; } - assert(info.lastKey >= pQueryHandle->window.skey && info.lastKey <= pQueryHandle->window.ekey); + assert(info.lastKey >= pTsdbReadHandle->window.skey && info.lastKey <= pTsdbReadHandle->window.ekey); } else { - assert(info.lastKey >= pQueryHandle->window.ekey && info.lastKey <= pQueryHandle->window.skey); + assert(info.lastKey >= pTsdbReadHandle->window.ekey && info.lastKey <= pTsdbReadHandle->window.skey); } taosArrayPush(pTableCheckInfo, &info); - tsdbDebug("%p check table uid:%"PRId64", tid:%d from lastKey:%"PRId64" 0x%"PRIx64, pQueryHandle, info.tableId.uid, - info.tableId.tid, info.lastKey, pQueryHandle->qId); + tsdbDebug("%p check table uid:%"PRId64" from lastKey:%"PRId64" 0x%"PRIx64, pTsdbReadHandle, info.tableId, info.lastKey, pTsdbReadHandle->qId); } } - taosArraySort(pTableCheckInfo, tsdbCheckInfoCompar); - +// taosArraySort(pTableCheckInfo, tsdbCheckInfoCompar); size_t gsize = taosArrayGetSize(pTableCheckInfo); - for (int32_t i = 0; i < gsize; ++i) { STableCheckInfo* pInfo = (STableCheckInfo*) taosArrayGet(pTableCheckInfo, i); taosArrayPush(pTable, &pInfo->pTableObj); @@ -323,22 +324,22 @@ static SArray* createCheckInfoFromTableGroup(STsdbQueryHandle* pQueryHandle, STa return pTableCheckInfo; } -static void resetCheckInfo(STsdbQueryHandle* pQueryHandle) { - size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); +static void resetCheckInfo(STsdbReadHandle* pTsdbReadHandle) { + size_t numOfTables = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); assert(numOfTables >= 1); // todo apply the lastkey of table check to avoid to load header file for (int32_t i = 0; i < numOfTables; ++i) { - STableCheckInfo* pCheckInfo = (STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, i); - pCheckInfo->lastKey = pQueryHandle->window.skey; + STableCheckInfo* pCheckInfo = (STableCheckInfo*) taosArrayGet(pTsdbReadHandle->pTableCheckInfo, i); + pCheckInfo->lastKey = pTsdbReadHandle->window.skey; pCheckInfo->iter = tSkipListDestroyIter(pCheckInfo->iter); pCheckInfo->iiter = tSkipListDestroyIter(pCheckInfo->iiter); pCheckInfo->initBuf = false; - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { - assert(pCheckInfo->lastKey >= pQueryHandle->window.skey); + if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { + assert(pCheckInfo->lastKey >= pTsdbReadHandle->window.skey); } else { - assert(pCheckInfo->lastKey <= pQueryHandle->window.skey); + assert(pCheckInfo->lastKey <= pTsdbReadHandle->window.skey); } } } @@ -358,38 +359,38 @@ static SArray* createCheckInfoFromCheckInfo(STableCheckInfo* pCheckInfo, TSKEY s return pNew; } -static bool emptyQueryTimewindow(STsdbQueryHandle* pQueryHandle) { - assert(pQueryHandle != NULL); +static bool emptyQueryTimewindow(STsdbReadHandle* pTsdbReadHandle) { + assert(pTsdbReadHandle != NULL); - STimeWindow* w = &pQueryHandle->window; - bool asc = ASCENDING_TRAVERSE(pQueryHandle->order); + STimeWindow* w = &pTsdbReadHandle->window; + bool asc = ASCENDING_TRAVERSE(pTsdbReadHandle->order); return ((asc && w->skey > w->ekey) || (!asc && w->ekey > w->skey)); } // Update the query time window according to the data time to live(TTL) information, in order to avoid to return // the expired data to client, even it is queried already. -static int64_t getEarliestValidTimestamp(STsdbRepo* pTsdb) { +static int64_t getEarliestValidTimestamp(STsdb* pTsdb) { STsdbCfg* pCfg = &pTsdb->config; int64_t now = taosGetTimestamp(pCfg->precision); return now - (tsTickPerDay[pCfg->precision] * pCfg->keep) + 1; // needs to add one tick } -static void setQueryTimewindow(STsdbQueryHandle* pQueryHandle, STsdbQueryCond* pCond) { - pQueryHandle->window = pCond->twindow; +static void setQueryTimewindow(STsdbReadHandle* pTsdbReadHandle, STsdbQueryCond* pCond) { + pTsdbReadHandle->window = pCond->twindow; bool updateTs = false; - int64_t startTs = getEarliestValidTimestamp(pQueryHandle->pTsdb); - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { - if (startTs > pQueryHandle->window.skey) { - pQueryHandle->window.skey = startTs; + int64_t startTs = getEarliestValidTimestamp(pTsdbReadHandle->pTsdb); + if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { + if (startTs > pTsdbReadHandle->window.skey) { + pTsdbReadHandle->window.skey = startTs; pCond->twindow.skey = startTs; updateTs = true; } } else { - if (startTs > pQueryHandle->window.ekey) { - pQueryHandle->window.ekey = startTs; + if (startTs > pTsdbReadHandle->window.ekey) { + pTsdbReadHandle->window.ekey = startTs; pCond->twindow.ekey = startTs; updateTs = true; } @@ -397,51 +398,50 @@ static void setQueryTimewindow(STsdbQueryHandle* pQueryHandle, STsdbQueryCond* p if (updateTs) { tsdbDebug("%p update the query time window, old:%" PRId64 " - %" PRId64 ", new:%" PRId64 " - %" PRId64 - ", 0x%" PRIx64, pQueryHandle, pCond->twindow.skey, pCond->twindow.ekey, pQueryHandle->window.skey, - pQueryHandle->window.ekey, pQueryHandle->qId); + ", 0x%" PRIx64, pTsdbReadHandle, pCond->twindow.skey, pCond->twindow.ekey, pTsdbReadHandle->window.skey, + pTsdbReadHandle->window.ekey, pTsdbReadHandle->qId); } } -static STsdbQueryHandle* tsdbQueryTablesImpl(STsdbRepo* tsdb, STsdbQueryCond* pCond, uint64_t qId, SMemRef* pMemRef) { - STsdbQueryHandle* pQueryHandle = calloc(1, sizeof(STsdbQueryHandle)); - if (pQueryHandle == NULL) { +static STsdbReadHandle* tsdbQueryTablesImpl(STsdb* tsdb, STsdbQueryCond* pCond, uint64_t qId, STsdbMemTable* pMemRef) { + STsdbReadHandle* pReadHandle = calloc(1, sizeof(STsdbReadHandle)); + if (pReadHandle == NULL) { goto _end; } - pQueryHandle->order = pCond->order; - pQueryHandle->pTsdb = tsdb; - pQueryHandle->type = TSDB_QUERY_TYPE_ALL; - pQueryHandle->cur.fid = INT32_MIN; - pQueryHandle->cur.win = TSWINDOW_INITIALIZER; - pQueryHandle->checkFiles = true; - pQueryHandle->activeIndex = 0; // current active table index - pQueryHandle->qId = qId; - pQueryHandle->allocSize = 0; - pQueryHandle->locateStart = false; - pQueryHandle->pMemRef = pMemRef; - pQueryHandle->loadType = pCond->type; + pReadHandle->order = pCond->order; + pReadHandle->pTsdb = tsdb; + pReadHandle->type = TSDB_QUERY_TYPE_ALL; + pReadHandle->cur.fid = INT32_MIN; + pReadHandle->cur.win = TSWINDOW_INITIALIZER; + pReadHandle->checkFiles = true; + pReadHandle->activeIndex = 0; // current active table index + pReadHandle->qId = qId; + pReadHandle->allocSize = 0; + pReadHandle->locateStart = false; + pReadHandle->loadType = pCond->type; - pQueryHandle->outputCapacity = ((STsdbRepo*)tsdb)->config.maxRowsPerFileBlock; - pQueryHandle->loadExternalRow = pCond->loadExternalRows; - pQueryHandle->currentLoadExternalRows = pCond->loadExternalRows; + pReadHandle->outputCapacity = 4096;//((STsdb*)tsdb)->config.maxRowsPerFileBlock; + pReadHandle->loadExternalRow = pCond->loadExternalRows; + pReadHandle->currentLoadExternalRows = pCond->loadExternalRows; - if (tsdbInitReadH(&pQueryHandle->rhelper, (STsdbRepo*)tsdb) != 0) { + if (tsdbInitReadH(&pReadHandle->rhelper, (STsdb*)tsdb) != 0) { goto _end; } - assert(pCond != NULL && pMemRef != NULL); - setQueryTimewindow(pQueryHandle, pCond); + assert(pCond != NULL); + setQueryTimewindow(pReadHandle, pCond); if (pCond->numOfCols > 0) { // allocate buffer in order to load data blocks from file - pQueryHandle->statis = calloc(pCond->numOfCols, sizeof(SDataStatis)); - if (pQueryHandle->statis == NULL) { + pReadHandle->statis = calloc(pCond->numOfCols, sizeof(SDataStatis)); + if (pReadHandle->statis == NULL) { goto _end; } // todo: use list instead of array? - pQueryHandle->pColumns = taosArrayInit(pCond->numOfCols, sizeof(SColumnInfoData)); - if (pQueryHandle->pColumns == NULL) { + pReadHandle->pColumns = taosArrayInit(pCond->numOfCols, sizeof(SColumnInfoData)); + if (pReadHandle->pColumns == NULL) { goto _end; } @@ -449,147 +449,144 @@ static STsdbQueryHandle* tsdbQueryTablesImpl(STsdbRepo* tsdb, STsdbQueryCond* pC SColumnInfoData colInfo = {{0}, 0}; colInfo.info = pCond->colList[i]; - colInfo.pData = calloc(1, EXTRA_BYTES + pQueryHandle->outputCapacity * pCond->colList[i].bytes); + colInfo.pData = calloc(1, EXTRA_BYTES + pReadHandle->outputCapacity * pCond->colList[i].bytes); if (colInfo.pData == NULL) { goto _end; } - taosArrayPush(pQueryHandle->pColumns, &colInfo); - pQueryHandle->statis[i].colId = colInfo.info.colId; + taosArrayPush(pReadHandle->pColumns, &colInfo); + pReadHandle->statis[i].colId = colInfo.info.colId; } - pQueryHandle->defaultLoadColumn = getDefaultLoadColumns(pQueryHandle, true); + pReadHandle->defaultLoadColumn = getDefaultLoadColumns(pReadHandle, true); } - STsdbMeta* pMeta = tsdbGetMeta(tsdb); - assert(pMeta != NULL); +// STsdbMeta* pMeta = NULL;//tsdbGetMeta(tsdb); +// assert(pMeta != NULL); - pQueryHandle->pDataCols = tdNewDataCols(pMeta->maxCols, pQueryHandle->pTsdb->config.maxRowsPerFileBlock); - if (pQueryHandle->pDataCols == NULL) { - tsdbError("%p failed to malloc buf for pDataCols, %"PRIu64, pQueryHandle, pQueryHandle->qId); + pReadHandle->pDataCols = tdNewDataCols(1000, pReadHandle->pTsdb->config.maxRowsPerFileBlock); + if (pReadHandle->pDataCols == NULL) { + tsdbError("%p failed to malloc buf for pDataCols, %"PRIu64, pReadHandle, pReadHandle->qId); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _end; } - tsdbInitDataBlockLoadInfo(&pQueryHandle->dataBlockLoadInfo); - tsdbInitCompBlockLoadInfo(&pQueryHandle->compBlockLoadInfo); + tsdbInitDataBlockLoadInfo(&pReadHandle->dataBlockLoadInfo); + tsdbInitCompBlockLoadInfo(&pReadHandle->compBlockLoadInfo); - return (TsdbQueryHandleT) pQueryHandle; + return (tsdbReadHandleT)pReadHandle; _end: - tsdbCleanupQueryHandle(pQueryHandle); +// tsdbCleanupQueryHandle(pTsdbReadHandle); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return NULL; } -TsdbQueryHandleT* tsdbQueryTables(STsdbRepo* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, uint64_t qId, SMemRef* pRef) { - STsdbQueryHandle* pQueryHandle = tsdbQueryTablesImpl(tsdb, pCond, qId, pRef); - if (pQueryHandle == NULL) { +tsdbReadHandleT* tsdbQueryTables(STsdb* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, uint64_t qId, void* pRef) { + STsdbReadHandle* pTsdbReadHandle = tsdbQueryTablesImpl(tsdb, pCond, qId, pRef); + if (pTsdbReadHandle == NULL) { return NULL; } - if (emptyQueryTimewindow(pQueryHandle)) { - return (TsdbQueryHandleT*) pQueryHandle; + if (emptyQueryTimewindow(pTsdbReadHandle)) { + return (tsdbReadHandleT*) pTsdbReadHandle; } - STsdbMeta* pMeta = tsdbGetMeta(tsdb); - assert(pMeta != NULL); - SArray* psTable = NULL; // todo apply the lastkey of table check to avoid to load header file - pQueryHandle->pTableCheckInfo = createCheckInfoFromTableGroup(pQueryHandle, groupList, pMeta, &psTable); - if (pQueryHandle->pTableCheckInfo == NULL) { - tsdbCleanupQueryHandle(pQueryHandle); + pTsdbReadHandle->pTableCheckInfo = createCheckInfoFromTableGroup(pTsdbReadHandle, groupList, &psTable); + if (pTsdbReadHandle->pTableCheckInfo == NULL) { +// tsdbCleanupQueryHandle(pTsdbReadHandle); taosArrayDestroy(psTable); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; return NULL; } - tsdbMayTakeMemSnapshot(pQueryHandle, psTable); +// tsdbMayTakeMemSnapshot(pTsdbReadHandle, psTable); - tsdbDebug("%p total numOfTable:%" PRIzu " in query, 0x%"PRIx64, pQueryHandle, taosArrayGetSize(pQueryHandle->pTableCheckInfo), pQueryHandle->qId); - return (TsdbQueryHandleT) pQueryHandle; + tsdbDebug("%p total numOfTable:%" PRIzu " in query, 0x%"PRIx64, pTsdbReadHandle, taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo), pTsdbReadHandle->qId); + return (tsdbReadHandleT) pTsdbReadHandle; } -void tsdbResetQueryHandle(TsdbQueryHandleT queryHandle, STsdbQueryCond *pCond) { - STsdbQueryHandle* pQueryHandle = queryHandle; +void tsdbResetQueryHandle(tsdbReadHandleT queryHandle, STsdbQueryCond *pCond) { + STsdbReadHandle* pTsdbReadHandle = queryHandle; - if (emptyQueryTimewindow(pQueryHandle)) { - if (pCond->order != pQueryHandle->order) { - pQueryHandle->order = pCond->order; - SWAP(pQueryHandle->window.skey, pQueryHandle->window.ekey, int64_t); + if (emptyQueryTimewindow(pTsdbReadHandle)) { + if (pCond->order != pTsdbReadHandle->order) { + pTsdbReadHandle->order = pCond->order; + SWAP(pTsdbReadHandle->window.skey, pTsdbReadHandle->window.ekey, int64_t); } return; } - pQueryHandle->order = pCond->order; - pQueryHandle->window = pCond->twindow; - pQueryHandle->type = TSDB_QUERY_TYPE_ALL; - pQueryHandle->cur.fid = -1; - pQueryHandle->cur.win = TSWINDOW_INITIALIZER; - pQueryHandle->checkFiles = true; - pQueryHandle->activeIndex = 0; // current active table index - pQueryHandle->locateStart = false; - pQueryHandle->loadExternalRow = pCond->loadExternalRows; + pTsdbReadHandle->order = pCond->order; + pTsdbReadHandle->window = pCond->twindow; + pTsdbReadHandle->type = TSDB_QUERY_TYPE_ALL; + pTsdbReadHandle->cur.fid = -1; + pTsdbReadHandle->cur.win = TSWINDOW_INITIALIZER; + pTsdbReadHandle->checkFiles = true; + pTsdbReadHandle->activeIndex = 0; // current active table index + pTsdbReadHandle->locateStart = false; + pTsdbReadHandle->loadExternalRow = pCond->loadExternalRows; if (ASCENDING_TRAVERSE(pCond->order)) { - assert(pQueryHandle->window.skey <= pQueryHandle->window.ekey); + assert(pTsdbReadHandle->window.skey <= pTsdbReadHandle->window.ekey); } else { - assert(pQueryHandle->window.skey >= pQueryHandle->window.ekey); + assert(pTsdbReadHandle->window.skey >= pTsdbReadHandle->window.ekey); } // allocate buffer in order to load data blocks from file - memset(pQueryHandle->statis, 0, sizeof(SDataStatis)); + memset(pTsdbReadHandle->statis, 0, sizeof(SDataStatis)); - tsdbInitDataBlockLoadInfo(&pQueryHandle->dataBlockLoadInfo); - tsdbInitCompBlockLoadInfo(&pQueryHandle->compBlockLoadInfo); + tsdbInitDataBlockLoadInfo(&pTsdbReadHandle->dataBlockLoadInfo); + tsdbInitCompBlockLoadInfo(&pTsdbReadHandle->compBlockLoadInfo); - resetCheckInfo(pQueryHandle); + resetCheckInfo(pTsdbReadHandle); } -void tsdbResetQueryHandleForNewTable(TsdbQueryHandleT queryHandle, STsdbQueryCond *pCond, STableGroupInfo* groupList) { - STsdbQueryHandle* pQueryHandle = queryHandle; +void tsdbResetQueryHandleForNewTable(tsdbReadHandleT queryHandle, STsdbQueryCond *pCond, STableGroupInfo* groupList) { + STsdbReadHandle* pTsdbReadHandle = queryHandle; - pQueryHandle->order = pCond->order; - pQueryHandle->window = pCond->twindow; - pQueryHandle->type = TSDB_QUERY_TYPE_ALL; - pQueryHandle->cur.fid = -1; - pQueryHandle->cur.win = TSWINDOW_INITIALIZER; - pQueryHandle->checkFiles = true; - pQueryHandle->activeIndex = 0; // current active table index - pQueryHandle->locateStart = false; - pQueryHandle->loadExternalRow = pCond->loadExternalRows; + pTsdbReadHandle->order = pCond->order; + pTsdbReadHandle->window = pCond->twindow; + pTsdbReadHandle->type = TSDB_QUERY_TYPE_ALL; + pTsdbReadHandle->cur.fid = -1; + pTsdbReadHandle->cur.win = TSWINDOW_INITIALIZER; + pTsdbReadHandle->checkFiles = true; + pTsdbReadHandle->activeIndex = 0; // current active table index + pTsdbReadHandle->locateStart = false; + pTsdbReadHandle->loadExternalRow = pCond->loadExternalRows; if (ASCENDING_TRAVERSE(pCond->order)) { - assert(pQueryHandle->window.skey <= pQueryHandle->window.ekey); + assert(pTsdbReadHandle->window.skey <= pTsdbReadHandle->window.ekey); } else { - assert(pQueryHandle->window.skey >= pQueryHandle->window.ekey); + assert(pTsdbReadHandle->window.skey >= pTsdbReadHandle->window.ekey); } // allocate buffer in order to load data blocks from file - memset(pQueryHandle->statis, 0, sizeof(SDataStatis)); + memset(pTsdbReadHandle->statis, 0, sizeof(SDataStatis)); - tsdbInitDataBlockLoadInfo(&pQueryHandle->dataBlockLoadInfo); - tsdbInitCompBlockLoadInfo(&pQueryHandle->compBlockLoadInfo); + tsdbInitDataBlockLoadInfo(&pTsdbReadHandle->dataBlockLoadInfo); + tsdbInitCompBlockLoadInfo(&pTsdbReadHandle->compBlockLoadInfo); SArray* pTable = NULL; - STsdbMeta* pMeta = tsdbGetMeta(pQueryHandle->pTsdb); +// STsdbMeta* pMeta = tsdbGetMeta(pTsdbReadHandle->pTsdb); - pQueryHandle->pTableCheckInfo = destroyTableCheckInfo(pQueryHandle->pTableCheckInfo); +// pTsdbReadHandle->pTableCheckInfo = destroyTableCheckInfo(pTsdbReadHandle->pTableCheckInfo); - pQueryHandle->pTableCheckInfo = createCheckInfoFromTableGroup(pQueryHandle, groupList, pMeta, &pTable); - if (pQueryHandle->pTableCheckInfo == NULL) { - tsdbCleanupQueryHandle(pQueryHandle); + pTsdbReadHandle->pTableCheckInfo = NULL;//createCheckInfoFromTableGroup(pTsdbReadHandle, groupList, pMeta, &pTable); + if (pTsdbReadHandle->pTableCheckInfo == NULL) { +// tsdbCleanupQueryHandle(pTsdbReadHandle); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; } - pQueryHandle->prev = doFreeColumnInfoData(pQueryHandle->prev); - pQueryHandle->next = doFreeColumnInfoData(pQueryHandle->next); +// pTsdbReadHandle->prev = doFreeColumnInfoData(pTsdbReadHandle->prev); +// pTsdbReadHandle->next = doFreeColumnInfoData(pTsdbReadHandle->next); } -TsdbQueryHandleT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, SMemRef* pMemRef) { +tsdbReadHandleT tsdbQueryLastRow(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, STsdbMemTable* pMemRef) { pCond->twindow = updateLastrowForEachGroup(groupList); // no qualified table @@ -597,56 +594,56 @@ TsdbQueryHandleT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STable return NULL; } - STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qId, pMemRef); - if (pQueryHandle == NULL) { + STsdbReadHandle *pTsdbReadHandle = (STsdbReadHandle*) tsdbQueryTables(tsdb, pCond, groupList, qId, pMemRef); + if (pTsdbReadHandle == NULL) { return NULL; } - int32_t code = checkForCachedLastRow(pQueryHandle, groupList); + int32_t code = checkForCachedLastRow(pTsdbReadHandle, groupList); if (code != TSDB_CODE_SUCCESS) { // set the numOfTables to be 0 terrno = code; return NULL; } assert(pCond->order == TSDB_ORDER_ASC && pCond->twindow.skey <= pCond->twindow.ekey); - if (pQueryHandle->cachelastrow) { - pQueryHandle->type = TSDB_QUERY_TYPE_LAST; + if (pTsdbReadHandle->cachelastrow) { + pTsdbReadHandle->type = TSDB_QUERY_TYPE_LAST; } - return pQueryHandle; + return pTsdbReadHandle; } - -TsdbQueryHandleT tsdbQueryCacheLast(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, SMemRef* pMemRef) { - STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qId, pMemRef); - if (pQueryHandle == NULL) { +#if 0 +tsdbReadHandleT tsdbQueryCacheLast(STsdb *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, uint64_t qId, STsdbMemTable* pMemRef) { + STsdbReadHandle *pTsdbReadHandle = (STsdbReadHandle*) tsdbQueryTables(tsdb, pCond, groupList, qId, pMemRef); + if (pTsdbReadHandle == NULL) { return NULL; } - int32_t code = checkForCachedLast(pQueryHandle); + int32_t code = checkForCachedLast(pTsdbReadHandle); if (code != TSDB_CODE_SUCCESS) { // set the numOfTables to be 0 terrno = code; return NULL; } - if (pQueryHandle->cachelastrow) { - pQueryHandle->type = TSDB_QUERY_TYPE_LAST; + if (pTsdbReadHandle->cachelastrow) { + pTsdbReadHandle->type = TSDB_QUERY_TYPE_LAST; } - return pQueryHandle; + return pTsdbReadHandle; } - -SArray* tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle) { +#endif +SArray* tsdbGetQueriedTableList(tsdbReadHandleT *pHandle) { assert(pHandle != NULL); - STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) pHandle; + STsdbReadHandle *pTsdbReadHandle = (STsdbReadHandle*) pHandle; - size_t size = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + size_t size = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); SArray* res = taosArrayInit(size, POINTER_BYTES); for(int32_t i = 0; i < size; ++i) { - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); + STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, i); taosArrayPush(res, &pCheckInfo->pTableObj); } @@ -668,11 +665,11 @@ static STableGroupInfo* trimTableGroup(STimeWindow* window, STableGroupInfo* pGr SArray* px = taosArrayInit(4, sizeof(STableKeyInfo)); for (int32_t j = 0; j < numOfTables; ++j) { STableKeyInfo* pInfo = (STableKeyInfo*)taosArrayGet(oneGroup, j); - if (window->skey <= pInfo->lastKey && ((STable*)pInfo->pTable)->lastKey != TSKEY_INITIAL_VAL) { - taosArrayPush(px, pInfo); - pNew->numOfTables += 1; - break; - } +// if (window->skey <= pInfo->lastKey && ((STable*)pInfo->pTable)->lastKey != TSKEY_INITIAL_VAL) { +// taosArrayPush(px, pInfo); +// pNew->numOfTables += 1; +// break; +// } } // there are no data in this group @@ -686,7 +683,7 @@ static STableGroupInfo* trimTableGroup(STimeWindow* window, STableGroupInfo* pGr return pNew; } -TsdbQueryHandleT tsdbQueryRowsInExternalWindow(STsdbRepo *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, uint64_t qId, SMemRef* pRef) { +tsdbReadHandleT tsdbQueryRowsInExternalWindow(STsdb *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, uint64_t qId, STsdbMemTable* pRef) { STableGroupInfo* pNew = trimTableGroup(&pCond->twindow, groupList); if (pNew->numOfTables == 0) { @@ -701,17 +698,14 @@ TsdbQueryHandleT tsdbQueryRowsInExternalWindow(STsdbRepo *tsdb, STsdbQueryCond* } } - STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, pNew, qId, pRef); - pQueryHandle->loadExternalRow = true; - pQueryHandle->currentLoadExternalRows = true; + STsdbReadHandle *pTsdbReadHandle = (STsdbReadHandle*) tsdbQueryTables(tsdb, pCond, pNew, qId, pRef); + pTsdbReadHandle->loadExternalRow = true; + pTsdbReadHandle->currentLoadExternalRows = true; - return pQueryHandle; + return pTsdbReadHandle; } -static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCheckInfo) { - STable* pTable = pCheckInfo->pTableObj; - assert(pTable != NULL); - +static bool initTableMemIterator(STsdbReadHandle* pHandle, STableCheckInfo* pCheckInfo) { if (pCheckInfo->initBuf) { return true; } @@ -720,33 +714,29 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh int32_t order = pHandle->order; // no data in buffer, abort - if (pHandle->pMemRef->snapshot.mem == NULL && pHandle->pMemRef->snapshot.imem == NULL) { - return false; - } +// if (pHandle->pMemTable->snapshot.mem == NULL && pHandle->pMemTable->snapshot.imem == NULL) { +// return false; +// } +// +// assert(pCheckInfo->iter == NULL && pCheckInfo->iiter == NULL); +// + STbData** pMem = NULL; + STbData** pIMem = NULL; - assert(pCheckInfo->iter == NULL && pCheckInfo->iiter == NULL); - - STableData* pMem = NULL; - STableData* pIMem = NULL; - - SMemTable* pMemT = pHandle->pMemRef->snapshot.mem; - SMemTable* pIMemT = pHandle->pMemRef->snapshot.imem; - - if (pMemT && pCheckInfo->tableId.tid < pMemT->maxTables) { - pMem = pMemT->tData[pCheckInfo->tableId.tid]; - if (pMem != NULL && pMem->uid == pCheckInfo->tableId.uid) { // check uid - TKEY tLastKey = keyToTkey(pCheckInfo->lastKey); + TKEY tLastKey = 0; /// keyToTkey(pCheckInfo->lastKey); + if (pHandle->pTsdb->mem != NULL) { + pMem = taosHashGet(pHandle->pTsdb->mem->pHashIdx, &pCheckInfo->tableId, sizeof(pCheckInfo->tableId)); + if (pMem != NULL) { pCheckInfo->iter = - tSkipListCreateIterFromVal(pMem->pData, (const char*)&tLastKey, TSDB_DATA_TYPE_TIMESTAMP, order); + tSkipListCreateIterFromVal((*pMem)->pData, (const char*)&tLastKey, TSDB_DATA_TYPE_TIMESTAMP, order); } } - if (pIMemT && pCheckInfo->tableId.tid < pIMemT->maxTables) { - pIMem = pIMemT->tData[pCheckInfo->tableId.tid]; - if (pIMem != NULL && pIMem->uid == pCheckInfo->tableId.uid) { // check uid - TKEY tLastKey = keyToTkey(pCheckInfo->lastKey); + if (pHandle->pTsdb->imem != NULL) { + pIMem = taosHashGet(pHandle->pTsdb->imem->pHashIdx, &pCheckInfo->tableId, sizeof(pCheckInfo->tableId)); + if (pIMem != NULL) { pCheckInfo->iiter = - tSkipListCreateIterFromVal(pIMem->pData, (const char*)&tLastKey, TSDB_DATA_TYPE_TIMESTAMP, order); + tSkipListCreateIterFromVal((*pIMem)->pData, (const char*)&tLastKey, TSDB_DATA_TYPE_TIMESTAMP, order); } } @@ -767,10 +757,9 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh SMemRow row = (SMemRow)SL_GET_NODE_DATA(node); TSKEY key = memRowKey(row); // first timestamp in buffer - tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in mem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 + tsdbDebug("%p uid:%" PRId64 ", check data in mem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 "-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", 0x%"PRIx64, - pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, key, order, pMem->keyFirst, pMem->keyLast, - pCheckInfo->lastKey, pMem->numOfRows, pHandle->qId); + pHandle, pCheckInfo->tableId, key, order, (*pMem)->keyMin, (*pMem)->keyMax, pCheckInfo->lastKey, (*pMem)->nrows, pHandle->qId); if (ASCENDING_TRAVERSE(order)) { assert(pCheckInfo->lastKey <= key); @@ -779,8 +768,7 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh } } else { - tsdbDebug("%p uid:%"PRId64", tid:%d no data in mem, 0x%"PRIx64, pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, - pHandle->qId); + tsdbDebug("%p uid:%"PRId64", no data in mem, 0x%"PRIx64, pHandle, pCheckInfo->tableId, pHandle->qId); } if (!imemEmpty) { @@ -789,10 +777,9 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh SMemRow row = (SMemRow)SL_GET_NODE_DATA(node); TSKEY key = memRowKey(row); // first timestamp in buffer - tsdbDebug("%p uid:%" PRId64 ", tid:%d check data in imem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 + tsdbDebug("%p uid:%" PRId64 ", check data in imem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 "-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", 0x%"PRIx64, - pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, key, order, pIMem->keyFirst, pIMem->keyLast, - pCheckInfo->lastKey, pIMem->numOfRows, pHandle->qId); + pHandle, pCheckInfo->tableId, key, order, (*pIMem)->keyMin, (*pIMem)->keyMax, pCheckInfo->lastKey, (*pIMem)->nrows, pHandle->qId); if (ASCENDING_TRAVERSE(order)) { assert(pCheckInfo->lastKey <= key); @@ -800,7 +787,7 @@ static bool initTableMemIterator(STsdbQueryHandle* pHandle, STableCheckInfo* pCh assert(pCheckInfo->lastKey >= key); } } else { - tsdbDebug("%p uid:%"PRId64", tid:%d no data in imem, 0x%"PRIx64, pHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, + tsdbDebug("%p uid:%"PRId64", no data in imem, 0x%"PRIx64, pHandle, pCheckInfo->tableId, pHandle->qId); } @@ -973,17 +960,13 @@ static bool moveToNextRowInMem(STableCheckInfo* pCheckInfo) { return hasNext; } -static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) { +static bool hasMoreDataInCache(STsdbReadHandle* pHandle) { STsdbCfg *pCfg = &pHandle->pTsdb->config; size_t size = taosArrayGetSize(pHandle->pTableCheckInfo); assert(pHandle->activeIndex < size && pHandle->activeIndex >= 0 && size >= 1); pHandle->cur.fid = INT32_MIN; STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); - - STable* pTable = pCheckInfo->pTableObj; - assert(pTable != NULL); - if (!pCheckInfo->initBuf) { initTableMemIterator(pHandle, pCheckInfo); } @@ -994,8 +977,8 @@ static bool hasMoreDataInCache(STsdbQueryHandle* pHandle) { } pCheckInfo->lastKey = memRowKey(row); // first timestamp in buffer - tsdbDebug("%p uid:%" PRId64", tid:%d check data in buffer from skey:%" PRId64 ", order:%d, 0x%"PRIx64, pHandle, - pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, pCheckInfo->lastKey, pHandle->order, pHandle->qId); + tsdbDebug("%p uid:%" PRId64", check data in buffer from skey:%" PRId64 ", order:%d, 0x%"PRIx64, pHandle, + pCheckInfo->tableId, pCheckInfo->lastKey, pHandle->order, pHandle->qId); // all data in mem are checked already. if ((pCheckInfo->lastKey > pHandle->window.ekey && ASCENDING_TRAVERSE(pHandle->order)) || @@ -1068,21 +1051,21 @@ static int32_t binarySearchForBlock(SBlock* pBlock, int32_t numOfBlocks, TSKEY s return midSlot; } -static int32_t loadBlockInfo(STsdbQueryHandle * pQueryHandle, int32_t index, int32_t* numOfBlocks) { +static int32_t loadBlockInfo(STsdbReadHandle * pTsdbReadHandle, int32_t index, int32_t* numOfBlocks) { int32_t code = 0; - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, index); + STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, index); pCheckInfo->numOfBlocks = 0; - if (tsdbSetReadTable(&pQueryHandle->rhelper, pCheckInfo->pTableObj) != TSDB_CODE_SUCCESS) { + if (tsdbSetReadTable(&pTsdbReadHandle->rhelper, pCheckInfo->pTableObj) != TSDB_CODE_SUCCESS) { code = terrno; return code; } - SBlockIdx* compIndex = pQueryHandle->rhelper.pBlkIdx; + SBlockIdx* compIndex = pTsdbReadHandle->rhelper.pBlkIdx; // no data block in this file, try next file - if (compIndex == NULL || compIndex->uid != pCheckInfo->tableId.uid) { + if (compIndex == NULL || compIndex->uid != pCheckInfo->tableId) { return 0; // no data blocks in the file belongs to pCheckInfo->pTable } @@ -1100,21 +1083,21 @@ static int32_t loadBlockInfo(STsdbQueryHandle * pQueryHandle, int32_t index, int pCheckInfo->compSize = compIndex->len; } - if (tsdbLoadBlockInfo(&(pQueryHandle->rhelper), (void*)(pCheckInfo->pCompInfo)) < 0) { + if (tsdbLoadBlockInfo(&(pTsdbReadHandle->rhelper), (void*)(pCheckInfo->pCompInfo)) < 0) { return terrno; } SBlockInfo* pCompInfo = pCheckInfo->pCompInfo; TSKEY s = TSKEY_INITIAL_VAL, e = TSKEY_INITIAL_VAL; - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { - assert(pCheckInfo->lastKey <= pQueryHandle->window.ekey && pQueryHandle->window.skey <= pQueryHandle->window.ekey); + if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { + assert(pCheckInfo->lastKey <= pTsdbReadHandle->window.ekey && pTsdbReadHandle->window.skey <= pTsdbReadHandle->window.ekey); } else { - assert(pCheckInfo->lastKey >= pQueryHandle->window.ekey && pQueryHandle->window.skey >= pQueryHandle->window.ekey); + assert(pCheckInfo->lastKey >= pTsdbReadHandle->window.ekey && pTsdbReadHandle->window.skey >= pTsdbReadHandle->window.ekey); } - s = MIN(pCheckInfo->lastKey, pQueryHandle->window.ekey); - e = MAX(pCheckInfo->lastKey, pQueryHandle->window.ekey); + s = MIN(pCheckInfo->lastKey, pTsdbReadHandle->window.ekey); + e = MAX(pCheckInfo->lastKey, pTsdbReadHandle->window.ekey); // discard the unqualified data block based on the query time window int32_t start = binarySearchForBlock(pCompInfo->blocks, compIndex->numOfBlocks, s, TSDB_ORDER_ASC); @@ -1139,26 +1122,26 @@ static int32_t loadBlockInfo(STsdbQueryHandle * pQueryHandle, int32_t index, int return 0; } -static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlocks) { +static int32_t getFileCompInfo(STsdbReadHandle* pTsdbReadHandle, int32_t* numOfBlocks) { // load all the comp offset value for all tables in this file int32_t code = TSDB_CODE_SUCCESS; *numOfBlocks = 0; - pQueryHandle->cost.headFileLoad += 1; + pTsdbReadHandle->cost.headFileLoad += 1; int64_t s = taosGetTimestampUs(); size_t numOfTables = 0; - if (pQueryHandle->loadType == BLOCK_LOAD_TABLE_SEQ_ORDER) { - code = loadBlockInfo(pQueryHandle, pQueryHandle->activeIndex, numOfBlocks); - } else if (pQueryHandle->loadType == BLOCK_LOAD_OFFSET_SEQ_ORDER) { - numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + if (pTsdbReadHandle->loadType == BLOCK_LOAD_TABLE_SEQ_ORDER) { + code = loadBlockInfo(pTsdbReadHandle, pTsdbReadHandle->activeIndex, numOfBlocks); + } else if (pTsdbReadHandle->loadType == BLOCK_LOAD_OFFSET_SEQ_ORDER) { + numOfTables = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); for (int32_t i = 0; i < numOfTables; ++i) { - code = loadBlockInfo(pQueryHandle, i, numOfBlocks); + code = loadBlockInfo(pTsdbReadHandle, i, numOfBlocks); if (code != TSDB_CODE_SUCCESS) { int64_t e = taosGetTimestampUs(); - pQueryHandle->cost.headFileLoadTime += (e - s); + pTsdbReadHandle->cost.headFileLoadTime += (e - s); return code; } } @@ -1167,57 +1150,57 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo } int64_t e = taosGetTimestampUs(); - pQueryHandle->cost.headFileLoadTime += (e - s); + pTsdbReadHandle->cost.headFileLoadTime += (e - s); return code; } -static int32_t doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SBlock* pBlock, STableCheckInfo* pCheckInfo, int32_t slotIndex) { +static int32_t doLoadFileDataBlock(STsdbReadHandle* pTsdbReadHandle, SBlock* pBlock, STableCheckInfo* pCheckInfo, int32_t slotIndex) { int64_t st = taosGetTimestampUs(); - STSchema *pSchema = tsdbGetTableSchema(pCheckInfo->pTableObj); - int32_t code = tdInitDataCols(pQueryHandle->pDataCols, pSchema); + STSchema *pSchema = NULL;//tsdbGetTableSchema(pCheckInfo->pTableObj); + int32_t code = tdInitDataCols(pTsdbReadHandle->pDataCols, pSchema); if (code != TSDB_CODE_SUCCESS) { - tsdbError("%p failed to malloc buf for pDataCols, 0x%"PRIx64, pQueryHandle, pQueryHandle->qId); + tsdbError("%p failed to malloc buf for pDataCols, 0x%"PRIx64, pTsdbReadHandle, pTsdbReadHandle->qId); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _error; } - code = tdInitDataCols(pQueryHandle->rhelper.pDCols[0], pSchema); + code = tdInitDataCols(pTsdbReadHandle->rhelper.pDCols[0], pSchema); if (code != TSDB_CODE_SUCCESS) { - tsdbError("%p failed to malloc buf for rhelper.pDataCols[0], 0x%"PRIx64, pQueryHandle, pQueryHandle->qId); + tsdbError("%p failed to malloc buf for rhelper.pDataCols[0], 0x%"PRIx64, pTsdbReadHandle, pTsdbReadHandle->qId); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _error; } - code = tdInitDataCols(pQueryHandle->rhelper.pDCols[1], pSchema); + code = tdInitDataCols(pTsdbReadHandle->rhelper.pDCols[1], pSchema); if (code != TSDB_CODE_SUCCESS) { - tsdbError("%p failed to malloc buf for rhelper.pDataCols[1], 0x%"PRIx64, pQueryHandle, pQueryHandle->qId); + tsdbError("%p failed to malloc buf for rhelper.pDataCols[1], 0x%"PRIx64, pTsdbReadHandle, pTsdbReadHandle->qId); terrno = TSDB_CODE_TDB_OUT_OF_MEMORY; goto _error; } - int16_t* colIds = pQueryHandle->defaultLoadColumn->pData; + int16_t* colIds = pTsdbReadHandle->defaultLoadColumn->pData; - int32_t ret = tsdbLoadBlockDataCols(&(pQueryHandle->rhelper), pBlock, pCheckInfo->pCompInfo, colIds, (int)(QH_GET_NUM_OF_COLS(pQueryHandle))); + int32_t ret = tsdbLoadBlockDataCols(&(pTsdbReadHandle->rhelper), pBlock, pCheckInfo->pCompInfo, colIds, (int)(QH_GET_NUM_OF_COLS(pTsdbReadHandle))); if (ret != TSDB_CODE_SUCCESS) { int32_t c = terrno; assert(c != TSDB_CODE_SUCCESS); goto _error; } - SDataBlockLoadInfo* pBlockLoadInfo = &pQueryHandle->dataBlockLoadInfo; + SDataBlockLoadInfo* pBlockLoadInfo = &pTsdbReadHandle->dataBlockLoadInfo; - pBlockLoadInfo->fileGroup = pQueryHandle->pFileGroup; - pBlockLoadInfo->slot = pQueryHandle->cur.slot; - pBlockLoadInfo->tid = pCheckInfo->pTableObj->tableId.tid; + pBlockLoadInfo->fileGroup = pTsdbReadHandle->pFileGroup; + pBlockLoadInfo->slot = pTsdbReadHandle->cur.slot; + pBlockLoadInfo->uid = pCheckInfo->pTableObj->uid; - SDataCols* pCols = pQueryHandle->rhelper.pDCols[0]; + SDataCols* pCols = pTsdbReadHandle->rhelper.pDCols[0]; assert(pCols->numOfRows != 0 && pCols->numOfRows <= pBlock->numOfRows); pBlock->numOfRows = pCols->numOfRows; // Convert from TKEY to TSKEY for primary timestamp column if current block has timestamp before 1970-01-01T00:00:00Z - if(pBlock->keyFirst < 0 && colIds[0] == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + if(pBlock->keyFirst < 0 && colIds[0] == PRIMARYKEY_TIMESTAMP_COL_ID) { int64_t* src = pCols->cols[0].pData; for(int32_t i = 0; i < pBlock->numOfRows; ++i) { src[i] = tdGetKey(src[i]); @@ -1225,60 +1208,60 @@ static int32_t doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SBlock* pBloc } int64_t elapsedTime = (taosGetTimestampUs() - st); - pQueryHandle->cost.blockLoadTime += elapsedTime; + pTsdbReadHandle->cost.blockLoadTime += elapsedTime; tsdbDebug("%p load file block into buffer, index:%d, brange:%"PRId64"-%"PRId64", rows:%d, elapsed time:%"PRId64 " us, 0x%"PRIx64, - pQueryHandle, slotIndex, pBlock->keyFirst, pBlock->keyLast, pBlock->numOfRows, elapsedTime, pQueryHandle->qId); + pTsdbReadHandle, slotIndex, pBlock->keyFirst, pBlock->keyLast, pBlock->numOfRows, elapsedTime, pTsdbReadHandle->qId); return TSDB_CODE_SUCCESS; _error: pBlock->numOfRows = 0; tsdbError("%p error occurs in loading file block, index:%d, brange:%"PRId64"-%"PRId64", rows:%d, 0x%"PRIx64, - pQueryHandle, slotIndex, pBlock->keyFirst, pBlock->keyLast, pBlock->numOfRows, pQueryHandle->qId); + pTsdbReadHandle, slotIndex, pBlock->keyFirst, pBlock->keyLast, pBlock->numOfRows, pTsdbReadHandle->qId); return terrno; } -static int32_t getEndPosInDataBlock(STsdbQueryHandle* pQueryHandle, SDataBlockInfo* pBlockInfo); -static int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, int32_t start, int32_t end); -static void moveDataToFront(STsdbQueryHandle* pQueryHandle, int32_t numOfRows, int32_t numOfCols); -static void doCheckGeneratedBlockRange(STsdbQueryHandle* pQueryHandle); -static void copyAllRemainRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SDataBlockInfo* pBlockInfo, int32_t endPos); +static int32_t getEndPosInDataBlock(STsdbReadHandle* pTsdbReadHandle, SDataBlockInfo* pBlockInfo); +static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t capacity, int32_t numOfRows, int32_t start, int32_t end); +static void moveDataToFront(STsdbReadHandle* pTsdbReadHandle, int32_t numOfRows, int32_t numOfCols); +static void doCheckGeneratedBlockRange(STsdbReadHandle* pTsdbReadHandle); +static void copyAllRemainRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, STableCheckInfo* pCheckInfo, SDataBlockInfo* pBlockInfo, int32_t endPos); -static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SBlock* pBlock, STableCheckInfo* pCheckInfo){ - SQueryFilePos* cur = &pQueryHandle->cur; - STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; +static int32_t handleDataMergeIfNeeded(STsdbReadHandle* pTsdbReadHandle, SBlock* pBlock, STableCheckInfo* pCheckInfo){ + SQueryFilePos* cur = &pTsdbReadHandle->cur; + STsdbCfg* pCfg = &pTsdbReadHandle->pTsdb->config; SDataBlockInfo binfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock); TSKEY key; int32_t code = TSDB_CODE_SUCCESS; - /*bool hasData = */ initTableMemIterator(pQueryHandle, pCheckInfo); + /*bool hasData = */ initTableMemIterator(pTsdbReadHandle, pCheckInfo); assert(cur->pos >= 0 && cur->pos <= binfo.rows); - key = extractFirstTraverseKey(pCheckInfo, pQueryHandle->order, pCfg->update); + key = extractFirstTraverseKey(pCheckInfo, pTsdbReadHandle->order, pCfg->update); if (key != TSKEY_INITIAL_VAL) { - tsdbDebug("%p key in mem:%"PRId64", 0x%"PRIx64, pQueryHandle, key, pQueryHandle->qId); + tsdbDebug("%p key in mem:%"PRId64", 0x%"PRIx64, pTsdbReadHandle, key, pTsdbReadHandle->qId); } else { - tsdbDebug("%p no data in mem, 0x%"PRIx64, pQueryHandle, pQueryHandle->qId); + tsdbDebug("%p no data in mem, 0x%"PRIx64, pTsdbReadHandle, pTsdbReadHandle->qId); } - if ((ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key <= binfo.window.ekey)) || - (!ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key >= binfo.window.skey))) { + if ((ASCENDING_TRAVERSE(pTsdbReadHandle->order) && (key != TSKEY_INITIAL_VAL && key <= binfo.window.ekey)) || + (!ASCENDING_TRAVERSE(pTsdbReadHandle->order) && (key != TSKEY_INITIAL_VAL && key >= binfo.window.skey))) { - if ((ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key < binfo.window.skey)) || - (!ASCENDING_TRAVERSE(pQueryHandle->order) && (key != TSKEY_INITIAL_VAL && key > binfo.window.ekey))) { + if ((ASCENDING_TRAVERSE(pTsdbReadHandle->order) && (key != TSKEY_INITIAL_VAL && key < binfo.window.skey)) || + (!ASCENDING_TRAVERSE(pTsdbReadHandle->order) && (key != TSKEY_INITIAL_VAL && key > binfo.window.ekey))) { // do not load file block into buffer - int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order) ? 1 : -1; + int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order) ? 1 : -1; - TSKEY maxKey = ASCENDING_TRAVERSE(pQueryHandle->order)? (binfo.window.skey - step):(binfo.window.ekey - step); - cur->rows = tsdbReadRowsFromCache(pCheckInfo, maxKey, pQueryHandle->outputCapacity, &cur->win, pQueryHandle); - pQueryHandle->realNumOfRows = cur->rows; + TSKEY maxKey = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? (binfo.window.skey - step):(binfo.window.ekey - step); + cur->rows = tsdbReadRowsFromCache(pCheckInfo, maxKey, pTsdbReadHandle->outputCapacity, &cur->win, pTsdbReadHandle); + pTsdbReadHandle->realNumOfRows = cur->rows; // update the last key value pCheckInfo->lastKey = cur->win.ekey + step; - if (!ASCENDING_TRAVERSE(pQueryHandle->order)) { + if (!ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { SWAP(cur->win.skey, cur->win.ekey, TSKEY); } @@ -1289,11 +1272,11 @@ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SBlock* p // return error, add test cases - if ((code = doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo, cur->slot)) != TSDB_CODE_SUCCESS) { + if ((code = doLoadFileDataBlock(pTsdbReadHandle, pBlock, pCheckInfo, cur->slot)) != TSDB_CODE_SUCCESS) { return code; } - doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock); + doMergeTwoLevelData(pTsdbReadHandle, pCheckInfo, pBlock); } else { /* * no data in cache, only load data from file @@ -1301,19 +1284,19 @@ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SBlock* p * * Here the buffer is not enough, so only part of file block can be loaded into memory buffer */ - assert(pQueryHandle->outputCapacity >= binfo.rows); - int32_t endPos = getEndPosInDataBlock(pQueryHandle, &binfo); + assert(pTsdbReadHandle->outputCapacity >= binfo.rows); + int32_t endPos = getEndPosInDataBlock(pTsdbReadHandle, &binfo); - if ((cur->pos == 0 && endPos == binfo.rows -1 && ASCENDING_TRAVERSE(pQueryHandle->order)) || - (cur->pos == (binfo.rows - 1) && endPos == 0 && (!ASCENDING_TRAVERSE(pQueryHandle->order)))) { - pQueryHandle->realNumOfRows = binfo.rows; + if ((cur->pos == 0 && endPos == binfo.rows -1 && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || + (cur->pos == (binfo.rows - 1) && endPos == 0 && (!ASCENDING_TRAVERSE(pTsdbReadHandle->order)))) { + pTsdbReadHandle->realNumOfRows = binfo.rows; cur->rows = binfo.rows; cur->win = binfo.window; cur->mixBlock = false; cur->blockCompleted = true; - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { + if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { cur->lastKey = binfo.window.ekey + 1; cur->pos = binfo.rows; } else { @@ -1321,17 +1304,17 @@ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SBlock* p cur->pos = -1; } } else { // partially copy to dest buffer - copyAllRemainRowsFromFileBlock(pQueryHandle, pCheckInfo, &binfo, endPos); + copyAllRemainRowsFromFileBlock(pTsdbReadHandle, pCheckInfo, &binfo, endPos); cur->mixBlock = true; } assert(cur->blockCompleted); if (cur->rows == binfo.rows) { tsdbDebug("%p whole file block qualified, brange:%"PRId64"-%"PRId64", rows:%d, lastKey:%"PRId64", %"PRIx64, - pQueryHandle, cur->win.skey, cur->win.ekey, cur->rows, cur->lastKey, pQueryHandle->qId); + pTsdbReadHandle, cur->win.skey, cur->win.ekey, cur->rows, cur->lastKey, pTsdbReadHandle->qId); } else { tsdbDebug("%p create data block from remain file block, brange:%"PRId64"-%"PRId64", rows:%d, total:%d, lastKey:%"PRId64", %"PRIx64, - pQueryHandle, cur->win.skey, cur->win.ekey, cur->rows, binfo.rows, cur->lastKey, pQueryHandle->qId); + pTsdbReadHandle, cur->win.skey, cur->win.ekey, cur->rows, binfo.rows, cur->lastKey, pTsdbReadHandle->qId); } } @@ -1339,58 +1322,58 @@ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SBlock* p return code; } -static int32_t loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SBlock* pBlock, STableCheckInfo* pCheckInfo, bool* exists) { - SQueryFilePos* cur = &pQueryHandle->cur; +static int32_t loadFileDataBlock(STsdbReadHandle* pTsdbReadHandle, SBlock* pBlock, STableCheckInfo* pCheckInfo, bool* exists) { + SQueryFilePos* cur = &pTsdbReadHandle->cur; int32_t code = TSDB_CODE_SUCCESS; - bool asc = ASCENDING_TRAVERSE(pQueryHandle->order); + bool asc = ASCENDING_TRAVERSE(pTsdbReadHandle->order); if (asc) { // query ended in/started from current block - if (pQueryHandle->window.ekey < pBlock->keyLast || pCheckInfo->lastKey > pBlock->keyFirst) { - if ((code = doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo, cur->slot)) != TSDB_CODE_SUCCESS) { + if (pTsdbReadHandle->window.ekey < pBlock->keyLast || pCheckInfo->lastKey > pBlock->keyFirst) { + if ((code = doLoadFileDataBlock(pTsdbReadHandle, pBlock, pCheckInfo, cur->slot)) != TSDB_CODE_SUCCESS) { *exists = false; return code; } - SDataCols* pTSCol = pQueryHandle->rhelper.pDCols[0]; + SDataCols* pTSCol = pTsdbReadHandle->rhelper.pDCols[0]; assert(pTSCol->cols->type == TSDB_DATA_TYPE_TIMESTAMP && pTSCol->numOfRows == pBlock->numOfRows); if (pCheckInfo->lastKey > pBlock->keyFirst) { cur->pos = - binarySearchForKey(pTSCol->cols[0].pData, pBlock->numOfRows, pCheckInfo->lastKey, pQueryHandle->order); + binarySearchForKey(pTSCol->cols[0].pData, pBlock->numOfRows, pCheckInfo->lastKey, pTsdbReadHandle->order); } else { cur->pos = 0; } assert(pCheckInfo->lastKey <= pBlock->keyLast); - doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock); + doMergeTwoLevelData(pTsdbReadHandle, pCheckInfo, pBlock); } else { // the whole block is loaded in to buffer cur->pos = asc? 0:(pBlock->numOfRows - 1); - code = handleDataMergeIfNeeded(pQueryHandle, pBlock, pCheckInfo); + code = handleDataMergeIfNeeded(pTsdbReadHandle, pBlock, pCheckInfo); } } else { //desc order, query ended in current block - if (pQueryHandle->window.ekey > pBlock->keyFirst || pCheckInfo->lastKey < pBlock->keyLast) { - if ((code = doLoadFileDataBlock(pQueryHandle, pBlock, pCheckInfo, cur->slot)) != TSDB_CODE_SUCCESS) { + if (pTsdbReadHandle->window.ekey > pBlock->keyFirst || pCheckInfo->lastKey < pBlock->keyLast) { + if ((code = doLoadFileDataBlock(pTsdbReadHandle, pBlock, pCheckInfo, cur->slot)) != TSDB_CODE_SUCCESS) { *exists = false; return code; } - SDataCols* pTsCol = pQueryHandle->rhelper.pDCols[0]; + SDataCols* pTsCol = pTsdbReadHandle->rhelper.pDCols[0]; if (pCheckInfo->lastKey < pBlock->keyLast) { - cur->pos = binarySearchForKey(pTsCol->cols[0].pData, pBlock->numOfRows, pCheckInfo->lastKey, pQueryHandle->order); + cur->pos = binarySearchForKey(pTsCol->cols[0].pData, pBlock->numOfRows, pCheckInfo->lastKey, pTsdbReadHandle->order); } else { cur->pos = pBlock->numOfRows - 1; } assert(pCheckInfo->lastKey >= pBlock->keyFirst); - doMergeTwoLevelData(pQueryHandle, pCheckInfo, pBlock); + doMergeTwoLevelData(pTsdbReadHandle, pCheckInfo, pBlock); } else { cur->pos = asc? 0:(pBlock->numOfRows-1); - code = handleDataMergeIfNeeded(pQueryHandle, pBlock, pCheckInfo); + code = handleDataMergeIfNeeded(pTsdbReadHandle, pBlock, pCheckInfo); } } - *exists = pQueryHandle->realNumOfRows > 0; + *exists = pTsdbReadHandle->realNumOfRows > 0; return code; } @@ -1456,11 +1439,11 @@ static int doBinarySearchKey(char* pValue, int num, TSKEY key, int order) { return midPos; } -static int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, int32_t start, int32_t end) { +static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t capacity, int32_t numOfRows, int32_t start, int32_t end) { char* pData = NULL; - int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1 : -1; + int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? 1 : -1; - SDataCols* pCols = pQueryHandle->rhelper.pDCols[0]; + SDataCols* pCols = pTsdbReadHandle->rhelper.pDCols[0]; TSKEY* tsArray = pCols->cols[0].pData; int32_t num = end - start + 1; @@ -1470,12 +1453,12 @@ static int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t c return numOfRows; } - int32_t requiredNumOfCols = (int32_t)taosArrayGetSize(pQueryHandle->pColumns); + int32_t requiredNumOfCols = (int32_t)taosArrayGetSize(pTsdbReadHandle->pColumns); //data in buffer has greater timestamp, copy data in file block int32_t i = 0, j = 0; while(i < requiredNumOfCols && j < pCols->numOfCols) { - SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); + SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); SDataCol* src = &pCols->cols[j]; if (src->colId < pColInfo->info.colId) { @@ -1485,7 +1468,7 @@ static int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t c int32_t bytes = pColInfo->info.bytes; - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { + if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; } else { pData = (char*)pColInfo->pData + (capacity - numOfRows - num) * pColInfo->info.bytes; @@ -1523,8 +1506,8 @@ static int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t c } while (i < requiredNumOfCols) { // the remain columns are all null data - SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { + SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); + if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; } else { pData = (char*)pColInfo->pData + (capacity - numOfRows - num) * pColInfo->info.bytes; @@ -1544,15 +1527,15 @@ static int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t c i++; } - pQueryHandle->cur.win.ekey = tsArray[end]; - pQueryHandle->cur.lastKey = tsArray[end] + step; + pTsdbReadHandle->cur.win.ekey = tsArray[end]; + pTsdbReadHandle->cur.lastKey = tsArray[end] + step; return numOfRows + num; } // Note: row1 always has high priority -static void mergeTwoRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfRows, - SMemRow row1, SMemRow row2, int32_t numOfCols, STable* pTable, +static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacity, int32_t numOfRows, + SMemRow row1, SMemRow row2, int32_t numOfCols, uint64_t uid, STSchema* pSchema1, STSchema* pSchema2, bool forceSetNull) { char* pData = NULL; STSchema* pSchema; @@ -1570,8 +1553,9 @@ static void mergeTwoRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t numOfColsOfRow1 = 0; if (pSchema1 == NULL) { - pSchema1 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row1)); + pSchema1 = metaGetTbTSchema(pTsdbReadHandle->pTsdb->pMeta, uid, 0); } + if(isRow1DataRow) { numOfColsOfRow1 = schemaNCols(pSchema1); } else { @@ -1582,7 +1566,7 @@ static void mergeTwoRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, if(row2) { isRow2DataRow = isDataRow(row2); if (pSchema2 == NULL) { - pSchema2 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row2)); + pSchema2 = metaGetTbTSchema(pTsdbReadHandle->pTsdb->pMeta, uid, 0); } if(isRow2DataRow) { numOfColsOfRow2 = schemaNCols(pSchema2); @@ -1594,9 +1578,9 @@ static void mergeTwoRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, int32_t i = 0, j = 0, k = 0; while(i < numOfCols && (j < numOfColsOfRow1 || k < numOfColsOfRow2)) { - SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); + SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { + if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; } else { pData = (char*)pColInfo->pData + (capacity - numOfRows - 1) * pColInfo->info.bytes; @@ -1699,7 +1683,7 @@ static void mergeTwoRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, SET_DOUBLE_PTR(pData, value); break; case TSDB_DATA_TYPE_TIMESTAMP: - if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { + if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) { *(TSKEY *)pData = tdGetKey(*(TKEY *)value); } else { *(TSKEY *)pData = *(TSKEY *)value; @@ -1730,8 +1714,8 @@ static void mergeTwoRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, if(forceSetNull) { while (i < numOfCols) { // the remain columns are all null data - SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { + SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); + if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; } else { pData = (char*)pColInfo->pData + (capacity - numOfRows - 1) * pColInfo->info.bytes; @@ -1748,29 +1732,29 @@ static void mergeTwoRowFromMem(STsdbQueryHandle* pQueryHandle, int32_t capacity, } } -static void moveDataToFront(STsdbQueryHandle* pQueryHandle, int32_t numOfRows, int32_t numOfCols) { - if (numOfRows == 0 || ASCENDING_TRAVERSE(pQueryHandle->order)) { +static void moveDataToFront(STsdbReadHandle* pTsdbReadHandle, int32_t numOfRows, int32_t numOfCols) { + if (numOfRows == 0 || ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { return; } // if the buffer is not full in case of descending order query, move the data in the front of the buffer - if (numOfRows < pQueryHandle->outputCapacity) { - int32_t emptySize = pQueryHandle->outputCapacity - numOfRows; + if (numOfRows < pTsdbReadHandle->outputCapacity) { + int32_t emptySize = pTsdbReadHandle->outputCapacity - numOfRows; for(int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); + SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); memmove((char*)pColInfo->pData, (char*)pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes); } } } -static void getQualifiedRowsPos(STsdbQueryHandle* pQueryHandle, int32_t startPos, int32_t endPos, int32_t numOfExisted, +static void getQualifiedRowsPos(STsdbReadHandle* pTsdbReadHandle, int32_t startPos, int32_t endPos, int32_t numOfExisted, int32_t* start, int32_t* end) { *start = -1; - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { + if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { int32_t remain = endPos - startPos + 1; - if (remain + numOfExisted > pQueryHandle->outputCapacity) { - *end = (pQueryHandle->outputCapacity - numOfExisted) + startPos - 1; + if (remain + numOfExisted > pTsdbReadHandle->outputCapacity) { + *end = (pTsdbReadHandle->outputCapacity - numOfExisted) + startPos - 1; } else { *end = endPos; } @@ -1778,8 +1762,8 @@ static void getQualifiedRowsPos(STsdbQueryHandle* pQueryHandle, int32_t startPos *start = startPos; } else { int32_t remain = (startPos - endPos) + 1; - if (remain + numOfExisted > pQueryHandle->outputCapacity) { - *end = startPos + 1 - (pQueryHandle->outputCapacity - numOfExisted); + if (remain + numOfExisted > pTsdbReadHandle->outputCapacity) { + *end = startPos + 1 - (pTsdbReadHandle->outputCapacity - numOfExisted); } else { *end = endPos; } @@ -1789,55 +1773,55 @@ static void getQualifiedRowsPos(STsdbQueryHandle* pQueryHandle, int32_t startPos } } -static void updateInfoAfterMerge(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, int32_t numOfRows, int32_t endPos) { - SQueryFilePos* cur = &pQueryHandle->cur; +static void updateInfoAfterMerge(STsdbReadHandle* pTsdbReadHandle, STableCheckInfo* pCheckInfo, int32_t numOfRows, int32_t endPos) { + SQueryFilePos* cur = &pTsdbReadHandle->cur; pCheckInfo->lastKey = cur->lastKey; - pQueryHandle->realNumOfRows = numOfRows; + pTsdbReadHandle->realNumOfRows = numOfRows; cur->rows = numOfRows; cur->pos = endPos; } -static void doCheckGeneratedBlockRange(STsdbQueryHandle* pQueryHandle) { - SQueryFilePos* cur = &pQueryHandle->cur; +static void doCheckGeneratedBlockRange(STsdbReadHandle* pTsdbReadHandle) { + SQueryFilePos* cur = &pTsdbReadHandle->cur; if (cur->rows > 0) { - if (ASCENDING_TRAVERSE(pQueryHandle->order)) { - assert(cur->win.skey >= pQueryHandle->window.skey && cur->win.ekey <= pQueryHandle->window.ekey); + if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { + assert(cur->win.skey >= pTsdbReadHandle->window.skey && cur->win.ekey <= pTsdbReadHandle->window.ekey); } else { - assert(cur->win.skey >= pQueryHandle->window.ekey && cur->win.ekey <= pQueryHandle->window.skey); + assert(cur->win.skey >= pTsdbReadHandle->window.ekey && cur->win.ekey <= pTsdbReadHandle->window.skey); } - SColumnInfoData* pColInfoData = taosArrayGet(pQueryHandle->pColumns, 0); + SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pColumns, 0); assert(cur->win.skey == ((TSKEY*)pColInfoData->pData)[0] && cur->win.ekey == ((TSKEY*)pColInfoData->pData)[cur->rows-1]); } else { - cur->win = pQueryHandle->window; + cur->win = pTsdbReadHandle->window; - int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1; - cur->lastKey = pQueryHandle->window.ekey + step; + int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? 1:-1; + cur->lastKey = pTsdbReadHandle->window.ekey + step; } } -static void copyAllRemainRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SDataBlockInfo* pBlockInfo, int32_t endPos) { - SQueryFilePos* cur = &pQueryHandle->cur; +static void copyAllRemainRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, STableCheckInfo* pCheckInfo, SDataBlockInfo* pBlockInfo, int32_t endPos) { + SQueryFilePos* cur = &pTsdbReadHandle->cur; - SDataCols* pCols = pQueryHandle->rhelper.pDCols[0]; + SDataCols* pCols = pTsdbReadHandle->rhelper.pDCols[0]; TSKEY* tsArray = pCols->cols[0].pData; - int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1; - int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pQueryHandle)); + int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? 1:-1; + int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pTsdbReadHandle)); int32_t pos = cur->pos; int32_t start = cur->pos; int32_t end = endPos; - if (!ASCENDING_TRAVERSE(pQueryHandle->order)) { + if (!ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { SWAP(start, end, int32_t); } - assert(pQueryHandle->outputCapacity >= (end - start + 1)); - int32_t numOfRows = doCopyRowsFromFileBlock(pQueryHandle, pQueryHandle->outputCapacity, 0, start, end); + assert(pTsdbReadHandle->outputCapacity >= (end - start + 1)); + int32_t numOfRows = doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, 0, start, end); // the time window should always be ascending order: skey <= ekey cur->win = (STimeWindow) {.skey = tsArray[start], .ekey = tsArray[end]}; @@ -1846,35 +1830,34 @@ static void copyAllRemainRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, STabl cur->blockCompleted = true; // if the buffer is not full in case of descending order query, move the data in the front of the buffer - moveDataToFront(pQueryHandle, numOfRows, numOfCols); + moveDataToFront(pTsdbReadHandle, numOfRows, numOfCols); // The value of pos may be -1 or pBlockInfo->rows, and it is invalid in both cases. pos = endPos + step; - updateInfoAfterMerge(pQueryHandle, pCheckInfo, numOfRows, pos); - doCheckGeneratedBlockRange(pQueryHandle); + updateInfoAfterMerge(pTsdbReadHandle, pCheckInfo, numOfRows, pos); + doCheckGeneratedBlockRange(pTsdbReadHandle); - tsdbDebug("%p uid:%" PRIu64",tid:%d data block created, mixblock:%d, brange:%"PRIu64"-%"PRIu64" rows:%d, 0x%"PRIx64, - pQueryHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, cur->mixBlock, cur->win.skey, - cur->win.ekey, cur->rows, pQueryHandle->qId); + tsdbDebug("%p uid:%" PRIu64", data block created, mixblock:%d, brange:%"PRIu64"-%"PRIu64" rows:%d, 0x%"PRIx64, + pTsdbReadHandle, pCheckInfo->tableId, cur->mixBlock, cur->win.skey, cur->win.ekey, cur->rows, pTsdbReadHandle->qId); } -int32_t getEndPosInDataBlock(STsdbQueryHandle* pQueryHandle, SDataBlockInfo* pBlockInfo) { +int32_t getEndPosInDataBlock(STsdbReadHandle* pTsdbReadHandle, SDataBlockInfo* pBlockInfo) { // NOTE: reverse the order to find the end position in data block int32_t endPos = -1; - int32_t order = ASCENDING_TRAVERSE(pQueryHandle->order)? TSDB_ORDER_DESC : TSDB_ORDER_ASC; + int32_t order = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? TSDB_ORDER_DESC : TSDB_ORDER_ASC; - SQueryFilePos* cur = &pQueryHandle->cur; - SDataCols* pCols = pQueryHandle->rhelper.pDCols[0]; + SQueryFilePos* cur = &pTsdbReadHandle->cur; + SDataCols* pCols = pTsdbReadHandle->rhelper.pDCols[0]; - if (ASCENDING_TRAVERSE(pQueryHandle->order) && pQueryHandle->window.ekey >= pBlockInfo->window.ekey) { + if (ASCENDING_TRAVERSE(pTsdbReadHandle->order) && pTsdbReadHandle->window.ekey >= pBlockInfo->window.ekey) { endPos = pBlockInfo->rows - 1; cur->mixBlock = (cur->pos != 0); - } else if (!ASCENDING_TRAVERSE(pQueryHandle->order) && pQueryHandle->window.ekey <= pBlockInfo->window.skey) { + } else if (!ASCENDING_TRAVERSE(pTsdbReadHandle->order) && pTsdbReadHandle->window.ekey <= pBlockInfo->window.skey) { endPos = 0; cur->mixBlock = (cur->pos != pBlockInfo->rows - 1); } else { assert(pCols->numOfRows > 0); - endPos = doBinarySearchKey(pCols->cols[0].pData, pCols->numOfRows, pQueryHandle->window.ekey, order); + endPos = doBinarySearchKey(pCols->cols[0].pData, pCols->numOfRows, pTsdbReadHandle->window.ekey, order); cur->mixBlock = true; } @@ -1883,33 +1866,33 @@ int32_t getEndPosInDataBlock(STsdbQueryHandle* pQueryHandle, SDataBlockInfo* pBl // only return the qualified data to client in terms of query time window, data rows in the same block but do not // be included in the query time window will be discarded -static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SBlock* pBlock) { - SQueryFilePos* cur = &pQueryHandle->cur; - SDataBlockInfo blockInfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock); - STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; +static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInfo* pCheckInfo, SBlock* pBlock) { + SQueryFilePos* cur = &pTsdbReadHandle->cur; + SDataBlockInfo blockInfo = {0};//GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock); + STsdbCfg* pCfg = &pTsdbReadHandle->pTsdb->config; - initTableMemIterator(pQueryHandle, pCheckInfo); + initTableMemIterator(pTsdbReadHandle, pCheckInfo); - SDataCols* pCols = pQueryHandle->rhelper.pDCols[0]; - assert(pCols->cols[0].type == TSDB_DATA_TYPE_TIMESTAMP && pCols->cols[0].colId == PRIMARYKEY_TIMESTAMP_COL_INDEX && + SDataCols* pCols = pTsdbReadHandle->rhelper.pDCols[0]; + assert(pCols->cols[0].type == TSDB_DATA_TYPE_TIMESTAMP && pCols->cols[0].colId == PRIMARYKEY_TIMESTAMP_COL_ID && cur->pos >= 0 && cur->pos < pBlock->numOfRows); TSKEY* tsArray = pCols->cols[0].pData; assert(pCols->numOfRows == pBlock->numOfRows && tsArray[0] == pBlock->keyFirst && tsArray[pBlock->numOfRows-1] == pBlock->keyLast); // for search the endPos, so the order needs to reverse - int32_t order = (pQueryHandle->order == TSDB_ORDER_ASC)? TSDB_ORDER_DESC:TSDB_ORDER_ASC; + int32_t order = (pTsdbReadHandle->order == TSDB_ORDER_ASC)? TSDB_ORDER_DESC:TSDB_ORDER_ASC; - int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1; - int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pQueryHandle)); + int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? 1:-1; + int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pTsdbReadHandle)); STable* pTable = pCheckInfo->pTableObj; - int32_t endPos = getEndPosInDataBlock(pQueryHandle, &blockInfo); + int32_t endPos = getEndPosInDataBlock(pTsdbReadHandle, &blockInfo); - tsdbDebug("%p uid:%" PRIu64",tid:%d start merge data block, file block range:%"PRIu64"-%"PRIu64" rows:%d, start:%d," + tsdbDebug("%p uid:%" PRIu64" start merge data block, file block range:%"PRIu64"-%"PRIu64" rows:%d, start:%d," "end:%d, 0x%"PRIx64, - pQueryHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, blockInfo.window.skey, blockInfo.window.ekey, - blockInfo.rows, cur->pos, endPos, pQueryHandle->qId); + pTsdbReadHandle, pCheckInfo->tableId, blockInfo.window.skey, blockInfo.window.ekey, + blockInfo.rows, cur->pos, endPos, pTsdbReadHandle->qId); // compared with the data from in-memory buffer, to generate the correct timestamp array list int32_t numOfRows = 0; @@ -1924,40 +1907,40 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* // no data in buffer, load data from file directly if (pCheckInfo->iiter == NULL && pCheckInfo->iter == NULL) { - copyAllRemainRowsFromFileBlock(pQueryHandle, pCheckInfo, &blockInfo, endPos); + copyAllRemainRowsFromFileBlock(pTsdbReadHandle, pCheckInfo, &blockInfo, endPos); return; } else if (pCheckInfo->iter != NULL || pCheckInfo->iiter != NULL) { SSkipListNode* node = NULL; do { SMemRow row2 = NULL; - SMemRow row1 = getSMemRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update, &row2); + SMemRow row1 = getSMemRowInTableMem(pCheckInfo, pTsdbReadHandle->order, pCfg->update, &row2); if (row1 == NULL) { break; } TSKEY key = memRowKey(row1); - if ((key > pQueryHandle->window.ekey && ASCENDING_TRAVERSE(pQueryHandle->order)) || - (key < pQueryHandle->window.ekey && !ASCENDING_TRAVERSE(pQueryHandle->order))) { + if ((key > pTsdbReadHandle->window.ekey && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || + (key < pTsdbReadHandle->window.ekey && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) { break; } - if (((pos > endPos || tsArray[pos] > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) || - ((pos < endPos || tsArray[pos] < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order))) { + if (((pos > endPos || tsArray[pos] > pTsdbReadHandle->window.ekey) && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || + ((pos < endPos || tsArray[pos] < pTsdbReadHandle->window.ekey) && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) { break; } - if ((key < tsArray[pos] && ASCENDING_TRAVERSE(pQueryHandle->order)) || - (key > tsArray[pos] && !ASCENDING_TRAVERSE(pQueryHandle->order))) { + if ((key < tsArray[pos] && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || + (key > tsArray[pos] && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) { if (rv1 != memRowVersion(row1)) { - pSchema1 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row1)); +// pSchema1 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row1)); rv1 = memRowVersion(row1); } if(row2 && rv2 != memRowVersion(row2)) { - pSchema2 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row2)); +// pSchema2 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row2)); rv2 = memRowVersion(row2); } - mergeTwoRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row1, row2, numOfCols, pTable, pSchema1, pSchema2, true); + mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, numOfRows, row1, row2, numOfCols, pTable, pSchema1, pSchema2, true); numOfRows += 1; if (cur->win.skey == TSKEY_INITIAL_VAL) { cur->win.skey = key; @@ -1971,19 +1954,19 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* } else if (key == tsArray[pos]) { // data in buffer has the same timestamp of data in file block, ignore it if (pCfg->update) { if(pCfg->update == TD_ROW_PARTIAL_UPDATE) { - doCopyRowsFromFileBlock(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, pos, pos); + doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, numOfRows, pos, pos); } if (rv1 != memRowVersion(row1)) { - pSchema1 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row1)); +// pSchema1 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row1)); rv1 = memRowVersion(row1); } if(row2 && rv2 != memRowVersion(row2)) { - pSchema2 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row2)); +// pSchema2 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row2)); rv2 = memRowVersion(row2); } bool forceSetNull = pCfg->update != TD_ROW_PARTIAL_UPDATE; - mergeTwoRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, row1, row2, numOfCols, pTable, pSchema1, pSchema2, forceSetNull); + mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, numOfRows, row1, row2, numOfCols, pTable, pSchema1, pSchema2, forceSetNull); numOfRows += 1; if (cur->win.skey == TSKEY_INITIAL_VAL) { cur->win.skey = key; @@ -1998,8 +1981,8 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* } else { moveToNextRowInMem(pCheckInfo); } - } else if ((key > tsArray[pos] && ASCENDING_TRAVERSE(pQueryHandle->order)) || - (key < tsArray[pos] && !ASCENDING_TRAVERSE(pQueryHandle->order))) { + } else if ((key > tsArray[pos] && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || + (key < tsArray[pos] && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) { if (cur->win.skey == TSKEY_INITIAL_VAL) { cur->win.skey = tsArray[pos]; } @@ -2016,38 +1999,38 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* } int32_t qstart = 0, qend = 0; - getQualifiedRowsPos(pQueryHandle, pos, end, numOfRows, &qstart, &qend); + getQualifiedRowsPos(pTsdbReadHandle, pos, end, numOfRows, &qstart, &qend); - numOfRows = doCopyRowsFromFileBlock(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, qstart, qend); + numOfRows = doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, numOfRows, qstart, qend); pos += (qend - qstart + 1) * step; - cur->win.ekey = ASCENDING_TRAVERSE(pQueryHandle->order)? tsArray[qend]:tsArray[qstart]; + cur->win.ekey = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? tsArray[qend]:tsArray[qstart]; cur->lastKey = cur->win.ekey + step; } - } while (numOfRows < pQueryHandle->outputCapacity); + } while (numOfRows < pTsdbReadHandle->outputCapacity); - if (numOfRows < pQueryHandle->outputCapacity) { + if (numOfRows < pTsdbReadHandle->outputCapacity) { /** * if cache is empty, load remain file block data. In contrast, if there are remain data in cache, do NOT * copy them all to result buffer, since it may be overlapped with file data block. */ if (node == NULL || - ((memRowKey((SMemRow)SL_GET_NODE_DATA(node)) > pQueryHandle->window.ekey) && - ASCENDING_TRAVERSE(pQueryHandle->order)) || - ((memRowKey((SMemRow)SL_GET_NODE_DATA(node)) < pQueryHandle->window.ekey) && - !ASCENDING_TRAVERSE(pQueryHandle->order))) { + ((memRowKey((SMemRow)SL_GET_NODE_DATA(node)) > pTsdbReadHandle->window.ekey) && + ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || + ((memRowKey((SMemRow)SL_GET_NODE_DATA(node)) < pTsdbReadHandle->window.ekey) && + !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) { // no data in cache or data in cache is greater than the ekey of time window, load data from file block if (cur->win.skey == TSKEY_INITIAL_VAL) { cur->win.skey = tsArray[pos]; } int32_t start = -1, end = -1; - getQualifiedRowsPos(pQueryHandle, pos, endPos, numOfRows, &start, &end); + getQualifiedRowsPos(pTsdbReadHandle, pos, endPos, numOfRows, &start, &end); - numOfRows = doCopyRowsFromFileBlock(pQueryHandle, pQueryHandle->outputCapacity, numOfRows, start, end); + numOfRows = doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, numOfRows, start, end); pos += (end - start + 1) * step; - cur->win.ekey = ASCENDING_TRAVERSE(pQueryHandle->order)? tsArray[end]:tsArray[start]; + cur->win.ekey = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? tsArray[end]:tsArray[start]; cur->lastKey = cur->win.ekey + step; cur->mixBlock = true; } @@ -2055,20 +2038,19 @@ static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* } cur->blockCompleted = - (((pos > endPos || cur->lastKey > pQueryHandle->window.ekey) && ASCENDING_TRAVERSE(pQueryHandle->order)) || - ((pos < endPos || cur->lastKey < pQueryHandle->window.ekey) && !ASCENDING_TRAVERSE(pQueryHandle->order))); + (((pos > endPos || cur->lastKey > pTsdbReadHandle->window.ekey) && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || + ((pos < endPos || cur->lastKey < pTsdbReadHandle->window.ekey) && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))); - if (!ASCENDING_TRAVERSE(pQueryHandle->order)) { + if (!ASCENDING_TRAVERSE(pTsdbReadHandle->order)) { SWAP(cur->win.skey, cur->win.ekey, TSKEY); } - moveDataToFront(pQueryHandle, numOfRows, numOfCols); - updateInfoAfterMerge(pQueryHandle, pCheckInfo, numOfRows, pos); - doCheckGeneratedBlockRange(pQueryHandle); + moveDataToFront(pTsdbReadHandle, numOfRows, numOfCols); + updateInfoAfterMerge(pTsdbReadHandle, pCheckInfo, numOfRows, pos); + doCheckGeneratedBlockRange(pTsdbReadHandle); - tsdbDebug("%p uid:%" PRIu64",tid:%d data block created, mixblock:%d, brange:%"PRIu64"-%"PRIu64" rows:%d, 0x%"PRIx64, - pQueryHandle, pCheckInfo->tableId.uid, pCheckInfo->tableId.tid, cur->mixBlock, cur->win.skey, - cur->win.ekey, cur->rows, pQueryHandle->qId); + tsdbDebug("%p uid:%" PRIu64", data block created, mixblock:%d, brange:%"PRIu64"-%"PRIu64" rows:%d, 0x%"PRIx64, + pTsdbReadHandle, pCheckInfo->tableId, cur->mixBlock, cur->win.skey, cur->win.ekey, cur->rows, pTsdbReadHandle->qId); } int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order) { @@ -2174,24 +2156,24 @@ static int32_t dataBlockOrderCompar(const void* pLeft, const void* pRight, void* return pLeftBlockInfoEx->compBlock->offset > pRightBlockInfoEx->compBlock->offset ? 1 : -1; } -static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numOfBlocks, int32_t* numOfAllocBlocks) { +static int32_t createDataBlocksInfo(STsdbReadHandle* pTsdbReadHandle, int32_t numOfBlocks, int32_t* numOfAllocBlocks) { size_t size = sizeof(STableBlockInfo) * numOfBlocks; - if (pQueryHandle->allocSize < size) { - pQueryHandle->allocSize = (int32_t)size; - char* tmp = realloc(pQueryHandle->pDataBlockInfo, pQueryHandle->allocSize); + if (pTsdbReadHandle->allocSize < size) { + pTsdbReadHandle->allocSize = (int32_t)size; + char* tmp = realloc(pTsdbReadHandle->pDataBlockInfo, pTsdbReadHandle->allocSize); if (tmp == NULL) { return TSDB_CODE_TDB_OUT_OF_MEMORY; } - pQueryHandle->pDataBlockInfo = (STableBlockInfo*) tmp; + pTsdbReadHandle->pDataBlockInfo = (STableBlockInfo*) tmp; } - memset(pQueryHandle->pDataBlockInfo, 0, size); + memset(pTsdbReadHandle->pDataBlockInfo, 0, size); *numOfAllocBlocks = numOfBlocks; // access data blocks according to the offset of each block in asc/desc order. - int32_t numOfTables = (int32_t)taosArrayGetSize(pQueryHandle->pTableCheckInfo); + int32_t numOfTables = (int32_t)taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); SBlockOrderSupporter sup = {0}; sup.numOfTables = numOfTables; @@ -2208,7 +2190,7 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO int32_t numOfQualTables = 0; for (int32_t j = 0; j < numOfTables; ++j) { - STableCheckInfo* pTableCheck = (STableCheckInfo*)taosArrayGet(pQueryHandle->pTableCheckInfo, j); + STableCheckInfo* pTableCheck = (STableCheckInfo*)taosArrayGet(pTsdbReadHandle->pTableCheckInfo, j); if (pTableCheck->numOfBlocks <= 0) { continue; } @@ -2239,16 +2221,16 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO // since there is only one table qualified, blocks are not sorted if (numOfQualTables == 1) { - memcpy(pQueryHandle->pDataBlockInfo, sup.pDataBlockInfo[0], sizeof(STableBlockInfo) * numOfBlocks); + memcpy(pTsdbReadHandle->pDataBlockInfo, sup.pDataBlockInfo[0], sizeof(STableBlockInfo) * numOfBlocks); cleanBlockOrderSupporter(&sup, numOfQualTables); - tsdbDebug("%p create data blocks info struct completed for 1 table, %d blocks not sorted 0x%"PRIx64, pQueryHandle, cnt, - pQueryHandle->qId); + tsdbDebug("%p create data blocks info struct completed for 1 table, %d blocks not sorted 0x%"PRIx64, pTsdbReadHandle, cnt, + pTsdbReadHandle->qId); return TSDB_CODE_SUCCESS; } - tsdbDebug("%p create data blocks info struct completed, %d blocks in %d tables 0x%"PRIx64, pQueryHandle, cnt, - numOfQualTables, pQueryHandle->qId); + tsdbDebug("%p create data blocks info struct completed, %d blocks in %d tables 0x%"PRIx64, pTsdbReadHandle, cnt, + numOfQualTables, pTsdbReadHandle->qId); assert(cnt <= numOfBlocks && numOfQualTables <= numOfTables); // the pTableQueryInfo[j]->numOfBlocks may be 0 sup.numOfTables = numOfQualTables; @@ -2267,7 +2249,7 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO int32_t index = sup.blockIndexArray[pos]++; STableBlockInfo* pBlocksInfo = sup.pDataBlockInfo[pos]; - pQueryHandle->pDataBlockInfo[numOfTotal++] = pBlocksInfo[index]; + pTsdbReadHandle->pDataBlockInfo[numOfTotal++] = pBlocksInfo[index]; // set data block index overflow, in order to disable the offset comparator if (sup.blockIndexArray[pos] >= sup.numOfBlocksPerTable[pos]) { @@ -2284,90 +2266,90 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO * } */ - tsdbDebug("%p %d data blocks sort completed, 0x%"PRIx64, pQueryHandle, cnt, pQueryHandle->qId); + tsdbDebug("%p %d data blocks sort completed, 0x%"PRIx64, pTsdbReadHandle, cnt, pTsdbReadHandle->qId); cleanBlockOrderSupporter(&sup, numOfTables); free(pTree); return TSDB_CODE_SUCCESS; } -static int32_t getFirstFileDataBlock(STsdbQueryHandle* pQueryHandle, bool* exists); +static int32_t getFirstFileDataBlock(STsdbReadHandle* pTsdbReadHandle, bool* exists); -static int32_t getDataBlockRv(STsdbQueryHandle* pQueryHandle, STableBlockInfo* pNext, bool *exists) { - int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1 : -1; - SQueryFilePos* cur = &pQueryHandle->cur; +static int32_t getDataBlockRv(STsdbReadHandle* pTsdbReadHandle, STableBlockInfo* pNext, bool *exists) { + int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? 1 : -1; + SQueryFilePos* cur = &pTsdbReadHandle->cur; while(1) { - int32_t code = loadFileDataBlock(pQueryHandle, pNext->compBlock, pNext->pTableCheckInfo, exists); + int32_t code = loadFileDataBlock(pTsdbReadHandle, pNext->compBlock, pNext->pTableCheckInfo, exists); if (code != TSDB_CODE_SUCCESS || *exists) { return code; } - if ((cur->slot == pQueryHandle->numOfBlocks - 1 && ASCENDING_TRAVERSE(pQueryHandle->order)) || - (cur->slot == 0 && !ASCENDING_TRAVERSE(pQueryHandle->order))) { + if ((cur->slot == pTsdbReadHandle->numOfBlocks - 1 && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || + (cur->slot == 0 && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) { // all data blocks in current file has been checked already, try next file if exists - return getFirstFileDataBlock(pQueryHandle, exists); + return getFirstFileDataBlock(pTsdbReadHandle, exists); } else { // next block of the same file cur->slot += step; cur->mixBlock = false; cur->blockCompleted = false; - pNext = &pQueryHandle->pDataBlockInfo[cur->slot]; + pNext = &pTsdbReadHandle->pDataBlockInfo[cur->slot]; } } } -static int32_t getFirstFileDataBlock(STsdbQueryHandle* pQueryHandle, bool* exists) { - pQueryHandle->numOfBlocks = 0; - SQueryFilePos* cur = &pQueryHandle->cur; +static int32_t getFirstFileDataBlock(STsdbReadHandle* pTsdbReadHandle, bool* exists) { + pTsdbReadHandle->numOfBlocks = 0; + SQueryFilePos* cur = &pTsdbReadHandle->cur; int32_t code = TSDB_CODE_SUCCESS; int32_t numOfBlocks = 0; - int32_t numOfTables = (int32_t)taosArrayGetSize(pQueryHandle->pTableCheckInfo); + int32_t numOfTables = (int32_t)taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); - STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; + STsdbCfg* pCfg = &pTsdbReadHandle->pTsdb->config; STimeWindow win = TSWINDOW_INITIALIZER; while (true) { - tsdbRLockFS(REPO_FS(pQueryHandle->pTsdb)); + tsdbRLockFS(REPO_FS(pTsdbReadHandle->pTsdb)); - if ((pQueryHandle->pFileGroup = tsdbFSIterNext(&pQueryHandle->fileIter)) == NULL) { - tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); + if ((pTsdbReadHandle->pFileGroup = tsdbFSIterNext(&pTsdbReadHandle->fileIter)) == NULL) { + tsdbUnLockFS(REPO_FS(pTsdbReadHandle->pTsdb)); break; } - tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, pQueryHandle->pFileGroup->fid, &win.skey, &win.ekey); + tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, pTsdbReadHandle->pFileGroup->fid, &win.skey, &win.ekey); // current file are not overlapped with query time window, ignore remain files - if ((ASCENDING_TRAVERSE(pQueryHandle->order) && win.skey > pQueryHandle->window.ekey) || - (!ASCENDING_TRAVERSE(pQueryHandle->order) && win.ekey < pQueryHandle->window.ekey)) { - tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); - tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, 0x%"PRIx64, pQueryHandle, - pQueryHandle->window.skey, pQueryHandle->window.ekey, pQueryHandle->qId); - pQueryHandle->pFileGroup = NULL; - assert(pQueryHandle->numOfBlocks == 0); + if ((ASCENDING_TRAVERSE(pTsdbReadHandle->order) && win.skey > pTsdbReadHandle->window.ekey) || + (!ASCENDING_TRAVERSE(pTsdbReadHandle->order) && win.ekey < pTsdbReadHandle->window.ekey)) { + tsdbUnLockFS(REPO_FS(pTsdbReadHandle->pTsdb)); + tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, 0x%"PRIx64, pTsdbReadHandle, + pTsdbReadHandle->window.skey, pTsdbReadHandle->window.ekey, pTsdbReadHandle->qId); + pTsdbReadHandle->pFileGroup = NULL; + assert(pTsdbReadHandle->numOfBlocks == 0); break; } - if (tsdbSetAndOpenReadFSet(&pQueryHandle->rhelper, pQueryHandle->pFileGroup) < 0) { - tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); + if (tsdbSetAndOpenReadFSet(&pTsdbReadHandle->rhelper, pTsdbReadHandle->pFileGroup) < 0) { + tsdbUnLockFS(REPO_FS(pTsdbReadHandle->pTsdb)); code = terrno; break; } - tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); + tsdbUnLockFS(REPO_FS(pTsdbReadHandle->pTsdb)); - if (tsdbLoadBlockIdx(&pQueryHandle->rhelper) < 0) { + if (tsdbLoadBlockIdx(&pTsdbReadHandle->rhelper) < 0) { code = terrno; break; } - if ((code = getFileCompInfo(pQueryHandle, &numOfBlocks)) != TSDB_CODE_SUCCESS) { + if ((code = getFileCompInfo(pTsdbReadHandle, &numOfBlocks)) != TSDB_CODE_SUCCESS) { break; } - tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, 0x%"PRIx64, pQueryHandle, numOfBlocks, numOfTables, - pQueryHandle->pFileGroup->fid, pQueryHandle->qId); + tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, 0x%"PRIx64, pTsdbReadHandle, numOfBlocks, numOfTables, + pTsdbReadHandle->pFileGroup->fid, pTsdbReadHandle->qId); assert(numOfBlocks >= 0); if (numOfBlocks == 0) { @@ -2375,20 +2357,20 @@ static int32_t getFirstFileDataBlock(STsdbQueryHandle* pQueryHandle, bool* exist } // todo return error code to query engine - if ((code = createDataBlocksInfo(pQueryHandle, numOfBlocks, &pQueryHandle->numOfBlocks)) != TSDB_CODE_SUCCESS) { + if ((code = createDataBlocksInfo(pTsdbReadHandle, numOfBlocks, &pTsdbReadHandle->numOfBlocks)) != TSDB_CODE_SUCCESS) { break; } - assert(numOfBlocks >= pQueryHandle->numOfBlocks); - if (pQueryHandle->numOfBlocks > 0) { + assert(numOfBlocks >= pTsdbReadHandle->numOfBlocks); + if (pTsdbReadHandle->numOfBlocks > 0) { break; } } // no data in file anymore - if (pQueryHandle->numOfBlocks <= 0 || code != TSDB_CODE_SUCCESS) { + if (pTsdbReadHandle->numOfBlocks <= 0 || code != TSDB_CODE_SUCCESS) { if (code == TSDB_CODE_SUCCESS) { - assert(pQueryHandle->pFileGroup == NULL); + assert(pTsdbReadHandle->pFileGroup == NULL); } cur->fid = INT32_MIN; // denote that there are no data in file anymore @@ -2396,12 +2378,12 @@ static int32_t getFirstFileDataBlock(STsdbQueryHandle* pQueryHandle, bool* exist return code; } - assert(pQueryHandle->pFileGroup != NULL && pQueryHandle->numOfBlocks > 0); - cur->slot = ASCENDING_TRAVERSE(pQueryHandle->order)? 0:pQueryHandle->numOfBlocks-1; - cur->fid = pQueryHandle->pFileGroup->fid; + assert(pTsdbReadHandle->pFileGroup != NULL && pTsdbReadHandle->numOfBlocks > 0); + cur->slot = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? 0:pTsdbReadHandle->numOfBlocks-1; + cur->fid = pTsdbReadHandle->pFileGroup->fid; - STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot]; - return getDataBlockRv(pQueryHandle, pBlockInfo, exists); + STableBlockInfo* pBlockInfo = &pTsdbReadHandle->pDataBlockInfo[cur->slot]; + return getDataBlockRv(pTsdbReadHandle, pBlockInfo, exists); } static bool isEndFileDataBlock(SQueryFilePos* cur, int32_t numOfBlocks, bool ascTrav) { @@ -2409,90 +2391,90 @@ static bool isEndFileDataBlock(SQueryFilePos* cur, int32_t numOfBlocks, bool asc return (cur->slot == numOfBlocks - 1 && ascTrav) || (cur->slot == 0 && !ascTrav); } -static void moveToNextDataBlockInCurrentFile(STsdbQueryHandle* pQueryHandle) { - int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1 : -1; +static void moveToNextDataBlockInCurrentFile(STsdbReadHandle* pTsdbReadHandle) { + int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? 1 : -1; - SQueryFilePos* cur = &pQueryHandle->cur; - assert(cur->slot < pQueryHandle->numOfBlocks && cur->slot >= 0); + SQueryFilePos* cur = &pTsdbReadHandle->cur; + assert(cur->slot < pTsdbReadHandle->numOfBlocks && cur->slot >= 0); cur->slot += step; cur->mixBlock = false; cur->blockCompleted = false; } - -int32_t tsdbGetFileBlocksDistInfo(TsdbQueryHandleT* queryHandle, STableBlockDist* pTableBlockInfo) { - STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) queryHandle; +#if 0 +int32_t tsdbGetFileBlocksDistInfo(tsdbReadHandleT* queryHandle, STableBlockDist* pTableBlockInfo) { + STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) queryHandle; pTableBlockInfo->totalSize = 0; pTableBlockInfo->totalRows = 0; - STsdbFS* pFileHandle = REPO_FS(pQueryHandle->pTsdb); + STsdbFS* pFileHandle = REPO_FS(pTsdbReadHandle->pTsdb); // find the start data block in file - pQueryHandle->locateStart = true; - STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; - int32_t fid = getFileIdFromKey(pQueryHandle->window.skey, pCfg->daysPerFile, pCfg->precision); + pTsdbReadHandle->locateStart = true; + STsdbCfg* pCfg = &pTsdbReadHandle->pTsdb->config; + int32_t fid = getFileIdFromKey(pTsdbReadHandle->window.skey, pCfg->daysPerFile, pCfg->precision); tsdbRLockFS(pFileHandle); - tsdbFSIterInit(&pQueryHandle->fileIter, pFileHandle, pQueryHandle->order); - tsdbFSIterSeek(&pQueryHandle->fileIter, fid); + tsdbFSIterInit(&pTsdbReadHandle->fileIter, pFileHandle, pTsdbReadHandle->order); + tsdbFSIterSeek(&pTsdbReadHandle->fileIter, fid); tsdbUnLockFS(pFileHandle); pTableBlockInfo->numOfFiles += 1; int32_t code = TSDB_CODE_SUCCESS; int32_t numOfBlocks = 0; - int32_t numOfTables = (int32_t)taosArrayGetSize(pQueryHandle->pTableCheckInfo); + int32_t numOfTables = (int32_t)taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); int defaultRows = TSDB_DEFAULT_BLOCK_ROWS(pCfg->maxRowsPerFileBlock); STimeWindow win = TSWINDOW_INITIALIZER; while (true) { numOfBlocks = 0; - tsdbRLockFS(REPO_FS(pQueryHandle->pTsdb)); + tsdbRLockFS(REPO_FS(pTsdbReadHandle->pTsdb)); - if ((pQueryHandle->pFileGroup = tsdbFSIterNext(&pQueryHandle->fileIter)) == NULL) { - tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); + if ((pTsdbReadHandle->pFileGroup = tsdbFSIterNext(&pTsdbReadHandle->fileIter)) == NULL) { + tsdbUnLockFS(REPO_FS(pTsdbReadHandle->pTsdb)); break; } - tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, pQueryHandle->pFileGroup->fid, &win.skey, &win.ekey); + tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, pTsdbReadHandle->pFileGroup->fid, &win.skey, &win.ekey); // current file are not overlapped with query time window, ignore remain files - if ((ASCENDING_TRAVERSE(pQueryHandle->order) && win.skey > pQueryHandle->window.ekey) || - (!ASCENDING_TRAVERSE(pQueryHandle->order) && win.ekey < pQueryHandle->window.ekey)) { - tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); - tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, 0x%"PRIx64, pQueryHandle, - pQueryHandle->window.skey, pQueryHandle->window.ekey, pQueryHandle->qId); - pQueryHandle->pFileGroup = NULL; + if ((ASCENDING_TRAVERSE(pTsdbReadHandle->order) && win.skey > pTsdbReadHandle->window.ekey) || + (!ASCENDING_TRAVERSE(pTsdbReadHandle->order) && win.ekey < pTsdbReadHandle->window.ekey)) { + tsdbUnLockFS(REPO_FS(pTsdbReadHandle->pTsdb)); + tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, 0x%"PRIx64, pTsdbReadHandle, + pTsdbReadHandle->window.skey, pTsdbReadHandle->window.ekey, pTsdbReadHandle->qId); + pTsdbReadHandle->pFileGroup = NULL; break; } pTableBlockInfo->numOfFiles += 1; - if (tsdbSetAndOpenReadFSet(&pQueryHandle->rhelper, pQueryHandle->pFileGroup) < 0) { - tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); + if (tsdbSetAndOpenReadFSet(&pTsdbReadHandle->rhelper, pTsdbReadHandle->pFileGroup) < 0) { + tsdbUnLockFS(REPO_FS(pTsdbReadHandle->pTsdb)); code = terrno; break; } - tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb)); + tsdbUnLockFS(REPO_FS(pTsdbReadHandle->pTsdb)); - if (tsdbLoadBlockIdx(&pQueryHandle->rhelper) < 0) { + if (tsdbLoadBlockIdx(&pTsdbReadHandle->rhelper) < 0) { code = terrno; break; } - if ((code = getFileCompInfo(pQueryHandle, &numOfBlocks)) != TSDB_CODE_SUCCESS) { + if ((code = getFileCompInfo(pTsdbReadHandle, &numOfBlocks)) != TSDB_CODE_SUCCESS) { break; } - tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, 0x%"PRIx64, pQueryHandle, numOfBlocks, numOfTables, - pQueryHandle->pFileGroup->fid, pQueryHandle->qId); + tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, 0x%"PRIx64, pTsdbReadHandle, numOfBlocks, numOfTables, + pTsdbReadHandle->pFileGroup->fid, pTsdbReadHandle->qId); if (numOfBlocks == 0) { continue; } for (int32_t i = 0; i < numOfTables; ++i) { - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); + STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, i); SBlock* pBlock = pCheckInfo->pCompInfo->blocks; for (int32_t j = 0; j < pCheckInfo->numOfBlocks; ++j) { @@ -2512,36 +2494,37 @@ int32_t tsdbGetFileBlocksDistInfo(TsdbQueryHandleT* queryHandle, STableBlockDist return code; } +#endif -static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists) { - STsdbFS* pFileHandle = REPO_FS(pQueryHandle->pTsdb); - SQueryFilePos* cur = &pQueryHandle->cur; +static int32_t getDataBlocksInFiles(STsdbReadHandle* pTsdbReadHandle, bool* exists) { + STsdbFS* pFileHandle = REPO_FS(pTsdbReadHandle->pTsdb); + SQueryFilePos* cur = &pTsdbReadHandle->cur; // find the start data block in file - if (!pQueryHandle->locateStart) { - pQueryHandle->locateStart = true; - STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; - int32_t fid = getFileIdFromKey(pQueryHandle->window.skey, pCfg->daysPerFile, pCfg->precision); + if (!pTsdbReadHandle->locateStart) { + pTsdbReadHandle->locateStart = true; + STsdbCfg* pCfg = &pTsdbReadHandle->pTsdb->config; + int32_t fid = getFileIdFromKey(pTsdbReadHandle->window.skey, pCfg->daysPerFile, pCfg->precision); tsdbRLockFS(pFileHandle); - tsdbFSIterInit(&pQueryHandle->fileIter, pFileHandle, pQueryHandle->order); - tsdbFSIterSeek(&pQueryHandle->fileIter, fid); + tsdbFSIterInit(&pTsdbReadHandle->fileIter, pFileHandle, pTsdbReadHandle->order); + tsdbFSIterSeek(&pTsdbReadHandle->fileIter, fid); tsdbUnLockFS(pFileHandle); - return getFirstFileDataBlock(pQueryHandle, exists); + return getFirstFileDataBlock(pTsdbReadHandle, exists); } else { // check if current file block is all consumed - STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot]; + STableBlockInfo* pBlockInfo = &pTsdbReadHandle->pDataBlockInfo[cur->slot]; STableCheckInfo* pCheckInfo = pBlockInfo->pTableCheckInfo; // current block is done, try next if ((!cur->mixBlock) || cur->blockCompleted) { // all data blocks in current file has been checked already, try next file if exists } else { - tsdbDebug("%p continue in current data block, index:%d, pos:%d, 0x%"PRIx64, pQueryHandle, cur->slot, cur->pos, - pQueryHandle->qId); - int32_t code = handleDataMergeIfNeeded(pQueryHandle, pBlockInfo->compBlock, pCheckInfo); - *exists = (pQueryHandle->realNumOfRows > 0); + tsdbDebug("%p continue in current data block, index:%d, pos:%d, 0x%"PRIx64, pTsdbReadHandle, cur->slot, cur->pos, + pTsdbReadHandle->qId); + int32_t code = handleDataMergeIfNeeded(pTsdbReadHandle, pBlockInfo->compBlock, pCheckInfo); + *exists = (pTsdbReadHandle->realNumOfRows > 0); if (code != TSDB_CODE_SUCCESS || *exists) { return code; @@ -2550,50 +2533,50 @@ static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists // current block is empty, try next block in file // all data blocks in current file has been checked already, try next file if exists - if (isEndFileDataBlock(cur, pQueryHandle->numOfBlocks, ASCENDING_TRAVERSE(pQueryHandle->order))) { - return getFirstFileDataBlock(pQueryHandle, exists); + if (isEndFileDataBlock(cur, pTsdbReadHandle->numOfBlocks, ASCENDING_TRAVERSE(pTsdbReadHandle->order))) { + return getFirstFileDataBlock(pTsdbReadHandle, exists); } else { - moveToNextDataBlockInCurrentFile(pQueryHandle); - STableBlockInfo* pNext = &pQueryHandle->pDataBlockInfo[cur->slot]; - return getDataBlockRv(pQueryHandle, pNext, exists); + moveToNextDataBlockInCurrentFile(pTsdbReadHandle); + STableBlockInfo* pNext = &pTsdbReadHandle->pDataBlockInfo[cur->slot]; + return getDataBlockRv(pTsdbReadHandle, pNext, exists); } } } -static bool doHasDataInBuffer(STsdbQueryHandle* pQueryHandle) { - size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); +static bool doHasDataInBuffer(STsdbReadHandle* pTsdbReadHandle) { + size_t numOfTables = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); - while (pQueryHandle->activeIndex < numOfTables) { - if (hasMoreDataInCache(pQueryHandle)) { + while (pTsdbReadHandle->activeIndex < numOfTables) { + if (hasMoreDataInCache(pTsdbReadHandle)) { return true; } - pQueryHandle->activeIndex += 1; + pTsdbReadHandle->activeIndex += 1; } // no data in memtable or imemtable, decrease the memory reference. // TODO !! -// tsdbMayUnTakeMemSnapshot(pQueryHandle); +// tsdbMayUnTakeMemSnapshot(pTsdbReadHandle); return false; } //todo not unref yet, since it is not support multi-group interpolation query -static UNUSED_FUNC void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle) { +static UNUSED_FUNC void changeQueryHandleForInterpQuery(tsdbReadHandleT pHandle) { // filter the queried time stamp in the first place - STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; + STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) pHandle; // starts from the buffer in case of descending timestamp order check data blocks - size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + size_t numOfTables = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); int32_t i = 0; while(i < numOfTables) { - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i); + STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, i); // the first qualified table for interpolation query - if ((pQueryHandle->window.skey <= pCheckInfo->pTableObj->lastKey) && - (pCheckInfo->pTableObj->lastKey != TSKEY_INITIAL_VAL)) { - break; - } +// if ((pTsdbReadHandle->window.skey <= pCheckInfo->pTableObj->lastKey) && +// (pCheckInfo->pTableObj->lastKey != TSKEY_INITIAL_VAL)) { +// break; +// } i++; } @@ -2603,35 +2586,34 @@ static UNUSED_FUNC void changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle return; } - STableCheckInfo info = *(STableCheckInfo*) taosArrayGet(pQueryHandle->pTableCheckInfo, i); - taosArrayClear(pQueryHandle->pTableCheckInfo); + STableCheckInfo info = *(STableCheckInfo*) taosArrayGet(pTsdbReadHandle->pTableCheckInfo, i); + taosArrayClear(pTsdbReadHandle->pTableCheckInfo); - info.lastKey = pQueryHandle->window.skey; - taosArrayPush(pQueryHandle->pTableCheckInfo, &info); + info.lastKey = pTsdbReadHandle->window.skey; + taosArrayPush(pTsdbReadHandle->pTableCheckInfo, &info); } static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win, - STsdbQueryHandle* pQueryHandle) { + STsdbReadHandle* pTsdbReadHandle) { int numOfRows = 0; - int32_t numOfCols = (int32_t)taosArrayGetSize(pQueryHandle->pColumns); - STsdbCfg* pCfg = &pQueryHandle->pTsdb->config; + int32_t numOfCols = (int32_t)taosArrayGetSize(pTsdbReadHandle->pColumns); + STsdbCfg* pCfg = &pTsdbReadHandle->pTsdb->config; win->skey = TSKEY_INITIAL_VAL; int64_t st = taosGetTimestampUs(); - STable* pTable = pCheckInfo->pTableObj; int16_t rv = -1; STSchema* pSchema = NULL; do { - SMemRow row = getSMemRowInTableMem(pCheckInfo, pQueryHandle->order, pCfg->update, NULL); + SMemRow row = getSMemRowInTableMem(pCheckInfo, pTsdbReadHandle->order, pCfg->update, NULL); if (row == NULL) { break; } TSKEY key = memRowKey(row); - if ((key > maxKey && ASCENDING_TRAVERSE(pQueryHandle->order)) || (key < maxKey && !ASCENDING_TRAVERSE(pQueryHandle->order))) { - tsdbDebug("%p key:%"PRIu64" beyond qrange:%"PRId64" - %"PRId64", no more data in buffer", pQueryHandle, key, pQueryHandle->window.skey, - pQueryHandle->window.ekey); + if ((key > maxKey && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || (key < maxKey && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) { + tsdbDebug("%p key:%"PRIu64" beyond qrange:%"PRId64" - %"PRId64", no more data in buffer", pTsdbReadHandle, key, pTsdbReadHandle->window.skey, + pTsdbReadHandle->window.ekey); break; } @@ -2642,10 +2624,10 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int win->ekey = key; if (rv != memRowVersion(row)) { - pSchema = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row)); + pSchema = metaGetTbTSchema(pTsdbReadHandle->pTsdb->pMeta, pCheckInfo->tableId, 0); rv = memRowVersion(row); } - mergeTwoRowFromMem(pQueryHandle, maxRowsToRead, numOfRows, row, NULL, numOfCols, pTable, pSchema, NULL, true); + mergeTwoRowFromMem(pTsdbReadHandle, maxRowsToRead, numOfRows, row, NULL, numOfCols, pCheckInfo->tableId, pSchema, NULL, true); if (++numOfRows >= maxRowsToRead) { moveToNextRowInMem(pCheckInfo); @@ -2657,24 +2639,24 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int assert(numOfRows <= maxRowsToRead); // if the buffer is not full in case of descending order query, move the data in the front of the buffer - if (!ASCENDING_TRAVERSE(pQueryHandle->order) && numOfRows < maxRowsToRead) { + if (!ASCENDING_TRAVERSE(pTsdbReadHandle->order) && numOfRows < maxRowsToRead) { int32_t emptySize = maxRowsToRead - numOfRows; for(int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, i); + SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); memmove((char*)pColInfo->pData, (char*)pColInfo->pData + emptySize * pColInfo->info.bytes, numOfRows * pColInfo->info.bytes); } } int64_t elapsedTime = taosGetTimestampUs() - st; - tsdbDebug("%p build data block from cache completed, elapsed time:%"PRId64" us, numOfRows:%d, numOfCols:%d, 0x%"PRIx64, pQueryHandle, - elapsedTime, numOfRows, numOfCols, pQueryHandle->qId); + tsdbDebug("%p build data block from cache completed, elapsed time:%"PRId64" us, numOfRows:%d, numOfCols:%d, 0x%"PRIx64, pTsdbReadHandle, + elapsedTime, numOfRows, numOfCols, pTsdbReadHandle->qId); return numOfRows; } static int32_t getAllTableList(STable* pSuperTable, SArray* list) { - SSkipListIterator* iter = tSkipListCreateIter(pSuperTable->pIndex); + SSkipListIterator* iter = NULL;//tSkipListCreateIter(pSuperTable->pIndex); while (tSkipListIterNext(iter)) { SSkipListNode* pNode = tSkipListIterGet(iter); @@ -2693,58 +2675,61 @@ static void destroyHelper(void* param) { return; } - tQueryInfo* pInfo = (tQueryInfo*)param; - if (pInfo->optr != TSDB_RELATION_IN) { - tfree(pInfo->q); - } else { - taosHashCleanup((SHashObj *)(pInfo->q)); - } +// tQueryInfo* pInfo = (tQueryInfo*)param; +// if (pInfo->optr != TSDB_RELATION_IN) { +// tfree(pInfo->q); +// } else { +// taosHashCleanup((SHashObj *)(pInfo->q)); +// } free(param); } -static bool loadBlockOfActiveTable(STsdbQueryHandle* pQueryHandle) { - if (pQueryHandle->checkFiles) { +#define TSDB_PREV_ROW 0x1 +#define TSDB_NEXT_ROW 0x2 + +static bool loadBlockOfActiveTable(STsdbReadHandle* pTsdbReadHandle) { + if (pTsdbReadHandle->checkFiles) { // check if the query range overlaps with the file data block bool exists = true; - int32_t code = getDataBlocksInFiles(pQueryHandle, &exists); + int32_t code = getDataBlocksInFiles(pTsdbReadHandle, &exists); if (code != TSDB_CODE_SUCCESS) { - pQueryHandle->checkFiles = false; + pTsdbReadHandle->checkFiles = false; return false; } if (exists) { - tsdbRetrieveDataBlock((TsdbQueryHandleT*) pQueryHandle, NULL); - if (pQueryHandle->currentLoadExternalRows && pQueryHandle->window.skey == pQueryHandle->window.ekey) { - SColumnInfoData* pColInfo = taosArrayGet(pQueryHandle->pColumns, 0); - assert(*(int64_t*)pColInfo->pData == pQueryHandle->window.skey); + tsdbRetrieveDataBlock((tsdbReadHandleT*) pTsdbReadHandle, NULL); + if (pTsdbReadHandle->currentLoadExternalRows && pTsdbReadHandle->window.skey == pTsdbReadHandle->window.ekey) { + SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, 0); + assert(*(int64_t*)pColInfo->pData == pTsdbReadHandle->window.skey); } - pQueryHandle->currentLoadExternalRows = false; // clear the flag, since the exact matched row is found. + pTsdbReadHandle->currentLoadExternalRows = false; // clear the flag, since the exact matched row is found. return exists; } - pQueryHandle->checkFiles = false; + pTsdbReadHandle->checkFiles = false; } - if (hasMoreDataInCache(pQueryHandle)) { - pQueryHandle->currentLoadExternalRows = false; + if (hasMoreDataInCache(pTsdbReadHandle)) { + pTsdbReadHandle->currentLoadExternalRows = false; return true; } // current result is empty - if (pQueryHandle->currentLoadExternalRows && pQueryHandle->window.skey == pQueryHandle->window.ekey && pQueryHandle->cur.rows == 0) { - SMemRef* pMemRef = pQueryHandle->pMemRef; + if (pTsdbReadHandle->currentLoadExternalRows && pTsdbReadHandle->window.skey == pTsdbReadHandle->window.ekey && pTsdbReadHandle->cur.rows == 0) { +// STsdbMemTable* pMemRef = pTsdbReadHandle->pMemTable; - doGetExternalRow(pQueryHandle, TSDB_PREV_ROW, pMemRef); - doGetExternalRow(pQueryHandle, TSDB_NEXT_ROW, pMemRef); +// doGetExternalRow(pTsdbReadHandle, TSDB_PREV_ROW, pMemRef); +// doGetExternalRow(pTsdbReadHandle, TSDB_NEXT_ROW, pMemRef); - bool result = tsdbGetExternalRow(pQueryHandle); + bool result = tsdbGetExternalRow(pTsdbReadHandle); - pQueryHandle->prev = doFreeColumnInfoData(pQueryHandle->prev); - pQueryHandle->next = doFreeColumnInfoData(pQueryHandle->next); - pQueryHandle->currentLoadExternalRows = false; +// pTsdbReadHandle->prev = doFreeColumnInfoData(pTsdbReadHandle->prev); +// pTsdbReadHandle->next = doFreeColumnInfoData(pTsdbReadHandle->next); + pTsdbReadHandle->currentLoadExternalRows = false; return result; } @@ -2752,26 +2737,26 @@ static bool loadBlockOfActiveTable(STsdbQueryHandle* pQueryHandle) { return false; } -static bool loadCachedLastRow(STsdbQueryHandle* pQueryHandle) { +static bool loadCachedLastRow(STsdbReadHandle* pTsdbReadHandle) { // the last row is cached in buffer, return it directly. - // here note that the pQueryHandle->window must be the TS_INITIALIZER - int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pQueryHandle)); - size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); + // here note that the pTsdbReadHandle->window must be the TS_INITIALIZER + int32_t numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pTsdbReadHandle)); + size_t numOfTables = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); assert(numOfTables > 0 && numOfCols > 0); - SQueryFilePos* cur = &pQueryHandle->cur; + SQueryFilePos* cur = &pTsdbReadHandle->cur; SMemRow pRow = NULL; TSKEY key = TSKEY_INITIAL_VAL; - int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1; + int32_t step = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? 1:-1; - if (++pQueryHandle->activeIndex < numOfTables) { - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); - int32_t ret = tsdbGetCachedLastRow(pCheckInfo->pTableObj, &pRow, &key); - if (ret != TSDB_CODE_SUCCESS) { - return false; - } - mergeTwoRowFromMem(pQueryHandle, pQueryHandle->outputCapacity, 0, pRow, NULL, numOfCols, pCheckInfo->pTableObj, NULL, NULL, true); + if (++pTsdbReadHandle->activeIndex < numOfTables) { + STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, pTsdbReadHandle->activeIndex); +// int32_t ret = tsdbGetCachedLastRow(pCheckInfo->pTableObj, &pRow, &key); +// if (ret != TSDB_CODE_SUCCESS) { +// return false; +// } + mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, 0, pRow, NULL, numOfCols, pCheckInfo->pTableObj, NULL, NULL, true); tfree(pRow); // update the last key value @@ -2791,192 +2776,191 @@ static bool loadCachedLastRow(STsdbQueryHandle* pQueryHandle) { -static bool loadCachedLast(STsdbQueryHandle* pQueryHandle) { - // the last row is cached in buffer, return it directly. - // here note that the pQueryHandle->window must be the TS_INITIALIZER - int32_t tgNumOfCols = (int32_t)QH_GET_NUM_OF_COLS(pQueryHandle); - size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); - int32_t numOfRows = 0; - assert(numOfTables > 0 && tgNumOfCols > 0); - SQueryFilePos* cur = &pQueryHandle->cur; - TSKEY priKey = TSKEY_INITIAL_VAL; - int32_t priIdx = -1; - SColumnInfoData* pColInfo = NULL; +//static bool loadCachedLast(STsdbReadHandle* pTsdbReadHandle) { +// // the last row is cached in buffer, return it directly. +// // here note that the pTsdbReadHandle->window must be the TS_INITIALIZER +// int32_t tgNumOfCols = (int32_t)QH_GET_NUM_OF_COLS(pTsdbReadHandle); +// size_t numOfTables = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); +// int32_t numOfRows = 0; +// assert(numOfTables > 0 && tgNumOfCols > 0); +// SQueryFilePos* cur = &pTsdbReadHandle->cur; +// TSKEY priKey = TSKEY_INITIAL_VAL; +// int32_t priIdx = -1; +// SColumnInfoData* pColInfo = NULL; +// +// while (++pTsdbReadHandle->activeIndex < numOfTables) { +// STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, pTsdbReadHandle->activeIndex); +// STable* pTable = pCheckInfo->pTableObj; +// char* pData = NULL; +// +// int32_t numOfCols = pTable->maxColNum; +// +// if (pTable->lastCols == NULL || pTable->maxColNum <= 0) { +// tsdbWarn("no last cached for table %s, uid:%" PRIu64 ",tid:%d", pTable->name->data, pTable->uid, pTable->tableId); +// continue; +// } +// +// int32_t i = 0, j = 0; +// while(i < tgNumOfCols && j < numOfCols) { +// pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); +// if (pTable->lastCols[j].colId < pColInfo->info.colId) { +// j++; +// continue; +// } else if (pTable->lastCols[j].colId > pColInfo->info.colId) { +// i++; +// continue; +// } +// +// pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; +// +// if (pTable->lastCols[j].bytes > 0) { +// void* value = pTable->lastCols[j].pData; +// switch (pColInfo->info.type) { +// case TSDB_DATA_TYPE_BINARY: +// case TSDB_DATA_TYPE_NCHAR: +// memcpy(pData, value, varDataTLen(value)); +// break; +// case TSDB_DATA_TYPE_NULL: +// case TSDB_DATA_TYPE_BOOL: +// case TSDB_DATA_TYPE_TINYINT: +// case TSDB_DATA_TYPE_UTINYINT: +// *(uint8_t *)pData = *(uint8_t *)value; +// break; +// case TSDB_DATA_TYPE_SMALLINT: +// case TSDB_DATA_TYPE_USMALLINT: +// *(uint16_t *)pData = *(uint16_t *)value; +// break; +// case TSDB_DATA_TYPE_INT: +// case TSDB_DATA_TYPE_UINT: +// *(uint32_t *)pData = *(uint32_t *)value; +// break; +// case TSDB_DATA_TYPE_BIGINT: +// case TSDB_DATA_TYPE_UBIGINT: +// *(uint64_t *)pData = *(uint64_t *)value; +// break; +// case TSDB_DATA_TYPE_FLOAT: +// SET_FLOAT_PTR(pData, value); +// break; +// case TSDB_DATA_TYPE_DOUBLE: +// SET_DOUBLE_PTR(pData, value); +// break; +// case TSDB_DATA_TYPE_TIMESTAMP: +// if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) { +// priKey = tdGetKey(*(TKEY *)value); +// priIdx = i; +// +// i++; +// j++; +// continue; +// } else { +// *(TSKEY *)pData = *(TSKEY *)value; +// } +// break; +// default: +// memcpy(pData, value, pColInfo->info.bytes); +// } +// +// for (int32_t n = 0; n < tgNumOfCols; ++n) { +// if (n == i) { +// continue; +// } +// +// pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, n); +// pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;; +// +// if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) { +//// *(TSKEY *)pData = pTable->lastCols[j].ts; +// continue; +// } +// +// if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { +// setVardataNull(pData, pColInfo->info.type); +// } else { +// setNull(pData, pColInfo->info.type, pColInfo->info.bytes); +// } +// } +// +// numOfRows++; +// assert(numOfRows < pTsdbReadHandle->outputCapacity); +// } +// +// i++; +// j++; +// } +// +// // leave the real ts column as the last row, because last function only (not stable) use the last row as res +// if (priKey != TSKEY_INITIAL_VAL) { +// pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, priIdx); +// pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; +// +// *(TSKEY *)pData = priKey; +// +// for (int32_t n = 0; n < tgNumOfCols; ++n) { +// if (n == priIdx) { +// continue; +// } +// +// pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, n); +// pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;; +// +// assert (pColInfo->info.colId != PRIMARYKEY_TIMESTAMP_COL_ID); +// +// if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { +// setVardataNull(pData, pColInfo->info.type); +// } else { +// setNull(pData, pColInfo->info.type, pColInfo->info.bytes); +// } +// } +// +// numOfRows++; +// } +// +// if (numOfRows > 0) { +// cur->rows = numOfRows; +// cur->mixBlock = true; +// +// return true; +// } +// } +// +// return false; +//} - while (++pQueryHandle->activeIndex < numOfTables) { - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); - STable* pTable = pCheckInfo->pTableObj; - char* pData = NULL; - - int32_t numOfCols = pTable->maxColNum; - - if (pTable->lastCols == NULL || pTable->maxColNum <= 0) { - tsdbWarn("no last cached for table %s, uid:%" PRIu64 ",tid:%d", pTable->name->data, pTable->tableId.uid, pTable->tableId.tid); - continue; - } - - int32_t i = 0, j = 0; - while(i < tgNumOfCols && j < numOfCols) { - pColInfo = taosArrayGet(pQueryHandle->pColumns, i); - if (pTable->lastCols[j].colId < pColInfo->info.colId) { - j++; - continue; - } else if (pTable->lastCols[j].colId > pColInfo->info.colId) { - i++; - continue; - } - - pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; - - if (pTable->lastCols[j].bytes > 0) { - void* value = pTable->lastCols[j].pData; - switch (pColInfo->info.type) { - case TSDB_DATA_TYPE_BINARY: - case TSDB_DATA_TYPE_NCHAR: - memcpy(pData, value, varDataTLen(value)); - break; - case TSDB_DATA_TYPE_NULL: - case TSDB_DATA_TYPE_BOOL: - case TSDB_DATA_TYPE_TINYINT: - case TSDB_DATA_TYPE_UTINYINT: - *(uint8_t *)pData = *(uint8_t *)value; - break; - case TSDB_DATA_TYPE_SMALLINT: - case TSDB_DATA_TYPE_USMALLINT: - *(uint16_t *)pData = *(uint16_t *)value; - break; - case TSDB_DATA_TYPE_INT: - case TSDB_DATA_TYPE_UINT: - *(uint32_t *)pData = *(uint32_t *)value; - break; - case TSDB_DATA_TYPE_BIGINT: - case TSDB_DATA_TYPE_UBIGINT: - *(uint64_t *)pData = *(uint64_t *)value; - break; - case TSDB_DATA_TYPE_FLOAT: - SET_FLOAT_PTR(pData, value); - break; - case TSDB_DATA_TYPE_DOUBLE: - SET_DOUBLE_PTR(pData, value); - break; - case TSDB_DATA_TYPE_TIMESTAMP: - if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - priKey = tdGetKey(*(TKEY *)value); - priIdx = i; - - i++; - j++; - continue; - } else { - *(TSKEY *)pData = *(TSKEY *)value; - } - break; - default: - memcpy(pData, value, pColInfo->info.bytes); - } - - for (int32_t n = 0; n < tgNumOfCols; ++n) { - if (n == i) { - continue; - } - - pColInfo = taosArrayGet(pQueryHandle->pColumns, n); - pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;; - - if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) { - *(TSKEY *)pData = pTable->lastCols[j].ts; - continue; - } - - if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { - setVardataNull(pData, pColInfo->info.type); - } else { - setNull(pData, pColInfo->info.type, pColInfo->info.bytes); - } - } - - numOfRows++; - assert(numOfRows < pQueryHandle->outputCapacity); - } - - i++; - j++; - } - - // leave the real ts column as the last row, because last function only (not stable) use the last row as res - if (priKey != TSKEY_INITIAL_VAL) { - pColInfo = taosArrayGet(pQueryHandle->pColumns, priIdx); - pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; - - *(TSKEY *)pData = priKey; - - for (int32_t n = 0; n < tgNumOfCols; ++n) { - if (n == priIdx) { - continue; - } - - pColInfo = taosArrayGet(pQueryHandle->pColumns, n); - pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;; - - assert (pColInfo->info.colId != PRIMARYKEY_TIMESTAMP_COL_INDEX); - - if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) { - setVardataNull(pData, pColInfo->info.type); - } else { - setNull(pData, pColInfo->info.type, pColInfo->info.bytes); - } - } - - numOfRows++; - } - - if (numOfRows > 0) { - cur->rows = numOfRows; - cur->mixBlock = true; - - return true; - } - } - - return false; -} - - -static bool loadDataBlockFromTableSeq(STsdbQueryHandle* pQueryHandle) { - size_t numOfTables = taosArrayGetSize(pQueryHandle->pTableCheckInfo); +static bool loadDataBlockFromTableSeq(STsdbReadHandle* pTsdbReadHandle) { + size_t numOfTables = taosArrayGetSize(pTsdbReadHandle->pTableCheckInfo); assert(numOfTables > 0); int64_t stime = taosGetTimestampUs(); - while(pQueryHandle->activeIndex < numOfTables) { - if (loadBlockOfActiveTable(pQueryHandle)) { + while(pTsdbReadHandle->activeIndex < numOfTables) { + if (loadBlockOfActiveTable(pTsdbReadHandle)) { return true; } - STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); + STableCheckInfo* pCheckInfo = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, pTsdbReadHandle->activeIndex); pCheckInfo->numOfBlocks = 0; - pQueryHandle->activeIndex += 1; - pQueryHandle->locateStart = false; - pQueryHandle->checkFiles = true; - pQueryHandle->cur.rows = 0; - pQueryHandle->currentLoadExternalRows = pQueryHandle->loadExternalRow; + pTsdbReadHandle->activeIndex += 1; + pTsdbReadHandle->locateStart = false; + pTsdbReadHandle->checkFiles = true; + pTsdbReadHandle->cur.rows = 0; + pTsdbReadHandle->currentLoadExternalRows = pTsdbReadHandle->loadExternalRow; terrno = TSDB_CODE_SUCCESS; int64_t elapsedTime = taosGetTimestampUs() - stime; - pQueryHandle->cost.checkForNextTime += elapsedTime; + pTsdbReadHandle->cost.checkForNextTime += elapsedTime; } return false; } // handle data in cache situation -bool tsdbNextDataBlock(TsdbQueryHandleT pHandle) { - STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; +bool tsdbNextDataBlock(tsdbReadHandleT pHandle) { + STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) pHandle; - if (emptyQueryTimewindow(pQueryHandle)) { - tsdbDebug("%p query window not overlaps with the data set, no result returned, 0x%"PRIx64, pQueryHandle, pQueryHandle->qId); + if (emptyQueryTimewindow(pTsdbReadHandle)) { + tsdbDebug("%p query window not overlaps with the data set, no result returned, 0x%"PRIx64, pTsdbReadHandle, pTsdbReadHandle->qId); return false; } @@ -2984,177 +2968,177 @@ bool tsdbNextDataBlock(TsdbQueryHandleT pHandle) { int64_t elapsedTime = stime; // TODO refactor: remove "type" - if (pQueryHandle->type == TSDB_QUERY_TYPE_LAST) { - if (pQueryHandle->cachelastrow == TSDB_CACHED_TYPE_LASTROW) { - return loadCachedLastRow(pQueryHandle); - } else if (pQueryHandle->cachelastrow == TSDB_CACHED_TYPE_LAST) { - return loadCachedLast(pQueryHandle); + if (pTsdbReadHandle->type == TSDB_QUERY_TYPE_LAST) { + if (pTsdbReadHandle->cachelastrow == TSDB_CACHED_TYPE_LASTROW) { +// return loadCachedLastRow(pTsdbReadHandle); + } else if (pTsdbReadHandle->cachelastrow == TSDB_CACHED_TYPE_LAST) { +// return loadCachedLast(pTsdbReadHandle); } } - if (pQueryHandle->loadType == BLOCK_LOAD_TABLE_SEQ_ORDER) { - return loadDataBlockFromTableSeq(pQueryHandle); + if (pTsdbReadHandle->loadType == BLOCK_LOAD_TABLE_SEQ_ORDER) { + return loadDataBlockFromTableSeq(pTsdbReadHandle); } else { // loadType == RR and Offset Order - if (pQueryHandle->checkFiles) { + if (pTsdbReadHandle->checkFiles) { // check if the query range overlaps with the file data block bool exists = true; - int32_t code = getDataBlocksInFiles(pQueryHandle, &exists); + int32_t code = getDataBlocksInFiles(pTsdbReadHandle, &exists); if (code != TSDB_CODE_SUCCESS) { - pQueryHandle->activeIndex = 0; - pQueryHandle->checkFiles = false; + pTsdbReadHandle->activeIndex = 0; + pTsdbReadHandle->checkFiles = false; return false; } if (exists) { - pQueryHandle->cost.checkForNextTime += (taosGetTimestampUs() - stime); + pTsdbReadHandle->cost.checkForNextTime += (taosGetTimestampUs() - stime); return exists; } - pQueryHandle->activeIndex = 0; - pQueryHandle->checkFiles = false; + pTsdbReadHandle->activeIndex = 0; + pTsdbReadHandle->checkFiles = false; } // TODO: opt by consider the scan order - bool ret = doHasDataInBuffer(pQueryHandle); + bool ret = doHasDataInBuffer(pTsdbReadHandle); terrno = TSDB_CODE_SUCCESS; elapsedTime = taosGetTimestampUs() - stime; - pQueryHandle->cost.checkForNextTime += elapsedTime; + pTsdbReadHandle->cost.checkForNextTime += elapsedTime; return ret; } } -static int32_t doGetExternalRow(STsdbQueryHandle* pQueryHandle, int16_t type, SMemRef* pMemRef) { - STsdbQueryHandle* pSecQueryHandle = NULL; +//static int32_t doGetExternalRow(STsdbReadHandle* pTsdbReadHandle, int16_t type, STsdbMemTable* pMemRef) { +// STsdbReadHandle* pSecQueryHandle = NULL; +// +// if (type == TSDB_PREV_ROW && pTsdbReadHandle->prev) { +// return TSDB_CODE_SUCCESS; +// } +// +// if (type == TSDB_NEXT_ROW && pTsdbReadHandle->next) { +// return TSDB_CODE_SUCCESS; +// } +// +// // prepare the structure +// int32_t numOfCols = (int32_t) QH_GET_NUM_OF_COLS(pTsdbReadHandle); +// +// if (type == TSDB_PREV_ROW) { +// pTsdbReadHandle->prev = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); +// if (pTsdbReadHandle->prev == NULL) { +// terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; +// goto out_of_memory; +// } +// } else { +// pTsdbReadHandle->next = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); +// if (pTsdbReadHandle->next == NULL) { +// terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; +// goto out_of_memory; +// } +// } +// +// SArray* row = (type == TSDB_PREV_ROW)? pTsdbReadHandle->prev : pTsdbReadHandle->next; +// +// for (int32_t i = 0; i < numOfCols; ++i) { +// SColumnInfoData* pCol = taosArrayGet(pTsdbReadHandle->pColumns, i); +// +// SColumnInfoData colInfo = {{0}, 0}; +// colInfo.info = pCol->info; +// colInfo.pData = calloc(1, pCol->info.bytes); +// if (colInfo.pData == NULL) { +// terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; +// goto out_of_memory; +// } +// +// taosArrayPush(row, &colInfo); +// } +// +// // load the previous row +// STsdbQueryCond cond = {.numOfCols = numOfCols, .loadExternalRows = false, .type = BLOCK_LOAD_OFFSET_SEQ_ORDER}; +// if (type == TSDB_PREV_ROW) { +// cond.order = TSDB_ORDER_DESC; +// cond.twindow = (STimeWindow){pTsdbReadHandle->window.skey, INT64_MIN}; +// } else { +// cond.order = TSDB_ORDER_ASC; +// cond.twindow = (STimeWindow){pTsdbReadHandle->window.skey, INT64_MAX}; +// } +// +// cond.colList = calloc(cond.numOfCols, sizeof(SColumnInfo)); +// if (cond.colList == NULL) { +// terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; +// goto out_of_memory; +// } +// +// for (int32_t i = 0; i < cond.numOfCols; ++i) { +// SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pColumns, i); +// memcpy(&cond.colList[i], &pColInfoData->info, sizeof(SColumnInfo)); +// } +// +// pSecQueryHandle = tsdbQueryTablesImpl(pTsdbReadHandle->pTsdb, &cond, pTsdbReadHandle->qId, pMemRef); +// tfree(cond.colList); +// +// // current table, only one table +// STableCheckInfo* pCurrent = taosArrayGet(pTsdbReadHandle->pTableCheckInfo, pTsdbReadHandle->activeIndex); +// +// SArray* psTable = NULL; +// pSecQueryHandle->pTableCheckInfo = createCheckInfoFromCheckInfo(pCurrent, pSecQueryHandle->window.skey, &psTable); +// if (pSecQueryHandle->pTableCheckInfo == NULL) { +// taosArrayDestroy(psTable); +// terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; +// goto out_of_memory; +// } +// +// +// tsdbMayTakeMemSnapshot(pSecQueryHandle, psTable); +// if (!tsdbNextDataBlock((void*)pSecQueryHandle)) { +// // no result in current query, free the corresponding result rows structure +// if (type == TSDB_PREV_ROW) { +// pTsdbReadHandle->prev = doFreeColumnInfoData(pTsdbReadHandle->prev); +// } else { +// pTsdbReadHandle->next = doFreeColumnInfoData(pTsdbReadHandle->next); +// } +// +// goto out_of_memory; +// } +// +// SDataBlockInfo blockInfo = {{0}, 0}; +// tsdbRetrieveDataBlockInfo((void*)pSecQueryHandle, &blockInfo); +// tsdbRetrieveDataBlock((void*)pSecQueryHandle, pSecQueryHandle->defaultLoadColumn); +// +// row = (type == TSDB_PREV_ROW)? pTsdbReadHandle->prev:pTsdbReadHandle->next; +// int32_t pos = (type == TSDB_PREV_ROW)?pSecQueryHandle->cur.rows - 1:0; +// +// for (int32_t i = 0; i < numOfCols; ++i) { +// SColumnInfoData* pCol = taosArrayGet(row, i); +// SColumnInfoData* s = taosArrayGet(pSecQueryHandle->pColumns, i); +// memcpy((char*)pCol->pData, (char*)s->pData + s->info.bytes * pos, pCol->info.bytes); +// } +// +//out_of_memory: +// tsdbCleanupQueryHandle(pSecQueryHandle); +// return terrno; +//} - if (type == TSDB_PREV_ROW && pQueryHandle->prev) { - return TSDB_CODE_SUCCESS; - } - - if (type == TSDB_NEXT_ROW && pQueryHandle->next) { - return TSDB_CODE_SUCCESS; - } - - // prepare the structure - int32_t numOfCols = (int32_t) QH_GET_NUM_OF_COLS(pQueryHandle); - - if (type == TSDB_PREV_ROW) { - pQueryHandle->prev = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); - if (pQueryHandle->prev == NULL) { - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto out_of_memory; - } - } else { - pQueryHandle->next = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); - if (pQueryHandle->next == NULL) { - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto out_of_memory; - } - } - - SArray* row = (type == TSDB_PREV_ROW)? pQueryHandle->prev : pQueryHandle->next; - - for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pCol = taosArrayGet(pQueryHandle->pColumns, i); - - SColumnInfoData colInfo = {{0}, 0}; - colInfo.info = pCol->info; - colInfo.pData = calloc(1, pCol->info.bytes); - if (colInfo.pData == NULL) { - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto out_of_memory; - } - - taosArrayPush(row, &colInfo); - } - - // load the previous row - STsdbQueryCond cond = {.numOfCols = numOfCols, .loadExternalRows = false, .type = BLOCK_LOAD_OFFSET_SEQ_ORDER}; - if (type == TSDB_PREV_ROW) { - cond.order = TSDB_ORDER_DESC; - cond.twindow = (STimeWindow){pQueryHandle->window.skey, INT64_MIN}; - } else { - cond.order = TSDB_ORDER_ASC; - cond.twindow = (STimeWindow){pQueryHandle->window.skey, INT64_MAX}; - } - - cond.colList = calloc(cond.numOfCols, sizeof(SColumnInfo)); - if (cond.colList == NULL) { - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto out_of_memory; - } - - for (int32_t i = 0; i < cond.numOfCols; ++i) { - SColumnInfoData* pColInfoData = taosArrayGet(pQueryHandle->pColumns, i); - memcpy(&cond.colList[i], &pColInfoData->info, sizeof(SColumnInfo)); - } - - pSecQueryHandle = tsdbQueryTablesImpl(pQueryHandle->pTsdb, &cond, pQueryHandle->qId, pMemRef); - tfree(cond.colList); - - // current table, only one table - STableCheckInfo* pCurrent = taosArrayGet(pQueryHandle->pTableCheckInfo, pQueryHandle->activeIndex); - - SArray* psTable = NULL; - pSecQueryHandle->pTableCheckInfo = createCheckInfoFromCheckInfo(pCurrent, pSecQueryHandle->window.skey, &psTable); - if (pSecQueryHandle->pTableCheckInfo == NULL) { - taosArrayDestroy(psTable); - terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto out_of_memory; - } - - - tsdbMayTakeMemSnapshot(pSecQueryHandle, psTable); - if (!tsdbNextDataBlock((void*)pSecQueryHandle)) { - // no result in current query, free the corresponding result rows structure - if (type == TSDB_PREV_ROW) { - pQueryHandle->prev = doFreeColumnInfoData(pQueryHandle->prev); - } else { - pQueryHandle->next = doFreeColumnInfoData(pQueryHandle->next); - } - - goto out_of_memory; - } - - SDataBlockInfo blockInfo = {{0}, 0}; - tsdbRetrieveDataBlockInfo((void*)pSecQueryHandle, &blockInfo); - tsdbRetrieveDataBlock((void*)pSecQueryHandle, pSecQueryHandle->defaultLoadColumn); - - row = (type == TSDB_PREV_ROW)? pQueryHandle->prev:pQueryHandle->next; - int32_t pos = (type == TSDB_PREV_ROW)?pSecQueryHandle->cur.rows - 1:0; - - for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pCol = taosArrayGet(row, i); - SColumnInfoData* s = taosArrayGet(pSecQueryHandle->pColumns, i); - memcpy((char*)pCol->pData, (char*)s->pData + s->info.bytes * pos, pCol->info.bytes); - } - -out_of_memory: - tsdbCleanupQueryHandle(pSecQueryHandle); - return terrno; -} - -bool tsdbGetExternalRow(TsdbQueryHandleT pHandle) { - STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*) pHandle; - SQueryFilePos* cur = &pQueryHandle->cur; +bool tsdbGetExternalRow(tsdbReadHandleT pHandle) { + STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*) pHandle; + SQueryFilePos* cur = &pTsdbReadHandle->cur; cur->fid = INT32_MIN; cur->mixBlock = true; - if (pQueryHandle->prev == NULL || pQueryHandle->next == NULL) { + if (pTsdbReadHandle->prev == NULL || pTsdbReadHandle->next == NULL) { cur->rows = 0; return false; } - int32_t numOfCols = (int32_t) QH_GET_NUM_OF_COLS(pQueryHandle); + int32_t numOfCols = (int32_t) QH_GET_NUM_OF_COLS(pTsdbReadHandle); for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pColInfoData = taosArrayGet(pQueryHandle->pColumns, i); - SColumnInfoData* first = taosArrayGet(pQueryHandle->prev, i); + SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pColumns, i); + SColumnInfoData* first = taosArrayGet(pTsdbReadHandle->prev, i); memcpy(pColInfoData->pData, first->pData, pColInfoData->info.bytes); - SColumnInfoData* sec = taosArrayGet(pQueryHandle->next, i); + SColumnInfoData* sec = taosArrayGet(pTsdbReadHandle->next, i); memcpy(((char*)pColInfoData->pData) + pColInfoData->info.bytes, sec->pData, pColInfoData->info.bytes); if (i == 0 && pColInfoData->info.type == TSDB_DATA_TYPE_TIMESTAMP) { @@ -3171,76 +3155,75 @@ bool tsdbGetExternalRow(TsdbQueryHandleT pHandle) { * if lastRow == NULL, return TSDB_CODE_TDB_NO_CACHE_LAST_ROW * else set pRes and return TSDB_CODE_SUCCESS and save lastKey */ -int32_t tsdbGetCachedLastRow(STable* pTable, SMemRow* pRes, TSKEY* lastKey) { - int32_t code = TSDB_CODE_SUCCESS; +//int32_t tsdbGetCachedLastRow(STable* pTable, SMemRow* pRes, TSKEY* lastKey) { +// int32_t code = TSDB_CODE_SUCCESS; +// +// TSDB_RLOCK_TABLE(pTable); +// +// if (!pTable->lastRow) { +// code = TSDB_CODE_TDB_NO_CACHE_LAST_ROW; +// goto out; +// } +// +// if (pRes) { +// *pRes = tdMemRowDup(pTable->lastRow); +// if (*pRes == NULL) { +// code = TSDB_CODE_TDB_OUT_OF_MEMORY; +// } +// } +// +//out: +// TSDB_RUNLOCK_TABLE(pTable); +// return code; +//} - TSDB_RLOCK_TABLE(pTable); - - if (!pTable->lastRow) { - code = TSDB_CODE_TDB_NO_CACHE_LAST_ROW; - goto out; - } - - if (pRes) { - *pRes = tdMemRowDup(pTable->lastRow); - if (*pRes == NULL) { - code = TSDB_CODE_TDB_OUT_OF_MEMORY; - } - } - -out: - TSDB_RUNLOCK_TABLE(pTable); - return code; +bool isTsdbCacheLastRow(tsdbReadHandleT* pTsdbReadHandle) { + return ((STsdbReadHandle *)pTsdbReadHandle)->cachelastrow > TSDB_CACHED_TYPE_NONE; } -bool isTsdbCacheLastRow(TsdbQueryHandleT* pQueryHandle) { - return ((STsdbQueryHandle *)pQueryHandle)->cachelastrow > TSDB_CACHED_TYPE_NONE; +int32_t checkForCachedLastRow(STsdbReadHandle* pTsdbReadHandle, STableGroupInfo *groupList) { + assert(pTsdbReadHandle != NULL && groupList != NULL); + +// TSKEY key = TSKEY_INITIAL_VAL; +// +// SArray* group = taosArrayGetP(groupList->pGroupList, 0); +// assert(group != NULL); +// +// STableKeyInfo* pInfo = (STableKeyInfo*)taosArrayGet(group, 0); +// +// int32_t code = 0; +// +// if (((STable*)pInfo->pTable)->lastRow) { +// code = tsdbGetCachedLastRow(pInfo->pTable, NULL, &key); +// if (code != TSDB_CODE_SUCCESS) { +// pTsdbReadHandle->cachelastrow = TSDB_CACHED_TYPE_NONE; +// } else { +// pTsdbReadHandle->cachelastrow = TSDB_CACHED_TYPE_LASTROW; +// } +// } +// +// // update the tsdb query time range +// if (pTsdbReadHandle->cachelastrow != TSDB_CACHED_TYPE_NONE) { +// pTsdbReadHandle->window = TSWINDOW_INITIALIZER; +// pTsdbReadHandle->checkFiles = false; +// pTsdbReadHandle->activeIndex = -1; // start from -1 +// } + + return TSDB_CODE_SUCCESS; } -int32_t checkForCachedLastRow(STsdbQueryHandle* pQueryHandle, STableGroupInfo *groupList) { - assert(pQueryHandle != NULL && groupList != NULL); - - TSKEY key = TSKEY_INITIAL_VAL; - - SArray* group = taosArrayGetP(groupList->pGroupList, 0); - assert(group != NULL); - - STableKeyInfo* pInfo = (STableKeyInfo*)taosArrayGet(group, 0); +int32_t checkForCachedLast(STsdbReadHandle* pTsdbReadHandle) { + assert(pTsdbReadHandle != NULL); int32_t code = 0; - - if (((STable*)pInfo->pTable)->lastRow) { - code = tsdbGetCachedLastRow(pInfo->pTable, NULL, &key); - if (code != TSDB_CODE_SUCCESS) { - pQueryHandle->cachelastrow = TSDB_CACHED_TYPE_NONE; - } else { - pQueryHandle->cachelastrow = TSDB_CACHED_TYPE_LASTROW; - } - } +// if (pTsdbReadHandle->pTsdb && atomic_load_8(&pTsdbReadHandle->pTsdb->hasCachedLastColumn)){ +// pTsdbReadHandle->cachelastrow = TSDB_CACHED_TYPE_LAST; +// } // update the tsdb query time range - if (pQueryHandle->cachelastrow != TSDB_CACHED_TYPE_NONE) { - pQueryHandle->window = TSWINDOW_INITIALIZER; - pQueryHandle->checkFiles = false; - pQueryHandle->activeIndex = -1; // start from -1 - } - - return code; -} - -int32_t checkForCachedLast(STsdbQueryHandle* pQueryHandle) { - assert(pQueryHandle != NULL); - - int32_t code = 0; - - if (pQueryHandle->pTsdb && atomic_load_8(&pQueryHandle->pTsdb->hasCachedLastColumn)){ - pQueryHandle->cachelastrow = TSDB_CACHED_TYPE_LAST; - } - - // update the tsdb query time range - if (pQueryHandle->cachelastrow) { - pQueryHandle->checkFiles = false; - pQueryHandle->activeIndex = -1; // start from -1 + if (pTsdbReadHandle->cachelastrow) { + pTsdbReadHandle->checkFiles = false; + pTsdbReadHandle->activeIndex = -1; // start from -1 } return code; @@ -3266,7 +3249,7 @@ STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList) { STableKeyInfo* pInfo = (STableKeyInfo*) taosArrayGet(pGroup, i); // if the lastKey equals to INT64_MIN, there is no data in this table - TSKEY lastKey = ((STable*)(pInfo->pTable))->lastKey; + TSKEY lastKey = 0;//((STable*)(pInfo->pTable))->lastKey; if (key < lastKey) { key = lastKey; @@ -3290,7 +3273,7 @@ STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList) { // keyInfo.pTable may be NULL here. if (pInfo->pTable != keyInfo.pTable) { - tsdbUnRefTable(pInfo->pTable); +// tsdbUnRefTable(pInfo->pTable); } } @@ -3322,23 +3305,23 @@ STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList) { return window; } -void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT* pQueryHandle, SDataBlockInfo* pDataBlockInfo) { - STsdbQueryHandle* pHandle = (STsdbQueryHandle*)pQueryHandle; +void tsdbRetrieveDataBlockInfo(tsdbReadHandleT* pTsdbReadHandle, SDataBlockInfo* pDataBlockInfo) { + STsdbReadHandle* pHandle = (STsdbReadHandle*)pTsdbReadHandle; SQueryFilePos* cur = &pHandle->cur; - STable* pTable = NULL; + + uint64_t uid = 0; // there are data in file if (pHandle->cur.fid != INT32_MIN) { STableBlockInfo* pBlockInfo = &pHandle->pDataBlockInfo[cur->slot]; - pTable = pBlockInfo->pTableCheckInfo->pTableObj; + uid = pBlockInfo->pTableCheckInfo->tableId; } else { STableCheckInfo* pCheckInfo = taosArrayGet(pHandle->pTableCheckInfo, pHandle->activeIndex); - pTable = pCheckInfo->pTableObj; + uid = pCheckInfo->tableId; } - pDataBlockInfo->uid = pTable->tableId.uid; - pDataBlockInfo->tid = pTable->tableId.tid; - pDataBlockInfo->rows = cur->rows; + pDataBlockInfo->uid = uid; + pDataBlockInfo->rows = cur->rows; pDataBlockInfo->window = cur->win; pDataBlockInfo->numOfCols = (int32_t)(QH_GET_NUM_OF_COLS(pHandle)); } @@ -3346,8 +3329,8 @@ void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT* pQueryHandle, SDataBlockInfo* p /* * return null for mixed data block, if not a complete file data block, the statistics value will always return NULL */ -int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataStatis** pBlockStatis) { - STsdbQueryHandle* pHandle = (STsdbQueryHandle*) pQueryHandle; +int32_t tsdbRetrieveDataBlockStatisInfo(tsdbReadHandleT* pTsdbReadHandle, SDataStatis** pBlockStatis) { + STsdbReadHandle* pHandle = (STsdbReadHandle*) pTsdbReadHandle; SQueryFilePos* c = &pHandle->cur; if (c->mixBlock) { @@ -3381,7 +3364,7 @@ int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataSta // always load the first primary timestamp column data SDataStatis* pPrimaryColStatis = &pHandle->statis[0]; - assert(pPrimaryColStatis->colId == PRIMARYKEY_TIMESTAMP_COL_INDEX); + assert(pPrimaryColStatis->colId == PRIMARYKEY_TIMESTAMP_COL_ID); pPrimaryColStatis->numOfNull = 0; pPrimaryColStatis->min = pBlockInfo->compBlock->keyFirst; @@ -3401,12 +3384,12 @@ int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataSta return TSDB_CODE_SUCCESS; } -SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) { +SArray* tsdbRetrieveDataBlock(tsdbReadHandleT* pTsdbReadHandle, SArray* pIdList) { /** * In the following two cases, the data has been loaded to SColumnInfoData. * 1. data is from cache, 2. data block is not completed qualified to query time range */ - STsdbQueryHandle* pHandle = (STsdbQueryHandle*)pQueryHandle; + STsdbReadHandle* pHandle = (STsdbReadHandle*)pTsdbReadHandle; if (pHandle->cur.fid == INT32_MIN) { return pHandle->pColumns; @@ -3417,14 +3400,14 @@ SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) { if (pHandle->cur.mixBlock) { return pHandle->pColumns; } else { - SDataBlockInfo binfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlockInfo->compBlock); + SDataBlockInfo binfo = {0};/*GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlockInfo->compBlock);*/ assert(pHandle->realNumOfRows <= binfo.rows); // data block has been loaded, todo extract method SDataBlockLoadInfo* pBlockLoadInfo = &pHandle->dataBlockLoadInfo; if (pBlockLoadInfo->slot == pHandle->cur.slot && pBlockLoadInfo->fileGroup->fid == pHandle->cur.fid && - pBlockLoadInfo->tid == pCheckInfo->pTableObj->tableId.tid) { + pBlockLoadInfo->uid == pCheckInfo->pTableObj->tid) { return pHandle->pColumns; } else { // only load the file block SBlock* pBlock = pBlockInfo->compBlock; @@ -3451,7 +3434,7 @@ SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) { } } } - +#if 0 void filterPrepare(void* expr, void* param) { tExprNode* pExpr = (tExprNode*)expr; if (pExpr->_node.info != NULL) { @@ -3550,11 +3533,12 @@ static int32_t tableGroupComparFn(const void *p1, const void *p2, const void *pa return 0; } +#endif static int tsdbCheckInfoCompar(const void* key1, const void* key2) { - if (((STableCheckInfo*)key1)->tableId.tid < ((STableCheckInfo*)key2)->tableId.tid) { + if (((STableCheckInfo*)key1)->tableId < ((STableCheckInfo*)key2)->tableId) { return -1; - } else if (((STableCheckInfo*)key1)->tableId.tid > ((STableCheckInfo*)key2)->tableId.tid) { + } else if (((STableCheckInfo*)key1)->tableId > ((STableCheckInfo*)key2)->tableId) { return 1; } else { ASSERT(false); @@ -3570,7 +3554,6 @@ void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTable STableKeyInfo info = {.pTable = pTable, .lastKey = skey}; taosArrayPush(g, &info); - tsdbRefTable(pTable); for (int32_t i = 1; i < numOfTables; ++i) { STable** prev = taosArrayGet(pTableList, i - 1); @@ -3579,8 +3562,7 @@ void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTable int32_t ret = compareFn(prev, p, pSupp); assert(ret == 0 || ret == -1); - tsdbRefTable(*p); - assert((*p)->type == TSDB_CHILD_TABLE); +// assert((*p)->type == TSDB_CHILD_TABLE); if (ret == 0) { STableKeyInfo info1 = {.pTable = *p, .lastKey = skey}; @@ -3597,6 +3579,7 @@ void createTableGroupImpl(SArray* pGroups, SArray* pTableList, size_t numOfTable taosArrayPush(pGroups, &g); } +#if 0 SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pCols, int32_t numOfOrderCols, TSKEY skey) { assert(pTableList != NULL); SArray* pTableGroup = taosArrayInit(1, POINTER_BYTES); @@ -3616,7 +3599,6 @@ SArray* createTableGroup(SArray* pTableList, STSchema* pTagSchema, SColIndex* pC for(int32_t i = 0; i < size; ++i) { STableKeyInfo *pKeyInfo = taosArrayGet(pTableList, i); - tsdbRefTable(pKeyInfo->pTable); STableKeyInfo info = {.pTable = pKeyInfo->pTable, .lastKey = skey}; taosArrayPush(sa, &info); @@ -3737,7 +3719,7 @@ static int32_t doQueryTableList(STable* pSTable, SArray* pRes, tExprNode* pExpr) return TSDB_CODE_SUCCESS; } -int32_t tsdbQuerySTableByTagCond(STsdbRepo* tsdb, uint64_t uid, TSKEY skey, const char* pTagCond, size_t len, +int32_t tsdbQuerySTableByTagCond(STsdb* tsdb, uint64_t uid, TSKEY skey, const char* pTagCond, size_t len, int16_t tagNameRelType, const char* tbnameCond, STableGroupInfo* pGroupInfo, SColIndex* pColIndex, int32_t numOfCols) { if (tsdbRLockRepoMeta(tsdb) < 0) goto _error; @@ -3752,7 +3734,7 @@ int32_t tsdbQuerySTableByTagCond(STsdbRepo* tsdb, uint64_t uid, TSKEY skey, cons } if (pTable->type != TSDB_SUPER_TABLE) { - tsdbError("%p query normal tag not allowed, uid:%" PRIu64 ", tid:%d, name:%s", tsdb, uid, pTable->tableId.tid, + tsdbError("%p query normal tag not allowed, uid:%" PRIu64 ", tid:%d, name:%s", tsdb, uid, pTable->tableId, pTable->name->data); terrno = TSDB_CODE_COM_OPS_NOT_SUPPORT; //basically, this error is caused by invalid sql issued by client @@ -3821,8 +3803,8 @@ int32_t tsdbQuerySTableByTagCond(STsdbRepo* tsdb, uint64_t uid, TSKEY skey, cons pGroupInfo->numOfTables = (uint32_t)taosArrayGetSize(res); pGroupInfo->pGroupList = createTableGroup(res, pTagSchema, pColIndex, numOfCols, skey); - tsdbDebug("%p stable tid:%d, uid:%"PRIu64" query, numOfTables:%u, belong to %" PRIzu " groups", tsdb, pTable->tableId.tid, - pTable->tableId.uid, pGroupInfo->numOfTables, taosArrayGetSize(pGroupInfo->pGroupList)); + tsdbDebug("%p stable tid:%d, uid:%"PRIu64" query, numOfTables:%u, belong to %" PRIzu " groups", tsdb, pTable->tableId, + pTable->uid, pGroupInfo->numOfTables, taosArrayGetSize(pGroupInfo->pGroupList)); taosArrayDestroy(res); @@ -3833,7 +3815,7 @@ int32_t tsdbQuerySTableByTagCond(STsdbRepo* tsdb, uint64_t uid, TSKEY skey, cons return terrno; } -int32_t tsdbGetOneTableGroup(STsdbRepo* tsdb, uint64_t uid, TSKEY startKey, STableGroupInfo* pGroupInfo) { +int32_t tsdbGetOneTableGroup(STsdb* tsdb, uint64_t uid, TSKEY startKey, STableGroupInfo* pGroupInfo) { if (tsdbRLockRepoMeta(tsdb) < 0) goto _error; STable* pTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid); @@ -3844,7 +3826,6 @@ int32_t tsdbGetOneTableGroup(STsdbRepo* tsdb, uint64_t uid, TSKEY startKey, STab } assert(pTable->type == TSDB_CHILD_TABLE || pTable->type == TSDB_NORMAL_TABLE || pTable->type == TSDB_STREAM_TABLE); - tsdbRefTable(pTable); if (tsdbUnlockRepoMeta(tsdb) < 0) goto _error; pGroupInfo->numOfTables = 1; @@ -3862,7 +3843,7 @@ int32_t tsdbGetOneTableGroup(STsdbRepo* tsdb, uint64_t uid, TSKEY startKey, STab return terrno; } -int32_t tsdbGetTableGroupFromIdList(STsdbRepo* tsdb, SArray* pTableIdList, STableGroupInfo* pGroupInfo) { +int32_t tsdbGetTableGroupFromIdList(STsdb* tsdb, SArray* pTableIdList, STableGroupInfo* pGroupInfo) { if (tsdbRLockRepoMeta(tsdb) < 0) { return terrno; } @@ -3889,8 +3870,6 @@ int32_t tsdbGetTableGroupFromIdList(STsdbRepo* tsdb, SArray* pTableIdList, STabl return terrno; } - tsdbRefTable(pTable); - STableKeyInfo info = {.pTable = pTable, .lastKey = id->key}; taosArrayPush(group, &info); } @@ -3938,42 +3917,42 @@ static void* destroyTableCheckInfo(SArray* pTableCheckInfo) { return NULL; } -void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) { - STsdbQueryHandle* pQueryHandle = (STsdbQueryHandle*)queryHandle; - if (pQueryHandle == NULL) { +void tsdbCleanupQueryHandle(tsdbReadHandleT queryHandle) { + STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*)queryHandle; + if (pTsdbReadHandle == NULL) { return; } - pQueryHandle->pColumns = doFreeColumnInfoData(pQueryHandle->pColumns); + pTsdbReadHandle->pColumns = doFreeColumnInfoData(pTsdbReadHandle->pColumns); - taosArrayDestroy(pQueryHandle->defaultLoadColumn); - tfree(pQueryHandle->pDataBlockInfo); - tfree(pQueryHandle->statis); + taosArrayDestroy(pTsdbReadHandle->defaultLoadColumn); + tfree(pTsdbReadHandle->pDataBlockInfo); + tfree(pTsdbReadHandle->statis); - if (!emptyQueryTimewindow(pQueryHandle)) { - tsdbMayUnTakeMemSnapshot(pQueryHandle); + if (!emptyQueryTimewindow(pTsdbReadHandle)) { + tsdbMayUnTakeMemSnapshot(pTsdbReadHandle); } else { - assert(pQueryHandle->pTableCheckInfo == NULL); + assert(pTsdbReadHandle->pTableCheckInfo == NULL); } - if (pQueryHandle->pTableCheckInfo != NULL) { - pQueryHandle->pTableCheckInfo = destroyTableCheckInfo(pQueryHandle->pTableCheckInfo); + if (pTsdbReadHandle->pTableCheckInfo != NULL) { + pTsdbReadHandle->pTableCheckInfo = destroyTableCheckInfo(pTsdbReadHandle->pTableCheckInfo); } - tsdbDestroyReadH(&pQueryHandle->rhelper); + tsdbDestroyReadH(&pTsdbReadHandle->rhelper); - tdFreeDataCols(pQueryHandle->pDataCols); - pQueryHandle->pDataCols = NULL; + tdFreeDataCols(pTsdbReadHandle->pDataCols); + pTsdbReadHandle->pDataCols = NULL; - pQueryHandle->prev = doFreeColumnInfoData(pQueryHandle->prev); - pQueryHandle->next = doFreeColumnInfoData(pQueryHandle->next); + pTsdbReadHandle->prev = doFreeColumnInfoData(pTsdbReadHandle->prev); + pTsdbReadHandle->next = doFreeColumnInfoData(pTsdbReadHandle->next); - SIOCostSummary* pCost = &pQueryHandle->cost; + SIOCostSummary* pCost = &pTsdbReadHandle->cost; tsdbDebug("%p :io-cost summary: head-file read cnt:%"PRIu64", head-file time:%"PRIu64" us, statis-info:%"PRId64" us, datablock:%" PRId64" us, check data:%"PRId64" us, 0x%"PRIx64, - pQueryHandle, pCost->headFileLoad, pCost->headFileLoadTime, pCost->statisInfoLoadTime, pCost->blockLoadTime, pCost->checkForNextTime, pQueryHandle->qId); + pTsdbReadHandle, pCost->headFileLoad, pCost->headFileLoadTime, pCost->statisInfoLoadTime, pCost->blockLoadTime, pCost->checkForNextTime, pTsdbReadHandle->qId); - tfree(pQueryHandle); + tfree(pTsdbReadHandle); } void tsdbDestroyTableGroup(STableGroupInfo *pGroupList) { @@ -4257,3 +4236,4 @@ void getTableListfromSkipList(tExprNode *pExpr, SSkipList *pSkipList, SArray *re //apply the hierarchical filter expression to every node in skiplist to find the qualified nodes applyFilterToSkipListNode(pSkipList, pExpr, result, param); } +#endif \ No newline at end of file diff --git a/source/libs/executor/CMakeLists.txt b/source/libs/executor/CMakeLists.txt index ba941ab22d..04b5fab4bf 100644 --- a/source/libs/executor/CMakeLists.txt +++ b/source/libs/executor/CMakeLists.txt @@ -1,12 +1,27 @@ aux_source_directory(src EXECUTOR_SRC) -add_library(executor ${EXECUTOR_SRC}) +#add_library(executor ${EXECUTOR_SRC}) + + +#target_link_libraries( +# executor +# PRIVATE os util common function parser planner qcom tsdb +#) + +add_library(executor STATIC ${EXECUTOR_SRC}) +#set_target_properties(executor PROPERTIES +# IMPORTED_LOCATION "${CMAKE_CURRENT_SOURCE_DIR}/libexecutor.a" +# INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_SOURCE_DIR}/include/libs/executor" +# ) +target_link_libraries(executor + PRIVATE os util common function parser planner qcom tsdb + ) + target_include_directories( - executor - PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/executor" - PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" + executor + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/executor" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) -target_link_libraries( - executor - PRIVATE os util common function parser planner qcom -) \ No newline at end of file +#if(${BUILD_TEST}) + ADD_SUBDIRECTORY(test) +#endif(${BUILD_TEST}) \ No newline at end of file diff --git a/source/libs/executor/inc/dataSinkMgt.h b/source/libs/executor/inc/dataSinkMgt.h index d13423b25d..d0057a213a 100644 --- a/source/libs/executor/inc/dataSinkMgt.h +++ b/source/libs/executor/inc/dataSinkMgt.h @@ -33,7 +33,7 @@ struct SDataSink; struct SSDataBlock; typedef struct SDataSinkMgtCfg { - uint32_t maxDataBlockNum; + uint32_t maxDataBlockNum; // todo: this should be numOfRows? uint32_t maxDataBlockNumPerQuery; } SDataSinkMgtCfg; diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 2c1bf71638..79766b22ac 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -38,7 +38,7 @@ #define GET_RES_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t)) #define GET_RES_EXT_WINDOW_KEY_LEN(_l) ((_l) + sizeof(uint64_t) + POINTER_BYTES) -#define GET_QID(_r) (((SQInfo*)((_r)->qinfo))->qId) +#define GET_TASKID(_t) (((SExecTaskInfo*)(_t))->id.queryId) #define curTimeWindowIndex(_winres) ((_winres)->curIndex) @@ -157,6 +157,6 @@ int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo); int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, struct STaskRuntimeEnv *pRuntimeEnv, int32_t* offset); -int32_t initUdfInfo(struct SUdfInfo* pUdfInfo); +//int32_t initUdfInfo(struct SUdfInfo* pUdfInfo); #endif // TDENGINE_QUERYUTIL_H diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 74ee4637c1..33fafb4074 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -29,15 +29,8 @@ #include "tpagedfile.h" #include "planner.h" - struct SColumnFilterElem; -typedef struct { - uint32_t numOfTables; - SArray *pGroupList; - SHashObj *map; // speedup acquire the tableQueryInfo by table uid -} STableGroupInfo; - typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order); #define IS_QUERY_KILLED(_q) ((_q)->code == TSDB_CODE_TSC_QUERY_CANCELLED) @@ -51,19 +44,19 @@ typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int #define NEEDTO_COMPRESS_QUERY(size) ((size) > tsCompressColData? 1 : 0) enum { - // when query starts to execute, this status will set - QUERY_NOT_COMPLETED = 0x1u, + // when this task starts to execute, this status will set + TASK_NOT_COMPLETED = 0x1u, - /* query is over + /* Task is over * 1. this status is used in one row result query process, e.g., count/sum/first/last/ avg...etc. * 2. when all data within queried time window, it is also denoted as query_completed */ - QUERY_COMPLETED = 0x2u, + TASK_COMPLETED = 0x2u, /* when the result is not completed return to client, this status will be * usually used in case of interval query with interpolation option */ - QUERY_OVER = 0x4u, + TASK_OVER = 0x4u, }; typedef struct SResultRowCell { @@ -129,6 +122,7 @@ typedef struct { } SOperatorProfResult; typedef struct STaskCostInfo { + int64_t created; int64_t start; int64_t end; @@ -246,13 +240,14 @@ typedef struct STaskIdInfo { uint64_t taskId; // this is a subplan id } STaskIdInfo; -typedef struct STaskInfo { +typedef struct SExecTaskInfo { STaskIdInfo id; char *content; uint32_t status; STimeWindow window; STaskCostInfo cost; int64_t owner; // if it is in execution + int32_t code; STableGroupInfo tableqinfoGroupInfo; // this is a group array list, including SArray structure pthread_mutex_t lock; // used to synchronize the rsp/query threads @@ -260,8 +255,9 @@ typedef struct STaskInfo { // int32_t dataReady; // denote if query result is ready or not // void* rspContext; // response context char *sql; // query sql string - jmp_buf env; -} STaskInfo; + jmp_buf env; // + struct SOperatorInfo *pRoot; +} SExecTaskInfo; typedef struct STaskRuntimeEnv { jmp_buf env; @@ -269,7 +265,7 @@ typedef struct STaskRuntimeEnv { uint32_t status; // query status void* qinfo; uint8_t scanFlag; // denotes reversed scan of data or not - void* pQueryHandle; + void* pTsdbReadHandle; int32_t prevGroupId; // previous executed group id bool enableGroupData; @@ -314,8 +310,8 @@ typedef struct SOperatorInfo { char *name; // name, used to show the query execution plan void *info; // extension attribution SExprInfo *pExpr; - STaskRuntimeEnv *pRuntimeEnv; - STaskInfo *pTaskInfo; + STaskRuntimeEnv *pRuntimeEnv; // todo remove it + SExecTaskInfo *pTaskInfo; struct SOperatorInfo **pDownstream; // downstram pointer list int32_t numOfDownstream; // number of downstream. The value is always ONE expect for join operator @@ -376,7 +372,7 @@ typedef struct STaskParam { } STaskParam; typedef struct STableScanInfo { - void *pQueryHandle; + void *pTsdbReadHandle; int32_t numOfBlocks; int32_t numOfSkipped; int32_t numOfBlockStatis; @@ -544,7 +540,7 @@ typedef struct SOrderOperatorInfo { void appendUpstream(SOperatorInfo* p, SOperatorInfo* pUpstream); SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, STaskRuntimeEnv* pRuntimeEnv, int32_t repeatTime, int32_t reverseTime); -SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime); +SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, SExecTaskInfo* pTaskInfo); SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, STaskRuntimeEnv* pRuntimeEnv); SOperatorInfo* createAggregateOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput); @@ -572,11 +568,11 @@ SOperatorInfo* createFilterOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI SOperatorInfo* createJoinOperatorInfo(SOperatorInfo** pUpstream, int32_t numOfUpstream, SSchema* pSchema, int32_t numOfOutput); SOperatorInfo* createOrderOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorInfo* upstream, SExprInfo* pExpr, int32_t numOfOutput, SOrder* pOrderVal); -SSDataBlock* doGlobalAggregate(void* param, bool* newgroup); -SSDataBlock* doMultiwayMergeSort(void* param, bool* newgroup); -SSDataBlock* doSLimit(void* param, bool* newgroup); +//SSDataBlock* doGlobalAggregate(void* param, bool* newgroup); +//SSDataBlock* doMultiwayMergeSort(void* param, bool* newgroup); +//SSDataBlock* doSLimit(void* param, bool* newgroup); -int32_t doCreateFilterInfo(SColumnInfo* pCols, int32_t numOfCols, int32_t numOfFilterCols, SSingleColumnFilterInfo** pFilterInfo, uint64_t qId); +//int32_t doCreateFilterInfo(SColumnInfo* pCols, int32_t numOfCols, int32_t numOfFilterCols, SSingleColumnFilterInfo** pFilterInfo, uint64_t qId); void doSetFilterColumnInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, SSDataBlock* pBlock); bool doFilterDataBlock(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFilterCols, int32_t numOfRows, int8_t* p); void doCompactSDataBlock(SSDataBlock* pBlock, int32_t numOfRows, int8_t* p); @@ -617,14 +613,14 @@ STableQueryInfo* createTmpTableQueryInfo(STimeWindow win); int32_t buildArithmeticExprFromMsg(SExprInfo *pArithExprInfo, void *pQueryMsg); -bool isQueryKilled(SQInfo *pQInfo); +bool isTaskKilled(SExecTaskInfo *pTaskInfo); int32_t checkForQueryBuf(size_t numOfTables); bool checkNeedToCompressQueryCol(SQInfo *pQInfo); bool doBuildResCheck(SQInfo* pQInfo); void setQueryStatus(STaskRuntimeEnv *pRuntimeEnv, int8_t status); bool onlyQueryTags(STaskAttr* pQueryAttr); -void destroyUdfInfo(struct SUdfInfo* pUdfInfo); +//void destroyUdfInfo(struct SUdfInfo* pUdfInfo); bool isValidQInfo(void *param); @@ -644,5 +640,7 @@ void freeQueryAttr(STaskAttr *pQuery); int32_t getMaximumIdleDurationSec(); void doInvokeUdf(struct SUdfInfo* pUdfInfo, SQLFunctionCtx *pCtx, int32_t idx, int32_t type); +void setTaskStatus(SExecTaskInfo *pTaskInfo, int8_t status); +int32_t doCreateExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, void* readerHandle); #endif // TDENGINE_EXECUTORIMPL_H diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index e8ecffb72c..5253add5af 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -547,7 +547,7 @@ static UNUSED_FUNC int32_t mergeIntoGroupResultImpl(STaskRuntimeEnv *pRuntimeEnv pTableQueryInfoList = malloc(POINTER_BYTES * size); if (pTableQueryInfoList == NULL || posList == NULL || pGroupResInfo->pRows == NULL || pGroupResInfo->pRows == NULL) { -// qError("QInfo:%"PRIu64" failed alloc memory", GET_QID(pRuntimeEnv)); +// qError("QInfo:%"PRIu64" failed alloc memory", GET_TASKID(pRuntimeEnv)); code = TSDB_CODE_QRY_OUT_OF_MEMORY; goto _end; } @@ -619,7 +619,7 @@ static UNUSED_FUNC int32_t mergeIntoGroupResultImpl(STaskRuntimeEnv *pRuntimeEnv int64_t endt = taosGetTimestampMs(); -// qDebug("QInfo:%"PRIx64" result merge completed for group:%d, elapsed time:%" PRId64 " ms", GET_QID(pRuntimeEnv), +// qDebug("QInfo:%"PRIx64" result merge completed for group:%d, elapsed time:%" PRId64 " ms", GET_TASKID(pRuntimeEnv), // pGroupResInfo->currentGroup, endt - startt); _end: @@ -641,13 +641,13 @@ int32_t mergeIntoGroupResult(SGroupResInfo* pGroupResInfo, STaskRuntimeEnv* pRun break; } -// qDebug("QInfo:%"PRIu64" no result in group %d, continue", GET_QID(pRuntimeEnv), pGroupResInfo->currentGroup); +// qDebug("QInfo:%"PRIu64" no result in group %d, continue", GET_TASKID(pRuntimeEnv), pGroupResInfo->currentGroup); cleanupGroupResInfo(pGroupResInfo); incNextGroup(pGroupResInfo); } // int64_t elapsedTime = taosGetTimestampUs() - st; -// qDebug("QInfo:%"PRIu64" merge res data into group, index:%d, total group:%d, elapsed time:%" PRId64 "us", GET_QID(pRuntimeEnv), +// qDebug("QInfo:%"PRIu64" merge res data into group, index:%d, total group:%d, elapsed time:%" PRId64 "us", GET_TASKID(pRuntimeEnv), // pGroupResInfo->currentGroup, pGroupResInfo->totalGroup, elapsedTime); return TSDB_CODE_SUCCESS; diff --git a/source/libs/executor/src/executorMain.c b/source/libs/executor/src/executorMain.c index 968380ea01..ab9e7a5211 100644 --- a/source/libs/executor/src/executorMain.c +++ b/source/libs/executor/src/executorMain.c @@ -13,11 +13,12 @@ * along with this program. If not, see . */ +#include +#include "exception.h" #include "os.h" #include "tcache.h" #include "tglobal.h" #include "tmsg.h" -#include "exception.h" #include "thash.h" #include "executorimpl.h" @@ -66,152 +67,24 @@ void freeParam(STaskParam *param) { tfree(param->prevResult); } -// todo parse json to get the operator tree. +int32_t qCreateExecTask(void* tsdb, int32_t vgId, SSubplan* pSubplan, qTaskInfo_t* pTaskInfo) { + assert(tsdb != NULL && pSubplan != NULL); -int32_t qCreateTask(void* tsdb, int32_t vgId, void* pQueryMsg, qTaskInfo_t* pTaskInfo, uint64_t taskId) { - assert(pQueryMsg != NULL && tsdb != NULL); - - int32_t code = TSDB_CODE_SUCCESS; -#if 0 - STaskParam param = {0}; - code = convertQueryMsg(pQueryMsg, ¶m); + int32_t code = doCreateExecTaskInfo(pSubplan, (SExecTaskInfo**) pTaskInfo, tsdb); if (code != TSDB_CODE_SUCCESS) { - goto _over; + goto _error; } - if (pQueryMsg->numOfTables <= 0) { - qError("Invalid number of tables to query, numOfTables:%d", pQueryMsg->numOfTables); - code = TSDB_CODE_QRY_INVALID_MSG; - goto _over; - } - - if (param.pTableIdList == NULL || taosArrayGetSize(param.pTableIdList) == 0) { - qError("qmsg:%p, SQueryTableReq wrong format", pQueryMsg); - code = TSDB_CODE_QRY_INVALID_MSG; - goto _over; - } - - SQueriedTableInfo info = { .numOfTags = pQueryMsg->numOfTags, .numOfCols = pQueryMsg->numOfCols, .colList = pQueryMsg->tableCols}; - if ((code = createQueryFunc(&info, pQueryMsg->numOfOutput, ¶m.pExprs, param.pExpr, param.pTagColumnInfo, - pQueryMsg->queryType, pQueryMsg, param.pUdfInfo)) != TSDB_CODE_SUCCESS) { - goto _over; - } - - if (param.pSecExpr != NULL) { - if ((code = createIndirectQueryFuncExprFromMsg(pQueryMsg, pQueryMsg->secondStageOutput, ¶m.pSecExprs, param.pSecExpr, param.pExprs, param.pUdfInfo)) != TSDB_CODE_SUCCESS) { - goto _over; - } - } - - if (param.colCond != NULL) { - if ((code = createQueryFilter(param.colCond, pQueryMsg->colCondLen, ¶m.pFilters)) != TSDB_CODE_SUCCESS) { - goto _over; - } - } - - param.pGroupbyExpr = createGroupbyExprFromMsg(pQueryMsg, param.pGroupColIndex, &code); - if ((param.pGroupbyExpr == NULL && pQueryMsg->numOfGroupCols != 0) || code != TSDB_CODE_SUCCESS) { - goto _over; - } - - bool isSTableQuery = false; - STableGroupInfo tableGroupInfo = {0}; - int64_t st = taosGetTimestampUs(); - - if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_TABLE_QUERY)) { - STableIdInfo *id = taosArrayGet(param.pTableIdList, 0); - - qDebug("qmsg:%p query normal table, uid:%"PRId64", tid:%d", pQueryMsg, id->uid, id->tid); - if ((code = tsdbGetOneTableGroup(tsdb, id->uid, pQueryMsg->window.skey, &tableGroupInfo)) != TSDB_CODE_SUCCESS) { - goto _over; - } - } else if (TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY|TSDB_QUERY_TYPE_STABLE_QUERY)) { - isSTableQuery = true; - - // also note there's possibility that only one table in the super table - if (!TSDB_QUERY_HAS_TYPE(pQueryMsg->queryType, TSDB_QUERY_TYPE_MULTITABLE_QUERY)) { - STableIdInfo *id = taosArrayGet(param.pTableIdList, 0); - - // group by normal column, do not pass the group by condition to tsdb to group table into different group - int32_t numOfGroupByCols = pQueryMsg->numOfGroupCols; - if (pQueryMsg->numOfGroupCols == 1 && !TSDB_COL_IS_TAG(param.pGroupColIndex->flag)) { - numOfGroupByCols = 0; - } - - qDebug("qmsg:%p query stable, uid:%"PRIu64", tid:%d", pQueryMsg, id->uid, id->tid); - code = tsdbQuerySTableByTagCond(tsdb, id->uid, pQueryMsg->window.skey, param.tagCond, pQueryMsg->tagCondLen, - pQueryMsg->tagNameRelType, param.tbnameCond, &tableGroupInfo, param.pGroupColIndex, numOfGroupByCols); - - if (code != TSDB_CODE_SUCCESS) { - qError("qmsg:%p failed to query stable, reason: %s", pQueryMsg, tstrerror(code)); - goto _over; - } - } else { - code = tsdbGetTableGroupFromIdList(tsdb, param.pTableIdList, &tableGroupInfo); - if (code != TSDB_CODE_SUCCESS) { - goto _over; - } - - qDebug("qmsg:%p query on %u tables in one group from client", pQueryMsg, tableGroupInfo.numOfTables); - } - - int64_t el = taosGetTimestampUs() - st; - qDebug("qmsg:%p tag filter completed, numOfTables:%u, elapsed time:%"PRId64"us", pQueryMsg, tableGroupInfo.numOfTables, el); - } else { - assert(0); - } - - code = checkForQueryBuf(tableGroupInfo.numOfTables); - if (code != TSDB_CODE_SUCCESS) { // not enough query buffer, abort - goto _over; - } - - assert(pQueryMsg->stableQuery == isSTableQuery); - (*pTaskInfo) = createQInfoImpl(pQueryMsg, param.pGroupbyExpr, param.pExprs, param.pSecExprs, &tableGroupInfo, - param.pTagColumnInfo, param.pFilters, vgId, param.sql, qId, param.pUdfInfo); - - param.sql = NULL; - param.pExprs = NULL; - param.pSecExprs = NULL; - param.pGroupbyExpr = NULL; - param.pTagColumnInfo = NULL; - param.pFilters = NULL; - - if ((*pTaskInfo) == NULL) { - code = TSDB_CODE_QRY_OUT_OF_MEMORY; - goto _over; - } - param.pUdfInfo = NULL; - - code = initQInfo(&pQueryMsg->tsBuf, tsdb, NULL, *pTaskInfo, ¶m, (char*)pQueryMsg, pQueryMsg->prevResultLen, NULL); - - _over: - if (param.pGroupbyExpr != NULL) { - taosArrayDestroy(param.pGroupbyExpr->columnInfo); - } - - tfree(param.colCond); - - destroyUdfInfo(param.pUdfInfo); - - taosArrayDestroy(param.pTableIdList); - param.pTableIdList = NULL; - - freeParam(¶m); - - for (int32_t i = 0; i < pQueryMsg->numOfCols; i++) { - SColumnInfo* column = pQueryMsg->tableCols + i; - freeColumnFilterInfo(column->flist.filterInfo, column->flist.numOfFilters); - } - - filterFreeInfo(param.pFilters); - - //pTaskInfo already freed in initQInfo, but *pTaskInfo may not pointer to null; + SDataSinkMgtCfg cfg = {.maxDataBlockNum = 1000}; + code = dsDataSinkMgtInit(&cfg); if (code != TSDB_CODE_SUCCESS) { - *pTaskInfo = NULL; + goto _error; } -#endif + DataSinkHandle pHandle = NULL; + code = dsCreateDataSinker(pSubplan->pDataSink, &pHandle); + + _error: // if failed to add ref for all tables in this query, abort current query return code; } @@ -250,7 +123,7 @@ int waitMoment(SQInfo* pQInfo){ while(used_ms < ms) { taosMsleep(1000); used_ms += 1000; - if(isQueryKilled(pQInfo)){ + if(isTaskKilled(pQInfo)){ printf("test check query is canceled, sleep break.%s\n", pQInfo->sql); break; } @@ -261,68 +134,64 @@ int waitMoment(SQInfo* pQInfo){ } #endif -bool qExecTask(qTaskInfo_t qinfo, uint64_t *qId) { - SQInfo *pQInfo = (SQInfo *)qinfo; - assert(pQInfo && pQInfo->signature == pQInfo); +bool qExecTask(qTaskInfo_t tinfo, SSDataBlock** pRes) { + SExecTaskInfo *pTaskInfo = (SExecTaskInfo *) tinfo; int64_t threadId = taosGetSelfPthreadId(); int64_t curOwner = 0; - if ((curOwner = atomic_val_compare_exchange_64(&pQInfo->owner, 0, threadId)) != 0) { - qError("QInfo:0x%"PRIx64"-%p qhandle is now executed by thread:%p", pQInfo->qId, pQInfo, (void*) curOwner); - pQInfo->code = TSDB_CODE_QRY_IN_EXEC; + if ((curOwner = atomic_val_compare_exchange_64(&pTaskInfo->owner, 0, threadId)) != 0) { + qError("QInfo:0x%"PRIx64"-%p qhandle is now executed by thread:%p", GET_TASKID(pTaskInfo), pTaskInfo, (void*) curOwner); + pTaskInfo->code = TSDB_CODE_QRY_IN_EXEC; return false; } - *qId = pQInfo->qId; - if(pQInfo->startExecTs == 0) - pQInfo->startExecTs = taosGetTimestampMs(); - - if (isQueryKilled(pQInfo)) { - qDebug("QInfo:0x%"PRIx64" it is already killed, abort", pQInfo->qId); - return doBuildResCheck(pQInfo); + if(pTaskInfo->cost.start == 0) { + pTaskInfo->cost.start = taosGetTimestampMs(); } - STaskRuntimeEnv* pRuntimeEnv = &pQInfo->runtimeEnv; - if (pRuntimeEnv->tableqinfoGroupInfo.numOfTables == 0) { - qDebug("QInfo:0x%"PRIx64" no table exists for query, abort", pQInfo->qId); -// setTaskStatus(pRuntimeEnv, QUERY_COMPLETED); - return doBuildResCheck(pQInfo); + if (isTaskKilled(pTaskInfo)) { + qDebug("QInfo:0x%"PRIx64" it is already killed, abort", GET_TASKID(pTaskInfo)); +// return doBuildResCheck(pTaskInfo); } +// STaskRuntimeEnv* pRuntimeEnv = &pTaskInfo->runtimeEnv; +// if (pTaskInfo->tableqinfoGroupInfo.numOfTables == 0) { +// qDebug("QInfo:0x%"PRIx64" no table exists for query, abort", GET_TASKID(pTaskInfo)); +// setTaskStatus(pTaskInfo, TASK_COMPLETED); +// return doBuildResCheck(pTaskInfo); +// } + // error occurs, record the error code and return to client - int32_t ret = setjmp(pQInfo->runtimeEnv.env); + int32_t ret = setjmp(pTaskInfo->env); if (ret != TSDB_CODE_SUCCESS) { - publishQueryAbortEvent(pQInfo, ret); - pQInfo->code = ret; - qDebug("QInfo:0x%"PRIx64" query abort due to error/cancel occurs, code:%s", pQInfo->qId, tstrerror(pQInfo->code)); - return doBuildResCheck(pQInfo); + publishQueryAbortEvent(pTaskInfo, ret); + pTaskInfo->code = ret; + qDebug("QInfo:0x%"PRIx64" query abort due to error/cancel occurs, code:%s", GET_TASKID(pTaskInfo), tstrerror(pTaskInfo->code)); +// return doBuildResCheck(pTaskInfo); } - qDebug("QInfo:0x%"PRIx64" query task is launched", pQInfo->qId); + qDebug("QInfo:0x%"PRIx64" query task is launched", GET_TASKID(pTaskInfo)); bool newgroup = false; - publishOperatorProfEvent(pRuntimeEnv->proot, QUERY_PROF_BEFORE_OPERATOR_EXEC); + publishOperatorProfEvent(pTaskInfo->pRoot, QUERY_PROF_BEFORE_OPERATOR_EXEC); int64_t st = taosGetTimestampUs(); - pRuntimeEnv->outputBuf = pRuntimeEnv->proot->exec(pRuntimeEnv->proot, &newgroup); - pQInfo->summary.elapsedTime += (taosGetTimestampUs() - st); -#ifdef TEST_IMPL - waitMoment(pQInfo); -#endif - publishOperatorProfEvent(pRuntimeEnv->proot, QUERY_PROF_AFTER_OPERATOR_EXEC); - pRuntimeEnv->resultInfo.total += GET_NUM_OF_RESULTS(pRuntimeEnv); + *pRes = pTaskInfo->pRoot->exec(pTaskInfo->pRoot, &newgroup); + // todo put the result into sink node. - if (isQueryKilled(pQInfo)) { - qDebug("QInfo:0x%"PRIx64" query is killed", pQInfo->qId); - } else if (GET_NUM_OF_RESULTS(pRuntimeEnv) == 0) { - qDebug("QInfo:0x%"PRIx64" over, %u tables queried, total %"PRId64" rows returned", pQInfo->qId, pRuntimeEnv->tableqinfoGroupInfo.numOfTables, - pRuntimeEnv->resultInfo.total); + pTaskInfo->cost.elapsedTime += (taosGetTimestampUs() - st); + publishOperatorProfEvent(pTaskInfo->pRoot, QUERY_PROF_AFTER_OPERATOR_EXEC); + + if (isTaskKilled(pTaskInfo)) { + qDebug("QInfo:0x%"PRIx64" query is killed", GET_TASKID(pTaskInfo)); +// } else if (GET_NUM_OF_RESULTS(pRuntimeEnv) == 0) { +// qDebug("QInfo:0x%"PRIx64" over, %u tables queried, total %"PRId64" rows returned", pTaskInfo->qId, pRuntimeEnv->tableqinfoGroupInfo.numOfTables, +// pRuntimeEnv->resultInfo.total); } else { - qDebug("QInfo:0x%"PRIx64" query paused, %d rows returned, total:%" PRId64 " rows", pQInfo->qId, - GET_NUM_OF_RESULTS(pRuntimeEnv), pRuntimeEnv->resultInfo.total); +// qDebug("QInfo:0x%"PRIx64" query paused, %d rows returned, total:%" PRId64 " rows", pTaskInfo->qId, +// GET_NUM_OF_RESULTS(pRuntimeEnv), pRuntimeEnv->resultInfo.total); } - - return doBuildResCheck(pQInfo); +// return doBuildResCheck(pTaskInfo); } int32_t qRetrieveQueryResultInfo(qTaskInfo_t qinfo, bool* buildRes, void* pRspContext) { @@ -398,13 +267,13 @@ int32_t qKillTask(qTaskInfo_t qinfo) { } int32_t qIsTaskCompleted(qTaskInfo_t qinfo) { - SQInfo *pQInfo = (SQInfo *)qinfo; + SExecTaskInfo *pTaskInfo = (SExecTaskInfo *)qinfo; - if (pQInfo == NULL || !isValidQInfo(pQInfo)) { + if (pTaskInfo == NULL /*|| !isValidQInfo(pTaskInfo)*/) { return TSDB_CODE_QRY_INVALID_QHANDLE; } - return isQueryKilled(pQInfo) || Q_STATUS_EQUAL(pQInfo->runtimeEnv.status, QUERY_OVER); + return isTaskKilled(pTaskInfo) || Q_STATUS_EQUAL(pTaskInfo->status, TASK_OVER); } void qDestroyTask(qTaskInfo_t qHandle) { diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index b00d37f828..db25e384be 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -26,6 +26,7 @@ #include "thash.h" #include "ttypes.h" #include "query.h" +#include "tsdb.h" #define IS_MAIN_SCAN(runtime) ((runtime)->scanFlag == MAIN_SCAN) #define IS_REVERSE_SCAN(runtime) ((runtime)->scanFlag == REVERSE_SCAN) @@ -204,12 +205,10 @@ static void destroyStateWindowOperatorInfo(void* param, int32_t numOfOutput); static void destroyAggOperatorInfo(void* param, int32_t numOfOutput); static void destroyOperatorInfo(SOperatorInfo* pOperator); -void setTaskStatus(STaskInfo *pTaskInfo, int8_t status); - static void doSetOperatorCompleted(SOperatorInfo* pOperator) { pOperator->status = OP_EXEC_DONE; if (pOperator->pTaskInfo != NULL) { - setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); + setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); } } @@ -1509,7 +1508,7 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SGroupbyOperatorInfo *pIn int16_t type = pColInfoData->info.type; if (type == TSDB_DATA_TYPE_FLOAT || type == TSDB_DATA_TYPE_DOUBLE) { - //qError("QInfo:0x%"PRIx64" group by not supported on double/float columns, abort", GET_QID(pRuntimeEnv)); + //qError("QInfo:0x%"PRIx64" group by not supported on double/float columns, abort", GET_TASKID(pRuntimeEnv)); return; } @@ -1908,7 +1907,7 @@ static SQLFunctionCtx* createSQLFunctionCtx(STaskRuntimeEnv* pRuntimeEnv, SExprI pCtx->param[2].i = pQueryAttr->window.ekey; pCtx->param[2].nType = TSDB_DATA_TYPE_BIGINT; } else if (functionId == FUNCTION_ARITHM) { - pCtx->param[1].pz = (char*) getScalarFuncSupport(pRuntimeEnv->scalarSup, i); +// pCtx->param[1].pz = (char*) getScalarFuncSupport(pRuntimeEnv->scalarSup, i); } } @@ -1940,7 +1939,7 @@ static void* destroySQLFunctionCtx(SQLFunctionCtx* pCtx, int32_t numOfOutput) { } static int32_t setupQueryRuntimeEnv(STaskRuntimeEnv *pRuntimeEnv, int32_t numOfTables, SArray* pOperator, void* merger) { - //qDebug("QInfo:0x%"PRIx64" setup runtime env", GET_QID(pRuntimeEnv)); + //qDebug("QInfo:0x%"PRIx64" setup runtime env", GET_TASKID(pRuntimeEnv)); STaskAttr *pQueryAttr = pRuntimeEnv->pQueryAttr; pRuntimeEnv->prevGroupId = INT32_MIN; @@ -1977,7 +1976,7 @@ static int32_t setupQueryRuntimeEnv(STaskRuntimeEnv *pRuntimeEnv, int32_t numOfT } } - //qDebug("QInfo:0x%"PRIx64" init runtime environment completed", GET_QID(pRuntimeEnv)); + //qDebug("QInfo:0x%"PRIx64" init runtime environment completed", GET_TASKID(pRuntimeEnv)); // group by normal column, sliding window query, interval query are handled by interval query processor // interval (down sampling operation) @@ -2164,8 +2163,8 @@ _clean: static void doFreeQueryHandle(STaskRuntimeEnv* pRuntimeEnv) { STaskAttr* pQueryAttr = pRuntimeEnv->pQueryAttr; -// tsdbCleanupQueryHandle(pRuntimeEnv->pQueryHandle); - pRuntimeEnv->pQueryHandle = NULL; +// tsdbCleanupQueryHandle(pRuntimeEnv->pTsdbReadHandle); + pRuntimeEnv->pTsdbReadHandle = NULL; // SMemRef* pMemRef = &pQueryAttr->memRef; // assert(pMemRef->ref == 0 && pMemRef->snapshot.imem == NULL && pMemRef->snapshot.mem == NULL); @@ -2191,7 +2190,7 @@ static void teardownQueryRuntimeEnv(STaskRuntimeEnv *pRuntimeEnv) { //qDebug("QInfo:0x%"PRIx64" teardown runtime env", pQInfo->qId); destroyScalarFuncSupport(pRuntimeEnv->scalarSup, pQueryAttr->numOfOutput); - destroyUdfInfo(pRuntimeEnv->pUdfInfo); +// destroyUdfInfo(pRuntimeEnv->pUdfInfo); destroyResultBuf(pRuntimeEnv->pResultBuf); doFreeQueryHandle(pRuntimeEnv); @@ -2224,18 +2223,18 @@ static bool needBuildResAfterQueryComplete(SQInfo* pQInfo) { return pQInfo->rspContext != NULL; } -bool isQueryKilled(SQInfo *pQInfo) { - if (IS_QUERY_KILLED(pQInfo)) { +bool isTaskKilled(SExecTaskInfo *pTaskInfo) { + if (IS_QUERY_KILLED(pTaskInfo)) { return true; } // query has been executed more than tsShellActivityTimer, and the retrieve has not arrived // abort current query execution. - if (pQInfo->owner != 0 && ((taosGetTimestampSec() - pQInfo->startExecTs/1000) > getMaximumIdleDurationSec()) && - (!needBuildResAfterQueryComplete(pQInfo))) { + if (pTaskInfo->owner != 0 && ((taosGetTimestampSec() - pTaskInfo->cost.start/1000) > 10*getMaximumIdleDurationSec()) + /*(!needBuildResAfterQueryComplete(pTaskInfo))*/) { - assert(pQInfo->startExecTs != 0); - //qDebug("QInfo:%" PRIu64 " retrieve not arrive beyond %d ms, abort current query execution, start:%" PRId64 + assert(pTaskInfo->cost.start != 0); +// qDebug("QInfo:%" PRIu64 " retrieve not arrive beyond %d ms, abort current query execution, start:%" PRId64 // ", current:%d", pQInfo->qId, 1, pQInfo->startExecTs, taosGetTimestampSec()); return true; } @@ -2887,7 +2886,7 @@ void doSetFilterColumnInfo(SSingleColumnFilterInfo* pFilterInfo, int32_t numOfFi } } -int32_t loadDataBlock(STaskInfo *pTaskInfo, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { +int32_t loadDataBlock(SExecTaskInfo *pTaskInfo, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { STaskCostInfo* pCost = &pTaskInfo->cost; pCost->totalBlocks += 1; @@ -2896,13 +2895,15 @@ int32_t loadDataBlock(STaskInfo *pTaskInfo, STableScanInfo* pTableScanInfo, SSDa pCost->totalCheckedRows += pBlock->info.rows; pCost->loadBlocks += 1; -// pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pQueryHandle, NULL); + pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pTsdbReadHandle, NULL); if (pBlock->pDataBlock == NULL) { return terrno; + } else { + return TSDB_CODE_SUCCESS; } } -int32_t loadDataBlockOnDemand(STaskInfo *pTaskInfo, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { +int32_t loadDataBlockOnDemand(SExecTaskInfo *pTaskInfo, STableScanInfo* pTableScanInfo, SSDataBlock* pBlock, uint32_t* status) { *status = BLK_DATA_NO_NEEDED; pBlock->pDataBlock = NULL; @@ -2991,10 +2992,10 @@ int32_t loadDataBlockOnDemand(STaskInfo *pTaskInfo, STableScanInfo* pTableScanIn } else if ((*status) == BLK_DATA_STATIS_NEEDED) { // this function never returns error? pCost->loadBlockStatis += 1; -// tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pQueryHandle, &pBlock->pBlockAgg); +// tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pTsdbReadHandle, &pBlock->pBlockAgg); if (pBlock->pBlockAgg == NULL) { // data block statistics does not exist, load data block -// pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pQueryHandle, NULL); +// pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pTsdbReadHandle, NULL); pCost->totalCheckedRows += pBlock->info.rows; } } else { @@ -3002,7 +3003,7 @@ int32_t loadDataBlockOnDemand(STaskInfo *pTaskInfo, STableScanInfo* pTableScanIn // load the data block statistics to perform further filter pCost->loadBlockStatis += 1; -// tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pQueryHandle, &pBlock->pBlockAgg); +// tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->pTsdbReadHandle, &pBlock->pBlockAgg); if (pQueryAttr->topBotQuery && pBlock->pBlockAgg != NULL) { { // set previous window @@ -3048,7 +3049,7 @@ int32_t loadDataBlockOnDemand(STaskInfo *pTaskInfo, STableScanInfo* pTableScanIn pCost->totalCheckedRows += pBlockInfo->rows; pCost->loadBlocks += 1; -// pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pQueryHandle, NULL); +// pBlock->pDataBlock = tsdbRetrieveDataBlock(pTableScanInfo->pTsdbReadHandle, NULL); // if (pBlock->pDataBlock == NULL) { // return terrno; // } @@ -3449,12 +3450,12 @@ void initCtxOutputBuffer(SQLFunctionCtx* pCtx, int32_t size) { } } -void setTaskStatus(STaskInfo *pTaskInfo, int8_t status) { - if (status == QUERY_NOT_COMPLETED) { +void setTaskStatus(SExecTaskInfo *pTaskInfo, int8_t status) { + if (status == TASK_NOT_COMPLETED) { pTaskInfo->status = status; } else { // QUERY_NOT_COMPLETED is not compatible with any other status, so clear its position first - CLEAR_QUERY_STATUS(pTaskInfo, QUERY_NOT_COMPLETED); + CLEAR_QUERY_STATUS(pTaskInfo, TASK_NOT_COMPLETED); pTaskInfo->status |= status; } } @@ -3476,6 +3477,10 @@ static void setupEnvForReverseScan(STableScanInfo *pTableScanInfo, SQLFunctionCt SWITCH_ORDER(pTableScanInfo->order); setupQueryRangeForReverseScan(pTableScanInfo); + + pTableScanInfo->times = 1; + pTableScanInfo->current = 0; + pTableScanInfo->reverseTimes = 0; } void finalizeQueryResult(SOperatorInfo* pOperator, SQLFunctionCtx* pCtx, SResultRowInfo* pResultRowInfo, int32_t* rowCellInfoOffset) { @@ -3700,10 +3705,10 @@ void setCtxTagForJoin(STaskRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx, SExprI // // int16_t tagType = pCtx[0].tag.nType; // if (tagType == TSDB_DATA_TYPE_BINARY || tagType == TSDB_DATA_TYPE_NCHAR) { -// //qDebug("QInfo:0x%"PRIx64" set tag value for join comparison, colId:%" PRId64 ", val:%s", GET_QID(pRuntimeEnv), +// //qDebug("QInfo:0x%"PRIx64" set tag value for join comparison, colId:%" PRId64 ", val:%s", GET_TASKID(pRuntimeEnv), //// pExprInfo->base.param[0].i, pCtx[0].tag.pz); // } else { -// //qDebug("QInfo:0x%"PRIx64" set tag value for join comparison, colId:%" PRId64 ", val:%" PRId64, GET_QID(pRuntimeEnv), +// //qDebug("QInfo:0x%"PRIx64" set tag value for join comparison, colId:%" PRId64 ", val:%" PRId64, GET_TASKID(pRuntimeEnv), //// pExprInfo->base.param[0].i, pCtx[0].tag.i); // } // } @@ -3723,9 +3728,9 @@ int32_t setTimestampListJoinInfo(STaskRuntimeEnv* pRuntimeEnv, SVariant* pTag, S // failed to find data with the specified tag value and vnodeId if (!tsBufIsValidElem(&elem)) { if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { - //qError("QInfo:0x%"PRIx64" failed to find tag:%s in ts_comp", GET_QID(pRuntimeEnv), pTag->pz); + //qError("QInfo:0x%"PRIx64" failed to find tag:%s in ts_comp", GET_TASKID(pRuntimeEnv), pTag->pz); } else { - //qError("QInfo:0x%"PRIx64" failed to find tag:%" PRId64 " in ts_comp", GET_QID(pRuntimeEnv), pTag->i); + //qError("QInfo:0x%"PRIx64" failed to find tag:%" PRId64 " in ts_comp", GET_TASKID(pRuntimeEnv), pTag->i); } return -1; @@ -3734,17 +3739,17 @@ int32_t setTimestampListJoinInfo(STaskRuntimeEnv* pRuntimeEnv, SVariant* pTag, S // Keep the cursor info of current table pTableQueryInfo->cur = tsBufGetCursor(pRuntimeEnv->pTsBuf); if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { - //qDebug("QInfo:0x%"PRIx64" find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_QID(pRuntimeEnv), pTag->pz, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); + //qDebug("QInfo:0x%"PRIx64" find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_TASKID(pRuntimeEnv), pTag->pz, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); } else { - //qDebug("QInfo:0x%"PRIx64" find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_QID(pRuntimeEnv), pTag->i, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); + //qDebug("QInfo:0x%"PRIx64" find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_TASKID(pRuntimeEnv), pTag->i, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); } } else { tsBufSetCursor(pRuntimeEnv->pTsBuf, &pTableQueryInfo->cur); if (pTag->nType == TSDB_DATA_TYPE_BINARY || pTag->nType == TSDB_DATA_TYPE_NCHAR) { - //qDebug("QInfo:0x%"PRIx64" find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_QID(pRuntimeEnv), pTag->pz, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); + //qDebug("QInfo:0x%"PRIx64" find tag:%s start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_TASKID(pRuntimeEnv), pTag->pz, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); } else { - //qDebug("QInfo:0x%"PRIx64" find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_QID(pRuntimeEnv), pTag->i, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); + //qDebug("QInfo:0x%"PRIx64" find tag:%"PRId64" start pos in ts_comp, blockIndex:%d, tsIndex:%d", GET_TASKID(pRuntimeEnv), pTag->i, pTableQueryInfo->cur.blockIndex, pTableQueryInfo->cur.tsIndex); } } @@ -3882,7 +3887,7 @@ static int32_t doCopyToSDataBlock(STaskRuntimeEnv* pRuntimeEnv, SGroupResInfo* p int32_t start = 0; int32_t step = -1; - //qDebug("QInfo:0x%"PRIx64" start to copy data from windowResInfo to output buf", GET_QID(pRuntimeEnv)); + //qDebug("QInfo:0x%"PRIx64" start to copy data from windowResInfo to output buf", GET_TASKID(pRuntimeEnv)); assert(orderType == TSDB_ORDER_ASC || orderType == TSDB_ORDER_DESC); if (orderType == TSDB_ORDER_ASC) { @@ -3927,7 +3932,7 @@ static int32_t doCopyToSDataBlock(STaskRuntimeEnv* pRuntimeEnv, SGroupResInfo* p } } - //qDebug("QInfo:0x%"PRIx64" copy data to query buf completed", GET_QID(pRuntimeEnv)); + //qDebug("QInfo:0x%"PRIx64" copy data to query buf completed", GET_TASKID(pRuntimeEnv)); pBlock->info.rows = numOfResult; return 0; } @@ -4055,7 +4060,7 @@ static void doCopyQueryResultToMsg(SQInfo *pQInfo, int32_t numOfRows, char *data //qDebug("QInfo:0x%"PRIx64" set %d subscribe info", pQInfo->qId, total); // Check if query is completed or not for stable query or normal table query respectively. - if (Q_STATUS_EQUAL(pRuntimeEnv->status, QUERY_COMPLETED) && pRuntimeEnv->proot->status == OP_EXEC_DONE) { + if (Q_STATUS_EQUAL(pRuntimeEnv->status, TASK_COMPLETED) && pRuntimeEnv->proot->status == OP_EXEC_DONE) { // setTaskStatus(pOperator->pTaskInfo, QUERY_OVER); } } @@ -4232,7 +4237,7 @@ void queryCostStatis(SQInfo *pQInfo) { // // assert(pQueryAttr->pos >= 0 && pQueryAttr->pos <= pBlockInfo->rows - 1); // -// SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); +// SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pTsdbReadHandle, NULL); // SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); // // // update the pQueryAttr->limit.offset value, and pQueryAttr->pos value @@ -4244,7 +4249,7 @@ void queryCostStatis(SQInfo *pQInfo) { // // int32_t numOfRes = tableApplyFunctionsOnBlock(pRuntimeEnv, pBlockInfo, NULL, binarySearchForKey, pDataBlock); // -// //qDebug("QInfo:0x%"PRIx64" check data block, brange:%" PRId64 "-%" PRId64 ", numBlocksOfStep:%d, numOfRes:%d, lastKey:%"PRId64, GET_QID(pRuntimeEnv), +// //qDebug("QInfo:0x%"PRIx64" check data block, brange:%" PRId64 "-%" PRId64 ", numBlocksOfStep:%d, numOfRes:%d, lastKey:%"PRId64, GET_TASKID(pRuntimeEnv), // pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes, pQuery->current->lastKey); //} @@ -4259,22 +4264,22 @@ void queryCostStatis(SQInfo *pQInfo) { // int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQueryAttr->order.order); // // STableQueryInfo* pTableQueryInfo = pRuntimeEnv->current; -// TsdbQueryHandleT pQueryHandle = pRuntimeEnv->pQueryHandle; +// TsdbQueryHandleT pTsdbReadHandle = pRuntimeEnv->pTsdbReadHandle; // // SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; -// while (tsdbNextDataBlock(pQueryHandle)) { -// if (isQueryKilled(pRuntimeEnv->qinfo)) { +// while (tsdbNextDataBlock(pTsdbReadHandle)) { +// if (isTaskKilled(pRuntimeEnv->qinfo)) { // longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); // } // -// tsdbRetrieveDataBlockInfo(pQueryHandle, &blockInfo); +// tsdbRetrieveDataBlockInfo(pTsdbReadHandle, &blockInfo); // // if (pQueryAttr->limit.offset > blockInfo.rows) { // pQueryAttr->limit.offset -= blockInfo.rows; // pTableQueryInfo->lastKey = (QUERY_IS_ASC_QUERY(pQueryAttr)) ? blockInfo.window.ekey : blockInfo.window.skey; // pTableQueryInfo->lastKey += step; // -// //qDebug("QInfo:0x%"PRIx64" skip rows:%d, offset:%" PRId64, GET_QID(pRuntimeEnv), blockInfo.rows, +// //qDebug("QInfo:0x%"PRIx64" skip rows:%d, offset:%" PRId64, GET_TASKID(pRuntimeEnv), blockInfo.rows, // pQuery->limit.offset); // } else { // find the appropriated start position in current block // updateOffsetVal(pRuntimeEnv, &blockInfo); @@ -4300,7 +4305,7 @@ void queryCostStatis(SQInfo *pQInfo) { // // // load the data block and check data remaining in current data block // // TODO optimize performance -// SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); +// SArray * pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pTsdbReadHandle, NULL); // SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); // // tw = *win; @@ -4323,7 +4328,7 @@ void queryCostStatis(SQInfo *pQInfo) { // pRuntimeEnv->resultRowInfo.curIndex = index; // restore the window index // // //qDebug("QInfo:0x%"PRIx64" check data block, brange:%" PRId64 "-%" PRId64 ", numOfRows:%d, numOfRes:%d, lastKey:%" PRId64, -// GET_QID(pRuntimeEnv), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes, +// GET_TASKID(pRuntimeEnv), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, numOfRes, // pQueryAttr->current->lastKey); // // return key; @@ -4365,8 +4370,8 @@ void queryCostStatis(SQInfo *pQInfo) { // STableQueryInfo *pTableQueryInfo = pRuntimeEnv->current; // // SDataBlockInfo blockInfo = SDATA_BLOCK_INITIALIZER; -// while (tsdbNextDataBlock(pRuntimeEnv->pQueryHandle)) { -// tsdbRetrieveDataBlockInfo(pRuntimeEnv->pQueryHandle, &blockInfo); +// while (tsdbNextDataBlock(pRuntimeEnv->pTsdbReadHandle)) { +// tsdbRetrieveDataBlockInfo(pRuntimeEnv->pTsdbReadHandle, &blockInfo); // // if (QUERY_IS_ASC_QUERY(pQueryAttr)) { // if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) { @@ -4412,7 +4417,7 @@ void queryCostStatis(SQInfo *pQInfo) { // */ // if ((tw.skey <= blockInfo.window.ekey && ascQuery) || (tw.ekey >= blockInfo.window.skey && !ascQuery)) { // -// SArray *pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pQueryHandle, NULL); +// SArray *pDataBlock = tsdbRetrieveDataBlock(pRuntimeEnv->pTsdbReadHandle, NULL); // SColumnInfoData *pColInfoData = taosArrayGet(pDataBlock, 0); // // if ((win.ekey > blockInfo.window.ekey && ascQuery) || (win.ekey < blockInfo.window.skey && !ascQuery)) { @@ -4486,7 +4491,7 @@ static int32_t setupQueryHandle(void* tsdb, STaskRuntimeEnv* pRuntimeEnv, int64_ terrno = TSDB_CODE_SUCCESS; if (isFirstLastRowQuery(pQueryAttr)) { - pRuntimeEnv->pQueryHandle = tsdbQueryLastRow(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); + pRuntimeEnv->pTsdbReadHandle = tsdbQueryLastRow(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); // update the query time window pQueryAttr->window = cond.twindow; @@ -4507,11 +4512,11 @@ static int32_t setupQueryHandle(void* tsdb, STaskRuntimeEnv* pRuntimeEnv, int64_ } } } else if (isCachedLastQuery(pQueryAttr)) { - pRuntimeEnv->pQueryHandle = tsdbQueryCacheLast(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); + pRuntimeEnv->pTsdbReadHandle = tsdbQueryCacheLast(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); } else if (pQueryAttr->pointInterpQuery) { - pRuntimeEnv->pQueryHandle = tsdbQueryRowsInExternalWindow(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); + pRuntimeEnv->pTsdbReadHandle = tsdbQueryRowsInExternalWindow(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); } else { - pRuntimeEnv->pQueryHandle = tsdbQueryTables(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); + pRuntimeEnv->pTsdbReadHandle = tsdbQueryTables(tsdb, &cond, &pQueryAttr->tableGroupInfo, qId, &pQueryAttr->memRef); } #endif return terrno; @@ -4543,19 +4548,19 @@ int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr switch(tbScanner) { // case OP_TableBlockInfoScan: { -// pRuntimeEnv->proot = createTableBlockInfoScanOperator(pRuntimeEnv->pQueryHandle, pRuntimeEnv); +// pRuntimeEnv->proot = createTableBlockInfoScanOperator(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv); // break; // } // case OP_TableSeqScan: { -// pRuntimeEnv->proot = createTableSeqScanOperator(pRuntimeEnv->pQueryHandle, pRuntimeEnv); +// pRuntimeEnv->proot = createTableSeqScanOperator(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv); // break; // } // case OP_DataBlocksOptScan: { -// pRuntimeEnv->proot = createDataBlocksOptScanInfo(pRuntimeEnv->pQueryHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr), pQueryAttr->needReverseScan? 1:0); +// pRuntimeEnv->proot = createDataBlocksOptScanInfo(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr), pQueryAttr->needReverseScan? 1:0); // break; // } // case OP_TableScan: { -// pRuntimeEnv->proot = createTableScanOperator(pRuntimeEnv->pQueryHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr)); +// pRuntimeEnv->proot = createTableScanOperator(pRuntimeEnv->pTsdbReadHandle, pRuntimeEnv, getNumOfScanTimes(pQueryAttr)); // break; // } default: { // do nothing @@ -4606,7 +4611,7 @@ int32_t doInitQInfo(SQInfo* pQInfo, STSBuf* pTsBuf, void* tsdb, void* sourceOptr return TSDB_CODE_SUCCESS; } -static void doTableQueryInfoTimeWindowCheck(STaskInfo* pTaskInfo, STableQueryInfo* pTableQueryInfo, int32_t order) { +static void doTableQueryInfoTimeWindowCheck(SExecTaskInfo* pTaskInfo, STableQueryInfo* pTableQueryInfo, int32_t order) { if (order == TSDB_ORDER_ASC) { assert( (pTableQueryInfo->win.skey <= pTableQueryInfo->win.ekey) && @@ -4679,20 +4684,20 @@ static SSDataBlock* doTableScanImpl(void* param, bool* newgroup) { SOperatorInfo *pOperator = (SOperatorInfo*) param; STableScanInfo *pTableScanInfo = pOperator->info; - STaskInfo *pTaskInfo = pOperator->pTaskInfo; + SExecTaskInfo *pTaskInfo = pOperator->pTaskInfo; SSDataBlock *pBlock = &pTableScanInfo->block; STableGroupInfo *pTableGroupInfo = &pOperator->pTaskInfo->tableqinfoGroupInfo; *newgroup = false; - while (/*tsdbNextDataBlock(pTableScanInfo->pQueryHandle)*/1) { - if (isQueryKilled(pOperator->pRuntimeEnv->qinfo)) { - longjmp(pOperator->pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED); + while (tsdbNextDataBlock(pTableScanInfo->pTsdbReadHandle)) { + if (isTaskKilled(pOperator->pTaskInfo)) { + longjmp(pOperator->pTaskInfo->env, TSDB_CODE_TSC_QUERY_CANCELLED); } pTableScanInfo->numOfBlocks += 1; -// tsdbRetrieveDataBlockInfo(pTableScanInfo->pQueryHandle, &pBlock->info); + tsdbRetrieveDataBlockInfo(pTableScanInfo->pTsdbReadHandle, &pBlock->info); // todo opt // if (pTableGroupInfo->numOfTables > 1 || (pRuntimeEnv->current == NULL && pTableGroupInfo->numOfTables == 1)) { @@ -4711,7 +4716,7 @@ static SSDataBlock* doTableScanImpl(void* param, bool* newgroup) { int32_t code = loadDataBlock(pTaskInfo, pTableScanInfo, pBlock, &status); // int32_t code = loadDataBlockOnDemand(pOperator->pRuntimeEnv, pTableScanInfo, pBlock, &status); if (code != TSDB_CODE_SUCCESS) { - longjmp(pOperator->pRuntimeEnv->env, code); + longjmp(pOperator->pTaskInfo->env, code); } // current block is ignored according to filter result by block statistics data, continue load the next block @@ -4729,7 +4734,7 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { SOperatorInfo* pOperator = (SOperatorInfo*) param; STableScanInfo *pTableScanInfo = pOperator->info; - STaskInfo *pTaskInfo = pOperator->pTaskInfo; + SExecTaskInfo *pTaskInfo = pOperator->pTaskInfo; SResultRowInfo* pResultRowInfo = pTableScanInfo->pResultRowInfo; *newgroup = false; @@ -4741,7 +4746,7 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { } if (++pTableScanInfo->current >= pTableScanInfo->times) { - if (pTableScanInfo->reverseTimes <= 0/* || isTsdbCacheLastRow(pTableScanInfo->pQueryHandle)*/) { + if (pTableScanInfo->reverseTimes <= 0/* || isTsdbCacheLastRow(pTableScanInfo->pTsdbReadHandle)*/) { return NULL; } else { break; @@ -4750,9 +4755,9 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { // do prepare for the next round table scan operation // STsdbQueryCond cond = createTsdbQueryCond(pQueryAttr, &pQueryAttr->window); -// tsdbResetQueryHandle(pTableScanInfo->pQueryHandle, &cond); +// tsdbResetQueryHandle(pTableScanInfo->pTsdbReadHandle, &cond); - setTaskStatus(pTaskInfo, QUERY_NOT_COMPLETED); + setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); pTableScanInfo->scanFlag = REPEAT_SCAN; // if (pTaskInfo->pTsBuf) { @@ -4764,8 +4769,8 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { pResultRowInfo->curPos = 0; } - //qDebug("QInfo:0x%"PRIx64" start to repeat scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64, -// GET_QID(pRuntimeEnv), cond.twindow.skey, cond.twindow.ekey); + qDebug("QInfo:0x%"PRIx64" start to repeat scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64, + GET_TASKID(pTaskInfo), pTaskInfo->window.skey, pTaskInfo->window.ekey); } SSDataBlock *p = NULL; @@ -4773,15 +4778,10 @@ static SSDataBlock* doTableScan(void* param, bool *newgroup) { if (pTableScanInfo->reverseTimes > 0) { setupEnvForReverseScan(pTableScanInfo, pTableScanInfo->pCtx, pTableScanInfo->numOfOutput); // STsdbQueryCond cond = createTsdbQueryCond(pQueryAttr, &pQueryAttr->window); -// tsdbResetQueryHandle(pTableScanInfo->pQueryHandle, &cond); +// tsdbResetQueryHandle(pTableScanInfo->pTsdbReadHandle, &cond); - //qDebug("QInfo:0x%"PRIx64" start to reverse scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64, -// GET_QID(pRuntimeEnv), cond.twindow.skey, cond.twindow.ekey); - - pTableScanInfo->times = 1; - pTableScanInfo->current = 0; - pTableScanInfo->reverseTimes = 0; -// pTableScanInfo->order = cond.order; + qDebug("QInfo:0x%"PRIx64" start to reverse scan data blocks due to query func required, qrange:%" PRId64 "-%" PRId64, + GET_TASKID(pTaskInfo), pTaskInfo->window.skey, pTaskInfo->window.ekey); if (pResultRowInfo->size > 0) { pResultRowInfo->curPos = pResultRowInfo->size - 1; @@ -4814,8 +4814,8 @@ static SSDataBlock* doBlockInfoScan(void* param, bool* newgroup) { tableBlockDist.maxRows = INT_MIN; tableBlockDist.minRows = INT_MAX; - tsdbGetFileBlocksDistInfo(pTableScanInfo->pQueryHandle, &tableBlockDist); - tableBlockDist.numOfRowsInMemTable = (int32_t) tsdbGetNumOfRowsInMemTable(pTableScanInfo->pQueryHandle); + tsdbGetFileBlocksDistInfo(pTableScanInfo->pTsdbReadHandle, &tableBlockDist); + tableBlockDist.numOfRowsInMemTable = (int32_t) tsdbGetNumOfRowsInMemTable(pTableScanInfo->pTsdbReadHandle); SSDataBlock* pBlock = &pTableScanInfo->block; pBlock->info.rows = 1; @@ -4842,11 +4842,11 @@ static SSDataBlock* doBlockInfoScan(void* param, bool* newgroup) { } -SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime) { +SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, int32_t order, int32_t numOfOutput, int32_t repeatTime, SExecTaskInfo* pTaskInfo) { assert(repeatTime > 0 && numOfOutput > 0); STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - pInfo->pQueryHandle = pTsdbQueryHandle; + pInfo->pTsdbReadHandle = pTsdbQueryHandle; pInfo->times = repeatTime; pInfo->reverseTimes = 0; pInfo->order = order; @@ -4862,6 +4862,7 @@ SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, int32_t order, in pOperator->numOfOutput = numOfOutput; pOperator->pRuntimeEnv = NULL; pOperator->exec = doTableScan; + pOperator->pTaskInfo = pTaskInfo; return pOperator; } @@ -4869,7 +4870,7 @@ SOperatorInfo* createTableScanOperator(void* pTsdbQueryHandle, int32_t order, in SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, STaskRuntimeEnv* pRuntimeEnv) { STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - pInfo->pQueryHandle = pTsdbQueryHandle; + pInfo->pTsdbReadHandle = pTsdbQueryHandle; pInfo->times = 1; pInfo->reverseTimes = 0; pInfo->order = pRuntimeEnv->pQueryAttr->order.order; @@ -4893,7 +4894,7 @@ SOperatorInfo* createTableSeqScanOperator(void* pTsdbQueryHandle, STaskRuntimeEn SOperatorInfo* createTableBlockInfoScanOperator(void* pTsdbQueryHandle, STaskRuntimeEnv* pRuntimeEnv) { STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - pInfo->pQueryHandle = pTsdbQueryHandle; + pInfo->pTsdbReadHandle = pTsdbQueryHandle; pInfo->block.pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); SColumnInfoData infoData = {{0}}; @@ -4977,7 +4978,7 @@ SOperatorInfo* createDataBlocksOptScanInfo(void* pTsdbQueryHandle, STaskRuntimeE assert(repeatTime > 0); STableScanInfo* pInfo = calloc(1, sizeof(STableScanInfo)); - pInfo->pQueryHandle = pTsdbQueryHandle; + pInfo->pTsdbReadHandle = pTsdbQueryHandle; pInfo->times = repeatTime; pInfo->reverseTimes = reverseTime; pInfo->current = 0; @@ -5145,7 +5146,7 @@ SOperatorInfo* createGlobalAggregateOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, S pOperator->numOfOutput = numOfOutput; pOperator->pRuntimeEnv = pRuntimeEnv; - pOperator->exec = doGlobalAggregate; +// pOperator->exec = doGlobalAggregate; pOperator->cleanup = destroyGlobalAggOperatorInfo; appendUpstream(pOperator, downstream); @@ -5188,7 +5189,7 @@ SOperatorInfo *createMultiwaySortOperatorInfo(STaskRuntimeEnv *pRuntimeEnv, SExp pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->numOfOutput = numOfOutput; pOperator->pExpr = pExpr; - pOperator->exec = doMultiwayMergeSort; +// pOperator->exec = doMultiwayMergeSort; pOperator->cleanup = destroyGlobalAggOperatorInfo; return pOperator; } @@ -5476,7 +5477,7 @@ static SSDataBlock* doProjectOperation(void* param, bool* newgroup) { assert(*newgroup == false); *newgroup = prevVal; - setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); + setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); break; } @@ -5644,7 +5645,7 @@ static SSDataBlock* doIntervalAgg(void* param, bool* newgroup) { pOperator->status = OP_RES_TO_RETURN; closeAllResultRows(&pIntervalInfo->resultRowInfo); - setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); + setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); finalizeQueryResult(pOperator, pIntervalInfo->pCtx, &pIntervalInfo->resultRowInfo, pIntervalInfo->rowCellInfoOffset); initGroupResInfo(&pRuntimeEnv->groupResInfo, &pIntervalInfo->resultRowInfo); @@ -5704,7 +5705,7 @@ static SSDataBlock* doAllIntervalAgg(void* param, bool* newgroup) { pOperator->status = OP_RES_TO_RETURN; closeAllResultRows(&pIntervalInfo->resultRowInfo); - setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); + setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); finalizeQueryResult(pOperator, pIntervalInfo->pCtx, &pIntervalInfo->resultRowInfo, pIntervalInfo->rowCellInfoOffset); initGroupResInfo(&pRuntimeEnv->groupResInfo, &pIntervalInfo->resultRowInfo); @@ -5767,7 +5768,7 @@ static SSDataBlock* doSTableIntervalAgg(void* param, bool* newgroup) { pOperator->status = OP_RES_TO_RETURN; pQueryAttr->order.order = order; // TODO : restore the order doCloseAllTimeWindow(pRuntimeEnv); - setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); + setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); if (pIntervalInfo->pRes->info.rows == 0 || !hasRemainData(&pRuntimeEnv->groupResInfo)) { @@ -5822,7 +5823,7 @@ static SSDataBlock* doAllSTableIntervalAgg(void* param, bool* newgroup) { pOperator->status = OP_RES_TO_RETURN; pQueryAttr->order.order = order; // TODO : restore the order doCloseAllTimeWindow(pRuntimeEnv); - setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); + setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); int64_t st = taosGetTimestampUs(); copyToSDataBlock(pRuntimeEnv, 3000, pIntervalInfo->pRes, pIntervalInfo->rowCellInfoOffset); @@ -5956,7 +5957,7 @@ static SSDataBlock* doStateWindowAgg(void *param, bool* newgroup) { pOperator->status = OP_RES_TO_RETURN; closeAllResultRows(&pBInfo->resultRowInfo); - setTaskStatus(pOperator->pTaskInfo, QUERY_COMPLETED); + setTaskStatus(pOperator->pTaskInfo, TASK_COMPLETED); finalizeQueryResult(pOperator, pBInfo->pCtx, &pBInfo->resultRowInfo, pBInfo->rowCellInfoOffset); initGroupResInfo(&pRuntimeEnv->groupResInfo, &pBInfo->resultRowInfo); @@ -6094,7 +6095,7 @@ static SSDataBlock* hashGroupbyAggregate(void* param, bool* newgroup) { static void doHandleRemainBlockForNewGroupImpl(SFillOperatorInfo *pInfo, STaskRuntimeEnv* pRuntimeEnv, bool* newgroup) { pInfo->totalInputRows = pInfo->existNewGroupBlock->info.rows; - int64_t ekey = Q_STATUS_EQUAL(pRuntimeEnv->status, QUERY_COMPLETED)?pRuntimeEnv->pQueryAttr->window.ekey:pInfo->existNewGroupBlock->info.window.ekey; + int64_t ekey = Q_STATUS_EQUAL(pRuntimeEnv->status, TASK_COMPLETED)?pRuntimeEnv->pQueryAttr->window.ekey:pInfo->existNewGroupBlock->info.window.ekey; taosResetFillInfo(pInfo->pFillInfo, getFillInfoStart(pInfo->pFillInfo)); taosFillSetStartInfo(pInfo->pFillInfo, pInfo->existNewGroupBlock->info.rows, ekey); @@ -6422,7 +6423,7 @@ SOperatorInfo* createFilterOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI SFilterOperatorInfo* pInfo = calloc(1, sizeof(SFilterOperatorInfo)); assert(numOfFilter > 0 && pCols != NULL); - doCreateFilterInfo(pCols, numOfOutput, numOfFilter, &pInfo->pFilterInfo, 0); +// doCreateFilterInfo(pCols, numOfOutput, numOfFilter, &pInfo->pFilterInfo, 0); pInfo->numOfFilterCols = numOfFilter; SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); @@ -6714,10 +6715,10 @@ SOperatorInfo* createSLimitOperatorInfo(STaskRuntimeEnv* pRuntimeEnv, SOperatorI SOperatorInfo* pOperator = calloc(1, sizeof(SOperatorInfo)); pOperator->name = "SLimitOperator"; -// pOperator->operatorType = OP_SLimit; + pOperator->operatorType = OP_SLimit; pOperator->blockingOptr = false; pOperator->status = OP_IN_EXECUTING; - pOperator->exec = doSLimit; +// pOperator->exec = doSLimit; pOperator->info = pInfo; pOperator->pRuntimeEnv = pRuntimeEnv; pOperator->cleanup = destroySlimitOperatorInfo; @@ -6799,14 +6800,14 @@ static SSDataBlock* doTagScan(void* param, bool* newgroup) { count += 1; } - //qDebug("QInfo:0x%"PRIx64" create (tableId, tag) info completed, rows:%d", GET_QID(pRuntimeEnv), count); + //qDebug("QInfo:0x%"PRIx64" create (tableId, tag) info completed, rows:%d", GET_TASKID(pRuntimeEnv), count); } else if (functionId == FUNCTION_COUNT) {// handle the "count(tbname)" query SColumnInfoData* pColInfo = taosArrayGet(pRes->pDataBlock, 0); *(int64_t*)pColInfo->pData = pInfo->totalTables; count = 1; pOperator->status = OP_EXEC_DONE; - //qDebug("QInfo:0x%"PRIx64" create count(tbname) query, res:%d rows:1", GET_QID(pRuntimeEnv), count); + //qDebug("QInfo:0x%"PRIx64" create count(tbname) query, res:%d rows:1", GET_TASKID(pRuntimeEnv), count); } else { // return only the tags|table name etc. SExprInfo* pExprInfo = &pOperator->pExpr[0]; // todo use the column list instead of exprinfo @@ -6845,11 +6846,11 @@ static SSDataBlock* doTagScan(void* param, bool* newgroup) { pOperator->status = OP_EXEC_DONE; } - //qDebug("QInfo:0x%"PRIx64" create tag values results completed, rows:%d", GET_QID(pRuntimeEnv), count); + //qDebug("QInfo:0x%"PRIx64" create tag values results completed, rows:%d", GET_TASKID(pRuntimeEnv), count); } if (pOperator->status == OP_EXEC_DONE) { - setTaskStatus(pOperator->pRuntimeEnv, QUERY_COMPLETED); + setTaskStatus(pOperator->pRuntimeEnv, TASK_COMPLETED); } pRes->info.rows = count; @@ -7174,90 +7175,52 @@ static int32_t deserializeColFilterInfo(SColumnFilterInfo* pColFilters, int16_t return TSDB_CODE_SUCCESS; } -/** - * { - "Id": { - "QueryId": 20, - "TemplateId": 0, - "SubplanId": 0 - }, - "Node": { - "Name": "TableScan", - "InputSchema": [{ - "Type": 9, - "ColId": 1, - "Bytes": 8 - }, { - "Type": 4, - "ColId": 2, - "Bytes": 4 - }, { - "Type": 8, - "ColId": 3, - "Bytes": 20 - }], - "TableScan": { - "TableId": 1, - "TableType": 3, - "Flag": 0, - "Window": { - "StartKey": 0, - "EndKey": 0 - } - } - }, - "DataSink": { - "Name": "Dispatch", - "Dispatch": { - } - } +static SExecTaskInfo* createExecTaskInfo(uint64_t queryId) { + SExecTaskInfo* pTaskInfo = calloc(1, sizeof(SExecTaskInfo)); + setTaskStatus(pTaskInfo, TASK_NOT_COMPLETED); + + pthread_mutex_init(&pTaskInfo->lock, NULL); + pTaskInfo->cost.created = taosGetTimestampMs(); + return pTaskInfo; } - */ -int32_t parseTaskInfo(const char* msg, int32_t len) { - cJSON* pJson = cJSON_Parse(msg); - if (NULL == pJson) { - return TSDB_CODE_INVALID_MSG; + +SOperatorInfo* doCreateOperatorTreeNode(SPhyNode* pPhyNode, SExecTaskInfo* pTaskInfo, void* param) { + if (pPhyNode->pChildren == NULL || taosArrayGetSize(pPhyNode->pChildren) == 0) { + if (pPhyNode->info.type == OP_TableScan) { + size_t numOfCols = taosArrayGetSize(pPhyNode->pTargets); + SOperatorInfo* pOperatorInfo = createTableScanOperator(param, TSDB_ORDER_ASC, numOfCols, 1, pTaskInfo); + pTaskInfo->pRoot = pOperatorInfo; + } } +} - cJSON* pSub = cJSON_GetObjectItem(pJson, "ID"); - if (NULL != pSub) { - printf("Id : %s\n", pSub->valuestring); - } +int32_t doCreateExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, void* readerHandle) { + STsdbQueryCond cond = {.order = TSDB_ORDER_ASC, .numOfCols = 2, .loadExternalRows = false}; + cond.twindow.skey = INT64_MIN; + cond.twindow.ekey = INT64_MAX; + cond.colList = calloc(cond.numOfCols, sizeof(SColumnInfo)); - cJSON* pNode = cJSON_GetObjectItem(pJson, "Node"); - if (pNode == NULL) { - return TSDB_CODE_INVALID_MSG; - } + // todo set the correct table column info + cond.colList[0].type = TSDB_DATA_TYPE_TIMESTAMP; + cond.colList[0].bytes = sizeof(uint64_t); + cond.colList[0].colId = 1; - cJSON* pNodeName = cJSON_GetObjectItem(pNode, "name"); - if (pNodeName == NULL) { - return TSDB_CODE_INVALID_MSG; - } + cond.colList[1].type = TSDB_DATA_TYPE_INT; + cond.colList[1].bytes = sizeof(int32_t); + cond.colList[1].colId = 2; - printf("node name is: %s\n", pNodeName->valuestring); - - cJSON* pNodeSchema = cJSON_GetObjectItem(pNode, "InputSchema"); - if (pNodeSchema == NULL) { - return TSDB_CODE_INVALID_MSG; - } - - cJSON* pOperator = cJSON_GetObjectItem(pNode, pNodeName->valuestring); - if (pOperator == NULL) { - return TSDB_CODE_INVALID_MSG; - } - - cJSON* pTableId = cJSON_GetObjectItem(pOperator, "tableId"); - if (pTableId == NULL) { - return TSDB_CODE_INVALID_MSG; - } - - cJSON* pTimeWindow = cJSON_GetObjectItem(pOperator, "window"); - if (pTimeWindow == NULL) { - return TSDB_CODE_INVALID_MSG; - } + STableGroupInfo group = {.numOfTables = 1, .pGroupList = taosArrayInit(1, POINTER_BYTES)}; + SArray* pa = taosArrayInit(1, sizeof(STableKeyInfo)); + STableKeyInfo info = {.pTable = NULL, .lastKey = 0, .uid = 1}; + taosArrayPush(pa, &info); + taosArrayPush(group.pGroupList, &pa); + *pTaskInfo = createExecTaskInfo((uint64_t)pPlan->id.queryId); + tsdbReadHandleT tsdbReadHandle = tsdbQueryTables(readerHandle, &cond, &group, (*pTaskInfo)->id.queryId, NULL); + doCreateOperatorTreeNode(pPlan->pNode, *pTaskInfo, tsdbReadHandle); + return TSDB_CODE_SUCCESS; } /** @@ -7704,7 +7667,7 @@ int32_t createQueryFunc(SQueriedTableInfo* pTableInfo, int32_t numOfOutput, SExp *pExprInfo = NULL; int32_t code = TSDB_CODE_SUCCESS; - code = initUdfInfo(pUdfInfo); +// code = initUdfInfo(pUdfInfo); if (code) { return code; } @@ -7996,8 +7959,8 @@ int32_t createFilterInfo(STaskAttr* pQueryAttr, uint64_t qId) { return TSDB_CODE_SUCCESS; } - doCreateFilterInfo(pQueryAttr->tableCols, pQueryAttr->numOfCols, pQueryAttr->numOfFilterCols, - &pQueryAttr->pFilterInfo, qId); +// doCreateFilterInfo(pQueryAttr->tableCols, pQueryAttr->numOfCols, pQueryAttr->numOfFilterCols, +// &pQueryAttr->pFilterInfo, qId); pQueryAttr->createFilterOperator = true; @@ -8505,7 +8468,7 @@ int32_t doDumpQueryResult(SQInfo *pQInfo, char *data, int8_t compressed, int32_t } // all data returned, set query over - if (Q_STATUS_EQUAL(pRuntimeEnv->status, QUERY_COMPLETED)) { + if (Q_STATUS_EQUAL(pRuntimeEnv->status, TASK_COMPLETED)) { // setTaskStatus(pOperator->pTaskInfo, QUERY_OVER); } } else { diff --git a/source/libs/executor/test/CMakeLists.txt b/source/libs/executor/test/CMakeLists.txt new file mode 100644 index 0000000000..ece84207c7 --- /dev/null +++ b/source/libs/executor/test/CMakeLists.txt @@ -0,0 +1,18 @@ + +MESSAGE(STATUS "build parser unit test") + +# GoogleTest requires at least C++11 +SET(CMAKE_CXX_STANDARD 11) +AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST) + +ADD_EXECUTABLE(executorTest ${SOURCE_LIST}) +TARGET_LINK_LIBRARIES( + executorTest + PUBLIC os util common transport gtest taos qcom executor function planner +) + +TARGET_INCLUDE_DIRECTORIES( + executorTest + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/executor/" + PRIVATE "${CMAKE_SOURCE_DIR}/source/libs/executor/inc" +) diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index e69de29bb2..5f5fddbe28 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -0,0 +1,221 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include +#include +#include +#include +#pragma GCC diagnostic ignored "-Wwrite-strings" + +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wsign-compare" +#include "os.h" + +#include "taos.h" +#include "tdef.h" +#include "tvariant.h" +#include "tep.h" +#include "trpc.h" +#include "stub.h" +#include "executor.h" + +/** +{ + "Id": { + "QueryId": 1.3108161807422521e+19, + "TemplateId": 0, + "SubplanId": 0 + }, + "Node": { + "Name": "TableScan", + "Targets": [{ + "Base": { + "Schema": { + "Type": 9, + "ColId": 5000, + "Bytes": 8 + }, + "Columns": [{ + "TableId": 1, + "Flag": 0, + "Info": { + "ColId": 1, + "Type": 9, + "Bytes": 8 + } + }], + "InterBytes": 0 + }, + "Expr": { + "Type": 4, + "Column": { + "Type": 9, + "ColId": 1, + "Bytes": 8 + } + } + }, { + "Base": { + "Schema": { + "Type": 4, + "ColId": 5001, + "Bytes": 4 + }, + "Columns": [{ + "TableId": 1, + "Flag": 0, + "Info": { + "ColId": 2, + "Type": 4, + "Bytes": 4 + } + }], + "InterBytes": 0 + }, + "Expr": { + "Type": 4, + "Column": { + "Type": 4, + "ColId": 2, + "Bytes": 4 + } + } + }], + "InputSchema": [{ + "Type": 9, + "ColId": 5000, + "Bytes": 8 + }, { + "Type": 4, + "ColId": 5001, + "Bytes": 4 + }], + "TableScan": { + "TableId": 1, + "TableType": 2, + "Flag": 0, + "Window": { + "StartKey": -9.2233720368547758e+18, + "EndKey": 9.2233720368547758e+18 + } + } + }, + "DataSink": { + "Name": "Dispatch", + "Dispatch": { + } + } +} + */ + +int main(int argc, char** argv) { + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +TEST(testCase, build_executor_tree_Test) { + + + const char* msg = "{\n" + "\t\"Id\":\t{\n" + "\t\t\"QueryId\":\t1.3108161807422521e+19,\n" + "\t\t\"TemplateId\":\t0,\n" + "\t\t\"SubplanId\":\t0\n" + "\t},\n" + "\t\"Node\":\t{\n" + "\t\t\"Name\":\t\"TableScan\",\n" + "\t\t\"Targets\":\t[{\n" + "\t\t\t\t\"Base\":\t{\n" + "\t\t\t\t\t\"Schema\":\t{\n" + "\t\t\t\t\t\t\"Type\":\t9,\n" + "\t\t\t\t\t\t\"ColId\":\t5000,\n" + "\t\t\t\t\t\t\"Bytes\":\t8\n" + "\t\t\t\t\t},\n" + "\t\t\t\t\t\"Columns\":\t[{\n" + "\t\t\t\t\t\t\t\"TableId\":\t1,\n" + "\t\t\t\t\t\t\t\"Flag\":\t0,\n" + "\t\t\t\t\t\t\t\"Info\":\t{\n" + "\t\t\t\t\t\t\t\t\"ColId\":\t1,\n" + "\t\t\t\t\t\t\t\t\"Type\":\t9,\n" + "\t\t\t\t\t\t\t\t\"Bytes\":\t8\n" + "\t\t\t\t\t\t\t}\n" + "\t\t\t\t\t\t}],\n" + "\t\t\t\t\t\"InterBytes\":\t0\n" + "\t\t\t\t},\n" + "\t\t\t\t\"Expr\":\t{\n" + "\t\t\t\t\t\"Type\":\t4,\n" + "\t\t\t\t\t\"Column\":\t{\n" + "\t\t\t\t\t\t\"Type\":\t9,\n" + "\t\t\t\t\t\t\"ColId\":\t1,\n" + "\t\t\t\t\t\t\"Bytes\":\t8\n" + "\t\t\t\t\t}\n" + "\t\t\t\t}\n" + "\t\t\t}, {\n" + "\t\t\t\t\"Base\":\t{\n" + "\t\t\t\t\t\"Schema\":\t{\n" + "\t\t\t\t\t\t\"Type\":\t4,\n" + "\t\t\t\t\t\t\"ColId\":\t5001,\n" + "\t\t\t\t\t\t\"Bytes\":\t4\n" + "\t\t\t\t\t},\n" + "\t\t\t\t\t\"Columns\":\t[{\n" + "\t\t\t\t\t\t\t\"TableId\":\t1,\n" + "\t\t\t\t\t\t\t\"Flag\":\t0,\n" + "\t\t\t\t\t\t\t\"Info\":\t{\n" + "\t\t\t\t\t\t\t\t\"ColId\":\t2,\n" + "\t\t\t\t\t\t\t\t\"Type\":\t4,\n" + "\t\t\t\t\t\t\t\t\"Bytes\":\t4\n" + "\t\t\t\t\t\t\t}\n" + "\t\t\t\t\t\t}],\n" + "\t\t\t\t\t\"InterBytes\":\t0\n" + "\t\t\t\t},\n" + "\t\t\t\t\"Expr\":\t{\n" + "\t\t\t\t\t\"Type\":\t4,\n" + "\t\t\t\t\t\"Column\":\t{\n" + "\t\t\t\t\t\t\"Type\":\t4,\n" + "\t\t\t\t\t\t\"ColId\":\t2,\n" + "\t\t\t\t\t\t\"Bytes\":\t4\n" + "\t\t\t\t\t}\n" + "\t\t\t\t}\n" + "\t\t\t}],\n" + "\t\t\"InputSchema\":\t[{\n" + "\t\t\t\t\"Type\":\t9,\n" + "\t\t\t\t\"ColId\":\t5000,\n" + "\t\t\t\t\"Bytes\":\t8\n" + "\t\t\t}, {\n" + "\t\t\t\t\"Type\":\t4,\n" + "\t\t\t\t\"ColId\":\t5001,\n" + "\t\t\t\t\"Bytes\":\t4\n" + "\t\t\t}],\n" + "\t\t\"TableScan\":\t{\n" + "\t\t\t\"TableId\":\t1,\n" + "\t\t\t\"TableType\":\t2,\n" + "\t\t\t\"Flag\":\t0,\n" + "\t\t\t\"Window\":\t{\n" + "\t\t\t\t\"StartKey\":\t-9.2233720368547758e+18,\n" + "\t\t\t\t\"EndKey\":\t9.2233720368547758e+18\n" + "\t\t\t}\n" + "\t\t}\n" + "\t},\n" + "\t\"DataSink\":\t{\n" + "\t\t\"Name\":\t\"Dispatch\",\n" + "\t\t\"Dispatch\":\t{\n" + "\t\t}\n" + "\t}\n" + "}"; + + SExecTaskInfo* pTaskInfo = nullptr; + int32_t code = qCreateExecTask((void*) 1, 2, NULL, (void**) &pTaskInfo); +} \ No newline at end of file diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index f440e6cdfe..5f17fcfaee 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -70,6 +70,9 @@ int32_t parseQuerySql(SParseContext* pCxt, SQueryNode** pQuery) { int32_t code = qParserValidateSqlNode(&pCxt->ctx, &info, pQueryInfo, pCxt->pMsg, pCxt->msgLen); if (code == TSDB_CODE_SUCCESS) { *pQuery = (SQueryNode*)pQueryInfo; + } else { + terrno = code; + return code; } } diff --git a/source/libs/planner/inc/plannerInt.h b/source/libs/planner/inc/plannerInt.h index 2a50752c88..be266bd415 100644 --- a/source/libs/planner/inc/plannerInt.h +++ b/source/libs/planner/inc/plannerInt.h @@ -62,7 +62,7 @@ typedef struct SQueryPlanNode { SSchema *pSchema; // the schema of the input SSDatablock int32_t numOfCols; // number of input columns SArray *pExpr; // the query functions or sql aggregations - int32_t numOfExpr; // number of result columns, which is also the number of pExprs + int32_t numOfExpr; // number of result columns, which is also the number of pExprs void *pExtInfo; // additional information // children operator to generated result for current node to process // in case of join, multiple prev nodes exist. diff --git a/source/libs/planner/src/physicalPlan.c b/source/libs/planner/src/physicalPlan.c index a38d110d5f..7ef434ccf5 100644 --- a/source/libs/planner/src/physicalPlan.c +++ b/source/libs/planner/src/physicalPlan.c @@ -88,16 +88,20 @@ static bool copySchema(SDataBlockSchema* dst, const SDataBlockSchema* src) { } static bool toDataBlockSchema(SQueryPlanNode* pPlanNode, SDataBlockSchema* dataBlockSchema) { - dataBlockSchema->numOfCols = pPlanNode->numOfCols; + dataBlockSchema->numOfCols = pPlanNode->numOfExpr; dataBlockSchema->pSchema = malloc(sizeof(SSlotSchema) * pPlanNode->numOfCols); if (NULL == dataBlockSchema->pSchema) { return false; } - memcpy(dataBlockSchema->pSchema, pPlanNode->pSchema, sizeof(SSlotSchema) * pPlanNode->numOfCols); + dataBlockSchema->resultRowSize = 0; - for (int32_t i = 0; i < dataBlockSchema->numOfCols; ++i) { + for (int32_t i = 0; i < pPlanNode->numOfExpr; ++i) { + SExprInfo* pExprInfo = taosArrayGetP(pPlanNode->pExpr, i); + memcpy(&dataBlockSchema->pSchema[i], &pExprInfo->base.resSchema, sizeof(SSlotSchema)); + dataBlockSchema->resultRowSize += dataBlockSchema->pSchema[i].bytes; } + return true; } @@ -284,7 +288,6 @@ static SPhyNode* createSingleTableScanNode(SQueryPlanNode* pPlanNode, SQueryTabl return createUserTableScanNode(pPlanNode, pTable, OP_TableScan); } - static SPhyNode* createTableScanNode(SPlanContext* pCxt, SQueryPlanNode* pPlanNode) { SQueryTableInfo* pTable = (SQueryTableInfo*)pPlanNode->pExtInfo; diff --git a/source/libs/planner/src/physicalPlanJson.c b/source/libs/planner/src/physicalPlanJson.c index d8918d27de..895788c550 100644 --- a/source/libs/planner/src/physicalPlanJson.c +++ b/source/libs/planner/src/physicalPlanJson.c @@ -154,9 +154,13 @@ static bool fromRawArrayWithAlloc(const cJSON* json, const char* name, FFromJson return fromItem(jArray, func, *array, itemSize, *size); } -static bool fromRawArray(const cJSON* json, const char* name, FFromJson func, void* array, int32_t itemSize, int32_t* size) { +static bool fromRawArray(const cJSON* json, const char* name, FFromJson func, void** array, int32_t itemSize, int32_t* size) { const cJSON* jArray = getArray(json, name, size); - return fromItem(jArray, func, array, itemSize, *size); + if (*array == NULL) { + *array = calloc(*size, itemSize); + } + + return fromItem(jArray, func, *array, itemSize, *size); } static char* getString(const cJSON* json, const char* name) { @@ -218,7 +222,8 @@ static bool dataBlockSchemaFromJson(const cJSON* json, void* obj) { SDataBlockSchema* schema = (SDataBlockSchema*)obj; schema->resultRowSize = getNumber(json, jkDataBlockSchemaResultRowSize); schema->precision = getNumber(json, jkDataBlockSchemaPrecision); - return fromRawArray(json, jkDataBlockSchemaSlotSchema, schemaFromJson, schema->pSchema, sizeof(SSlotSchema), &schema->numOfCols); + + return fromRawArray(json, jkDataBlockSchemaSlotSchema, schemaFromJson, (void**) &(schema->pSchema), sizeof(SSlotSchema), &schema->numOfCols); } static const char* jkColumnFilterInfoLowerRelOptr = "LowerRelOptr"; @@ -920,6 +925,8 @@ int32_t subPlanToString(const SSubplan* subplan, char** str, int32_t* len) { } *str = cJSON_Print(json); + + printf("%s\n", *str); *len = strlen(*str) + 1; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index f80a631413..de3454ddbb 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -56,35 +56,43 @@ void qDestroyQueryDag(struct SQueryDag* pDag) { tfree(pDag); } -int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, uint64_t requestId) { - SQueryPlanNode* logicPlan; - int32_t code = createQueryPlan(pNode, &logicPlan); +int32_t qCreateQueryDag(const struct SQueryNode* pNode, struct SQueryDag** pDag, SSchema** pSchema, uint32_t* numOfResCols, uint64_t requestId) { + SQueryPlanNode* pLogicPlan; + int32_t code = createQueryPlan(pNode, &pLogicPlan); if (TSDB_CODE_SUCCESS != code) { - destroyQueryPlan(logicPlan); + destroyQueryPlan(pLogicPlan); return code; } - // - if (logicPlan->info.type != QNODE_MODIFY) { -// char* str = NULL; -// queryPlanToString(logicPlan, &str); -// printf("%s\n", str); + if (pLogicPlan->info.type != QNODE_MODIFY) { + char* str = NULL; + queryPlanToString(pLogicPlan, &str); + printf("%s\n", str); } - code = optimizeQueryPlan(logicPlan); + int32_t numOfOutput = pLogicPlan->numOfExpr; + *pSchema = calloc(numOfOutput, sizeof(SSchema)); + *numOfResCols = numOfOutput; + + for(int32_t i = 0; i < numOfOutput; ++i) { + SExprInfo* pExprInfo = taosArrayGetP(pLogicPlan->pExpr, i); + memcpy(&(*pSchema)[i], pExprInfo->pExpr->pSchema, sizeof(SSchema)); + } + + code = optimizeQueryPlan(pLogicPlan); if (TSDB_CODE_SUCCESS != code) { - destroyQueryPlan(logicPlan); + destroyQueryPlan(pLogicPlan); return code; } - code = createDag(logicPlan, NULL, pDag, requestId); + code = createDag(pLogicPlan, NULL, pDag, requestId); if (TSDB_CODE_SUCCESS != code) { - destroyQueryPlan(logicPlan); + destroyQueryPlan(pLogicPlan); qDestroyQueryDag(*pDag); return code; } - destroyQueryPlan(logicPlan); + destroyQueryPlan(pLogicPlan); return TSDB_CODE_SUCCESS; } diff --git a/source/libs/planner/test/phyPlanTests.cpp b/source/libs/planner/test/phyPlanTests.cpp index 3f6f5ab77b..0d3820cc77 100644 --- a/source/libs/planner/test/phyPlanTests.cpp +++ b/source/libs/planner/test/phyPlanTests.cpp @@ -62,7 +62,10 @@ protected: } SQueryDag* dag = nullptr; uint64_t requestId = 20; - code = qCreateQueryDag(query, &dag, requestId); + SSchema *schema = NULL; + uint32_t numOfOutput = 0; + + code = qCreateQueryDag(query, &dag, &schema, &numOfOutput, requestId); dag_.reset(dag); return code; } diff --git a/source/libs/qworker/CMakeLists.txt b/source/libs/qworker/CMakeLists.txt index ea3e97057b..a3db9c6992 100644 --- a/source/libs/qworker/CMakeLists.txt +++ b/source/libs/qworker/CMakeLists.txt @@ -1,15 +1,31 @@ aux_source_directory(src QWORKER_SRC) -add_library(qworker ${QWORKER_SRC}) +#add_library(qworker ${QWORKER_SRC}) +#target_include_directories( +# qworker +# PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/qworker" +# PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +#) +# +#target_link_libraries( +# qworker +# PRIVATE os util transport planner qcom executor +#) + +add_library(qworker STATIC ${QWORKER_SRC}) target_include_directories( qworker PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/qworker" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" ) -target_link_libraries( - qworker - PRIVATE os util transport planner qcom -) +#set_target_properties(qworker PROPERTIES +# IMPORTED_LOCATION "${CMAKE_CURRENT_SOURCE_DIR}/libqworker.a" +# INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_SOURCE_DIR}/include/libs/qworker" +# ) + +target_link_libraries(qworker + PRIVATE os util transport planner qcom executor + ) if(${BUILD_TEST}) ADD_SUBDIRECTORY(test) diff --git a/source/libs/qworker/inc/qworkerInt.h b/source/libs/qworker/inc/qworkerInt.h index 91927c370a..691177fa83 100644 --- a/source/libs/qworker/inc/qworkerInt.h +++ b/source/libs/qworker/inc/qworkerInt.h @@ -67,15 +67,16 @@ typedef struct SQWTaskStatus { bool drop; } SQWTaskStatus; -typedef struct SQWorkerResCache { - SRWLatch lock; - void *data; -} SQWorkerResCache; +typedef struct SQWorkerTaskHandleCache { + SRWLatch lock; + qTaskInfo_t taskHandle; + DataSinkHandle sinkHandle; +} SQWorkerTaskHandleCache; typedef struct SQWSchStatus { int32_t lastAccessTs; // timestamp in second SRWLatch tasksLock; - SHashObj *tasksHash; // key:queryId+taskId, value: SQWorkerTaskStatus + SHashObj *tasksHash; // key:queryId+taskId, value: SQWTaskStatus } SQWSchStatus; // Qnode/Vnode level task management @@ -83,7 +84,7 @@ typedef struct SQWorkerMgmt { SQWorkerCfg cfg; SRWLatch schLock; SRWLatch resLock; - SHashObj *schHash; //key: schedulerId, value: SQWorkerSchStatus + SHashObj *schHash; //key: schedulerId, value: SQWSchStatus SHashObj *resHash; //key: queryId+taskId, value: SQWorkerResCache } SQWorkerMgmt; diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 15e894fd61..5f2f6214d8 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -1,9 +1,11 @@ #include "qworker.h" -#include "tname.h" +#include +#include "executor.h" #include "planner.h" #include "query.h" #include "qworkerInt.h" #include "tmsg.h" +#include "tname.h" int32_t qwValidateStatus(int8_t oriStatus, int8_t newStatus) { int32_t code = 0; @@ -89,12 +91,13 @@ int32_t qwUpdateTaskInfo(SQWTaskStatus *task, int8_t type, void *data) { return TSDB_CODE_SUCCESS; } -int32_t qwAddTaskResCache(SQWorkerMgmt *mgmt, uint64_t qId, uint64_t tId, void *data) { +int32_t qwAddTaskAndSinkToCache(SQWorkerMgmt *mgmt, uint64_t qId, uint64_t tId, qTaskInfo_t taskHandle, DataSinkHandle sinkHandle) { char id[sizeof(qId) + sizeof(tId)] = {0}; QW_SET_QTID(id, qId, tId); SQWorkerResCache resCache = {0}; - resCache.data = data; + resCache.taskHandle = taskHandle; + resCache.sinkHandle = sinkHandle; QW_LOCK(QW_WRITE, &mgmt->resLock); if (0 != taosHashPut(mgmt->resHash, id, sizeof(id), &resCache, sizeof(SQWorkerResCache))) { @@ -1011,7 +1014,7 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { bool queryDone = false; bool queryRsped = false; bool needStop = false; - SSubplan *plan = NULL; + struct SSubplan *plan = NULL; QW_ERR_JRET(qwCheckTaskCancelDrop(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, &needStop)); if (needStop) { @@ -1025,10 +1028,8 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { QW_ERR_JRET(code); } - //TODO call executer to init subquery - code = 0; // return error directly - //TODO call executer to init subquery - + qTaskInfo_t pTaskInfo = NULL; + code = qCreateExecTask(node, 0, (struct SSubplan *)plan, &pTaskInfo); if (code) { QW_ERR_JRET(code); } else { @@ -1039,22 +1040,19 @@ int32_t qWorkerProcessQueryMsg(void *node, void *qWorkerMgmt, SRpcMsg *pMsg) { queryRsped = true; - //TODO call executer to execute subquery - code = 0; - void *data = NULL; + SSDataBlock* pRes = NULL; + code = qExecTask(pTaskInfo, &pRes); + queryDone = false; - //TODO call executer to execute subquery if (code) { QW_ERR_JRET(code); } else { - QW_ERR_JRET(qwAddTaskResCache(qWorkerMgmt, msg->queryId, msg->taskId, data)); - + QW_ERR_JRET(qwAddTaskAndSinkToCache(qWorkerMgmt, msg->queryId, msg->taskId, pTaskInfo, NULL)); QW_ERR_JRET(qwUpdateTaskStatus(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, JOB_TASK_STATUS_PARTIAL_SUCCEED)); } _return: - if (queryRsped) { code = qwCheckAndSendReadyRsp(qWorkerMgmt, msg->sId, msg->queryId, msg->taskId, pMsg, code); } else { diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 4a0d62a06f..cbd07f2a07 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -269,7 +269,6 @@ int32_t schRecordTaskExecNode(SSchJob *pJob, SSchTask *pTask, SQueryNodeAddr *ad int32_t schValidateAndBuildJob(SQueryDag *pDag, SSchJob *pJob) { int32_t code = 0; - pJob->queryId = pDag->queryId; if (pDag->numOfSubplans <= 0) { @@ -367,7 +366,6 @@ int32_t schValidateAndBuildJob(SQueryDag *pDag, SSchJob *pJob) { SCH_ERR_JRET(schBuildTaskRalation(pJob, planToTask)); _return: - if (planToTask) { taosHashCleanup(planToTask); } @@ -762,6 +760,8 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch break; } + break; + } case TDMT_VND_QUERY_RSP: { SQueryTableRsp *rsp = (SQueryTableRsp *)msg; @@ -773,6 +773,8 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch break; } + break; + } case TDMT_VND_RES_READY_RSP: { SResReadyRsp *rsp = (SResReadyRsp *)msg; @@ -784,6 +786,8 @@ int32_t schHandleResponseMsg(SSchJob *pJob, SSchTask *pTask, int32_t msgType, ch break; } + break; + } case TDMT_VND_FETCH_RSP: { SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)msg; @@ -1325,7 +1329,7 @@ int32_t scheduleExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, void } int32_t scheduleAsyncExecJob(void *transport, SArray *nodeList, SQueryDag* pDag, void** pJob) { - if (NULL == transport || /*NULL == nodeList || */NULL == pDag || NULL == pDag->pSubplans || NULL == pJob) { + if (NULL == transport || NULL == pDag || NULL == pDag->pSubplans || NULL == pJob) { SCH_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } @@ -1485,8 +1489,4 @@ void scheduleFreeJob(void *job) { void schedulerDestroy(void) { if (schMgmt.jobs) { - taosHashCleanup(schMgmt.jobs); //TODO - schMgmt.jobs = NULL; - } - } - + taosHashCleanup(sch \ No newline at end of file