From 6c7235592b2344c5c45c25d8b48eb93bdbf094ac Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 6 Oct 2021 23:28:43 +0800 Subject: [PATCH 1/9] [TD-10529] add query module APIs. --- include/libs/catalog/catalog.h | 90 ++++++++++++++++++++++++++++ source/libs/catalog/inc/catalogInt.h | 7 +++ source/libs/catalog/src/catalog.c | 4 +- 3 files changed, 100 insertions(+), 1 deletion(-) diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index de37807cf6..395116b3bc 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -20,6 +20,96 @@ extern "C" { #endif +#include "os.h" +#include "thash.h" +#include "tarray.h" +#include "taosdef.h" +#include "transport.h" + +struct SCatalog; + +typedef struct SMetaReq { + char clusterId[TSDB_CLUSTER_ID_LEN]; + SArray *pTableName; // table full name + SArray *pVgroup; // vgroup id + SArray *pUdf; // udf name + bool qNodeEpset; // valid qnode +} SMetaReq; + +typedef struct SMetaData { + SArray *pTableMeta; // tableMeta + SArray *pVgroupInfo; // vgroupInfo list + SArray *pUdfList; // udf info list + SEpSet *pEpSet; // qnode epset list +} SMetaData; + +typedef struct STableComInfo { + uint8_t numOfTags; // the number of tags in schema + uint8_t precision; // the number of precision + int16_t numOfColumns; // the number of columns + int32_t rowSize; // row size of the schema +} STableComInfo; + +/* + * ASSERT(sizeof(SCTableMeta) == 24) + * ASSERT(tableType == TSDB_CHILD_TABLE) + * The cached child table meta info. For each child table, 24 bytes are required to keep the essential table info. + */ +typedef struct SCTableMeta { + int32_t vgId:24; + int8_t tableType; + uint32_t tid; + uint64_t uid; + uint64_t suid; +} SCTableMeta; + +typedef struct SSchema { + uint8_t type; + char name[TSDB_COL_NAME_LEN]; + int16_t colId; + int16_t bytes; +} SSchema; + +/* + * Note that the first 24 bytes of STableMeta are identical to SCTableMeta, it is safe to cast a STableMeta to be a SCTableMeta. + */ +typedef struct STableMeta { + int32_t vgId:24; + int8_t tableType; + uint32_t tid; + uint64_t uid; + uint64_t suid; + // if the table is TSDB_CHILD_TABLE, the following information is acquired from the corresponding super table meta info + int16_t sversion; + int16_t tversion; + STableComInfo tableInfo; + SSchema schema[]; +} STableMeta; + +/** + * Catalog service object, which is utilized to hold tableMeta (meta/vgroupInfo/udfInfo) at the client-side. + * There is ONLY one SCatalog object for one process space, and this function returns a singleton. + * @param pMgmtEps + * @return + */ +struct SCatalog* getCatalogHandle(const SEpSet* pMgmtEps); + +/** + * Get the required meta data from mnode. + * Note that this is a synchronized API and is also thread-safety. + * @param pCatalog + * @param pMetaReq + * @param pMetaData + * @return + */ +int32_t catalogGetMetaData(struct SCatalog* pCatalog, const SMetaReq* pMetaReq, SMetaData* pMetaData); + +/** + * Destroy catalog service handle + * @param pCatalog + */ +void destroyCatalog(struct SCatalog* pCatalog); + #ifdef __cplusplus } #endif diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index f6f8a61faf..5b50bbff4c 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -20,6 +20,13 @@ extern "C" { #endif +#include "catalog.h" + +typedef struct SCatalog { + void *pMsgSender; // used to send messsage to mnode to fetch necessary metadata + SHashObj *pData; // items cached for each cluster, the hash key is the cluster-id, returned by mgmt node +} SCatalog; + #ifdef __cplusplus } #endif diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 6dea4a4e57..c553e1bfbf 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -11,4 +11,6 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . - */ \ No newline at end of file + */ + +#include "catalogInt.h" From 4a843ca06a101248ef6e292a14630ba8b80a7682 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 6 Oct 2021 23:31:59 +0800 Subject: [PATCH 2/9] [TD-10529]remove unused APIs. --- include/client/taos.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/include/client/taos.h b/include/client/taos.h index edb1552b81..6fa30737e7 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -172,10 +172,6 @@ DLL_EXPORT int taos_load_table_info(TAOS *taos, const char* tableNameList); DLL_EXPORT int taos_insert_lines(TAOS* taos, char* lines[], int numLines); -DLL_EXPORT int taos_insert_telnet_lines(TAOS* taos, char* lines[], int numLines); - -DLL_EXPORT int taos_insert_json_payload(TAOS* taos, char* payload); - #ifdef __cplusplus } #endif From d845a13c75455ffc4d202a5ee88b05268dc8b57b Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 6 Oct 2021 23:34:38 +0800 Subject: [PATCH 3/9] [TD-10529]Add APIs of executor. --- include/libs/executor/executor.h | 165 +++++++++++++++++++++++++++++++ 1 file changed, 165 insertions(+) diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 15416724c6..246b08f693 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -20,6 +20,171 @@ extern "C" { #endif +typedef void* qinfo_t; + +/** + * create the qinfo object according to QueryTableMsg + * @param tsdb + * @param pQueryTableMsg + * @param qinfo + * @return + */ +int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryTableMsg, qinfo_t* qinfo, uint64_t qId); + + +/** + * the main query execution function, including query on both table and multiple tables, + * which are decided according to the tag or table name query conditions + * + * @param qinfo + * @return + */ +bool qTableQuery(qinfo_t qinfo, uint64_t *qId); + +/** + * Retrieve the produced results information, if current query is not paused or completed, + * this function will be blocked to wait for the query execution completed or paused, + * in which case enough results have been produced already. + * + * @param qinfo + * @return + */ +int32_t qRetrieveQueryResultInfo(qinfo_t qinfo, bool* buildRes, void* pRspContext); + +/** + * + * Retrieve the actual results to fill the response message payload. + * Note that this function must be executed after qRetrieveQueryResultInfo is invoked. + * + * @param qinfo qinfo object + * @param pRsp response message + * @param contLen payload length + * @return + */ +int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp** pRsp, int32_t* contLen, bool* continueExec); + +/** + * return the transporter context (RPC) + * @param qinfo + * @return + */ +void* qGetResultRetrieveMsg(qinfo_t qinfo); + +/** + * kill the ongoing query and free the query handle and corresponding resources automatically + * @param qinfo qhandle + * @return + */ +int32_t qKillQuery(qinfo_t qinfo); + +/** + * return whether query is completed or not + * @param qinfo + * @return + */ +int32_t qIsQueryCompleted(qinfo_t qinfo); + +/** + * destroy query info structure + * @param qHandle + */ +void qDestroyQueryInfo(qinfo_t qHandle); + +/** + * Get the queried table uid + * @param qHandle + * @return + */ +int64_t qGetQueriedTableUid(qinfo_t qHandle); + +/** + * Extract the qualified table id list, and than pass them to the TSDB driver to load the required table data blocks. + * + * @param iter the table iterator to traverse all tables belongs to a super table, or an invert index + * @return + */ +int32_t qGetQualifiedTableIdList(void* pTableList, const char* tagCond, int32_t tagCondLen, SArray* pTableIdList); + +/** + * Create the table group according to the group by tags info + * @param pTableIdList + * @param skey + * @param groupInfo + * @param groupByIndex + * @param numOfIndex + * @return + */ +int32_t qCreateTableGroupByGroupExpr(SArray* pTableIdList, TSKEY skey, STableGroupInfo groupInfo, SColIndex* groupByIndex, int32_t numOfIndex); + +/** + * Update the table id list of a given query. + * @param uid child table uid + * @param type operation type: ADD|DROP + * @return + */ +int32_t qUpdateQueriedTableIdList(qinfo_t qinfo, int64_t uid, int32_t type); + +//================================================================================================ +// query handle management +/** + * Query handle mgmt object + * @param vgId + * @return + */ +void* qOpenQueryMgmt(int32_t vgId); + +/** + * broadcast the close information and wait for all query stop. + * @param pExecutor + */ +void qQueryMgmtNotifyClosed(void* pExecutor); + +/** + * Re-open the query handle management module when opening the vnode again. + * @param pExecutor + */ +void qQueryMgmtReOpen(void *pExecutor); + +/** + * Close query mgmt and clean up resources. + * @param pExecutor + */ +void qCleanupQueryMgmt(void* pExecutor); + +/** + * Add the query into the query mgmt object + * @param pMgmt + * @param qId + * @param qInfo + * @return + */ +void** qRegisterQInfo(void* pMgmt, uint64_t qId, void *qInfo); + +/** + * acquire the query handle according to the key from query mgmt object. + * @param pMgmt + * @param key + * @return + */ +void** qAcquireQInfo(void* pMgmt, uint64_t key); + +/** + * release the query handle and decrease the reference count in cache + * @param pMgmt + * @param pQInfo + * @param freeHandle + * @return + */ +void** qReleaseQInfo(void* pMgmt, void* pQInfo); + +/** + * De-register the query handle from the management module and free it immediately. + * @param pMgmt + * @param pQInfo + * @return + */ +void** qDeregisterQInfo(void* pMgmt, void* pQInfo); + #ifdef __cplusplus } #endif From a00f4d676c9cf1618a1df72f97837ef67e1c2f7c Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 6 Oct 2021 23:42:20 +0800 Subject: [PATCH 4/9] [TD-10529]Add APIs of parser/planner/scheduler. --- include/libs/parser/parser.h | 45 +++++++++++++++++++++++++++++ include/libs/planner/planner.h | 46 ++++++++++++++++++++++++++++++ include/libs/scheduler/scheduler.h | 35 +++++++++++++++++++++++ 3 files changed, 126 insertions(+) diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index 082dd4bcdd..b9a6a156cc 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -20,6 +20,51 @@ extern "C" { #endif +#include "catalog.h" +#include "common.h" + +struct SQueryStmtInfo; +struct SInsertStmtInfo; + +/** + * True will be returned if the input sql string is insert, false otherwise. + * @param pStr sql string + * @param length length of the sql string + * @return + */ +bool qIsInsertSql(const char* pStr, size_t length); + +/** + * Parse the sql statement and then return the SQueryStmtInfo as the result of bounded AST. + * @param pSql sql string + * @param length length of the sql string + * @param id operator id, generated by uuid generator + * @param msg extended error message if exists. + * @return error code + */ +int32_t qParseQuerySql(const char* pStr, size_t length, struct SQueryStmtInfo** pQueryInfo, int64_t id, char* msg); + +/** + * Parse the insert sql statement. + * @param pStr sql string + * @param length length of the sql string + * @param pInsertParam data in binary format to submit to vnode directly. + * @param id operator id, generated by uuid generator. + * @param msg extended error message if exists to help avoid the problem in sql statement. + * @return + */ +int32_t qParseInsertSql(const char* pStr, size_t length, struct SInsertStmtInfo** pInsertInfo, int64_t id, char* msg); + +/** + * Convert a normal sql statement to only query tags information to enable that the subscribe client can be aware quickly of the true vgroup ids that + * involved in the subscribe procedure. + * @param pSql + * @param length + * @param pConvertSql + * @return + */ +int32_t qParserConvertSql(const char* pStr, size_t length, char** pConvertSql); + #ifdef __cplusplus } #endif diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index 6de7a59653..33209e5dd2 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -20,6 +20,52 @@ extern "C" { #endif +/** + * Optimize the query execution plan, currently not implement yet. + * @param pQueryNode + * @return + */ +int32_t qOptimizeQueryPlan(SQueryNode* pQueryNode); + +/** + * Create the query plan according to the bound AST, which is in the form of pQueryInfo + * @param pQueryInfo + * @param pQueryNode + * @return + */ +int32_t qCreateQueryPlan(const SQueryInfo* pQueryInfo, SQueryNode* pQueryNode); + +/** + * Convert the query plan to string, in order to display it in the shell. + * @param pQueryNode + * @return + */ +int32_t qQueryPlanToString(SQueryNode* pQueryNode, char** str); + +/** + * Restore the SQL statement according to the logic query plan. + * @param pQueryNode + * @param sql + * @return + */ +int32_t qQueryPlanToSql(SQueryNode* pQueryNode, char** sql); + +/** + * Create the physical plan for the query, according to the logic plan. + * @param pQueryNode + * @param pPhyNode + * @return + */ +int32_t qCreatePhysicalPlan(SQueryNode* pQueryNode, SEpSet* pQnode, SQueryPhyNode *pPhyNode); + +/** + * Convert to physical plan to string to enable to print it out in the shell. + * @param pPhyNode + * @param str + * @return + */ +int32_t qPhyPlanToString(SQueryPhyNode *pPhyNode, char** str); + #ifdef __cplusplus } #endif diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index 205d88970f..8863e55741 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -20,6 +20,41 @@ extern "C" { #endif +#define QUERY_TASK_MERGE 1 +#define QUERY_TASK_PARTIAL 2 + +/** + * create query job from the physical execution plan + * @param pPhyNode + * @param pJob + * @return + */ +int32_t qCreateQueryJob(const SQueryPhyNode* pPhyNode, SQueryJob* pJob); + +/** + * Process the query job, generated according to the query physical plan. + * This is a synchronized API, and is also thread-safety. + * @param pJob + * @return + */ +int32_t qProcessQueryJob(SQueryJob* pJob); + +/** + * The SSqlObj should not be here???? + * @param pSql + * @param pVgroupId + * @param pRetVgroupId + * @return + */ +SArray* qGetInvolvedVgroupIdList(SSqlObj* pSql, SArray* pVgroupId, SArray* pRetVgroupId); + +/** + * Cancel query job + * @param pJob + * @return + */ +int32_t qKillQueryJob(SQueryJob* pJob); + #ifdef __cplusplus } #endif From c6d23aa82d5222114f175c316623e3152d73a29d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Wed, 6 Oct 2021 23:46:48 +0800 Subject: [PATCH 5/9] [TD-10529] --- include/libs/transport/transport.h | 18 +++ include/os/os.h | 1 + source/libs/catalog/CMakeLists.txt | 5 + source/libs/parser/CMakeLists.txt | 12 ++ source/libs/parser/inc/parserInt.h | 175 +++++++++++++++++++++++++++++ source/libs/parser/src/parser.c | 6 +- 6 files changed, 216 insertions(+), 1 deletion(-) diff --git a/include/libs/transport/transport.h b/include/libs/transport/transport.h index 2733108825..e7660e4d66 100644 --- a/include/libs/transport/transport.h +++ b/include/libs/transport/transport.h @@ -20,6 +20,24 @@ extern "C" { #endif +typedef void* SEpSet; + +typedef struct SEpAddr { + char fqdn[TSDB_FQDN_LEN]; + uint16_t port; +} SEpAddr; + +typedef struct SVgroup { + int32_t vgId; + int8_t numOfEps; + SEpAddr epAddr[TSDB_MAX_REPLICA]; +} SVgroup; + +typedef struct SVgroupsInfo { + int32_t numOfVgroups; + SVgroup vgroups[]; +} SVgroupsInfo; + #ifdef __cplusplus } #endif diff --git a/include/os/os.h b/include/os/os.h index 259aef8143..83aca8ac55 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -27,6 +27,7 @@ extern "C" { #include #include #include +#include #include "osAtomic.h" #include "osDef.h" diff --git a/source/libs/catalog/CMakeLists.txt b/source/libs/catalog/CMakeLists.txt index ae331bf392..ff3e62700a 100644 --- a/source/libs/catalog/CMakeLists.txt +++ b/source/libs/catalog/CMakeLists.txt @@ -4,4 +4,9 @@ target_include_directories( catalog PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/catalog" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) + +target_link_libraries( + catalog + PRIVATE os util common transport ) \ No newline at end of file diff --git a/source/libs/parser/CMakeLists.txt b/source/libs/parser/CMakeLists.txt index e69de29bb2..8f6d8d2880 100644 --- a/source/libs/parser/CMakeLists.txt +++ b/source/libs/parser/CMakeLists.txt @@ -0,0 +1,12 @@ +aux_source_directory(src PARSER_SRC) +add_library(parser ${PARSER_SRC}) +target_include_directories( + parser + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/parser" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) + +target_link_libraries( + parser + PRIVATE os util common catalog transport +) \ No newline at end of file diff --git a/source/libs/parser/inc/parserInt.h b/source/libs/parser/inc/parserInt.h index 2c76f1bb96..a3e84cdb22 100644 --- a/source/libs/parser/inc/parserInt.h +++ b/source/libs/parser/inc/parserInt.h @@ -20,6 +20,181 @@ extern "C" { #endif +#include "catalog.h" +#include "parser.h" +#include "tname.h" +#include "astGen.h" + +typedef struct SField { + char name[TSDB_COL_NAME_LEN]; + uint8_t type; + int16_t bytes; +} SField; + +typedef struct SInterval { + int32_t tz; // query client timezone + char intervalUnit; + char slidingUnit; + char offsetUnit; + int64_t interval; + int64_t sliding; + int64_t offset; +} SInterval; + +typedef struct SSessionWindow { + int64_t gap; // gap between two session window(in microseconds) + int32_t primaryColId; // primary timestamp column +} SSessionWindow; + +typedef struct SGroupbyExpr { + int16_t tableIndex; + SArray* columnInfo; // SArray, group by columns information + int16_t numOfGroupCols; // todo remove it + int16_t orderIndex; // order by column index + int16_t orderType; // order by type: asc/desc +} SGroupbyExpr; + +typedef struct SFieldInfo { + int16_t numOfOutput; // number of column in result + SField *final; + SArray *internalField; // SArray +} SFieldInfo; + +typedef struct SLimitVal { + int64_t limit; + int64_t offset; +} SLimitVal; + +typedef struct SOrderVal { + uint32_t order; + int32_t orderColId; +} SOrderVal; + +typedef struct SCond { + uint64_t uid; + int32_t len; // length of tag query condition data + char * cond; +} SCond; + +typedef struct SJoinNode { + uint64_t uid; + int16_t tagColId; + SArray* tsJoin; + SArray* tagJoin; +} SJoinNode; + +typedef struct SJoinInfo { + bool hasJoin; + SJoinNode *joinTables[TSDB_MAX_JOIN_TABLE_NUM]; +} SJoinInfo; + +typedef struct STagCond { + int16_t relType; // relation between tbname list and query condition, including : TK_AND or TK_OR + SCond tbnameCond; // tbname query condition, only support tbname query condition on one table + SJoinInfo joinInfo; // join condition, only support two tables join currently + SArray *pCond; // for different table, the query condition must be seperated +} STagCond; + +typedef struct STableMetaInfo { + STableMeta *pTableMeta; // table meta, cached in client side and acquired by name + uint32_t tableMetaSize; + size_t tableMetaCapacity; + SVgroupsInfo *vgroupList; + SArray *pVgroupTables; // SArray + + /* + * 1. keep the vgroup index during the multi-vnode super table projection query + * 2. keep the vgroup index for multi-vnode insertion + */ + int32_t vgroupIndex; + SName name; + char aliasName[TSDB_TABLE_NAME_LEN]; // alias name of table specified in query sql + SArray *tagColList; // SArray, involved tag columns +} STableMetaInfo; + +typedef struct SInsertStmtInfo { + SHashObj *pTableBlockHashList; // data block for each table + SArray *pDataBlocks; // SArray. Merged submit block for each vgroup + int8_t schemaAttached; // denote if submit block is built with table schema or not + uint8_t payloadType; // EPayloadType. 0: K-V payload for non-prepare insert, 1: rawPayload for prepare insert + uint32_t insertType; // insert data from [file|sql statement| bound statement] + char *sql; // current sql statement position +} SInsertStmtInfo; + +typedef struct SQueryStmtInfo { + int16_t command; // the command may be different for each subclause, so keep it seperately. + uint32_t type; // query/insert type + STimeWindow window; // the whole query time window + + SInterval interval; // tumble time window + SSessionWindow sessionWindow; // session time window + + SGroupbyExpr groupbyExpr; // groupby tags info + SArray * colList; // SArray + SFieldInfo fieldsInfo; + SArray * exprList; // SArray + SArray * exprList1; // final exprlist in case of arithmetic expression exists + SLimitVal limit; + SLimitVal slimit; + STagCond tagCond; + + SArray * colCond; + + SOrderVal order; + int16_t numOfTables; + int16_t curTableIdx; + STableMetaInfo **pTableMetaInfo; + struct STSBuf *tsBuf; + + int16_t fillType; // final result fill type + int64_t * fillVal; // default value for fill + int32_t numOfFillVal; // fill value size + + char * msg; // pointer to the pCmd->payload to keep error message temporarily + int64_t clauseLimit; // limit for current sub clause + + int64_t prjOffset; // offset value in the original sql expression, only applied at client side + int64_t vgroupLimit; // table limit in case of super table projection query + global order + limit + + int32_t udColumnId; // current user-defined constant output field column id, monotonically decreases from TSDB_UD_COLUMN_INDEX + bool distinct; // distinct tag or not + bool onlyHasTagCond; + int32_t bufLen; + char* buf; + SArray *pUdfInfo; + + struct SQueryStmtInfo *sibling; // sibling + SArray *pUpstream; // SArray + struct SQueryStmtInfo *pDownstream; + int32_t havingFieldNum; + bool stableQuery; + bool groupbyColumn; + bool simpleAgg; + bool arithmeticOnAgg; + bool projectionQuery; + bool hasFilter; + bool onlyTagQuery; + bool orderProjectQuery; + bool stateWindow; + bool globalMerge; + bool multigroupResult; +} SQueryStmtInfo; + +/** + * validate the AST by pNode + * @param pNode + * @return SQueryNode a bounded AST with essential meta data from local buffer or mgmt node + */ +int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, const SSqlNode* pNode, SQueryStmtInfo* pQueryInfo); + +/** + * + * @param pSqlNode + * @param pMetaInfo + * @return + */ +int32_t qParserExtractRequestedMetaInfo(const SSqlNode* pSqlNode, SMetaReq* pMetaInfo); + #ifdef __cplusplus } #endif diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 6dea4a4e57..28574fa306 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -11,4 +11,8 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . - */ \ No newline at end of file + */ + +#include "parserInt.h" +#include "astGen.h" +#include "ttoken.h" \ No newline at end of file From ee9013e4de66c728fd7edf1bc8b4a5bc89ab3f6f Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 8 Oct 2021 10:23:44 +0800 Subject: [PATCH 6/9] [TD-10529]refactor & add required APIs. --- include/{client => common}/taos.h | 0 include/libs/catalog/catalog.h | 8 +- include/libs/executor/executor.h | 3 +- include/libs/parser/parser.h | 143 +++++++++++++++++++- include/libs/planner/planner.h | 56 +++++++- include/libs/scheduler/scheduler.h | 17 +-- include/os/os.h | 3 + source/client/src/client.c | 9 +- source/common/CMakeLists.txt | 5 + source/libs/parser/inc/parserInt.h | 163 ++--------------------- source/libs/parser/src/parser.c | 41 +++++- source/libs/parser/test/parserTests.cpp | 14 ++ source/libs/planner/CMakeLists.txt | 5 + source/libs/planner/inc/plannerInt.h | 33 +++++ source/libs/planner/src/planner.c | 42 +++++- source/libs/scheduler/CMakeLists.txt | 13 ++ source/libs/scheduler/inc/schedulerInt.h | 19 +++ source/libs/scheduler/src/scheduler.c | 4 +- 18 files changed, 392 insertions(+), 186 deletions(-) rename include/{client => common}/taos.h (100%) diff --git a/include/client/taos.h b/include/common/taos.h similarity index 100% rename from include/client/taos.h rename to include/common/taos.h diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index 395116b3bc..1af3e16f20 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -25,6 +25,7 @@ extern "C" { #include "tarray.h" #include "taosdef.h" #include "transport.h" +#include "common.h" struct SCatalog; @@ -63,13 +64,6 @@ typedef struct SCTableMeta { uint64_t suid; } SCTableMeta; -typedef struct SSchema { - uint8_t type; - char name[TSDB_COL_NAME_LEN]; - int16_t colId; - int16_t bytes; -} SSchema; - /* * Note that the first 24 bytes of STableMeta are identical to SCTableMeta, it is safe to cast a STableMeta to be a SCTableMeta. */ diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 246b08f693..c3c7d740f7 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -29,8 +29,7 @@ typedef void* qinfo_t; * @param qinfo * @return */ -int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableMsg* pQueryTableMsg, qinfo_t* qinfo, uint64_t qId); - +int32_t qCreateQueryInfo(void* tsdb, int32_t vgId, SQueryTableInfo* pQueryTableMsg, qinfo_t* qinfo, uint64_t qId); /** * the main query execution function, including query on both table and multiple tables, diff --git a/include/libs/parser/parser.h b/include/libs/parser/parser.h index b9a6a156cc..4890dd674a 100644 --- a/include/libs/parser/parser.h +++ b/include/libs/parser/parser.h @@ -22,8 +22,149 @@ extern "C" { #include "catalog.h" #include "common.h" +#include "tname.h" + +typedef struct SInterval { + int32_t tz; // query client timezone + char intervalUnit; + char slidingUnit; + char offsetUnit; + int64_t interval; + int64_t sliding; + int64_t offset; +} SInterval; + +typedef struct SSessionWindow { + int64_t gap; // gap between two session window(in microseconds) + int32_t primaryColId; // primary timestamp column +} SSessionWindow; + +typedef struct SGroupbyExpr { + int16_t tableIndex; + SArray* columnInfo; // SArray, group by columns information + int16_t orderIndex; // order by column index + int16_t orderType; // order by type: asc/desc +} SGroupbyExpr; + +typedef struct SField { + char name[TSDB_COL_NAME_LEN]; + uint8_t type; + int16_t bytes; +} SField; + +typedef struct SFieldInfo { + int16_t numOfOutput; // number of column in result + SField *final; + SArray *internalField; // SArray +} SFieldInfo; + +typedef struct SLimit { + int64_t limit; + int64_t offset; +} SLimit; + +typedef struct SOrder { + uint32_t order; + int32_t orderColId; +} SOrder; + +typedef struct SCond { + uint64_t uid; + int32_t len; // length of tag query condition data + char * cond; +} SCond; + +typedef struct SJoinNode { + uint64_t uid; + int16_t tagColId; + SArray* tsJoin; + SArray* tagJoin; +} SJoinNode; + +typedef struct SJoinInfo { + bool hasJoin; + SJoinNode *joinTables[TSDB_MAX_JOIN_TABLE_NUM]; +} SJoinInfo; + +typedef struct STagCond { + int16_t relType; // relation between tbname list and query condition, including : TK_AND or TK_OR + SCond tbnameCond; // tbname query condition, only support tbname query condition on one table + SJoinInfo joinInfo; // join condition, only support two tables join currently + SArray *pCond; // for different table, the query condition must be seperated +} STagCond; + +typedef struct STableMetaInfo { + STableMeta *pTableMeta; // table meta, cached in client side and acquired by name + uint32_t tableMetaSize; + size_t tableMetaCapacity; + SVgroupsInfo *vgroupList; + SArray *pVgroupTables; // SArray + + /* + * 1. keep the vgroup index during the multi-vnode super table projection query + * 2. keep the vgroup index for multi-vnode insertion + */ + int32_t vgroupIndex; + SName name; + char aliasName[TSDB_TABLE_NAME_LEN]; // alias name of table specified in query sql + SArray *tagColList; // SArray, involved tag columns +} STableMetaInfo; + +typedef struct SQueryStmtInfo { + int16_t command; // the command may be different for each subclause, so keep it seperately. + uint32_t type; // query/insert type + STimeWindow window; // the whole query time window + SInterval interval; // tumble time window + SSessionWindow sessionWindow; // session time window + SGroupbyExpr groupbyExpr; // groupby tags info + SArray * colList; // SArray + SFieldInfo fieldsInfo; + SArray * exprList; // SArray + SArray * exprList1; // final exprlist in case of arithmetic expression exists + SLimit limit; + SLimit slimit; + STagCond tagCond; + SArray * colCond; + SOrder order; + int16_t numOfTables; + int16_t curTableIdx; + STableMetaInfo **pTableMetaInfo; + struct STSBuf *tsBuf; + + int16_t fillType; // final result fill type + int64_t * fillVal; // default value for fill + int32_t numOfFillVal; // fill value size + + char * msg; // pointer to the pCmd->payload to keep error message temporarily + int64_t clauseLimit; // limit for current sub clause + + int64_t prjOffset; // offset value in the original sql expression, only applied at client side + int64_t vgroupLimit; // table limit in case of super table projection query + global order + limit + + int32_t udColumnId; // current user-defined constant output field column id, monotonically decreases from TSDB_UD_COLUMN_INDEX + bool distinct; // distinct tag or not + bool onlyHasTagCond; + int32_t bufLen; + char* buf; + SArray *pUdfInfo; + + struct SQueryStmtInfo *sibling; // sibling + SArray *pUpstream; // SArray + struct SQueryStmtInfo *pDownstream; + int32_t havingFieldNum; + bool stableQuery; + bool groupbyColumn; + bool simpleAgg; + bool arithmeticOnAgg; + bool projectionQuery; + bool hasFilter; + bool onlyTagQuery; + bool orderProjectQuery; + bool stateWindow; + bool globalMerge; + bool multigroupResult; +} SQueryStmtInfo; -struct SQueryStmtInfo; struct SInsertStmtInfo; /** diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index 33209e5dd2..cbd9b6f89e 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -20,12 +20,35 @@ extern "C" { #endif +#define QUERY_TYPE_MERGE 1 +#define QUERY_TYPE_PARTIAL 2 + +struct SEpSet; +struct SQueryNode; +struct SQueryPhyNode; +struct SQueryStmtInfo; + +typedef struct SSubquery { + int64_t queryId; // the subquery id created by qnode + int32_t type; // QUERY_TYPE_MERGE|QUERY_TYPE_PARTIAL + int32_t level; // the execution level of current subquery, starting from 0. + SArray *pUpstream; // the upstream,from which to fetch the result + struct SQueryPhyNode *pNode; // physical plan of current subquery +} SSubquery; + +typedef struct SQueryJob { + SArray **pSubqueries; + int32_t numOfLevels; + int32_t currentLevel; +} SQueryJob; + + /** * Optimize the query execution plan, currently not implement yet. * @param pQueryNode * @return */ -int32_t qOptimizeQueryPlan(SQueryNode* pQueryNode); +int32_t qOptimizeQueryPlan(struct SQueryNode* pQueryNode); /** * Create the query plan according to the bound AST, which is in the form of pQueryInfo @@ -33,14 +56,14 @@ int32_t qOptimizeQueryPlan(SQueryNode* pQueryNode); * @param pQueryNode * @return */ -int32_t qCreateQueryPlan(const SQueryInfo* pQueryInfo, SQueryNode* pQueryNode); +int32_t qCreateQueryPlan(const struct SQueryStmtInfo* pQueryInfo, struct SQueryNode* pQueryNode); /** * Convert the query plan to string, in order to display it in the shell. * @param pQueryNode * @return */ -int32_t qQueryPlanToString(SQueryNode* pQueryNode, char** str); +int32_t qQueryPlanToString(struct SQueryNode* pQueryNode, char** str); /** * Restore the SQL statement according to the logic query plan. @@ -48,7 +71,7 @@ int32_t qQueryPlanToString(SQueryNode* pQueryNode, char** str); * @param sql * @return */ -int32_t qQueryPlanToSql(SQueryNode* pQueryNode, char** sql); +int32_t qQueryPlanToSql(struct SQueryNode* pQueryNode, char** sql); /** * Create the physical plan for the query, according to the logic plan. @@ -56,7 +79,7 @@ int32_t qQueryPlanToSql(SQueryNode* pQueryNode, char** sql); * @param pPhyNode * @return */ -int32_t qCreatePhysicalPlan(SQueryNode* pQueryNode, SEpSet* pQnode, SQueryPhyNode *pPhyNode); +int32_t qCreatePhysicalPlan(struct SQueryNode* pQueryNode, struct SEpSet* pQnode, struct SQueryPhyNode *pPhyNode); /** * Convert to physical plan to string to enable to print it out in the shell. @@ -64,7 +87,28 @@ int32_t qCreatePhysicalPlan(SQueryNode* pQueryNode, SEpSet* pQnode, SQueryPhyNod * @param str * @return */ -int32_t qPhyPlanToString(SQueryPhyNode *pPhyNode, char** str); +int32_t qPhyPlanToString(struct SQueryPhyNode *pPhyNode, char** str); + +/** + * Destroy the query plan object. + * @return + */ +void* qDestroyQueryPlan(struct SQueryNode* pQueryNode); + +/** + * Destroy the physical plan. + * @param pQueryPhyNode + * @return + */ +void* qDestroyQueryPhyPlan(struct SQueryPhyNode* pQueryPhyNode); + +/** + * Create the query job from the physical execution plan + * @param pPhyNode + * @param pJob + * @return + */ +int32_t qCreateQueryJob(const struct SQueryPhyNode* pPhyNode, struct SQueryJob** pJob); #ifdef __cplusplus } diff --git a/include/libs/scheduler/scheduler.h b/include/libs/scheduler/scheduler.h index 8863e55741..d9653046cf 100644 --- a/include/libs/scheduler/scheduler.h +++ b/include/libs/scheduler/scheduler.h @@ -20,16 +20,7 @@ extern "C" { #endif -#define QUERY_TASK_MERGE 1 -#define QUERY_TASK_PARTIAL 2 - -/** - * create query job from the physical execution plan - * @param pPhyNode - * @param pJob - * @return - */ -int32_t qCreateQueryJob(const SQueryPhyNode* pPhyNode, SQueryJob* pJob); +struct SQueryJob; /** * Process the query job, generated according to the query physical plan. @@ -37,7 +28,7 @@ int32_t qCreateQueryJob(const SQueryPhyNode* pPhyNode, SQueryJob* pJob); * @param pJob * @return */ -int32_t qProcessQueryJob(SQueryJob* pJob); +int32_t qProcessQueryJob(struct SQueryJob* pJob); /** * The SSqlObj should not be here???? @@ -46,14 +37,14 @@ int32_t qProcessQueryJob(SQueryJob* pJob); * @param pRetVgroupId * @return */ -SArray* qGetInvolvedVgroupIdList(SSqlObj* pSql, SArray* pVgroupId, SArray* pRetVgroupId); +//SArray* qGetInvolvedVgroupIdList(struct SSqlObj* pSql, SArray* pVgroupId, SArray* pRetVgroupId); /** * Cancel query job * @param pJob * @return */ -int32_t qKillQueryJob(SQueryJob* pJob); +int32_t qKillQueryJob(struct SQueryJob* pJob); #ifdef __cplusplus } diff --git a/include/os/os.h b/include/os/os.h index 83aca8ac55..f009c92163 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -28,6 +28,9 @@ extern "C" { #include #include #include +#include +#include +#include #include "osAtomic.h" #include "osDef.h" diff --git a/source/client/src/client.c b/source/client/src/client.c index 6dea4a4e57..77c4aa1b2d 100644 --- a/source/client/src/client.c +++ b/source/client/src/client.c @@ -11,4 +11,11 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . - */ \ No newline at end of file + */ + +//#include "taos.h" + +//TAOS_RES *taos_query(TAOS *taos, const char *sql) { +// +//} + diff --git a/source/common/CMakeLists.txt b/source/common/CMakeLists.txt index 1273a61c02..59612a22e6 100644 --- a/source/common/CMakeLists.txt +++ b/source/common/CMakeLists.txt @@ -4,4 +4,9 @@ target_include_directories( common PUBLIC "${CMAKE_SOURCE_DIR}/include/common" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) + +target_link_libraries( + common + PRIVATE os util ) \ No newline at end of file diff --git a/source/libs/parser/inc/parserInt.h b/source/libs/parser/inc/parserInt.h index a3e84cdb22..ce1504b4e2 100644 --- a/source/libs/parser/inc/parserInt.h +++ b/source/libs/parser/inc/parserInt.h @@ -21,96 +21,10 @@ extern "C" { #endif #include "catalog.h" -#include "parser.h" #include "tname.h" -#include "astGen.h" +#include "astGenerator.h" -typedef struct SField { - char name[TSDB_COL_NAME_LEN]; - uint8_t type; - int16_t bytes; -} SField; - -typedef struct SInterval { - int32_t tz; // query client timezone - char intervalUnit; - char slidingUnit; - char offsetUnit; - int64_t interval; - int64_t sliding; - int64_t offset; -} SInterval; - -typedef struct SSessionWindow { - int64_t gap; // gap between two session window(in microseconds) - int32_t primaryColId; // primary timestamp column -} SSessionWindow; - -typedef struct SGroupbyExpr { - int16_t tableIndex; - SArray* columnInfo; // SArray, group by columns information - int16_t numOfGroupCols; // todo remove it - int16_t orderIndex; // order by column index - int16_t orderType; // order by type: asc/desc -} SGroupbyExpr; - -typedef struct SFieldInfo { - int16_t numOfOutput; // number of column in result - SField *final; - SArray *internalField; // SArray -} SFieldInfo; - -typedef struct SLimitVal { - int64_t limit; - int64_t offset; -} SLimitVal; - -typedef struct SOrderVal { - uint32_t order; - int32_t orderColId; -} SOrderVal; - -typedef struct SCond { - uint64_t uid; - int32_t len; // length of tag query condition data - char * cond; -} SCond; - -typedef struct SJoinNode { - uint64_t uid; - int16_t tagColId; - SArray* tsJoin; - SArray* tagJoin; -} SJoinNode; - -typedef struct SJoinInfo { - bool hasJoin; - SJoinNode *joinTables[TSDB_MAX_JOIN_TABLE_NUM]; -} SJoinInfo; - -typedef struct STagCond { - int16_t relType; // relation between tbname list and query condition, including : TK_AND or TK_OR - SCond tbnameCond; // tbname query condition, only support tbname query condition on one table - SJoinInfo joinInfo; // join condition, only support two tables join currently - SArray *pCond; // for different table, the query condition must be seperated -} STagCond; - -typedef struct STableMetaInfo { - STableMeta *pTableMeta; // table meta, cached in client side and acquired by name - uint32_t tableMetaSize; - size_t tableMetaCapacity; - SVgroupsInfo *vgroupList; - SArray *pVgroupTables; // SArray - - /* - * 1. keep the vgroup index during the multi-vnode super table projection query - * 2. keep the vgroup index for multi-vnode insertion - */ - int32_t vgroupIndex; - SName name; - char aliasName[TSDB_TABLE_NAME_LEN]; // alias name of table specified in query sql - SArray *tagColList; // SArray, involved tag columns -} STableMetaInfo; +struct SSqlNode; typedef struct SInsertStmtInfo { SHashObj *pTableBlockHashList; // data block for each table @@ -121,71 +35,16 @@ typedef struct SInsertStmtInfo { char *sql; // current sql statement position } SInsertStmtInfo; -typedef struct SQueryStmtInfo { - int16_t command; // the command may be different for each subclause, so keep it seperately. - uint32_t type; // query/insert type - STimeWindow window; // the whole query time window - - SInterval interval; // tumble time window - SSessionWindow sessionWindow; // session time window - - SGroupbyExpr groupbyExpr; // groupby tags info - SArray * colList; // SArray - SFieldInfo fieldsInfo; - SArray * exprList; // SArray - SArray * exprList1; // final exprlist in case of arithmetic expression exists - SLimitVal limit; - SLimitVal slimit; - STagCond tagCond; - - SArray * colCond; - - SOrderVal order; - int16_t numOfTables; - int16_t curTableIdx; - STableMetaInfo **pTableMetaInfo; - struct STSBuf *tsBuf; - - int16_t fillType; // final result fill type - int64_t * fillVal; // default value for fill - int32_t numOfFillVal; // fill value size - - char * msg; // pointer to the pCmd->payload to keep error message temporarily - int64_t clauseLimit; // limit for current sub clause - - int64_t prjOffset; // offset value in the original sql expression, only applied at client side - int64_t vgroupLimit; // table limit in case of super table projection query + global order + limit - - int32_t udColumnId; // current user-defined constant output field column id, monotonically decreases from TSDB_UD_COLUMN_INDEX - bool distinct; // distinct tag or not - bool onlyHasTagCond; - int32_t bufLen; - char* buf; - SArray *pUdfInfo; - - struct SQueryStmtInfo *sibling; // sibling - SArray *pUpstream; // SArray - struct SQueryStmtInfo *pDownstream; - int32_t havingFieldNum; - bool stableQuery; - bool groupbyColumn; - bool simpleAgg; - bool arithmeticOnAgg; - bool projectionQuery; - bool hasFilter; - bool onlyTagQuery; - bool orderProjectQuery; - bool stateWindow; - bool globalMerge; - bool multigroupResult; -} SQueryStmtInfo; - /** - * validate the AST by pNode - * @param pNode - * @return SQueryNode a bounded AST with essential meta data from local buffer or mgmt node + * Validate the sql info, according to the corresponding metadata info from catalog. + * @param pCatalog + * @param pSqlInfo + * @param pQueryInfo a bounded AST with essential meta data from local buffer or mgmt node + * @param id + * @param msg + * @return */ -int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, const SSqlNode* pNode, SQueryStmtInfo* pQueryInfo); +int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pSqlInfo, SQueryStmtInfo* pQueryInfo, int64_t id, char* msg); /** * @@ -193,7 +52,7 @@ int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, const SSqlNode* pNode, * @param pMetaInfo * @return */ -int32_t qParserExtractRequestedMetaInfo(const SSqlNode* pSqlNode, SMetaReq* pMetaInfo); +int32_t qParserExtractRequestedMetaInfo(const struct SSqlNode* pSqlNode, SMetaReq* pMetaInfo); #ifdef __cplusplus } diff --git a/source/libs/parser/src/parser.c b/source/libs/parser/src/parser.c index 28574fa306..be1d93ecf8 100644 --- a/source/libs/parser/src/parser.c +++ b/source/libs/parser/src/parser.c @@ -14,5 +14,42 @@ */ #include "parserInt.h" -#include "astGen.h" -#include "ttoken.h" \ No newline at end of file +#include "ttoken.h" +#include "astGenerator.h" + +bool qIsInsertSql(const char* pStr, size_t length) { + return false; +} + +int32_t qParseQuerySql(const char* pStr, size_t length, struct SQueryStmtInfo** pQueryInfo, int64_t id, char* msg) { + *pQueryInfo = calloc(1, sizeof(SQueryStmtInfo)); + if (*pQueryInfo == NULL) { + return -1; // set correct error code. + } + + SSqlInfo info = genAST(pStr); + if (!info.valid) { + strcpy(msg, info.msg); + return -1; // set correct error code. + } + + struct SCatalog* pCatalog = getCatalogHandle(NULL); + int32_t code = qParserValidateSqlNode(pCatalog, &info, *pQueryInfo, id, msg); + if (code != 0) { + return code; + } + + return 0; +} + +int32_t qParseInsertSql(const char* pStr, size_t length, struct SInsertStmtInfo** pInsertInfo, int64_t id, char* msg) { + return 0; +} + +int32_t qParserConvertSql(const char* pStr, size_t length, char** pConvertSql) { + return 0; +} + +int32_t qParserExtractRequestedMetaInfo(const struct SSqlNode* pSqlNode, SMetaReq* pMetaInfo) { + return 0; +} \ No newline at end of file diff --git a/source/libs/parser/test/parserTests.cpp b/source/libs/parser/test/parserTests.cpp index e69de29bb2..6dea4a4e57 100644 --- a/source/libs/parser/test/parserTests.cpp +++ b/source/libs/parser/test/parserTests.cpp @@ -0,0 +1,14 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ \ No newline at end of file diff --git a/source/libs/planner/CMakeLists.txt b/source/libs/planner/CMakeLists.txt index 8b3eb96c14..52506b4556 100644 --- a/source/libs/planner/CMakeLists.txt +++ b/source/libs/planner/CMakeLists.txt @@ -4,4 +4,9 @@ target_include_directories( planner PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/planner" PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) + +target_link_libraries( + planner + PRIVATE os util common catalog parser ) \ No newline at end of file diff --git a/source/libs/planner/inc/plannerInt.h b/source/libs/planner/inc/plannerInt.h index 4005508ed7..fc08951c92 100644 --- a/source/libs/planner/inc/plannerInt.h +++ b/source/libs/planner/inc/plannerInt.h @@ -20,6 +20,39 @@ extern "C" { #endif +#include "common.h" +#include "tarray.h" +#include "planner.h" + +typedef struct SQueryNodeBasicInfo { + int32_t type; + char *name; +} SQueryNodeBasicInfo; + +typedef struct SQueryTableInfo { + char *tableName; + uint64_t uid; + int32_t tid; +} SQueryTableInfo; + +typedef struct SQueryNode { + SQueryNodeBasicInfo info; + SQueryTableInfo tableInfo; + SSchema *pSchema; // the schema of the input SSDatablock + int32_t numOfCols; // number of input columns + struct SExprInfo *pExpr; // the query functions or sql aggregations + int32_t numOfOutput; // number of result columns, which is also the number of pExprs + void *pExtInfo; // additional information + // previous operator to generated result for current node to process + // in case of join, multiple prev nodes exist. + SArray *pPrevNodes; // upstream nodes + struct SQueryNode *nextNode; +} SQueryNode; + +typedef struct SQueryPhyNode { + +} SQueryPhyNode; + #ifdef __cplusplus } #endif diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index 6dea4a4e57..8e72fb2758 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -11,4 +11,44 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . - */ \ No newline at end of file + */ + +#include "os.h" +#include "plannerInt.h" +#include "parser.h" + +int32_t qOptimizeQueryPlan(struct SQueryNode* pQueryNode) { + return 0; +} + +int32_t qCreateQueryPlan(const struct SQueryStmtInfo* pQueryInfo, struct SQueryNode* pQueryNode) { + return 0; +} + +int32_t qQueryPlanToString(struct SQueryNode* pQueryNode, char** str) { + return 0; +} + +int32_t qQueryPlanToSql(struct SQueryNode* pQueryNode, char** sql) { + return 0; +} + +int32_t qCreatePhysicalPlan(struct SQueryNode* pQueryNode, struct SEpSet* pQnode, struct SQueryPhyNode *pPhyNode) { + return 0; +} + +int32_t qPhyPlanToString(struct SQueryPhyNode *pPhyNode, char** str) { + return 0; +} + +void* qDestroyQueryPlan(struct SQueryNode* pQueryNode) { + return NULL; +} + +void* qDestroyQueryPhyPlan(struct SQueryPhyNode* pQueryPhyNode) { + return NULL; +} + +int32_t qCreateQueryJob(const struct SQueryPhyNode* pPhyNode, struct SQueryJob** pJob) { + return 0; +} \ No newline at end of file diff --git a/source/libs/scheduler/CMakeLists.txt b/source/libs/scheduler/CMakeLists.txt index e69de29bb2..770a6b02c2 100644 --- a/source/libs/scheduler/CMakeLists.txt +++ b/source/libs/scheduler/CMakeLists.txt @@ -0,0 +1,13 @@ +aux_source_directory(src SCHEDULER_SRC) +add_library(scheduler ${SCHEDULER_SRC}) + +target_include_directories( + scheduler + PUBLIC "${CMAKE_SOURCE_DIR}/include/libs/scheduler" + PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc" +) + +target_link_libraries( + scheduler + PRIVATE os util planner +) \ No newline at end of file diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index 3e2cf2d37f..778e3b241d 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -20,6 +20,25 @@ extern "C" { #endif +#include "os.h" +#include "tarray.h" +#include "planner.h" +#include "scheduler.h" + +typedef struct SSubquery { + int64_t taskId; // the task id created by qnode + int32_t type; + int32_t level; + struct SQueryPhyNode *pNode; + SArray *pUpstream; +} SSubquery; + +typedef struct SQuery { + SArray **pSubquery; + int32_t numOfLevels; + int32_t currentLevel; +} SQuery; + #ifdef __cplusplus } #endif diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 6dea4a4e57..37f6240f9b 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -11,4 +11,6 @@ * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . - */ \ No newline at end of file + */ + +#include "schedulerInt.h" \ No newline at end of file From 2f5eb2ab2dc560498d5446b4f494ef1081207d19 Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 8 Oct 2021 10:25:09 +0800 Subject: [PATCH 7/9] [TD-10529]add new files. --- include/common/common.h | 37 ++ include/common/taosdef.h | 448 +++++++++++++++++++++++ include/common/tname.h | 59 +++ include/common/ttypes.h | 215 +++++++++++ include/util/thash.h | 206 +++++++++++ source/libs/parser/inc/astGenerator.h | 360 ++++++++++++++++++ source/libs/parser/inc/insertParser.h | 19 + source/libs/parser/inc/parserUtil.h | 28 ++ source/libs/parser/inc/ttoken.h | 185 ++++++++++ source/libs/parser/inc/ttokendef.h | 228 ++++++++++++ source/libs/parser/inc/tvariant.h | 66 ++++ source/libs/parser/src/astGenerator.c | 507 ++++++++++++++++++++++++++ source/libs/parser/src/astValidate.c | 26 ++ 13 files changed, 2384 insertions(+) create mode 100644 include/common/common.h create mode 100644 include/common/taosdef.h create mode 100644 include/common/tname.h create mode 100644 include/common/ttypes.h create mode 100644 include/util/thash.h create mode 100644 source/libs/parser/inc/astGenerator.h create mode 100644 source/libs/parser/inc/insertParser.h create mode 100644 source/libs/parser/inc/parserUtil.h create mode 100644 source/libs/parser/inc/ttoken.h create mode 100644 source/libs/parser/inc/ttokendef.h create mode 100644 source/libs/parser/inc/tvariant.h create mode 100644 source/libs/parser/src/astGenerator.c create mode 100644 source/libs/parser/src/astValidate.c diff --git a/include/common/common.h b/include/common/common.h new file mode 100644 index 0000000000..c03b978c98 --- /dev/null +++ b/include/common/common.h @@ -0,0 +1,37 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_COMMON_H +#define TDENGINE_COMMON_H + +typedef struct STimeWindow { + TSKEY skey; + TSKEY ekey; +} STimeWindow; + +typedef struct { + int32_t dataLen; + char name[TSDB_TABLE_FNAME_LEN]; + char *data; +} STagData; + +typedef struct SSchema { + uint8_t type; + char name[TSDB_COL_NAME_LEN]; + int16_t colId; + int16_t bytes; +} SSchema; + +#endif // TDENGINE_COMMON_H diff --git a/include/common/taosdef.h b/include/common/taosdef.h new file mode 100644 index 0000000000..dd5bbace8c --- /dev/null +++ b/include/common/taosdef.h @@ -0,0 +1,448 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_TAOS_DEF_H +#define TDENGINE_TAOS_DEF_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "os.h" + +#define TSDB__packed + +#define TSKEY int64_t + +#define TSWINDOW_INITIALIZER ((STimeWindow) {INT64_MIN, INT64_MAX}) +#define TSWINDOW_DESC_INITIALIZER ((STimeWindow) {INT64_MAX, INT64_MIN}) +#define IS_TSWINDOW_SPECIFIED(win) (((win).skey != INT64_MIN) || ((win).ekey != INT64_MAX)) + +#define TSKEY_INITIAL_VAL INT64_MIN + +// Bytes for each type. +extern const int32_t TYPE_BYTES[15]; + +// TODO: replace and remove code below +#define CHAR_BYTES sizeof(int8_t) +#define SHORT_BYTES sizeof(int16_t) +#define INT_BYTES sizeof(int32_t) +#define LONG_BYTES sizeof(int64_t) +#define FLOAT_BYTES sizeof(float) +#define DOUBLE_BYTES sizeof(double) +#define POINTER_BYTES sizeof(ptrdiff_t) // 8 by default assert(sizeof(ptrdiff_t) == sizseof(void*) +#define TSDB_KEYSIZE sizeof(TSKEY) +#define TSDB_NCHAR_SIZE sizeof(int32_t) + +// NULL definition +#define TSDB_DATA_BOOL_NULL 0x02 +#define TSDB_DATA_TINYINT_NULL 0x80 +#define TSDB_DATA_SMALLINT_NULL 0x8000 +#define TSDB_DATA_INT_NULL 0x80000000L +#define TSDB_DATA_BIGINT_NULL 0x8000000000000000L +#define TSDB_DATA_TIMESTAMP_NULL TSDB_DATA_BIGINT_NULL + +#define TSDB_DATA_FLOAT_NULL 0x7FF00000 // it is an NAN +#define TSDB_DATA_DOUBLE_NULL 0x7FFFFF0000000000L // an NAN +#define TSDB_DATA_NCHAR_NULL 0xFFFFFFFF +#define TSDB_DATA_BINARY_NULL 0xFF + +#define TSDB_DATA_UTINYINT_NULL 0xFF +#define TSDB_DATA_USMALLINT_NULL 0xFFFF +#define TSDB_DATA_UINT_NULL 0xFFFFFFFF +#define TSDB_DATA_UBIGINT_NULL 0xFFFFFFFFFFFFFFFFL + +#define TSDB_DATA_NULL_STR "NULL" +#define TSDB_DATA_NULL_STR_L "null" +#define TSDB_DEFAULT_USER "root" + +#ifdef _TD_POWER_ +#define TSDB_DEFAULT_PASS "powerdb" +#elif (_TD_TQ_ == true) +#define TSDB_DEFAULT_PASS "tqueue" +#else +#define TSDB_DEFAULT_PASS "taosdata" +#endif + +#define SHELL_MAX_PASSWORD_LEN 20 + +#define TSDB_TRUE 1 +#define TSDB_FALSE 0 +#define TSDB_OK 0 +#define TSDB_ERR -1 + +#define TS_PATH_DELIMITER "." +#define TSQL_TBNAME "TBNAME" +#define TSQL_TBNAME_L "tbname" +#define TSQL_BLOCK_DIST "_BLOCK_DIST" +#define TSQL_BLOCK_DIST_L "_block_dist" + +#define TSDB_TIME_PRECISION_MILLI 0 +#define TSDB_TIME_PRECISION_MICRO 1 +#define TSDB_TIME_PRECISION_NANO 2 + +#define TSDB_TIME_PRECISION_MILLI_STR "ms" +#define TSDB_TIME_PRECISION_MICRO_STR "us" +#define TSDB_TIME_PRECISION_NANO_STR "ns" + +#define TSDB_TICK_PER_SECOND(precision) ((int64_t)((precision)==TSDB_TIME_PRECISION_MILLI ? 1e3L : ((precision)==TSDB_TIME_PRECISION_MICRO ? 1e6L : 1e9L))) + +#define T_MEMBER_SIZE(type, member) sizeof(((type *)0)->member) +#define T_APPEND_MEMBER(dst, ptr, type, member) \ +do {\ + memcpy((void *)(dst), (void *)(&((ptr)->member)), T_MEMBER_SIZE(type, member));\ + dst = (void *)((char *)(dst) + T_MEMBER_SIZE(type, member));\ +} while(0) +#define T_READ_MEMBER(src, type, target) \ +do { \ + (target) = *(type *)(src); \ + (src) = (void *)((char *)src + sizeof(type));\ +} while(0) + +#define GET_INT8_VAL(x) (*(int8_t *)(x)) +#define GET_INT16_VAL(x) (*(int16_t *)(x)) +#define GET_INT32_VAL(x) (*(int32_t *)(x)) +#define GET_INT64_VAL(x) (*(int64_t *)(x)) +#define GET_UINT8_VAL(x) (*(uint8_t*) (x)) +#define GET_UINT16_VAL(x) (*(uint16_t *)(x)) +#define GET_UINT32_VAL(x) (*(uint32_t *)(x)) +#define GET_UINT64_VAL(x) (*(uint64_t *)(x)) + +#ifdef _TD_ARM_32 + float taos_align_get_float(const char* pBuf); + double taos_align_get_double(const char* pBuf); + + #define GET_FLOAT_VAL(x) taos_align_get_float(x) + #define GET_DOUBLE_VAL(x) taos_align_get_double(x) + #define SET_FLOAT_VAL(x, y) { float z = (float)(y); (*(int32_t*) x = *(int32_t*)(&z)); } + #define SET_DOUBLE_VAL(x, y) { double z = (double)(y); (*(int64_t*) x = *(int64_t*)(&z)); } + #define SET_FLOAT_PTR(x, y) { (*(int32_t*) x = *(int32_t*)y); } + #define SET_DOUBLE_PTR(x, y) { (*(int64_t*) x = *(int64_t*)y); } +#else + #define GET_FLOAT_VAL(x) (*(float *)(x)) + #define GET_DOUBLE_VAL(x) (*(double *)(x)) + #define SET_FLOAT_VAL(x, y) { (*(float *)(x)) = (float)(y); } + #define SET_DOUBLE_VAL(x, y) { (*(double *)(x)) = (double)(y); } + #define SET_FLOAT_PTR(x, y) { (*(float *)(x)) = (*(float *)(y)); } + #define SET_DOUBLE_PTR(x, y) { (*(double *)(x)) = (*(double *)(y)); } +#endif + +// TODO: check if below is necessary +#define TSDB_RELATION_INVALID 0 +#define TSDB_RELATION_LESS 1 +#define TSDB_RELATION_GREATER 2 +#define TSDB_RELATION_EQUAL 3 +#define TSDB_RELATION_LESS_EQUAL 4 +#define TSDB_RELATION_GREATER_EQUAL 5 +#define TSDB_RELATION_NOT_EQUAL 6 +#define TSDB_RELATION_LIKE 7 +#define TSDB_RELATION_ISNULL 8 +#define TSDB_RELATION_NOTNULL 9 +#define TSDB_RELATION_IN 10 + +#define TSDB_RELATION_AND 11 +#define TSDB_RELATION_OR 12 +#define TSDB_RELATION_NOT 13 + +#define TSDB_RELATION_MATCH 14 +#define TSDB_RELATION_NMATCH 15 + +#define TSDB_BINARY_OP_ADD 30 +#define TSDB_BINARY_OP_SUBTRACT 31 +#define TSDB_BINARY_OP_MULTIPLY 32 +#define TSDB_BINARY_OP_DIVIDE 33 +#define TSDB_BINARY_OP_REMAINDER 34 + + +#define IS_RELATION_OPTR(op) (((op) >= TSDB_RELATION_LESS) && ((op) < TSDB_RELATION_IN)) +#define IS_ARITHMETIC_OPTR(op) (((op) >= TSDB_BINARY_OP_ADD) && ((op) <= TSDB_BINARY_OP_REMAINDER)) + +#define TS_PATH_DELIMITER_LEN 1 + +#define TSDB_UNI_LEN 24 +#define TSDB_USER_LEN TSDB_UNI_LEN + +// ACCOUNT is a 32 bit positive integer +// this is the length of its string representation, including the terminator zero +#define TSDB_ACCT_ID_LEN 11 + +#define TSDB_MAX_COLUMNS 4096 +#define TSDB_MIN_COLUMNS 2 //PRIMARY COLUMN(timestamp) + other columns + +#define TSDB_NODE_NAME_LEN 64 +#define TSDB_TABLE_NAME_LEN 193 // it is a null-terminated string +#define TSDB_DB_NAME_LEN 33 +#define TSDB_FUNC_NAME_LEN 65 +#define TSDB_FUNC_CODE_LEN (65535 - 512) +#define TSDB_FUNC_BUF_SIZE 512 +#define TSDB_TYPE_STR_MAX_LEN 32 +#define TSDB_TABLE_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN + TSDB_TABLE_NAME_LEN) +#define TSDB_COL_NAME_LEN 65 +#define TSDB_MAX_SAVED_SQL_LEN TSDB_MAX_COLUMNS * 64 +#define TSDB_MAX_SQL_LEN TSDB_PAYLOAD_SIZE +#define TSDB_MAX_SQL_SHOW_LEN 512 +#define TSDB_MAX_ALLOWED_SQL_LEN (1*1024*1024u) // sql length should be less than 1mb + +#define TSDB_APPNAME_LEN TSDB_UNI_LEN + +/** + * In some scenarios uint16_t (0~65535) is used to store the row len. + * - Firstly, we use 65531(65535 - 4), as the SDataRow/SKVRow contains 4 bits header. + * - Secondly, if all cols are VarDataT type except primary key, we need 4 bits to store the offset, thus + * the final value is 65531-(4096-1)*4 = 49151. + */ +#define TSDB_MAX_BYTES_PER_ROW 49151 +#define TSDB_MAX_TAGS_LEN 16384 +#define TSDB_MAX_TAGS 128 +#define TSDB_MAX_TAG_CONDITIONS 1024 + +#define TSDB_AUTH_LEN 16 +#define TSDB_KEY_LEN 16 +#define TSDB_VERSION_LEN 12 +#define TSDB_LOCALE_LEN 64 +#define TSDB_TIMEZONE_LEN 96 +#define TSDB_LABEL_LEN 8 + +#define TSDB_CLUSTER_ID_LEN 40 +#define TSDB_FQDN_LEN 128 +#define TSDB_EP_LEN (TSDB_FQDN_LEN + 6) +#define TSDB_IPv4ADDR_LEN 16 +#define TSDB_FILENAME_LEN 128 +#define TSDB_SHOW_SQL_LEN 512 +#define TSDB_SHOW_SUBQUERY_LEN 1000 +#define TSDB_SLOW_QUERY_SQL_LEN 512 + +#define TSDB_STEP_NAME_LEN 32 +#define TSDB_STEP_DESC_LEN 128 + +#define TSDB_MQTT_HOSTNAME_LEN 64 +#define TSDB_MQTT_PORT_LEN 8 +#define TSDB_MQTT_USER_LEN 24 +#define TSDB_MQTT_PASS_LEN 24 +#define TSDB_MQTT_TOPIC_LEN 64 +#define TSDB_MQTT_CLIENT_ID_LEN 32 + +#define TSDB_DB_TYPE_DEFAULT 0 +#define TSDB_DB_TYPE_TOPIC 1 + +#define TSDB_DEFAULT_PKT_SIZE 65480 //same as RPC_MAX_UDP_SIZE + +#define TSDB_PAYLOAD_SIZE TSDB_DEFAULT_PKT_SIZE +#define TSDB_DEFAULT_PAYLOAD_SIZE 5120 // default payload size, greater than PATH_MAX value +#define TSDB_EXTRA_PAYLOAD_SIZE 128 // extra bytes for auth +#define TSDB_CQ_SQL_SIZE 1024 +#define TSDB_MIN_VNODES 64 +#define TSDB_MAX_VNODES 2048 +#define TSDB_MIN_VNODES_PER_DB 2 +#define TSDB_MAX_VNODES_PER_DB 64 + +#define TSDB_DNODE_ROLE_ANY 0 +#define TSDB_DNODE_ROLE_MGMT 1 +#define TSDB_DNODE_ROLE_VNODE 2 + +#define TSDB_MAX_REPLICA 5 + +#define TSDB_TBNAME_COLUMN_INDEX (-1) +#define TSDB_UD_COLUMN_INDEX (-1000) +#define TSDB_RES_COL_ID (-5000) + +#define TSDB_MULTI_TABLEMETA_MAX_NUM 100000 // maximum batch size allowed to load table meta + +#define TSDB_MIN_CACHE_BLOCK_SIZE 1 +#define TSDB_MAX_CACHE_BLOCK_SIZE 128 // 128MB for each vnode +#define TSDB_DEFAULT_CACHE_BLOCK_SIZE 16 + +#define TSDB_MIN_TOTAL_BLOCKS 3 +#define TSDB_MAX_TOTAL_BLOCKS 10000 +#define TSDB_DEFAULT_TOTAL_BLOCKS 6 + +#define TSDB_MIN_TABLES 4 +#define TSDB_MAX_TABLES 10000000 +#define TSDB_DEFAULT_TABLES 1000000 +#define TSDB_TABLES_STEP 1000 +#define TSDB_META_COMPACT_RATIO 0 // disable tsdb meta compact by default + +#define TSDB_MIN_DAYS_PER_FILE 1 +#define TSDB_MAX_DAYS_PER_FILE 3650 +#define TSDB_DEFAULT_DAYS_PER_FILE 10 + +#define TSDB_MIN_KEEP 1 // data in db to be reserved. +#define TSDB_MAX_KEEP 365000 // data in db to be reserved. +#define TSDB_DEFAULT_KEEP 3650 // ten years + +#define TSDB_DEFAULT_MIN_ROW_FBLOCK 100 +#define TSDB_MIN_MIN_ROW_FBLOCK 10 +#define TSDB_MAX_MIN_ROW_FBLOCK 1000 + +#define TSDB_DEFAULT_MAX_ROW_FBLOCK 4096 +#define TSDB_MIN_MAX_ROW_FBLOCK 200 +#define TSDB_MAX_MAX_ROW_FBLOCK 10000 + +#define TSDB_MIN_COMMIT_TIME 30 +#define TSDB_MAX_COMMIT_TIME 40960 +#define TSDB_DEFAULT_COMMIT_TIME 3600 + +#define TSDB_MIN_PRECISION TSDB_TIME_PRECISION_MILLI +#define TSDB_MAX_PRECISION TSDB_TIME_PRECISION_NANO +#define TSDB_DEFAULT_PRECISION TSDB_TIME_PRECISION_MILLI + +#define TSDB_MIN_COMP_LEVEL 0 +#define TSDB_MAX_COMP_LEVEL 2 +#define TSDB_DEFAULT_COMP_LEVEL 2 + +#define TSDB_MIN_WAL_LEVEL 0 +#define TSDB_MAX_WAL_LEVEL 2 +#define TSDB_DEFAULT_WAL_LEVEL 1 + +#define TSDB_MIN_DB_UPDATE 0 +#define TSDB_MAX_DB_UPDATE 2 +#define TSDB_DEFAULT_DB_UPDATE_OPTION 0 + +#define TSDB_MIN_DB_CACHE_LAST_ROW 0 +#define TSDB_MAX_DB_CACHE_LAST_ROW 3 +#define TSDB_DEFAULT_CACHE_LAST_ROW 0 + +#define TSDB_MIN_FSYNC_PERIOD 0 +#define TSDB_MAX_FSYNC_PERIOD 180000 // millisecond +#define TSDB_DEFAULT_FSYNC_PERIOD 3000 // three second + +#define TSDB_MIN_DB_REPLICA_OPTION 1 +#define TSDB_MAX_DB_REPLICA_OPTION 3 +#define TSDB_DEFAULT_DB_REPLICA_OPTION 1 + +#define TSDB_MIN_DB_PARTITON_OPTION 0 +#define TSDB_MAX_DB_PARTITON_OPTION 1000 +#define TSDB_DEFAULT_DB_PARTITON_OPTION 4 + +#define TSDB_MIN_DB_QUORUM_OPTION 1 +#define TSDB_MAX_DB_QUORUM_OPTION 2 +#define TSDB_DEFAULT_DB_QUORUM_OPTION 1 + +#define TSDB_MAX_JOIN_TABLE_NUM 10 +#define TSDB_MAX_UNION_CLAUSE 5 + +#define TSDB_MAX_FIELD_LEN 16384 +#define TSDB_MAX_BINARY_LEN (TSDB_MAX_FIELD_LEN-TSDB_KEYSIZE) // keep 16384 +#define TSDB_MAX_NCHAR_LEN (TSDB_MAX_FIELD_LEN-TSDB_KEYSIZE) // keep 16384 +#define PRIMARYKEY_TIMESTAMP_COL_INDEX 0 + +#define TSDB_MAX_RPC_THREADS 5 + +#define TSDB_QUERY_TYPE_NON_TYPE 0x00u // none type +#define TSDB_QUERY_TYPE_FREE_RESOURCE 0x01u // free qhandle at vnode + +#define TSDB_UDF_TYPE_SCALAR 1 +#define TSDB_UDF_TYPE_AGGREGATE 2 + +/* + * TODO remove the following definitions + * 1. ordinary sub query for select * from super_table + * 2. all sqlobj generated by createSubqueryObj with this flag + */ +#define TSDB_QUERY_TYPE_SUBQUERY 0x02u +#define TSDB_QUERY_TYPE_STABLE_SUBQUERY 0x04u // two-stage subquery for super table + +#define TSDB_QUERY_TYPE_TABLE_QUERY 0x08u // query ordinary table; below only apply to client side +#define TSDB_QUERY_TYPE_STABLE_QUERY 0x10u // query on super table +#define TSDB_QUERY_TYPE_JOIN_QUERY 0x20u // join query +#define TSDB_QUERY_TYPE_PROJECTION_QUERY 0x40u // select *,columns... query +#define TSDB_QUERY_TYPE_JOIN_SEC_STAGE 0x80u // join sub query at the second stage + +#define TSDB_QUERY_TYPE_TAG_FILTER_QUERY 0x400u +#define TSDB_QUERY_TYPE_INSERT 0x100u // insert type +#define TSDB_QUERY_TYPE_MULTITABLE_QUERY 0x200u +#define TSDB_QUERY_TYPE_FILE_INSERT 0x400u // insert data from file +#define TSDB_QUERY_TYPE_STMT_INSERT 0x800u // stmt insert type +#define TSDB_QUERY_TYPE_NEST_SUBQUERY 0x1000u // nested sub query + +#define TSDB_QUERY_HAS_TYPE(x, _type) (((x) & (_type)) != 0) +#define TSDB_QUERY_SET_TYPE(x, _type) ((x) |= (_type)) +#define TSDB_QUERY_CLEAR_TYPE(x, _type) ((x) &= (~_type)) +#define TSDB_QUERY_RESET_TYPE(x) ((x) = TSDB_QUERY_TYPE_NON_TYPE) + +#define TSDB_ORDER_ASC 1 +#define TSDB_ORDER_DESC 2 + +#define TSDB_DEFAULT_CLUSTER_HASH_SIZE 1 +#define TSDB_DEFAULT_MNODES_HASH_SIZE 5 +#define TSDB_DEFAULT_DNODES_HASH_SIZE 10 +#define TSDB_DEFAULT_ACCOUNTS_HASH_SIZE 10 +#define TSDB_DEFAULT_USERS_HASH_SIZE 20 +#define TSDB_DEFAULT_DBS_HASH_SIZE 100 +#define TSDB_DEFAULT_VGROUPS_HASH_SIZE 100 +#define TSDB_DEFAULT_STABLES_HASH_SIZE 100 +#define TSDB_DEFAULT_CTABLES_HASH_SIZE 20000 + +#define TSDB_PORT_DNODESHELL 0 +#define TSDB_PORT_DNODEDNODE 5 +#define TSDB_PORT_SYNC 10 +#define TSDB_PORT_HTTP 11 +#define TSDB_PORT_ARBITRATOR 12 + +#define TSDB_MAX_WAL_SIZE (1024*1024*3) +#define TSDB_ARB_DUMMY_TIME 4765104000000 // 2121-01-01 00:00:00.000, :P + +typedef enum { + TAOS_QTYPE_RPC = 1, + TAOS_QTYPE_FWD = 2, + TAOS_QTYPE_WAL = 3, + TAOS_QTYPE_CQ = 4, + TAOS_QTYPE_QUERY = 5 +} EQType; + +#define TSDB_MAX_TIERS 3 +#define TSDB_MAX_DISKS_PER_TIER 16 +#define TSDB_MAX_DISKS (TSDB_MAX_TIERS * TSDB_MAX_DISKS_PER_TIER) + +typedef enum { + TSDB_SUPER_TABLE = 1, // super table + TSDB_CHILD_TABLE = 2, // table created from super table + TSDB_NORMAL_TABLE = 3, // ordinary table + TSDB_STREAM_TABLE = 4, // table created from stream computing + TSDB_TEMP_TABLE = 5, // temp table created by nest query + TSDB_TABLE_MAX = 6 +} ETableType; + +typedef enum { + TSDB_MOD_MNODE = 1, + TSDB_MOD_HTTP = 2, + TSDB_MOD_MONITOR = 3, + TSDB_MOD_MQTT = 4, + TSDB_MOD_MAX = 5 +} EModuleType; + +typedef enum { + TSDB_CHECK_ITEM_NETWORK, + TSDB_CHECK_ITEM_MEM, + TSDB_CHECK_ITEM_CPU, + TSDB_CHECK_ITEM_DISK, + TSDB_CHECK_ITEM_OS, + TSDB_CHECK_ITEM_ACCESS, + TSDB_CHECK_ITEM_VERSION, + TSDB_CHECK_ITEM_DATAFILE, + TSDB_CHECK_ITEM_MAX +} ECheckItemType; + +typedef enum { + TD_ROW_DISCARD_UPDATE = 0, + TD_ROW_OVERWRITE_UPDATE = 1, + TD_ROW_PARTIAL_UPDATE = 2 +} TDUpdateConfig; + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/common/tname.h b/include/common/tname.h new file mode 100644 index 0000000000..18526f54d4 --- /dev/null +++ b/include/common/tname.h @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_TNAME_H +#define TDENGINE_TNAME_H + +#define TSDB_DB_NAME_T 1 +#define TSDB_TABLE_NAME_T 2 + +#define T_NAME_ACCT 0x1u +#define T_NAME_DB 0x2u +#define T_NAME_TABLE 0x4u + +typedef struct SName { + uint8_t type; //db_name_t, table_name_t + char acctId[TSDB_ACCT_ID_LEN]; + char dbname[TSDB_DB_NAME_LEN]; + char tname[TSDB_TABLE_NAME_LEN]; +} SName; + +int32_t tNameExtractFullName(const SName* name, char* dst); + +int32_t tNameLen(const SName* name); + +SName* tNameDup(const SName* name); + +bool tIsValidName(const SName* name); + +const char* tNameGetTableName(const SName* name); + +int32_t tNameGetDbName(const SName* name, char* dst); + +int32_t tNameGetFullDbName(const SName* name, char* dst); + +bool tNameIsEmpty(const SName* name); + +void tNameAssign(SName* dst, const SName* src); + +int32_t tNameFromString(SName* dst, const char* str, uint32_t type); + +int32_t tNameSetAcctId(SName* dst, const char* acct); + +#if 0 +int32_t tNameSetDbName(SName* dst, const char* acct, SToken* dbToken); +#endif + +#endif // TDENGINE_TNAME_H diff --git a/include/common/ttypes.h b/include/common/ttypes.h new file mode 100644 index 0000000000..6fe6e11d05 --- /dev/null +++ b/include/common/ttypes.h @@ -0,0 +1,215 @@ +#ifndef TDENGINE_TTYPE_H +#define TDENGINE_TTYPE_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "taosdef.h" + +// ----------------- For variable data types such as TSDB_DATA_TYPE_BINARY and TSDB_DATA_TYPE_NCHAR +typedef int32_t VarDataOffsetT; +typedef int16_t VarDataLenT; // maxVarDataLen: 32767 +typedef uint16_t TDRowLenT; // not including overhead: 0 ~ 65535 +typedef uint32_t TDRowTLenT; // total length, including overhead + +typedef struct tstr { + VarDataLenT len; + char data[]; +} tstr; + +#pragma pack(push, 1) +typedef struct { + VarDataLenT len; + uint8_t data; +} SBinaryNullT; + +typedef struct { + VarDataLenT len; + uint32_t data; +} SNCharNullT; +#pragma pack(pop) + +#define VARSTR_HEADER_SIZE sizeof(VarDataLenT) + +#define varDataLen(v) ((VarDataLenT *)(v))[0] +#define varDataTLen(v) (sizeof(VarDataLenT) + varDataLen(v)) +#define varDataVal(v) ((void *)((char *)v + VARSTR_HEADER_SIZE)) +#define varDataCopy(dst, v) memcpy((dst), (void*) (v), varDataTLen(v)) +#define varDataLenByData(v) (*(VarDataLenT *)(((char*)(v)) - VARSTR_HEADER_SIZE)) +#define varDataSetLen(v, _len) (((VarDataLenT *)(v))[0] = (VarDataLenT) (_len)) +#define IS_VAR_DATA_TYPE(t) (((t) == TSDB_DATA_TYPE_BINARY) || ((t) == TSDB_DATA_TYPE_NCHAR)) + +#define varDataNetLen(v) (htons(((VarDataLenT *)(v))[0])) +#define varDataNetTLen(v) (sizeof(VarDataLenT) + varDataNetLen(v)) + + +// this data type is internally used only in 'in' query to hold the values +#define TSDB_DATA_TYPE_POINTER_ARRAY (1000) +#define TSDB_DATA_TYPE_VALUE_ARRAY (1001) + +#define GET_TYPED_DATA(_v, _finalType, _type, _data) \ + do { \ + switch (_type) { \ + case TSDB_DATA_TYPE_BOOL: \ + case TSDB_DATA_TYPE_TINYINT: \ + (_v) = (_finalType)GET_INT8_VAL(_data); \ + break; \ + case TSDB_DATA_TYPE_UTINYINT: \ + (_v) = (_finalType)GET_UINT8_VAL(_data); \ + break; \ + case TSDB_DATA_TYPE_SMALLINT: \ + (_v) = (_finalType)GET_INT16_VAL(_data); \ + break; \ + case TSDB_DATA_TYPE_USMALLINT: \ + (_v) = (_finalType)GET_UINT16_VAL(_data); \ + break; \ + case TSDB_DATA_TYPE_TIMESTAMP:\ + case TSDB_DATA_TYPE_BIGINT: \ + (_v) = (_finalType)(GET_INT64_VAL(_data)); \ + break; \ + case TSDB_DATA_TYPE_UBIGINT: \ + (_v) = (_finalType)(GET_UINT64_VAL(_data)); \ + break; \ + case TSDB_DATA_TYPE_FLOAT: \ + (_v) = (_finalType)GET_FLOAT_VAL(_data); \ + break; \ + case TSDB_DATA_TYPE_DOUBLE: \ + (_v) = (_finalType)GET_DOUBLE_VAL(_data); \ + break; \ + case TSDB_DATA_TYPE_UINT: \ + (_v) = (_finalType)GET_UINT32_VAL(_data); \ + break; \ + default: \ + (_v) = (_finalType)GET_INT32_VAL(_data); \ + break; \ + } \ + } while (0) + +#define SET_TYPED_DATA(_v, _type, _data) \ + do { \ + switch (_type) { \ + case TSDB_DATA_TYPE_BOOL: \ + case TSDB_DATA_TYPE_TINYINT: \ + *(int8_t *)(_v) = (int8_t)(_data); \ + break; \ + case TSDB_DATA_TYPE_UTINYINT: \ + *(uint8_t *)(_v) = (uint8_t)(_data); \ + break; \ + case TSDB_DATA_TYPE_SMALLINT: \ + *(int16_t *)(_v) = (int16_t)(_data); \ + break; \ + case TSDB_DATA_TYPE_USMALLINT: \ + *(uint16_t *)(_v) = (uint16_t)(_data); \ + break; \ + case TSDB_DATA_TYPE_BIGINT: \ + *(int64_t *)(_v) = (int64_t)(_data); \ + break; \ + case TSDB_DATA_TYPE_UBIGINT: \ + *(uint64_t *)(_v) = (uint64_t)(_data); \ + break; \ + case TSDB_DATA_TYPE_FLOAT: \ + *(float *)(_v) = (float)(_data); \ + break; \ + case TSDB_DATA_TYPE_DOUBLE: \ + *(double *)(_v) = (double)(_data); \ + break; \ + case TSDB_DATA_TYPE_UINT: \ + *(uint32_t *)(_v) = (uint32_t)(_data); \ + break; \ + default: \ + *(int32_t *)(_v) = (int32_t)(_data); \ + break; \ + } \ + } while (0) + +#define IS_SIGNED_NUMERIC_TYPE(_t) ((_t) >= TSDB_DATA_TYPE_TINYINT && (_t) <= TSDB_DATA_TYPE_BIGINT) +#define IS_UNSIGNED_NUMERIC_TYPE(_t) ((_t) >= TSDB_DATA_TYPE_UTINYINT && (_t) <= TSDB_DATA_TYPE_UBIGINT) +#define IS_FLOAT_TYPE(_t) ((_t) == TSDB_DATA_TYPE_FLOAT || (_t) == TSDB_DATA_TYPE_DOUBLE) + +#define IS_NUMERIC_TYPE(_t) ((IS_SIGNED_NUMERIC_TYPE(_t)) || (IS_UNSIGNED_NUMERIC_TYPE(_t)) || (IS_FLOAT_TYPE(_t))) + +#define IS_VALID_TINYINT(_t) ((_t) > INT8_MIN && (_t) <= INT8_MAX) +#define IS_VALID_SMALLINT(_t) ((_t) > INT16_MIN && (_t) <= INT16_MAX) +#define IS_VALID_INT(_t) ((_t) > INT32_MIN && (_t) <= INT32_MAX) +#define IS_VALID_BIGINT(_t) ((_t) > INT64_MIN && (_t) <= INT64_MAX) +#define IS_VALID_UTINYINT(_t) ((_t) >= 0 && (_t) < UINT8_MAX) +#define IS_VALID_USMALLINT(_t) ((_t) >= 0 && (_t) < UINT16_MAX) +#define IS_VALID_UINT(_t) ((_t) >= 0 && (_t) < UINT32_MAX) +#define IS_VALID_UBIGINT(_t) ((_t) >= 0 && (_t) < UINT64_MAX) +#define IS_VALID_FLOAT(_t) ((_t) >= -FLT_MAX && (_t) <= FLT_MAX) +#define IS_VALID_DOUBLE(_t) ((_t) >= -DBL_MAX && (_t) <= DBL_MAX) + +static FORCE_INLINE bool isNull(const void *val, int32_t type) { + switch (type) { + case TSDB_DATA_TYPE_BOOL: + return *(uint8_t *)val == TSDB_DATA_BOOL_NULL; + case TSDB_DATA_TYPE_TINYINT: + return *(uint8_t *)val == TSDB_DATA_TINYINT_NULL; + case TSDB_DATA_TYPE_SMALLINT: + return *(uint16_t *)val == TSDB_DATA_SMALLINT_NULL; + case TSDB_DATA_TYPE_INT: + return *(uint32_t *)val == TSDB_DATA_INT_NULL; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: + return *(uint64_t *)val == TSDB_DATA_BIGINT_NULL; + case TSDB_DATA_TYPE_FLOAT: + return *(uint32_t *)val == TSDB_DATA_FLOAT_NULL; + case TSDB_DATA_TYPE_DOUBLE: + return *(uint64_t *)val == TSDB_DATA_DOUBLE_NULL; + case TSDB_DATA_TYPE_NCHAR: + return varDataLen(val) == sizeof(int32_t) && *(uint32_t*) varDataVal(val) == TSDB_DATA_NCHAR_NULL; + case TSDB_DATA_TYPE_BINARY: + return varDataLen(val) == sizeof(int8_t) && *(uint8_t *) varDataVal(val) == TSDB_DATA_BINARY_NULL; + case TSDB_DATA_TYPE_UTINYINT: + return *(uint8_t*) val == TSDB_DATA_UTINYINT_NULL; + case TSDB_DATA_TYPE_USMALLINT: + return *(uint16_t*) val == TSDB_DATA_USMALLINT_NULL; + case TSDB_DATA_TYPE_UINT: + return *(uint32_t*) val == TSDB_DATA_UINT_NULL; + case TSDB_DATA_TYPE_UBIGINT: + return *(uint64_t*) val == TSDB_DATA_UBIGINT_NULL; + + default: + return false; + }; +} + +typedef struct tDataTypeDescriptor { + int16_t type; + int16_t nameLen; + int32_t bytes; + char * name; + int64_t minValue; + int64_t maxValue; + int (*compFunc)(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, + char algorithm, char *const buffer, int bufferSize); + int (*decompFunc)(const char *const input, int compressedSize, const int nelements, char *const output, + int outputSize, char algorithm, char *const buffer, int bufferSize); + void (*statisFunc)(const void *pData, int32_t numofrow, int64_t *min, int64_t *max, int64_t *sum, + int16_t *minindex, int16_t *maxindex, int16_t *numofnull); +} tDataTypeDescriptor; + +extern tDataTypeDescriptor tDataTypes[15]; + +bool isValidDataType(int32_t type); + +void setVardataNull(void* val, int32_t type); +void setNull(void *val, int32_t type, int32_t bytes); +void setNullN(void *val, int32_t type, int32_t bytes, int32_t numOfElems); +const void *getNullValue(int32_t type); + +void assignVal(char *val, const char *src, int32_t len, int32_t type); +void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf); +void operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type); +void* getDataMin(int32_t type); +void* getDataMax(int32_t type); + + +#define SET_DOUBLE_NULL(v) (*(uint64_t *)(v) = TSDB_DATA_DOUBLE_NULL) + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_TTYPE_H diff --git a/include/util/thash.h b/include/util/thash.h new file mode 100644 index 0000000000..e863396f38 --- /dev/null +++ b/include/util/thash.h @@ -0,0 +1,206 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_HASH_H +#define TDENGINE_HASH_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "tarray.h" +#include "tlockfree.h" + +typedef uint32_t (*_hash_fn_t)(const char *, uint32_t); +typedef int32_t (*_equal_fn_t)(const void*, const void*); +typedef void (*_hash_before_fn_t)(void *); +typedef void (*_hash_free_fn_t)(void *); + +#define HASH_MAX_CAPACITY (1024 * 1024 * 16) +#define HASH_DEFAULT_LOAD_FACTOR (0.75) + +#define HASH_INDEX(v, c) ((v) & ((c)-1)) + +/** + * murmur hash algorithm + * @key usually string + * @len key length + * @seed hash seed + * @out an int32 value + */ +uint32_t MurmurHash3_32(const char *key, uint32_t len); + +/** + * + * @param key + * @param len + * @return + */ +uint32_t taosIntHash_32(const char *key, uint32_t len); +uint32_t taosIntHash_64(const char *key, uint32_t len); + +_hash_fn_t taosGetDefaultHashFunction(int32_t type); + +typedef struct SHashNode { + struct SHashNode *next; + uint32_t hashVal; // the hash value of key + uint32_t dataLen; // length of data + uint32_t keyLen; // length of the key + uint16_t count; // reference count + int8_t removed; // flag to indicate removed + char data[]; +} SHashNode; + +#define GET_HASH_NODE_KEY(_n) ((char*)(_n) + sizeof(SHashNode) + (_n)->dataLen) +#define GET_HASH_NODE_DATA(_n) ((char*)(_n) + sizeof(SHashNode)) +#define GET_HASH_PNODE(_n) ((SHashNode *)((char*)(_n) - sizeof(SHashNode))) + +typedef enum SHashLockTypeE { + HASH_NO_LOCK = 0, + HASH_ENTRY_LOCK = 1, +} SHashLockTypeE; + +typedef struct SHashEntry { + int32_t num; // number of elements in current entry + SRWLatch latch; // entry latch + SHashNode *next; +} SHashEntry; + +typedef struct SHashObj { + SHashEntry **hashList; + uint32_t capacity; // number of slots + uint32_t size; // number of elements in hash table + + _hash_fn_t hashFp; // hash function + _hash_free_fn_t freeFp; // hash node free callback function + _equal_fn_t equalFp; // equal function + _hash_before_fn_t callbackFp; // function invoked before return the value to caller + + SRWLatch lock; // read-write spin lock + SHashLockTypeE type; // lock type + bool enableUpdate; // enable update + SArray *pMemBlock; // memory block allocated for SHashEntry +} SHashObj; + +/** + * init the hash table + * + * @param capacity initial capacity of the hash table + * @param fn hash function to generate the hash value + * @param threadsafe thread safe or not + * @return + */ +SHashObj *taosHashInit(size_t capacity, _hash_fn_t fn, bool update, SHashLockTypeE type); + +/** + * return the size of hash table + * @param pHashObj + * @return + */ +int32_t taosHashGetSize(const SHashObj *pHashObj); + +/** + * put element into hash table, if the element with the same key exists, update it + * @param pHashObj + * @param key + * @param keyLen + * @param data + * @param size + * @return + */ +int32_t taosHashPut(SHashObj *pHashObj, const void *key, size_t keyLen, void *data, size_t size); + +/** + * return the payload data with the specified key + * + * @param pHashObj + * @param key + * @param keyLen + * @return + */ +void *taosHashGet(SHashObj *pHashObj, const void *key, size_t keyLen); + +/** + * Clone the result to destination buffer + * @param pHashObj + * @param key + * @param keyLen + * @param destBuf + * @return + */ +void *taosHashGetClone(SHashObj *pHashObj, const void *key, size_t keyLen, void* destBuf); + +/** + * remove item with the specified key + * @param pHashObj + * @param key + * @param keyLen + */ +int32_t taosHashRemove(SHashObj *pHashObj, const void *key, size_t keyLen); + +/** + * Clear the hash table. + * @param pHashObj + */ +void taosHashClear(SHashObj *pHashObj); + +/** + * Clean up hash table and release all allocated resources. + * @param handle + */ +void taosHashCleanup(SHashObj *pHashObj); + +/** + * Get the max overflow link list length + * @param pHashObj + * @return + */ +int32_t taosHashGetMaxOverflowLinkLength(const SHashObj *pHashObj); + +/** + * Get the hash table size + * @param pHashObj + * @return + */ +size_t taosHashGetMemSize(const SHashObj *pHashObj); + +/** + * Create the hash table iterator + * @param pHashObj + * @param p + * @return + */ +void *taosHashIterate(SHashObj *pHashObj, void *p); + +/** + * Cancel the hash table iterator + * @param pHashObj + * @param p + */ +void taosHashCancelIterate(SHashObj *pHashObj, void *p); + +/** + * Get the corresponding key information for a given data in hash table + * @param pHashObj + * @param data + * @return + */ +int32_t taosHashGetKey(SHashObj *pHashObj, void *data, void** key, size_t* keyLen); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_HASH_H diff --git a/source/libs/parser/inc/astGenerator.h b/source/libs/parser/inc/astGenerator.h new file mode 100644 index 0000000000..b6d7545f3b --- /dev/null +++ b/source/libs/parser/inc/astGenerator.h @@ -0,0 +1,360 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_ASTGENERATOR_H +#define TDENGINE_ASTGENERATOR_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "ttoken.h" +#include "tvariant.h" +#include "parser.h" + +#define ParseTOKENTYPE SToken + +#define NON_ARITHMEIC_EXPR 0 +#define NORMAL_ARITHMETIC 1 +#define AGG_ARIGHTMEIC 2 + +enum SQL_NODE_TYPE { + SQL_NODE_TABLE_COLUMN= 1, + SQL_NODE_SQLFUNCTION = 2, + SQL_NODE_VALUE = 3, + SQL_NODE_EXPR = 4, +}; + +enum SQL_NODE_FROM_TYPE { + SQL_NODE_FROM_SUBQUERY = 1, + SQL_NODE_FROM_TABLELIST = 2, +}; + +//enum SQL_EXPR_FLAG { +// EXPR_FLAG_TS_ERROR = 1, +// EXPR_FLAG_NS_TIMESTAMP = 2, +// EXPR_FLAG_TIMESTAMP_VAR = 3, +//}; + +extern char tTokenTypeSwitcher[13]; + +#define toTSDBType(x) \ + do { \ + if ((x) >= tListLen(tTokenTypeSwitcher)) { \ + (x) = TSDB_DATA_TYPE_BINARY; \ + } else { \ + (x) = tTokenTypeSwitcher[(x)]; \ + } \ + } while (0) + +#define TPARSER_HAS_TOKEN(_t) ((_t).n > 0) +#define TPARSER_SET_NONE_TOKEN(_t) ((_t).n = 0) + +typedef struct SListItem { + SVariant pVar; + uint8_t sortOrder; +} SListItem; + +typedef struct SIntervalVal { + int32_t token; + SToken interval; + SToken offset; +} SIntervalVal; + +typedef struct SSessionWindowVal { + SToken col; + SToken gap; +} SSessionWindowVal; + +typedef struct SWindowStateVal { + SToken col; +} SWindowStateVal; + +struct SRelationInfo; + +typedef struct SSqlNode { + struct SArray *pSelNodeList; // select clause + struct SRelationInfo *from; // from clause SArray + struct tSqlExpr *pWhere; // where clause [optional] + SArray *pGroupby; // groupby clause, only for tags[optional], SArray + SArray *pSortOrder; // orderby [optional], SArray + SArray *fillType; // fill type[optional], SArray + SIntervalVal interval; // (interval, interval_offset) [optional] + SSessionWindowVal sessionVal; // session window [optional] + SWindowStateVal windowstateVal; // window_state(col) [optional] + SToken sliding; // sliding window [optional] + SLimit limit; // limit offset [optional] + SLimit slimit; // group limit offset [optional] + SToken sqlstr; // sql string in select clause + struct tSqlExpr *pHaving; // having clause [optional] +} SSqlNode; + +typedef struct SRelElementPair { + union { + SToken tableName; + SArray *pSubquery; + }; + + SToken aliasName; +} SRelElementPair; + +typedef struct SRelationInfo { + int32_t type; // nested query|table name list + SArray *list; // SArray +} SRelationInfo; + +typedef struct SCreatedTableInfo { + SToken name; // table name token + SToken stableName; // super table name token , for using clause + SArray *pTagNames; // create by using super table, tag name + SArray *pTagVals; // create by using super table, tag value + char *fullname; // table full name + STagData tagdata; // true tag data, super table full name is in STagData + int8_t igExist; // ignore if exists +} SCreatedTableInfo; + +typedef struct SCreateTableSql { + SToken name; // table name, create table [name] xxx + int8_t type; // create normal table/from super table/ stream + bool existCheck; + + struct { + SArray *pTagColumns; // SArray + SArray *pColumns; // SArray + } colInfo; + + SArray *childTableInfo; // SArray + SSqlNode *pSelect; +} SCreateTableSql; + +typedef struct SAlterTableInfo { + SToken name; + int16_t tableType; + int16_t type; + STagData tagData; + SArray *pAddColumns; // SArray + SArray *varList; // set t=val or: change src dst, SArray +} SAlterTableInfo; + +typedef struct SCreateDbInfo { + SToken dbname; + int32_t replica; + int32_t cacheBlockSize; + int32_t maxTablesPerVnode; + int32_t numOfBlocks; + int32_t daysPerFile; + int32_t minRowsPerBlock; + int32_t maxRowsPerBlock; + int32_t fsyncPeriod; + int64_t commitTime; + int32_t walLevel; + int32_t quorum; + int32_t compressionLevel; + SToken precision; + bool ignoreExists; + int8_t update; + int8_t cachelast; + SArray *keep; + int8_t dbType; + int16_t partitions; +} SCreateDbInfo; + +typedef struct SCreateFuncInfo { + SToken name; + SToken path; + int32_t type; + int32_t bufSize; + SField output; +} SCreateFuncInfo; + +typedef struct SCreateAcctInfo { + int32_t maxUsers; + int32_t maxDbs; + int32_t maxTimeSeries; + int32_t maxStreams; + int32_t maxPointsPerSecond; + int64_t maxStorage; + int64_t maxQueryTime; + int32_t maxConnections; + SToken stat; +} SCreateAcctInfo; + +typedef struct SShowInfo { + uint8_t showType; + SToken prefix; + SToken pattern; +} SShowInfo; + +typedef struct SUserInfo { + SToken user; + SToken passwd; + SToken privilege; + int16_t type; +} SUserInfo; + +typedef struct SMiscInfo { + SArray *a; // SArray + bool existsCheck; + int16_t dbType; + int16_t tableType; + SUserInfo user; + union { + SCreateDbInfo dbOpt; + SCreateAcctInfo acctOpt; + SCreateFuncInfo funcOpt; + SShowInfo showOpt; + SToken id; + }; +} SMiscInfo; + +typedef struct SSqlInfo { + int32_t type; + bool valid; + SArray *list; // todo refactor + char msg[256]; + SArray *funcs; + union { + SCreateTableSql *pCreateTableInfo; + SAlterTableInfo *pAlterInfo; + SMiscInfo *pMiscInfo; + }; +} SSqlInfo; + +typedef struct tSqlExpr { + uint16_t type; // sql node type + uint32_t tokenId; // TK_LE: less than(binary expr) + + // The complete string of the function(col, param), and the function name is kept in exprToken + struct { + SToken operand; + struct SArray *paramList; // function parameters list + } Expr; + + SToken columnName; // table column info + SVariant value; // the use input value + SToken exprToken; // original sql expr string or function name of sql function + struct tSqlExpr *pLeft; // the left child + struct tSqlExpr *pRight; // the right child +} tSqlExpr; + +// used in select clause. select from xxx +typedef struct tSqlExprItem { + tSqlExpr *pNode; // The list of expressions + char *aliasName; // alias name, null-terminated string + bool distinct; +} tSqlExprItem; + +SArray *tListItemAppend(SArray *pList, SVariant *pVar, uint8_t sortOrder); +SArray *tListItemInsert(SArray *pList, SVariant *pVar, uint8_t sortOrder, int32_t index); +SArray *tListItemAppendToken(SArray *pList, SToken *pAliasToken, uint8_t sortOrder); + +SRelationInfo *setTableNameList(SRelationInfo *pRelationInfo, SToken *pName, SToken *pAlias); +void * destroyRelationInfo(SRelationInfo *pFromInfo); +SRelationInfo *addSubquery(SRelationInfo *pRelationInfo, SArray *pSub, SToken *pAlias); + +// sql expr leaf node +tSqlExpr *tSqlExprCreateIdValue(SToken *pToken, int32_t optrType); +tSqlExpr *tSqlExprCreateFunction(SArray *pParam, SToken *pFuncToken, SToken *endToken, int32_t optType); +SArray * tAppendFuncName(SArray *pList, SToken *pToken); + +tSqlExpr *tSqlExprCreate(tSqlExpr *pLeft, tSqlExpr *pRight, int32_t optrType); +tSqlExpr *tSqlExprClone(tSqlExpr *pSrc); +void tSqlExprCompact(tSqlExpr **pExpr); +bool tSqlExprIsLeaf(tSqlExpr *pExpr); +bool tSqlExprIsParentOfLeaf(tSqlExpr *pExpr); +void tSqlExprDestroy(tSqlExpr *pExpr); +SArray * tSqlExprListAppend(SArray *pList, tSqlExpr *pNode, SToken *pDistinct, SToken *pToken); +void tSqlExprListDestroy(SArray *pList); + +SSqlNode *tSetQuerySqlNode(SToken *pSelectToken, SArray *pSelNodeList, SRelationInfo *pFrom, tSqlExpr *pWhere, + SArray *pGroupby, SArray *pSortOrder, SIntervalVal *pInterval, SSessionWindowVal *ps, + SWindowStateVal *pw, SToken *pSliding, SArray *pFill, SLimit *pLimit, SLimit *pgLimit, tSqlExpr *pHaving); +int32_t tSqlExprCompare(tSqlExpr *left, tSqlExpr *right); + +SCreateTableSql *tSetCreateTableInfo(SArray *pCols, SArray *pTags, SSqlNode *pSelect, int32_t type); + +SAlterTableInfo * tSetAlterTableInfo(SToken *pTableName, SArray *pCols, SArray *pVals, int32_t type, + int16_t tableTable); +SCreatedTableInfo createNewChildTableInfo(SToken *pTableName, SArray *pTagNames, SArray *pTagVals, SToken *pToken, + SToken *igExists); + +void destroyAllSqlNode(SArray *pSqlNode); +void destroySqlNode(SSqlNode *pSql); +void freeCreateTableInfo(void* p); + +SSqlInfo *setSqlInfo(SSqlInfo *pInfo, void *pSqlExprInfo, SToken *pTableName, int32_t type); +SArray *setSubclause(SArray *pList, void *pSqlNode); +SArray *appendSelectClause(SArray *pList, void *pSubclause); + +void setCreatedTableName(SSqlInfo *pInfo, SToken *pTableNameToken, SToken *pIfNotExists); + +void SqlInfoDestroy(SSqlInfo *pInfo); + +void setDCLSqlElems(SSqlInfo *pInfo, int32_t type, int32_t nParams, ...); +void setDropDbTableInfo(SSqlInfo *pInfo, int32_t type, SToken* pToken, SToken* existsCheck,int16_t dbType,int16_t tableType); +void setShowOptions(SSqlInfo *pInfo, int32_t type, SToken* prefix, SToken* pPatterns); + +void setCreateDbInfo(SSqlInfo *pInfo, int32_t type, SToken *pToken, SCreateDbInfo *pDB, SToken *pIgExists); + +void setCreateAcctSql(SSqlInfo *pInfo, int32_t type, SToken *pName, SToken *pPwd, SCreateAcctInfo *pAcctInfo); +void setCreateUserSql(SSqlInfo *pInfo, SToken *pName, SToken *pPasswd); +void setKillSql(SSqlInfo *pInfo, int32_t type, SToken *ip); +void setAlterUserSql(SSqlInfo *pInfo, int16_t type, SToken *pName, SToken* pPwd, SToken *pPrivilege); + +void setCompactVnodeSql(SSqlInfo *pInfo, int32_t type, SArray *pParam); + +void setDefaultCreateDbOption(SCreateDbInfo *pDBInfo); +void setDefaultCreateTopicOption(SCreateDbInfo *pDBInfo); + +// prefix show db.tables; +void tSetDbName(SToken *pCpxName, SToken *pDb); + +void tSetColumnInfo(struct SField *pField, SToken *pName, struct SField *pType); +void tSetColumnType(struct SField *pField, SToken *type); + +/** + * The main parse function. + * @param yyp The parser + * @param yymajor The major token code number + * @param yyminor The value for the token + */ +void Parse(void *yyp, int yymajor, ParseTOKENTYPE yyminor, SSqlInfo *); + +/** + * Free the allocated resources in case of failure. + * @param p The parser to be deleted + * @param freeProc Function used to reclaim memory + */ +void ParseFree(void *p, void (*freeProc)(void *)); + +/** + * Allocated callback function. + * @param mallocProc The parser allocator + * @return + */ +void *ParseAlloc(void *(*mallocProc)(size_t)); + +/** + * + * @param str sql string + * @return sql ast + */ +SSqlInfo genAST(const char *str); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_ASTGENERATOR_H diff --git a/source/libs/parser/inc/insertParser.h b/source/libs/parser/inc/insertParser.h new file mode 100644 index 0000000000..49e678cd54 --- /dev/null +++ b/source/libs/parser/inc/insertParser.h @@ -0,0 +1,19 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_INSERTPARSER_H +#define TDENGINE_INSERTPARSER_H + +#endif // TDENGINE_INSERTPARSER_H diff --git a/source/libs/parser/inc/parserUtil.h b/source/libs/parser/inc/parserUtil.h new file mode 100644 index 0000000000..4e6af0a901 --- /dev/null +++ b/source/libs/parser/inc/parserUtil.h @@ -0,0 +1,28 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_PARSERUTIL_H +#define TDENGINE_PARSERUTIL_H + +#ifdef __cplusplus +extern "C" { +#endif + + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_PARSERUTIL_H diff --git a/source/libs/parser/inc/ttoken.h b/source/libs/parser/inc/ttoken.h new file mode 100644 index 0000000000..bacabe299e --- /dev/null +++ b/source/libs/parser/inc/ttoken.h @@ -0,0 +1,185 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_TTOKEN_H +#define TDENGINE_TTOKEN_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "os.h" +#include "ttokendef.h" + +// used to denote the minimum unite in sql parsing +typedef struct SToken { + uint32_t n; + uint32_t type; + char *z; +} SToken; + +/** + * check if it is a number or not + * @param pToken + * @return + */ +#define isNumber(tk) \ +((tk)->type == TK_INTEGER || (tk)->type == TK_FLOAT || (tk)->type == TK_HEX || (tk)->type == TK_BIN) + +/** + * tokenizer for sql string + * @param z + * @param tokenType + * @return + */ +uint32_t tGetToken(char *z, uint32_t *tokenType); + +/** + * enhanced tokenizer for sql string. + * + * @param str + * @param i + * @param isPrevOptr + * @return + */ +SToken tStrGetToken(char *str, int32_t *i, bool isPrevOptr); + +/** + * check if it is a keyword or not + * @param z + * @param len + * @return + */ +bool taosIsKeyWordToken(const char *z, int32_t len); + +/** + * check if it is a token or not + * @param pToken + * @return token type, if it is not a number, TK_ILLEGAL will return + */ +static FORCE_INLINE int32_t tGetNumericStringType(const SToken* pToken) { + const char* z = pToken->z; + int32_t type = TK_ILLEGAL; + + uint32_t i = 0; + for(; i < pToken->n; ++i) { + switch (z[i]) { + case '+': + case '-': { + break; + } + + case '.': { + /* + * handle the the float number with out integer part + * .123 + * .123e4 + */ + if (!isdigit(z[i+1])) { + return TK_ILLEGAL; + } + + for (i += 2; isdigit(z[i]); i++) { + } + + if ((z[i] == 'e' || z[i] == 'E') && + (isdigit(z[i + 1]) || ((z[i + 1] == '+' || z[i + 1] == '-') && isdigit(z[i + 2])))) { + i += 2; + while (isdigit(z[i])) { + i++; + } + } + + type = TK_FLOAT; + goto _end; + } + + case '0': { + char next = z[i + 1]; + if (next == 'b') { // bin number + type = TK_BIN; + for (i += 2; (z[i] == '0' || z[i] == '1'); ++i) { + } + + goto _end; + } else if (next == 'x') { //hex number + type = TK_HEX; + for (i += 2; isdigit(z[i]) || (z[i] >= 'a' && z[i] <= 'f') || (z[i] >= 'A' && z[i] <= 'F'); ++i) { + } + + goto _end; + } + } + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': { + type = TK_INTEGER; + for (; isdigit(z[i]); i++) { + } + + int32_t seg = 0; + while (z[i] == '.' && isdigit(z[i + 1])) { + i += 2; + + while (isdigit(z[i])) { + i++; + } + + seg++; + type = TK_FLOAT; + } + + if (seg > 1) { + return TK_ILLEGAL; + } + + if ((z[i] == 'e' || z[i] == 'E') && + (isdigit(z[i + 1]) || ((z[i + 1] == '+' || z[i + 1] == '-') && isdigit(z[i + 2])))) { + i += 2; + while (isdigit(z[i])) { + i++; + } + + type = TK_FLOAT; + } + + goto _end; + } + default: + return TK_ILLEGAL; + } + } + + _end: + return (i < pToken->n)? TK_ILLEGAL:type; +} + +void taosCleanupKeywordsTable(); + +SToken tscReplaceStrToken(char **str, SToken *token, const char* newToken); + +SToken taosTokenDup(SToken* pToken, char* buf, int32_t len); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_TTOKEN_H diff --git a/source/libs/parser/inc/ttokendef.h b/source/libs/parser/inc/ttokendef.h new file mode 100644 index 0000000000..9b0ad2cf13 --- /dev/null +++ b/source/libs/parser/inc/ttokendef.h @@ -0,0 +1,228 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_TTOKENDEF_H +#define TDENGINE_TTOKENDEF_H + +#define TK_ID 1 +#define TK_BOOL 2 +#define TK_TINYINT 3 +#define TK_SMALLINT 4 +#define TK_INTEGER 5 +#define TK_BIGINT 6 +#define TK_FLOAT 7 +#define TK_DOUBLE 8 +#define TK_STRING 9 +#define TK_TIMESTAMP 10 +#define TK_BINARY 11 +#define TK_NCHAR 12 +#define TK_OR 13 +#define TK_AND 14 +#define TK_NOT 15 +#define TK_EQ 16 +#define TK_NE 17 +#define TK_ISNULL 18 +#define TK_NOTNULL 19 +#define TK_IS 20 +#define TK_LIKE 21 +#define TK_MATCH 22 +#define TK_NMATCH 23 +#define TK_GLOB 24 +#define TK_BETWEEN 25 +#define TK_IN 26 +#define TK_GT 27 +#define TK_GE 28 +#define TK_LT 29 +#define TK_LE 30 +#define TK_BITAND 31 +#define TK_BITOR 32 +#define TK_LSHIFT 33 +#define TK_RSHIFT 34 +#define TK_PLUS 35 +#define TK_MINUS 36 +#define TK_DIVIDE 37 +#define TK_TIMES 38 +#define TK_STAR 39 +#define TK_SLASH 40 +#define TK_REM 41 +#define TK_CONCAT 42 +#define TK_UMINUS 43 +#define TK_UPLUS 44 +#define TK_BITNOT 45 +#define TK_SHOW 46 +#define TK_DATABASES 47 +#define TK_TOPICS 48 +#define TK_FUNCTIONS 49 +#define TK_MNODES 50 +#define TK_DNODES 51 +#define TK_ACCOUNTS 52 +#define TK_USERS 53 +#define TK_MODULES 54 +#define TK_QUERIES 55 +#define TK_CONNECTIONS 56 +#define TK_STREAMS 57 +#define TK_VARIABLES 58 +#define TK_SCORES 59 +#define TK_GRANTS 60 +#define TK_VNODES 61 +#define TK_DOT 62 +#define TK_CREATE 63 +#define TK_TABLE 64 +#define TK_STABLE 65 +#define TK_DATABASE 66 +#define TK_TABLES 67 +#define TK_STABLES 68 +#define TK_VGROUPS 69 +#define TK_DROP 70 +#define TK_TOPIC 71 +#define TK_FUNCTION 72 +#define TK_DNODE 73 +#define TK_USER 74 +#define TK_ACCOUNT 75 +#define TK_USE 76 +#define TK_DESCRIBE 77 +#define TK_DESC 78 +#define TK_ALTER 79 +#define TK_PASS 80 +#define TK_PRIVILEGE 81 +#define TK_LOCAL 82 +#define TK_COMPACT 83 +#define TK_LP 84 +#define TK_RP 85 +#define TK_IF 86 +#define TK_EXISTS 87 +#define TK_AS 88 +#define TK_OUTPUTTYPE 89 +#define TK_AGGREGATE 90 +#define TK_BUFSIZE 91 +#define TK_PPS 92 +#define TK_TSERIES 93 +#define TK_DBS 94 +#define TK_STORAGE 95 +#define TK_QTIME 96 +#define TK_CONNS 97 +#define TK_STATE 98 +#define TK_COMMA 99 +#define TK_KEEP 100 +#define TK_CACHE 101 +#define TK_REPLICA 102 +#define TK_QUORUM 103 +#define TK_DAYS 104 +#define TK_MINROWS 105 +#define TK_MAXROWS 106 +#define TK_BLOCKS 107 +#define TK_CTIME 108 +#define TK_WAL 109 +#define TK_FSYNC 110 +#define TK_COMP 111 +#define TK_PRECISION 112 +#define TK_UPDATE 113 +#define TK_CACHELAST 114 +#define TK_PARTITIONS 115 +#define TK_UNSIGNED 116 +#define TK_TAGS 117 +#define TK_USING 118 +#define TK_NULL 119 +#define TK_NOW 120 +#define TK_SELECT 121 +#define TK_UNION 122 +#define TK_ALL 123 +#define TK_DISTINCT 124 +#define TK_FROM 125 +#define TK_VARIABLE 126 +#define TK_INTERVAL 127 +#define TK_EVERY 128 +#define TK_SESSION 129 +#define TK_STATE_WINDOW 130 +#define TK_FILL 131 +#define TK_SLIDING 132 +#define TK_ORDER 133 +#define TK_BY 134 +#define TK_ASC 135 +#define TK_GROUP 136 +#define TK_HAVING 137 +#define TK_LIMIT 138 +#define TK_OFFSET 139 +#define TK_SLIMIT 140 +#define TK_SOFFSET 141 +#define TK_WHERE 142 +#define TK_RESET 143 +#define TK_QUERY 144 +#define TK_SYNCDB 145 +#define TK_ADD 146 +#define TK_COLUMN 147 +#define TK_MODIFY 148 +#define TK_TAG 149 +#define TK_CHANGE 150 +#define TK_SET 151 +#define TK_KILL 152 +#define TK_CONNECTION 153 +#define TK_STREAM 154 +#define TK_COLON 155 +#define TK_ABORT 156 +#define TK_AFTER 157 +#define TK_ATTACH 158 +#define TK_BEFORE 159 +#define TK_BEGIN 160 +#define TK_CASCADE 161 +#define TK_CLUSTER 162 +#define TK_CONFLICT 163 +#define TK_COPY 164 +#define TK_DEFERRED 165 +#define TK_DELIMITERS 166 +#define TK_DETACH 167 +#define TK_EACH 168 +#define TK_END 169 +#define TK_EXPLAIN 170 +#define TK_FAIL 171 +#define TK_FOR 172 +#define TK_IGNORE 173 +#define TK_IMMEDIATE 174 +#define TK_INITIALLY 175 +#define TK_INSTEAD 176 +#define TK_KEY 177 +#define TK_OF 178 +#define TK_RAISE 179 +#define TK_REPLACE 180 +#define TK_RESTRICT 181 +#define TK_ROW 182 +#define TK_STATEMENT 183 +#define TK_TRIGGER 184 +#define TK_VIEW 185 +#define TK_IPTOKEN 186 +#define TK_SEMI 187 +#define TK_NONE 188 +#define TK_PREV 189 +#define TK_LINEAR 190 +#define TK_IMPORT 191 +#define TK_TBNAME 192 +#define TK_JOIN 193 +#define TK_INSERT 194 +#define TK_INTO 195 +#define TK_VALUES 196 + + +#define TK_SPACE 300 +#define TK_COMMENT 301 +#define TK_ILLEGAL 302 +#define TK_HEX 303 // hex number 0x123 +#define TK_OCT 304 // oct number +#define TK_BIN 305 // bin format data 0b111 +#define TK_FILE 306 +#define TK_QUESTION 307 // denoting the placeholder of "?",when invoking statement bind query + +#endif + + diff --git a/source/libs/parser/inc/tvariant.h b/source/libs/parser/inc/tvariant.h new file mode 100644 index 0000000000..941d1920cb --- /dev/null +++ b/source/libs/parser/inc/tvariant.h @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_TVARIANT_H +#define TDENGINE_TVARIANT_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "tarray.h" +#include "ttoken.h" + +// variant, each number/string/field_id has a corresponding struct during parsing sql +typedef struct SVariant { + uint32_t nType; + int32_t nLen; // only used for string, for number, it is useless + union { + int64_t i64; + uint64_t u64; + double d; + char *pz; + wchar_t *wpz; + SArray *arr; // only for 'in' query to hold value list, not value for a field + }; +} SVariant; + +bool taosVariantIsValid(SVariant *pVar); + +void taosVariantCreate(SVariant *pVar, SStrToken *token); + +void taosVariantCreateFromBinary(SVariant *pVar, const char *pz, size_t len, uint32_t type); + +void taosVariantDestroy(SVariant *pV); + +void taosVariantAssign(SVariant *pDst, const SVariant *pSrc); + +int32_t taosVariantCompare(const SVariant* p1, const SVariant* p2); + +int32_t taosVariantToString(SVariant *pVar, char *dst); + +int32_t taosVariantDump(SVariant *pVariant, char *payload, int16_t type, bool includeLengthPrefix); + +#if 0 +int32_t taosVariantDumpEx(SVariant *pVariant, char *payload, int16_t type, bool includeLengthPrefix, bool *converted, char *extInfo); +#endif + +int32_t taosVariantTypeSetType(SVariant *pVariant, char type); + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_TVARIANT_H diff --git a/source/libs/parser/src/astGenerator.c b/source/libs/parser/src/astGenerator.c new file mode 100644 index 0000000000..74642ea811 --- /dev/null +++ b/source/libs/parser/src/astGenerator.c @@ -0,0 +1,507 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "os.h" +#include "taos.h" +#include "astGenerator.h" + +int32_t tStrToInteger(const char* z, int16_t type, int32_t n, int64_t* value, bool issigned) { + errno = 0; + int32_t ret = 0; + + char* endPtr = NULL; + if (type == TK_FLOAT) { + double v = strtod(z, &endPtr); + if ((errno == ERANGE && v == HUGE_VALF) || isinf(v) || isnan(v)) { + ret = -1; + } else if ((issigned && (v < INT64_MIN || v > INT64_MAX)) || ((!issigned) && (v < 0 || v > UINT64_MAX))) { + ret = -1; + } else { + *value = (int64_t) round(v); + } + + errno = 0; + return ret; + } + + int32_t radix = 10; + if (type == TK_HEX) { + radix = 16; + } else if (type == TK_BIN) { + radix = 2; + } + + // the string may be overflow according to errno + if (!issigned) { + const char *p = z; + while(*p != 0 && *p == ' ') p++; + if (*p != 0 && *p == '-') { return -1;} + + *value = strtoull(z, &endPtr, radix); + } else { + *value = strtoll(z, &endPtr, radix); + } + + // not a valid integer number, return error + if (endPtr - z != n || errno == ERANGE) { + ret = -1; + } + + errno = 0; + return ret; +} + +SArray *tListItemAppend(SArray *pList, SVariant *pVar, uint8_t sortOrder) { + if (pList == NULL) { + pList = taosArrayInit(4, sizeof(SListItem)); + } + + if (pVar == NULL) { + return pList; + } + + /* + * Here we do not employ the assign function, since we need the pz attribute of structure , which is the point to char string. + * Otherwise, the original pointer may be lost, which causes memory leak. + */ + SListItem item; + item.pVar = *pVar; + item.sortOrder = sortOrder; + + taosArrayPush(pList, &item); + return pList; +} + +SArray *tListItemInsert(SArray *pList, SVariant *pVar, uint8_t sortOrder, int32_t index) { + if (pList == NULL || pVar == NULL || index >= taosArrayGetSize(pList)) { + return tListItemAppend(pList, pVar, sortOrder); + } + + SListItem item; + item.pVar = *pVar; + item.sortOrder = sortOrder; + + taosArrayInsert(pList, index, &item); + return pList; +} + +SArray *tListItemAppendToken(SArray *pList, SToken *pAliasToken, uint8_t sortOrder) { + if (pList == NULL) { + pList = taosArrayInit(4, sizeof(SListItem)); + } + + if (pAliasToken) { + SListItem item; + taosVariantCreate(&item.pVar, pAliasToken); + item.sortOrder = sortOrder; + + taosArrayPush(pList, &item); + } + + return pList; +} + +SRelationInfo *setTableNameList(SRelationInfo *pRelationInfo, SToken *pName, SToken *pAlias) { + if (pRelationInfo == NULL) { + pRelationInfo = calloc(1, sizeof(SRelationInfo)); + pRelationInfo->list = taosArrayInit(4, sizeof(SRelElementPair)); + } + + pRelationInfo->type = SQL_NODE_FROM_TABLELIST; + SRelElementPair p = {.tableName = *pName}; + if (pAlias != NULL) { + p.aliasName = *pAlias; + } else { + TPARSER_SET_NONE_TOKEN(p.aliasName); + } + + taosArrayPush(pRelationInfo->list, &p); + return pRelationInfo; +} + +void *destroyRelationInfo(SRelationInfo *pRelationInfo) { + if (pRelationInfo == NULL) { + return NULL; + } + + if (pRelationInfo->type == SQL_NODE_FROM_TABLELIST) { + taosArrayDestroy(pRelationInfo->list); + } else { + size_t size = taosArrayGetSize(pRelationInfo->list); + for(int32_t i = 0; i < size; ++i) { + SArray* pa = taosArrayGetP(pRelationInfo->list, i); + destroyAllSqlNode(pa); + } + taosArrayDestroy(pRelationInfo->list); + } + + tfree(pRelationInfo); + return NULL; +} + +SRelationInfo *addSubquery(SRelationInfo *pRelationInfo, SArray *pSub, SToken *pAlias) { + if (pRelationInfo == NULL) { + pRelationInfo = calloc(1, sizeof(SRelationInfo)); + pRelationInfo->list = taosArrayInit(4, sizeof(SRelElementPair)); + } + + pRelationInfo->type = SQL_NODE_FROM_SUBQUERY; + + SRelElementPair p = {.pSubquery = pSub}; + if (pAlias != NULL) { + p.aliasName = *pAlias; + } else { + TPARSER_SET_NONE_TOKEN(p.aliasName); + } + + taosArrayPush(pRelationInfo->list, &p); + return pRelationInfo; +} + +// sql expr leaf node +// todo Evalute the value during the validation process of AST. +tSqlExpr *tSqlExprCreateIdValue(SToken *pToken, int32_t optrType) { + tSqlExpr *pSqlExpr = calloc(1, sizeof(tSqlExpr)); + + if (pToken != NULL) { + pSqlExpr->exprToken = *pToken; + } + + if (optrType == TK_NULL) { +// if (pToken) { +// pToken->type = TSDB_DATA_TYPE_NULL; +// tVariantCreate(&pSqlExpr->value, pToken); +// } + pSqlExpr->tokenId = optrType; + pSqlExpr->type = SQL_NODE_VALUE; + } else if (optrType == TK_INTEGER || optrType == TK_STRING || optrType == TK_FLOAT || optrType == TK_BOOL) { +// if (pToken) { +// toTSDBType(pToken->type); +// tVariantCreate(&pSqlExpr->value, pToken); +// } + pSqlExpr->tokenId = optrType; + pSqlExpr->type = SQL_NODE_VALUE; + } else if (optrType == TK_NOW) { + // use nanosecond by default TODO set value after getting database precision +// pSqlExpr->value.i64 = taosGetTimestamp(TSDB_TIME_PRECISION_NANO); +// pSqlExpr->value.nType = TSDB_DATA_TYPE_BIGINT; + pSqlExpr->tokenId = TK_TIMESTAMP; // TK_TIMESTAMP used to denote the time value is in microsecond + pSqlExpr->type = SQL_NODE_VALUE; +// pSqlExpr->flags |= 1 << EXPR_FLAG_NS_TIMESTAMP; + } else if (optrType == TK_VARIABLE) { + // use nanosecond by default + // TODO set value after getting database precision +// if (pToken) { +// char unit = 0; +// int32_t ret = parseAbsoluteDuration(pToken->z, pToken->n, &pSqlExpr->value.i64, &unit, TSDB_TIME_PRECISION_NANO); +// if (ret != TSDB_CODE_SUCCESS) { +// terrno = TSDB_CODE_TSC_SQL_SYNTAX_ERROR; +// } +// } + +// pSqlExpr->flags |= 1 << EXPR_FLAG_NS_TIMESTAMP; +// pSqlExpr->flags |= 1 << EXPR_FLAG_TIMESTAMP_VAR; +// pSqlExpr->value.nType = TSDB_DATA_TYPE_BIGINT; + pSqlExpr->tokenId = TK_TIMESTAMP; + pSqlExpr->type = SQL_NODE_VALUE; + } else { + // Here it must be the column name (tk_id) if it is not a number or string. + assert(optrType == TK_ID || optrType == TK_ALL); + if (pToken != NULL) { + pSqlExpr->columnName = *pToken; + } + + pSqlExpr->tokenId = optrType; + pSqlExpr->type = SQL_NODE_TABLE_COLUMN; + } + + return pSqlExpr; +} + +tSqlExpr *tSqlExprCreateFunction(SArray *pParam, SToken *pFuncToken, SToken *endToken, int32_t optType) { + if (pFuncToken == NULL) { + return NULL; + } + + tSqlExpr *pExpr = calloc(1, sizeof(tSqlExpr)); + pExpr->tokenId = optType; + pExpr->type = SQL_NODE_SQLFUNCTION; + pExpr->Expr.paramList = pParam; + + int32_t len = (int32_t)((endToken->z + endToken->n) - pFuncToken->z); + pExpr->Expr.operand = (*pFuncToken); + + pExpr->exprToken.n = len; + pExpr->exprToken.z = pFuncToken->z; + pExpr->exprToken.type = pFuncToken->type; + + return pExpr; +} + +SArray *tAppendFuncName(SArray *pList, SToken *pToken) { + assert(pList != NULL && pToken != NULL); + taosArrayPush(pList, pToken); + return pList; +} + +tSqlExpr *tSqlExprCreate(tSqlExpr *pLeft, tSqlExpr *pRight, int32_t optrType) { + tSqlExpr *pExpr = calloc(1, sizeof(tSqlExpr)); + pExpr->type = SQL_NODE_EXPR; + + if (pLeft != NULL && pRight != NULL && (optrType != TK_IN)) { + char* endPos = pRight->exprToken.z + pRight->exprToken.n; + pExpr->exprToken.z = pLeft->exprToken.z; + pExpr->exprToken.n = (uint32_t)(endPos - pExpr->exprToken.z); + pExpr->exprToken.type = pLeft->exprToken.type; + } + + if ((pLeft != NULL && pRight != NULL) && + (optrType == TK_PLUS || optrType == TK_MINUS || optrType == TK_STAR || optrType == TK_DIVIDE || optrType == TK_REM)) { + /* + * if a exprToken is noted as the TK_TIMESTAMP, the time precision is microsecond + * Otherwise, the time precision is adaptive, determined by the time precision from databases. + */ + if ((pLeft->tokenId == TK_INTEGER && pRight->tokenId == TK_INTEGER) || + (pLeft->tokenId == TK_TIMESTAMP && pRight->tokenId == TK_TIMESTAMP)) { + pExpr->value.nType = TSDB_DATA_TYPE_BIGINT; + pExpr->tokenId = pLeft->tokenId; + pExpr->type = SQL_NODE_VALUE; + + switch (optrType) { + case TK_PLUS: { + pExpr->value.i64 = pLeft->value.i64 + pRight->value.i64; + break; + } + + case TK_MINUS: { + pExpr->value.i64 = pLeft->value.i64 - pRight->value.i64; + break; + } + case TK_STAR: { + pExpr->value.i64 = pLeft->value.i64 * pRight->value.i64; + break; + } + case TK_DIVIDE: { + pExpr->tokenId = TK_FLOAT; + pExpr->value.nType = TSDB_DATA_TYPE_DOUBLE; + pExpr->value.d = (double)pLeft->value.i64 / pRight->value.i64; + break; + } + case TK_REM: { + pExpr->value.i64 = pLeft->value.i64 % pRight->value.i64; + break; + } + } + + tSqlExprDestroy(pLeft); + tSqlExprDestroy(pRight); + } else if ((pLeft->tokenId == TK_FLOAT && pRight->tokenId == TK_INTEGER) || + (pLeft->tokenId == TK_INTEGER && pRight->tokenId == TK_FLOAT) || + (pLeft->tokenId == TK_FLOAT && pRight->tokenId == TK_FLOAT)) { + pExpr->value.nType = TSDB_DATA_TYPE_DOUBLE; + pExpr->tokenId = TK_FLOAT; + pExpr->type = SQL_NODE_VALUE; + + double left = (pLeft->value.nType == TSDB_DATA_TYPE_DOUBLE) ? pLeft->value.d : pLeft->value.i64; + double right = (pRight->value.nType == TSDB_DATA_TYPE_DOUBLE) ? pRight->value.d : pRight->value.i64; + + switch (optrType) { + case TK_PLUS: { + pExpr->value.d = left + right; + break; + } + case TK_MINUS: { + pExpr->value.d = left - right; + break; + } + case TK_STAR: { + pExpr->value.d = left * right; + break; + } + case TK_DIVIDE: { + pExpr->value.d = left / right; + break; + } + case TK_REM: { + pExpr->value.d = left - ((int64_t)(left / right)) * right; + break; + } + } + + tSqlExprDestroy(pLeft); + tSqlExprDestroy(pRight); + + } else { + pExpr->tokenId = optrType; + pExpr->pLeft = pLeft; + pExpr->pRight = pRight; + } + } else if (optrType == TK_IN) { + pExpr->tokenId = optrType; + pExpr->pLeft = pLeft; + + tSqlExpr *pRSub = calloc(1, sizeof(tSqlExpr)); + pRSub->tokenId = TK_SET; // TODO refactor ..... + pRSub->Expr.paramList = (SArray *)pRight; + + pExpr->pRight = pRSub; + } else { + pExpr->tokenId = optrType; + pExpr->pLeft = pLeft; + + if (pLeft != NULL && pRight == NULL) { + pRight = calloc(1, sizeof(tSqlExpr)); + } + + pExpr->pRight = pRight; + } + + return pExpr; +} + +tSqlExpr *tSqlExprClone(tSqlExpr *pSrc); +void tSqlExprCompact(tSqlExpr **pExpr); +bool tSqlExprIsLeaf(tSqlExpr *pExpr); +bool tSqlExprIsParentOfLeaf(tSqlExpr *pExpr); +void tSqlExprDestroy(tSqlExpr *pExpr); +SArray * tSqlExprListAppend(SArray *pList, tSqlExpr *pNode, SToken *pDistinct, SToken *pToken); +void tSqlExprListDestroy(SArray *pList); + +SSqlNode *tSetQuerySqlNode(SToken *pSelectToken, SArray *pSelNodeList, SRelationInfo *pFrom, tSqlExpr *pWhere, + SArray *pGroupby, SArray *pSortOrder, SIntervalVal *pInterval, SSessionWindowVal *ps, + SWindowStateVal *pw, SToken *pSliding, SArray *pFill, SLimit *pLimit, SLimit *pgLimit, tSqlExpr *pHaving); +int32_t tSqlExprCompare(tSqlExpr *left, tSqlExpr *right); + +SCreateTableSql *tSetCreateTableInfo(SArray *pCols, SArray *pTags, SSqlNode *pSelect, int32_t type); + +SAlterTableInfo * tSetAlterTableInfo(SToken *pTableName, SArray *pCols, SArray *pVals, int32_t type, + int16_t tableTable); +SCreatedTableInfo createNewChildTableInfo(SToken *pTableName, SArray *pTagNames, SArray *pTagVals, SToken *pToken, + SToken *igExists); + +void destroyAllSqlNode(SArray *pSqlNode); +void destroySqlNode(SSqlNode *pSql); +void freeCreateTableInfo(void* p); + +SSqlInfo *setSqlInfo(SSqlInfo *pInfo, void *pSqlExprInfo, SToken *pTableName, int32_t type); +SArray *setSubclause(SArray *pList, void *pSqlNode); +SArray *appendSelectClause(SArray *pList, void *pSubclause); + +void setCreatedTableName(SSqlInfo *pInfo, SToken *pTableNameToken, SToken *pIfNotExists); + +void SqlInfoDestroy(SSqlInfo *pInfo); + +void setDCLSqlElems(SSqlInfo *pInfo, int32_t type, int32_t nParams, ...); +void setDropDbTableInfo(SSqlInfo *pInfo, int32_t type, SToken* pToken, SToken* existsCheck,int16_t dbType,int16_t tableType); +void setShowOptions(SSqlInfo *pInfo, int32_t type, SToken* prefix, SToken* pPatterns); + +void setCreateDbInfo(SSqlInfo *pInfo, int32_t type, SToken *pToken, SCreateDbInfo *pDB, SToken *pIgExists); + +void setCreateAcctSql(SSqlInfo *pInfo, int32_t type, SToken *pName, SToken *pPwd, SCreateAcctInfo *pAcctInfo); +void setCreateUserSql(SSqlInfo *pInfo, SToken *pName, SToken *pPasswd); +void setKillSql(SSqlInfo *pInfo, int32_t type, SToken *ip); +void setAlterUserSql(SSqlInfo *pInfo, int16_t type, SToken *pName, SToken* pPwd, SToken *pPrivilege); + +void setCompactVnodeSql(SSqlInfo *pInfo, int32_t type, SArray *pParam); + +void setDefaultCreateDbOption(SCreateDbInfo *pDBInfo); +void setDefaultCreateTopicOption(SCreateDbInfo *pDBInfo); + +// prefix show db.tables; +void tSetDbName(SToken *pCpxName, SToken *pDb); + +void tSetColumnInfo(struct SField *pField, SToken *pName, struct SField *pType); +void tSetColumnType(struct SField *pField, SToken *type); + +/** + * + * @param yyp The parser + * @param yymajor The major token code number + * @param yyminor The value for the token + */ +void Parse(void *yyp, int yymajor, ParseTOKENTYPE yyminor, SSqlInfo *); + +/** + * + * @param p The parser to be deleted + * @param freeProc Function used to reclaim memory + */ +void ParseFree(void *p, void (*freeProc)(void *)); + +/** + * + * @param mallocProc The parser allocator + * @return + */ +void *ParseAlloc(void *(*mallocProc)(size_t)); + +SSqlInfo genAST(const char *pStr) { + void *pParser = ParseAlloc(malloc); + + SSqlInfo sqlInfo = {0}; + + sqlInfo.valid = true; + sqlInfo.funcs = taosArrayInit(4, sizeof(SToken)); + + int32_t i = 0; + while (1) { + SToken t0 = {0}; + + if (pStr[i] == 0) { + Parse(pParser, 0, t0, &sqlInfo); + goto abort_parse; + } + + t0.n = tGetToken((char *)&pStr[i], &t0.type); + t0.z = (char *)(pStr + i); + i += t0.n; + + switch (t0.type) { + case TK_SPACE: + case TK_COMMENT: { + break; + } + case TK_SEMI: { + Parse(pParser, 0, t0, &sqlInfo); + goto abort_parse; + } + + case TK_QUESTION: + case TK_ILLEGAL: { + snprintf(sqlInfo.msg, tListLen(sqlInfo.msg), "unrecognized token: \"%s\"", t0.z); + sqlInfo.valid = false; + goto abort_parse; + } + + case TK_HEX: + case TK_OCT: + case TK_BIN: { + snprintf(sqlInfo.msg, tListLen(sqlInfo.msg), "unsupported token: \"%s\"", t0.z); + sqlInfo.valid = false; + goto abort_parse; + } + + default: + Parse(pParser, t0.type, t0, &sqlInfo); + if (sqlInfo.valid == false) { + goto abort_parse; + } + } + } + +abort_parse: + ParseFree(pParser, free); + return sqlInfo; +} diff --git a/source/libs/parser/src/astValidate.c b/source/libs/parser/src/astValidate.c new file mode 100644 index 0000000000..7718dc0758 --- /dev/null +++ b/source/libs/parser/src/astValidate.c @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + + +#include "parserInt.h" + +int32_t qParserValidateSqlNode(struct SCatalog* pCatalog, SSqlInfo* pInfo, SQueryStmtInfo* pQueryInfo, int64_t id, char* msg) { + //1. if it is a query, get the meta info and continue. + + + +// qParserExtractRequestedMetaInfo(pInfo->) + return 0; +} From e477c355f39abdac4899f92b07b57e5eaa010cfc Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 8 Oct 2021 10:41:20 +0800 Subject: [PATCH 8/9] [TD-10529]add new files. --- include/common/common.h | 2 + include/common/tcompression.h | 370 ++++++++++++ source/common/src/ttypes.c | 684 +++++++++++++++++++++++ source/libs/parser/inc/tvariant.h | 2 +- source/libs/planner/CMakeLists.txt | 2 +- source/libs/scheduler/inc/schedulerInt.h | 8 - src/client/src/tscServer.c | 2 +- src/common/src/ttypes.c | 4 +- src/dnode/src/dnodeMain.c | 42 +- src/mnode/src/mnodeTable.c | 39 +- src/query/src/qExecutor.c | 10 +- src/query/src/qResultbuf.c | 4 +- src/query/src/qTsbuf.c | 6 +- src/query/src/qUtil.c | 6 +- src/tsdb/inc/tsdbint.h | 18 +- src/util/src/tcompression.c | 5 +- 16 files changed, 1125 insertions(+), 79 deletions(-) create mode 100644 include/common/tcompression.h create mode 100644 source/common/src/ttypes.c diff --git a/include/common/common.h b/include/common/common.h index c03b978c98..fb66902019 100644 --- a/include/common/common.h +++ b/include/common/common.h @@ -16,6 +16,8 @@ #ifndef TDENGINE_COMMON_H #define TDENGINE_COMMON_H +#include "taosdef.h" + typedef struct STimeWindow { TSKEY skey; TSKEY ekey; diff --git a/include/common/tcompression.h b/include/common/tcompression.h new file mode 100644 index 0000000000..a03da91b44 --- /dev/null +++ b/include/common/tcompression.h @@ -0,0 +1,370 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_TCOMPRESSION_H +#define TDENGINE_TCOMPRESSION_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include "taosdef.h" + +#define COMP_OVERFLOW_BYTES 2 +#define BITS_PER_BYTE 8 + +// Masks +#define INT64MASK(_x) ((((uint64_t)1) << _x) - 1) +#define INT32MASK(_x) (((uint32_t)1 << _x) - 1) +#define INT8MASK(_x) (((uint8_t)1 << _x) - 1) + +// Compression algorithm +#define NO_COMPRESSION 0 +#define ONE_STAGE_COMP 1 +#define TWO_STAGE_COMP 2 + +// +// compressed data first byte foramt +// ------ 7 bit ---- | ---- 1 bit ---- +// algorithm mode +// + +// compression data mode save first byte lower 1 bit +#define MODE_NOCOMPRESS 0 // original data +#define MODE_COMPRESS 1 // compatible old compress + +// compression algorithm save first byte higher 7 bit +#define ALGO_SZ_LOSSY 1 // SZ compress + +#define HEAD_MODE(x) x%2 +#define HEAD_ALGO(x) x/2 + +extern int tsCompressINTImp(const char *const input, const int nelements, char *const output, const char type); +extern int tsDecompressINTImp(const char *const input, const int nelements, char *const output, const char type); +extern int tsCompressBoolImp(const char *const input, const int nelements, char *const output); +extern int tsDecompressBoolImp(const char *const input, const int nelements, char *const output); +extern int tsCompressStringImp(const char *const input, int inputSize, char *const output, int outputSize); +extern int tsDecompressStringImp(const char *const input, int compressedSize, char *const output, int outputSize); +extern int tsCompressTimestampImp(const char *const input, const int nelements, char *const output); +extern int tsDecompressTimestampImp(const char *const input, const int nelements, char *const output); +extern int tsCompressDoubleImp(const char *const input, const int nelements, char *const output); +extern int tsDecompressDoubleImp(const char *const input, const int nelements, char *const output); +extern int tsCompressFloatImp(const char *const input, const int nelements, char *const output); +extern int tsDecompressFloatImp(const char *const input, const int nelements, char *const output); +// lossy +extern int tsCompressFloatLossyImp(const char * input, const int nelements, char *const output); +extern int tsDecompressFloatLossyImp(const char * input, int compressedSize, const int nelements, char *const output); +extern int tsCompressDoubleLossyImp(const char * input, const int nelements, char *const output); +extern int tsDecompressDoubleLossyImp(const char * input, int compressedSize, const int nelements, char *const output); + +#ifdef TD_TSZ +extern bool lossyFloat; +extern bool lossyDouble; +// init call +int tsCompressInit(); +// exit call +void tsCompressExit(); +#endif + +static FORCE_INLINE int tsCompressTinyint(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, char algorithm, + char *const buffer, int bufferSize) { + if (algorithm == ONE_STAGE_COMP) { + return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_TINYINT); + } else if (algorithm == TWO_STAGE_COMP) { + int len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_TINYINT); + return tsCompressStringImp(buffer, len, output, outputSize); + } else { + assert(0); + return -1; + } +} + +static FORCE_INLINE int tsDecompressTinyint(const char *const input, int compressedSize, const int nelements, char *const output, + int outputSize, char algorithm, char *const buffer, int bufferSize) { + if (algorithm == ONE_STAGE_COMP) { + return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_TINYINT); + } else if (algorithm == TWO_STAGE_COMP) { + if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; + return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_TINYINT); + } else { + assert(0); + return -1; + } +} + +static FORCE_INLINE int tsCompressSmallint(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, char algorithm, + char *const buffer, int bufferSize) { + if (algorithm == ONE_STAGE_COMP) { + return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_SMALLINT); + } else if (algorithm == TWO_STAGE_COMP) { + int len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_SMALLINT); + return tsCompressStringImp(buffer, len, output, outputSize); + } else { + assert(0); + return -1; + } +} + +static FORCE_INLINE int tsDecompressSmallint(const char *const input, int compressedSize, const int nelements, char *const output, + int outputSize, char algorithm, char *const buffer, int bufferSize) { + if (algorithm == ONE_STAGE_COMP) { + return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_SMALLINT); + } else if (algorithm == TWO_STAGE_COMP) { + if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; + return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_SMALLINT); + } else { + assert(0); + return -1; + } +} + +static FORCE_INLINE int tsCompressInt(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, char algorithm, + char *const buffer, int bufferSize) { + if (algorithm == ONE_STAGE_COMP) { + return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_INT); + } else if (algorithm == TWO_STAGE_COMP) { + int len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_INT); + return tsCompressStringImp(buffer, len, output, outputSize); + } else { + assert(0); + return -1; + } +} + +static FORCE_INLINE int tsDecompressInt(const char *const input, int compressedSize, const int nelements, char *const output, + int outputSize, char algorithm, char *const buffer, int bufferSize) { + if (algorithm == ONE_STAGE_COMP) { + return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_INT); + } else if (algorithm == TWO_STAGE_COMP) { + if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; + return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_INT); + } else { + assert(0); + return -1; + } +} + +static FORCE_INLINE int tsCompressBigint(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, + char algorithm, char *const buffer, int bufferSize) { + if (algorithm == ONE_STAGE_COMP) { + return tsCompressINTImp(input, nelements, output, TSDB_DATA_TYPE_BIGINT); + } else if (algorithm == TWO_STAGE_COMP) { + int len = tsCompressINTImp(input, nelements, buffer, TSDB_DATA_TYPE_BIGINT); + return tsCompressStringImp(buffer, len, output, outputSize); + } else { + assert(0); + return -1; + } +} + +static FORCE_INLINE int tsDecompressBigint(const char *const input, int compressedSize, const int nelements, char *const output, + int outputSize, char algorithm, char *const buffer, int bufferSize) { + if (algorithm == ONE_STAGE_COMP) { + return tsDecompressINTImp(input, nelements, output, TSDB_DATA_TYPE_BIGINT); + } else if (algorithm == TWO_STAGE_COMP) { + if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; + return tsDecompressINTImp(buffer, nelements, output, TSDB_DATA_TYPE_BIGINT); + } else { + assert(0); + return -1; + } +} + +static FORCE_INLINE int tsCompressBool(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, + char algorithm, char *const buffer, int bufferSize) { + if (algorithm == ONE_STAGE_COMP) { + return tsCompressBoolImp(input, nelements, output); + } else if (algorithm == TWO_STAGE_COMP) { + int len = tsCompressBoolImp(input, nelements, buffer); + return tsCompressStringImp(buffer, len, output, outputSize); + } else { + assert(0); + return -1; + } +} + +static FORCE_INLINE int tsDecompressBool(const char *const input, int compressedSize, const int nelements, char *const output, + int outputSize, char algorithm, char *const buffer, int bufferSize) { + if (algorithm == ONE_STAGE_COMP) { + return tsDecompressBoolImp(input, nelements, output); + } else if (algorithm == TWO_STAGE_COMP) { + if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; + return tsDecompressBoolImp(buffer, nelements, output); + } else { + assert(0); + return -1; + } +} + +static FORCE_INLINE int tsCompressString(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, + char algorithm, char *const buffer, int bufferSize) { + return tsCompressStringImp(input, inputSize, output, outputSize); +} + +static FORCE_INLINE int tsDecompressString(const char *const input, int compressedSize, const int nelements, char *const output, + int outputSize, char algorithm, char *const buffer, int bufferSize) { + return tsDecompressStringImp(input, compressedSize, output, outputSize); +} + +static FORCE_INLINE int tsCompressFloat(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, + char algorithm, char *const buffer, int bufferSize) { +#ifdef TD_TSZ + // lossy mode + if(lossyFloat) { + return tsCompressFloatLossyImp(input, nelements, output); + // lossless mode + } else { +#endif + if (algorithm == ONE_STAGE_COMP) { + return tsCompressFloatImp(input, nelements, output); + } else if (algorithm == TWO_STAGE_COMP) { + int len = tsCompressFloatImp(input, nelements, buffer); + return tsCompressStringImp(buffer, len, output, outputSize); + } else { + assert(0); + return -1; + } +#ifdef TD_TSZ + } +#endif +} + +static FORCE_INLINE int tsDecompressFloat(const char *const input, int compressedSize, const int nelements, char *const output, + int outputSize, char algorithm, char *const buffer, int bufferSize) { +#ifdef TD_TSZ + if(HEAD_ALGO(input[0]) == ALGO_SZ_LOSSY){ + // decompress lossy + return tsDecompressFloatLossyImp(input, compressedSize, nelements, output); + } else { +#endif + // decompress lossless + if (algorithm == ONE_STAGE_COMP) { + return tsDecompressFloatImp(input, nelements, output); + } else if (algorithm == TWO_STAGE_COMP) { + if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; + return tsDecompressFloatImp(buffer, nelements, output); + } else { + assert(0); + return -1; + } +#ifdef TD_TSZ + } +#endif +} + + +static FORCE_INLINE int tsCompressDouble(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, + char algorithm, char *const buffer, int bufferSize) { +#ifdef TD_TSZ + if(lossyDouble){ + // lossy mode + return tsCompressDoubleLossyImp(input, nelements, output); + } else { +#endif + // lossless mode + if (algorithm == ONE_STAGE_COMP) { + return tsCompressDoubleImp(input, nelements, output); + } else if (algorithm == TWO_STAGE_COMP) { + int len = tsCompressDoubleImp(input, nelements, buffer); + return tsCompressStringImp(buffer, len, output, outputSize); + } else { + assert(0); + return -1; + } +#ifdef TD_TSZ + } +#endif +} + +static FORCE_INLINE int tsDecompressDouble(const char *const input, int compressedSize, const int nelements, char *const output, + int outputSize, char algorithm, char *const buffer, int bufferSize) { + #ifdef TD_TSZ + if(HEAD_ALGO(input[0]) == ALGO_SZ_LOSSY){ + // decompress lossy + return tsDecompressDoubleLossyImp(input, compressedSize, nelements, output); + } else { + #endif + // decompress lossless + if (algorithm == ONE_STAGE_COMP) { + return tsDecompressDoubleImp(input, nelements, output); + } else if (algorithm == TWO_STAGE_COMP) { + if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; + return tsDecompressDoubleImp(buffer, nelements, output); + } else { + assert(0); + return -1; + } +#ifdef TD_TSZ + } +#endif +} + +#ifdef TD_TSZ +// +// lossy float double +// +static FORCE_INLINE int tsCompressFloatLossy(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, + char algorithm, char *const buffer, int bufferSize) { + return tsCompressFloatLossyImp(input, nelements, output); +} + +static FORCE_INLINE int tsDecompressFloatLossy(const char *const input, int compressedSize, const int nelements, char *const output, + int outputSize, char algorithm, char *const buffer, int bufferSize){ + return tsDecompressFloatLossyImp(input, compressedSize, nelements, output); +} + +static FORCE_INLINE int tsCompressDoubleLossy(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, + char algorithm, char *const buffer, int bufferSize){ + return tsCompressDoubleLossyImp(input, nelements, output); +} + +static FORCE_INLINE int tsDecompressDoubleLossy(const char *const input, int compressedSize, const int nelements, char *const output, + int outputSize, char algorithm, char *const buffer, int bufferSize){ + return tsDecompressDoubleLossyImp(input, compressedSize, nelements, output); +} + +#endif + +static FORCE_INLINE int tsCompressTimestamp(const char *const input, int inputSize, const int nelements, char *const output, int outputSize, + char algorithm, char *const buffer, int bufferSize) { + if (algorithm == ONE_STAGE_COMP) { + return tsCompressTimestampImp(input, nelements, output); + } else if (algorithm == TWO_STAGE_COMP) { + int len = tsCompressTimestampImp(input, nelements, buffer); + return tsCompressStringImp(buffer, len, output, outputSize); + } else { + assert(0); + return -1; + } +} + +static FORCE_INLINE int tsDecompressTimestamp(const char *const input, int compressedSize, const int nelements, char *const output, + int outputSize, char algorithm, char *const buffer, int bufferSize) { + if (algorithm == ONE_STAGE_COMP) { + return tsDecompressTimestampImp(input, nelements, output); + } else if (algorithm == TWO_STAGE_COMP) { + if (tsDecompressStringImp(input, compressedSize, buffer, bufferSize) < 0) return -1; + return tsDecompressTimestampImp(buffer, nelements, output); + } else { + assert(0); + return -1; + } +} + +#ifdef __cplusplus +} +#endif + +#endif // TDENGINE_TCOMPRESSION_H \ No newline at end of file diff --git a/source/common/src/ttypes.c b/source/common/src/ttypes.c new file mode 100644 index 0000000000..bec8793b72 --- /dev/null +++ b/source/common/src/ttypes.c @@ -0,0 +1,684 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "taos.h" +#include "os.h" +#include "ttypes.h" +#include "tcompression.h" + +const int32_t TYPE_BYTES[15] = { + -1, // TSDB_DATA_TYPE_NULL + CHAR_BYTES, // TSDB_DATA_TYPE_BOOL + CHAR_BYTES, // TSDB_DATA_TYPE_TINYINT + SHORT_BYTES, // TSDB_DATA_TYPE_SMALLINT + INT_BYTES, // TSDB_DATA_TYPE_INT + sizeof(int64_t), // TSDB_DATA_TYPE_BIGINT + FLOAT_BYTES, // TSDB_DATA_TYPE_FLOAT + DOUBLE_BYTES, // TSDB_DATA_TYPE_DOUBLE + sizeof(VarDataOffsetT), // TSDB_DATA_TYPE_BINARY + sizeof(TSKEY), // TSDB_DATA_TYPE_TIMESTAMP + sizeof(VarDataOffsetT), // TSDB_DATA_TYPE_NCHAR + CHAR_BYTES, // TSDB_DATA_TYPE_UTINYINT + SHORT_BYTES, // TSDB_DATA_TYPE_USMALLINT + INT_BYTES, // TSDB_DATA_TYPE_UINT + sizeof(uint64_t), // TSDB_DATA_TYPE_UBIGINT +}; + +#define DO_STATICS(__sum, __min, __max, __minIndex, __maxIndex, _list, _index) \ + do { \ + (__sum) += (_list)[(_index)]; \ + if ((__min) > (_list)[(_index)]) { \ + (__min) = (_list)[(_index)]; \ + (__minIndex) = (_index); \ + } \ + \ + if ((__max) < (_list)[(_index)]) { \ + (__max) = (_list)[(_index)]; \ + (__maxIndex) = (_index); \ + } \ + } while (0) + +static void getStatics_bool(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, + int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { + int8_t *data = (int8_t *)pData; + *min = INT64_MAX; + *max = INT64_MIN; + *minIndex = 0; + *maxIndex = 0; + + assert(numOfRow <= INT16_MAX); + + for (int32_t i = 0; i < numOfRow; ++i) { + if (data[i] == TSDB_DATA_BOOL_NULL) { + (*numOfNull) += 1; + continue; + } + + DO_STATICS(*sum, *min, *max, *minIndex, *maxIndex, data, i); + } +} + +static void getStatics_i8(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, + int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { + int8_t *data = (int8_t *)pData; + *min = INT64_MAX; + *max = INT64_MIN; + *minIndex = 0; + *maxIndex = 0; + + assert(numOfRow <= INT16_MAX); + + for (int32_t i = 0; i < numOfRow; ++i) { + if (((uint8_t)data[i]) == TSDB_DATA_TINYINT_NULL) { + (*numOfNull) += 1; + continue; + } + + DO_STATICS(*sum, *min, *max, *minIndex, *maxIndex, data, i); + } +} + +static void getStatics_u8(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, + int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { + uint8_t *data = (uint8_t *)pData; + uint64_t _min = UINT64_MAX; + uint64_t _max = 0; + uint64_t _sum = 0; + + *minIndex = 0; + *maxIndex = 0; + + assert(numOfRow <= INT16_MAX); + + for (int32_t i = 0; i < numOfRow; ++i) { + if (((uint8_t)data[i]) == TSDB_DATA_UTINYINT_NULL) { + (*numOfNull) += 1; + continue; + } + + DO_STATICS(_sum, _min, _max, *minIndex, *maxIndex, data, i); + } + + *min = _min; + *max = _max; + *sum = _sum; +} + +static void getStatics_i16(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, + int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { + int16_t *data = (int16_t *)pData; + *min = INT64_MAX; + *max = INT64_MIN; + *minIndex = 0; + *maxIndex = 0; + + assert(numOfRow <= INT16_MAX); + + for (int32_t i = 0; i < numOfRow; ++i) { + if (((uint16_t)data[i]) == TSDB_DATA_SMALLINT_NULL) { + (*numOfNull) += 1; + continue; + } + + DO_STATICS(*sum, *min, *max, *minIndex, *maxIndex, data, i); + } + +} + +static void getStatics_u16(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, + int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { + uint16_t *data = (uint16_t *)pData; + uint64_t _min = UINT64_MAX; + uint64_t _max = 0; + uint64_t _sum = 0; + + *minIndex = 0; + *maxIndex = 0; + + assert(numOfRow <= INT16_MAX); + + for (int32_t i = 0; i < numOfRow; ++i) { + if (((uint16_t)data[i]) == TSDB_DATA_USMALLINT_NULL) { + (*numOfNull) += 1; + continue; + } + + DO_STATICS(_sum, _min, _max, *minIndex, *maxIndex, data, i); + } + + *min = _min; + *max = _max; + *sum = _sum; +} + +static void getStatics_i32(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, + int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { + int32_t *data = (int32_t *)pData; + *min = INT64_MAX; + *max = INT64_MIN; + *minIndex = 0; + *maxIndex = 0; + + assert(numOfRow <= INT16_MAX); + + for (int32_t i = 0; i < numOfRow; ++i) { + if (((uint32_t)data[i]) == TSDB_DATA_INT_NULL) { + (*numOfNull) += 1; + continue; + } + + DO_STATICS(*sum, *min, *max, *minIndex, *maxIndex, data, i); + } +} + +static void getStatics_u32(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, + int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { + uint32_t *data = (uint32_t *)pData; + uint64_t _min = UINT64_MAX; + uint64_t _max = 0; + uint64_t _sum = 0; + + *minIndex = 0; + *maxIndex = 0; + + assert(numOfRow <= INT16_MAX); + + for (int32_t i = 0; i < numOfRow; ++i) { + if (((uint32_t)data[i]) == TSDB_DATA_UINT_NULL) { + (*numOfNull) += 1; + continue; + } + + DO_STATICS(_sum, _min, _max, *minIndex, *maxIndex, data, i); + } + + *min = _min; + *max = _max; + *sum = _sum; +} + +static void getStatics_i64(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, + int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { + int64_t *data = (int64_t *)pData; + *min = INT64_MAX; + *max = INT64_MIN; + *minIndex = 0; + *maxIndex = 0; + + assert(numOfRow <= INT16_MAX); + + for (int32_t i = 0; i < numOfRow; ++i) { + if (((uint64_t)data[i]) == TSDB_DATA_BIGINT_NULL) { + (*numOfNull) += 1; + continue; + } + + DO_STATICS(*sum, *min, *max, *minIndex, *maxIndex, data, i); + } +} + +static void getStatics_u64(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, + int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { + uint64_t *data = (uint64_t *)pData; + uint64_t _min = UINT64_MAX; + uint64_t _max = 0; + uint64_t _sum = 0; + + *minIndex = 0; + *maxIndex = 0; + + assert(numOfRow <= INT16_MAX); + + for (int32_t i = 0; i < numOfRow; ++i) { + if (((uint64_t)data[i]) == TSDB_DATA_UBIGINT_NULL) { + (*numOfNull) += 1; + continue; + } + + DO_STATICS(_sum, _min, _max, *minIndex, *maxIndex, data, i); + } + + *min = _min; + *max = _max; + *sum = _sum; +} + +static void getStatics_f(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, + int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { + float *data = (float *)pData; + float fmin = FLT_MAX; + float fmax = -FLT_MAX; + double dsum = 0; + *minIndex = 0; + *maxIndex = 0; + + assert(numOfRow <= INT16_MAX); + + for (int32_t i = 0; i < numOfRow; ++i) { + if ((*(uint32_t*)&(data[i])) == TSDB_DATA_FLOAT_NULL) { + (*numOfNull) += 1; + continue; + } + + float fv = GET_FLOAT_VAL((const char*)&(data[i])); + + dsum += fv; + if (fmin > fv) { + fmin = fv; + *minIndex = i; + } + + if (fmax < fv) { + fmax = fv; + *maxIndex = i; + } + } + + SET_DOUBLE_VAL(sum, dsum); + SET_DOUBLE_VAL(max, fmax); + SET_DOUBLE_VAL(min, fmin); +} + +static void getStatics_d(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, + int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { + double *data = (double *)pData; + double dmin = DBL_MAX; + double dmax = -DBL_MAX; + double dsum = 0; + *minIndex = 0; + *maxIndex = 0; + + assert(numOfRow <= INT16_MAX); + + for (int32_t i = 0; i < numOfRow; ++i) { + if ((*(uint64_t*)&(data[i])) == TSDB_DATA_DOUBLE_NULL) { + (*numOfNull) += 1; + continue; + } + + double dv = 0; + dv = GET_DOUBLE_VAL((const char*)&(data[i])); + dsum += dv; + if (dmin > dv) { + dmin = dv; + *minIndex = i; + } + + if (dmax < dv) { + dmax = dv; + *maxIndex = i; + } + } + + SET_DOUBLE_PTR(sum, &dsum); + SET_DOUBLE_PTR(max, &dmax); + SET_DOUBLE_PTR(min, &dmin); +} + +static void getStatics_bin(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, + int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { + const char* data = pData; + assert(numOfRow <= INT16_MAX); + + for (int32_t i = 0; i < numOfRow; ++i) { + if (isNull(data, TSDB_DATA_TYPE_BINARY)) { + (*numOfNull) += 1; + } + + data += varDataTLen(data); + } + + *sum = 0; + *max = 0; + *min = 0; + *minIndex = 0; + *maxIndex = 0; +} + +static void getStatics_nchr(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, + int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { + const char* data = pData; + assert(numOfRow <= INT16_MAX); + + for (int32_t i = 0; i < numOfRow; ++i) { + if (isNull(data, TSDB_DATA_TYPE_NCHAR)) { + (*numOfNull) += 1; + } + + data += varDataTLen(data); + } + + *sum = 0; + *max = 0; + *min = 0; + *minIndex = 0; + *maxIndex = 0; +} + +tDataTypeDescriptor tDataTypes[15] = { + {TSDB_DATA_TYPE_NULL, 6, 1, "NOTYPE", 0, 0, NULL, NULL, NULL}, + {TSDB_DATA_TYPE_BOOL, 4, CHAR_BYTES, "BOOL", false, true, tsCompressBool, tsDecompressBool, getStatics_bool}, + {TSDB_DATA_TYPE_TINYINT, 7, CHAR_BYTES, "TINYINT", INT8_MIN, INT8_MAX, tsCompressTinyint, tsDecompressTinyint, getStatics_i8}, + {TSDB_DATA_TYPE_SMALLINT, 8, SHORT_BYTES, "SMALLINT", INT16_MIN, INT16_MAX, tsCompressSmallint, tsDecompressSmallint, getStatics_i16}, + {TSDB_DATA_TYPE_INT, 3, INT_BYTES, "INT", INT32_MIN, INT32_MAX, tsCompressInt, tsDecompressInt, getStatics_i32}, + {TSDB_DATA_TYPE_BIGINT, 6, LONG_BYTES, "BIGINT", INT64_MIN, INT64_MAX, tsCompressBigint, tsDecompressBigint, getStatics_i64}, + {TSDB_DATA_TYPE_FLOAT, 5, FLOAT_BYTES, "FLOAT", 0, 0, tsCompressFloat, tsDecompressFloat, getStatics_f}, + {TSDB_DATA_TYPE_DOUBLE, 6, DOUBLE_BYTES, "DOUBLE", 0, 0, tsCompressDouble, tsDecompressDouble, getStatics_d}, + {TSDB_DATA_TYPE_BINARY, 6, 0, "BINARY", 0, 0, tsCompressString, tsDecompressString, getStatics_bin}, + {TSDB_DATA_TYPE_TIMESTAMP, 9, LONG_BYTES, "TIMESTAMP", INT64_MIN, INT64_MAX, tsCompressTimestamp, tsDecompressTimestamp, getStatics_i64}, + {TSDB_DATA_TYPE_NCHAR, 5, 8, "NCHAR", 0, 0, tsCompressString, tsDecompressString, getStatics_nchr}, + {TSDB_DATA_TYPE_UTINYINT, 16, CHAR_BYTES, "TINYINT UNSIGNED", 0, UINT8_MAX, tsCompressTinyint, tsDecompressTinyint, getStatics_u8}, + {TSDB_DATA_TYPE_USMALLINT, 17, SHORT_BYTES, "SMALLINT UNSIGNED", 0, UINT16_MAX, tsCompressSmallint, tsDecompressSmallint, getStatics_u16}, + {TSDB_DATA_TYPE_UINT, 12, INT_BYTES, "INT UNSIGNED", 0, UINT32_MAX, tsCompressInt, tsDecompressInt, getStatics_u32}, + {TSDB_DATA_TYPE_UBIGINT, 15, LONG_BYTES, "BIGINT UNSIGNED", 0, UINT64_MAX, tsCompressBigint, tsDecompressBigint, getStatics_u64}, +}; + +char tTokenTypeSwitcher[13] = { + TSDB_DATA_TYPE_NULL, // no type + TSDB_DATA_TYPE_BINARY, // TK_ID + TSDB_DATA_TYPE_BOOL, // TK_BOOL + TSDB_DATA_TYPE_BIGINT, // TK_TINYINT + TSDB_DATA_TYPE_BIGINT, // TK_SMALLINT + TSDB_DATA_TYPE_BIGINT, // TK_INTEGER + TSDB_DATA_TYPE_BIGINT, // TK_BIGINT + TSDB_DATA_TYPE_DOUBLE, // TK_FLOAT + TSDB_DATA_TYPE_DOUBLE, // TK_DOUBLE + TSDB_DATA_TYPE_BINARY, // TK_STRING + TSDB_DATA_TYPE_BIGINT, // TK_TIMESTAMP + TSDB_DATA_TYPE_BINARY, // TK_BINARY + TSDB_DATA_TYPE_NCHAR, // TK_NCHAR +}; + +float floatMin = -FLT_MAX, floatMax = FLT_MAX; +double doubleMin = -DBL_MAX, doubleMax = DBL_MAX; + +FORCE_INLINE void* getDataMin(int32_t type) { + switch (type) { + case TSDB_DATA_TYPE_FLOAT: + return &floatMin; + case TSDB_DATA_TYPE_DOUBLE: + return &doubleMin; + default: + return &tDataTypes[type].minValue; + } +} + +FORCE_INLINE void* getDataMax(int32_t type) { + switch (type) { + case TSDB_DATA_TYPE_FLOAT: + return &floatMax; + case TSDB_DATA_TYPE_DOUBLE: + return &doubleMax; + default: + return &tDataTypes[type].maxValue; + } +} + +bool isValidDataType(int32_t type) { + return type >= TSDB_DATA_TYPE_NULL && type <= TSDB_DATA_TYPE_UBIGINT; +} + +void setVardataNull(void* val, int32_t type) { + if (type == TSDB_DATA_TYPE_BINARY) { + varDataSetLen(val, sizeof(int8_t)); + *(uint8_t*) varDataVal(val) = TSDB_DATA_BINARY_NULL; + } else if (type == TSDB_DATA_TYPE_NCHAR) { + varDataSetLen(val, sizeof(int32_t)); + *(uint32_t*) varDataVal(val) = TSDB_DATA_NCHAR_NULL; + } else { + assert(0); + } +} + +void setNull(void *val, int32_t type, int32_t bytes) { setNullN(val, type, bytes, 1); } + +#define POINTER_SHIFT(p, b) ((void *)((char *)(p) + (b))) + +void setNullN(void *val, int32_t type, int32_t bytes, int32_t numOfElems) { + switch (type) { + case TSDB_DATA_TYPE_BOOL: + for (int32_t i = 0; i < numOfElems; ++i) { + *(uint8_t *)(POINTER_SHIFT(val, i * tDataTypes[type].bytes)) = TSDB_DATA_BOOL_NULL; + } + break; + case TSDB_DATA_TYPE_TINYINT: + for (int32_t i = 0; i < numOfElems; ++i) { + *(uint8_t *)(POINTER_SHIFT(val, i * tDataTypes[type].bytes)) = TSDB_DATA_TINYINT_NULL; + } + break; + case TSDB_DATA_TYPE_SMALLINT: + for (int32_t i = 0; i < numOfElems; ++i) { + *(uint16_t *)(POINTER_SHIFT(val, i * tDataTypes[type].bytes)) = TSDB_DATA_SMALLINT_NULL; + } + break; + case TSDB_DATA_TYPE_INT: + for (int32_t i = 0; i < numOfElems; ++i) { + *(uint32_t *)(POINTER_SHIFT(val, i * tDataTypes[type].bytes)) = TSDB_DATA_INT_NULL; + } + break; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: + for (int32_t i = 0; i < numOfElems; ++i) { + *(uint64_t *)(POINTER_SHIFT(val, i * tDataTypes[type].bytes)) = TSDB_DATA_BIGINT_NULL; + } + break; + case TSDB_DATA_TYPE_UTINYINT: + for (int32_t i = 0; i < numOfElems; ++i) { + *(uint8_t *)(POINTER_SHIFT(val, i * tDataTypes[type].bytes)) = TSDB_DATA_UTINYINT_NULL; + } + break; + case TSDB_DATA_TYPE_USMALLINT: + for (int32_t i = 0; i < numOfElems; ++i) { + *(uint16_t *)(POINTER_SHIFT(val, i * tDataTypes[type].bytes)) = TSDB_DATA_USMALLINT_NULL; + } + break; + case TSDB_DATA_TYPE_UINT: + for (int32_t i = 0; i < numOfElems; ++i) { + *(uint32_t *)(POINTER_SHIFT(val, i * tDataTypes[type].bytes)) = TSDB_DATA_UINT_NULL; + } + break; + case TSDB_DATA_TYPE_UBIGINT: + for (int32_t i = 0; i < numOfElems; ++i) { + *(uint64_t *)(POINTER_SHIFT(val, i * tDataTypes[type].bytes)) = TSDB_DATA_UBIGINT_NULL; + } + break; + case TSDB_DATA_TYPE_FLOAT: + for (int32_t i = 0; i < numOfElems; ++i) { + *(uint32_t *)(POINTER_SHIFT(val, i * tDataTypes[type].bytes)) = TSDB_DATA_FLOAT_NULL; + } + break; + case TSDB_DATA_TYPE_DOUBLE: + for (int32_t i = 0; i < numOfElems; ++i) { + *(uint64_t *)(POINTER_SHIFT(val, i * tDataTypes[type].bytes)) = TSDB_DATA_DOUBLE_NULL; + } + break; + case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_BINARY: + for (int32_t i = 0; i < numOfElems; ++i) { + setVardataNull(POINTER_SHIFT(val, i * bytes), type); + } + break; + default: { + for (int32_t i = 0; i < numOfElems; ++i) { + *(uint32_t *)(POINTER_SHIFT(val, i * tDataTypes[TSDB_DATA_TYPE_INT].bytes)) = TSDB_DATA_INT_NULL; + } + break; + } + } +} + +static uint8_t nullBool = TSDB_DATA_BOOL_NULL; +static uint8_t nullTinyInt = TSDB_DATA_TINYINT_NULL; +static uint16_t nullSmallInt = TSDB_DATA_SMALLINT_NULL; +static uint32_t nullInt = TSDB_DATA_INT_NULL; +static uint64_t nullBigInt = TSDB_DATA_BIGINT_NULL; +static uint32_t nullFloat = TSDB_DATA_FLOAT_NULL; +static uint64_t nullDouble = TSDB_DATA_DOUBLE_NULL; +static uint8_t nullTinyIntu = TSDB_DATA_UTINYINT_NULL; +static uint16_t nullSmallIntu = TSDB_DATA_USMALLINT_NULL; +static uint32_t nullIntu = TSDB_DATA_UINT_NULL; +static uint64_t nullBigIntu = TSDB_DATA_UBIGINT_NULL; +static SBinaryNullT nullBinary = {1, TSDB_DATA_BINARY_NULL}; +static SNCharNullT nullNchar = {4, TSDB_DATA_NCHAR_NULL}; + +static const void *nullValues[] = { + &nullBool, &nullTinyInt, &nullSmallInt, &nullInt, &nullBigInt, + &nullFloat, &nullDouble, &nullBinary, &nullBigInt, &nullNchar, + &nullTinyIntu, &nullSmallIntu, &nullIntu, &nullBigIntu, +}; + +const void *getNullValue(int32_t type) { + assert(type >= TSDB_DATA_TYPE_BOOL && type <= TSDB_DATA_TYPE_UBIGINT); + return nullValues[type - 1]; +} + +void assignVal(char *val, const char *src, int32_t len, int32_t type) { + switch (type) { + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: + *((int8_t *)val) = GET_INT8_VAL(src); + break; + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: + *((int16_t *)val) = GET_INT16_VAL(src); + break; + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: + *((int32_t *)val) = GET_INT32_VAL(src); + break; + + case TSDB_DATA_TYPE_FLOAT: + SET_FLOAT_VAL(val, GET_FLOAT_VAL(src)); + break; + case TSDB_DATA_TYPE_DOUBLE: + SET_DOUBLE_VAL(val, GET_DOUBLE_VAL(src)); + break; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: + *((int64_t *)val) = GET_INT64_VAL(src); + break; + case TSDB_DATA_TYPE_BINARY: + varDataCopy(val, src); + break; + case TSDB_DATA_TYPE_NCHAR: + varDataCopy(val, src); + break; + default: { + memcpy(val, src, len); + break; + } + } +} + +void operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type) { + if (optr == TSDB_BINARY_OP_ADD) { + switch (type) { + case TSDB_DATA_TYPE_TINYINT: + *((int8_t *)dst) = GET_INT8_VAL(s1) + GET_INT8_VAL(s2); + break; + case TSDB_DATA_TYPE_UTINYINT: + *((uint8_t *)dst) = GET_UINT8_VAL(s1) + GET_UINT8_VAL(s2); + break; + case TSDB_DATA_TYPE_SMALLINT: + *((int16_t *)dst) = GET_INT16_VAL(s1) + GET_INT16_VAL(s2); + break; + case TSDB_DATA_TYPE_USMALLINT: + *((uint16_t *)dst) = GET_UINT16_VAL(s1) + GET_UINT16_VAL(s2); + break; + case TSDB_DATA_TYPE_INT: + *((int32_t *)dst) = GET_INT32_VAL(s1) + GET_INT32_VAL(s2); + break; + case TSDB_DATA_TYPE_UINT: + *((uint32_t *)dst) = GET_UINT32_VAL(s1) + GET_UINT32_VAL(s2); + break; + case TSDB_DATA_TYPE_BIGINT: + *((int64_t *)dst) = GET_INT64_VAL(s1) + GET_INT64_VAL(s2); + break; + case TSDB_DATA_TYPE_UBIGINT: + *((uint64_t *)dst) = GET_UINT64_VAL(s1) + GET_UINT64_VAL(s2); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + *((int64_t *)dst) = GET_INT64_VAL(s1) + GET_INT64_VAL(s2); + break; + case TSDB_DATA_TYPE_FLOAT: + SET_FLOAT_VAL(dst, GET_FLOAT_VAL(s1) + GET_FLOAT_VAL(s2)); + break; + case TSDB_DATA_TYPE_DOUBLE: + SET_DOUBLE_VAL(dst, GET_DOUBLE_VAL(s1) + GET_DOUBLE_VAL(s2)); + break; + default: { + assert(0); + break; + } + } + } else { + assert(0); + } +} + +#define SWAP(a, b, c) \ + do { \ + typeof(a) __tmp = (a); \ + (a) = (b); \ + (b) = __tmp; \ + } while (0) + + +void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf) { + switch (type) { + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: { + SWAP(*(int32_t *)(pLeft), *(int32_t *)(pRight), int32_t); + break; + } + + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: { + SWAP(*(int64_t *)(pLeft), *(int64_t *)(pRight), int64_t); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + SWAP(*(double *)(pLeft), *(double *)(pRight), double); + break; + } + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: { + SWAP(*(int16_t *)(pLeft), *(int16_t *)(pRight), int16_t); + break; + } + + case TSDB_DATA_TYPE_FLOAT: { + SWAP(*(float *)(pLeft), *(float *)(pRight), float); + break; + } + + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: { + SWAP(*(int8_t *)(pLeft), *(int8_t *)(pRight), int8_t); + break; + } + + default: { + memcpy(buf, pLeft, size); + memcpy(pLeft, pRight, size); + memcpy(pRight, buf, size); + break; + } + } +} diff --git a/source/libs/parser/inc/tvariant.h b/source/libs/parser/inc/tvariant.h index 941d1920cb..4eb3120a79 100644 --- a/source/libs/parser/inc/tvariant.h +++ b/source/libs/parser/inc/tvariant.h @@ -39,7 +39,7 @@ typedef struct SVariant { bool taosVariantIsValid(SVariant *pVar); -void taosVariantCreate(SVariant *pVar, SStrToken *token); +void taosVariantCreate(SVariant *pVar, SToken *token); void taosVariantCreateFromBinary(SVariant *pVar, const char *pz, size_t len, uint32_t type); diff --git a/source/libs/planner/CMakeLists.txt b/source/libs/planner/CMakeLists.txt index 52506b4556..b00d92336d 100644 --- a/source/libs/planner/CMakeLists.txt +++ b/source/libs/planner/CMakeLists.txt @@ -8,5 +8,5 @@ target_include_directories( target_link_libraries( planner - PRIVATE os util common catalog parser + PRIVATE os util common catalog parser transport ) \ No newline at end of file diff --git a/source/libs/scheduler/inc/schedulerInt.h b/source/libs/scheduler/inc/schedulerInt.h index 778e3b241d..b1b128e200 100644 --- a/source/libs/scheduler/inc/schedulerInt.h +++ b/source/libs/scheduler/inc/schedulerInt.h @@ -25,14 +25,6 @@ extern "C" { #include "planner.h" #include "scheduler.h" -typedef struct SSubquery { - int64_t taskId; // the task id created by qnode - int32_t type; - int32_t level; - struct SQueryPhyNode *pNode; - SArray *pUpstream; -} SSubquery; - typedef struct SQuery { SArray **pSubquery; int32_t numOfLevels; diff --git a/src/client/src/tscServer.c b/src/client/src/tscServer.c index b87ec92ff1..7f57c7d5c9 100644 --- a/src/client/src/tscServer.c +++ b/src/client/src/tscServer.c @@ -13,7 +13,7 @@ * along with this program. If not, see . */ -#include +#include #include "os.h" #include "qPlan.h" #include "qTableMeta.h" diff --git a/src/common/src/ttypes.c b/src/common/src/ttypes.c index 08bfc2e9aa..33f5d754b4 100644 --- a/src/common/src/ttypes.c +++ b/src/common/src/ttypes.c @@ -14,9 +14,9 @@ */ #include "os.h" -#include "ttype.h" +#include "tcompression.h" #include "ttokendef.h" -#include "tscompression.h" +#include "ttype.h" const int32_t TYPE_BYTES[15] = { -1, // TSDB_DATA_TYPE_NULL diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c index c6f6e976f6..242aad5af7 100644 --- a/src/dnode/src/dnodeMain.c +++ b/src/dnode/src/dnodeMain.c @@ -14,36 +14,36 @@ */ #define _DEFAULT_SOURCE -#include "os.h" -#include "taos.h" -#include "tnote.h" -#include "ttimer.h" -#include "tconfig.h" -#include "tfile.h" -#include "twal.h" -#include "tfs.h" -#include "tsync.h" -#include "dnodeStep.h" -#include "dnodePeer.h" -#include "dnodeModule.h" -#include "dnodeEps.h" -#include "dnodeMInfos.h" #include "dnodeCfg.h" #include "dnodeCheck.h" -#include "dnodeVRead.h" -#include "dnodeVWrite.h" -#include "dnodeVMgmt.h" -#include "dnodeVnodes.h" +#include "dnodeEps.h" +#include "dnodeMInfos.h" +#include "dnodeMPeer.h" #include "dnodeMRead.h" #include "dnodeMWrite.h" -#include "dnodeMPeer.h" +#include "dnodeModule.h" +#include "dnodePeer.h" #include "dnodeShell.h" +#include "dnodeStep.h" #include "dnodeTelemetry.h" -#include "module.h" +#include "dnodeVMgmt.h" +#include "dnodeVRead.h" +#include "dnodeVWrite.h" +#include "dnodeVnodes.h" #include "mnode.h" +#include "module.h" +#include "os.h" #include "qScript.h" +#include "taos.h" #include "tcache.h" -#include "tscompression.h" +#include "tcompression.h" +#include "tconfig.h" +#include "tfile.h" +#include "tfs.h" +#include "tnote.h" +#include "tsync.h" +#include "ttimer.h" +#include "twal.h" #if !defined(_MODULE) || !defined(_TD_LINUX) int32_t moduleStart() { return 0; } diff --git a/src/mnode/src/mnodeTable.c b/src/mnode/src/mnodeTable.c index a6158906a7..44dae0aed1 100644 --- a/src/mnode/src/mnodeTable.c +++ b/src/mnode/src/mnodeTable.c @@ -14,35 +14,34 @@ */ #define _DEFAULT_SOURCE -#include "os.h" -#include "taosmsg.h" -#include "tutil.h" -#include "taoserror.h" -#include "taosmsg.h" -#include "tscompression.h" -#include "tname.h" -#include "tidpool.h" -#include "tglobal.h" -#include "tcompare.h" -#include "tdataformat.h" -#include "tgrant.h" -#include "tqueue.h" +#include "mnodeTable.h" +#include "dnode.h" #include "hash.h" #include "mnode.h" -#include "dnode.h" -#include "mnodeDef.h" -#include "mnodeInt.h" #include "mnodeAcct.h" #include "mnodeDb.h" +#include "mnodeDef.h" #include "mnodeDnode.h" +#include "mnodeFunc.h" +#include "mnodeInt.h" +#include "mnodePeer.h" +#include "mnodeRead.h" #include "mnodeSdb.h" #include "mnodeShow.h" -#include "mnodeTable.h" #include "mnodeVgroup.h" #include "mnodeWrite.h" -#include "mnodeRead.h" -#include "mnodePeer.h" -#include "mnodeFunc.h" +#include "os.h" +#include "taoserror.h" +#include "taosmsg.h" +#include "tcompare.h" +#include "tcompression.h" +#include "tdataformat.h" +#include "tglobal.h" +#include "tgrant.h" +#include "tidpool.h" +#include "tname.h" +#include "tqueue.h" +#include "tutil.h" #define ALTER_CTABLE_RETRY_TIMES 3 #define CREATE_CTABLE_RETRY_TIMES 10 diff --git a/src/query/src/qExecutor.c b/src/query/src/qExecutor.c index 54a5423219..ec576eec8b 100644 --- a/src/query/src/qExecutor.c +++ b/src/query/src/qExecutor.c @@ -19,17 +19,17 @@ #include "exception.h" #include "hash.h" -#include "texpr.h" #include "qExecutor.h" #include "qResultbuf.h" +#include "qScript.h" #include "qUtil.h" #include "queryLog.h" -#include "tlosertree.h" -#include "ttype.h" #include "tcompare.h" -#include "tscompression.h" -#include "qScript.h" +#include "tcompression.h" +#include "texpr.h" +#include "tlosertree.h" #include "tscLog.h" +#include "ttype.h" #define IS_MASTER_SCAN(runtime) ((runtime)->scanFlag == MASTER_SCAN) #define IS_REVERSE_SCAN(runtime) ((runtime)->scanFlag == REVERSE_SCAN) diff --git a/src/query/src/qResultbuf.c b/src/query/src/qResultbuf.c index 63eba51d6b..3724362e29 100644 --- a/src/query/src/qResultbuf.c +++ b/src/query/src/qResultbuf.c @@ -1,10 +1,10 @@ #include "qResultbuf.h" -#include "stddef.h" -#include "tscompression.h" #include "hash.h" #include "qExtbuffer.h" #include "queryLog.h" +#include "stddef.h" #include "taoserror.h" +#include "tcompression.h" #define GET_DATA_PAYLOAD(_p) ((char *)(_p)->pData + POINTER_BYTES) #define NO_IN_MEM_AVAILABLE_PAGES(_b) (listNEles((_b)->lruList) >= (_b)->inMemPages) diff --git a/src/query/src/qTsbuf.c b/src/query/src/qTsbuf.c index 4cf05dd2c7..92ca841272 100644 --- a/src/query/src/qTsbuf.c +++ b/src/query/src/qTsbuf.c @@ -1,8 +1,8 @@ #include "qTsbuf.h" -#include "taoserror.h" -#include "tscompression.h" -#include "tutil.h" #include "queryLog.h" +#include "taoserror.h" +#include "tcompression.h" +#include "tutil.h" static int32_t getDataStartOffset(); static void TSBufUpdateGroupInfo(STSBuf* pTSBuf, int32_t index, STSGroupBlockInfo* pBlockInfo); diff --git a/src/query/src/qUtil.c b/src/query/src/qUtil.c index bc27e094db..f52d6de5d2 100644 --- a/src/query/src/qUtil.c +++ b/src/query/src/qUtil.c @@ -19,10 +19,10 @@ #include "qExecutor.h" #include "qUtil.h" -#include "tbuffer.h" -#include "tlosertree.h" #include "queryLog.h" -#include "tscompression.h" +#include "tbuffer.h" +#include "tcompression.h" +#include "tlosertree.h" typedef struct SCompSupporter { STableQueryInfo **pTableQueryInfo; diff --git a/src/tsdb/inc/tsdbint.h b/src/tsdb/inc/tsdbint.h index 80e9297579..2354269c6e 100644 --- a/src/tsdb/inc/tsdbint.h +++ b/src/tsdb/inc/tsdbint.h @@ -26,20 +26,20 @@ // #include // #include +#include "hash.h" #include "os.h" -#include "tlog.h" #include "taosdef.h" #include "taoserror.h" -#include "tchecksum.h" -#include "tskiplist.h" -#include "tdataformat.h" -#include "tcoding.h" -#include "tscompression.h" -#include "tlockfree.h" -#include "tlist.h" -#include "hash.h" #include "tarray.h" +#include "tchecksum.h" +#include "tcoding.h" +#include "tcompression.h" +#include "tdataformat.h" #include "tfs.h" +#include "tlist.h" +#include "tlockfree.h" +#include "tlog.h" +#include "tskiplist.h" #include "tsocket.h" #include "tsdb.h" diff --git a/src/util/src/tcompression.c b/src/util/src/tcompression.c index 48bba75926..5d16018606 100644 --- a/src/util/src/tcompression.c +++ b/src/util/src/tcompression.c @@ -53,10 +53,9 @@ #include "td_sz.h" #endif #include "taosdef.h" -#include "tscompression.h" -#include "tulog.h" +#include "tcompression.h" #include "tglobal.h" - +#include "tulog.h" static const int TEST_NUMBER = 1; #define is_bigendian() ((*(char *)&TEST_NUMBER) == 0) From bb97be2a5f8c23adc36f486791584b4f1906f27d Mon Sep 17 00:00:00 2001 From: Haojun Liao Date: Fri, 8 Oct 2021 11:55:50 +0800 Subject: [PATCH 9/9] [td-10529] merge 3.0. --- include/common/tname.h | 59 ++ source/common/src/ttypes.c | 684 ++++++++++++++++++++++ source/libs/executor/inc/tarithoperator.h | 32 + source/libs/executor/src/tarithoperator.h | 32 + src/dnode/src/dnodeMain.c | 0 5 files changed, 807 insertions(+) create mode 100644 include/common/tname.h create mode 100644 source/common/src/ttypes.c create mode 100644 source/libs/executor/inc/tarithoperator.h create mode 100644 source/libs/executor/src/tarithoperator.h delete mode 100644 src/dnode/src/dnodeMain.c diff --git a/include/common/tname.h b/include/common/tname.h new file mode 100644 index 0000000000..18526f54d4 --- /dev/null +++ b/include/common/tname.h @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef TDENGINE_TNAME_H +#define TDENGINE_TNAME_H + +#define TSDB_DB_NAME_T 1 +#define TSDB_TABLE_NAME_T 2 + +#define T_NAME_ACCT 0x1u +#define T_NAME_DB 0x2u +#define T_NAME_TABLE 0x4u + +typedef struct SName { + uint8_t type; //db_name_t, table_name_t + char acctId[TSDB_ACCT_ID_LEN]; + char dbname[TSDB_DB_NAME_LEN]; + char tname[TSDB_TABLE_NAME_LEN]; +} SName; + +int32_t tNameExtractFullName(const SName* name, char* dst); + +int32_t tNameLen(const SName* name); + +SName* tNameDup(const SName* name); + +bool tIsValidName(const SName* name); + +const char* tNameGetTableName(const SName* name); + +int32_t tNameGetDbName(const SName* name, char* dst); + +int32_t tNameGetFullDbName(const SName* name, char* dst); + +bool tNameIsEmpty(const SName* name); + +void tNameAssign(SName* dst, const SName* src); + +int32_t tNameFromString(SName* dst, const char* str, uint32_t type); + +int32_t tNameSetAcctId(SName* dst, const char* acct); + +#if 0 +int32_t tNameSetDbName(SName* dst, const char* acct, SToken* dbToken); +#endif + +#endif // TDENGINE_TNAME_H diff --git a/source/common/src/ttypes.c b/source/common/src/ttypes.c new file mode 100644 index 0000000000..bec8793b72 --- /dev/null +++ b/source/common/src/ttypes.c @@ -0,0 +1,684 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "taos.h" +#include "os.h" +#include "ttypes.h" +#include "tcompression.h" + +const int32_t TYPE_BYTES[15] = { + -1, // TSDB_DATA_TYPE_NULL + CHAR_BYTES, // TSDB_DATA_TYPE_BOOL + CHAR_BYTES, // TSDB_DATA_TYPE_TINYINT + SHORT_BYTES, // TSDB_DATA_TYPE_SMALLINT + INT_BYTES, // TSDB_DATA_TYPE_INT + sizeof(int64_t), // TSDB_DATA_TYPE_BIGINT + FLOAT_BYTES, // TSDB_DATA_TYPE_FLOAT + DOUBLE_BYTES, // TSDB_DATA_TYPE_DOUBLE + sizeof(VarDataOffsetT), // TSDB_DATA_TYPE_BINARY + sizeof(TSKEY), // TSDB_DATA_TYPE_TIMESTAMP + sizeof(VarDataOffsetT), // TSDB_DATA_TYPE_NCHAR + CHAR_BYTES, // TSDB_DATA_TYPE_UTINYINT + SHORT_BYTES, // TSDB_DATA_TYPE_USMALLINT + INT_BYTES, // TSDB_DATA_TYPE_UINT + sizeof(uint64_t), // TSDB_DATA_TYPE_UBIGINT +}; + +#define DO_STATICS(__sum, __min, __max, __minIndex, __maxIndex, _list, _index) \ + do { \ + (__sum) += (_list)[(_index)]; \ + if ((__min) > (_list)[(_index)]) { \ + (__min) = (_list)[(_index)]; \ + (__minIndex) = (_index); \ + } \ + \ + if ((__max) < (_list)[(_index)]) { \ + (__max) = (_list)[(_index)]; \ + (__maxIndex) = (_index); \ + } \ + } while (0) + +static void getStatics_bool(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, + int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { + int8_t *data = (int8_t *)pData; + *min = INT64_MAX; + *max = INT64_MIN; + *minIndex = 0; + *maxIndex = 0; + + assert(numOfRow <= INT16_MAX); + + for (int32_t i = 0; i < numOfRow; ++i) { + if (data[i] == TSDB_DATA_BOOL_NULL) { + (*numOfNull) += 1; + continue; + } + + DO_STATICS(*sum, *min, *max, *minIndex, *maxIndex, data, i); + } +} + +static void getStatics_i8(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, + int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { + int8_t *data = (int8_t *)pData; + *min = INT64_MAX; + *max = INT64_MIN; + *minIndex = 0; + *maxIndex = 0; + + assert(numOfRow <= INT16_MAX); + + for (int32_t i = 0; i < numOfRow; ++i) { + if (((uint8_t)data[i]) == TSDB_DATA_TINYINT_NULL) { + (*numOfNull) += 1; + continue; + } + + DO_STATICS(*sum, *min, *max, *minIndex, *maxIndex, data, i); + } +} + +static void getStatics_u8(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, + int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { + uint8_t *data = (uint8_t *)pData; + uint64_t _min = UINT64_MAX; + uint64_t _max = 0; + uint64_t _sum = 0; + + *minIndex = 0; + *maxIndex = 0; + + assert(numOfRow <= INT16_MAX); + + for (int32_t i = 0; i < numOfRow; ++i) { + if (((uint8_t)data[i]) == TSDB_DATA_UTINYINT_NULL) { + (*numOfNull) += 1; + continue; + } + + DO_STATICS(_sum, _min, _max, *minIndex, *maxIndex, data, i); + } + + *min = _min; + *max = _max; + *sum = _sum; +} + +static void getStatics_i16(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, + int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { + int16_t *data = (int16_t *)pData; + *min = INT64_MAX; + *max = INT64_MIN; + *minIndex = 0; + *maxIndex = 0; + + assert(numOfRow <= INT16_MAX); + + for (int32_t i = 0; i < numOfRow; ++i) { + if (((uint16_t)data[i]) == TSDB_DATA_SMALLINT_NULL) { + (*numOfNull) += 1; + continue; + } + + DO_STATICS(*sum, *min, *max, *minIndex, *maxIndex, data, i); + } + +} + +static void getStatics_u16(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, + int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { + uint16_t *data = (uint16_t *)pData; + uint64_t _min = UINT64_MAX; + uint64_t _max = 0; + uint64_t _sum = 0; + + *minIndex = 0; + *maxIndex = 0; + + assert(numOfRow <= INT16_MAX); + + for (int32_t i = 0; i < numOfRow; ++i) { + if (((uint16_t)data[i]) == TSDB_DATA_USMALLINT_NULL) { + (*numOfNull) += 1; + continue; + } + + DO_STATICS(_sum, _min, _max, *minIndex, *maxIndex, data, i); + } + + *min = _min; + *max = _max; + *sum = _sum; +} + +static void getStatics_i32(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, + int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { + int32_t *data = (int32_t *)pData; + *min = INT64_MAX; + *max = INT64_MIN; + *minIndex = 0; + *maxIndex = 0; + + assert(numOfRow <= INT16_MAX); + + for (int32_t i = 0; i < numOfRow; ++i) { + if (((uint32_t)data[i]) == TSDB_DATA_INT_NULL) { + (*numOfNull) += 1; + continue; + } + + DO_STATICS(*sum, *min, *max, *minIndex, *maxIndex, data, i); + } +} + +static void getStatics_u32(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, + int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { + uint32_t *data = (uint32_t *)pData; + uint64_t _min = UINT64_MAX; + uint64_t _max = 0; + uint64_t _sum = 0; + + *minIndex = 0; + *maxIndex = 0; + + assert(numOfRow <= INT16_MAX); + + for (int32_t i = 0; i < numOfRow; ++i) { + if (((uint32_t)data[i]) == TSDB_DATA_UINT_NULL) { + (*numOfNull) += 1; + continue; + } + + DO_STATICS(_sum, _min, _max, *minIndex, *maxIndex, data, i); + } + + *min = _min; + *max = _max; + *sum = _sum; +} + +static void getStatics_i64(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, + int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { + int64_t *data = (int64_t *)pData; + *min = INT64_MAX; + *max = INT64_MIN; + *minIndex = 0; + *maxIndex = 0; + + assert(numOfRow <= INT16_MAX); + + for (int32_t i = 0; i < numOfRow; ++i) { + if (((uint64_t)data[i]) == TSDB_DATA_BIGINT_NULL) { + (*numOfNull) += 1; + continue; + } + + DO_STATICS(*sum, *min, *max, *minIndex, *maxIndex, data, i); + } +} + +static void getStatics_u64(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, + int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { + uint64_t *data = (uint64_t *)pData; + uint64_t _min = UINT64_MAX; + uint64_t _max = 0; + uint64_t _sum = 0; + + *minIndex = 0; + *maxIndex = 0; + + assert(numOfRow <= INT16_MAX); + + for (int32_t i = 0; i < numOfRow; ++i) { + if (((uint64_t)data[i]) == TSDB_DATA_UBIGINT_NULL) { + (*numOfNull) += 1; + continue; + } + + DO_STATICS(_sum, _min, _max, *minIndex, *maxIndex, data, i); + } + + *min = _min; + *max = _max; + *sum = _sum; +} + +static void getStatics_f(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, + int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { + float *data = (float *)pData; + float fmin = FLT_MAX; + float fmax = -FLT_MAX; + double dsum = 0; + *minIndex = 0; + *maxIndex = 0; + + assert(numOfRow <= INT16_MAX); + + for (int32_t i = 0; i < numOfRow; ++i) { + if ((*(uint32_t*)&(data[i])) == TSDB_DATA_FLOAT_NULL) { + (*numOfNull) += 1; + continue; + } + + float fv = GET_FLOAT_VAL((const char*)&(data[i])); + + dsum += fv; + if (fmin > fv) { + fmin = fv; + *minIndex = i; + } + + if (fmax < fv) { + fmax = fv; + *maxIndex = i; + } + } + + SET_DOUBLE_VAL(sum, dsum); + SET_DOUBLE_VAL(max, fmax); + SET_DOUBLE_VAL(min, fmin); +} + +static void getStatics_d(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, + int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { + double *data = (double *)pData; + double dmin = DBL_MAX; + double dmax = -DBL_MAX; + double dsum = 0; + *minIndex = 0; + *maxIndex = 0; + + assert(numOfRow <= INT16_MAX); + + for (int32_t i = 0; i < numOfRow; ++i) { + if ((*(uint64_t*)&(data[i])) == TSDB_DATA_DOUBLE_NULL) { + (*numOfNull) += 1; + continue; + } + + double dv = 0; + dv = GET_DOUBLE_VAL((const char*)&(data[i])); + dsum += dv; + if (dmin > dv) { + dmin = dv; + *minIndex = i; + } + + if (dmax < dv) { + dmax = dv; + *maxIndex = i; + } + } + + SET_DOUBLE_PTR(sum, &dsum); + SET_DOUBLE_PTR(max, &dmax); + SET_DOUBLE_PTR(min, &dmin); +} + +static void getStatics_bin(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, + int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { + const char* data = pData; + assert(numOfRow <= INT16_MAX); + + for (int32_t i = 0; i < numOfRow; ++i) { + if (isNull(data, TSDB_DATA_TYPE_BINARY)) { + (*numOfNull) += 1; + } + + data += varDataTLen(data); + } + + *sum = 0; + *max = 0; + *min = 0; + *minIndex = 0; + *maxIndex = 0; +} + +static void getStatics_nchr(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, + int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { + const char* data = pData; + assert(numOfRow <= INT16_MAX); + + for (int32_t i = 0; i < numOfRow; ++i) { + if (isNull(data, TSDB_DATA_TYPE_NCHAR)) { + (*numOfNull) += 1; + } + + data += varDataTLen(data); + } + + *sum = 0; + *max = 0; + *min = 0; + *minIndex = 0; + *maxIndex = 0; +} + +tDataTypeDescriptor tDataTypes[15] = { + {TSDB_DATA_TYPE_NULL, 6, 1, "NOTYPE", 0, 0, NULL, NULL, NULL}, + {TSDB_DATA_TYPE_BOOL, 4, CHAR_BYTES, "BOOL", false, true, tsCompressBool, tsDecompressBool, getStatics_bool}, + {TSDB_DATA_TYPE_TINYINT, 7, CHAR_BYTES, "TINYINT", INT8_MIN, INT8_MAX, tsCompressTinyint, tsDecompressTinyint, getStatics_i8}, + {TSDB_DATA_TYPE_SMALLINT, 8, SHORT_BYTES, "SMALLINT", INT16_MIN, INT16_MAX, tsCompressSmallint, tsDecompressSmallint, getStatics_i16}, + {TSDB_DATA_TYPE_INT, 3, INT_BYTES, "INT", INT32_MIN, INT32_MAX, tsCompressInt, tsDecompressInt, getStatics_i32}, + {TSDB_DATA_TYPE_BIGINT, 6, LONG_BYTES, "BIGINT", INT64_MIN, INT64_MAX, tsCompressBigint, tsDecompressBigint, getStatics_i64}, + {TSDB_DATA_TYPE_FLOAT, 5, FLOAT_BYTES, "FLOAT", 0, 0, tsCompressFloat, tsDecompressFloat, getStatics_f}, + {TSDB_DATA_TYPE_DOUBLE, 6, DOUBLE_BYTES, "DOUBLE", 0, 0, tsCompressDouble, tsDecompressDouble, getStatics_d}, + {TSDB_DATA_TYPE_BINARY, 6, 0, "BINARY", 0, 0, tsCompressString, tsDecompressString, getStatics_bin}, + {TSDB_DATA_TYPE_TIMESTAMP, 9, LONG_BYTES, "TIMESTAMP", INT64_MIN, INT64_MAX, tsCompressTimestamp, tsDecompressTimestamp, getStatics_i64}, + {TSDB_DATA_TYPE_NCHAR, 5, 8, "NCHAR", 0, 0, tsCompressString, tsDecompressString, getStatics_nchr}, + {TSDB_DATA_TYPE_UTINYINT, 16, CHAR_BYTES, "TINYINT UNSIGNED", 0, UINT8_MAX, tsCompressTinyint, tsDecompressTinyint, getStatics_u8}, + {TSDB_DATA_TYPE_USMALLINT, 17, SHORT_BYTES, "SMALLINT UNSIGNED", 0, UINT16_MAX, tsCompressSmallint, tsDecompressSmallint, getStatics_u16}, + {TSDB_DATA_TYPE_UINT, 12, INT_BYTES, "INT UNSIGNED", 0, UINT32_MAX, tsCompressInt, tsDecompressInt, getStatics_u32}, + {TSDB_DATA_TYPE_UBIGINT, 15, LONG_BYTES, "BIGINT UNSIGNED", 0, UINT64_MAX, tsCompressBigint, tsDecompressBigint, getStatics_u64}, +}; + +char tTokenTypeSwitcher[13] = { + TSDB_DATA_TYPE_NULL, // no type + TSDB_DATA_TYPE_BINARY, // TK_ID + TSDB_DATA_TYPE_BOOL, // TK_BOOL + TSDB_DATA_TYPE_BIGINT, // TK_TINYINT + TSDB_DATA_TYPE_BIGINT, // TK_SMALLINT + TSDB_DATA_TYPE_BIGINT, // TK_INTEGER + TSDB_DATA_TYPE_BIGINT, // TK_BIGINT + TSDB_DATA_TYPE_DOUBLE, // TK_FLOAT + TSDB_DATA_TYPE_DOUBLE, // TK_DOUBLE + TSDB_DATA_TYPE_BINARY, // TK_STRING + TSDB_DATA_TYPE_BIGINT, // TK_TIMESTAMP + TSDB_DATA_TYPE_BINARY, // TK_BINARY + TSDB_DATA_TYPE_NCHAR, // TK_NCHAR +}; + +float floatMin = -FLT_MAX, floatMax = FLT_MAX; +double doubleMin = -DBL_MAX, doubleMax = DBL_MAX; + +FORCE_INLINE void* getDataMin(int32_t type) { + switch (type) { + case TSDB_DATA_TYPE_FLOAT: + return &floatMin; + case TSDB_DATA_TYPE_DOUBLE: + return &doubleMin; + default: + return &tDataTypes[type].minValue; + } +} + +FORCE_INLINE void* getDataMax(int32_t type) { + switch (type) { + case TSDB_DATA_TYPE_FLOAT: + return &floatMax; + case TSDB_DATA_TYPE_DOUBLE: + return &doubleMax; + default: + return &tDataTypes[type].maxValue; + } +} + +bool isValidDataType(int32_t type) { + return type >= TSDB_DATA_TYPE_NULL && type <= TSDB_DATA_TYPE_UBIGINT; +} + +void setVardataNull(void* val, int32_t type) { + if (type == TSDB_DATA_TYPE_BINARY) { + varDataSetLen(val, sizeof(int8_t)); + *(uint8_t*) varDataVal(val) = TSDB_DATA_BINARY_NULL; + } else if (type == TSDB_DATA_TYPE_NCHAR) { + varDataSetLen(val, sizeof(int32_t)); + *(uint32_t*) varDataVal(val) = TSDB_DATA_NCHAR_NULL; + } else { + assert(0); + } +} + +void setNull(void *val, int32_t type, int32_t bytes) { setNullN(val, type, bytes, 1); } + +#define POINTER_SHIFT(p, b) ((void *)((char *)(p) + (b))) + +void setNullN(void *val, int32_t type, int32_t bytes, int32_t numOfElems) { + switch (type) { + case TSDB_DATA_TYPE_BOOL: + for (int32_t i = 0; i < numOfElems; ++i) { + *(uint8_t *)(POINTER_SHIFT(val, i * tDataTypes[type].bytes)) = TSDB_DATA_BOOL_NULL; + } + break; + case TSDB_DATA_TYPE_TINYINT: + for (int32_t i = 0; i < numOfElems; ++i) { + *(uint8_t *)(POINTER_SHIFT(val, i * tDataTypes[type].bytes)) = TSDB_DATA_TINYINT_NULL; + } + break; + case TSDB_DATA_TYPE_SMALLINT: + for (int32_t i = 0; i < numOfElems; ++i) { + *(uint16_t *)(POINTER_SHIFT(val, i * tDataTypes[type].bytes)) = TSDB_DATA_SMALLINT_NULL; + } + break; + case TSDB_DATA_TYPE_INT: + for (int32_t i = 0; i < numOfElems; ++i) { + *(uint32_t *)(POINTER_SHIFT(val, i * tDataTypes[type].bytes)) = TSDB_DATA_INT_NULL; + } + break; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: + for (int32_t i = 0; i < numOfElems; ++i) { + *(uint64_t *)(POINTER_SHIFT(val, i * tDataTypes[type].bytes)) = TSDB_DATA_BIGINT_NULL; + } + break; + case TSDB_DATA_TYPE_UTINYINT: + for (int32_t i = 0; i < numOfElems; ++i) { + *(uint8_t *)(POINTER_SHIFT(val, i * tDataTypes[type].bytes)) = TSDB_DATA_UTINYINT_NULL; + } + break; + case TSDB_DATA_TYPE_USMALLINT: + for (int32_t i = 0; i < numOfElems; ++i) { + *(uint16_t *)(POINTER_SHIFT(val, i * tDataTypes[type].bytes)) = TSDB_DATA_USMALLINT_NULL; + } + break; + case TSDB_DATA_TYPE_UINT: + for (int32_t i = 0; i < numOfElems; ++i) { + *(uint32_t *)(POINTER_SHIFT(val, i * tDataTypes[type].bytes)) = TSDB_DATA_UINT_NULL; + } + break; + case TSDB_DATA_TYPE_UBIGINT: + for (int32_t i = 0; i < numOfElems; ++i) { + *(uint64_t *)(POINTER_SHIFT(val, i * tDataTypes[type].bytes)) = TSDB_DATA_UBIGINT_NULL; + } + break; + case TSDB_DATA_TYPE_FLOAT: + for (int32_t i = 0; i < numOfElems; ++i) { + *(uint32_t *)(POINTER_SHIFT(val, i * tDataTypes[type].bytes)) = TSDB_DATA_FLOAT_NULL; + } + break; + case TSDB_DATA_TYPE_DOUBLE: + for (int32_t i = 0; i < numOfElems; ++i) { + *(uint64_t *)(POINTER_SHIFT(val, i * tDataTypes[type].bytes)) = TSDB_DATA_DOUBLE_NULL; + } + break; + case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_BINARY: + for (int32_t i = 0; i < numOfElems; ++i) { + setVardataNull(POINTER_SHIFT(val, i * bytes), type); + } + break; + default: { + for (int32_t i = 0; i < numOfElems; ++i) { + *(uint32_t *)(POINTER_SHIFT(val, i * tDataTypes[TSDB_DATA_TYPE_INT].bytes)) = TSDB_DATA_INT_NULL; + } + break; + } + } +} + +static uint8_t nullBool = TSDB_DATA_BOOL_NULL; +static uint8_t nullTinyInt = TSDB_DATA_TINYINT_NULL; +static uint16_t nullSmallInt = TSDB_DATA_SMALLINT_NULL; +static uint32_t nullInt = TSDB_DATA_INT_NULL; +static uint64_t nullBigInt = TSDB_DATA_BIGINT_NULL; +static uint32_t nullFloat = TSDB_DATA_FLOAT_NULL; +static uint64_t nullDouble = TSDB_DATA_DOUBLE_NULL; +static uint8_t nullTinyIntu = TSDB_DATA_UTINYINT_NULL; +static uint16_t nullSmallIntu = TSDB_DATA_USMALLINT_NULL; +static uint32_t nullIntu = TSDB_DATA_UINT_NULL; +static uint64_t nullBigIntu = TSDB_DATA_UBIGINT_NULL; +static SBinaryNullT nullBinary = {1, TSDB_DATA_BINARY_NULL}; +static SNCharNullT nullNchar = {4, TSDB_DATA_NCHAR_NULL}; + +static const void *nullValues[] = { + &nullBool, &nullTinyInt, &nullSmallInt, &nullInt, &nullBigInt, + &nullFloat, &nullDouble, &nullBinary, &nullBigInt, &nullNchar, + &nullTinyIntu, &nullSmallIntu, &nullIntu, &nullBigIntu, +}; + +const void *getNullValue(int32_t type) { + assert(type >= TSDB_DATA_TYPE_BOOL && type <= TSDB_DATA_TYPE_UBIGINT); + return nullValues[type - 1]; +} + +void assignVal(char *val, const char *src, int32_t len, int32_t type) { + switch (type) { + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: + *((int8_t *)val) = GET_INT8_VAL(src); + break; + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: + *((int16_t *)val) = GET_INT16_VAL(src); + break; + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: + *((int32_t *)val) = GET_INT32_VAL(src); + break; + + case TSDB_DATA_TYPE_FLOAT: + SET_FLOAT_VAL(val, GET_FLOAT_VAL(src)); + break; + case TSDB_DATA_TYPE_DOUBLE: + SET_DOUBLE_VAL(val, GET_DOUBLE_VAL(src)); + break; + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: + *((int64_t *)val) = GET_INT64_VAL(src); + break; + case TSDB_DATA_TYPE_BINARY: + varDataCopy(val, src); + break; + case TSDB_DATA_TYPE_NCHAR: + varDataCopy(val, src); + break; + default: { + memcpy(val, src, len); + break; + } + } +} + +void operateVal(void *dst, void *s1, void *s2, int32_t optr, int32_t type) { + if (optr == TSDB_BINARY_OP_ADD) { + switch (type) { + case TSDB_DATA_TYPE_TINYINT: + *((int8_t *)dst) = GET_INT8_VAL(s1) + GET_INT8_VAL(s2); + break; + case TSDB_DATA_TYPE_UTINYINT: + *((uint8_t *)dst) = GET_UINT8_VAL(s1) + GET_UINT8_VAL(s2); + break; + case TSDB_DATA_TYPE_SMALLINT: + *((int16_t *)dst) = GET_INT16_VAL(s1) + GET_INT16_VAL(s2); + break; + case TSDB_DATA_TYPE_USMALLINT: + *((uint16_t *)dst) = GET_UINT16_VAL(s1) + GET_UINT16_VAL(s2); + break; + case TSDB_DATA_TYPE_INT: + *((int32_t *)dst) = GET_INT32_VAL(s1) + GET_INT32_VAL(s2); + break; + case TSDB_DATA_TYPE_UINT: + *((uint32_t *)dst) = GET_UINT32_VAL(s1) + GET_UINT32_VAL(s2); + break; + case TSDB_DATA_TYPE_BIGINT: + *((int64_t *)dst) = GET_INT64_VAL(s1) + GET_INT64_VAL(s2); + break; + case TSDB_DATA_TYPE_UBIGINT: + *((uint64_t *)dst) = GET_UINT64_VAL(s1) + GET_UINT64_VAL(s2); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + *((int64_t *)dst) = GET_INT64_VAL(s1) + GET_INT64_VAL(s2); + break; + case TSDB_DATA_TYPE_FLOAT: + SET_FLOAT_VAL(dst, GET_FLOAT_VAL(s1) + GET_FLOAT_VAL(s2)); + break; + case TSDB_DATA_TYPE_DOUBLE: + SET_DOUBLE_VAL(dst, GET_DOUBLE_VAL(s1) + GET_DOUBLE_VAL(s2)); + break; + default: { + assert(0); + break; + } + } + } else { + assert(0); + } +} + +#define SWAP(a, b, c) \ + do { \ + typeof(a) __tmp = (a); \ + (a) = (b); \ + (b) = __tmp; \ + } while (0) + + +void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size, void* buf) { + switch (type) { + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_UINT: { + SWAP(*(int32_t *)(pLeft), *(int32_t *)(pRight), int32_t); + break; + } + + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_UBIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: { + SWAP(*(int64_t *)(pLeft), *(int64_t *)(pRight), int64_t); + break; + } + case TSDB_DATA_TYPE_DOUBLE: { + SWAP(*(double *)(pLeft), *(double *)(pRight), double); + break; + } + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_USMALLINT: { + SWAP(*(int16_t *)(pLeft), *(int16_t *)(pRight), int16_t); + break; + } + + case TSDB_DATA_TYPE_FLOAT: { + SWAP(*(float *)(pLeft), *(float *)(pRight), float); + break; + } + + case TSDB_DATA_TYPE_BOOL: + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_UTINYINT: { + SWAP(*(int8_t *)(pLeft), *(int8_t *)(pRight), int8_t); + break; + } + + default: { + memcpy(buf, pLeft, size); + memcpy(pLeft, pRight, size); + memcpy(pRight, buf, size); + break; + } + } +} diff --git a/source/libs/executor/inc/tarithoperator.h b/source/libs/executor/inc/tarithoperator.h new file mode 100644 index 0000000000..e47cb5c1cb --- /dev/null +++ b/source/libs/executor/inc/tarithoperator.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_COMMON_QARITHMETICOPERATOR_H_ +#define _TD_COMMON_QARITHMETICOPERATOR_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +typedef void (*_arithmetic_operator_fn_t)(void *left, int32_t numLeft, int32_t leftType, void *right, int32_t numRight, + int32_t rightType, void *output, int32_t order); + +_arithmetic_operator_fn_t getArithmeticOperatorFn(int32_t arithmeticOptr); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_COMMON_QARITHMETICOPERATOR_H_*/ diff --git a/source/libs/executor/src/tarithoperator.h b/source/libs/executor/src/tarithoperator.h new file mode 100644 index 0000000000..e47cb5c1cb --- /dev/null +++ b/source/libs/executor/src/tarithoperator.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_COMMON_QARITHMETICOPERATOR_H_ +#define _TD_COMMON_QARITHMETICOPERATOR_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +typedef void (*_arithmetic_operator_fn_t)(void *left, int32_t numLeft, int32_t leftType, void *right, int32_t numRight, + int32_t rightType, void *output, int32_t order); + +_arithmetic_operator_fn_t getArithmeticOperatorFn(int32_t arithmeticOptr); + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_COMMON_QARITHMETICOPERATOR_H_*/ diff --git a/src/dnode/src/dnodeMain.c b/src/dnode/src/dnodeMain.c deleted file mode 100644 index e69de29bb2..0000000000